| | |
|---|
| | | default=sys.maxsize, |
|---|
| | | help="The maximum number update step to train", |
|---|
| | | ) |
|---|
| | | parser.add_argument( |
|---|
| | | "--batch_interval", |
|---|
| | | type=int, |
|---|
| | | default=10000, |
|---|
| | | help="The batch interval for saving model.", |
|---|
| | | ) |
|---|
| | | group.add_argument( |
|---|
| | | "--patience", |
|---|
| | | type=int_or_none, |
|---|
| | |
|---|
| | | "and exclude_keys excludes keys of model states for the initialization." |
|---|
| | | "e.g.\n" |
|---|
| | | " # Load all parameters" |
|---|
| | | " --init_param some/where/model.pth\n" |
|---|
| | | " --init_param some/where/model.pb\n" |
|---|
| | | " # Load only decoder parameters" |
|---|
| | | " --init_param some/where/model.pth:decoder:decoder\n" |
|---|
| | | " --init_param some/where/model.pb:decoder:decoder\n" |
|---|
| | | " # Load only decoder parameters excluding decoder.embed" |
|---|
| | | " --init_param some/where/model.pth:decoder:decoder:decoder.embed\n" |
|---|
| | | " --init_param some/where/model.pth:decoder:decoder:decoder.embed\n", |
|---|
| | | " --init_param some/where/model.pb:decoder:decoder:decoder.embed\n" |
|---|
| | | " --init_param some/where/model.pb:decoder:decoder:decoder.embed\n", |
|---|
| | | ) |
|---|
| | | group.add_argument( |
|---|
| | | "--ignore_init_mismatch", |
|---|
| | |
|---|
| | | from funasr.datasets.large_datasets.build_dataloader import ArkDataLoader |
|---|
| | | train_iter_factory = ArkDataLoader(args.train_data_file, args.token_list, args.dataset_conf, |
|---|
| | | frontend_conf=args.frontend_conf if hasattr(args, "frontend_conf") else None, |
|---|
| | | seg_dict_file=args.seg_dict_file if hasattr(args, |
|---|
| | | "seg_dict_file") else None, |
|---|
| | | seg_dict_file=args.seg_dict_file if hasattr(args, "seg_dict_file") else None, |
|---|
| | | punc_dict_file=args.punc_list if hasattr(args, "punc_list") else None, |
|---|
| | | bpemodel_file=args.bpemodel if hasattr(args, "bpemodel") else None, |
|---|
| | | mode="train") |
|---|
| | | valid_iter_factory = ArkDataLoader(args.valid_data_file, args.token_list, args.dataset_conf, |
|---|
| | | frontend_conf=args.frontend_conf if hasattr(args, "frontend_conf") else None, |
|---|
| | | seg_dict_file=args.seg_dict_file if hasattr(args, |
|---|
| | | "seg_dict_file") else None, |
|---|
| | | seg_dict_file=args.seg_dict_file if hasattr(args, "seg_dict_file") else None, |
|---|
| | | punc_dict_file=args.punc_list if hasattr(args, "punc_list") else None, |
|---|
| | | bpemodel_file=args.bpemodel if hasattr(args, "bpemodel") else None, |
|---|
| | | mode="eval") |
|---|
| | | elif args.dataset_type == "small": |
|---|
| | | train_iter_factory = cls.build_iter_factory( |
|---|
| | |
|---|
| | | ) -> AbsIterFactory: |
|---|
| | | assert check_argument_types() |
|---|
| | | |
|---|
| | | if args.frontend_conf is not None and "fs" in args.frontend_conf: |
|---|
| | | dest_sample_rate = args.frontend_conf["fs"] |
|---|
| | | else: |
|---|
| | | dest_sample_rate = 16000 |
|---|
| | | |
|---|
| | | dataset = ESPnetDataset( |
|---|
| | | iter_options.data_path_and_name_and_type, |
|---|
| | | float_dtype=args.train_dtype, |
|---|
| | | preprocess=iter_options.preprocess_fn, |
|---|
| | | max_cache_size=iter_options.max_cache_size, |
|---|
| | | max_cache_fd=iter_options.max_cache_fd, |
|---|
| | | dest_sample_rate=args.frontend_conf["fs"], |
|---|
| | | dest_sample_rate=dest_sample_rate, |
|---|
| | | ) |
|---|
| | | cls.check_task_requirements( |
|---|
| | | dataset, args.allow_variable_data_keys, train=iter_options.train |
|---|
