From 584d0bc0ebb0de360c8cc3c05c26e376b943fefa Mon Sep 17 00:00:00 2001
From: aky15 <ankeyuthu@gmail.com>
Date: 星期二, 23 五月 2023 17:49:01 +0800
Subject: [PATCH] Merge pull request #538 from alibaba-damo-academy/dev_aky2
---
funasr/tasks/abs_task.py | 23 ++++-------------------
1 files changed, 4 insertions(+), 19 deletions(-)
diff --git a/funasr/tasks/abs_task.py b/funasr/tasks/abs_task.py
index fd4e190..0fb77a9 100644
--- a/funasr/tasks/abs_task.py
+++ b/funasr/tasks/abs_task.py
@@ -1376,25 +1376,10 @@
# 7. Build iterator factories
if args.dataset_type == "large":
- from funasr.datasets.large_datasets.build_dataloader import ArkDataLoader
- train_iter_factory = ArkDataLoader(args.train_data_file, args.token_list, args.dataset_conf,
- frontend_conf=args.frontend_conf if hasattr(args,
- "frontend_conf") else None,
- seg_dict_file=args.seg_dict_file if hasattr(args,
- "seg_dict_file") else None,
- punc_dict_file=args.punc_list if hasattr(args,
- "punc_list") else None,
- bpemodel_file=args.bpemodel if hasattr(args, "bpemodel") else None,
- mode="train")
- valid_iter_factory = ArkDataLoader(args.valid_data_file, args.token_list, args.dataset_conf,
- frontend_conf=args.frontend_conf if hasattr(args,
- "frontend_conf") else None,
- seg_dict_file=args.seg_dict_file if hasattr(args,
- "seg_dict_file") else None,
- punc_dict_file=args.punc_list if hasattr(args,
- "punc_list") else None,
- bpemodel_file=args.bpemodel if hasattr(args, "bpemodel") else None,
- mode="eval")
+ from funasr.datasets.large_datasets.build_dataloader import LargeDataLoader
+ train_iter_factory = LargeDataLoader(args, mode="train")
+ valid_iter_factory = LargeDataLoader(args, mode="eval")
+
elif args.dataset_type == "small":
train_iter_factory = cls.build_iter_factory(
args=args,
--
Gitblit v1.9.1