From d783b24ba7d8a03dabfa2139fcbf40c216e0ea3d Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: 星期四, 16 三月 2023 19:34:52 +0800
Subject: [PATCH] Merge pull request #199 from alibaba-damo-academy/dev_xw
---
funasr/datasets/large_datasets/build_dataloader.py | 33 ++++++++++++++++++++++++++-------
1 files changed, 26 insertions(+), 7 deletions(-)
diff --git a/funasr/datasets/large_datasets/build_dataloader.py b/funasr/datasets/large_datasets/build_dataloader.py
index 37fbb7c..0ad1889 100644
--- a/funasr/datasets/large_datasets/build_dataloader.py
+++ b/funasr/datasets/large_datasets/build_dataloader.py
@@ -21,16 +21,35 @@
symbol_table[char] = i
return symbol_table
+def load_seg_dict(seg_dict_file):
+ seg_dict = {}
+ assert isinstance(seg_dict_file, str)
+ with open(seg_dict_file, "r", encoding="utf8") as f:
+ lines = f.readlines()
+ for line in lines:
+ s = line.strip().split()
+ key = s[0]
+ value = s[1:]
+ seg_dict[key] = " ".join(value)
+ return seg_dict
class ArkDataLoader(AbsIterFactory):
- def __init__(self, data_list, dict_file, config_file, mode="train"):
- symbol_table = read_symbol_table(dict_file)
- with open(config_file, "r") as fin:
- configs = yaml.load(fin, Loader=yaml.FullLoader)
- self.dataset_conf = configs["dataset_conf"]
+ def __init__(self, data_list, dict_file, dataset_conf, frontend_conf=None, seg_dict_file=None, punc_dict_file=None, mode="train"):
+ symbol_table = read_symbol_table(dict_file) if dict_file is not None else None
+ if seg_dict_file is not None:
+ seg_dict = load_seg_dict(seg_dict_file)
+ else:
+ seg_dict = None
+ if punc_dict_file is not None:
+ punc_dict = read_symbol_table(punc_dict_file)
+ else:
+ punc_dict = None
+ self.dataset_conf = dataset_conf
+ self.frontend_conf = frontend_conf
logging.info("dataloader config: {}".format(self.dataset_conf))
- self.dataset = Dataset(data_list, symbol_table,
- self.dataset_conf, mode=mode)
+ batch_mode = self.dataset_conf.get("batch_mode", "padding")
+ self.dataset = Dataset(data_list, symbol_table, seg_dict, punc_dict,
+ self.dataset_conf, self.frontend_conf, mode=mode, batch_mode=batch_mode)
def build_iter(self, epoch, shuffle=True):
self.dataset.set_epoch(epoch)
--
Gitblit v1.9.1