From fcc9c89eaba9a4e36c54958aeedeec7ab3756cd7 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期一, 13 二月 2023 17:43:31 +0800
Subject: [PATCH] Merge branch 'main' of github.com:alibaba-damo-academy/FunASR add
---
funasr/datasets/large_datasets/build_dataloader.py | 28 +++++++++++++++++++++-------
1 files changed, 21 insertions(+), 7 deletions(-)
diff --git a/funasr/datasets/large_datasets/build_dataloader.py b/funasr/datasets/large_datasets/build_dataloader.py
index 37fbb7c..8f7fd0b 100644
--- a/funasr/datasets/large_datasets/build_dataloader.py
+++ b/funasr/datasets/large_datasets/build_dataloader.py
@@ -21,16 +21,30 @@
symbol_table[char] = i
return symbol_table
+def load_seg_dict(seg_dict_file):
+ seg_dict = {}
+ assert isinstance(seg_dict_file, str)
+ with open(seg_dict_file, "r", encoding="utf8") as f:
+ lines = f.readlines()
+ for line in lines:
+ s = line.strip().split()
+ key = s[0]
+ value = s[1:]
+ seg_dict[key] = " ".join(value)
+ return seg_dict
class ArkDataLoader(AbsIterFactory):
- def __init__(self, data_list, dict_file, config_file, mode="train"):
- symbol_table = read_symbol_table(dict_file)
- with open(config_file, "r") as fin:
- configs = yaml.load(fin, Loader=yaml.FullLoader)
- self.dataset_conf = configs["dataset_conf"]
+ def __init__(self, data_list, dict_file, dataset_conf, seg_dict_file=None, mode="train"):
+ symbol_table = read_symbol_table(dict_file) if dict_file is not None else None
+ if seg_dict_file is not None:
+ seg_dict = load_seg_dict(seg_dict_file)
+ else:
+ seg_dict = None
+ self.dataset_conf = dataset_conf
logging.info("dataloader config: {}".format(self.dataset_conf))
- self.dataset = Dataset(data_list, symbol_table,
- self.dataset_conf, mode=mode)
+ batch_mode = self.dataset_conf.get("batch_mode", "padding")
+ self.dataset = Dataset(data_list, symbol_table, seg_dict,
+ self.dataset_conf, mode=mode, batch_mode=batch_mode)
def build_iter(self, epoch, shuffle=True):
self.dataset.set_epoch(epoch)
--
Gitblit v1.9.1