From d2dc3af1a69ee4075bcfc0c83dc0fb8e3fc1db4e Mon Sep 17 00:00:00 2001
From: yhliang <68215459+yhliang-aslp@users.noreply.github.com>
Date: 星期四, 11 五月 2023 16:31:40 +0800
Subject: [PATCH] Merge pull request #492 from alibaba-damo-academy/dev_smohan
---
funasr/datasets/large_datasets/dataset.py | 17 +++++++----------
1 files changed, 7 insertions(+), 10 deletions(-)
diff --git a/funasr/datasets/large_datasets/dataset.py b/funasr/datasets/large_datasets/dataset.py
index 500257c..8c224d8 100644
--- a/funasr/datasets/large_datasets/dataset.py
+++ b/funasr/datasets/large_datasets/dataset.py
@@ -28,7 +28,7 @@
class AudioDataset(IterableDataset):
- def __init__(self, scp_lists, data_names, data_types, frontend_conf=None, shuffle=True, mode="train", pre_hwlist=None, pre_prob=0.0):
+ def __init__(self, scp_lists, data_names, data_types, frontend_conf=None, shuffle=True, mode="train"):
self.scp_lists = scp_lists
self.data_names = data_names
self.data_types = data_types
@@ -40,8 +40,6 @@
self.world_size = 1
self.worker_id = 0
self.num_workers = 1
- self.pre_hwlist = pre_hwlist
- self.pre_prob = pre_prob
def set_epoch(self, epoch):
self.epoch = epoch
@@ -103,7 +101,7 @@
if data_type == "kaldi_ark":
ark_reader = ReadHelper('ark:{}'.format(data_file))
reader_list.append(ark_reader)
- elif data_type == "text" or data_type == "sound":
+ elif data_type == "text" or data_type == "sound" or data_type == 'text_hotword':
text_reader = open(data_file, "r")
reader_list.append(text_reader)
elif data_type == "none":
@@ -178,13 +176,13 @@
data_types = conf.get("data_types", "kaldi_ark,text")
pre_hwfile = conf.get("pre_hwlist", None)
- pre_prob = conf.get("pre_prob", 0)
+ pre_prob = conf.get("pre_prob", 0) # unused yet
hw_config = {"sample_rate": conf.get("sample_rate", 0.6),
"double_rate": conf.get("double_rate", 0.1),
"hotword_min_length": conf.get("hotword_min_length", 2),
- "hotword_max_length": conf.get("hotword_max_length", 8)}
-
+ "hotword_max_length": conf.get("hotword_max_length", 8),
+ "pre_prob": conf.get("pre_prob", 0.0)}
if pre_hwfile is not None:
pre_hwlist = []
@@ -193,15 +191,14 @@
pre_hwlist.append(line.strip())
else:
pre_hwlist = None
- # logging.warning("Previous hwlist: {}".format(pre_hwlist))
+
dataset = AudioDataset(scp_lists,
data_names,
data_types,
frontend_conf=frontend_conf,
shuffle=shuffle,
mode=mode,
- pre_hwlist=pre_hwlist,
- pre_prob=pre_prob)
+ )
filter_conf = conf.get('filter_conf', {})
filter_fn = partial(filter, **filter_conf)
--
Gitblit v1.9.1