From 54931dd4e1a099d7d6f144c4e12e5453deb3aa26 Mon Sep 17 00:00:00 2001
From: 雾聪 <wucong.lyb@alibaba-inc.com>
Date: 星期三, 28 六月 2023 10:41:57 +0800
Subject: [PATCH] Merge branch 'main' of https://github.com/alibaba-damo-academy/FunASR into main
---
funasr/build_utils/build_streaming_iterator.py | 16 +++++++++++++---
1 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/funasr/build_utils/build_streaming_iterator.py b/funasr/build_utils/build_streaming_iterator.py
index 57cf8cf..1b16cf4 100644
--- a/funasr/build_utils/build_streaming_iterator.py
+++ b/funasr/build_utils/build_streaming_iterator.py
@@ -17,17 +17,27 @@
mc: bool = False,
dtype: str = np.float32,
num_workers: int = 1,
+ use_collate_fn: bool = True,
+ preprocess_fn=None,
ngpu: int = 0,
- train: bool=False,
+ train: bool = False,
) -> DataLoader:
"""Build DataLoader using iterable dataset"""
assert check_argument_types()
# preprocess
- preprocess_fn = build_preprocess(preprocess_args, train)
+ if preprocess_fn is not None:
+ preprocess_fn = preprocess_fn
+ elif preprocess_args is not None:
+ preprocess_args.task_name = task_name
+ preprocess_fn = build_preprocess(preprocess_args, train)
+ else:
+ preprocess_fn = None
# collate
- if task_name in ["punc", "lm"]:
+ if not use_collate_fn:
+ collate_fn = None
+ elif task_name in ["punc", "lm"]:
collate_fn = CommonCollateFn(int_pad_value=0)
else:
collate_fn = CommonCollateFn(float_pad_value=0.0, int_pad_value=-1)
--
Gitblit v1.9.1