From 55b45487c75e683cafe63dba29e32be8243b6cc7 Mon Sep 17 00:00:00 2001
From: jmwang66 <wangjiaming.wjm@alibaba-inc.com>
Date: 星期一, 06 二月 2023 16:59:00 +0800
Subject: [PATCH] update data2vec pretrain: dataset
---
funasr/datasets/large_datasets/utils/filter.py | 24 ++++++++++++++++--------
1 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/funasr/datasets/large_datasets/utils/filter.py b/funasr/datasets/large_datasets/utils/filter.py
index 91ba4be..1260a47 100644
--- a/funasr/datasets/large_datasets/utils/filter.py
+++ b/funasr/datasets/large_datasets/utils/filter.py
@@ -6,13 +6,21 @@
speech_length_max=15000,
token_length_min=0,
token_length_max=200):
- assert "speech" in data
- assert "text" in data
+ assert "speech" in data or "text" in data
- if "sampling_rate" in data:
- speech_length = (data["speech"].shape[0] / data["sampling_rate"]) * 1000.
+ if "speech" in data and "text" in data:
+ if "sampling_rate" in data:
+ speech_length = (data["speech"].shape[0] / data["sampling_rate"]) * 1000.
+ else:
+ speech_length = data["speech"].shape[0]
+ num_tokens = len(data['text'])
+ return speech_length_min < speech_length < speech_length_max and token_length_min < num_tokens < token_length_max
+ elif "speech" in data:
+ if "sampling_rate" in data:
+ speech_length = (data["speech"].shape[0] / data["sampling_rate"]) * 1000.
+ else:
+ speech_length = data["speech"].shape[0]
+ return speech_length_min < speech_length < speech_length_max
else:
- speech_length = data["speech"].shape[0]
- num_tokens = len(data['text'])
-
- return speech_length_min < speech_length < speech_length_max and token_length_min < num_tokens < token_length_max
+ num_tokens = len(data['text'])
+ return token_length_min < num_tokens < token_length_max
--
Gitblit v1.9.1