From c553a8db1712c2a5deeef5bbb68bd1fdf8d61ab7 Mon Sep 17 00:00:00 2001 From: 游雁 <zhifu.gzf@alibaba-inc.com> Date: 星期四, 13 六月 2024 17:38:01 +0800 Subject: [PATCH] decoding --- funasr/datasets/large_datasets/utils/filter.py | 34 +++++++++++++++++++++++----------- 1 files changed, 23 insertions(+), 11 deletions(-) diff --git a/funasr/datasets/large_datasets/utils/filter.py b/funasr/datasets/large_datasets/utils/filter.py index 5dc911f..adc8fa0 100644 --- a/funasr/datasets/large_datasets/utils/filter.py +++ b/funasr/datasets/large_datasets/utils/filter.py @@ -1,15 +1,27 @@ #!/usr/bin/env python -def filter(data, - min_length=10, - max_length=10000, - min_token_length=0, - max_token_length=200): - assert "speech" in data - assert "text" in data +def filter( + data, speech_length_min=100, speech_length_max=15000, token_length_min=0, token_length_max=200 +): + assert "speech" in data or "text" in data - num_frames = data["speech"].shape[0] - num_tokens = len(data['text']) - - return min_length < num_frames < max_length and min_token_length < num_tokens < max_token_length \ No newline at end of file + if "speech" in data and "text" in data: + if "sampling_rate" in data: + speech_length = (data["speech"].shape[0] / data["sampling_rate"]) * 1000.0 + else: + speech_length = data["speech"].shape[0] + num_tokens = len(data["text"]) + return ( + speech_length_min < speech_length < speech_length_max + and token_length_min < num_tokens < token_length_max + ) + elif "speech" in data: + if "sampling_rate" in data: + speech_length = (data["speech"].shape[0] / data["sampling_rate"]) * 1000.0 + else: + speech_length = data["speech"].shape[0] + return speech_length_min < speech_length < speech_length_max + else: + num_tokens = len(data["text"]) + return token_length_min < num_tokens < token_length_max -- Gitblit v1.9.1