From 2ddfc27d5b69e5c1a38021554e97dae958328c20 Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: 星期四, 01 二月 2024 17:29:28 +0800
Subject: [PATCH] Funasr1.0 (#1343)
---
funasr/datasets/audio_datasets/samplers.py | 10 +++++++---
1 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/funasr/datasets/audio_datasets/samplers.py b/funasr/datasets/audio_datasets/samplers.py
index 0d93098..535df5d 100644
--- a/funasr/datasets/audio_datasets/samplers.py
+++ b/funasr/datasets/audio_datasets/samplers.py
@@ -26,6 +26,8 @@
self.max_token_length = kwargs.get("max_token_length", 5000)
self.shuffle_idx = np.arange(self.total_samples)
self.shuffle = shuffle and is_training
+ self.length_scale_source = kwargs.get("length_scale_source", 1.0)
+
def __len__(self):
return (self.total_samples-1) // self.batch_size + 1
@@ -53,8 +55,10 @@
idx_map = self.shuffle_idx[idx]
# prompt = self.dataset.indexed_dataset[idx_map]["prompt"]
- sample_len_cur = self.dataset.get_source_len(idx_map) + \
- self.dataset.get_target_len(idx_map)
+ target_len = self.dataset.get_target_len(idx_map) if self.batch_type == 'length' else 0.0
+ source_len = self.dataset.get_source_len(idx_map) / self.length_scale_source
+ sample_len_cur = source_len + target_len
+
datalen_with_index.append([idx, sample_len_cur])
@@ -66,7 +70,7 @@
max_token_cur = max(max_token, sample_len_cur_raw)
max_token_padding = 1 + num_sample
- if self.batch_type == 'length':
+ if self.batch_type != 'example':
max_token_padding *= max_token_cur
if max_token_padding <= self.batch_size:
batch.append(idx)
--
Gitblit v1.9.1