From 3d12aa567443d603941636ab6308b4aafc5a80b0 Mon Sep 17 00:00:00 2001
From: 维石 <shixian.shi@alibaba-inc.com>
Date: 星期四, 18 四月 2024 10:14:16 +0800
Subject: [PATCH] update seaco finetune
---
funasr/datasets/audio_datasets/espnet_samplers.py | 7 ++++---
1 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/funasr/datasets/audio_datasets/espnet_samplers.py b/funasr/datasets/audio_datasets/espnet_samplers.py
index 1524a6a..bca0753 100644
--- a/funasr/datasets/audio_datasets/espnet_samplers.py
+++ b/funasr/datasets/audio_datasets/espnet_samplers.py
@@ -56,11 +56,12 @@
self.shuffle = shuffle and is_training
self.drop_last = drop_last
- # self.total_size = len(self.dataset)
- # self.num_samples = int(math.ceil(self.total_size / self.num_replicas))
+ self.total_size = len(self.dataset)
+ self.num_samples = int(math.ceil(self.total_size / self.num_replicas))
self.epoch = 0
self.sort_size = sort_size * num_replicas
self.max_token_length = kwargs.get("max_token_length", 2048)
+ self.min_token_length = kwargs.get("min_token_length", 0)
self.length_scale_source = kwargs.get("length_scale_source", 1.0)
@@ -85,7 +86,7 @@
for idx in sorted_indices:
original_sample_length = self.dataset.get_source_len(idx)
- if original_sample_length > self.max_token_length: # Skip samples that exceed the max length
+ if original_sample_length < self.min_token_length or original_sample_length > self.max_token_length: # Skip samples that exceed the max length
continue
# Set sample_length based on the batch type
sample_length = 1 if self.batch_type == "example" else original_sample_length
--
Gitblit v1.9.1