From 6e69d784e4814c3dbe35e8f70c6cf4b920c8b20b Mon Sep 17 00:00:00 2001
From: 天地 <tiandiweizun@gmail.com>
Date: 星期三, 19 三月 2025 23:10:13 +0800
Subject: [PATCH] 1. bug fix:list(mean)和list(var)，由于mean和var是numpy，导致写入到文件的格式错误，参考上面的话，大概率是list(mean.tolist())，其实外层list没有必要 (#2437)
---
funasr/datasets/openai_datasets/index_ds.py | 12 +++++++++++-
1 files changed, 11 insertions(+), 1 deletions(-)
diff --git a/funasr/datasets/openai_datasets/index_ds.py b/funasr/datasets/openai_datasets/index_ds.py
index cc518f8..010d2d5 100644
--- a/funasr/datasets/openai_datasets/index_ds.py
+++ b/funasr/datasets/openai_datasets/index_ds.py
@@ -15,7 +15,8 @@
def __init__(self, path: str, **kwargs):
super().__init__()
- self.max_source_length = kwargs.get("max_source_length", 2048)
+
+ self.max_source_length = kwargs.get("max_source_length", 3000)
self.min_source_length = kwargs.get("min_source_length", 0)
self.max_target_length = kwargs.get("max_target_length", 2048)
self.min_target_length = kwargs.get("min_target_length", 0)
@@ -52,6 +53,15 @@
data = data_dict["messages"]
speech_length = data_dict.get("speech_length", -1) // 8
text_length = data_dict.get("text_length", 0)
+ if speech_length > self.max_source_length:
+ logging.info(
+ "speech_length: {speech_length} > {self.max_source_length}, drop it"
+ )
+ continue
+ if text_length > self.max_target_length:
+ continue
+
+ self.max_target_length = kwargs.get("max_target_length", 2048)
system, user, assistant = [], [], []
for i, item in enumerate(data):
--
Gitblit v1.9.1