From d92cd5ae037ae85ab9730499d99e5c1bd475eed2 Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 06 二月 2024 21:22:21 +0800
Subject: [PATCH] Funasr1.0 (#1362)
---
funasr/datasets/audio_datasets/index_ds.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
1 files changed, 52 insertions(+), 2 deletions(-)
diff --git a/funasr/datasets/audio_datasets/index_ds.py b/funasr/datasets/audio_datasets/index_ds.py
index c94d209..008b08f 100644
--- a/funasr/datasets/audio_datasets/index_ds.py
+++ b/funasr/datasets/audio_datasets/index_ds.py
@@ -6,8 +6,8 @@
from funasr.register import tables
-@tables.register("index_ds_classes", "IndexDSJsonl")
-class IndexDSJsonl(torch.utils.data.Dataset):
+@tables.register("index_ds_classes", "IndexDSJsonlRankSplit")
+class IndexDSJsonlRankSplit(torch.utils.data.Dataset):
def __init__(self, path):
super().__init__()
@@ -66,3 +66,53 @@
def get_target_len(self, data_dict):
return data_dict["target_len"] if "target_len" in data_dict else 0
+
+@tables.register("index_ds_classes", "IndexDSJsonl")
+@tables.register("index_ds_classes", "IndexDSJsonlRankFull")
+class IndexDSJsonlRankFull(torch.utils.data.Dataset):
+
+ def __init__(self, path):
+ super().__init__()
+
+ contents = []
+ with open(path, encoding='utf-8') as fin:
+ for line in fin:
+ data = json.loads(line.strip())
+ if "text" in data: # for sft
+ self.contents.append(data['text'])
+ if "source" in data: # for speech lab pretrain
+ prompt = data.get("prompt", "<ASR>")
+ source = data["source"]
+ target = data["target"]
+ source_len = data.get("source_len", 1)
+ target_len = data.get("target_len", 0)
+
+ contents.append({"source": source,
+ "prompt": prompt,
+ "target": target,
+ "source_len": source_len,
+ "target_len": target_len,
+ }
+ )
+
+ self.contents = contents
+
+ logging.info(
+ "total_num of samplers across ranks: {}".format(len(self.contents)))
+
+ def __len__(self):
+ return len(self.contents)
+
+ def __getitem__(self, index):
+ try:
+ data = self.contents[index]
+ except:
+ print(index)
+ return data
+
+ def get_source_len(self, data_dict):
+ return data_dict.get("source_len", 1)
+
+ def get_target_len(self, data_dict):
+
+ return data_dict.get("target_len", 0)
--
Gitblit v1.9.1