From 861147c7308b91068ffa02724fdf74ee623a909e Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: 星期三, 24 四月 2024 16:03:38 +0800
Subject: [PATCH] Dev gzf exp (#1654)
---
funasr/datasets/audio_datasets/scp2jsonl.py | 61 +++++++++++++++++++-----------
1 files changed, 38 insertions(+), 23 deletions(-)
diff --git a/funasr/datasets/audio_datasets/scp2jsonl.py b/funasr/datasets/audio_datasets/scp2jsonl.py
index 0764298..f6ceb69 100644
--- a/funasr/datasets/audio_datasets/scp2jsonl.py
+++ b/funasr/datasets/audio_datasets/scp2jsonl.py
@@ -9,8 +9,9 @@
import torch.distributed as dist
-
-def gen_jsonl_from_wav_text_list(path, data_type_list=("source", "target"), jsonl_file_out:str=None, **kwargs):
+def gen_jsonl_from_wav_text_list(
+ path, data_type_list=("source", "target"), jsonl_file_out: str = None, **kwargs
+):
try:
rank = dist.get_rank()
world_size = dist.get_world_size()
@@ -25,18 +26,27 @@
for data_type, data_file in zip(data_type_list, path):
json_dict[data_type] = {}
with open(data_file, "r") as f:
-
+
data_file_lists = f.readlines()
- lines_for_each_th = (len(data_file_lists)-1)//cpu_cores + 1
+ lines_for_each_th = (len(data_file_lists) - 1) // cpu_cores + 1
task_num = cpu_cores if len(data_file_lists) > cpu_cores else 1
# import pdb;pdb.set_trace()
if task_num > 1:
with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_cores) as executor:
-
- futures = [executor.submit(parse_context_length, data_file_lists[i*lines_for_each_th:(i+1)*lines_for_each_th], data_type) for i in range(task_num)]
-
+
+ futures = [
+ executor.submit(
+ parse_context_length,
+ data_file_lists[
+ i * lines_for_each_th : (i + 1) * lines_for_each_th
+ ],
+ data_type,
+ )
+ for i in range(task_num)
+ ]
+
for future in concurrent.futures.as_completed(futures):
-
+
json_dict[data_type].update(future.result())
else:
res = parse_context_length(data_file_lists, data_type)
@@ -48,19 +58,19 @@
for data_file in data_type_list:
jsonl_line.update(json_dict[data_file][key])
jsonl_line = json.dumps(jsonl_line, ensure_ascii=False)
- f.write(jsonl_line+"\n")
+ f.write(jsonl_line + "\n")
f.flush()
print(f"processed {len(json_dict[data_type_list[0]])} samples")
-
+
else:
pass
-
+
if world_size > 1:
dist.barrier()
-
-
+
+
def parse_context_length(data_list: list, data_type: str):
-
+
res = {}
for i, line in enumerate(data_list):
key, line = line.strip().split(maxsplit=1)
@@ -68,26 +78,33 @@
if os.path.exists(line):
waveform, _ = librosa.load(line, sr=16000)
sample_num = len(waveform)
- context_len = int(sample_num/16000*1000/10)
+ context_len = int(sample_num / 16000 * 1000 / 10)
else:
context_len = len(line.split()) if " " in line else len(line)
res[key] = {data_type: line, f"{data_type}_len": context_len}
return res
-
+
@hydra.main(config_name=None, version_base=None)
def main_hydra(cfg: DictConfig):
-
+
kwargs = OmegaConf.to_container(cfg, resolve=True)
print(kwargs)
- scp_file_list = kwargs.get("scp_file_list", ("/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"))
+ scp_file_list = kwargs.get(
+ "scp_file_list",
+ ("/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"),
+ )
if isinstance(scp_file_list, str):
scp_file_list = eval(scp_file_list)
data_type_list = kwargs.get("data_type_list", ("source", "target"))
- jsonl_file_out = kwargs.get("jsonl_file_out", "/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl")
- gen_jsonl_from_wav_text_list(scp_file_list, data_type_list=data_type_list, jsonl_file_out=jsonl_file_out)
-
+ jsonl_file_out = kwargs.get(
+ "jsonl_file_out", "/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl"
+ )
+ gen_jsonl_from_wav_text_list(
+ scp_file_list, data_type_list=data_type_list, jsonl_file_out=jsonl_file_out
+ )
+
"""
python -m funasr.datasets.audio_datasets.scp2jsonl \
@@ -98,5 +115,3 @@
if __name__ == "__main__":
main_hydra()
-
-
\ No newline at end of file
--
Gitblit v1.9.1