From e9d2cfc3a134b00f4e98271fbee3838d1ccecbcc Mon Sep 17 00:00:00 2001
From: VirtuosoQ <2416050435@qq.com>
Date: 星期五, 26 四月 2024 14:59:30 +0800
Subject: [PATCH] FunASR java http client
---
funasr/datasets/audio_datasets/scp2jsonl.py | 44 ++++++++++++++++++++++++++------------------
1 files changed, 26 insertions(+), 18 deletions(-)
diff --git a/funasr/datasets/audio_datasets/scp2jsonl.py b/funasr/datasets/audio_datasets/scp2jsonl.py
index c60c6f5..0764298 100644
--- a/funasr/datasets/audio_datasets/scp2jsonl.py
+++ b/funasr/datasets/audio_datasets/scp2jsonl.py
@@ -19,7 +19,7 @@
world_size = 1
cpu_cores = os.cpu_count() or 1
-
+ print(f"convert wav.scp text to jsonl, ncpu: {cpu_cores}")
if rank == 0:
json_dict = {}
for data_type, data_file in zip(data_type_list, path):
@@ -29,15 +29,19 @@
data_file_lists = f.readlines()
lines_for_each_th = (len(data_file_lists)-1)//cpu_cores + 1
task_num = cpu_cores if len(data_file_lists) > cpu_cores else 1
- with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_cores) as executor:
-
- futures = [executor.submit(parse_context_length, data_file_lists[i*lines_for_each_th:(i+1)*lines_for_each_th], data_type) for i in range(task_num)]
+ # import pdb;pdb.set_trace()
+ if task_num > 1:
+ with concurrent.futures.ThreadPoolExecutor(max_workers=cpu_cores) as executor:
- for future in concurrent.futures.as_completed(futures):
-
- json_dict[data_type].update(future.result())
- # print(json_dict)
+ futures = [executor.submit(parse_context_length, data_file_lists[i*lines_for_each_th:(i+1)*lines_for_each_th], data_type) for i in range(task_num)]
+ for future in concurrent.futures.as_completed(futures):
+
+ json_dict[data_type].update(future.result())
+ else:
+ res = parse_context_length(data_file_lists, data_type)
+ json_dict[data_type].update(res)
+
with open(jsonl_file_out, "w") as f:
for key in json_dict[data_type_list[0]].keys():
jsonl_line = {"key": key}
@@ -46,6 +50,7 @@
jsonl_line = json.dumps(jsonl_line, ensure_ascii=False)
f.write(jsonl_line+"\n")
f.flush()
+ print(f"processed {len(json_dict[data_type_list[0]])} samples")
else:
pass
@@ -63,31 +68,34 @@
if os.path.exists(line):
waveform, _ = librosa.load(line, sr=16000)
sample_num = len(waveform)
- context_len = int(sample_num//16000*1000/10)
+ context_len = int(sample_num/16000*1000/10)
else:
- context_len = len(line)
+ context_len = len(line.split()) if " " in line else len(line)
res[key] = {data_type: line, f"{data_type}_len": context_len}
return res
@hydra.main(config_name=None, version_base=None)
def main_hydra(cfg: DictConfig):
- """
- python funasr/datasets/audio_datasets/scp2jsonl.py \
- ++scp_file_list='["/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"]' \
- ++data_type_list='["source", "target"]' \
- ++jsonl_file_out=/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl
-
- """
-
+
kwargs = OmegaConf.to_container(cfg, resolve=True)
+ print(kwargs)
scp_file_list = kwargs.get("scp_file_list", ("/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"))
+ if isinstance(scp_file_list, str):
+ scp_file_list = eval(scp_file_list)
data_type_list = kwargs.get("data_type_list", ("source", "target"))
jsonl_file_out = kwargs.get("jsonl_file_out", "/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl")
gen_jsonl_from_wav_text_list(scp_file_list, data_type_list=data_type_list, jsonl_file_out=jsonl_file_out)
+"""
+python -m funasr.datasets.audio_datasets.scp2jsonl \
+++scp_file_list='["/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"]' \
+++data_type_list='["source", "target"]' \
+++jsonl_file_out=/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl
+"""
+
if __name__ == "__main__":
main_hydra()
--
Gitblit v1.9.1