From 0acf868dbb02642348e3addfec8e56974facfc2f Mon Sep 17 00:00:00 2001
From: hnluo <haoneng.lhn@alibaba-inc.com>
Date: 星期一, 29 五月 2023 10:40:21 +0800
Subject: [PATCH] Merge pull request #558 from alibaba-damo-academy/dev_wjm2
---
egs/wenetspeech/conformer/local/extract_meta.py | 114 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 114 insertions(+), 0 deletions(-)
diff --git a/egs/wenetspeech/conformer/local/extract_meta.py b/egs/wenetspeech/conformer/local/extract_meta.py
new file mode 100755
index 0000000..6074162
--- /dev/null
+++ b/egs/wenetspeech/conformer/local/extract_meta.py
@@ -0,0 +1,114 @@
+# Copyright 2021 Xiaomi Corporation (Author: Yongqing Wang)
+# Mobvoi Inc(Author: Di Wu, Binbin Zhang)
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import os
+import sys
+
+
+def get_args():
+ parser = argparse.ArgumentParser(
+ description="""
+ This script is used to process raw json dataset of WenetSpeech,
+ where the long wav is splitinto segments and
+ data of wenet format is generated.
+ """
+ )
+ parser.add_argument("input_json", help="""Input json file of WenetSpeech""")
+ parser.add_argument("output_dir", help="""Output dir for prepared data""")
+
+ args = parser.parse_args()
+ return args
+
+
+def meta_analysis(input_json, output_dir):
+ input_dir = os.path.dirname(input_json)
+
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+
+ try:
+ with open(input_json, "r") as injson:
+ json_data = json.load(injson)
+ except Exception:
+ sys.exit(f"Failed to load input json file: {input_json}")
+ else:
+ if json_data["audios"] is not None:
+ with open(f"{output_dir}/text", "w") as utt2text, open(
+ f"{output_dir}/segments", "w"
+ ) as segments, open(f"{output_dir}/utt2dur", "w") as utt2dur, open(
+ f"{output_dir}/wav.scp", "w"
+ ) as wavscp, open(
+ f"{output_dir}/utt2subsets", "w"
+ ) as utt2subsets, open(
+ f"{output_dir}/reco2dur", "w"
+ ) as reco2dur:
+ for long_audio in json_data["audios"]:
+ try:
+ long_audio_path = os.path.realpath(
+ os.path.join(input_dir, long_audio["path"])
+ )
+ aid = long_audio["aid"]
+ segments_lists = long_audio["segments"]
+ duration = long_audio["duration"]
+ assert os.path.exists(long_audio_path)
+ except AssertionError:
+ print(
+ f"""Warning: {aid} something is wrong,
+ maybe AssertionError, skipped"""
+ )
+ continue
+ except Exception:
+ print(
+ f"""Warning: {aid} something is wrong, maybe the
+ error path: {long_audio_path}, skipped"""
+ )
+ continue
+ else:
+ wavscp.write(f"{aid}\t{long_audio_path}\n")
+ reco2dur.write(f"{aid}\t{duration}\n")
+ for segment_file in segments_lists:
+ try:
+ sid = segment_file["sid"]
+ start_time = segment_file["begin_time"]
+ end_time = segment_file["end_time"]
+ dur = end_time - start_time
+ text = segment_file["text"]
+ segment_subsets = segment_file["subsets"]
+ except Exception:
+ print(
+ f"""Warning: {segment_file} something
+ is wrong, skipped"""
+ )
+ continue
+ else:
+ utt2text.write(f"{sid}\t{text}\n")
+ segments.write(
+ f"{sid}\t{aid}\t{start_time}\t{end_time}\n"
+ )
+ utt2dur.write(f"{sid}\t{dur}\n")
+ segment_sub_names = " ".join(segment_subsets)
+ utt2subsets.write(f"{sid}\t{segment_sub_names}\n")
+
+
+def main():
+ args = get_args()
+
+ meta_analysis(args.input_json, args.output_dir)
+
+
+if __name__ == "__main__":
+ main()
--
Gitblit v1.9.1