# Copyright 2021 Xiaomi Corporation (Author: Yongqing Wang)
|
# Mobvoi Inc(Author: Di Wu, Binbin Zhang)
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
# you may not use this file except in compliance with the License.
|
# You may obtain a copy of the License at
|
#
|
# http://www.apache.org/licenses/LICENSE-2.0
|
#
|
# Unless required by applicable law or agreed to in writing, software
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
# See the License for the specific language governing permissions and
|
# limitations under the License.
|
|
import argparse
|
import json
|
import os
|
import sys
|
|
|
def get_args():
|
parser = argparse.ArgumentParser(
|
description="""
|
This script is used to process raw json dataset of WenetSpeech,
|
where the long wav is splitinto segments and
|
data of wenet format is generated.
|
"""
|
)
|
parser.add_argument("input_json", help="""Input json file of WenetSpeech""")
|
parser.add_argument("output_dir", help="""Output dir for prepared data""")
|
|
args = parser.parse_args()
|
return args
|
|
|
def meta_analysis(input_json, output_dir):
|
input_dir = os.path.dirname(input_json)
|
|
if not os.path.exists(output_dir):
|
os.makedirs(output_dir)
|
|
try:
|
with open(input_json, "r") as injson:
|
json_data = json.load(injson)
|
except Exception:
|
sys.exit(f"Failed to load input json file: {input_json}")
|
else:
|
if json_data["audios"] is not None:
|
with open(f"{output_dir}/text", "w") as utt2text, open(
|
f"{output_dir}/segments", "w"
|
) as segments, open(f"{output_dir}/utt2dur", "w") as utt2dur, open(
|
f"{output_dir}/wav.scp", "w"
|
) as wavscp, open(
|
f"{output_dir}/utt2subsets", "w"
|
) as utt2subsets, open(
|
f"{output_dir}/reco2dur", "w"
|
) as reco2dur:
|
for long_audio in json_data["audios"]:
|
try:
|
long_audio_path = os.path.realpath(
|
os.path.join(input_dir, long_audio["path"])
|
)
|
aid = long_audio["aid"]
|
segments_lists = long_audio["segments"]
|
duration = long_audio["duration"]
|
assert os.path.exists(long_audio_path)
|
except AssertionError:
|
print(
|
f"""Warning: {aid} something is wrong,
|
maybe AssertionError, skipped"""
|
)
|
continue
|
except Exception:
|
print(
|
f"""Warning: {aid} something is wrong, maybe the
|
error path: {long_audio_path}, skipped"""
|
)
|
continue
|
else:
|
wavscp.write(f"{aid}\t{long_audio_path}\n")
|
reco2dur.write(f"{aid}\t{duration}\n")
|
for segment_file in segments_lists:
|
try:
|
sid = segment_file["sid"]
|
start_time = segment_file["begin_time"]
|
end_time = segment_file["end_time"]
|
dur = end_time - start_time
|
text = segment_file["text"]
|
segment_subsets = segment_file["subsets"]
|
except Exception:
|
print(
|
f"""Warning: {segment_file} something
|
is wrong, skipped"""
|
)
|
continue
|
else:
|
utt2text.write(f"{sid}\t{text}\n")
|
segments.write(
|
f"{sid}\t{aid}\t{start_time}\t{end_time}\n"
|
)
|
utt2dur.write(f"{sid}\t{dur}\n")
|
segment_sub_names = " ".join(segment_subsets)
|
utt2subsets.write(f"{sid}\t{segment_sub_names}\n")
|
|
|
def main():
|
args = get_args()
|
|
meta_analysis(args.input_json, args.output_dir)
|
|
|
if __name__ == "__main__":
|
main()
|
