From 3d9f094e9652d4b84894c6fd4eae39a4a753b0f0 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 16 五月 2023 23:48:00 +0800
Subject: [PATCH] train
---
funasr/utils/prepare_data.py | 23 ++++++++++++++++++++++-
1 files changed, 22 insertions(+), 1 deletions(-)
diff --git a/funasr/utils/prepare_data.py b/funasr/utils/prepare_data.py
index 347aab5..3f55170 100644
--- a/funasr/utils/prepare_data.py
+++ b/funasr/utils/prepare_data.py
@@ -27,7 +27,7 @@
parts = line.strip().split()
if len(parts) < 2:
continue
- text_dict[parts[0]] = " ".join(parts[1:]).lower()
+ text_dict[parts[0]] = " ".join(parts[1:])
filter_count = 0
with open(wav_file, "w") as f_wav, open(text_file, "w") as f_text:
for sample_name, wav_path in wav_dict.items():
@@ -181,8 +181,29 @@
["{}/{}/wav.scp".format(args.data_dir, args.valid_set), data_names[0], data_types[0]],
["{}/{}/text".format(args.data_dir, args.valid_set), data_names[1], data_types[1]]
]
+ if args.embed_path is not None:
+ args.train_data_path_and_name_and_type.append(
+ [os.path.join(args.embed_path, "embeds", args.train_set, "embeds.scp"), "embed", "kaldi_ark"])
+ args.valid_data_path_and_name_and_type.append(
+ [os.path.join(args.embed_path, "embeds", args.valid_set, "embeds.scp"), "embed", "kaldi_ark"])
else:
args.train_data_file = os.path.join(args.data_dir, args.train_set, "data.list")
args.valid_data_file = os.path.join(args.data_dir, args.valid_set, "data.list")
+ if args.embed_path is not None:
+ if not distributed or distributed_option.dist_rank == 0:
+ for d in [args.train_set, args.valid_set]:
+ file = os.path.join(args.data_dir, d, "data.list")
+ with open(file) as f:
+ lines = f.readlines()
+ out_file = os.path.join(args.data_dir, d, "data_with_embed.list")
+ with open(out_file, "w") as out_f:
+ for line in lines:
+ parts = line.strip().split()
+ idx = parts[0].split("/")[-2]
+ embed_file = os.path.join(args.embed_path, "embeds", args.valid_set, "ark",
+ "embeds.{}.ark".format(idx))
+ out_f.write(parts[0] + " " + parts[1] + " " + embed_file + "\n")
+ args.train_data_file = os.path.join(args.data_dir, args.train_set, "data_with_embed.list")
+ args.valid_data_file = os.path.join(args.data_dir, args.valid_set, "data_with_embed.list")
if distributed:
dist.barrier()
--
Gitblit v1.9.1