From b37e7b15e02fe8ac42c6591650dc2bfe86b1e8ac Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期三, 22 五月 2024 15:11:08 +0800
Subject: [PATCH] wenetspeech
---
examples/aishell/paraformer/utils/text2token.py | 9 +++++++++
funasr/bin/compute_audio_cmvn.py | 10 ++++++----
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/examples/aishell/paraformer/utils/text2token.py b/examples/aishell/paraformer/utils/text2token.py
index 263cb9d..a89679f 100755
--- a/examples/aishell/paraformer/utils/text2token.py
+++ b/examples/aishell/paraformer/utils/text2token.py
@@ -60,6 +60,12 @@
read from SI1279.PHN file -> "sil b r ih sil k s aa r er n aa l
sil t er n ih sil t ih v sil" """,
)
+ parser.add_argument(
+ "--text_format",
+ default="text",
+ type=str,
+ help="text, jsonl",
+ )
return parser
@@ -82,6 +88,9 @@
line = f.readline()
n = args.nchar
while line:
+ if args.text_format == "jsonl":
+ data = json.loads(line.strip())
+ line = data["target"]
x = line.split()
print(" ".join(x[: args.skip_ncols]), end=" ")
a = " ".join(x[args.skip_ncols :])
diff --git a/funasr/bin/compute_audio_cmvn.py b/funasr/bin/compute_audio_cmvn.py
index 91bb2ac..697da94 100644
--- a/funasr/bin/compute_audio_cmvn.py
+++ b/funasr/bin/compute_audio_cmvn.py
@@ -52,7 +52,7 @@
frontend=frontend,
tokenizer=None,
is_training=False,
- **kwargs.get("dataset_conf")
+ **kwargs.get("dataset_conf"),
)
# dataloader
@@ -68,11 +68,13 @@
dataset_train, collate_fn=dataset_train.collator, **batch_sampler_train
)
- iter_stop = int(kwargs.get("scale", 1.0) * len(dataloader_train))
-
total_frames = 0
for batch_idx, batch in enumerate(dataloader_train):
- if batch_idx >= iter_stop:
+ iter_stop = int(kwargs.get("scale", -1.0) * len(dataloader_train))
+ log_step = iter_stop // 100
+ if batch_idx % log_step == 0:
+ print(f"prcessed: {batch_idx}/{iter_stop}")
+ if batch_idx >= iter_stop and iter_stop > 0.0:
break
fbank = batch["speech"].numpy()[0, :, :]
--
Gitblit v1.9.1