From 55e7a2d880b3322fd89bdb1a27373ffbd842f280 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期三, 22 五月 2024 16:33:17 +0800
Subject: [PATCH] wenetspeech
---
examples/aishell/paraformer/utils/text2token.py | 17 +++++++++++------
1 files changed, 11 insertions(+), 6 deletions(-)
diff --git a/examples/aishell/paraformer/utils/text2token.py b/examples/aishell/paraformer/utils/text2token.py
index 56c3913..a89679f 100755
--- a/examples/aishell/paraformer/utils/text2token.py
+++ b/examples/aishell/paraformer/utils/text2token.py
@@ -37,9 +37,7 @@
help="number of characters to split, i.e., \
aabb -> a a b b with -n 1 and aa bb with -n 2",
)
- parser.add_argument(
- "--skip-ncols", "-s", default=0, type=int, help="skip first n columns"
- )
+ parser.add_argument("--skip-ncols", "-s", default=0, type=int, help="skip first n columns")
parser.add_argument("--space", default="<space>", type=str, help="space symbol")
parser.add_argument(
"--non-lang-syms",
@@ -62,6 +60,12 @@
read from SI1279.PHN file -> "sil b r ih sil k s aa r er n aa l
sil t er n ih sil t ih v sil" """,
)
+ parser.add_argument(
+ "--text_format",
+ default="text",
+ type=str,
+ help="text, jsonl",
+ )
return parser
@@ -80,12 +84,13 @@
else:
f = codecs.getreader("utf-8")(sys.stdin if is_python2 else sys.stdin.buffer)
- sys.stdout = codecs.getwriter("utf-8")(
- sys.stdout if is_python2 else sys.stdout.buffer
- )
+ sys.stdout = codecs.getwriter("utf-8")(sys.stdout if is_python2 else sys.stdout.buffer)
line = f.readline()
n = args.nchar
while line:
+ if args.text_format == "jsonl":
+ data = json.loads(line.strip())
+ line = data["target"]
x = line.split()
print(" ".join(x[: args.skip_ncols]), end=" ")
a = " ".join(x[args.skip_ncols :])
--
Gitblit v1.9.1