From 28ccfbfc51068a663a80764e14074df5edf2b5ba Mon Sep 17 00:00:00 2001
From: kongdeqiang <kongdeqiang960204@163.com>
Date: 星期五, 13 三月 2026 17:41:41 +0800
Subject: [PATCH] 提交
---
funasr/bin/tokenize_text.py | 20 +++++++-------------
1 files changed, 7 insertions(+), 13 deletions(-)
diff --git a/funasr/bin/tokenize_text.py b/funasr/bin/tokenize_text.py
index dc565d0..8c2aae0 100755
--- a/funasr/bin/tokenize_text.py
+++ b/funasr/bin/tokenize_text.py
@@ -7,12 +7,11 @@
from typing import List
from typing import Optional
-from typeguard import check_argument_types
from funasr.utils.cli_utils import get_commandline_args
-from funasr.text.build_tokenizer import build_tokenizer
-from funasr.text.cleaner import TextCleaner
-from funasr.text.phoneme_tokenizer import g2p_choices
+from funasr.tokenizer.build_tokenizer import build_tokenizer
+from funasr.tokenizer.cleaner import TextCleaner
+from funasr.tokenizer.phoneme_tokenizer import g2p_classes
from funasr.utils.types import str2bool
from funasr.utils.types import str_or_none
@@ -81,7 +80,6 @@
cleaner: Optional[str],
g2p: Optional[str],
):
- assert check_argument_types()
logging.basicConfig(
level=log_level,
@@ -135,7 +133,7 @@
if not write_vocabulary:
return
-
+
## FIXME
## del duplicate add_symbols in counter
for symbol_and_id in add_symbol:
@@ -199,12 +197,8 @@
help="The verbose level of logging",
)
- parser.add_argument(
- "--input", "-i", required=True, help="Input text. - indicates sys.stdin"
- )
- parser.add_argument(
- "--output", "-o", required=True, help="Output text. - indicates sys.stdout"
- )
+ parser.add_argument("--input", "-i", required=True, help="Input text. - indicates sys.stdin")
+ parser.add_argument("--output", "-o", required=True, help="Output text. - indicates sys.stdout")
parser.add_argument(
"--field",
"-f",
@@ -241,7 +235,7 @@
parser.add_argument(
"--g2p",
type=str_or_none,
- choices=g2p_choices,
+ choices=g2p_classes,
default=None,
help="Specify g2p method if --token_type=phn",
)
--
Gitblit v1.9.1