From 9b4e9cc8a0311e5243d69b73ed073e7ea441982e Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期三, 27 三月 2024 16:05:29 +0800
Subject: [PATCH] train update
---
funasr/tokenizer/char_tokenizer.py | 11 +++++++----
1 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py
index 0f40b5e..28b4f40 100644
--- a/funasr/tokenizer/char_tokenizer.py
+++ b/funasr/tokenizer/char_tokenizer.py
@@ -50,10 +50,11 @@
def text2tokens(self, line: Union[str, list]) -> List[str]:
- if self.split_with_space:
+ # if self.split_with_space:
+
+ if self.seg_dict is not None:
tokens = line.strip().split(" ")
- if self.seg_dict is not None:
- tokens = seg_tokenize(tokens, self.seg_dict)
+ tokens = seg_tokenize(tokens, self.seg_dict)
else:
tokens = []
while len(line) != 0:
@@ -66,7 +67,9 @@
else:
t = line[0]
if t == " ":
- t = "<space>"
+ # t = "<space>"
+ line = line[1:]
+ continue
tokens.append(t)
line = line[1:]
return tokens
--
Gitblit v1.9.1