From a951500963c62330c1ca3f1f018d1d5ebfbb8b0f Mon Sep 17 00:00:00 2001
From: VirtuosoQ <2416050435@qq.com>
Date: 星期一, 29 四月 2024 16:47:23 +0800
Subject: [PATCH] 16:47 commit

---
 funasr/tokenizer/char_tokenizer.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py
index 2efc0b0..92c6e67 100644
--- a/funasr/tokenizer/char_tokenizer.py
+++ b/funasr/tokenizer/char_tokenizer.py
@@ -93,7 +93,8 @@
 	return seg_dict
 
 def seg_tokenize(txt, seg_dict):
-	pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
+	# pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
+	pattern = re.compile(r"([\u4E00-\u9FA5A-Za-z0-9])")
 	out_txt = ""
 	for word in txt:
 		word = word.lower()

--
Gitblit v1.9.1