From 22d71ff774d409f8b413f9955a7c0efef5b3e288 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 23 四月 2024 09:56:06 +0800
Subject: [PATCH] wechat

---
 funasr/tokenizer/char_tokenizer.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py
index 2efc0b0..92c6e67 100644
--- a/funasr/tokenizer/char_tokenizer.py
+++ b/funasr/tokenizer/char_tokenizer.py
@@ -93,7 +93,8 @@
 	return seg_dict
 
 def seg_tokenize(txt, seg_dict):
-	pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
+	# pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
+	pattern = re.compile(r"([\u4E00-\u9FA5A-Za-z0-9])")
 	out_txt = ""
 	for word in txt:
 		word = word.lower()

--
Gitblit v1.9.1