From 2ac38adbe5f4e1374a079e032ed4b504351a207c Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 23 四月 2024 18:08:57 +0800
Subject: [PATCH] Dev gzf exp (#1647)

---
 funasr/tokenizer/char_tokenizer.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py
index 2efc0b0..92c6e67 100644
--- a/funasr/tokenizer/char_tokenizer.py
+++ b/funasr/tokenizer/char_tokenizer.py
@@ -93,7 +93,8 @@
 	return seg_dict
 
 def seg_tokenize(txt, seg_dict):
-	pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
+	# pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
+	pattern = re.compile(r"([\u4E00-\u9FA5A-Za-z0-9])")
 	out_txt = ""
 	for word in txt:
 		word = word.lower()

--
Gitblit v1.9.1