From e9d2cfc3a134b00f4e98271fbee3838d1ccecbcc Mon Sep 17 00:00:00 2001
From: VirtuosoQ <2416050435@qq.com>
Date: 星期五, 26 四月 2024 14:59:30 +0800
Subject: [PATCH] FunASR java http  client

---
 funasr/tokenizer/char_tokenizer.py |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py
index 2efc0b0..92c6e67 100644
--- a/funasr/tokenizer/char_tokenizer.py
+++ b/funasr/tokenizer/char_tokenizer.py
@@ -93,7 +93,8 @@
 	return seg_dict
 
 def seg_tokenize(txt, seg_dict):
-	pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
+	# pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
+	pattern = re.compile(r"([\u4E00-\u9FA5A-Za-z0-9])")
 	out_txt = ""
 	for word in txt:
 		word = word.lower()

--
Gitblit v1.9.1