From e5819185cfb4b2cc8541c51274934e529fb3ac88 Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: 星期一, 09 一月 2023 19:58:54 +0800
Subject: [PATCH] Merge pull request #29 from alibaba-damo-academy/dev
---
funasr/utils/postprocess_utils.py | 12 ++++--------
1 files changed, 4 insertions(+), 8 deletions(-)
diff --git a/funasr/utils/postprocess_utils.py b/funasr/utils/postprocess_utils.py
index 72080ae..b18cd19 100644
--- a/funasr/utils/postprocess_utils.py
+++ b/funasr/utils/postprocess_utils.py
@@ -5,17 +5,15 @@
def isChinese(ch: str):
- if '\u4e00' <= ch <= '\u9fff':
+ if '\u4e00' <= ch <= '\u9fff' or '\u0030' <= ch <= '\u0039':
return True
return False
def isAllChinese(word: Union[List[Any], str]):
word_lists = []
- table = str.maketrans('', '', string.punctuation)
for i in word:
- cur = i.translate(table)
- cur = cur.replace(' ', '')
+ cur = i.replace(' ', '')
cur = cur.replace('</s>', '')
cur = cur.replace('<s>', '')
word_lists.append(cur)
@@ -31,10 +29,8 @@
def isAllAlpha(word: Union[List[Any], str]):
word_lists = []
- table = str.maketrans('', '', string.punctuation)
for i in word:
- cur = i.translate(table)
- cur = cur.replace(' ', '')
+ cur = i.replace(' ', '')
cur = cur.replace('</s>', '')
cur = cur.replace('<s>', '')
word_lists.append(cur)
@@ -43,7 +39,7 @@
return False
for ch in word_lists:
- if ch.isalpha() is False:
+ if ch.isalpha() is False and ch != "'":
return False
elif ch.isalpha() is True and isChinese(ch) is True:
return False
--
Gitblit v1.9.1