From fc08b62d05723cdc1ce021bb8ba044ca014fb1f7 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期一, 13 三月 2023 18:38:41 +0800
Subject: [PATCH] readme
---
funasr/utils/postprocess_utils.py | 14 +++++++++++---
1 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/funasr/utils/postprocess_utils.py b/funasr/utils/postprocess_utils.py
index 4da0d59..2475548 100644
--- a/funasr/utils/postprocess_utils.py
+++ b/funasr/utils/postprocess_utils.py
@@ -6,7 +6,7 @@
def isChinese(ch: str):
- if '\u4e00' <= ch <= '\u9fff' or '\u0030' <= ch <= '\u0039':
+ if '\u4e00' <= ch <= '\u9fff' or '\u0030' <= ch <= '\u0039' or ch == '@':
return True
return False
@@ -17,6 +17,8 @@
cur = i.replace(' ', '')
cur = cur.replace('</s>', '')
cur = cur.replace('<s>', '')
+ cur = cur.replace('<unk>', '')
+ cur = cur.replace('<OOV>', '')
word_lists.append(cur)
if len(word_lists) == 0:
@@ -34,6 +36,8 @@
cur = i.replace(' ', '')
cur = cur.replace('</s>', '')
cur = cur.replace('<s>', '')
+ cur = cur.replace('<unk>', '')
+ cur = cur.replace('<OOV>', '')
word_lists.append(cur)
if len(word_lists) == 0:
@@ -144,7 +148,7 @@
else:
word = i.decode('utf-8')
- if word in ['<s>', '</s>', '<unk>']:
+ if word in ['<s>', '</s>', '<unk>', '<OOV>']:
continue
else:
middle_lists.append(word)
@@ -232,5 +236,9 @@
return sentence, ts_lists, real_word_lists
else:
word_lists = abbr_dispose(word_lists)
+ real_word_lists = []
+ for ch in word_lists:
+ if ch != ' ':
+ real_word_lists.append(ch)
sentence = ''.join(word_lists).strip()
- return sentence
+ return sentence, real_word_lists
--
Gitblit v1.9.1