From cbbb3007436bf7cefa45af9a5aad60c696da9732 Mon Sep 17 00:00:00 2001
From: Lizerui9926 <110582652+Lizerui9926@users.noreply.github.com>
Date: 星期一, 03 四月 2023 20:01:28 +0800
Subject: [PATCH] Merge pull request #324 from alibaba-damo-academy/dev_lhn2
---
funasr/punctuation/text_preprocessor.py | 21 ---------------------
1 files changed, 0 insertions(+), 21 deletions(-)
diff --git a/funasr/punctuation/text_preprocessor.py b/funasr/punctuation/text_preprocessor.py
index 3d2c19e..c9c4bac 100644
--- a/funasr/punctuation/text_preprocessor.py
+++ b/funasr/punctuation/text_preprocessor.py
@@ -1,24 +1,3 @@
-def split_words(text: str):
- words = []
- segs = text.split()
- for seg in segs:
- # There is no space in seg.
- current_word = ""
- for c in seg:
- if len(c.encode()) == 1:
- # This is an ASCII char.
- current_word += c
- else:
- # This is a Chinese char.
- if len(current_word) > 0:
- words.append(current_word)
- current_word = ""
- words.append(c)
- if len(current_word) > 0:
- words.append(current_word)
- return words
-
-
def split_to_mini_sentence(words: list, word_limit: int = 20):
assert word_limit > 1
if len(words) <= word_limit:
--
Gitblit v1.9.1