From 4d907718f39e2b0f7a0c714c2e3de289e742fc61 Mon Sep 17 00:00:00 2001
From: Carl <415692979@qq.com>
Date: 星期四, 28 三月 2024 13:42:00 +0800
Subject: [PATCH] 修正commit 87b62d68957a2194b017a43b6c2a15424a05a984 引入的英文整句标点预测导致末尾两个单词中间的空格被删除的问题。 (#1556)
---
funasr/datasets/large_datasets/utils/tokenize.py | 8 ++++----
1 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/funasr/datasets/large_datasets/utils/tokenize.py b/funasr/datasets/large_datasets/utils/tokenize.py
index 7fcd0eb..34a97c1 100644
--- a/funasr/datasets/large_datasets/utils/tokenize.py
+++ b/funasr/datasets/large_datasets/utils/tokenize.py
@@ -54,15 +54,15 @@
length = len(text)
if 'hw_tag' in data:
+ pre_index = None
if hw_config['pre_hwlist'] is not None and hw_config['pre_prob'] > 0:
# enable preset hotword detect in sampling
- pre_index = None
for hw in hw_config['pre_hwlist']:
- _find = data['text'][0].find(hw)
+ hw = " ".join(seg_tokenize(hw, seg_dict))
+ _find = " ".join(text).find(hw)
if _find != -1:
- _find = data['text'][0][:_find].count(" ") # bpe sometimes
+ # _find = text[:_find].count(" ") # bpe sometimes
pre_index = [_find, _find + max(hw.count(" "), 1)]
- import pdb; pdb.set_trace()
break
hotword_indxs = sample_hotword(length, **hw_config, pre_index=pre_index)
data['hotword_indxs'] = hotword_indxs
--
Gitblit v1.9.1