From 24f73665e2d8ea8e4de2fe4f900bc539d7f7b989 Mon Sep 17 00:00:00 2001
From: hnluo <haoneng.lhn@alibaba-inc.com>
Date: 星期一, 17 四月 2023 15:49:45 +0800
Subject: [PATCH] Merge pull request #367 from alibaba-damo-academy/dev_lhn2

---
 funasr/datasets/large_datasets/utils/tokenize.py |   10 +++++++---
 1 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/funasr/datasets/large_datasets/utils/tokenize.py b/funasr/datasets/large_datasets/utils/tokenize.py
index a016e4e..5a2f921 100644
--- a/funasr/datasets/large_datasets/utils/tokenize.py
+++ b/funasr/datasets/large_datasets/utils/tokenize.py
@@ -28,12 +28,16 @@
 def tokenize(data,
              vocab=None,
              seg_dict=None,
-             punc_dict=None):
+             punc_dict=None,
+             bpe_tokenizer=None):
     assert "text" in data
     assert isinstance(vocab, dict)
     text = data["text"]
     token = []
     vad = -2
+
+    if bpe_tokenizer is not None:
+        text = bpe_tokenizer.text2tokens("".join(text))
 
     if seg_dict is not None:
         assert isinstance(seg_dict, dict)
@@ -43,8 +47,8 @@
     length = len(text)
     for i in range(length):
         x = text[i]
-        if i == length-1 and "punc" in data and text[i].startswith("vad:"):
-            vad = x[-1][4:]
+        if i == length-1 and "punc" in data and x.startswith("vad:"):
+            vad = x[4:]
             if len(vad) == 0:
                 vad = -1
             else:

--
Gitblit v1.9.1