update

shixian.shi

2023-06-27 41481008e2e51eb930799e92fb75cce603637988

update

 funasr/datasets/large_datasets/utils/tokenize.py

@@ -59,7 +59,7 @@
            pre_index = None
            for hw in hw_config['pre_hwlist']:
                hw = " ".join(seg_tokenize(hw, seg_dict))
                _find = text.find(hw)
                _find = " ".join(text.find(hw))
                if _find != -1:
                    _find = text[:_find].count(" ")  # bpe sometimes
                    pre_index = [_find, _find + max(hw.count(" "), 1)]