zhifu gao
2024-03-21 4482bbcbb912f699a4faecaafd65aa15aec64a51
funasr/datasets/large_datasets/utils/tokenize.py
@@ -54,16 +54,15 @@
    length = len(text)
    if 'hw_tag' in data:
        pre_index = None
        if hw_config['pre_hwlist'] is not None and hw_config['pre_prob'] > 0:
            # enable preset hotword detect in sampling
            pre_index = None
            for hw in hw_config['pre_hwlist']:
                hw = " ".join(seg_tokenize(hw))
                _find = text.find(hw)
                hw = " ".join(seg_tokenize(hw, seg_dict))
                _find = " ".join(text).find(hw)
                if _find != -1:
                    _find = text[:_find].count(" ")  # bpe sometimes
                    # _find = text[:_find].count(" ")  # bpe sometimes
                    pre_index = [_find, _find + max(hw.count(" "), 1)]
                    import pdb; pdb.set_trace()
                    break
        hotword_indxs = sample_hotword(length, **hw_config, pre_index=pre_index)
        data['hotword_indxs'] = hotword_indxs