FUNASR训练
blame | 历史 | 补丁 | 提交 | 提交对比 | ignore whitespace
游雁
2023-07-05 4e2fe544ae37174a3e09dfcdbbdae5abfe711e53 
 funasr/datasets/large_datasets/utils/tokenize.py


@@ -59,11 +59,10 @@


            pre_index = None


            for hw in hw_config['pre_hwlist']:


                hw = " ".join(seg_tokenize(hw, seg_dict))


                _find = " ".join(text.find(hw))


                _find = " ".join(text).find(hw)


                if _find != -1:


                    _find = text[:_find].count(" ")  # bpe sometimes


                    # _find = text[:_find].count(" ")  # bpe sometimes


                    pre_index = [_find, _find + max(hw.count(" "), 1)]


                    import pdb; pdb.set_trace()


                    break


        hotword_indxs = sample_hotword(length, **hw_config, pre_index=pre_index)


        data['hotword_indxs'] = hotword_indxs