SWHL
2023-02-21 f2a208d0d2e6a7805c44cd83faa7c313e01e594c
funasr/datasets/preprocessor.py
@@ -363,7 +363,7 @@
            if self.split_with_space:
                tokens = text.strip().split(" ")
                if self.seg_dict is not None:
                    tokens = forward_segment("".join(tokens).lower(), self.seg_dict)
                    tokens = forward_segment("".join(tokens), self.seg_dict)
                    tokens = seg_tokenize(tokens, self.seg_dict)
            else:
                tokens = self.tokenizer.text2tokens(text)