add transformerLM egs(aishell2)

wucong.lyb

2023-02-21 526d2e544e92b474eaa082ad96735e01ceccb2cc

 funasr/datasets/preprocessor.py

@@ -363,7 +363,7 @@
            if self.split_with_space:
                tokens = text.strip().split(" ")
                if self.seg_dict is not None:
                    tokens = forward_segment("".join(tokens).lower(), self.seg_dict)
                    tokens = forward_segment("".join(tokens), self.seg_dict)
                    tokens = seg_tokenize(tokens, self.seg_dict)
            else:
                tokens = self.tokenizer.text2tokens(text)