funasr/tokenizer/char_tokenizer.py
@@ -5,9 +5,9 @@ import warnings from funasr.tokenizer.abs_tokenizer import BaseTokenizer from funasr.utils.register import register_class from funasr.register import tables @register_class("tokenizer_classes", "CharTokenizer") @tables.register("tokenizer_classes", "CharTokenizer") class CharTokenizer(BaseTokenizer): def __init__( self, @@ -59,4 +59,4 @@ def tokens2text(self, tokens: Iterable[str]) -> str: tokens = [t if t != self.space_symbol else " " for t in tokens] return "".join(tokens) return "".join(tokens)
