funasr/tokenizer/char_tokenizer.py
@@ -4,10 +4,10 @@ from typing import Union import warnings from funasr.tokenizer.abs_tokenizer import AbsTokenizer from funasr.tokenizer.abs_tokenizer import BaseTokenizer from funasr.register import tables @tables.register("tokenizer_classes", "CharTokenizer") class CharTokenizer(BaseTokenizer): def __init__( self, @@ -59,4 +59,4 @@ def tokens2text(self, tokens: Iterable[str]) -> str: tokens = [t if t != self.space_symbol else " " for t in tokens] return "".join(tokens) return "".join(tokens)
