funasr/tokenizer/abs_tokenizer.py
@@ -21,11 +21,12 @@ class BaseTokenizer(ABC): def __init__(self, token_list: Union[Path, str, Iterable[str]], def __init__(self, token_list: Union[Path, str, Iterable[str]]=None, unk_symbol: str = "<unk>", **kwargs, ): if token_list is not None: if isinstance(token_list, (Path, str)): token_list = Path(token_list) self.token_list_repr = str(token_list)
