batch

游雁

2024-04-29 8f596af4be1c2e5c4e4b4a7008ba96f412d40fca

 funasr/tokenizer/word_tokenizer.py

@@ -14,14 +14,12 @@
        delimiter: str = None,
        non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
        remove_non_linguistic_symbols: bool = False,
        **kwargs,
    ):
        self.delimiter = delimiter

        if not remove_non_linguistic_symbols and non_linguistic_symbols is not None:
            warnings.warn(
                "non_linguistic_symbols is only used "
                "when remove_non_linguistic_symbols = True"
                "non_linguistic_symbols is only used " "when remove_non_linguistic_symbols = True"
            )

        if non_linguistic_symbols is None: