v1.0.29

游雁

2024-07-01 e8f68b44dd65ede9278e89a5277a5cf66d546375

 funasr/tokenizer/word_tokenizer.py

@@ -14,14 +14,12 @@
        delimiter: str = None,
        non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
        remove_non_linguistic_symbols: bool = False,
        **kwargs,
    ):
        self.delimiter = delimiter

        if not remove_non_linguistic_symbols and non_linguistic_symbols is not None:
            warnings.warn(
                "non_linguistic_symbols is only used "
                "when remove_non_linguistic_symbols = True"
                "non_linguistic_symbols is only used " "when remove_non_linguistic_symbols = True"
            )

        if non_linguistic_symbols is None: