batch

游雁

2024-04-29 f57b68121a526baea43b2e93f4540d8a2995f633

 funasr/tokenizer/sentencepiece_tokenizer.py

@@ -8,11 +8,10 @@
from funasr.tokenizer.abs_tokenizer import BaseTokenizer
from funasr.register import tables


@tables.register("tokenizer_classes", "SentencepiecesTokenizer")
class SentencepiecesTokenizer(BaseTokenizer):
    def __init__(self, bpemodel: Union[Path, str],
                 **kwargs
                 ):
    def __init__(self, bpemodel: Union[Path, str], **kwargs):
        super().__init__(**kwargs)
        self.bpemodel = str(bpemodel)
        # NOTE(kamo):
@@ -45,4 +44,4 @@

    def decode(self, line: List[int]):
        self._build_sentence_piece_processor()
        return self.sp.DecodeIds(line)
        return self.sp.DecodeIds(line)