Dev gzf funasr2 (#1111)

zhifu gao

2023-11-23 7dadb793e639d2b7f918f2f915e928a63e016ea5

 funasr/tokenizer/build_tokenizer.py

File was renamed from funasr/text/build_tokenizer.py
@@ -3,11 +3,11 @@
from typing import Union


from funasr.text.abs_tokenizer import AbsTokenizer
from funasr.text.char_tokenizer import CharTokenizer
from funasr.text.phoneme_tokenizer import PhonemeTokenizer
from funasr.text.sentencepiece_tokenizer import SentencepiecesTokenizer
from funasr.text.word_tokenizer import WordTokenizer
from funasr.tokenizer.abs_tokenizer import AbsTokenizer
from funasr.tokenizer.char_tokenizer import CharTokenizer
from funasr.tokenizer.phoneme_tokenizer import PhonemeTokenizer
from funasr.tokenizer.sentencepiece_tokenizer import SentencepiecesTokenizer
from funasr.tokenizer.word_tokenizer import WordTokenizer


def build_tokenizer(