1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
|
| try:
| from transformers import AutoTokenizer
| except:
| # print("If you want to use hugging, please `pip install -U transformers`")
| pass
|
| from funasr.register import tables
|
| @tables.register("tokenizer_classes", "HuggingfaceTokenizer")
| def HuggingfaceTokenizer(init_param_path, **kwargs):
|
| tokenizer = AutoTokenizer.from_pretrained(init_param_path)
|
| return tokenizer
|
|
