update

游雁

2024-03-26 e84f17adca2d8a81bc2d0229b9531e7eb0a7705c

 funasr/tokenizer/hf_tokenizer.py

@@ -1,14 +1,15 @@

try:
   from transformers import AutoTokenizer
except:
   print("If you want to use hugging, please `pip install -U transformers`")


from funasr.register import tables

@tables.register("tokenizer_classes", "HuggingfaceTokenizer")
def HuggingfaceTokenizer(init_param_path, **kwargs):

   try:
      from transformers import AutoTokenizer
   except:
      # print("If you want to use hugging, please `pip install -U transformers`")
      pass
   tokenizer = AutoTokenizer.from_pretrained(init_param_path)
   
   return tokenizer