16:37 java_http_client

VirtuosoQ

2024-04-28 476dc3f30c014e0d2ebdc46ce0283ddbfe63eeb8

 funasr/tokenizer/hf_tokenizer.py

@@ -1,15 +1,15 @@

try:
   from transformers import AutoTokenizer
except:
   # print("If you want to use hugging, please `pip install -U transformers`")
   pass


from funasr.register import tables

@tables.register("tokenizer_classes", "HuggingfaceTokenizer")
def HuggingfaceTokenizer(init_param_path, **kwargs):

   try:
      from transformers import AutoTokenizer
   except:
      # print("If you want to use hugging, please `pip install -U transformers`")
      pass
   tokenizer = AutoTokenizer.from_pretrained(init_param_path)
   
   return tokenizer