Merge pull request #1791 from dtlzhuangz/zhuangzhong_dev

Shi Xian

2024-06-11 7a9c0414b685b6f2ad7538017e2a1215e1e76901

 funasr/tokenizer/hf_tokenizer.py

@@ -1,16 +1,13 @@



from funasr.register import tables


@tables.register("tokenizer_classes", "HuggingfaceTokenizer")
def HuggingfaceTokenizer(init_param_path, **kwargs):
   try:
      from transformers import AutoTokenizer
   except:
      # print("If you want to use hugging, please `pip install -U transformers`")
      pass
   tokenizer = AutoTokenizer.from_pretrained(init_param_path)
	
   return tokenizer
    try:
        from transformers import AutoTokenizer
    except:
        # print("If you want to use hugging, please `pip install -U transformers`")
        pass
    tokenizer = AutoTokenizer.from_pretrained(init_param_path)

    return tokenizer