Dev gzf exp (#1665)

zhifu gao

2024-04-26 1cdb3cc28d4d89a576cc06e5cd8eb80da1f3a3aa

 funasr/tokenizer/hf_tokenizer.py

@@ -1,15 +1,13 @@

try:
   from transformers import AutoTokenizer
except:
   print("If you want to use hugging, please `pip install -U transformers`")

from funasr.register import tables


@tables.register("tokenizer_classes", "HuggingfaceTokenizer")
def HuggingfaceTokenizer(init_param_path, **kwargs):
    try:
        from transformers import AutoTokenizer
    except:
        # print("If you want to use hugging, please `pip install -U transformers`")
        pass
    tokenizer = AutoTokenizer.from_pretrained(init_param_path)

   tokenizer = AutoTokenizer.from_pretrained(init_param_path)
	
   return tokenizer

    return tokenizer