From 2779602177ae5374547c7a7e17de0b11a166326d Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期一, 29 四月 2024 15:08:46 +0800
Subject: [PATCH] Merge branch 'dev_gzf_exp' of github.com:alibaba-damo-academy/FunASR into dev_gzf_exp merge
---
funasr/tokenizer/hf_tokenizer.py | 19 ++++++++-----------
1 files changed, 8 insertions(+), 11 deletions(-)
diff --git a/funasr/tokenizer/hf_tokenizer.py b/funasr/tokenizer/hf_tokenizer.py
index b6043e9..1ac7ae2 100644
--- a/funasr/tokenizer/hf_tokenizer.py
+++ b/funasr/tokenizer/hf_tokenizer.py
@@ -1,16 +1,13 @@
-
-
-
from funasr.register import tables
+
@tables.register("tokenizer_classes", "HuggingfaceTokenizer")
def HuggingfaceTokenizer(init_param_path, **kwargs):
- try:
- from transformers import AutoTokenizer
- except:
- # print("If you want to use hugging, please `pip install -U transformers`")
- pass
- tokenizer = AutoTokenizer.from_pretrained(init_param_path)
-
- return tokenizer
+ try:
+ from transformers import AutoTokenizer
+ except:
+ # print("If you want to use hugging, please `pip install -U transformers`")
+ pass
+ tokenizer = AutoTokenizer.from_pretrained(init_param_path)
+ return tokenizer
--
Gitblit v1.9.1