From c6574bf4f4390a9c0dfdd5aa2c4f39eb85e1557e Mon Sep 17 00:00:00 2001
From: Shi Xian <40013335+R1ckShi@users.noreply.github.com>
Date: 星期二, 09 四月 2024 10:41:11 +0800
Subject: [PATCH] Merge pull request #1596 from wanghuii1/main
---
funasr/models/sense_voice/whisper_lib/tokenizer.py | 11 ++++++++---
1 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/funasr/models/sense_voice/whisper_lib/tokenizer.py b/funasr/models/sense_voice/whisper_lib/tokenizer.py
index e941fb2..463ce83 100644
--- a/funasr/models/sense_voice/whisper_lib/tokenizer.py
+++ b/funasr/models/sense_voice/whisper_lib/tokenizer.py
@@ -363,8 +363,10 @@
@lru_cache(maxsize=None)
-def get_encoding(name: str = "gpt2", num_languages: int = 99):
- vocab_path = os.path.join(os.path.dirname(__file__), "assets", f"{name}.tiktoken")
+def get_encoding(name: str = "gpt2", num_languages: int = 99, vocab_path:str=None):
+ if vocab_path is None:
+ vocab_path = os.path.join(os.path.dirname(__file__), "assets", f"{name}.tiktoken")
+
ranks = {
base64.b64decode(token): int(rank)
for token, rank in (line.split() for line in open(vocab_path) if line)
@@ -423,6 +425,7 @@
language: Optional[str] = None,
task: Optional[str] = None, # Literal["transcribe", "translate", None]
encoding_path: Optional[str] = None,
+ vocab_path: Optional[str] = None,
) -> Tokenizer:
if language is not None:
language = language.lower()
@@ -443,7 +446,9 @@
if encoding_path is not None:
encoding_name = encoding_path
- encoding = get_encoding(name=encoding_name, num_languages=num_languages)
+
+ encoding = get_encoding(name=encoding_name, num_languages=num_languages, vocab_path=vocab_path)
+
return Tokenizer(
encoding=encoding, num_languages=num_languages, language=language, task=task
--
Gitblit v1.9.1