From 3958472fb2a9bbac5cb2a30e3fb11925c7b5d3d8 Mon Sep 17 00:00:00 2001
From: Yabin Li <wucong.lyb@alibaba-inc.com>
Date: 星期三, 20 三月 2024 19:21:33 +0800
Subject: [PATCH] Update SDK_advanced_guide_offline_en_zh.md
---
funasr/auto/auto_model.py | 122 +++++++++++++++++++++++++++++-----------
1 files changed, 88 insertions(+), 34 deletions(-)
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py
index 921ede8..bd786d0 100644
--- a/funasr/auto/auto_model.py
+++ b/funasr/auto/auto_model.py
@@ -14,6 +14,7 @@
import numpy as np
from tqdm import tqdm
+from funasr.utils.misc import deep_update
from funasr.register import tables
from funasr.utils.load_utils import load_bytes
from funasr.download.file import download_from_url
@@ -23,11 +24,12 @@
from funasr.utils.load_utils import load_audio_text_image_video
from funasr.train_utils.set_all_random_seed import set_all_random_seed
from funasr.train_utils.load_pretrained_model import load_pretrained_model
-from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk
+from funasr.utils import export_utils
try:
+ from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk
from funasr.models.campplus.cluster_backend import ClusterBackend
except:
- print("If you want to use the speaker diarization, please `pip install hdbscan`")
+ print("Notice: If you want to use the speaker diarization, please `pip install hdbscan`")
def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None):
@@ -41,11 +43,12 @@
"""
data_list = []
key_list = []
- filelist = [".scp", ".txt", ".json", ".jsonl"]
+ filelist = [".scp", ".txt", ".json", ".jsonl", ".text"]
chars = string.ascii_letters + string.digits
if isinstance(data_in, str) and data_in.startswith('http'): # url
data_in = download_from_url(data_in)
+
if isinstance(data_in, str) and os.path.exists(data_in): # wav_path; filelist: wav.scp, file.jsonl;text.txt;
_, file_extension = os.path.splitext(data_in)
file_extension = file_extension.lower()
@@ -65,7 +68,8 @@
data_list.append(data)
key_list.append(key)
else:
- key = "rand_key_" + ''.join(random.choice(chars) for _ in range(13))
+ if key is None:
+ key = "rand_key_" + ''.join(random.choice(chars) for _ in range(13))
data_list = [data_in]
key_list = [key]
elif isinstance(data_in, (list, tuple)):
@@ -97,31 +101,37 @@
def __init__(self, **kwargs):
if not kwargs.get("disable_log", True):
tables.print()
-
+
model, kwargs = self.build_model(**kwargs)
# if vad_model is not None, build vad model else None
vad_model = kwargs.get("vad_model", None)
- vad_kwargs = kwargs.get("vad_model_revision", None)
+ vad_kwargs = {} if kwargs.get("vad_kwargs", {}) is None else kwargs.get("vad_kwargs", {})
if vad_model is not None:
logging.info("Building VAD model.")
- vad_kwargs = {"model": vad_model, "model_revision": vad_kwargs, "device": kwargs["device"]}
+ vad_kwargs["model"] = vad_model
+ vad_kwargs["model_revision"] = kwargs.get("vad_model_revision", None)
+ vad_kwargs["device"] = kwargs["device"]
vad_model, vad_kwargs = self.build_model(**vad_kwargs)
# if punc_model is not None, build punc model else None
punc_model = kwargs.get("punc_model", None)
- punc_kwargs = kwargs.get("punc_model_revision", None)
+ punc_kwargs = {} if kwargs.get("punc_kwargs", {}) is None else kwargs.get("punc_kwargs", {})
if punc_model is not None:
logging.info("Building punc model.")
- punc_kwargs = {"model": punc_model, "model_revision": punc_kwargs, "device": kwargs["device"]}
+ punc_kwargs["model"] = punc_model
+ punc_kwargs["model_revision"] = kwargs.get("punc_model_revision", None)
+ punc_kwargs["device"] = kwargs["device"]
punc_model, punc_kwargs = self.build_model(**punc_kwargs)
# if spk_model is not None, build spk model else None
spk_model = kwargs.get("spk_model", None)
- spk_kwargs = kwargs.get("spk_model_revision", None)
+ spk_kwargs = {} if kwargs.get("spk_kwargs", {}) is None else kwargs.get("spk_kwargs", {})
if spk_model is not None:
logging.info("Building SPK model.")
- spk_kwargs = {"model": spk_model, "model_revision": spk_kwargs, "device": kwargs["device"]}
+ spk_kwargs["model"] = spk_model
+ spk_kwargs["model_revision"] = kwargs.get("spk_model_revision", None)
+ spk_kwargs["device"] = kwargs["device"]
spk_model, spk_kwargs = self.build_model(**spk_kwargs)
self.cb_model = ClusterBackend().to(kwargs["device"])
spk_mode = kwargs.get("spk_mode", 'punc_segment')
@@ -142,46 +152,45 @@
def build_model(self, **kwargs):
assert "model" in kwargs
if "model_conf" not in kwargs:
- logging.info("download models from model hub: {}".format(kwargs.get("model_hub", "ms")))
+ logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms")))
kwargs = download_model(**kwargs)
set_all_random_seed(kwargs.get("seed", 0))
-
+
device = kwargs.get("device", "cuda")
if not torch.cuda.is_available() or kwargs.get("ngpu", 1) == 0:
device = "cpu"
kwargs["batch_size"] = 1
kwargs["device"] = device
-
- if kwargs.get("ncpu", None):
- torch.set_num_threads(kwargs.get("ncpu"))
+
+ torch.set_num_threads(kwargs.get("ncpu", 4))
# build tokenizer
tokenizer = kwargs.get("tokenizer", None)
if tokenizer is not None:
tokenizer_class = tables.tokenizer_classes.get(tokenizer)
- tokenizer = tokenizer_class(**kwargs["tokenizer_conf"])
- kwargs["tokenizer"] = tokenizer
-
+ tokenizer = tokenizer_class(**kwargs.get("tokenizer_conf", {}))
kwargs["token_list"] = tokenizer.token_list if hasattr(tokenizer, "token_list") else None
kwargs["token_list"] = tokenizer.get_vocab() if hasattr(tokenizer, "get_vocab") else kwargs["token_list"]
vocab_size = len(kwargs["token_list"]) if kwargs["token_list"] is not None else -1
else:
vocab_size = -1
+ kwargs["tokenizer"] = tokenizer
# build frontend
frontend = kwargs.get("frontend", None)
kwargs["input_size"] = None
if frontend is not None:
frontend_class = tables.frontend_classes.get(frontend)
- frontend = frontend_class(**kwargs["frontend_conf"])
- kwargs["frontend"] = frontend
+ frontend = frontend_class(**kwargs.get("frontend_conf", {}))
kwargs["input_size"] = frontend.output_size() if hasattr(frontend, "output_size") else None
-
+ kwargs["frontend"] = frontend
# build model
model_class = tables.model_classes.get(kwargs["model"])
- model = model_class(**kwargs, **kwargs["model_conf"], vocab_size=vocab_size)
-
+ model_conf = {}
+ deep_update(model_conf, kwargs.get("model_conf", {}))
+ deep_update(model_conf, kwargs)
+ model = model_class(**model_conf, vocab_size=vocab_size)
model.to(device)
# init_param
@@ -204,7 +213,7 @@
def __call__(self, *args, **cfg):
kwargs = self.kwargs
- kwargs.update(cfg)
+ deep_update(kwargs, cfg)
res = self.model(*args, kwargs)
return res
@@ -217,16 +226,16 @@
def inference(self, input, input_len=None, model=None, kwargs=None, key=None, **cfg):
kwargs = self.kwargs if kwargs is None else kwargs
- kwargs.update(cfg)
+ deep_update(kwargs, cfg)
model = self.model if model is None else model
model.eval()
batch_size = kwargs.get("batch_size", 1)
# if kwargs.get("device", "cpu") == "cpu":
# batch_size = 1
-
+
key_list, data_list = prepare_data_iterator(input, input_len=input_len, data_type=kwargs.get("data_type", None), key=key)
-
+
speed_stats = {}
asr_result_list = []
num_samples = len(data_list)
@@ -239,13 +248,17 @@
data_batch = data_list[beg_idx:end_idx]
key_batch = key_list[beg_idx:end_idx]
batch = {"data_in": data_batch, "key": key_batch}
+
if (end_idx - beg_idx) == 1 and kwargs.get("data_type", None) == "fbank": # fbank
batch["data_in"] = data_batch[0]
batch["data_lengths"] = input_len
time1 = time.perf_counter()
with torch.no_grad():
- results, meta_data = model.inference(**batch, **kwargs)
+ res = model.inference(**batch, **kwargs)
+ if isinstance(res, (list, tuple)):
+ results = res[0]
+ meta_data = res[1] if len(res) > 1 else {}
time2 = time.perf_counter()
asr_result_list.extend(results)
@@ -276,7 +289,7 @@
def inference_with_vad(self, input, input_len=None, **cfg):
kwargs = self.kwargs
# step.1: compute the vad model
- self.vad_kwargs.update(cfg)
+ deep_update(self.vad_kwargs, cfg)
beg_vad = time.time()
res = self.inference(input, input_len=input_len, model=self.vad_model, kwargs=self.vad_kwargs, **cfg)
end_vad = time.time()
@@ -284,8 +297,8 @@
# step.2 compute asr model
model = self.model
- kwargs.update(cfg)
- batch_size = int(kwargs.get("batch_size_s", 300))*1000
+ deep_update(kwargs, cfg)
+ batch_size = max(int(kwargs.get("batch_size_s", 300))*1000, 1)
batch_size_threshold_ms = int(kwargs.get("batch_size_threshold_s", 60))*1000
kwargs["batch_size"] = batch_size
@@ -299,7 +312,8 @@
key = res[i]["key"]
vadsegments = res[i]["value"]
input_i = data_list[i]
- speech = load_audio_text_image_video(input_i, fs=kwargs["frontend"].fs, audio_fs=kwargs.get("fs", 16000))
+ fs = kwargs["frontend"].fs if hasattr(kwargs["frontend"], "fs") else 16000
+ speech = load_audio_text_image_video(input_i, fs=fs, audio_fs=kwargs.get("fs", 16000))
speech_lengths = len(speech)
n = len(vadsegments)
data_with_index = [(vadsegments[i], i) for i in range(n)]
@@ -396,7 +410,7 @@
if return_raw_text:
result['raw_text'] = ''
else:
- self.punc_kwargs.update(cfg)
+ deep_update(self.punc_kwargs, cfg)
punc_res = self.inference(result["text"], model=self.punc_model, kwargs=self.punc_kwargs, **cfg)
raw_text = copy.copy(result["text"])
if return_raw_text: result['raw_text'] = raw_text
@@ -464,3 +478,43 @@
# f"time_escape_all: {time_escape_total_all_samples:0.3f}")
return results_ret_list
+ def export(self, input=None, **cfg):
+
+ """
+
+ :param input:
+ :param type:
+ :param quantize:
+ :param fallback_num:
+ :param calib_num:
+ :param opset_version:
+ :param cfg:
+ :return:
+ """
+
+ device = cfg.get("device", "cpu")
+ model = self.model.to(device=device)
+ kwargs = self.kwargs
+ deep_update(kwargs, cfg)
+ kwargs["device"] = device
+ del kwargs["model"]
+ model.eval()
+
+ type = kwargs.get("type", "onnx")
+
+ key_list, data_list = prepare_data_iterator(input, input_len=None, data_type=kwargs.get("data_type", None), key=None)
+
+ with torch.no_grad():
+
+ if type == "onnx":
+ export_dir = export_utils.export_onnx(
+ model=model,
+ data_in=data_list,
+ **kwargs)
+ else:
+ export_dir = export_utils.export_torchscripts(
+ model=model,
+ data_in=data_list,
+ **kwargs)
+
+ return export_dir
\ No newline at end of file
--
Gitblit v1.9.1