From fae73ee41441a9ed5d31720a17acd6f15cfd90c6 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 19 三月 2024 11:14:59 +0800
Subject: [PATCH] vad conf
---
examples/industrial_data_pretraining/paraformer/demo.py | 9 +++++----
funasr/auto/auto_model.py | 32 ++++++++++++++++++++------------
examples/industrial_data_pretraining/whisper/demo.py | 1 +
examples/industrial_data_pretraining/whisper/demo_from_openai.py | 6 +++++-
4 files changed, 31 insertions(+), 17 deletions(-)
diff --git a/examples/industrial_data_pretraining/paraformer/demo.py b/examples/industrial_data_pretraining/paraformer/demo.py
index 965f8f3..499791f 100644
--- a/examples/industrial_data_pretraining/paraformer/demo.py
+++ b/examples/industrial_data_pretraining/paraformer/demo.py
@@ -7,10 +7,11 @@
model = AutoModel(model="iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
model_revision="v2.0.4",
- # vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
- # vad_model_revision="v2.0.4",
- # punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
- # punc_model_revision="v2.0.4",
+ vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+ vad_model_revision="v2.0.4",
+ vad_kwargs={"max_single_segment_time": 60},
+ punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
+ punc_model_revision="v2.0.4",
# spk_model="iic/speech_campplus_sv_zh-cn_16k-common",
# spk_model_revision="v2.0.2",
)
diff --git a/examples/industrial_data_pretraining/whisper/demo.py b/examples/industrial_data_pretraining/whisper/demo.py
index ddebbdf..a073f68 100644
--- a/examples/industrial_data_pretraining/whisper/demo.py
+++ b/examples/industrial_data_pretraining/whisper/demo.py
@@ -10,6 +10,7 @@
model = AutoModel(model="iic/Whisper-large-v3",
model_revision="v2.0.5",
vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+ vad_kwargs={"max_single_segment_time": 30},
)
res = model.generate(
diff --git a/examples/industrial_data_pretraining/whisper/demo_from_openai.py b/examples/industrial_data_pretraining/whisper/demo_from_openai.py
index 5cac06b..9a22764 100644
--- a/examples/industrial_data_pretraining/whisper/demo_from_openai.py
+++ b/examples/industrial_data_pretraining/whisper/demo_from_openai.py
@@ -10,7 +10,11 @@
# model = AutoModel(model="Whisper-small", hub="openai")
# model = AutoModel(model="Whisper-medium", hub="openai")
# model = AutoModel(model="Whisper-large-v2", hub="openai")
-model = AutoModel(model="Whisper-large-v3", hub="openai", vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",)
+model = AutoModel(model="Whisper-large-v3",
+ vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+ vad_kwargs={"max_single_segment_time": 30},
+ hub="openai",
+ )
res = model.generate(
language=None,
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py
index 69aef28..39f91e9 100644
--- a/funasr/auto/auto_model.py
+++ b/funasr/auto/auto_model.py
@@ -68,7 +68,8 @@
data_list.append(data)
key_list.append(key)
else:
- key = "rand_key_" + ''.join(random.choice(chars) for _ in range(13))
+ if key is None:
+ key = "rand_key_" + ''.join(random.choice(chars) for _ in range(13))
data_list = [data_in]
key_list = [key]
elif isinstance(data_in, (list, tuple)):
@@ -105,18 +106,23 @@
# if vad_model is not None, build vad model else None
vad_model = kwargs.get("vad_model", None)
- vad_kwargs = kwargs.get("vad_model_revision", None)
if vad_model is not None:
logging.info("Building VAD model.")
- vad_kwargs = {"model": vad_model, "model_revision": vad_kwargs, "device": kwargs["device"]}
+ vad_kwargs = {} if kwargs.get("vad_kwargs", {}) is None else kwargs.get("vad_kwargs", {})
+ vad_kwargs["model"] = vad_model
+ vad_kwargs["model_revision"] = kwargs.get("vad_model_revision", None)
+ vad_kwargs["device"] = kwargs["device"]
vad_model, vad_kwargs = self.build_model(**vad_kwargs)
# if punc_model is not None, build punc model else None
punc_model = kwargs.get("punc_model", None)
- punc_kwargs = kwargs.get("punc_model_revision", None)
+
if punc_model is not None:
logging.info("Building punc model.")
- punc_kwargs = {"model": punc_model, "model_revision": punc_kwargs, "device": kwargs["device"]}
+ punc_kwargs = {} if kwargs.get("punc_kwargs", {}) is None else kwargs.get("punc_kwargs", {})
+ punc_kwargs["model"] = punc_model
+ punc_kwargs["model_revision"] = kwargs.get("punc_model_revision", None)
+ punc_kwargs["device"] = kwargs["device"]
punc_model, punc_kwargs = self.build_model(**punc_kwargs)
# if spk_model is not None, build spk model else None
@@ -124,7 +130,10 @@
spk_kwargs = kwargs.get("spk_model_revision", None)
if spk_model is not None:
logging.info("Building SPK model.")
- spk_kwargs = {"model": spk_model, "model_revision": spk_kwargs, "device": kwargs["device"]}
+ spk_kwargs = {} if kwargs.get("spk_kwargs", {}) is None else kwargs.get("spk_kwargs", {})
+ spk_kwargs["model"] = spk_model
+ spk_kwargs["model_revision"] = kwargs.get("spk_model_revision", None)
+ spk_kwargs["device"] = kwargs["device"]
spk_model, spk_kwargs = self.build_model(**spk_kwargs)
self.cb_model = ClusterBackend().to(kwargs["device"])
spk_mode = kwargs.get("spk_mode", 'punc_segment')
@@ -162,10 +171,7 @@
tokenizer = kwargs.get("tokenizer", None)
if tokenizer is not None:
tokenizer_class = tables.tokenizer_classes.get(tokenizer)
- tokenizer_conf = kwargs.get("tokenizer_conf", {})
- tokenizer = tokenizer_class(**tokenizer_conf)
-
-
+ tokenizer = tokenizer_class(**kwargs.get("tokenizer_conf", {}))
kwargs["token_list"] = tokenizer.token_list if hasattr(tokenizer, "token_list") else None
kwargs["token_list"] = tokenizer.get_vocab() if hasattr(tokenizer, "get_vocab") else kwargs["token_list"]
vocab_size = len(kwargs["token_list"]) if kwargs["token_list"] is not None else -1
@@ -178,12 +184,14 @@
kwargs["input_size"] = None
if frontend is not None:
frontend_class = tables.frontend_classes.get(frontend)
- frontend = frontend_class(**kwargs["frontend_conf"])
+ frontend = frontend_class(**kwargs.get("frontend_conf", {}))
kwargs["input_size"] = frontend.output_size() if hasattr(frontend, "output_size") else None
kwargs["frontend"] = frontend
# build model
model_class = tables.model_classes.get(kwargs["model"])
- model = model_class(**kwargs, **kwargs.get("model_conf", {}), vocab_size=vocab_size)
+ model_conf = kwargs.get("model_conf", {})
+ deep_update(model_conf, kwargs)
+ model = model_class(**model_conf, vocab_size=vocab_size)
model.to(device)
# init_param
--
Gitblit v1.9.1