From 580b11b57ac4b62f7e2acda73813a4e10e8e4cd3 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 10 十月 2023 17:17:29 +0800
Subject: [PATCH] v0.8.0
---
funasr/bin/asr_infer.py | 807 ++++++++++++++++++++++++++++++---------------------------
1 files changed, 426 insertions(+), 381 deletions(-)
diff --git a/funasr/bin/asr_infer.py b/funasr/bin/asr_infer.py
index 5946271..3117e5d 100644
--- a/funasr/bin/asr_infer.py
+++ b/funasr/bin/asr_infer.py
@@ -1,65 +1,44 @@
-# -*- encoding: utf-8 -*-
#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
# MIT License (https://opensource.org/licenses/MIT)
-import argparse
-import logging
-import sys
-import time
-import copy
-import os
+
import codecs
+import copy
+import logging
+import os
+import re
import tempfile
-import requests
from pathlib import Path
+from typing import Any
+from typing import Dict
+from typing import List
from typing import Optional
-from typing import Sequence
from typing import Tuple
from typing import Union
-from typing import Dict
-from typing import Any
-from typing import List
import numpy as np
+import requests
import torch
from packaging.version import parse as V
-from typeguard import check_argument_types
-from typeguard import check_return_type
-from funasr.fileio.datadir_writer import DatadirWriter
+from funasr.build_utils.build_model_from_file import build_model_from_file
+from funasr.models.e2e_asr_contextual_paraformer import NeatContextualParaformer
+from funasr.models.e2e_asr_paraformer import BiCifParaformer, ContextualParaformer
+from funasr.models.frontend.wav_frontend import WavFrontend, WavFrontendOnline
from funasr.modules.beam_search.beam_search import BeamSearch
-# from funasr.modules.beam_search.beam_search import BeamSearchPara as BeamSearch
from funasr.modules.beam_search.beam_search import Hypothesis
+from funasr.modules.beam_search.beam_search_sa_asr import Hypothesis as HypothesisSAASR
from funasr.modules.beam_search.beam_search_transducer import BeamSearchTransducer
from funasr.modules.beam_search.beam_search_transducer import Hypothesis as HypothesisTransducer
-from funasr.modules.beam_search.beam_search_sa_asr import Hypothesis as HypothesisSAASR
from funasr.modules.scorers.ctc import CTCPrefixScorer
from funasr.modules.scorers.length_bonus import LengthBonus
-from funasr.modules.subsampling import TooShortUttError
-from funasr.tasks.asr import ASRTask
-from funasr.tasks.lm import LMTask
+from funasr.build_utils.build_asr_model import frontend_choices
from funasr.text.build_tokenizer import build_tokenizer
from funasr.text.token_id_converter import TokenIDConverter
from funasr.torch_utils.device_funcs import to_device
-from funasr.torch_utils.set_all_random_seed import set_all_random_seed
-from funasr.utils import config_argparse
-from funasr.utils.cli_utils import get_commandline_args
-from funasr.utils.types import str2bool
-from funasr.utils.types import str2triple_str
-from funasr.utils.types import str_or_none
-from funasr.utils import asr_utils, wav_utils, postprocess_utils
-from funasr.models.frontend.wav_frontend import WavFrontend, WavFrontendOnline
-from funasr.models.e2e_asr_paraformer import BiCifParaformer, ContextualParaformer
-from funasr.models.e2e_asr_contextual_paraformer import NeatContextualParaformer
-from funasr.export.models.e2e_asr_paraformer import Paraformer as Paraformer_export
from funasr.utils.timestamp_tools import ts_prediction_lfr6_standard
-from funasr.bin.tp_infer import Speech2Timestamp
-from funasr.bin.vad_infer import Speech2VadSegment
-from funasr.bin.punc_infer import Text2Punc
-from funasr.utils.vad_utils import slice_padding_fbank
-from funasr.tasks.vad import VADTask
-from funasr.utils.timestamp_tools import time_stamp_sentence, ts_prediction_lfr6_standard
-from funasr.tasks.asr import frontend_choices
+
class Speech2Text:
"""Speech2Text class
@@ -72,36 +51,35 @@
[(text, token, token_int, hypothesis object), ...]
"""
-
+
def __init__(
- self,
- asr_train_config: Union[Path, str] = None,
- asr_model_file: Union[Path, str] = None,
- cmvn_file: Union[Path, str] = None,
- lm_train_config: Union[Path, str] = None,
- lm_file: Union[Path, str] = None,
- token_type: str = None,
- bpemodel: str = None,
- device: str = "cpu",
- maxlenratio: float = 0.0,
- minlenratio: float = 0.0,
- batch_size: int = 1,
- dtype: str = "float32",
- beam_size: int = 20,
- ctc_weight: float = 0.5,
- lm_weight: float = 1.0,
- ngram_weight: float = 0.9,
- penalty: float = 0.0,
- nbest: int = 1,
- streaming: bool = False,
- frontend_conf: dict = None,
- **kwargs,
+ self,
+ asr_train_config: Union[Path, str] = None,
+ asr_model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
+ lm_train_config: Union[Path, str] = None,
+ lm_file: Union[Path, str] = None,
+ token_type: str = None,
+ bpemodel: str = None,
+ device: str = "cpu",
+ maxlenratio: float = 0.0,
+ minlenratio: float = 0.0,
+ batch_size: int = 1,
+ dtype: str = "float32",
+ beam_size: int = 20,
+ ctc_weight: float = 0.5,
+ lm_weight: float = 1.0,
+ ngram_weight: float = 0.9,
+ penalty: float = 0.0,
+ nbest: int = 1,
+ streaming: bool = False,
+ frontend_conf: dict = None,
+ **kwargs,
):
- assert check_argument_types()
-
+
# 1. Build ASR model
scorers = {}
- asr_model, asr_train_args = ASRTask.build_model_from_file(
+ asr_model, asr_train_args = build_model_from_file(
asr_train_config, asr_model_file, cmvn_file, device
)
frontend = None
@@ -109,16 +87,15 @@
if asr_train_args.frontend == 'wav_frontend':
frontend = WavFrontend(cmvn_file=cmvn_file, **asr_train_args.frontend_conf)
else:
- from funasr.tasks.asr import frontend_choices
frontend_class = frontend_choices.get_class(asr_train_args.frontend)
frontend = frontend_class(**asr_train_args.frontend_conf).eval()
-
+
logging.info("asr_model: {}".format(asr_model))
logging.info("asr_train_args: {}".format(asr_train_args))
asr_model.to(dtype=getattr(torch, dtype)).eval()
-
+
decoder = asr_model.decoder
-
+
ctc = CTCPrefixScorer(ctc=asr_model.ctc, eos=asr_model.eos)
token_list = asr_model.token_list
scorers.update(
@@ -126,24 +103,24 @@
ctc=ctc,
length_bonus=LengthBonus(len(token_list)),
)
-
+
# 2. Build Language model
if lm_train_config is not None:
- lm, lm_train_args = LMTask.build_model_from_file(
+ lm, lm_train_args = build_model_from_file(
lm_train_config, lm_file, None, device
)
scorers["lm"] = lm.lm
-
+
# 3. Build ngram model
# ngram is not supported now
ngram = None
scorers["ngram"] = ngram
-
+
# 4. Build BeamSearch object
# transducer is not supported now
beam_search_transducer = None
from funasr.modules.beam_search.beam_search import BeamSearch
-
+
weights = dict(
decoder=1.0 - ctc_weight,
ctc=ctc_weight,
@@ -161,13 +138,13 @@
token_list=token_list,
pre_beam_score_key=None if ctc_weight == 1.0 else "full",
)
-
+
# 5. [Optional] Build Text converter: e.g. bpe-sym -> Text
if token_type is None:
token_type = asr_train_args.token_type
if bpemodel is None:
bpemodel = asr_train_args.bpemodel
-
+
if token_type is None:
tokenizer = None
elif token_type == "bpe":
@@ -179,7 +156,7 @@
tokenizer = build_tokenizer(token_type=token_type)
converter = TokenIDConverter(token_list=token_list)
logging.info(f"Text tokenizer: {tokenizer}")
-
+
self.asr_model = asr_model
self.asr_train_args = asr_train_args
self.converter = converter
@@ -192,10 +169,10 @@
self.dtype = dtype
self.nbest = nbest
self.frontend = frontend
-
+
@torch.no_grad()
def __call__(
- self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None
+ self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None
) -> List[
Tuple[
Optional[str],
@@ -212,12 +189,11 @@
text, token, token_int, hyp
"""
- assert check_argument_types()
-
+
# Input as audio signal
if isinstance(speech, np.ndarray):
speech = torch.tensor(speech)
-
+
if self.frontend is not None:
feats, feats_len = self.frontend.forward(speech, speech_lengths)
feats = to_device(feats, device=self.device)
@@ -228,48 +204,48 @@
feats_len = speech_lengths
lfr_factor = max(1, (feats.size()[-1] // 80) - 1)
batch = {"speech": feats, "speech_lengths": feats_len}
-
+
# a. To device
batch = to_device(batch, device=self.device)
-
+
# b. Forward Encoder
enc, _ = self.asr_model.encode(**batch)
if isinstance(enc, tuple):
enc = enc[0]
assert len(enc) == 1, len(enc)
-
+
# c. Passed the encoder result and the beam search
nbest_hyps = self.beam_search(
x=enc[0], maxlenratio=self.maxlenratio, minlenratio=self.minlenratio
)
-
+
nbest_hyps = nbest_hyps[: self.nbest]
-
+
results = []
for hyp in nbest_hyps:
assert isinstance(hyp, (Hypothesis)), type(hyp)
-
+
# remove sos/eos and get results
last_pos = -1
if isinstance(hyp.yseq, list):
token_int = hyp.yseq[1:last_pos]
else:
token_int = hyp.yseq[1:last_pos].tolist()
-
+
# remove blank symbol id, which is assumed to be 0
token_int = list(filter(lambda x: x != 0, token_int))
-
+
# Change integer-ids to tokens
token = self.converter.ids2tokens(token_int)
-
+
if self.tokenizer is not None:
text = self.tokenizer.tokens2text(token)
else:
text = None
results.append((text, token, token_int, hyp))
-
- assert check_return_type(results)
+
return results
+
class Speech2TextParaformer:
"""Speech2Text class
@@ -304,15 +280,15 @@
nbest: int = 1,
frontend_conf: dict = None,
hotword_list_or_file: str = None,
+ clas_scale: float = 1.0,
+ decoding_ind: int = 0,
**kwargs,
):
- assert check_argument_types()
# 1. Build ASR model
scorers = {}
- from funasr.tasks.asr import ASRTaskParaformer as ASRTask
- asr_model, asr_train_args = ASRTask.build_model_from_file(
- asr_train_config, asr_model_file, cmvn_file, device
+ asr_model, asr_train_args = build_model_from_file(
+ asr_train_config, asr_model_file, cmvn_file, device, mode="paraformer"
)
frontend = None
if asr_train_args.frontend is not None and asr_train_args.frontend_conf is not None:
@@ -334,8 +310,8 @@
# 2. Build Language model
if lm_train_config is not None:
- lm, lm_train_args = LMTask.build_model_from_file(
- lm_train_config, lm_file, device
+ lm, lm_train_args = build_model_from_file(
+ lm_train_config, lm_file, None, device, task_name="lm"
)
scorers["lm"] = lm.lm
@@ -396,10 +372,12 @@
self.asr_train_args = asr_train_args
self.converter = converter
self.tokenizer = tokenizer
+ self.cmvn_file = cmvn_file
# 6. [Optional] Build hotword list from str, local file or url
self.hotword_list = None
self.hotword_list = self.generate_hotwords_list(hotword_list_or_file)
+ self.clas_scale = clas_scale
is_use_lm = lm_weight != 0.0 and lm_file is not None
if (ctc_weight == 0.0 or asr_model.ctc == None) and not is_use_lm:
@@ -414,13 +392,14 @@
self.nbest = nbest
self.frontend = frontend
self.encoder_downsampling_factor = 1
+ self.decoding_ind = decoding_ind
if asr_train_args.encoder == "data2vec_encoder" or asr_train_args.encoder_conf["input_layer"] == "conv2d":
self.encoder_downsampling_factor = 4
@torch.no_grad()
def __call__(
self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None,
- begin_time: int = 0, end_time: int = None,
+ decoding_ind: int = None, begin_time: int = 0, end_time: int = None,
):
"""Inference
@@ -430,7 +409,6 @@
text, token, token_int, hyp
"""
- assert check_argument_types()
# Input as audio signal
if isinstance(speech, np.ndarray):
@@ -451,7 +429,9 @@
batch = to_device(batch, device=self.device)
# b. Forward Encoder
- enc, enc_len = self.asr_model.encode(**batch)
+ if decoding_ind is None:
+ decoding_ind = 0 if self.decoding_ind is None else self.decoding_ind
+ enc, enc_len = self.asr_model.encode(**batch, ind=decoding_ind)
if isinstance(enc, tuple):
enc = enc[0]
# assert len(enc) == 1, len(enc)
@@ -463,18 +443,25 @@
pre_token_length = pre_token_length.round().long()
if torch.max(pre_token_length) < 1:
return []
- if not isinstance(self.asr_model, ContextualParaformer) and not isinstance(self.asr_model, NeatContextualParaformer):
+ if not isinstance(self.asr_model, ContextualParaformer) and \
+ not isinstance(self.asr_model, NeatContextualParaformer):
if self.hotword_list:
logging.warning("Hotword is given but asr model is not a ContextualParaformer.")
- decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds, pre_token_length)
+ decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds,
+ pre_token_length)
decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]
else:
- decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds, pre_token_length, hw_list=self.hotword_list)
+ decoder_outs = self.asr_model.cal_decoder_with_predictor(enc,
+ enc_len,
+ pre_acoustic_embeds,
+ pre_token_length,
+ hw_list=self.hotword_list,
+ clas_scale=self.clas_scale)
decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]
if isinstance(self.asr_model, BiCifParaformer):
_, _, us_alphas, us_peaks = self.asr_model.calc_predictor_timestamp(enc, enc_len,
- pre_token_length) # test no bias cif2
+ pre_token_length) # test no bias cif2
results = []
b, n, d = decoder_out.size()
@@ -488,15 +475,20 @@
nbest_hyps = nbest_hyps[: self.nbest]
else:
- yseq = am_scores.argmax(dim=-1)
- score = am_scores.max(dim=-1)[0]
- score = torch.sum(score, dim=-1)
- # pad with mask tokens to ensure compatibility with sos/eos tokens
- yseq = torch.tensor(
- [self.asr_model.sos] + yseq.tolist() + [self.asr_model.eos], device=yseq.device
- )
+ if pre_token_length[i] == 0:
+ yseq = torch.tensor(
+ [self.asr_model.sos] + [self.asr_model.eos], device=pre_acoustic_embeds.device
+ )
+ score = torch.tensor(0.0, device=pre_acoustic_embeds.device)
+ else:
+ yseq = am_scores.argmax(dim=-1)
+ score = am_scores.max(dim=-1)[0]
+ score = torch.sum(score, dim=-1)
+ # pad with mask tokens to ensure compatibility with sos/eos tokens
+ yseq = torch.tensor(
+ [self.asr_model.sos] + yseq.tolist() + [self.asr_model.eos], device=yseq.device
+ )
nbest_hyps = [Hypothesis(yseq=yseq, score=score)]
-
for hyp in nbest_hyps:
assert isinstance(hyp, (Hypothesis)), type(hyp)
@@ -519,17 +511,53 @@
text = None
timestamp = []
if isinstance(self.asr_model, BiCifParaformer):
- _, timestamp = ts_prediction_lfr6_standard(us_alphas[i][:enc_len[i]*3],
- us_peaks[i][:enc_len[i]*3],
- copy.copy(token),
- vad_offset=begin_time)
+ _, timestamp = ts_prediction_lfr6_standard(us_alphas[i][:enc_len[i] * 3],
+ us_peaks[i][:enc_len[i] * 3],
+ copy.copy(token),
+ vad_offset=begin_time)
results.append((text, token, token_int, hyp, timestamp, enc_len_batch_total, lfr_factor))
-
- # assert check_return_type(results)
return results
def generate_hotwords_list(self, hotword_list_or_file):
+ def load_seg_dict(seg_dict_file):
+ seg_dict = {}
+ assert isinstance(seg_dict_file, str)
+ with open(seg_dict_file, "r", encoding="utf8") as f:
+ lines = f.readlines()
+ for line in lines:
+ s = line.strip().split()
+ key = s[0]
+ value = s[1:]
+ seg_dict[key] = " ".join(value)
+ return seg_dict
+
+ def seg_tokenize(txt, seg_dict):
+ pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
+ out_txt = ""
+ for word in txt:
+ word = word.lower()
+ if word in seg_dict:
+ out_txt += seg_dict[word] + " "
+ else:
+ if pattern.match(word):
+ for char in word:
+ if char in seg_dict:
+ out_txt += seg_dict[char] + " "
+ else:
+ out_txt += "<unk>" + " "
+ else:
+ out_txt += "<unk>" + " "
+ return out_txt.strip().split()
+
+ seg_dict = None
+ if self.cmvn_file is not None:
+ model_dir = os.path.dirname(self.cmvn_file)
+ seg_dict_file = os.path.join(model_dir, 'seg_dict')
+ if os.path.exists(seg_dict_file):
+ seg_dict = load_seg_dict(seg_dict_file)
+ else:
+ seg_dict = None
# for None
if hotword_list_or_file is None:
hotword_list = None
@@ -541,8 +569,11 @@
with codecs.open(hotword_list_or_file, 'r') as fin:
for line in fin.readlines():
hw = line.strip()
+ hw_list = hw.split()
+ if seg_dict is not None:
+ hw_list = seg_tokenize(hw_list, seg_dict)
hotword_str_list.append(hw)
- hotword_list.append(self.converter.tokens2ids([i for i in hw]))
+ hotword_list.append(self.converter.tokens2ids(hw_list))
hotword_list.append([self.asr_model.sos])
hotword_str_list.append('<s>')
logging.info("Initialized hotword list from file: {}, hotword list: {}."
@@ -562,8 +593,11 @@
with codecs.open(hotword_list_or_file, 'r') as fin:
for line in fin.readlines():
hw = line.strip()
+ hw_list = hw.split()
+ if seg_dict is not None:
+ hw_list = seg_tokenize(hw_list, seg_dict)
hotword_str_list.append(hw)
- hotword_list.append(self.converter.tokens2ids([i for i in hw]))
+ hotword_list.append(self.converter.tokens2ids(hw_list))
hotword_list.append([self.asr_model.sos])
hotword_str_list.append('<s>')
logging.info("Initialized hotword list from file: {}, hotword list: {}."
@@ -575,13 +609,17 @@
hotword_str_list = []
for hw in hotword_list_or_file.strip().split():
hotword_str_list.append(hw)
- hotword_list.append(self.converter.tokens2ids([i for i in hw]))
+ hw_list = hw.strip().split()
+ if seg_dict is not None:
+ hw_list = seg_tokenize(hw_list, seg_dict)
+ hotword_list.append(self.converter.tokens2ids(hw_list))
hotword_list.append([self.asr_model.sos])
hotword_str_list.append('<s>')
logging.info("Hotword list: {}.".format(hotword_str_list))
else:
hotword_list = None
return hotword_list
+
class Speech2TextParaformerOnline:
"""Speech2Text class
@@ -618,13 +656,11 @@
hotword_list_or_file: str = None,
**kwargs,
):
- assert check_argument_types()
# 1. Build ASR model
scorers = {}
- from funasr.tasks.asr import ASRTaskParaformer as ASRTask
- asr_model, asr_train_args = ASRTask.build_model_from_file(
- asr_train_config, asr_model_file, cmvn_file, device
+ asr_model, asr_train_args = build_model_from_file(
+ asr_train_config, asr_model_file, cmvn_file, device, mode="paraformer"
)
frontend = None
if asr_train_args.frontend is not None and asr_train_args.frontend_conf is not None:
@@ -646,8 +682,8 @@
# 2. Build Language model
if lm_train_config is not None:
- lm, lm_train_args = LMTask.build_model_from_file(
- lm_train_config, lm_file, device
+ lm, lm_train_args = build_model_from_file(
+ lm_train_config, lm_file, None, device, task_name="lm"
)
scorers["lm"] = lm.lm
@@ -739,7 +775,6 @@
text, token, token_int, hyp
"""
- assert check_argument_types()
results = []
cache_en = cache["encoder"]
if speech.shape[1] < 16 * 60 and cache_en["is_final"]:
@@ -781,7 +816,7 @@
enc_len_batch_total = torch.sum(enc_len).item() * self.encoder_downsampling_factor
predictor_outs = self.asr_model.calc_predictor_chunk(enc, cache)
- pre_acoustic_embeds, pre_token_length= predictor_outs[0], predictor_outs[1]
+ pre_acoustic_embeds, pre_token_length = predictor_outs[0], predictor_outs[1]
if torch.max(pre_token_length) < 1:
return []
decoder_outs = self.asr_model.cal_decoder_with_predictor_chunk(enc, pre_acoustic_embeds, cache)
@@ -823,12 +858,19 @@
# Change integer-ids to tokens
token = self.converter.ids2tokens(token_int)
- token = " ".join(token)
+ postprocessed_result = ""
+ for item in token:
+ if item.endswith('@@'):
+ postprocessed_result += item[:-2]
+ elif re.match('^[a-zA-Z]+$', item):
+ postprocessed_result += item + " "
+ else:
+ postprocessed_result += item
- results.append(token)
+ results.append(postprocessed_result)
- # assert check_return_type(results)
return results
+
class Speech2TextUniASR:
"""Speech2Text class
@@ -867,13 +909,11 @@
frontend_conf: dict = None,
**kwargs,
):
- assert check_argument_types()
# 1. Build ASR model
scorers = {}
- from funasr.tasks.asr import ASRTaskUniASR as ASRTask
- asr_model, asr_train_args = ASRTask.build_model_from_file(
- asr_train_config, asr_model_file, cmvn_file, device
+ asr_model, asr_train_args = build_model_from_file(
+ asr_train_config, asr_model_file, cmvn_file, device, mode="uniasr"
)
frontend = None
if asr_train_args.frontend is not None and asr_train_args.frontend_conf is not None:
@@ -899,8 +939,8 @@
# 2. Build Language model
if lm_train_config is not None:
- lm, lm_train_args = LMTask.build_model_from_file(
- lm_train_config, lm_file, device
+ lm, lm_train_args = build_model_from_file(
+ lm_train_config, lm_file, device, "lm"
)
scorers["lm"] = lm.lm
@@ -992,7 +1032,6 @@
text, token, token_int, hyp
"""
- assert check_argument_types()
# Input as audio signal
if isinstance(speech, np.ndarray):
@@ -1060,9 +1099,8 @@
text = None
results.append((text, token, token_int, hyp))
- assert check_return_type(results)
return results
-
+
class Speech2TextMFCCA:
"""Speech2Text class
@@ -1075,45 +1113,43 @@
[(text, token, token_int, hypothesis object), ...]
"""
-
+
def __init__(
- self,
- asr_train_config: Union[Path, str] = None,
- asr_model_file: Union[Path, str] = None,
- cmvn_file: Union[Path, str] = None,
- lm_train_config: Union[Path, str] = None,
- lm_file: Union[Path, str] = None,
- token_type: str = None,
- bpemodel: str = None,
- device: str = "cpu",
- maxlenratio: float = 0.0,
- minlenratio: float = 0.0,
- batch_size: int = 1,
- dtype: str = "float32",
- beam_size: int = 20,
- ctc_weight: float = 0.5,
- lm_weight: float = 1.0,
- ngram_weight: float = 0.9,
- penalty: float = 0.0,
- nbest: int = 1,
- streaming: bool = False,
- **kwargs,
+ self,
+ asr_train_config: Union[Path, str] = None,
+ asr_model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
+ lm_train_config: Union[Path, str] = None,
+ lm_file: Union[Path, str] = None,
+ token_type: str = None,
+ bpemodel: str = None,
+ device: str = "cpu",
+ maxlenratio: float = 0.0,
+ minlenratio: float = 0.0,
+ batch_size: int = 1,
+ dtype: str = "float32",
+ beam_size: int = 20,
+ ctc_weight: float = 0.5,
+ lm_weight: float = 1.0,
+ ngram_weight: float = 0.9,
+ penalty: float = 0.0,
+ nbest: int = 1,
+ streaming: bool = False,
+ **kwargs,
):
- assert check_argument_types()
-
+
# 1. Build ASR model
- from funasr.tasks.asr import ASRTaskMFCCA as ASRTask
scorers = {}
- asr_model, asr_train_args = ASRTask.build_model_from_file(
+ asr_model, asr_train_args = build_model_from_file(
asr_train_config, asr_model_file, cmvn_file, device
)
-
+
logging.info("asr_model: {}".format(asr_model))
logging.info("asr_train_args: {}".format(asr_train_args))
asr_model.to(dtype=getattr(torch, dtype)).eval()
-
+
decoder = asr_model.decoder
-
+
ctc = CTCPrefixScorer(ctc=asr_model.ctc, eos=asr_model.eos)
token_list = asr_model.token_list
scorers.update(
@@ -1121,11 +1157,11 @@
ctc=ctc,
length_bonus=LengthBonus(len(token_list)),
)
-
+
# 2. Build Language model
if lm_train_config is not None:
- lm, lm_train_args = LMTask.build_model_from_file(
- lm_train_config, lm_file, device
+ lm, lm_train_args = build_model_from_file(
+ lm_train_config, lm_file, None, device, task_name="lm"
)
lm.to(device)
scorers["lm"] = lm.lm
@@ -1133,11 +1169,11 @@
# ngram is not supported now
ngram = None
scorers["ngram"] = ngram
-
+
# 4. Build BeamSearch object
# transducer is not supported now
beam_search_transducer = None
-
+
weights = dict(
decoder=1.0 - ctc_weight,
ctc=ctc_weight,
@@ -1161,7 +1197,7 @@
token_type = asr_train_args.token_type
if bpemodel is None:
bpemodel = asr_train_args.bpemodel
-
+
if token_type is None:
tokenizer = None
elif token_type == "bpe":
@@ -1173,7 +1209,7 @@
tokenizer = build_tokenizer(token_type=token_type)
converter = TokenIDConverter(token_list=token_list)
logging.info(f"Text tokenizer: {tokenizer}")
-
+
self.asr_model = asr_model
self.asr_train_args = asr_train_args
self.converter = converter
@@ -1185,10 +1221,10 @@
self.device = device
self.dtype = dtype
self.nbest = nbest
-
+
@torch.no_grad()
def __call__(
- self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None
+ self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None
) -> List[
Tuple[
Optional[str],
@@ -1205,7 +1241,6 @@
text, token, token_int, hyp
"""
- assert check_argument_types()
# Input as audio signal
if isinstance(speech, np.ndarray):
speech = torch.tensor(speech)
@@ -1216,46 +1251,45 @@
# lenghts: (1,)
lengths = speech.new_full([1], dtype=torch.long, fill_value=speech.size(1))
batch = {"speech": speech, "speech_lengths": lengths}
-
+
# a. To device
batch = to_device(batch, device=self.device)
-
+
# b. Forward Encoder
enc, _ = self.asr_model.encode(**batch)
-
+
assert len(enc) == 1, len(enc)
-
+
# c. Passed the encoder result and the beam search
nbest_hyps = self.beam_search(
x=enc[0], maxlenratio=self.maxlenratio, minlenratio=self.minlenratio
)
-
+
nbest_hyps = nbest_hyps[: self.nbest]
-
+
results = []
for hyp in nbest_hyps:
assert isinstance(hyp, (Hypothesis)), type(hyp)
-
+
# remove sos/eos and get results
last_pos = -1
if isinstance(hyp.yseq, list):
token_int = hyp.yseq[1:last_pos]
else:
token_int = hyp.yseq[1:last_pos].tolist()
-
+
# remove blank symbol id, which is assumed to be 0
token_int = list(filter(lambda x: x != 0, token_int))
-
+
# Change integer-ids to tokens
token = self.converter.ids2tokens(token_int)
-
+
if self.tokenizer is not None:
text = self.tokenizer.tokens2text(token)
else:
text = None
results.append((text, token, token_int, hyp))
-
- assert check_return_type(results)
+
return results
@@ -1283,45 +1317,44 @@
right_context: Number of frames in right context AFTER subsampling.
display_partial_hypotheses: Whether to display partial hypotheses.
"""
-
+
def __init__(
- self,
- asr_train_config: Union[Path, str] = None,
- asr_model_file: Union[Path, str] = None,
- cmvn_file: Union[Path, str] = None,
- beam_search_config: Dict[str, Any] = None,
- lm_train_config: Union[Path, str] = None,
- lm_file: Union[Path, str] = None,
- token_type: str = None,
- bpemodel: str = None,
- device: str = "cpu",
- beam_size: int = 5,
- dtype: str = "float32",
- lm_weight: float = 1.0,
- quantize_asr_model: bool = False,
- quantize_modules: List[str] = None,
- quantize_dtype: str = "qint8",
- nbest: int = 1,
- streaming: bool = False,
- simu_streaming: bool = False,
- chunk_size: int = 16,
- left_context: int = 32,
- right_context: int = 0,
- display_partial_hypotheses: bool = False,
+ self,
+ asr_train_config: Union[Path, str] = None,
+ asr_model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
+ beam_search_config: Dict[str, Any] = None,
+ lm_train_config: Union[Path, str] = None,
+ lm_file: Union[Path, str] = None,
+ token_type: str = None,
+ bpemodel: str = None,
+ device: str = "cpu",
+ beam_size: int = 5,
+ dtype: str = "float32",
+ lm_weight: float = 1.0,
+ quantize_asr_model: bool = False,
+ quantize_modules: List[str] = None,
+ quantize_dtype: str = "qint8",
+ nbest: int = 1,
+ streaming: bool = False,
+ fake_streaming: bool = False,
+ full_utt: bool = False,
+ chunk_size: int = 16,
+ left_context: int = 32,
+ right_context: int = 0,
+ display_partial_hypotheses: bool = False,
) -> None:
"""Construct a Speech2Text object."""
super().__init__()
-
- assert check_argument_types()
- from funasr.tasks.asr import ASRTransducerTask
- asr_model, asr_train_args = ASRTransducerTask.build_model_from_file(
+
+ asr_model, asr_train_args = build_model_from_file(
asr_train_config, asr_model_file, cmvn_file, device
)
-
+
frontend = None
if asr_train_args.frontend is not None and asr_train_args.frontend_conf is not None:
frontend = WavFrontend(cmvn_file=cmvn_file, **asr_train_args.frontend_conf)
-
+
if quantize_asr_model:
if quantize_modules is not None:
if not all([q in ["LSTM", "Linear"] for q in quantize_modules]):
@@ -1329,36 +1362,36 @@
"Only 'Linear' and 'LSTM' modules are currently supported"
" by PyTorch and in --quantize_modules"
)
-
+
q_config = set([getattr(torch.nn, q) for q in quantize_modules])
else:
q_config = {torch.nn.Linear}
-
+
if quantize_dtype == "float16" and (V(torch.__version__) < V("1.5.0")):
raise ValueError(
"float16 dtype for dynamic quantization is not supported with torch"
" version < 1.5.0. Switching to qint8 dtype instead."
)
q_dtype = getattr(torch, quantize_dtype)
-
+
asr_model = torch.quantization.quantize_dynamic(
asr_model, q_config, dtype=q_dtype
).eval()
else:
asr_model.to(dtype=getattr(torch, dtype)).eval()
-
+
if lm_train_config is not None:
- lm, lm_train_args = LMTask.build_model_from_file(
- lm_train_config, lm_file, device
+ lm, lm_train_args = build_model_from_file(
+ lm_train_config, lm_file, None, device, task_name="lm"
)
lm_scorer = lm.lm
else:
lm_scorer = None
-
+
# 4. Build BeamSearch object
if beam_search_config is None:
beam_search_config = {}
-
+
beam_search = BeamSearchTransducer(
asr_model.decoder,
asr_model.joint_network,
@@ -1368,14 +1401,14 @@
nbest=nbest,
**beam_search_config,
)
-
+
token_list = asr_model.token_list
-
+
if token_type is None:
token_type = asr_train_args.token_type
if bpemodel is None:
bpemodel = asr_train_args.bpemodel
-
+
if token_type is None:
tokenizer = None
elif token_type == "bpe":
@@ -1387,60 +1420,62 @@
tokenizer = build_tokenizer(token_type=token_type)
converter = TokenIDConverter(token_list=token_list)
logging.info(f"Text tokenizer: {tokenizer}")
-
+
self.asr_model = asr_model
self.asr_train_args = asr_train_args
self.device = device
self.dtype = dtype
self.nbest = nbest
-
+
self.converter = converter
self.tokenizer = tokenizer
-
+
self.beam_search = beam_search
self.streaming = streaming
- self.simu_streaming = simu_streaming
+ self.fake_streaming = fake_streaming
+ self.full_utt = full_utt
self.chunk_size = max(chunk_size, 0)
self.left_context = left_context
self.right_context = max(right_context, 0)
-
+
if not streaming or chunk_size == 0:
self.streaming = False
self.asr_model.encoder.dynamic_chunk_training = False
-
- if not simu_streaming or chunk_size == 0:
- self.simu_streaming = False
+
+ if not fake_streaming or chunk_size == 0:
+ self.fake_streaming = False
self.asr_model.encoder.dynamic_chunk_training = False
-
+
self.frontend = frontend
self.window_size = self.chunk_size + self.right_context
-
+
if self.streaming:
self._ctx = self.asr_model.encoder.get_encoder_input_size(
self.window_size
)
-
+ self._right_ctx = right_context
+
self.last_chunk_length = (
- self.asr_model.encoder.embed.min_frame_length + self.right_context + 1
+ self.asr_model.encoder.embed.min_frame_length + self.right_context + 1
)
self.reset_inference_cache()
-
+
def reset_inference_cache(self) -> None:
"""Reset Speech2Text parameters."""
self.frontend_cache = None
-
+
self.asr_model.encoder.reset_streaming_cache(
self.left_context, device=self.device
)
self.beam_search.reset_inference_cache()
-
+
self.num_processed_frames = torch.tensor([[0]], device=self.device)
-
+
@torch.no_grad()
def streaming_decode(
- self,
- speech: Union[torch.Tensor, np.ndarray],
- is_final: bool = True,
+ self,
+ speech: Union[torch.Tensor, np.ndarray],
+ is_final: bool = True,
) -> List[HypothesisTransducer]:
"""Speech2Text streaming call.
Args:
@@ -1458,13 +1493,13 @@
)
speech = torch.cat([speech, pad],
dim=0) # feats, feats_length = self.apply_frontend(speech, is_final=is_final)
-
+
feats = speech.unsqueeze(0).to(getattr(torch, self.dtype))
feats_lengths = feats.new_full([1], dtype=torch.long, fill_value=feats.size(1))
-
+
if self.asr_model.normalize is not None:
feats, feats_lengths = self.asr_model.normalize(feats, feats_lengths)
-
+
feats = to_device(feats, device=self.device)
feats_lengths = to_device(feats_lengths, device=self.device)
enc_out = self.asr_model.encoder.chunk_forward(
@@ -1476,16 +1511,47 @@
right_context=self.right_context,
)
nbest_hyps = self.beam_search(enc_out[0], is_final=is_final)
-
+
self.num_processed_frames += self.chunk_size
-
+
if is_final:
self.reset_inference_cache()
-
+
return nbest_hyps
-
+
@torch.no_grad()
- def simu_streaming_decode(self, speech: Union[torch.Tensor, np.ndarray]) -> List[HypothesisTransducer]:
+ def fake_streaming_decode(self, speech: Union[torch.Tensor, np.ndarray]) -> List[HypothesisTransducer]:
+ """Speech2Text call.
+ Args:
+ speech: Speech data. (S)
+ Returns:
+ nbest_hypothesis: N-best hypothesis.
+ """
+
+ if isinstance(speech, np.ndarray):
+ speech = torch.tensor(speech)
+
+ if self.frontend is not None:
+ speech = torch.unsqueeze(speech, axis=0)
+ speech_lengths = speech.new_full([1], dtype=torch.long, fill_value=speech.size(1))
+ feats, feats_lengths = self.frontend(speech, speech_lengths)
+ else:
+ feats = speech.unsqueeze(0).to(getattr(torch, self.dtype))
+ feats_lengths = feats.new_full([1], dtype=torch.long, fill_value=feats.size(1))
+
+ if self.asr_model.normalize is not None:
+ feats, feats_lengths = self.asr_model.normalize(feats, feats_lengths)
+
+ feats = to_device(feats, device=self.device)
+ feats_lengths = to_device(feats_lengths, device=self.device)
+ enc_out = self.asr_model.encoder.simu_chunk_forward(feats, feats_lengths, self.chunk_size, self.left_context,
+ self.right_context)
+ nbest_hyps = self.beam_search(enc_out[0])
+
+ return nbest_hyps
+
+ @torch.no_grad()
+ def full_utt_decode(self, speech: Union[torch.Tensor, np.ndarray]) -> List[HypothesisTransducer]:
"""Speech2Text call.
Args:
speech: Speech data. (S)
@@ -1493,24 +1559,28 @@
nbest_hypothesis: N-best hypothesis.
"""
assert check_argument_types()
-
+
if isinstance(speech, np.ndarray):
speech = torch.tensor(speech)
-
- feats = speech.unsqueeze(0).to(getattr(torch, self.dtype))
- feats_lengths = feats.new_full([1], dtype=torch.long, fill_value=feats.size(1))
-
+
+ if self.frontend is not None:
+ speech = torch.unsqueeze(speech, axis=0)
+ speech_lengths = speech.new_full([1], dtype=torch.long, fill_value=speech.size(1))
+ feats, feats_lengths = self.frontend(speech, speech_lengths)
+ else:
+ feats = speech.unsqueeze(0).to(getattr(torch, self.dtype))
+ feats_lengths = feats.new_full([1], dtype=torch.long, fill_value=feats.size(1))
+
if self.asr_model.normalize is not None:
feats, feats_lengths = self.asr_model.normalize(feats, feats_lengths)
-
+
feats = to_device(feats, device=self.device)
feats_lengths = to_device(feats_lengths, device=self.device)
- enc_out = self.asr_model.encoder.simu_chunk_forward(feats, feats_lengths, self.chunk_size, self.left_context,
- self.right_context)
+ enc_out = self.asr_model.encoder.full_utt_forward(feats, feats_lengths)
nbest_hyps = self.beam_search(enc_out[0])
-
+
return nbest_hyps
-
+
@torch.no_grad()
def __call__(self, speech: Union[torch.Tensor, np.ndarray]) -> List[HypothesisTransducer]:
"""Speech2Text call.
@@ -1519,23 +1589,27 @@
Returns:
nbest_hypothesis: N-best hypothesis.
"""
- assert check_argument_types()
-
+
if isinstance(speech, np.ndarray):
speech = torch.tensor(speech)
-
- feats = speech.unsqueeze(0).to(getattr(torch, self.dtype))
- feats_lengths = feats.new_full([1], dtype=torch.long, fill_value=feats.size(1))
-
+
+ if self.frontend is not None:
+ speech = torch.unsqueeze(speech, axis=0)
+ speech_lengths = speech.new_full([1], dtype=torch.long, fill_value=speech.size(1))
+ feats, feats_lengths = self.frontend(speech, speech_lengths)
+ else:
+ feats = speech.unsqueeze(0).to(getattr(torch, self.dtype))
+ feats_lengths = feats.new_full([1], dtype=torch.long, fill_value=feats.size(1))
+
feats = to_device(feats, device=self.device)
feats_lengths = to_device(feats_lengths, device=self.device)
-
- enc_out, _ = self.asr_model.encoder(feats, feats_lengths)
-
+
+ enc_out, _, _ = self.asr_model.encoder(feats, feats_lengths)
+
nbest_hyps = self.beam_search(enc_out[0])
-
+
return nbest_hyps
-
+
def hypotheses_to_results(self, nbest_hyps: List[HypothesisTransducer]) -> List[Any]:
"""Build partial or final results from the hypotheses.
Args:
@@ -1544,47 +1618,20 @@
results: Results containing different representation for the hypothesis.
"""
results = []
-
+
for hyp in nbest_hyps:
token_int = list(filter(lambda x: x != 0, hyp.yseq))
-
+
token = self.converter.ids2tokens(token_int)
-
+
if self.tokenizer is not None:
text = self.tokenizer.tokens2text(token)
else:
text = None
results.append((text, token, token_int, hyp))
-
- assert check_return_type(results)
-
+
+
return results
-
- @staticmethod
- def from_pretrained(
- model_tag: Optional[str] = None,
- **kwargs: Optional[Any],
- ) -> Speech2Text:
- """Build Speech2Text instance from the pretrained model.
- Args:
- model_tag: Model tag of the pretrained models.
- Return:
- : Speech2Text instance.
- """
- if model_tag is not None:
- try:
- from espnet_model_zoo.downloader import ModelDownloader
-
- except ImportError:
- logging.error(
- "`espnet_model_zoo` is not installed. "
- "Please install via `pip install -U espnet_model_zoo`."
- )
- raise
- d = ModelDownloader()
- kwargs.update(**d.download_and_unpack(model_tag))
-
- return Speech2TextTransducer(**kwargs)
class Speech2TextSAASR:
@@ -1598,53 +1645,53 @@
[(text, token, token_int, hypothesis object), ...]
"""
-
+
def __init__(
- self,
- asr_train_config: Union[Path, str] = None,
- asr_model_file: Union[Path, str] = None,
- cmvn_file: Union[Path, str] = None,
- lm_train_config: Union[Path, str] = None,
- lm_file: Union[Path, str] = None,
- token_type: str = None,
- bpemodel: str = None,
- device: str = "cpu",
- maxlenratio: float = 0.0,
- minlenratio: float = 0.0,
- batch_size: int = 1,
- dtype: str = "float32",
- beam_size: int = 20,
- ctc_weight: float = 0.5,
- lm_weight: float = 1.0,
- ngram_weight: float = 0.9,
- penalty: float = 0.0,
- nbest: int = 1,
- streaming: bool = False,
- frontend_conf: dict = None,
- **kwargs,
+ self,
+ asr_train_config: Union[Path, str] = None,
+ asr_model_file: Union[Path, str] = None,
+ cmvn_file: Union[Path, str] = None,
+ lm_train_config: Union[Path, str] = None,
+ lm_file: Union[Path, str] = None,
+ token_type: str = None,
+ bpemodel: str = None,
+ device: str = "cpu",
+ maxlenratio: float = 0.0,
+ minlenratio: float = 0.0,
+ batch_size: int = 1,
+ dtype: str = "float32",
+ beam_size: int = 20,
+ ctc_weight: float = 0.5,
+ lm_weight: float = 1.0,
+ ngram_weight: float = 0.9,
+ penalty: float = 0.0,
+ nbest: int = 1,
+ streaming: bool = False,
+ frontend_conf: dict = None,
+ **kwargs,
):
- assert check_argument_types()
-
+
# 1. Build ASR model
- from funasr.tasks.sa_asr import ASRTask
scorers = {}
- asr_model, asr_train_args = ASRTask.build_model_from_file(
+ asr_model, asr_train_args = build_model_from_file(
asr_train_config, asr_model_file, cmvn_file, device
)
frontend = None
if asr_train_args.frontend is not None and asr_train_args.frontend_conf is not None:
- if asr_train_args.frontend == 'wav_frontend':
- frontend = WavFrontend(cmvn_file=cmvn_file, **asr_train_args.frontend_conf)
+ from funasr.tasks.sa_asr import frontend_choices
+ if asr_train_args.frontend == 'wav_frontend' or asr_train_args.frontend == "multichannelfrontend":
+ frontend_class = frontend_choices.get_class(asr_train_args.frontend)
+ frontend = frontend_class(cmvn_file=cmvn_file, **asr_train_args.frontend_conf).eval()
else:
frontend_class = frontend_choices.get_class(asr_train_args.frontend)
frontend = frontend_class(**asr_train_args.frontend_conf).eval()
-
+
logging.info("asr_model: {}".format(asr_model))
logging.info("asr_train_args: {}".format(asr_train_args))
asr_model.to(dtype=getattr(torch, dtype)).eval()
-
+
decoder = asr_model.decoder
-
+
ctc = CTCPrefixScorer(ctc=asr_model.ctc, eos=asr_model.eos)
token_list = asr_model.token_list
scorers.update(
@@ -1652,24 +1699,24 @@
ctc=ctc,
length_bonus=LengthBonus(len(token_list)),
)
-
+
# 2. Build Language model
if lm_train_config is not None:
- lm, lm_train_args = LMTask.build_model_from_file(
- lm_train_config, lm_file, None, device
+ lm, lm_train_args = build_model_from_file(
+ lm_train_config, lm_file, None, device, task_name="lm"
)
scorers["lm"] = lm.lm
-
+
# 3. Build ngram model
# ngram is not supported now
ngram = None
scorers["ngram"] = ngram
-
+
# 4. Build BeamSearch object
# transducer is not supported now
beam_search_transducer = None
from funasr.modules.beam_search.beam_search_sa_asr import BeamSearch
-
+
weights = dict(
decoder=1.0 - ctc_weight,
ctc=ctc_weight,
@@ -1687,13 +1734,13 @@
token_list=token_list,
pre_beam_score_key=None if ctc_weight == 1.0 else "full",
)
-
+
# 5. [Optional] Build Text converter: e.g. bpe-sym -> Text
if token_type is None:
token_type = asr_train_args.token_type
if bpemodel is None:
bpemodel = asr_train_args.bpemodel
-
+
if token_type is None:
tokenizer = None
elif token_type == "bpe":
@@ -1705,7 +1752,7 @@
tokenizer = build_tokenizer(token_type=token_type)
converter = TokenIDConverter(token_list=token_list)
logging.info(f"Text tokenizer: {tokenizer}")
-
+
self.asr_model = asr_model
self.asr_train_args = asr_train_args
self.converter = converter
@@ -1718,11 +1765,11 @@
self.dtype = dtype
self.nbest = nbest
self.frontend = frontend
-
+
@torch.no_grad()
def __call__(
- self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray],
- profile: Union[torch.Tensor, np.ndarray], profile_lengths: Union[torch.Tensor, np.ndarray]
+ self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray],
+ profile: Union[torch.Tensor, np.ndarray], profile_lengths: Union[torch.Tensor, np.ndarray]
) -> List[
Tuple[
Optional[str],
@@ -1740,15 +1787,14 @@
text, text_id, token, token_int, hyp
"""
- assert check_argument_types()
-
+
# Input as audio signal
if isinstance(speech, np.ndarray):
speech = torch.tensor(speech)
-
+
if isinstance(profile, np.ndarray):
profile = torch.tensor(profile)
-
+
if self.frontend is not None:
feats, feats_len = self.frontend.forward(speech, speech_lengths)
feats = to_device(feats, device=self.device)
@@ -1759,10 +1805,10 @@
feats_len = speech_lengths
lfr_factor = max(1, (feats.size()[-1] // 80) - 1)
batch = {"speech": feats, "speech_lengths": feats_len}
-
+
# a. To device
batch = to_device(batch, device=self.device)
-
+
# b. Forward Encoder
asr_enc, _, spk_enc = self.asr_model.encode(**batch)
if isinstance(asr_enc, tuple):
@@ -1771,30 +1817,30 @@
spk_enc = spk_enc[0]
assert len(asr_enc) == 1, len(asr_enc)
assert len(spk_enc) == 1, len(spk_enc)
-
+
# c. Passed the encoder result and the beam search
nbest_hyps = self.beam_search(
asr_enc[0], spk_enc[0], profile[0], maxlenratio=self.maxlenratio, minlenratio=self.minlenratio
)
-
+
nbest_hyps = nbest_hyps[: self.nbest]
-
+
results = []
for hyp in nbest_hyps:
assert isinstance(hyp, (HypothesisSAASR)), type(hyp)
-
+
# remove sos/eos and get results
last_pos = -1
if isinstance(hyp.yseq, list):
token_int = hyp.yseq[1: last_pos]
else:
token_int = hyp.yseq[1: last_pos].tolist()
-
+
spk_weigths = torch.stack(hyp.spk_weigths, dim=0)
-
+
token_ori = self.converter.ids2tokens(token_int)
text_ori = self.tokenizer.tokens2text(token_ori)
-
+
text_ori_spklist = text_ori.split('$')
cur_index = 0
spk_choose = []
@@ -1806,32 +1852,31 @@
spk_weights_local = spk_weights_local.mean(dim=0)
spk_choose_local = spk_weights_local.argmax(-1)
spk_choose.append(spk_choose_local.item() + 1)
-
+
# remove blank symbol id, which is assumed to be 0
token_int = list(filter(lambda x: x != 0, token_int))
-
+
# Change integer-ids to tokens
token = self.converter.ids2tokens(token_int)
-
+
if self.tokenizer is not None:
text = self.tokenizer.tokens2text(token)
else:
text = None
-
+
text_spklist = text.split('$')
assert len(spk_choose) == len(text_spklist)
-
+
spk_list = []
for i in range(len(text_spklist)):
text_split = text_spklist[i]
n = len(text_split)
spk_list.append(str(spk_choose[i]) * n)
-
+
text_id = '$'.join(spk_list)
-
+
assert len(text) == len(text_id)
-
+
results.append((text, text_id, token, token_int, hyp))
-
- assert check_return_type(results)
+
return results
--
Gitblit v1.9.1