From 6427c834dfd97b1f05c6659cdc7ccf010bf82fe1 Mon Sep 17 00:00:00 2001
From: 嘉渊 <wangjiaming.wjm@alibaba-inc.com>
Date: 星期一, 24 四月 2023 19:50:07 +0800
Subject: [PATCH] update
---
funasr/bin/tp_inference.py | 155 +++++++++++++++++++++------------------------------
1 files changed, 65 insertions(+), 90 deletions(-)
diff --git a/funasr/bin/tp_inference.py b/funasr/bin/tp_inference.py
index b3f15d4..6360b17 100644
--- a/funasr/bin/tp_inference.py
+++ b/funasr/bin/tp_inference.py
@@ -1,5 +1,6 @@
import argparse
import logging
+from optparse import Option
import sys
import json
from pathlib import Path
@@ -11,15 +12,12 @@
from typing import Union
from typing import Dict
-import math
import numpy as np
import torch
from typeguard import check_argument_types
-from typeguard import check_return_type
from funasr.fileio.datadir_writer import DatadirWriter
-from funasr.modules.scorers.scorer_interface import BatchScorerInterface
-from funasr.modules.subsampling import TooShortUttError
+from funasr.datasets.preprocessor import LMPreprocessor
from funasr.tasks.asr import ASRTaskAligner as ASRTask
from funasr.torch_utils.device_funcs import to_device
from funasr.torch_utils.set_all_random_seed import set_all_random_seed
@@ -28,9 +26,10 @@
from funasr.utils.types import str2bool
from funasr.utils.types import str2triple_str
from funasr.utils.types import str_or_none
-from funasr.utils import asr_utils, wav_utils, postprocess_utils
from funasr.models.frontend.wav_frontend import WavFrontend
from funasr.text.token_id_converter import TokenIDConverter
+from funasr.utils.timestamp_tools import ts_prediction_lfr6_standard
+
header_colors = '\033[95m'
end_colors = '\033[0m'
@@ -41,68 +40,13 @@
'model_fs': 16000
}
-def time_stamp_lfr6_advance(us_alphas, us_cif_peak, char_list):
- START_END_THRESHOLD = 5
- MAX_TOKEN_DURATION = 12
- TIME_RATE = 10.0 * 6 / 1000 / 3 # 3 times upsampled
- if len(us_cif_peak.shape) == 2:
- alphas, cif_peak = us_alphas[0], us_cif_peak[0] # support inference batch_size=1 only
- else:
- alphas, cif_peak = us_alphas, us_cif_peak
- num_frames = cif_peak.shape[0]
- if char_list[-1] == '</s>':
- char_list = char_list[:-1]
- # char_list = [i for i in text]
- timestamp_list = []
- new_char_list = []
- # for bicif model trained with large data, cif2 actually fires when a character starts
- # so treat the frames between two peaks as the duration of the former token
- fire_place = torch.where(cif_peak>1.0-1e-4)[0].cpu().numpy() - 3.2 # total offset
- num_peak = len(fire_place)
- assert num_peak == len(char_list) + 1 # number of peaks is supposed to be number of tokens + 1
- # begin silence
- if fire_place[0] > START_END_THRESHOLD:
- # char_list.insert(0, '<sil>')
- timestamp_list.append([0.0, fire_place[0]*TIME_RATE])
- new_char_list.append('<sil>')
- # tokens timestamp
- for i in range(len(fire_place)-1):
- new_char_list.append(char_list[i])
- if MAX_TOKEN_DURATION < 0 or fire_place[i+1] - fire_place[i] < MAX_TOKEN_DURATION:
- timestamp_list.append([fire_place[i]*TIME_RATE, fire_place[i+1]*TIME_RATE])
- else:
- # cut the duration to token and sil of the 0-weight frames last long
- _split = fire_place[i] + MAX_TOKEN_DURATION
- timestamp_list.append([fire_place[i]*TIME_RATE, _split*TIME_RATE])
- timestamp_list.append([_split*TIME_RATE, fire_place[i+1]*TIME_RATE])
- new_char_list.append('<sil>')
- # tail token and end silence
- # new_char_list.append(char_list[-1])
- if num_frames - fire_place[-1] > START_END_THRESHOLD:
- _end = (num_frames + fire_place[-1]) * 0.5
- # _end = fire_place[-1]
- timestamp_list[-1][1] = _end*TIME_RATE
- timestamp_list.append([_end*TIME_RATE, num_frames*TIME_RATE])
- new_char_list.append("<sil>")
- else:
- timestamp_list[-1][1] = num_frames*TIME_RATE
- assert len(new_char_list) == len(timestamp_list)
- res_str = ""
- for char, timestamp in zip(new_char_list, timestamp_list):
- res_str += "{} {} {};".format(char, str(timestamp[0]+0.0005)[:5], str(timestamp[1]+0.0005)[:5])
- res = []
- for char, timestamp in zip(char_list, timestamp_list):
- if char != '<sil>':
- res.append([int(timestamp[0] * 1000), int(timestamp[1] * 1000)])
- return res_str, res
-
class SpeechText2Timestamp:
def __init__(
self,
- tp_train_config: Union[Path, str] = None,
- tp_model_file: Union[Path, str] = None,
- tp_cmvn_file: Union[Path, str] = None,
+ timestamp_infer_config: Union[Path, str] = None,
+ timestamp_model_file: Union[Path, str] = None,
+ timestamp_cmvn_file: Union[Path, str] = None,
device: str = "cpu",
dtype: str = "float32",
**kwargs,
@@ -110,11 +54,14 @@
assert check_argument_types()
# 1. Build ASR model
tp_model, tp_train_args = ASRTask.build_model_from_file(
- tp_train_config, tp_model_file, device
+ timestamp_infer_config, timestamp_model_file, device
)
+ if 'cuda' in device:
+ tp_model = tp_model.cuda() # force model to cuda
+
frontend = None
if tp_train_args.frontend is not None:
- frontend = WavFrontend(cmvn_file=tp_cmvn_file, **tp_train_args.frontend_conf)
+ frontend = WavFrontend(cmvn_file=timestamp_cmvn_file, **tp_train_args.frontend_conf)
logging.info("tp_model: {}".format(tp_model))
logging.info("tp_train_args: {}".format(tp_train_args))
@@ -148,11 +95,11 @@
# Input as audio signal
if isinstance(speech, np.ndarray):
speech = torch.tensor(speech)
-
if self.frontend is not None:
feats, feats_len = self.frontend.forward(speech, speech_lengths)
feats = to_device(feats, device=self.device)
feats_len = feats_len.int()
+ self.tp_model.frontend = None
else:
feats = speech
feats_len = speech_lengths
@@ -169,8 +116,8 @@
enc = enc[0]
# c. Forward Predictor
- _, _, us_alphas, us_cif_peak = self.tp_model.calc_predictor_timestamp(enc, enc_len, text_lengths.to(self.device)+1)
- return us_alphas, us_cif_peak
+ _, _, us_alphas, us_peaks = self.tp_model.calc_predictor_timestamp(enc, enc_len, text_lengths.to(self.device)+1)
+ return us_alphas, us_peaks
def inference(
@@ -178,9 +125,9 @@
ngpu: int,
log_level: Union[int, str],
data_path_and_name_and_type,
- tp_train_config: Optional[str],
- tp_model_file: Optional[str],
- tp_cmvn_file: Optional[str] = None,
+ timestamp_infer_config: Optional[str],
+ timestamp_model_file: Optional[str],
+ timestamp_cmvn_file: Optional[str] = None,
raw_inputs: Union[np.ndarray, torch.Tensor] = None,
key_file: Optional[str] = None,
allow_variable_data_keys: bool = False,
@@ -188,21 +135,25 @@
dtype: str = "float32",
seed: int = 0,
num_workers: int = 1,
+ split_with_space: bool = True,
+ seg_dict_file: Optional[str] = None,
**kwargs,
):
inference_pipeline = inference_modelscope(
batch_size=batch_size,
ngpu=ngpu,
log_level=log_level,
- tp_train_config=tp_train_config,
- tp_model_file=tp_model_file,
- tp_cmvn_file=tp_cmvn_file,
+ timestamp_infer_config=timestamp_infer_config,
+ timestamp_model_file=timestamp_model_file,
+ timestamp_cmvn_file=timestamp_cmvn_file,
key_file=key_file,
allow_variable_data_keys=allow_variable_data_keys,
output_dir=output_dir,
dtype=dtype,
seed=seed,
num_workers=num_workers,
+ split_with_space=split_with_space,
+ seg_dict_file=seg_dict_file,
**kwargs,
)
return inference_pipeline(data_path_and_name_and_type, raw_inputs)
@@ -213,9 +164,9 @@
ngpu: int,
log_level: Union[int, str],
# data_path_and_name_and_type,
- tp_train_config: Optional[str],
- tp_model_file: Optional[str],
- tp_cmvn_file: Optional[str] = None,
+ timestamp_infer_config: Optional[str],
+ timestamp_model_file: Optional[str],
+ timestamp_cmvn_file: Optional[str] = None,
# raw_inputs: Union[np.ndarray, torch.Tensor] = None,
key_file: Optional[str] = None,
allow_variable_data_keys: bool = False,
@@ -223,6 +174,8 @@
dtype: str = "float32",
seed: int = 0,
num_workers: int = 1,
+ split_with_space: bool = True,
+ seg_dict_file: Optional[str] = None,
**kwargs,
):
assert check_argument_types()
@@ -240,20 +193,32 @@
device = "cuda"
else:
device = "cpu"
-
# 1. Set random-seed
set_all_random_seed(seed)
# 2. Build speech2vadsegment
speechtext2timestamp_kwargs = dict(
- tp_train_config=tp_train_config,
- tp_model_file=tp_model_file,
- tp_cmvn_file=tp_cmvn_file,
+ timestamp_infer_config=timestamp_infer_config,
+ timestamp_model_file=timestamp_model_file,
+ timestamp_cmvn_file=timestamp_cmvn_file,
device=device,
dtype=dtype,
)
logging.info("speechtext2timestamp_kwargs: {}".format(speechtext2timestamp_kwargs))
speechtext2timestamp = SpeechText2Timestamp(**speechtext2timestamp_kwargs)
+
+ preprocessor = LMPreprocessor(
+ train=False,
+ token_type=speechtext2timestamp.tp_train_args.token_type,
+ token_list=speechtext2timestamp.tp_train_args.token_list,
+ bpemodel=None,
+ text_cleaner=None,
+ g2p_type=None,
+ text_name="text",
+ non_linguistic_symbols=speechtext2timestamp.tp_train_args.non_linguistic_symbols,
+ split_with_space=split_with_space,
+ seg_dict_file=seg_dict_file,
+ )
def _forward(
data_path_and_name_and_type,
@@ -275,14 +240,11 @@
batch_size=batch_size,
key_file=key_file,
num_workers=num_workers,
- preprocess_fn=ASRTask.build_preprocess_fn(speechtext2timestamp.tp_train_args, False),
+ preprocess_fn=preprocessor,
collate_fn=ASRTask.build_collate_fn(speechtext2timestamp.tp_train_args, False),
allow_variable_data_keys=allow_variable_data_keys,
inference=True,
)
-
- finish_count = 0
- file_count = 1
tp_result_list = []
for keys, batch in loader:
@@ -300,9 +262,10 @@
for batch_id in range(_bs):
key = keys[batch_id]
token = speechtext2timestamp.converter.ids2tokens(batch['text'][batch_id])
- ts_str, ts_list = time_stamp_lfr6_advance(us_alphas[batch_id], us_cif_peak[batch_id], token)
+ ts_str, ts_list = ts_prediction_lfr6_standard(us_alphas[batch_id], us_cif_peak[batch_id], token, force_time_shift=-3.0)
logging.warning(ts_str)
- tp_result_list.append({'text':"".join([i for i in token if i != '<sil>']), 'timestamp': ts_list})
+ item = {'key': key, 'value': ts_str, 'timestamp':ts_list}
+ tp_result_list.append(item)
return tp_result_list
return _forward
@@ -365,17 +328,17 @@
group = parser.add_argument_group("The model configuration related")
group.add_argument(
- "--tp_train_config",
+ "--timestamp_infer_config",
type=str,
help="VAD infer configuration",
)
group.add_argument(
- "--tp_model_file",
+ "--timestamp_model_file",
type=str,
help="VAD model parameter file",
)
group.add_argument(
- "--tp_cmvn_file",
+ "--timestamp_cmvn_file",
type=str,
help="Global cmvn file",
)
@@ -387,6 +350,18 @@
default=1,
help="The batch size for inference",
)
+ group.add_argument(
+ "--seg_dict_file",
+ type=str,
+ default=None,
+ help="The batch size for inference",
+ )
+ group.add_argument(
+ "--split_with_space",
+ type=bool,
+ default=False,
+ help="The batch size for inference",
+ )
return parser
--
Gitblit v1.9.1