From 28ccfbfc51068a663a80764e14074df5edf2b5ba Mon Sep 17 00:00:00 2001
From: kongdeqiang <kongdeqiang960204@163.com>
Date: 星期五, 13 三月 2026 17:41:41 +0800
Subject: [PATCH] 提交
---
runtime/python/onnxruntime/funasr_onnx/utils/frontend.py | 257 +++++++++++++++++++++++++++++++--------------------
1 files changed, 156 insertions(+), 101 deletions(-)
diff --git a/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py b/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py
index 295e7b5..54f9deb 100644
--- a/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py
+++ b/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py
@@ -2,6 +2,7 @@
from pathlib import Path
from typing import Any, Dict, Iterable, List, NamedTuple, Set, Tuple, Union
import copy
+from functools import lru_cache
import numpy as np
import kaldi_native_fbank as knf
@@ -11,22 +12,21 @@
logger_initialized = {}
-class WavFrontend():
- """Conventional frontend structure for ASR.
- """
+class WavFrontend:
+ """Conventional frontend structure for ASR."""
def __init__(
- self,
- cmvn_file: str = None,
- fs: int = 16000,
- window: str = 'hamming',
- n_mels: int = 80,
- frame_length: int = 25,
- frame_shift: int = 10,
- lfr_m: int = 1,
- lfr_n: int = 1,
- dither: float = 1.0,
- **kwargs,
+ self,
+ cmvn_file: str = None,
+ fs: int = 16000,
+ window: str = "hamming",
+ n_mels: int = 80,
+ frame_length: int = 25,
+ frame_shift: int = 10,
+ lfr_m: int = 1,
+ lfr_n: int = 1,
+ dither: float = 1.0,
+ **kwargs,
) -> None:
opts = knf.FbankOptions()
@@ -46,26 +46,24 @@
self.cmvn_file = cmvn_file
if self.cmvn_file:
- self.cmvn = self.load_cmvn()
+ self.cmvn = load_cmvn(self.cmvn_file)
self.fbank_fn = None
self.fbank_beg_idx = 0
self.reset_status()
- def fbank(self,
- waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+ def fbank(self, waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
waveform = waveform * (1 << 15)
- self.fbank_fn = knf.OnlineFbank(self.opts)
- self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist())
- frames = self.fbank_fn.num_frames_ready
+ fbank_fn = knf.OnlineFbank(self.opts)
+ fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist())
+ frames = fbank_fn.num_frames_ready
mat = np.empty([frames, self.opts.mel_opts.num_bins])
for i in range(frames):
- mat[i, :] = self.fbank_fn.get_frame(i)
+ mat[i, :] = fbank_fn.get_frame(i)
feat = mat.astype(np.float32)
feat_len = np.array(mat.shape[0]).astype(np.int32)
return feat, feat_len
- def fbank_online(self,
- waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
+ def fbank_online(self, waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
waveform = waveform * (1 << 15)
# self.fbank_fn = knf.OnlineFbank(self.opts)
self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist())
@@ -103,12 +101,11 @@
T = T + (lfr_m - 1) // 2
for i in range(T_lfr):
if lfr_m <= T - i * lfr_n:
- LFR_inputs.append(
- (inputs[i * lfr_n:i * lfr_n + lfr_m]).reshape(1, -1))
+ LFR_inputs.append((inputs[i * lfr_n : i * lfr_n + lfr_m]).reshape(1, -1))
else:
# process last LFR frame
num_padding = lfr_m - (T - i * lfr_n)
- frame = inputs[i * lfr_n:].reshape(-1)
+ frame = inputs[i * lfr_n :].reshape(-1)
for _ in range(num_padding):
frame = np.hstack((frame, inputs[-1]))
@@ -126,31 +123,47 @@
inputs = (inputs + means) * vars
return inputs
- def load_cmvn(self,) -> np.ndarray:
- with open(self.cmvn_file, 'r', encoding='utf-8') as f:
- lines = f.readlines()
+@lru_cache()
+def load_cmvn(cmvn_file: Union[str, Path]) -> np.ndarray:
+ """load cmvn file to numpy array.
- means_list = []
- vars_list = []
- for i in range(len(lines)):
- line_item = lines[i].split()
- if line_item[0] == '<AddShift>':
- line_item = lines[i + 1].split()
- if line_item[0] == '<LearnRateCoef>':
- add_shift_line = line_item[3:(len(line_item) - 1)]
- means_list = list(add_shift_line)
- continue
- elif line_item[0] == '<Rescale>':
- line_item = lines[i + 1].split()
- if line_item[0] == '<LearnRateCoef>':
- rescale_line = line_item[3:(len(line_item) - 1)]
- vars_list = list(rescale_line)
- continue
+ Args:
+ cmvn_file (Union[str, Path]): cmvn file path.
- means = np.array(means_list).astype(np.float64)
- vars = np.array(vars_list).astype(np.float64)
- cmvn = np.array([means, vars])
- return cmvn
+ Raises:
+ FileNotFoundError: cmvn file not exits.
+
+ Returns:
+ np.ndarray: cmvn array. shape is (2, dim).The first row is means, the second row is vars.
+ """
+
+ cmvn_file = Path(cmvn_file)
+ if not cmvn_file.exists():
+ raise FileNotFoundError("cmvn file not exits")
+
+ with open(cmvn_file, "r", encoding="utf-8") as f:
+ lines = f.readlines()
+ means_list = []
+ vars_list = []
+ for i in range(len(lines)):
+ line_item = lines[i].split()
+ if line_item[0] == "<AddShift>":
+ line_item = lines[i + 1].split()
+ if line_item[0] == "<LearnRateCoef>":
+ add_shift_line = line_item[3 : (len(line_item) - 1)]
+ means_list = list(add_shift_line)
+ continue
+ elif line_item[0] == "<Rescale>":
+ line_item = lines[i + 1].split()
+ if line_item[0] == "<LearnRateCoef>":
+ rescale_line = line_item[3 : (len(line_item) - 1)]
+ vars_list = list(rescale_line)
+ continue
+
+ means = np.array(means_list).astype(np.float64)
+ vars = np.array(vars_list).astype(np.float64)
+ cmvn = np.array([means, vars])
+ return cmvn
class WavFrontendOnline(WavFrontend):
@@ -158,8 +171,12 @@
super().__init__(**kwargs)
# self.fbank_fn = knf.OnlineFbank(self.opts)
# add variables
- self.frame_sample_length = int(self.opts.frame_opts.frame_length_ms * self.opts.frame_opts.samp_freq / 1000)
- self.frame_shift_sample_length = int(self.opts.frame_opts.frame_shift_ms * self.opts.frame_opts.samp_freq / 1000)
+ self.frame_sample_length = int(
+ self.opts.frame_opts.frame_length_ms * self.opts.frame_opts.samp_freq / 1000
+ )
+ self.frame_shift_sample_length = int(
+ self.opts.frame_opts.frame_shift_ms * self.opts.frame_opts.samp_freq / 1000
+ )
self.waveform = None
self.reserve_waveforms = None
self.input_cache = None
@@ -167,23 +184,26 @@
@staticmethod
# inputs has catted the cache
- def apply_lfr(inputs: np.ndarray, lfr_m: int, lfr_n: int, is_final: bool = False) -> Tuple[
- np.ndarray, np.ndarray, int]:
+ def apply_lfr(
+ inputs: np.ndarray, lfr_m: int, lfr_n: int, is_final: bool = False
+ ) -> Tuple[np.ndarray, np.ndarray, int]:
"""
Apply lfr with data
"""
LFR_inputs = []
T = inputs.shape[0] # include the right context
- T_lfr = int(np.ceil((T - (lfr_m - 1) // 2) / lfr_n)) # minus the right context: (lfr_m - 1) // 2
+ T_lfr = int(
+ np.ceil((T - (lfr_m - 1) // 2) / lfr_n)
+ ) # minus the right context: (lfr_m - 1) // 2
splice_idx = T_lfr
for i in range(T_lfr):
if lfr_m <= T - i * lfr_n:
- LFR_inputs.append((inputs[i * lfr_n:i * lfr_n + lfr_m]).reshape(1, -1))
+ LFR_inputs.append((inputs[i * lfr_n : i * lfr_n + lfr_m]).reshape(1, -1))
else: # process last LFR frame
if is_final:
num_padding = lfr_m - (T - i * lfr_n)
- frame = (inputs[i * lfr_n:]).reshape(-1)
+ frame = (inputs[i * lfr_n :]).reshape(-1)
for _ in range(num_padding):
frame = np.hstack((frame, inputs[-1]))
LFR_inputs.append(frame)
@@ -197,24 +217,27 @@
return LFR_outputs.astype(np.float32), lfr_splice_cache, splice_idx
@staticmethod
- def compute_frame_num(sample_length: int, frame_sample_length: int, frame_shift_sample_length: int) -> int:
+ def compute_frame_num(
+ sample_length: int, frame_sample_length: int, frame_shift_sample_length: int
+ ) -> int:
frame_num = int((sample_length - frame_sample_length) / frame_shift_sample_length + 1)
return frame_num if frame_num >= 1 and sample_length >= frame_sample_length else 0
-
def fbank(
- self,
- input: np.ndarray,
- input_lengths: np.ndarray
+ self, input: np.ndarray, input_lengths: np.ndarray
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
self.fbank_fn = knf.OnlineFbank(self.opts)
batch_size = input.shape[0]
if self.input_cache is None:
self.input_cache = np.empty((batch_size, 0), dtype=np.float32)
input = np.concatenate((self.input_cache, input), axis=1)
- frame_num = self.compute_frame_num(input.shape[-1], self.frame_sample_length, self.frame_shift_sample_length)
+ frame_num = self.compute_frame_num(
+ input.shape[-1], self.frame_sample_length, self.frame_shift_sample_length
+ )
# update self.in_cache
- self.input_cache = input[:, -(input.shape[-1] - frame_num * self.frame_shift_sample_length):]
+ self.input_cache = input[
+ :, -(input.shape[-1] - frame_num * self.frame_shift_sample_length) :
+ ]
waveforms = np.empty(0, dtype=np.float32)
feats_pad = np.empty(0, dtype=np.float32)
feats_lens = np.empty(0, dtype=np.int32)
@@ -225,9 +248,15 @@
for i in range(batch_size):
waveform = input[i]
waveforms.append(
- waveform[:((frame_num - 1) * self.frame_shift_sample_length + self.frame_sample_length)])
+ waveform[
+ : (
+ (frame_num - 1) * self.frame_shift_sample_length
+ + self.frame_sample_length
+ )
+ ]
+ )
waveform = waveform * (1 << 15)
-
+
self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist())
frames = self.fbank_fn.num_frames_ready
mat = np.empty([frames, self.opts.mel_opts.num_bins])
@@ -249,22 +278,20 @@
return self.fbanks, self.fbanks_lens
def lfr_cmvn(
- self,
- input: np.ndarray,
- input_lengths: np.ndarray,
- is_final: bool = False
+ self, input: np.ndarray, input_lengths: np.ndarray, is_final: bool = False
) -> Tuple[np.ndarray, np.ndarray, List[int]]:
batch_size = input.shape[0]
feats = []
feats_lens = []
lfr_splice_frame_idxs = []
for i in range(batch_size):
- mat = input[i, :input_lengths[i], :]
+ mat = input[i, : input_lengths[i], :]
lfr_splice_frame_idx = -1
if self.lfr_m != 1 or self.lfr_n != 1:
# update self.lfr_splice_cache in self.apply_lfr
- mat, self.lfr_splice_cache[i], lfr_splice_frame_idx = self.apply_lfr(mat, self.lfr_m, self.lfr_n,
- is_final)
+ mat, self.lfr_splice_cache[i], lfr_splice_frame_idx = self.apply_lfr(
+ mat, self.lfr_m, self.lfr_n, is_final
+ )
if self.cmvn_file is not None:
mat = self.apply_cmvn(mat)
feat_length = mat.shape[0]
@@ -276,47 +303,70 @@
feats_pad = np.array(feats)
return feats_pad, feats_lens, lfr_splice_frame_idxs
-
def extract_fbank(
- self, input: np.ndarray, input_lengths: np.ndarray, is_final: bool = False
+ self, input: np.ndarray, input_lengths: np.ndarray, is_final: bool = False
) -> Tuple[np.ndarray, np.ndarray]:
batch_size = input.shape[0]
- assert batch_size == 1, 'we support to extract feature online only when the batch size is equal to 1 now'
+ assert (
+ batch_size == 1
+ ), "we support to extract feature online only when the batch size is equal to 1 now"
waveforms, feats, feats_lengths = self.fbank(input, input_lengths) # input shape: B T D
if feats.shape[0]:
- self.waveforms = waveforms if self.reserve_waveforms is None else np.concatenate(
- (self.reserve_waveforms, waveforms), axis=1)
+ self.waveforms = (
+ waveforms
+ if self.reserve_waveforms is None
+ else np.concatenate((self.reserve_waveforms, waveforms), axis=1)
+ )
if not self.lfr_splice_cache:
for i in range(batch_size):
- self.lfr_splice_cache.append(np.expand_dims(feats[i][0, :], axis=0).repeat((self.lfr_m - 1) // 2, axis=0))
-
+ self.lfr_splice_cache.append(
+ np.expand_dims(feats[i][0, :], axis=0).repeat((self.lfr_m - 1) // 2, axis=0)
+ )
+
if feats_lengths[0] + self.lfr_splice_cache[0].shape[0] >= self.lfr_m:
lfr_splice_cache_np = np.stack(self.lfr_splice_cache) # B T D
feats = np.concatenate((lfr_splice_cache_np, feats), axis=1)
feats_lengths += lfr_splice_cache_np[0].shape[0]
frame_from_waveforms = int(
- (self.waveforms.shape[1] - self.frame_sample_length) / self.frame_shift_sample_length + 1)
+ (self.waveforms.shape[1] - self.frame_sample_length)
+ / self.frame_shift_sample_length
+ + 1
+ )
minus_frame = (self.lfr_m - 1) // 2 if self.reserve_waveforms is None else 0
- feats, feats_lengths, lfr_splice_frame_idxs = self.lfr_cmvn(feats, feats_lengths, is_final)
+ feats, feats_lengths, lfr_splice_frame_idxs = self.lfr_cmvn(
+ feats, feats_lengths, is_final
+ )
if self.lfr_m == 1:
self.reserve_waveforms = None
else:
reserve_frame_idx = lfr_splice_frame_idxs[0] - minus_frame
# print('reserve_frame_idx: ' + str(reserve_frame_idx))
# print('frame_frame: ' + str(frame_from_waveforms))
- self.reserve_waveforms = self.waveforms[:, reserve_frame_idx * self.frame_shift_sample_length:frame_from_waveforms * self.frame_shift_sample_length]
- sample_length = (frame_from_waveforms - 1) * self.frame_shift_sample_length + self.frame_sample_length
+ self.reserve_waveforms = self.waveforms[
+ :,
+ reserve_frame_idx
+ * self.frame_shift_sample_length : frame_from_waveforms
+ * self.frame_shift_sample_length,
+ ]
+ sample_length = (
+ frame_from_waveforms - 1
+ ) * self.frame_shift_sample_length + self.frame_sample_length
self.waveforms = self.waveforms[:, :sample_length]
else:
# update self.reserve_waveforms and self.lfr_splice_cache
- self.reserve_waveforms = self.waveforms[:,
- :-(self.frame_sample_length - self.frame_shift_sample_length)]
+ self.reserve_waveforms = self.waveforms[
+ :, : -(self.frame_sample_length - self.frame_shift_sample_length)
+ ]
for i in range(batch_size):
- self.lfr_splice_cache[i] = np.concatenate((self.lfr_splice_cache[i], feats[i]), axis=0)
+ self.lfr_splice_cache[i] = np.concatenate(
+ (self.lfr_splice_cache[i], feats[i]), axis=0
+ )
return np.empty(0, dtype=np.float32), feats_lengths
else:
if is_final:
- self.waveforms = waveforms if self.reserve_waveforms is None else self.reserve_waveforms
+ self.waveforms = (
+ waveforms if self.reserve_waveforms is None else self.reserve_waveforms
+ )
feats = np.stack(self.lfr_splice_cache)
feats_lengths = np.zeros(batch_size, dtype=np.int32) + feats.shape[1]
feats, feats_lengths, _ = self.lfr_cmvn(feats, feats_lengths, is_final)
@@ -333,13 +383,14 @@
self.input_cache = None
self.lfr_splice_cache = []
+
def load_bytes(input):
middle_data = np.frombuffer(input, dtype=np.int16)
middle_data = np.asarray(middle_data)
- if middle_data.dtype.kind not in 'iu':
+ if middle_data.dtype.kind not in "iu":
raise TypeError("'middle_data' must be an array of integers")
- dtype = np.dtype('float32')
- if dtype.kind != 'f':
+ dtype = np.dtype("float32")
+ if dtype.kind != "f":
raise TypeError("'dtype' must be a floating point type")
i = np.iinfo(middle_data.dtype)
@@ -349,9 +400,8 @@
return array
-class SinusoidalPositionEncoderOnline():
- '''Streaming Positional encoding.
- '''
+class SinusoidalPositionEncoderOnline:
+ """Streaming Positional encoding."""
def encode(self, positions: np.ndarray = None, depth: int = None, dtype: np.dtype = np.float32):
batch_size = positions.shape[0]
@@ -365,29 +415,34 @@
def forward(self, x, start_idx=0):
batch_size, timesteps, input_dim = x.shape
- positions = np.arange(1, timesteps+1+start_idx)[None, :]
+ positions = np.arange(1, timesteps + 1 + start_idx)[None, :]
position_encoding = self.encode(positions, input_dim, x.dtype)
- return x + position_encoding[:, start_idx: start_idx + timesteps]
+ return x + position_encoding[:, start_idx : start_idx + timesteps]
def test():
path = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav"
import librosa
+
cmvn_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/am.mvn"
config_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/config.yaml"
from funasr.runtime.python.onnxruntime.rapid_paraformer.utils.utils import read_yaml
+
config = read_yaml(config_file)
waveform, _ = librosa.load(path, sr=None)
frontend = WavFrontend(
cmvn_file=cmvn_file,
- **config['frontend_conf'],
+ **config["frontend_conf"],
)
- speech, _ = frontend.fbank_online(waveform) #1d, (sample,), numpy
- feat, feat_len = frontend.lfr_cmvn(speech) # 2d, (frame, 450), np.float32 -> torch, torch.from_numpy(), dtype, (1, frame, 450)
-
- frontend.reset_status() # clear cache
+ speech, _ = frontend.fbank_online(waveform) # 1d, (sample,), numpy
+ feat, feat_len = frontend.lfr_cmvn(
+ speech
+ ) # 2d, (frame, 450), np.float32 -> torch, torch.from_numpy(), dtype, (1, frame, 450)
+
+ frontend.reset_status() # clear cache
return feat, feat_len
-if __name__ == '__main__':
+
+if __name__ == "__main__":
test()
--
Gitblit v1.9.1