From 94de39dde2e616a01683c518023d0fab72b4e103 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期一, 19 二月 2024 22:21:50 +0800
Subject: [PATCH] aishell example
---
runtime/onnxruntime/src/paraformer.cpp | 143 +++++++++++++++++++++++++++--------------------
1 files changed, 81 insertions(+), 62 deletions(-)
diff --git a/runtime/onnxruntime/src/paraformer.cpp b/runtime/onnxruntime/src/paraformer.cpp
index 11ed0ff..c56421c 100644
--- a/runtime/onnxruntime/src/paraformer.cpp
+++ b/runtime/onnxruntime/src/paraformer.cpp
@@ -19,10 +19,11 @@
// offline
void Paraformer::InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){
+ LoadConfigFromYaml(am_config.c_str());
// knf options
fbank_opts_.frame_opts.dither = 0;
fbank_opts_.mel_opts.num_bins = n_mels;
- fbank_opts_.frame_opts.samp_freq = MODEL_SAMPLE_RATE;
+ fbank_opts_.frame_opts.samp_freq = asr_sample_rate;
fbank_opts_.frame_opts.window_type = window_type;
fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
fbank_opts_.frame_opts.frame_length_ms = frame_length;
@@ -65,7 +66,6 @@
for (auto& item : m_strOutputNames)
m_szOutputNames.push_back(item.c_str());
vocab = new Vocab(am_config.c_str());
- LoadConfigFromYaml(am_config.c_str());
phone_set_ = new PhoneSet(am_config.c_str());
LoadCmvn(am_cmvn.c_str());
}
@@ -77,7 +77,7 @@
// knf options
fbank_opts_.frame_opts.dither = 0;
fbank_opts_.mel_opts.num_bins = n_mels;
- fbank_opts_.frame_opts.samp_freq = MODEL_SAMPLE_RATE;
+ fbank_opts_.frame_opts.samp_freq = asr_sample_rate;
fbank_opts_.frame_opts.window_type = window_type;
fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
fbank_opts_.frame_opts.frame_length_ms = frame_length;
@@ -187,13 +187,13 @@
}
void Paraformer::InitLm(const std::string &lm_file,
- const std::string &lm_cfg_file) {
+ const std::string &lm_cfg_file,
+ const std::string &lex_file) {
try {
lm_ = std::shared_ptr<fst::Fst<fst::StdArc>>(
fst::Fst<fst::StdArc>::Read(lm_file));
if (lm_){
- if (vocab) { delete vocab; }
- vocab = new Vocab(lm_cfg_file.c_str());
+ lm_vocab = new Vocab(lm_cfg_file.c_str(), lex_file.c_str());
LOG(INFO) << "Successfully load lm file " << lm_file;
}else{
LOG(ERROR) << "Failed to load lm file " << lm_file;
@@ -215,6 +215,9 @@
}
try{
+ YAML::Node frontend_conf = config["frontend_conf"];
+ this->asr_sample_rate = frontend_conf["fs"].as<int>();
+
YAML::Node lang_conf = config["lang"];
if (lang_conf.IsDefined()){
language = lang_conf.as<string>();
@@ -256,6 +259,9 @@
this->cif_threshold = predictor_conf["threshold"].as<double>();
this->tail_alphas = predictor_conf["tail_threshold"].as<double>();
+
+ this->asr_sample_rate = frontend_conf["fs"].as<int>();
+
}catch(exception const &e){
LOG(ERROR) << "Error when load argument from vad config YAML.";
@@ -300,10 +306,18 @@
Paraformer::~Paraformer()
{
- if(vocab)
+ if(vocab){
delete vocab;
- if(seg_dict)
+ }
+ if(lm_vocab){
+ delete lm_vocab;
+ }
+ if(seg_dict){
delete seg_dict;
+ }
+ if(phone_set_){
+ delete phone_set_;
+ }
}
void Paraformer::StartUtterance()
@@ -318,30 +332,20 @@
{
}
-vector<float> Paraformer::FbankKaldi(float sample_rate, const float* waves, int len) {
+void Paraformer::FbankKaldi(float sample_rate, const float* waves, int len, std::vector<std::vector<float>> &asr_feats) {
knf::OnlineFbank fbank_(fbank_opts_);
std::vector<float> buf(len);
for (int32_t i = 0; i != len; ++i) {
buf[i] = waves[i] * 32768;
}
fbank_.AcceptWaveform(sample_rate, buf.data(), buf.size());
- //fbank_->InputFinished();
+
int32_t frames = fbank_.NumFramesReady();
- int32_t feature_dim = fbank_opts_.mel_opts.num_bins;
- vector<float> features(frames * feature_dim);
- float *p = features.data();
- //std::cout << "samples " << len << std::endl;
- //std::cout << "fbank frames " << frames << std::endl;
- //std::cout << "fbank dim " << feature_dim << std::endl;
- //std::cout << "feature size " << features.size() << std::endl;
-
for (int32_t i = 0; i != frames; ++i) {
- const float *f = fbank_.GetFrame(i);
- std::copy(f, f + feature_dim, p);
- p += feature_dim;
+ const float *frame = fbank_.GetFrame(i);
+ std::vector<float> frame_vector(frame, frame + fbank_opts_.mel_opts.num_bins);
+ asr_feats.emplace_back(frame_vector);
}
-
- return features;
}
void Paraformer::LoadCmvn(const char *filename)
@@ -416,56 +420,66 @@
return wfst_decoder->FinalizeDecode(is_stamp, us_alphas, us_cif_peak);
}
-vector<float> Paraformer::ApplyLfr(const std::vector<float> &in)
-{
- int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;
- int32_t in_num_frames = in.size() / in_feat_dim;
- int32_t out_num_frames =
- (in_num_frames - lfr_m) / lfr_n + 1;
- int32_t out_feat_dim = in_feat_dim * lfr_m;
+void Paraformer::LfrCmvn(std::vector<std::vector<float>> &asr_feats) {
- std::vector<float> out(out_num_frames * out_feat_dim);
+ std::vector<std::vector<float>> out_feats;
+ int T = asr_feats.size();
+ int T_lrf = ceil(1.0 * T / lfr_n);
- const float *p_in = in.data();
- float *p_out = out.data();
-
- for (int32_t i = 0; i != out_num_frames; ++i) {
- std::copy(p_in, p_in + out_feat_dim, p_out);
-
- p_out += out_feat_dim;
- p_in += lfr_n * in_feat_dim;
+ // Pad frames at start(copy first frame)
+ for (int i = 0; i < (lfr_m - 1) / 2; i++) {
+ asr_feats.insert(asr_feats.begin(), asr_feats[0]);
}
-
- return out;
- }
-
- void Paraformer::ApplyCmvn(std::vector<float> *v)
- {
- int32_t dim = means_list_.size();
- int32_t num_frames = v->size() / dim;
-
- float *p = v->data();
-
- for (int32_t i = 0; i != num_frames; ++i) {
- for (int32_t k = 0; k != dim; ++k) {
- p[k] = (p[k] + means_list_[k]) * vars_list_[k];
- }
-
- p += dim;
+ // Merge lfr_m frames as one,lfr_n frames per window
+ T = T + (lfr_m - 1) / 2;
+ std::vector<float> p;
+ for (int i = 0; i < T_lrf; i++) {
+ if (lfr_m <= T - i * lfr_n) {
+ for (int j = 0; j < lfr_m; j++) {
+ p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end());
+ }
+ out_feats.emplace_back(p);
+ p.clear();
+ } else {
+ // Fill to lfr_m frames at last window if less than lfr_m frames (copy last frame)
+ int num_padding = lfr_m - (T - i * lfr_n);
+ for (int j = 0; j < (asr_feats.size() - i * lfr_n); j++) {
+ p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end());
+ }
+ for (int j = 0; j < num_padding; j++) {
+ p.insert(p.end(), asr_feats[asr_feats.size() - 1].begin(), asr_feats[asr_feats.size() - 1].end());
+ }
+ out_feats.emplace_back(p);
+ p.clear();
+ }
}
- }
+ // Apply cmvn
+ for (auto &out_feat: out_feats) {
+ for (int j = 0; j < means_list_.size(); j++) {
+ out_feat[j] = (out_feat[j] + means_list_[j]) * vars_list_[j];
+ }
+ }
+ asr_feats = out_feats;
+}
string Paraformer::Forward(float* din, int len, bool input_finished, const std::vector<std::vector<float>> &hw_emb, void* decoder_handle)
{
WfstDecoder* wfst_decoder = (WfstDecoder*)decoder_handle;
int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;
- std::vector<float> wav_feats = FbankKaldi(MODEL_SAMPLE_RATE, din, len);
- wav_feats = ApplyLfr(wav_feats);
- ApplyCmvn(&wav_feats);
+ std::vector<std::vector<float>> asr_feats;
+ FbankKaldi(asr_sample_rate, din, len, asr_feats);
+ if(asr_feats.size() == 0){
+ return "";
+ }
+ LfrCmvn(asr_feats);
int32_t feat_dim = lfr_m*in_feat_dim;
- int32_t num_frames = wav_feats.size() / feat_dim;
- //std::cout << "feat in: " << num_frames << " " << feat_dim << std::endl;
+ int32_t num_frames = asr_feats.size();
+
+ std::vector<float> wav_feats;
+ for (const auto &frame_feat: asr_feats) {
+ wav_feats.insert(wav_feats.end(), frame_feat.begin(), frame_feat.end());
+ }
#ifdef _WIN_X86
Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
@@ -675,6 +689,11 @@
return vocab;
}
+Vocab* Paraformer::GetLmVocab()
+{
+ return lm_vocab;
+}
+
PhoneSet* Paraformer::GetPhoneSet()
{
return phone_set_;
--
Gitblit v1.9.1