From ffb05b9ae7eccc47416e9e7fae9dea54d400a245 Mon Sep 17 00:00:00 2001
From: 雾聪 <wucong.lyb@alibaba-inc.com>
Date: 星期四, 10 八月 2023 19:05:51 +0800
Subject: [PATCH] Merge branch 'main' of https://github.com/alibaba-damo-academy/FunASR into main
---
funasr/runtime/onnxruntime/src/paraformer.cpp | 299 ++++++++++++++++++++++++++++++++++++++++++-----------------
1 files changed, 212 insertions(+), 87 deletions(-)
diff --git a/funasr/runtime/onnxruntime/src/paraformer.cpp b/funasr/runtime/onnxruntime/src/paraformer.cpp
index 493dd6d..ef2a182 100644
--- a/funasr/runtime/onnxruntime/src/paraformer.cpp
+++ b/funasr/runtime/onnxruntime/src/paraformer.cpp
@@ -1,77 +1,208 @@
+/**
+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+ * MIT License (https://opensource.org/licenses/MIT)
+*/
+
#include "precomp.h"
using namespace std;
-using namespace paraformer;
-Paraformer::Paraformer(const char* path,int thread_num, bool quantize, bool use_vad, bool use_punc)
-:env_(ORT_LOGGING_LEVEL_ERROR, "paraformer"),session_options{}{
- string model_path;
- string cmvn_path;
- string config_path;
+namespace funasr {
- // VAD model
- if(use_vad){
- string vad_path = PathAppend(path, "vad_model.onnx");
- string mvn_path = PathAppend(path, "vad.mvn");
- vad_handle = make_unique<FsmnVad>();
- vad_handle->InitVad(vad_path, mvn_path, MODEL_SAMPLE_RATE, VAD_MAX_LEN, VAD_SILENCE_DYRATION, VAD_SPEECH_NOISE_THRES);
- }
+Paraformer::Paraformer()
+:env_(ORT_LOGGING_LEVEL_ERROR, "paraformer"),session_options_{}{
+}
- // PUNC model
- if(use_punc){
- punc_handle = make_unique<CTTransformer>(path, thread_num);
- }
-
- if(quantize)
- {
- model_path = PathAppend(path, "model_quant.onnx");
- }else{
- model_path = PathAppend(path, "model.onnx");
- }
- cmvn_path = PathAppend(path, "am.mvn");
- config_path = PathAppend(path, "config.yaml");
-
+// offline
+void Paraformer::InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){
// knf options
- fbank_opts.frame_opts.dither = 0;
- fbank_opts.mel_opts.num_bins = 80;
- fbank_opts.frame_opts.samp_freq = MODEL_SAMPLE_RATE;
- fbank_opts.frame_opts.window_type = "hamming";
- fbank_opts.frame_opts.frame_shift_ms = 10;
- fbank_opts.frame_opts.frame_length_ms = 25;
- fbank_opts.energy_floor = 0;
- fbank_opts.mel_opts.debug_mel = false;
+ fbank_opts_.frame_opts.dither = 0;
+ fbank_opts_.mel_opts.num_bins = n_mels;
+ fbank_opts_.frame_opts.samp_freq = MODEL_SAMPLE_RATE;
+ fbank_opts_.frame_opts.window_type = window_type;
+ fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
+ fbank_opts_.frame_opts.frame_length_ms = frame_length;
+ fbank_opts_.energy_floor = 0;
+ fbank_opts_.mel_opts.debug_mel = false;
// fbank_ = std::make_unique<knf::OnlineFbank>(fbank_opts);
- // session_options.SetInterOpNumThreads(1);
- session_options.SetIntraOpNumThreads(thread_num);
- session_options.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+ // session_options_.SetInterOpNumThreads(1);
+ session_options_.SetIntraOpNumThreads(thread_num);
+ session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
// DisableCpuMemArena can improve performance
- session_options.DisableCpuMemArena();
+ session_options_.DisableCpuMemArena();
-#ifdef _WIN32
- wstring wstrPath = strToWstr(model_path);
- m_session = std::make_unique<Ort::Session>(env_, model_path.c_str(), session_options);
-#else
- m_session = std::make_unique<Ort::Session>(env_, model_path.c_str(), session_options);
-#endif
+ try {
+ m_session_ = std::make_unique<Ort::Session>(env_, am_model.c_str(), session_options_);
+ LOG(INFO) << "Successfully load model from " << am_model;
+ } catch (std::exception const &e) {
+ LOG(ERROR) << "Error when load am onnx model: " << e.what();
+ exit(0);
+ }
string strName;
- GetInputName(m_session.get(), strName);
+ GetInputName(m_session_.get(), strName);
m_strInputNames.push_back(strName.c_str());
- GetInputName(m_session.get(), strName,1);
+ GetInputName(m_session_.get(), strName,1);
m_strInputNames.push_back(strName);
- GetOutputName(m_session.get(), strName);
+ GetOutputName(m_session_.get(), strName);
m_strOutputNames.push_back(strName);
- GetOutputName(m_session.get(), strName,1);
+ GetOutputName(m_session_.get(), strName,1);
m_strOutputNames.push_back(strName);
for (auto& item : m_strInputNames)
m_szInputNames.push_back(item.c_str());
for (auto& item : m_strOutputNames)
m_szOutputNames.push_back(item.c_str());
- vocab = new Vocab(config_path.c_str());
- LoadCmvn(cmvn_path.c_str());
+ vocab = new Vocab(am_config.c_str());
+ LoadCmvn(am_cmvn.c_str());
+}
+
+// online
+void Paraformer::InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){
+
+ LoadOnlineConfigFromYaml(am_config.c_str());
+ // knf options
+ fbank_opts_.frame_opts.dither = 0;
+ fbank_opts_.mel_opts.num_bins = n_mels;
+ fbank_opts_.frame_opts.samp_freq = MODEL_SAMPLE_RATE;
+ fbank_opts_.frame_opts.window_type = window_type;
+ fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
+ fbank_opts_.frame_opts.frame_length_ms = frame_length;
+ fbank_opts_.energy_floor = 0;
+ fbank_opts_.mel_opts.debug_mel = false;
+
+ // session_options_.SetInterOpNumThreads(1);
+ session_options_.SetIntraOpNumThreads(thread_num);
+ session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+ // DisableCpuMemArena can improve performance
+ session_options_.DisableCpuMemArena();
+
+ try {
+ encoder_session_ = std::make_unique<Ort::Session>(env_, en_model.c_str(), session_options_);
+ LOG(INFO) << "Successfully load model from " << en_model;
+ } catch (std::exception const &e) {
+ LOG(ERROR) << "Error when load am encoder model: " << e.what();
+ exit(0);
+ }
+
+ try {
+ decoder_session_ = std::make_unique<Ort::Session>(env_, de_model.c_str(), session_options_);
+ LOG(INFO) << "Successfully load model from " << de_model;
+ } catch (std::exception const &e) {
+ LOG(ERROR) << "Error when load am decoder model: " << e.what();
+ exit(0);
+ }
+
+ // encoder
+ string strName;
+ GetInputName(encoder_session_.get(), strName);
+ en_strInputNames.push_back(strName.c_str());
+ GetInputName(encoder_session_.get(), strName,1);
+ en_strInputNames.push_back(strName);
+
+ GetOutputName(encoder_session_.get(), strName);
+ en_strOutputNames.push_back(strName);
+ GetOutputName(encoder_session_.get(), strName,1);
+ en_strOutputNames.push_back(strName);
+ GetOutputName(encoder_session_.get(), strName,2);
+ en_strOutputNames.push_back(strName);
+
+ for (auto& item : en_strInputNames)
+ en_szInputNames_.push_back(item.c_str());
+ for (auto& item : en_strOutputNames)
+ en_szOutputNames_.push_back(item.c_str());
+
+ // decoder
+ int de_input_len = 4 + fsmn_layers;
+ int de_out_len = 2 + fsmn_layers;
+ for(int i=0;i<de_input_len; i++){
+ GetInputName(decoder_session_.get(), strName, i);
+ de_strInputNames.push_back(strName.c_str());
+ }
+
+ for(int i=0;i<de_out_len; i++){
+ GetOutputName(decoder_session_.get(), strName,i);
+ de_strOutputNames.push_back(strName);
+ }
+
+ for (auto& item : de_strInputNames)
+ de_szInputNames_.push_back(item.c_str());
+ for (auto& item : de_strOutputNames)
+ de_szOutputNames_.push_back(item.c_str());
+
+ vocab = new Vocab(am_config.c_str());
+ LoadCmvn(am_cmvn.c_str());
+}
+
+// 2pass
+void Paraformer::InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){
+ // online
+ InitAsr(en_model, de_model, am_cmvn, am_config, thread_num);
+
+ // offline
+ try {
+ m_session_ = std::make_unique<Ort::Session>(env_, am_model.c_str(), session_options_);
+ LOG(INFO) << "Successfully load model from " << am_model;
+ } catch (std::exception const &e) {
+ LOG(ERROR) << "Error when load am onnx model: " << e.what();
+ exit(0);
+ }
+
+ string strName;
+ GetInputName(m_session_.get(), strName);
+ m_strInputNames.push_back(strName.c_str());
+ GetInputName(m_session_.get(), strName,1);
+ m_strInputNames.push_back(strName);
+
+ GetOutputName(m_session_.get(), strName);
+ m_strOutputNames.push_back(strName);
+ GetOutputName(m_session_.get(), strName,1);
+ m_strOutputNames.push_back(strName);
+
+ for (auto& item : m_strInputNames)
+ m_szInputNames.push_back(item.c_str());
+ for (auto& item : m_strOutputNames)
+ m_szOutputNames.push_back(item.c_str());
+}
+
+void Paraformer::LoadOnlineConfigFromYaml(const char* filename){
+
+ YAML::Node config;
+ try{
+ config = YAML::LoadFile(filename);
+ }catch(exception const &e){
+ LOG(ERROR) << "Error loading file, yaml file error or not exist.";
+ exit(-1);
+ }
+
+ try{
+ YAML::Node frontend_conf = config["frontend_conf"];
+ YAML::Node encoder_conf = config["encoder_conf"];
+ YAML::Node decoder_conf = config["decoder_conf"];
+ YAML::Node predictor_conf = config["predictor_conf"];
+
+ this->window_type = frontend_conf["window"].as<string>();
+ this->n_mels = frontend_conf["n_mels"].as<int>();
+ this->frame_length = frontend_conf["frame_length"].as<int>();
+ this->frame_shift = frontend_conf["frame_shift"].as<int>();
+ this->lfr_m = frontend_conf["lfr_m"].as<int>();
+ this->lfr_n = frontend_conf["lfr_n"].as<int>();
+
+ this->encoder_size = encoder_conf["output_size"].as<int>();
+ this->fsmn_dims = encoder_conf["output_size"].as<int>();
+
+ this->fsmn_layers = decoder_conf["num_blocks"].as<int>();
+ this->fsmn_lorder = decoder_conf["kernel_size"].as<int>()-1;
+
+ this->cif_threshold = predictor_conf["threshold"].as<double>();
+ this->tail_alphas = predictor_conf["tail_threshold"].as<double>();
+
+ }catch(exception const &e){
+ LOG(ERROR) << "Error when load argument from vad config YAML.";
+ exit(-1);
+ }
}
Paraformer::~Paraformer()
@@ -84,20 +215,16 @@
{
}
-vector<std::vector<int>> Paraformer::VadSeg(std::vector<float>& pcm_data){
- return vad_handle->Infer(pcm_data);
-}
-
-string Paraformer::AddPunc(const char* sz_input){
- return punc_handle->AddPunc(sz_input);
-}
-
vector<float> Paraformer::FbankKaldi(float sample_rate, const float* waves, int len) {
- knf::OnlineFbank fbank_(fbank_opts);
- fbank_.AcceptWaveform(sample_rate, waves, len);
+ knf::OnlineFbank fbank_(fbank_opts_);
+ std::vector<float> buf(len);
+ for (int32_t i = 0; i != len; ++i) {
+ buf[i] = waves[i] * 32768;
+ }
+ fbank_.AcceptWaveform(sample_rate, buf.data(), buf.size());
//fbank_->InputFinished();
int32_t frames = fbank_.NumFramesReady();
- int32_t feature_dim = fbank_opts.mel_opts.num_bins;
+ int32_t feature_dim = fbank_opts_.mel_opts.num_bins;
vector<float> features(frames * feature_dim);
float *p = features.data();
@@ -113,6 +240,10 @@
void Paraformer::LoadCmvn(const char *filename)
{
ifstream cmvn_stream(filename);
+ if (!cmvn_stream.is_open()) {
+ LOG(ERROR) << "Failed to open file: " << filename;
+ exit(0);
+ }
string line;
while (getline(cmvn_stream, line)) {
@@ -124,7 +255,7 @@
vector<string> means_lines{istream_iterator<string>{means_lines_stream}, istream_iterator<string>{}};
if (means_lines[0] == "<LearnRateCoef>") {
for (int j = 3; j < means_lines.size() - 1; j++) {
- means_list.push_back(stof(means_lines[j]));
+ means_list_.push_back(stof(means_lines[j]));
}
continue;
}
@@ -135,7 +266,7 @@
vector<string> vars_lines{istream_iterator<string>{vars_lines_stream}, istream_iterator<string>{}};
if (vars_lines[0] == "<LearnRateCoef>") {
for (int j = 3; j < vars_lines.size() - 1; j++) {
- vars_list.push_back(stof(vars_lines[j])*scale);
+ vars_list_.push_back(stof(vars_lines[j])*scale);
}
continue;
}
@@ -143,14 +274,14 @@
}
}
-string Paraformer::GreedySearch(float * in, int n_len )
+string Paraformer::GreedySearch(float * in, int n_len, int64_t token_nums)
{
vector<int> hyps;
int Tmax = n_len;
for (int i = 0; i < Tmax; i++) {
int max_idx;
float max_val;
- FindMax(in + i * 8404, 8404, max_val, max_idx);
+ FindMax(in + i * token_nums, token_nums, max_val, max_idx);
hyps.push_back(max_idx);
}
@@ -159,11 +290,11 @@
vector<float> Paraformer::ApplyLfr(const std::vector<float> &in)
{
- int32_t in_feat_dim = fbank_opts.mel_opts.num_bins;
+ int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;
int32_t in_num_frames = in.size() / in_feat_dim;
int32_t out_num_frames =
- (in_num_frames - lfr_window_size) / lfr_window_shift + 1;
- int32_t out_feat_dim = in_feat_dim * lfr_window_size;
+ (in_num_frames - lfr_m) / lfr_n + 1;
+ int32_t out_feat_dim = in_feat_dim * lfr_m;
std::vector<float> out(out_num_frames * out_feat_dim);
@@ -174,7 +305,7 @@
std::copy(p_in, p_in + out_feat_dim, p_out);
p_out += out_feat_dim;
- p_in += lfr_window_shift * in_feat_dim;
+ p_in += lfr_n * in_feat_dim;
}
return out;
@@ -182,29 +313,29 @@
void Paraformer::ApplyCmvn(std::vector<float> *v)
{
- int32_t dim = means_list.size();
+ int32_t dim = means_list_.size();
int32_t num_frames = v->size() / dim;
float *p = v->data();
for (int32_t i = 0; i != num_frames; ++i) {
for (int32_t k = 0; k != dim; ++k) {
- p[k] = (p[k] + means_list[k]) * vars_list[k];
+ p[k] = (p[k] + means_list_[k]) * vars_list_[k];
}
p += dim;
}
}
-string Paraformer::Forward(float* din, int len, int flag)
+string Paraformer::Forward(float* din, int len, bool input_finished)
{
- int32_t in_feat_dim = fbank_opts.mel_opts.num_bins;
+ int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;
std::vector<float> wav_feats = FbankKaldi(MODEL_SAMPLE_RATE, din, len);
wav_feats = ApplyLfr(wav_feats);
ApplyCmvn(&wav_feats);
- int32_t feat_dim = lfr_window_size*in_feat_dim;
+ int32_t feat_dim = lfr_m*in_feat_dim;
int32_t num_frames = wav_feats.size() / feat_dim;
#ifdef _WIN_X86
@@ -232,31 +363,25 @@
string result;
try {
- auto outputTensor = m_session->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), input_onnx.size(), m_szOutputNames.data(), m_szOutputNames.size());
+ auto outputTensor = m_session_->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), input_onnx.size(), m_szOutputNames.data(), m_szOutputNames.size());
std::vector<int64_t> outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape();
int64_t outputCount = std::accumulate(outputShape.begin(), outputShape.end(), 1, std::multiplies<int64_t>());
float* floatData = outputTensor[0].GetTensorMutableData<float>();
auto encoder_out_lens = outputTensor[1].GetTensorMutableData<int64_t>();
- result = GreedySearch(floatData, *encoder_out_lens);
+ result = GreedySearch(floatData, *encoder_out_lens, outputShape[2]);
}
catch (std::exception const &e)
{
- printf(e.what());
+ LOG(ERROR)<<e.what();
}
return result;
}
-string Paraformer::ForwardChunk(float* din, int len, int flag)
-{
-
- printf("Not Imp!!!!!!\n");
- return "Hello";
-}
-
string Paraformer::Rescoring()
{
- printf("Not Imp!!!!!!\n");
- return "Hello";
+ LOG(ERROR)<<"Not Imp!!!!!!";
+ return "";
}
+} // namespace funasr
--
Gitblit v1.9.1