/**
|
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
|
* MIT License (https://opensource.org/licenses/MIT)
|
*/
|
|
#include "precomp.h"
|
|
using namespace std;
|
|
namespace funasr {
|
|
Paraformer::Paraformer()
|
:env_(ORT_LOGGING_LEVEL_ERROR, "paraformer"),session_options_{}{
|
}
|
|
// offline
|
void Paraformer::InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){
|
// knf options
|
fbank_opts_.frame_opts.dither = 0;
|
fbank_opts_.mel_opts.num_bins = n_mels;
|
fbank_opts_.frame_opts.samp_freq = MODEL_SAMPLE_RATE;
|
fbank_opts_.frame_opts.window_type = window_type;
|
fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
|
fbank_opts_.frame_opts.frame_length_ms = frame_length;
|
fbank_opts_.energy_floor = 0;
|
fbank_opts_.mel_opts.debug_mel = false;
|
// fbank_ = std::make_unique<knf::OnlineFbank>(fbank_opts);
|
|
// session_options_.SetInterOpNumThreads(1);
|
session_options_.SetIntraOpNumThreads(thread_num);
|
session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
|
// DisableCpuMemArena can improve performance
|
session_options_.DisableCpuMemArena();
|
|
try {
|
m_session_ = std::make_unique<Ort::Session>(env_, am_model.c_str(), session_options_);
|
LOG(INFO) << "Successfully load model from " << am_model;
|
} catch (std::exception const &e) {
|
LOG(ERROR) << "Error when load am onnx model: " << e.what();
|
exit(0);
|
}
|
|
string strName;
|
GetInputName(m_session_.get(), strName);
|
m_strInputNames.push_back(strName.c_str());
|
GetInputName(m_session_.get(), strName,1);
|
m_strInputNames.push_back(strName);
|
|
GetOutputName(m_session_.get(), strName);
|
m_strOutputNames.push_back(strName);
|
GetOutputName(m_session_.get(), strName,1);
|
m_strOutputNames.push_back(strName);
|
|
for (auto& item : m_strInputNames)
|
m_szInputNames.push_back(item.c_str());
|
for (auto& item : m_strOutputNames)
|
m_szOutputNames.push_back(item.c_str());
|
vocab = new Vocab(am_config.c_str());
|
LoadCmvn(am_cmvn.c_str());
|
}
|
|
// online
|
void Paraformer::InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){
|
|
LoadOnlineConfigFromYaml(am_config.c_str());
|
// knf options
|
fbank_opts_.frame_opts.dither = 0;
|
fbank_opts_.mel_opts.num_bins = n_mels;
|
fbank_opts_.frame_opts.samp_freq = MODEL_SAMPLE_RATE;
|
fbank_opts_.frame_opts.window_type = window_type;
|
fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
|
fbank_opts_.frame_opts.frame_length_ms = frame_length;
|
fbank_opts_.energy_floor = 0;
|
fbank_opts_.mel_opts.debug_mel = false;
|
|
// session_options_.SetInterOpNumThreads(1);
|
session_options_.SetIntraOpNumThreads(thread_num);
|
session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
|
// DisableCpuMemArena can improve performance
|
session_options_.DisableCpuMemArena();
|
|
try {
|
encoder_session_ = std::make_unique<Ort::Session>(env_, en_model.c_str(), session_options_);
|
LOG(INFO) << "Successfully load model from " << en_model;
|
} catch (std::exception const &e) {
|
LOG(ERROR) << "Error when load am encoder model: " << e.what();
|
exit(0);
|
}
|
|
try {
|
decoder_session_ = std::make_unique<Ort::Session>(env_, de_model.c_str(), session_options_);
|
LOG(INFO) << "Successfully load model from " << de_model;
|
} catch (std::exception const &e) {
|
LOG(ERROR) << "Error when load am decoder model: " << e.what();
|
exit(0);
|
}
|
|
// encoder
|
string strName;
|
GetInputName(encoder_session_.get(), strName);
|
en_strInputNames.push_back(strName.c_str());
|
GetInputName(encoder_session_.get(), strName,1);
|
en_strInputNames.push_back(strName);
|
|
GetOutputName(encoder_session_.get(), strName);
|
en_strOutputNames.push_back(strName);
|
GetOutputName(encoder_session_.get(), strName,1);
|
en_strOutputNames.push_back(strName);
|
GetOutputName(encoder_session_.get(), strName,2);
|
en_strOutputNames.push_back(strName);
|
|
for (auto& item : en_strInputNames)
|
en_szInputNames_.push_back(item.c_str());
|
for (auto& item : en_strOutputNames)
|
en_szOutputNames_.push_back(item.c_str());
|
|
// decoder
|
int de_input_len = 4 + fsmn_layers;
|
int de_out_len = 2 + fsmn_layers;
|
for(int i=0;i<de_input_len; i++){
|
GetInputName(decoder_session_.get(), strName, i);
|
de_strInputNames.push_back(strName.c_str());
|
}
|
|
for(int i=0;i<de_out_len; i++){
|
GetOutputName(decoder_session_.get(), strName,i);
|
de_strOutputNames.push_back(strName);
|
}
|
|
for (auto& item : de_strInputNames)
|
de_szInputNames_.push_back(item.c_str());
|
for (auto& item : de_strOutputNames)
|
de_szOutputNames_.push_back(item.c_str());
|
|
vocab = new Vocab(am_config.c_str());
|
LoadCmvn(am_cmvn.c_str());
|
}
|
|
// 2pass
|
void Paraformer::InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){
|
// online
|
InitAsr(en_model, de_model, am_cmvn, am_config, thread_num);
|
|
// offline
|
try {
|
m_session_ = std::make_unique<Ort::Session>(env_, am_model.c_str(), session_options_);
|
LOG(INFO) << "Successfully load model from " << am_model;
|
} catch (std::exception const &e) {
|
LOG(ERROR) << "Error when load am onnx model: " << e.what();
|
exit(0);
|
}
|
|
string strName;
|
GetInputName(m_session_.get(), strName);
|
m_strInputNames.push_back(strName.c_str());
|
GetInputName(m_session_.get(), strName,1);
|
m_strInputNames.push_back(strName);
|
|
GetOutputName(m_session_.get(), strName);
|
m_strOutputNames.push_back(strName);
|
GetOutputName(m_session_.get(), strName,1);
|
m_strOutputNames.push_back(strName);
|
|
for (auto& item : m_strInputNames)
|
m_szInputNames.push_back(item.c_str());
|
for (auto& item : m_strOutputNames)
|
m_szOutputNames.push_back(item.c_str());
|
}
|
|
void Paraformer::LoadOnlineConfigFromYaml(const char* filename){
|
|
YAML::Node config;
|
try{
|
config = YAML::LoadFile(filename);
|
}catch(exception const &e){
|
LOG(ERROR) << "Error loading file, yaml file error or not exist.";
|
exit(-1);
|
}
|
|
try{
|
YAML::Node frontend_conf = config["frontend_conf"];
|
YAML::Node encoder_conf = config["encoder_conf"];
|
YAML::Node decoder_conf = config["decoder_conf"];
|
YAML::Node predictor_conf = config["predictor_conf"];
|
|
this->window_type = frontend_conf["window"].as<string>();
|
this->n_mels = frontend_conf["n_mels"].as<int>();
|
this->frame_length = frontend_conf["frame_length"].as<int>();
|
this->frame_shift = frontend_conf["frame_shift"].as<int>();
|
this->lfr_m = frontend_conf["lfr_m"].as<int>();
|
this->lfr_n = frontend_conf["lfr_n"].as<int>();
|
|
this->encoder_size = encoder_conf["output_size"].as<int>();
|
this->fsmn_dims = encoder_conf["output_size"].as<int>();
|
|
this->fsmn_layers = decoder_conf["num_blocks"].as<int>();
|
this->fsmn_lorder = decoder_conf["kernel_size"].as<int>()-1;
|
|
this->cif_threshold = predictor_conf["threshold"].as<double>();
|
this->tail_alphas = predictor_conf["tail_threshold"].as<double>();
|
|
}catch(exception const &e){
|
LOG(ERROR) << "Error when load argument from vad config YAML.";
|
exit(-1);
|
}
|
}
|
|
Paraformer::~Paraformer()
|
{
|
if(vocab)
|
delete vocab;
|
}
|
|
void Paraformer::Reset()
|
{
|
}
|
|
vector<float> Paraformer::FbankKaldi(float sample_rate, const float* waves, int len) {
|
knf::OnlineFbank fbank_(fbank_opts_);
|
std::vector<float> buf(len);
|
for (int32_t i = 0; i != len; ++i) {
|
buf[i] = waves[i] * 32768;
|
}
|
fbank_.AcceptWaveform(sample_rate, buf.data(), buf.size());
|
//fbank_->InputFinished();
|
int32_t frames = fbank_.NumFramesReady();
|
int32_t feature_dim = fbank_opts_.mel_opts.num_bins;
|
vector<float> features(frames * feature_dim);
|
float *p = features.data();
|
|
for (int32_t i = 0; i != frames; ++i) {
|
const float *f = fbank_.GetFrame(i);
|
std::copy(f, f + feature_dim, p);
|
p += feature_dim;
|
}
|
|
return features;
|
}
|
|
void Paraformer::LoadCmvn(const char *filename)
|
{
|
ifstream cmvn_stream(filename);
|
if (!cmvn_stream.is_open()) {
|
LOG(ERROR) << "Failed to open file: " << filename;
|
exit(0);
|
}
|
string line;
|
|
while (getline(cmvn_stream, line)) {
|
istringstream iss(line);
|
vector<string> line_item{istream_iterator<string>{iss}, istream_iterator<string>{}};
|
if (line_item[0] == "<AddShift>") {
|
getline(cmvn_stream, line);
|
istringstream means_lines_stream(line);
|
vector<string> means_lines{istream_iterator<string>{means_lines_stream}, istream_iterator<string>{}};
|
if (means_lines[0] == "<LearnRateCoef>") {
|
for (int j = 3; j < means_lines.size() - 1; j++) {
|
means_list_.push_back(stof(means_lines[j]));
|
}
|
continue;
|
}
|
}
|
else if (line_item[0] == "<Rescale>") {
|
getline(cmvn_stream, line);
|
istringstream vars_lines_stream(line);
|
vector<string> vars_lines{istream_iterator<string>{vars_lines_stream}, istream_iterator<string>{}};
|
if (vars_lines[0] == "<LearnRateCoef>") {
|
for (int j = 3; j < vars_lines.size() - 1; j++) {
|
vars_list_.push_back(stof(vars_lines[j])*scale);
|
}
|
continue;
|
}
|
}
|
}
|
}
|
|
string Paraformer::GreedySearch(float * in, int n_len, int64_t token_nums)
|
{
|
vector<int> hyps;
|
int Tmax = n_len;
|
for (int i = 0; i < Tmax; i++) {
|
int max_idx;
|
float max_val;
|
FindMax(in + i * token_nums, token_nums, max_val, max_idx);
|
hyps.push_back(max_idx);
|
}
|
|
return vocab->Vector2StringV2(hyps);
|
}
|
|
vector<float> Paraformer::ApplyLfr(const std::vector<float> &in)
|
{
|
int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;
|
int32_t in_num_frames = in.size() / in_feat_dim;
|
int32_t out_num_frames =
|
(in_num_frames - lfr_m) / lfr_n + 1;
|
int32_t out_feat_dim = in_feat_dim * lfr_m;
|
|
std::vector<float> out(out_num_frames * out_feat_dim);
|
|
const float *p_in = in.data();
|
float *p_out = out.data();
|
|
for (int32_t i = 0; i != out_num_frames; ++i) {
|
std::copy(p_in, p_in + out_feat_dim, p_out);
|
|
p_out += out_feat_dim;
|
p_in += lfr_n * in_feat_dim;
|
}
|
|
return out;
|
}
|
|
void Paraformer::ApplyCmvn(std::vector<float> *v)
|
{
|
int32_t dim = means_list_.size();
|
int32_t num_frames = v->size() / dim;
|
|
float *p = v->data();
|
|
for (int32_t i = 0; i != num_frames; ++i) {
|
for (int32_t k = 0; k != dim; ++k) {
|
p[k] = (p[k] + means_list_[k]) * vars_list_[k];
|
}
|
|
p += dim;
|
}
|
}
|
|
string Paraformer::Forward(float* din, int len, bool input_finished)
|
{
|
|
int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;
|
std::vector<float> wav_feats = FbankKaldi(MODEL_SAMPLE_RATE, din, len);
|
wav_feats = ApplyLfr(wav_feats);
|
ApplyCmvn(&wav_feats);
|
|
int32_t feat_dim = lfr_m*in_feat_dim;
|
int32_t num_frames = wav_feats.size() / feat_dim;
|
|
#ifdef _WIN_X86
|
Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
|
#else
|
Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
|
#endif
|
|
const int64_t input_shape_[3] = {1, num_frames, feat_dim};
|
Ort::Value onnx_feats = Ort::Value::CreateTensor<float>(m_memoryInfo,
|
wav_feats.data(),
|
wav_feats.size(),
|
input_shape_,
|
3);
|
|
const int64_t paraformer_length_shape[1] = {1};
|
std::vector<int32_t> paraformer_length;
|
paraformer_length.emplace_back(num_frames);
|
Ort::Value onnx_feats_len = Ort::Value::CreateTensor<int32_t>(
|
m_memoryInfo, paraformer_length.data(), paraformer_length.size(), paraformer_length_shape, 1);
|
|
std::vector<Ort::Value> input_onnx;
|
input_onnx.emplace_back(std::move(onnx_feats));
|
input_onnx.emplace_back(std::move(onnx_feats_len));
|
|
string result;
|
try {
|
auto outputTensor = m_session_->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), input_onnx.size(), m_szOutputNames.data(), m_szOutputNames.size());
|
std::vector<int64_t> outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape();
|
|
int64_t outputCount = std::accumulate(outputShape.begin(), outputShape.end(), 1, std::multiplies<int64_t>());
|
float* floatData = outputTensor[0].GetTensorMutableData<float>();
|
auto encoder_out_lens = outputTensor[1].GetTensorMutableData<int64_t>();
|
result = GreedySearch(floatData, *encoder_out_lens, outputShape[2]);
|
}
|
catch (std::exception const &e)
|
{
|
LOG(ERROR)<<e.what();
|
}
|
|
return result;
|
}
|
|
string Paraformer::Rescoring()
|
{
|
LOG(ERROR)<<"Not Imp!!!!!!";
|
return "";
|
}
|
} // namespace funasr
|
