/** * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. * MIT License (https://opensource.org/licenses/MIT) */ #include "precomp.h" using namespace std; namespace funasr { Paraformer::Paraformer() :env_(ORT_LOGGING_LEVEL_ERROR, "paraformer"),session_options_{}{ } // offline void Paraformer::InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){ // knf options fbank_opts_.frame_opts.dither = 0; fbank_opts_.mel_opts.num_bins = n_mels; fbank_opts_.frame_opts.samp_freq = MODEL_SAMPLE_RATE; fbank_opts_.frame_opts.window_type = window_type; fbank_opts_.frame_opts.frame_shift_ms = frame_shift; fbank_opts_.frame_opts.frame_length_ms = frame_length; fbank_opts_.energy_floor = 0; fbank_opts_.mel_opts.debug_mel = false; // fbank_ = std::make_unique(fbank_opts); // session_options_.SetInterOpNumThreads(1); session_options_.SetIntraOpNumThreads(thread_num); session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL); // DisableCpuMemArena can improve performance session_options_.DisableCpuMemArena(); try { m_session_ = std::make_unique(env_, am_model.c_str(), session_options_); LOG(INFO) << "Successfully load model from " << am_model; } catch (std::exception const &e) { LOG(ERROR) << "Error when load am onnx model: " << e.what(); exit(0); } string strName; GetInputName(m_session_.get(), strName); m_strInputNames.push_back(strName.c_str()); GetInputName(m_session_.get(), strName,1); m_strInputNames.push_back(strName); GetOutputName(m_session_.get(), strName); m_strOutputNames.push_back(strName); GetOutputName(m_session_.get(), strName,1); m_strOutputNames.push_back(strName); for (auto& item : m_strInputNames) m_szInputNames.push_back(item.c_str()); for (auto& item : m_strOutputNames) m_szOutputNames.push_back(item.c_str()); vocab = new Vocab(am_config.c_str()); LoadCmvn(am_cmvn.c_str()); } // online void Paraformer::InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){ LoadOnlineConfigFromYaml(am_config.c_str()); // knf options fbank_opts_.frame_opts.dither = 0; fbank_opts_.mel_opts.num_bins = n_mels; fbank_opts_.frame_opts.samp_freq = MODEL_SAMPLE_RATE; fbank_opts_.frame_opts.window_type = window_type; fbank_opts_.frame_opts.frame_shift_ms = frame_shift; fbank_opts_.frame_opts.frame_length_ms = frame_length; fbank_opts_.energy_floor = 0; fbank_opts_.mel_opts.debug_mel = false; // session_options_.SetInterOpNumThreads(1); session_options_.SetIntraOpNumThreads(thread_num); session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL); // DisableCpuMemArena can improve performance session_options_.DisableCpuMemArena(); try { encoder_session_ = std::make_unique(env_, en_model.c_str(), session_options_); LOG(INFO) << "Successfully load model from " << en_model; } catch (std::exception const &e) { LOG(ERROR) << "Error when load am encoder model: " << e.what(); exit(0); } try { decoder_session_ = std::make_unique(env_, de_model.c_str(), session_options_); LOG(INFO) << "Successfully load model from " << de_model; } catch (std::exception const &e) { LOG(ERROR) << "Error when load am decoder model: " << e.what(); exit(0); } // encoder string strName; GetInputName(encoder_session_.get(), strName); en_strInputNames.push_back(strName.c_str()); GetInputName(encoder_session_.get(), strName,1); en_strInputNames.push_back(strName); GetOutputName(encoder_session_.get(), strName); en_strOutputNames.push_back(strName); GetOutputName(encoder_session_.get(), strName,1); en_strOutputNames.push_back(strName); GetOutputName(encoder_session_.get(), strName,2); en_strOutputNames.push_back(strName); for (auto& item : en_strInputNames) en_szInputNames_.push_back(item.c_str()); for (auto& item : en_strOutputNames) en_szOutputNames_.push_back(item.c_str()); // decoder int de_input_len = 4 + fsmn_layers; int de_out_len = 2 + fsmn_layers; for(int i=0;i(env_, am_model.c_str(), session_options_); LOG(INFO) << "Successfully load model from " << am_model; } catch (std::exception const &e) { LOG(ERROR) << "Error when load am onnx model: " << e.what(); exit(0); } string strName; GetInputName(m_session_.get(), strName); m_strInputNames.push_back(strName.c_str()); GetInputName(m_session_.get(), strName,1); m_strInputNames.push_back(strName); GetOutputName(m_session_.get(), strName); m_strOutputNames.push_back(strName); GetOutputName(m_session_.get(), strName,1); m_strOutputNames.push_back(strName); for (auto& item : m_strInputNames) m_szInputNames.push_back(item.c_str()); for (auto& item : m_strOutputNames) m_szOutputNames.push_back(item.c_str()); } void Paraformer::LoadOnlineConfigFromYaml(const char* filename){ YAML::Node config; try{ config = YAML::LoadFile(filename); }catch(exception const &e){ LOG(ERROR) << "Error loading file, yaml file error or not exist."; exit(-1); } try{ YAML::Node frontend_conf = config["frontend_conf"]; YAML::Node encoder_conf = config["encoder_conf"]; YAML::Node decoder_conf = config["decoder_conf"]; YAML::Node predictor_conf = config["predictor_conf"]; this->window_type = frontend_conf["window"].as(); this->n_mels = frontend_conf["n_mels"].as(); this->frame_length = frontend_conf["frame_length"].as(); this->frame_shift = frontend_conf["frame_shift"].as(); this->lfr_m = frontend_conf["lfr_m"].as(); this->lfr_n = frontend_conf["lfr_n"].as(); this->encoder_size = encoder_conf["output_size"].as(); this->fsmn_dims = encoder_conf["output_size"].as(); this->fsmn_layers = decoder_conf["num_blocks"].as(); this->fsmn_lorder = decoder_conf["kernel_size"].as()-1; this->cif_threshold = predictor_conf["threshold"].as(); this->tail_alphas = predictor_conf["tail_threshold"].as(); }catch(exception const &e){ LOG(ERROR) << "Error when load argument from vad config YAML."; exit(-1); } } Paraformer::~Paraformer() { if(vocab) delete vocab; } void Paraformer::Reset() { } vector Paraformer::FbankKaldi(float sample_rate, const float* waves, int len) { knf::OnlineFbank fbank_(fbank_opts_); std::vector buf(len); for (int32_t i = 0; i != len; ++i) { buf[i] = waves[i] * 32768; } fbank_.AcceptWaveform(sample_rate, buf.data(), buf.size()); //fbank_->InputFinished(); int32_t frames = fbank_.NumFramesReady(); int32_t feature_dim = fbank_opts_.mel_opts.num_bins; vector features(frames * feature_dim); float *p = features.data(); for (int32_t i = 0; i != frames; ++i) { const float *f = fbank_.GetFrame(i); std::copy(f, f + feature_dim, p); p += feature_dim; } return features; } void Paraformer::LoadCmvn(const char *filename) { ifstream cmvn_stream(filename); if (!cmvn_stream.is_open()) { LOG(ERROR) << "Failed to open file: " << filename; exit(0); } string line; while (getline(cmvn_stream, line)) { istringstream iss(line); vector line_item{istream_iterator{iss}, istream_iterator{}}; if (line_item[0] == "") { getline(cmvn_stream, line); istringstream means_lines_stream(line); vector means_lines{istream_iterator{means_lines_stream}, istream_iterator{}}; if (means_lines[0] == "") { for (int j = 3; j < means_lines.size() - 1; j++) { means_list_.push_back(stof(means_lines[j])); } continue; } } else if (line_item[0] == "") { getline(cmvn_stream, line); istringstream vars_lines_stream(line); vector vars_lines{istream_iterator{vars_lines_stream}, istream_iterator{}}; if (vars_lines[0] == "") { for (int j = 3; j < vars_lines.size() - 1; j++) { vars_list_.push_back(stof(vars_lines[j])*scale); } continue; } } } } string Paraformer::GreedySearch(float * in, int n_len, int64_t token_nums) { vector hyps; int Tmax = n_len; for (int i = 0; i < Tmax; i++) { int max_idx; float max_val; FindMax(in + i * token_nums, token_nums, max_val, max_idx); hyps.push_back(max_idx); } return vocab->Vector2StringV2(hyps); } vector Paraformer::ApplyLfr(const std::vector &in) { int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins; int32_t in_num_frames = in.size() / in_feat_dim; int32_t out_num_frames = (in_num_frames - lfr_m) / lfr_n + 1; int32_t out_feat_dim = in_feat_dim * lfr_m; std::vector out(out_num_frames * out_feat_dim); const float *p_in = in.data(); float *p_out = out.data(); for (int32_t i = 0; i != out_num_frames; ++i) { std::copy(p_in, p_in + out_feat_dim, p_out); p_out += out_feat_dim; p_in += lfr_n * in_feat_dim; } return out; } void Paraformer::ApplyCmvn(std::vector *v) { int32_t dim = means_list_.size(); int32_t num_frames = v->size() / dim; float *p = v->data(); for (int32_t i = 0; i != num_frames; ++i) { for (int32_t k = 0; k != dim; ++k) { p[k] = (p[k] + means_list_[k]) * vars_list_[k]; } p += dim; } } string Paraformer::Forward(float* din, int len, bool input_finished) { int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins; std::vector wav_feats = FbankKaldi(MODEL_SAMPLE_RATE, din, len); wav_feats = ApplyLfr(wav_feats); ApplyCmvn(&wav_feats); int32_t feat_dim = lfr_m*in_feat_dim; int32_t num_frames = wav_feats.size() / feat_dim; #ifdef _WIN_X86 Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); #else Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); #endif const int64_t input_shape_[3] = {1, num_frames, feat_dim}; Ort::Value onnx_feats = Ort::Value::CreateTensor(m_memoryInfo, wav_feats.data(), wav_feats.size(), input_shape_, 3); const int64_t paraformer_length_shape[1] = {1}; std::vector paraformer_length; paraformer_length.emplace_back(num_frames); Ort::Value onnx_feats_len = Ort::Value::CreateTensor( m_memoryInfo, paraformer_length.data(), paraformer_length.size(), paraformer_length_shape, 1); std::vector input_onnx; input_onnx.emplace_back(std::move(onnx_feats)); input_onnx.emplace_back(std::move(onnx_feats_len)); string result; try { auto outputTensor = m_session_->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), input_onnx.size(), m_szOutputNames.data(), m_szOutputNames.size()); std::vector outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape(); int64_t outputCount = std::accumulate(outputShape.begin(), outputShape.end(), 1, std::multiplies()); float* floatData = outputTensor[0].GetTensorMutableData(); auto encoder_out_lens = outputTensor[1].GetTensorMutableData(); result = GreedySearch(floatData, *encoder_out_lens, outputShape[2]); } catch (std::exception const &e) { LOG(ERROR)<