From c2e4e3c2e9be855277d9f4fa9cd0544892ff829a Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期三, 30 八月 2023 09:57:30 +0800
Subject: [PATCH] Merge branch 'main' of github.com:alibaba-damo-academy/FunASR add
---
funasr/runtime/onnxruntime/src/paraformer.cpp | 217 +++++++++++++++++++++++++++++++++++++++++++++++++++--
1 files changed, 207 insertions(+), 10 deletions(-)
diff --git a/funasr/runtime/onnxruntime/src/paraformer.cpp b/funasr/runtime/onnxruntime/src/paraformer.cpp
index e2c695c..c36a86d 100644
--- a/funasr/runtime/onnxruntime/src/paraformer.cpp
+++ b/funasr/runtime/onnxruntime/src/paraformer.cpp
@@ -4,13 +4,17 @@
*/
#include "precomp.h"
+#include "paraformer.h"
+#include "encode_converter.h"
+#include <cstddef>
using namespace std;
-
namespace funasr {
Paraformer::Paraformer()
-:env_(ORT_LOGGING_LEVEL_ERROR, "paraformer"),session_options_{}{
+:use_hotword(false),
+ env_(ORT_LOGGING_LEVEL_ERROR, "paraformer"),session_options_{},
+ hw_env_(ORT_LOGGING_LEVEL_ERROR, "paraformer_hw"),hw_session_options{} {
}
// offline
@@ -45,6 +49,10 @@
m_strInputNames.push_back(strName.c_str());
GetInputName(m_session_.get(), strName,1);
m_strInputNames.push_back(strName);
+ if (use_hotword) {
+ GetInputName(m_session_.get(), strName, 2);
+ m_strInputNames.push_back(strName);
+ }
size_t numOutputNodes = m_session_->GetOutputCount();
for(int index=0; index<numOutputNodes; index++){
@@ -206,10 +214,47 @@
}
}
+void Paraformer::InitHwCompiler(const std::string &hw_model, int thread_num) {
+ hw_session_options.SetIntraOpNumThreads(thread_num);
+ hw_session_options.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
+ // DisableCpuMemArena can improve performance
+ hw_session_options.DisableCpuMemArena();
+
+ try {
+ hw_m_session = std::make_unique<Ort::Session>(hw_env_, hw_model.c_str(), hw_session_options);
+ LOG(INFO) << "Successfully load model from " << hw_model;
+ } catch (std::exception const &e) {
+ LOG(ERROR) << "Error when load hw compiler onnx model: " << e.what();
+ exit(0);
+ }
+
+ string strName;
+ GetInputName(hw_m_session.get(), strName);
+ hw_m_strInputNames.push_back(strName.c_str());
+ //GetInputName(hw_m_session.get(), strName,1);
+ //hw_m_strInputNames.push_back(strName);
+
+ GetOutputName(hw_m_session.get(), strName);
+ hw_m_strOutputNames.push_back(strName);
+
+ for (auto& item : hw_m_strInputNames)
+ hw_m_szInputNames.push_back(item.c_str());
+ for (auto& item : hw_m_strOutputNames)
+ hw_m_szOutputNames.push_back(item.c_str());
+ // if init hotword compiler is called, this is a hotword paraformer model
+ use_hotword = true;
+}
+
+void Paraformer::InitSegDict(const std::string &seg_dict_model) {
+ seg_dict = new SegDict(seg_dict_model.c_str());
+}
+
Paraformer::~Paraformer()
{
if(vocab)
delete vocab;
+ if(seg_dict)
+ delete seg_dict;
}
void Paraformer::Reset()
@@ -228,6 +273,10 @@
int32_t feature_dim = fbank_opts_.mel_opts.num_bins;
vector<float> features(frames * feature_dim);
float *p = features.data();
+ //std::cout << "samples " << len << std::endl;
+ //std::cout << "fbank frames " << frames << std::endl;
+ //std::cout << "fbank dim " << feature_dim << std::endl;
+ //std::cout << "feature size " << features.size() << std::endl;
for (int32_t i = 0; i != frames; ++i) {
const float *f = fbank_.GetFrame(i);
@@ -389,11 +438,11 @@
}
}
string stamp_str="";
- for (i=0; i<timestamp_list.size(); i++) {
- stamp_str += std::to_string(timestamp_list[i][0]);
+ for (i=0; i<timestamp_merge.size(); i++) {
+ stamp_str += std::to_string(timestamp_merge[i][0]);
stamp_str += ", ";
- stamp_str += std::to_string(timestamp_list[i][1]);
- if(i!=timestamp_list.size()-1){
+ stamp_str += std::to_string(timestamp_merge[i][1]);
+ if(i!=timestamp_merge.size()-1){
stamp_str += ",";
}
}
@@ -426,7 +475,9 @@
if (char_list.back() == "</s>") {
char_list.pop_back();
}
-
+ if (char_list.empty()) {
+ return ;
+ }
vector<vector<float>> timestamp_list;
vector<string> new_char_list;
vector<float> fire_place;
@@ -441,6 +492,9 @@
if(num_peak != (int)char_list.size() + 1){
float sum = std::accumulate(us_alphas.begin(), us_alphas.end(), 0.0f);
float scale = sum/((int)char_list.size() + 1);
+ if(scale == 0){
+ return;
+ }
cif_peak.clear();
sum = 0.0;
for(auto &alpha:us_alphas){
@@ -458,6 +512,11 @@
fire_place.push_back(i + total_offset);
}
}
+ }
+
+ num_peak = fire_place.size();
+ if(fire_place.size() == 0){
+ return;
}
// begin silence
@@ -481,6 +540,10 @@
}
// tail token and end silence
+ if(timestamp_list.size()==0){
+ LOG(ERROR)<<"timestamp_list's size is 0!";
+ return;
+ }
if (num_frames - fire_place.back() > START_END_THRESHOLD) {
float _end = (num_frames + fire_place.back()) / 2.0;
timestamp_list.back()[1] = _end * TIME_RATE;
@@ -549,7 +612,7 @@
}
}
-string Paraformer::Forward(float* din, int len, bool input_finished)
+string Paraformer::Forward(float* din, int len, bool input_finished, const std::vector<std::vector<float>> &hw_emb)
{
int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;
@@ -559,6 +622,7 @@
int32_t feat_dim = lfr_m*in_feat_dim;
int32_t num_frames = wav_feats.size() / feat_dim;
+ //std::cout << "feat in: " << num_frames << " " << feat_dim << std::endl;
#ifdef _WIN_X86
Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
@@ -578,15 +642,41 @@
paraformer_length.emplace_back(num_frames);
Ort::Value onnx_feats_len = Ort::Value::CreateTensor<int32_t>(
m_memoryInfo, paraformer_length.data(), paraformer_length.size(), paraformer_length_shape, 1);
-
+
std::vector<Ort::Value> input_onnx;
input_onnx.emplace_back(std::move(onnx_feats));
input_onnx.emplace_back(std::move(onnx_feats_len));
- string result;
+ std::vector<float> embedding;
+ try{
+ if (use_hotword) {
+ if(hw_emb.size()<=0){
+ LOG(ERROR) << "hw_emb is null";
+ return "";
+ }
+ //PrintMat(hw_emb, "input_clas_emb");
+ const int64_t hotword_shape[3] = {1, hw_emb.size(), hw_emb[0].size()};
+ embedding.reserve(hw_emb.size() * hw_emb[0].size());
+ for (auto item : hw_emb) {
+ embedding.insert(embedding.end(), item.begin(), item.end());
+ }
+ //LOG(INFO) << "hotword shape " << hotword_shape[0] << " " << hotword_shape[1] << " " << hotword_shape[2] << " size " << embedding.size();
+ Ort::Value onnx_hw_emb = Ort::Value::CreateTensor<float>(
+ m_memoryInfo, embedding.data(), embedding.size(), hotword_shape, 3);
+
+ input_onnx.emplace_back(std::move(onnx_hw_emb));
+ }
+ }catch (std::exception const &e)
+ {
+ LOG(ERROR)<<e.what();
+ return "";
+ }
+
+ string result="";
try {
auto outputTensor = m_session_->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), input_onnx.size(), m_szOutputNames.data(), m_szOutputNames.size());
std::vector<int64_t> outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape();
+ //LOG(INFO) << "paraformer out shape " << outputShape[0] << " " << outputShape[1] << " " << outputShape[2];
int64_t outputCount = std::accumulate(outputShape.begin(), outputShape.end(), 1, std::multiplies<int64_t>());
float* floatData = outputTensor[0].GetTensorMutableData<float>();
@@ -610,6 +700,17 @@
}else{
result = GreedySearch(floatData, *encoder_out_lens, outputShape[2]);
}
+// int pos = 0;
+// std::vector<std::vector<float>> logits;
+// for (int j = 0; j < outputShape[1]; j++)
+// {
+// std::vector<float> vec_token;
+// vec_token.insert(vec_token.begin(), floatData + pos, floatData + pos + outputShape[2]);
+// logits.push_back(vec_token);
+// pos += outputShape[2];
+// }
+// //PrintMat(logits, "logits_out");
+// result = GreedySearch(floatData, *encoder_out_lens, outputShape[2]);
}
catch (std::exception const &e)
{
@@ -619,6 +720,102 @@
return result;
}
+
+std::vector<std::vector<float>> Paraformer::CompileHotwordEmbedding(std::string &hotwords) {
+ int embedding_dim = encoder_size;
+ std::vector<std::vector<float>> hw_emb;
+ if (!use_hotword) {
+ std::vector<float> vec(embedding_dim, 0);
+ hw_emb.push_back(vec);
+ return hw_emb;
+ }
+ int max_hotword_len = 10;
+ std::vector<int32_t> hotword_matrix;
+ std::vector<int32_t> lengths;
+ int hotword_size = 1;
+ int real_hw_size = 0;
+ if (!hotwords.empty()) {
+ std::vector<std::string> hotword_array = split(hotwords, ' ');
+ hotword_size = hotword_array.size() + 1;
+ hotword_matrix.reserve(hotword_size * max_hotword_len);
+ for (auto hotword : hotword_array) {
+ std::vector<std::string> chars;
+ if (EncodeConverter::IsAllChineseCharactor((const U8CHAR_T*)hotword.c_str(), hotword.size())) {
+ KeepChineseCharacterAndSplit(hotword, chars);
+ } else {
+ // for english
+ std::vector<std::string> words = split(hotword, ' ');
+ for (auto word : words) {
+ std::vector<string> tokens = seg_dict->GetTokensByWord(word);
+ chars.insert(chars.end(), tokens.begin(), tokens.end());
+ }
+ }
+ if(chars.size()==0){
+ continue;
+ }
+ std::vector<int32_t> hw_vector(max_hotword_len, 0);
+ int vector_len = std::min(max_hotword_len, (int)chars.size());
+ for (int i=0; i<chars.size(); i++) {
+ std::cout << chars[i] << " ";
+ hw_vector[i] = vocab->GetIdByToken(chars[i]);
+ }
+ std::cout << std::endl;
+ lengths.push_back(vector_len);
+ real_hw_size += 1;
+ hotword_matrix.insert(hotword_matrix.end(), hw_vector.begin(), hw_vector.end());
+ }
+ hotword_size = real_hw_size + 1;
+ }
+ std::vector<int32_t> blank_vec(max_hotword_len, 0);
+ blank_vec[0] = 1;
+ hotword_matrix.insert(hotword_matrix.end(), blank_vec.begin(), blank_vec.end());
+ lengths.push_back(1);
+
+#ifdef _WIN_X86
+ Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
+#else
+ Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
+#endif
+
+ const int64_t input_shape_[2] = {hotword_size, max_hotword_len};
+ Ort::Value onnx_hotword = Ort::Value::CreateTensor<int32_t>(m_memoryInfo,
+ (int32_t*)hotword_matrix.data(),
+ hotword_size * max_hotword_len,
+ input_shape_,
+ 2);
+ LOG(INFO) << "clas shape " << hotword_size << " " << max_hotword_len << std::endl;
+
+ std::vector<Ort::Value> input_onnx;
+ input_onnx.emplace_back(std::move(onnx_hotword));
+
+ std::vector<std::vector<float>> result;
+ try {
+ auto outputTensor = hw_m_session->Run(Ort::RunOptions{nullptr}, hw_m_szInputNames.data(), input_onnx.data(), input_onnx.size(), hw_m_szOutputNames.data(), hw_m_szOutputNames.size());
+ std::vector<int64_t> outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape();
+
+ int64_t outputCount = std::accumulate(outputShape.begin(), outputShape.end(), 1, std::multiplies<int64_t>());
+ float* floatData = outputTensor[0].GetTensorMutableData<float>(); // shape [max_hotword_len, hotword_size, dim]
+ // get embedding by real hotword length
+ assert(outputShape[0] == max_hotword_len);
+ assert(outputShape[1] == hotword_size);
+ embedding_dim = outputShape[2];
+
+ for (int j = 0; j < hotword_size; j++)
+ {
+ int start_pos = hotword_size * (lengths[j] - 1) * embedding_dim + j * embedding_dim;
+ std::vector<float> embedding;
+ embedding.insert(embedding.begin(), floatData + start_pos, floatData + start_pos + embedding_dim);
+ result.push_back(embedding);
+ }
+ }
+ catch (std::exception const &e)
+ {
+ LOG(ERROR)<<e.what();
+ }
+ //PrintMat(result, "clas_embedding_output");
+ return result;
+}
+
string Paraformer::Rescoring()
{
LOG(ERROR)<<"Not Imp!!!!!!";
--
Gitblit v1.9.1