From 1d7bbbffb6a024a33859b48a7a656d0455dc0be1 Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: 星期一, 16 十月 2023 11:47:59 +0800
Subject: [PATCH] Update README.md
---
funasr/runtime/onnxruntime/src/paraformer.cpp | 106 ++++++++++++++++++++++++++++++++++++-----------------
1 files changed, 72 insertions(+), 34 deletions(-)
diff --git a/funasr/runtime/onnxruntime/src/paraformer.cpp b/funasr/runtime/onnxruntime/src/paraformer.cpp
index cbaab2d..5bbaeef 100644
--- a/funasr/runtime/onnxruntime/src/paraformer.cpp
+++ b/funasr/runtime/onnxruntime/src/paraformer.cpp
@@ -37,11 +37,11 @@
session_options_.DisableCpuMemArena();
try {
- m_session_ = std::make_unique<Ort::Session>(env_, am_model.c_str(), session_options_);
+ m_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(am_model).c_str(), session_options_);
LOG(INFO) << "Successfully load model from " << am_model;
} catch (std::exception const &e) {
LOG(ERROR) << "Error when load am onnx model: " << e.what();
- exit(0);
+ exit(-1);
}
string strName;
@@ -65,6 +65,7 @@
for (auto& item : m_strOutputNames)
m_szOutputNames.push_back(item.c_str());
vocab = new Vocab(am_config.c_str());
+ LoadConfigFromYaml(am_config.c_str());
LoadCmvn(am_cmvn.c_str());
}
@@ -89,19 +90,19 @@
session_options_.DisableCpuMemArena();
try {
- encoder_session_ = std::make_unique<Ort::Session>(env_, en_model.c_str(), session_options_);
+ encoder_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(en_model).c_str(), session_options_);
LOG(INFO) << "Successfully load model from " << en_model;
} catch (std::exception const &e) {
LOG(ERROR) << "Error when load am encoder model: " << e.what();
- exit(0);
+ exit(-1);
}
try {
- decoder_session_ = std::make_unique<Ort::Session>(env_, de_model.c_str(), session_options_);
+ decoder_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(de_model).c_str(), session_options_);
LOG(INFO) << "Successfully load model from " << de_model;
} catch (std::exception const &e) {
LOG(ERROR) << "Error when load am decoder model: " << e.what();
- exit(0);
+ exit(-1);
}
// encoder
@@ -152,11 +153,11 @@
// offline
try {
- m_session_ = std::make_unique<Ort::Session>(env_, am_model.c_str(), session_options_);
+ m_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(am_model).c_str(), session_options_);
LOG(INFO) << "Successfully load model from " << am_model;
} catch (std::exception const &e) {
LOG(ERROR) << "Error when load am onnx model: " << e.what();
- exit(0);
+ exit(-1);
}
string strName;
@@ -164,16 +165,44 @@
m_strInputNames.push_back(strName.c_str());
GetInputName(m_session_.get(), strName,1);
m_strInputNames.push_back(strName);
+
+ if (use_hotword) {
+ GetInputName(m_session_.get(), strName, 2);
+ m_strInputNames.push_back(strName);
+ }
- GetOutputName(m_session_.get(), strName);
- m_strOutputNames.push_back(strName);
- GetOutputName(m_session_.get(), strName,1);
- m_strOutputNames.push_back(strName);
+ // support time stamp
+ size_t numOutputNodes = m_session_->GetOutputCount();
+ for(int index=0; index<numOutputNodes; index++){
+ GetOutputName(m_session_.get(), strName, index);
+ m_strOutputNames.push_back(strName);
+ }
for (auto& item : m_strInputNames)
m_szInputNames.push_back(item.c_str());
for (auto& item : m_strOutputNames)
m_szOutputNames.push_back(item.c_str());
+}
+
+void Paraformer::LoadConfigFromYaml(const char* filename){
+
+ YAML::Node config;
+ try{
+ config = YAML::LoadFile(filename);
+ }catch(exception const &e){
+ LOG(ERROR) << "Error loading file, yaml file error or not exist.";
+ exit(-1);
+ }
+
+ try{
+ YAML::Node lang_conf = config["lang"];
+ if (lang_conf.IsDefined()){
+ language = lang_conf.as<string>();
+ }
+ }catch(exception const &e){
+ LOG(ERROR) << "Error when load argument from vad config YAML.";
+ exit(-1);
+ }
}
void Paraformer::LoadOnlineConfigFromYaml(const char* filename){
@@ -221,11 +250,11 @@
hw_session_options.DisableCpuMemArena();
try {
- hw_m_session = std::make_unique<Ort::Session>(hw_env_, hw_model.c_str(), hw_session_options);
+ hw_m_session = std::make_unique<Ort::Session>(hw_env_, ORTSTRING(hw_model).c_str(), hw_session_options);
LOG(INFO) << "Successfully load model from " << hw_model;
} catch (std::exception const &e) {
LOG(ERROR) << "Error when load hw compiler onnx model: " << e.what();
- exit(0);
+ exit(-1);
}
string strName;
@@ -292,7 +321,7 @@
ifstream cmvn_stream(filename);
if (!cmvn_stream.is_open()) {
LOG(ERROR) << "Failed to open file: " << filename;
- exit(0);
+ exit(-1);
}
string line;
@@ -335,7 +364,7 @@
hyps.push_back(max_idx);
}
if(!is_stamp){
- return vocab->Vector2StringV2(hyps);
+ return vocab->Vector2StringV2(hyps, language);
}else{
std::vector<string> char_list;
std::vector<std::vector<float>> timestamp_list;
@@ -438,11 +467,11 @@
}
}
string stamp_str="";
- for (i=0; i<timestamp_list.size(); i++) {
- stamp_str += std::to_string(timestamp_list[i][0]);
+ for (i=0; i<timestamp_merge.size(); i++) {
+ stamp_str += std::to_string(timestamp_merge[i][0]);
stamp_str += ", ";
- stamp_str += std::to_string(timestamp_list[i][1]);
- if(i!=timestamp_list.size()-1){
+ stamp_str += std::to_string(timestamp_merge[i][1]);
+ if(i!=timestamp_merge.size()-1){
stamp_str += ",";
}
}
@@ -475,7 +504,9 @@
if (char_list.back() == "</s>") {
char_list.pop_back();
}
-
+ if (char_list.empty()) {
+ return ;
+ }
vector<vector<float>> timestamp_list;
vector<string> new_char_list;
vector<float> fire_place;
@@ -490,6 +521,9 @@
if(num_peak != (int)char_list.size() + 1){
float sum = std::accumulate(us_alphas.begin(), us_alphas.end(), 0.0f);
float scale = sum/((int)char_list.size() + 1);
+ if(scale == 0){
+ return;
+ }
cif_peak.clear();
sum = 0.0;
for(auto &alpha:us_alphas){
@@ -507,6 +541,11 @@
fire_place.push_back(i + total_offset);
}
}
+ }
+
+ num_peak = fire_place.size();
+ if(fire_place.size() == 0){
+ return;
}
// begin silence
@@ -530,6 +569,10 @@
}
// tail token and end silence
+ if(timestamp_list.size()==0){
+ LOG(ERROR)<<"timestamp_list's size is 0!";
+ return;
+ }
if (num_frames - fire_place.back() > START_END_THRESHOLD) {
float _end = (num_frames + fire_place.back()) / 2.0;
timestamp_list.back()[1] = _end * TIME_RATE;
@@ -641,7 +684,7 @@
return "";
}
//PrintMat(hw_emb, "input_clas_emb");
- const int64_t hotword_shape[3] = {1, hw_emb.size(), hw_emb[0].size()};
+ const int64_t hotword_shape[3] = {1, static_cast<int64_t>(hw_emb.size()), static_cast<int64_t>(hw_emb[0].size())};
embedding.reserve(hw_emb.size() * hw_emb[0].size());
for (auto item : hw_emb) {
embedding.insert(embedding.end(), item.begin(), item.end());
@@ -658,7 +701,7 @@
return "";
}
- string result;
+ string result="";
try {
auto outputTensor = m_session_->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), input_onnx.size(), m_szOutputNames.data(), m_szOutputNames.size());
std::vector<int64_t> outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape();
@@ -686,17 +729,6 @@
}else{
result = GreedySearch(floatData, *encoder_out_lens, outputShape[2]);
}
-// int pos = 0;
-// std::vector<std::vector<float>> logits;
-// for (int j = 0; j < outputShape[1]; j++)
-// {
-// std::vector<float> vec_token;
-// vec_token.insert(vec_token.begin(), floatData + pos, floatData + pos + outputShape[2]);
-// logits.push_back(vec_token);
-// pos += outputShape[2];
-// }
-// //PrintMat(logits, "logits_out");
-// result = GreedySearch(floatData, *encoder_out_lens, outputShape[2]);
}
catch (std::exception const &e)
{
@@ -719,6 +751,7 @@
std::vector<int32_t> hotword_matrix;
std::vector<int32_t> lengths;
int hotword_size = 1;
+ int real_hw_size = 0;
if (!hotwords.empty()) {
std::vector<std::string> hotword_array = split(hotwords, ' ');
hotword_size = hotword_array.size() + 1;
@@ -735,6 +768,9 @@
chars.insert(chars.end(), tokens.begin(), tokens.end());
}
}
+ if(chars.size()==0){
+ continue;
+ }
std::vector<int32_t> hw_vector(max_hotword_len, 0);
int vector_len = std::min(max_hotword_len, (int)chars.size());
for (int i=0; i<chars.size(); i++) {
@@ -743,8 +779,10 @@
}
std::cout << std::endl;
lengths.push_back(vector_len);
+ real_hw_size += 1;
hotword_matrix.insert(hotword_matrix.end(), hw_vector.begin(), hw_vector.end());
}
+ hotword_size = real_hw_size + 1;
}
std::vector<int32_t> blank_vec(max_hotword_len, 0);
blank_vec[0] = 1;
--
Gitblit v1.9.1