From 28ccfbfc51068a663a80764e14074df5edf2b5ba Mon Sep 17 00:00:00 2001
From: kongdeqiang <kongdeqiang960204@163.com>
Date: 星期五, 13 三月 2026 17:41:41 +0800
Subject: [PATCH] 提交
---
runtime/onnxruntime/src/audio.cpp | 175 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 175 insertions(+), 0 deletions(-)
diff --git a/runtime/onnxruntime/src/audio.cpp b/runtime/onnxruntime/src/audio.cpp
index 40ea871..22a9ecd 100644
--- a/runtime/onnxruntime/src/audio.cpp
+++ b/runtime/onnxruntime/src/audio.cpp
@@ -133,6 +133,7 @@
};
~AudioWindow(){
free(window);
+ window = nullptr;
};
int put(int val)
{
@@ -162,6 +163,7 @@
AudioFrame::~AudioFrame(){
if(data != nullptr){
free(data);
+ data = nullptr;
}
}
int AudioFrame::SetStart(int val)
@@ -221,12 +223,15 @@
{
if (speech_buff != nullptr) {
free(speech_buff);
+ speech_buff = nullptr;
}
if (speech_data != nullptr) {
free(speech_data);
+ speech_data = nullptr;
}
if (speech_char != nullptr) {
free(speech_char);
+ speech_char = nullptr;
}
ClearQueue(frame_queue);
ClearQueue(asr_online_queue);
@@ -271,6 +276,7 @@
speech_len = samples.size();
if (speech_data != nullptr) {
free(speech_data);
+ speech_data = nullptr;
}
speech_data = (float*)malloc(sizeof(float) * speech_len);
memset(speech_data, 0, sizeof(float) * speech_len);
@@ -402,9 +408,11 @@
if (speech_data != nullptr) {
free(speech_data);
+ speech_data = nullptr;
}
if (speech_char != nullptr) {
free(speech_char);
+ speech_char = nullptr;
}
offset = 0;
@@ -584,6 +592,7 @@
if (speech_data != nullptr) {
free(speech_data);
+ speech_data = nullptr;
}
speech_len = (resampled_buffers.size()) / 2;
@@ -615,9 +624,11 @@
WaveHeader header;
if (speech_data != nullptr) {
free(speech_data);
+ speech_data = nullptr;
}
if (speech_buff != nullptr) {
free(speech_buff);
+ speech_buff = nullptr;
}
offset = 0;
@@ -690,6 +701,7 @@
WaveHeader header;
if (speech_char != nullptr) {
free(speech_char);
+ speech_char = nullptr;
}
offset = 0;
std::ifstream is(filename, std::ifstream::binary);
@@ -729,9 +741,11 @@
WaveHeader header;
if (speech_data != nullptr) {
free(speech_data);
+ speech_data = nullptr;
}
if (speech_buff != nullptr) {
free(speech_buff);
+ speech_buff = nullptr;
}
std::memcpy(&header, buf, sizeof(header));
@@ -774,6 +788,7 @@
{
if (speech_data != nullptr) {
free(speech_data);
+ speech_data = nullptr;
}
speech_len = n_buf_len / 2;
@@ -807,6 +822,7 @@
{
if (speech_data != nullptr) {
free(speech_data);
+ speech_data = nullptr;
}
speech_len = n_buf_len / 2;
@@ -844,9 +860,11 @@
{
if (speech_data != nullptr) {
free(speech_data);
+ speech_data = nullptr;
}
if (speech_buff != nullptr) {
free(speech_buff);
+ speech_buff = nullptr;
}
offset = 0;
@@ -899,6 +917,7 @@
{
if (speech_char != nullptr) {
free(speech_char);
+ speech_char = nullptr;
}
offset = 0;
@@ -926,6 +945,7 @@
{
if (speech_char != nullptr) {
free(speech_char);
+ speech_char = nullptr;
}
FILE* fp;
@@ -1003,6 +1023,90 @@
}
}
+int Audio::Fetch(float**& dout, int*& len, int*& flag, float*& start_time, int batch_size, int &batch_in)
+{
+ batch_in = std::min((int)frame_queue.size(), batch_size);
+ if (batch_in == 0){
+ return 0;
+ } else{
+ // init
+ dout = new float*[batch_in];
+ len = new int[batch_in];
+ flag = new int[batch_in];
+ start_time = new float[batch_in];
+
+ for(int idx=0; idx < batch_in; idx++){
+ AudioFrame *frame = frame_queue.front();
+ frame_queue.pop();
+
+ start_time[idx] = (float)(frame->GetStart())/ dest_sample_rate;
+ dout[idx] = speech_data + frame->GetStart();
+ len[idx] = frame->GetLen();
+ delete frame;
+ flag[idx] = S_END;
+ }
+ return 1;
+ }
+}
+
+int Audio::FetchDynamic(float**& dout, int*& len, int*& flag, float*& start_time, int batch_size, int &batch_in)
+{
+ //compute batch size
+ queue<AudioFrame *> frame_batch;
+ int max_acc = 300*1000*seg_sample;
+ int max_sent = 60*1000*seg_sample;
+ int bs_acc = 0;
+ int max_len = 0;
+ int max_batch = 1;
+ #ifdef USE_GPU
+ max_batch = batch_size;
+ #endif
+ max_batch = std::min(max_batch, (int)frame_queue.size());
+
+ for(int idx=0; idx < max_batch; idx++){
+ AudioFrame *frame = frame_queue.front();
+ int length = frame->GetLen();
+ if(length >= max_sent){
+ if(bs_acc == 0){
+ bs_acc++;
+ frame_batch.push(frame);
+ frame_queue.pop();
+ }
+ break;
+ }
+ max_len = std::max(max_len, frame->GetLen());
+ if(max_len*(bs_acc+1) > max_acc){
+ break;
+ }
+ bs_acc++;
+ frame_batch.push(frame);
+ frame_queue.pop();
+ }
+
+ batch_in = (int)frame_batch.size();
+ if (batch_in == 0){
+ return 0;
+ } else{
+ // init
+ dout = new float*[batch_in];
+ len = new int[batch_in];
+ flag = new int[batch_in];
+ start_time = new float[batch_in];
+
+ for(int idx=0; idx < batch_in; idx++){
+ AudioFrame *frame = frame_batch.front();
+ frame_batch.pop();
+
+ start_time[idx] = (float)(frame->GetStart())/ dest_sample_rate;
+ dout[idx] = speech_data + frame->GetStart();
+ len[idx] = frame->GetLen();
+ delete frame;
+ flag[idx] = S_END;
+ }
+ return 1;
+ }
+}
+
void Audio::Padding()
{
float num_samples = speech_len;
@@ -1030,6 +1134,7 @@
new_data[tmp_off + i] = speech_data[ii];
}
free(speech_data);
+ speech_data = nullptr;
speech_data = new_data;
speech_len = num_new_samples;
@@ -1064,6 +1169,76 @@
}
}
+void Audio::CutSplit(OfflineStream* offline_stream, std::vector<int> &index_vector)
+{
+ std::unique_ptr<VadModel> vad_online_handle = make_unique<FsmnVadOnline>((FsmnVad*)(offline_stream->vad_handle).get());
+ AudioFrame *frame;
+
+ frame = frame_queue.front();
+ frame_queue.pop();
+ int sp_len = frame->GetLen();
+ delete frame;
+ frame = nullptr;
+
+ int step = dest_sample_rate*1;
+ bool is_final=false;
+ vector<std::vector<int>> vad_segments;
+ for (int sample_offset = 0; sample_offset < speech_len; sample_offset += std::min(step, speech_len - sample_offset)) {
+ if (sample_offset + step >= speech_len - 1) {
+ step = speech_len - sample_offset;
+ is_final = true;
+ } else {
+ is_final = false;
+ }
+ std::vector<float> pcm_data(speech_data+sample_offset, speech_data+sample_offset+step);
+ vector<std::vector<int>> cut_segments = vad_online_handle->Infer(pcm_data, is_final);
+ vad_segments.insert(vad_segments.end(), cut_segments.begin(), cut_segments.end());
+ }
+
+ int speech_start_i = -1, speech_end_i =-1;
+ std::vector<AudioFrame*> vad_frames;
+ for(vector<int> vad_segment:vad_segments)
+ {
+ if(vad_segment.size() != 2){
+ LOG(ERROR) << "Size of vad_segment is not 2.";
+ break;
+ }
+ if(vad_segment[0] != -1){
+ speech_start_i = vad_segment[0];
+ }
+ if(vad_segment[1] != -1){
+ speech_end_i = vad_segment[1];
+ }
+
+ if(speech_start_i!=-1 && speech_end_i!=-1){
+ int start = speech_start_i*seg_sample;
+ int end = speech_end_i*seg_sample;
+ frame = new AudioFrame(end-start);
+ frame->SetStart(start);
+ frame->SetEnd(end);
+ vad_frames.push_back(frame);
+ frame = nullptr;
+ speech_start_i=-1;
+ speech_end_i=-1;
+ }
+
+ }
+ // sort
+ {
+ index_vector.clear();
+ index_vector.resize(vad_frames.size());
+ for (int i = 0; i < index_vector.size(); ++i) {
+ index_vector[i] = i;
+ }
+ std::sort(index_vector.begin(), index_vector.end(), [&vad_frames](const int a, const int b) {
+ return vad_frames[a]->len < vad_frames[b]->len;
+ });
+ for (int idx : index_vector) {
+ frame_queue.push(vad_frames[idx]);
+ }
+ }
+}
+
void Audio::Split(VadModel* vad_obj, vector<std::vector<int>>& vad_segments, bool input_finished)
{
AudioFrame *frame;
--
Gitblit v1.9.1