From ae609ca0c64056622888d5eddfca09a92defc30b Mon Sep 17 00:00:00 2001
From: Yabin Li <wucong.lyb@alibaba-inc.com>
Date: 星期二, 11 七月 2023 10:57:30 +0800
Subject: [PATCH] Dev ffmpeg (#727)
---
funasr/runtime/onnxruntime/src/audio.cpp | 363 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 363 insertions(+), 0 deletions(-)
diff --git a/funasr/runtime/onnxruntime/src/audio.cpp b/funasr/runtime/onnxruntime/src/audio.cpp
index 23d0010..85633b7 100644
--- a/funasr/runtime/onnxruntime/src/audio.cpp
+++ b/funasr/runtime/onnxruntime/src/audio.cpp
@@ -9,6 +9,15 @@
#include "audio.h"
#include "precomp.h"
+extern "C" {
+#include <libavutil/opt.h>
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/channel_layout.h>
+#include <libavutil/samplefmt.h>
+#include <libswresample/swresample.h>
+}
+
using namespace std;
namespace funasr {
@@ -220,6 +229,334 @@
memset(speech_data, 0, sizeof(float) * speech_len);
copy(samples.begin(), samples.end(), speech_data);
}
+
+bool Audio::FfmpegLoad(const char *filename){
+ // from file
+ AVFormatContext* formatContext = avformat_alloc_context();
+ if (avformat_open_input(&formatContext, filename, NULL, NULL) != 0) {
+ printf("Error: Could not open input file.");
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ return false;
+ }
+
+ if (avformat_find_stream_info(formatContext, NULL) < 0) {
+ printf("Error: Could not find stream information.");
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ return false;
+ }
+ const AVCodec* codec = NULL;
+ AVCodecParameters* codecParameters = NULL;
+ int audioStreamIndex = av_find_best_stream(formatContext, AVMEDIA_TYPE_AUDIO, -1, -1, &codec, 0);
+ if (audioStreamIndex >= 0) {
+ codecParameters = formatContext->streams[audioStreamIndex]->codecpar;
+ }
+ AVCodecContext* codecContext = avcodec_alloc_context3(codec);
+ if (!codecContext) {
+ fprintf(stderr, "Failed to allocate codec context\n");
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ return false;
+ }
+ if (avcodec_parameters_to_context(codecContext, codecParameters) != 0) {
+ printf("Error: Could not copy codec parameters to codec context.");
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ avcodec_free_context(&codecContext);
+ return false;
+ }
+ if (avcodec_open2(codecContext, codec, NULL) < 0) {
+ printf("Error: Could not open audio decoder.");
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ avcodec_free_context(&codecContext);
+ return false;
+ }
+ SwrContext *swr_ctx = swr_alloc_set_opts(
+ nullptr, // allocate a new context
+ AV_CH_LAYOUT_MONO, // output channel layout (stereo)
+ AV_SAMPLE_FMT_S16, // output sample format (signed 16-bit)
+ 16000, // output sample rate (same as input)
+ av_get_default_channel_layout(codecContext->channels), // input channel layout
+ codecContext->sample_fmt, // input sample format
+ codecContext->sample_rate, // input sample rate
+ 0, // logging level
+ nullptr // parent context
+ );
+ if (swr_ctx == nullptr) {
+ std::cerr << "Could not initialize resampler" << std::endl;
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ avcodec_free_context(&codecContext);
+ return false;
+ }
+ if (swr_init(swr_ctx) != 0) {
+ std::cerr << "Could not initialize resampler" << std::endl;
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ avcodec_free_context(&codecContext);
+ swr_free(&swr_ctx);
+ return false;
+ }
+
+ // to pcm
+ AVPacket* packet = av_packet_alloc();
+ AVFrame* frame = av_frame_alloc();
+ std::vector<uint8_t> resampled_buffers;
+ while (av_read_frame(formatContext, packet) >= 0) {
+ if (packet->stream_index == audioStreamIndex) {
+ if (avcodec_send_packet(codecContext, packet) >= 0) {
+ while (avcodec_receive_frame(codecContext, frame) >= 0) {
+ // Resample audio if necessary
+ std::vector<uint8_t> resampled_buffer;
+ int in_samples = frame->nb_samples;
+ uint8_t **in_data = frame->extended_data;
+ int out_samples = av_rescale_rnd(in_samples,
+ 16000,
+ codecContext->sample_rate,
+ AV_ROUND_DOWN);
+
+ int resampled_size = out_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16);
+ if (resampled_buffer.size() < resampled_size) {
+ resampled_buffer.resize(resampled_size);
+ }
+ uint8_t *resampled_data = resampled_buffer.data();
+ int ret = swr_convert(
+ swr_ctx,
+ &resampled_data, // output buffer
+ resampled_size, // output buffer size
+ (const uint8_t **)(frame->data), //(const uint8_t **)(frame->extended_data)
+ in_samples // input buffer size
+ );
+ if (ret < 0) {
+ std::cerr << "Error resampling audio" << std::endl;
+ break;
+ }
+ std::copy(resampled_buffer.begin(), resampled_buffer.end(), std::back_inserter(resampled_buffers));
+ }
+ }
+ }
+ av_packet_unref(packet);
+ }
+
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ avcodec_free_context(&codecContext);
+ swr_free(&swr_ctx);
+ av_packet_free(&packet);
+ av_frame_free(&frame);
+
+ if (speech_data != NULL) {
+ free(speech_data);
+ }
+ if (speech_buff != NULL) {
+ free(speech_buff);
+ }
+ offset = 0;
+
+ speech_len = (resampled_buffers.size()) / 2;
+ speech_buff = (int16_t*)malloc(sizeof(int16_t) * speech_len);
+ if (speech_buff)
+ {
+ memset(speech_buff, 0, sizeof(int16_t) * speech_len);
+ memcpy((void*)speech_buff, (const void*)resampled_buffers.data(), speech_len * sizeof(int16_t));
+
+ speech_data = (float*)malloc(sizeof(float) * speech_len);
+ memset(speech_data, 0, sizeof(float) * speech_len);
+
+ float scale = 1;
+ if (data_type == 1) {
+ scale = 32768;
+ }
+ for (int32_t i = 0; i != speech_len; ++i) {
+ speech_data[i] = (float)speech_buff[i] / scale;
+ }
+
+ AudioFrame* frame = new AudioFrame(speech_len);
+ frame_queue.push(frame);
+
+ return true;
+ }
+ else
+ return false;
+
+}
+
+bool Audio::FfmpegLoad(const char* buf, int n_file_len){
+ // from buf
+ char* buf_copy = (char *)malloc(n_file_len);
+ memcpy(buf_copy, buf, n_file_len);
+
+ AVIOContext* avio_ctx = avio_alloc_context(
+ (unsigned char*)buf_copy, // buffer
+ n_file_len, // buffer size
+ 0, // write flag (0 for read-only)
+ nullptr, // opaque pointer (not used here)
+ nullptr, // read callback (not used here)
+ nullptr, // write callback (not used here)
+ nullptr // seek callback (not used here)
+ );
+ AVFormatContext* formatContext = avformat_alloc_context();
+ formatContext->pb = avio_ctx;
+ if (avformat_open_input(&formatContext, "", NULL, NULL) != 0) {
+ printf("Error: Could not open input file.");
+ avio_context_free(&avio_ctx);
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ return false;
+ }
+
+ if (avformat_find_stream_info(formatContext, NULL) < 0) {
+ printf("Error: Could not find stream information.");
+ avio_context_free(&avio_ctx);
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ return false;
+ }
+ const AVCodec* codec = NULL;
+ AVCodecParameters* codecParameters = NULL;
+ int audioStreamIndex = av_find_best_stream(formatContext, AVMEDIA_TYPE_AUDIO, -1, -1, &codec, 0);
+ if (audioStreamIndex >= 0) {
+ codecParameters = formatContext->streams[audioStreamIndex]->codecpar;
+ }
+ AVCodecContext* codecContext = avcodec_alloc_context3(codec);
+ if (!codecContext) {
+ fprintf(stderr, "Failed to allocate codec context\n");
+ avio_context_free(&avio_ctx);
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ return false;
+ }
+ if (avcodec_parameters_to_context(codecContext, codecParameters) != 0) {
+ printf("Error: Could not copy codec parameters to codec context.");
+ avio_context_free(&avio_ctx);
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ avcodec_free_context(&codecContext);
+ return false;
+ }
+ if (avcodec_open2(codecContext, codec, NULL) < 0) {
+ printf("Error: Could not open audio decoder.");
+ avio_context_free(&avio_ctx);
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ avcodec_free_context(&codecContext);
+ return false;
+ }
+ SwrContext *swr_ctx = swr_alloc_set_opts(
+ nullptr, // allocate a new context
+ AV_CH_LAYOUT_MONO, // output channel layout (stereo)
+ AV_SAMPLE_FMT_S16, // output sample format (signed 16-bit)
+ 16000, // output sample rate (same as input)
+ av_get_default_channel_layout(codecContext->channels), // input channel layout
+ codecContext->sample_fmt, // input sample format
+ codecContext->sample_rate, // input sample rate
+ 0, // logging level
+ nullptr // parent context
+ );
+ if (swr_ctx == nullptr) {
+ std::cerr << "Could not initialize resampler" << std::endl;
+ avio_context_free(&avio_ctx);
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ avcodec_free_context(&codecContext);
+ return false;
+ }
+ if (swr_init(swr_ctx) != 0) {
+ std::cerr << "Could not initialize resampler" << std::endl;
+ avio_context_free(&avio_ctx);
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ avcodec_free_context(&codecContext);
+ swr_free(&swr_ctx);
+ return false;
+ }
+
+ // to pcm
+ AVPacket* packet = av_packet_alloc();
+ AVFrame* frame = av_frame_alloc();
+ std::vector<uint8_t> resampled_buffers;
+ while (av_read_frame(formatContext, packet) >= 0) {
+ if (packet->stream_index == audioStreamIndex) {
+ if (avcodec_send_packet(codecContext, packet) >= 0) {
+ while (avcodec_receive_frame(codecContext, frame) >= 0) {
+ // Resample audio if necessary
+ std::vector<uint8_t> resampled_buffer;
+ int in_samples = frame->nb_samples;
+ uint8_t **in_data = frame->extended_data;
+ int out_samples = av_rescale_rnd(in_samples,
+ 16000,
+ codecContext->sample_rate,
+ AV_ROUND_DOWN);
+
+ int resampled_size = out_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16);
+ if (resampled_buffer.size() < resampled_size) {
+ resampled_buffer.resize(resampled_size);
+ }
+ uint8_t *resampled_data = resampled_buffer.data();
+ int ret = swr_convert(
+ swr_ctx,
+ &resampled_data, // output buffer
+ resampled_size, // output buffer size
+ (const uint8_t **)(frame->data), //(const uint8_t **)(frame->extended_data)
+ in_samples // input buffer size
+ );
+ if (ret < 0) {
+ std::cerr << "Error resampling audio" << std::endl;
+ break;
+ }
+ std::copy(resampled_buffer.begin(), resampled_buffer.end(), std::back_inserter(resampled_buffers));
+ }
+ }
+ }
+ av_packet_unref(packet);
+ }
+
+ avio_context_free(&avio_ctx);
+ avformat_close_input(&formatContext);
+ avformat_free_context(formatContext);
+ avcodec_free_context(&codecContext);
+ swr_free(&swr_ctx);
+ av_packet_free(&packet);
+ av_frame_free(&frame);
+
+ if (speech_data != NULL) {
+ free(speech_data);
+ }
+ if (speech_buff != NULL) {
+ free(speech_buff);
+ }
+ offset = 0;
+
+ speech_len = (resampled_buffers.size()) / 2;
+ speech_buff = (int16_t*)malloc(sizeof(int16_t) * speech_len);
+ if (speech_buff)
+ {
+ memset(speech_buff, 0, sizeof(int16_t) * speech_len);
+ memcpy((void*)speech_buff, (const void*)resampled_buffers.data(), speech_len * sizeof(int16_t));
+
+ speech_data = (float*)malloc(sizeof(float) * speech_len);
+ memset(speech_data, 0, sizeof(float) * speech_len);
+
+ float scale = 1;
+ if (data_type == 1) {
+ scale = 32768;
+ }
+ for (int32_t i = 0; i != speech_len; ++i) {
+ speech_data[i] = (float)speech_buff[i] / scale;
+ }
+
+ AudioFrame* frame = new AudioFrame(speech_len);
+ frame_queue.push(frame);
+
+ return true;
+ }
+ else
+ return false;
+
+}
+
bool Audio::LoadWav(const char *filename, int32_t* sampling_rate)
{
@@ -507,6 +844,32 @@
return true;
}
+bool Audio::LoadOthers2Char(const char* filename)
+{
+ if (speech_char != NULL) {
+ free(speech_char);
+ }
+
+ FILE* fp;
+ fp = fopen(filename, "rb");
+ if (fp == nullptr)
+ {
+ LOG(ERROR) << "Failed to read " << filename;
+ return false;
+ }
+ fseek(fp, 0, SEEK_END);
+ uint32_t n_file_len = ftell(fp);
+ fseek(fp, 0, SEEK_SET);
+
+ speech_len = n_file_len;
+ speech_char = (char *)malloc(n_file_len);
+ memset(speech_char, 0, n_file_len);
+ fread(speech_char, 1, n_file_len, fp);
+ fclose(fp);
+
+ return true;
+}
+
int Audio::FetchChunck(float *&dout, int len)
{
if (offset >= speech_align_len) {
--
Gitblit v1.9.1