python/FunASR-XL.git

			@@ -193,18 +193,28 @@
			return 0;
			}

			Audio::Audio(int data_type) : data_type(data_type)
			Audio::Audio(int data_type) : dest_sample_rate(MODEL_SAMPLE_RATE), data_type(data_type)
			{
			speech_buff = NULL;
			speech_data = NULL;
			align_size = 1360;
			seg_sample = dest_sample_rate / 1000;
			}

			Audio::Audio(int data_type, int size) : data_type(data_type)
			Audio::Audio(int model_sample_rate, int data_type) : dest_sample_rate(model_sample_rate), data_type(data_type)
			{
			speech_buff = NULL;
			speech_data = NULL;
			align_size = 1360;
			seg_sample = dest_sample_rate / 1000;
			}

			Audio::Audio(int model_sample_rate, int data_type, int size) : dest_sample_rate(model_sample_rate), data_type(data_type)
			{
			speech_buff = NULL;
			speech_data = NULL;
			align_size = (float)size;
			seg_sample = dest_sample_rate / 1000;
			}

			Audio::~Audio()
			@@ -218,32 +228,43 @@
			if (speech_char != NULL) {
			free(speech_char);
			}
			ClearQueue(frame_queue);
			ClearQueue(asr_online_queue);
			ClearQueue(asr_offline_queue);
			}

			void Audio::ClearQueue(std::queue<AudioFrame*>& q) {
			while (!q.empty()) {
			AudioFrame* frame = q.front();
			delete frame;
			q.pop();
			}
			}

			void Audio::Disp()
			{
			LOG(INFO) << "Audio time is " << (float)speech_len / MODEL_SAMPLE_RATE << " s. len is " << speech_len;
			LOG(INFO) << "Audio time is " << (float)speech_len / dest_sample_rate << " s. len is " << speech_len;
			}

			float Audio::GetTimeLen()
			{
			return (float)speech_len / MODEL_SAMPLE_RATE;
			return (float)speech_len / dest_sample_rate;
			}

			void Audio::WavResample(int32_t sampling_rate, const float *waveform,
			int32_t n)
			{
			LOG(INFO) << "Creating a resampler:\n"
			<< " in_sample_rate: "<< sampling_rate << "\n"
			<< " output_sample_rate: " << static_cast<int32_t>(MODEL_SAMPLE_RATE);
			LOG(INFO) << "Creating a resampler: "
			<< " in_sample_rate: "<< sampling_rate
			<< " output_sample_rate: " << static_cast<int32_t>(dest_sample_rate);
			float min_freq =
			std::min<int32_t>(sampling_rate, MODEL_SAMPLE_RATE);
			std::min<int32_t>(sampling_rate, dest_sample_rate);
			float lowpass_cutoff = 0.99 * 0.5 * min_freq;

			int32_t lowpass_filter_width = 6;

			auto resampler = std::make_unique<LinearResample>(
			sampling_rate, MODEL_SAMPLE_RATE, lowpass_cutoff, lowpass_filter_width);
			sampling_rate, dest_sample_rate, lowpass_cutoff, lowpass_filter_width);
			std::vector<float> samples;
			resampler->Resample(waveform, n, true, &samples);
			//reset speech_data
			@@ -311,7 +332,7 @@
			nullptr, // allocate a new context
			AV_CH_LAYOUT_MONO, // output channel layout (stereo)
			AV_SAMPLE_FMT_S16, // output sample format (signed 16-bit)
			16000, // output sample rate (same as input)
			dest_sample_rate, // output sample rate (same as input)
			av_get_default_channel_layout(codecContext->channels), // input channel layout
			codecContext->sample_fmt, // input sample format
			codecContext->sample_rate, // input sample rate
			@@ -344,30 +365,28 @@
			while (avcodec_receive_frame(codecContext, frame) >= 0) {
			// Resample audio if necessary
			std::vector<uint8_t> resampled_buffer;
			int in_samples = frame->nb_samples;
			uint8_t **in_data = frame->extended_data;
			int out_samples = av_rescale_rnd(in_samples,
			16000,
			int out_samples = av_rescale_rnd(swr_get_delay(swr_ctx, codecContext->sample_rate) + frame->nb_samples,
			dest_sample_rate,
			codecContext->sample_rate,
			AV_ROUND_DOWN);

			int resampled_size = out_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16);
			if (resampled_buffer.size() < resampled_size) {
			resampled_buffer.resize(resampled_size);
			}
			}
			uint8_t *resampled_data = resampled_buffer.data();
			int ret = swr_convert(
			swr_ctx,
			&resampled_data, // output buffer
			resampled_size, // output buffer size
			(const uint8_t )(frame->data), //(const uint8_t )(frame->extended_data)
			in_samples // input buffer size
			out_samples, // output buffer size
			(const uint8_t **)(frame->data), // choose channel
			frame->nb_samples // input buffer size
			);
			if (ret < 0) {
			LOG(ERROR) << "Error resampling audio";
			break;
			}
			std::copy(resampled_buffer.begin(), resampled_buffer.end(), std::back_inserter(resampled_buffers));
			resampled_buffers.insert(resampled_buffers.end(), resampled_buffer.begin(), resampled_buffer.begin() + resampled_size);
			}
			}
			}
			@@ -384,9 +403,6 @@
			if (speech_data != NULL) {
			free(speech_data);
			}
			if (speech_buff != NULL) {
			free(speech_buff);
			}
			if (speech_char != NULL) {
			free(speech_char);
			}
			@@ -399,30 +415,25 @@
			}

			speech_len = (resampled_buffers.size()) / 2;
			speech_buff = (int16_t)malloc(sizeof(int16_t) speech_len);
			if (speech_buff)
			{
			memset(speech_buff, 0, sizeof(int16_t) * speech_len);
			memcpy((void)speech_buff, (const void)resampled_buffers.data(), speech_len * sizeof(int16_t));

			speech_data = (float)malloc(sizeof(float) speech_len);
			speech_data = (float)malloc(sizeof(float) speech_len);
			if(speech_data){
			memset(speech_data, 0, sizeof(float) * speech_len);

			float scale = 1;
			if (data_type == 1) {
			scale = 32768;
			scale = 32768.0f;
			}
			for (int32_t i = 0; i != speech_len; ++i) {
			speech_data[i] = (float)speech_buff[i] / scale;
			for (int32_t i = 0; i < speech_len; ++i) {
			int16_t val = (int16_t)((resampled_buffers[2 * i + 1] << 8) \| resampled_buffers[2 * i]);
			speech_data[i] = (float)val / scale;
			}

			AudioFrame* frame = new AudioFrame(speech_len);
			frame_queue.push(frame);

			return true;
			}
			else
			}else{
			return false;
			}

			#endif
			}

			@@ -443,6 +454,10 @@
			nullptr, // write callback (not used here)
			nullptr // seek callback (not used here)
			);
			if (!avio_ctx) {
			av_free(buf_copy);
			return false;
			}
			AVFormatContext* formatContext = avformat_alloc_context();
			formatContext->pb = avio_ctx;
			if (avformat_open_input(&formatContext, "", NULL, NULL) != 0) {
			@@ -494,7 +509,7 @@
			nullptr, // allocate a new context
			AV_CH_LAYOUT_MONO, // output channel layout (stereo)
			AV_SAMPLE_FMT_S16, // output sample format (signed 16-bit)
			16000, // output sample rate (same as input)
			dest_sample_rate, // output sample rate (same as input)
			av_get_default_channel_layout(codecContext->channels), // input channel layout
			codecContext->sample_fmt, // input sample format
			codecContext->sample_rate, // input sample rate
			@@ -529,37 +544,37 @@
			while (avcodec_receive_frame(codecContext, frame) >= 0) {
			// Resample audio if necessary
			std::vector<uint8_t> resampled_buffer;
			int in_samples = frame->nb_samples;
			uint8_t **in_data = frame->extended_data;
			int out_samples = av_rescale_rnd(in_samples,
			16000,
			int out_samples = av_rescale_rnd(swr_get_delay(swr_ctx, codecContext->sample_rate) + frame->nb_samples,
			dest_sample_rate,
			codecContext->sample_rate,
			AV_ROUND_DOWN);

			int resampled_size = out_samples * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16);
			if (resampled_buffer.size() < resampled_size) {
			resampled_buffer.resize(resampled_size);
			}
			}
			uint8_t *resampled_data = resampled_buffer.data();
			int ret = swr_convert(
			swr_ctx,
			&resampled_data, // output buffer
			resampled_size, // output buffer size
			(const uint8_t )(frame->data), //(const uint8_t )(frame->extended_data)
			in_samples // input buffer size
			out_samples, // output buffer size
			(const uint8_t **)(frame->data), // choose channel: channel_data
			frame->nb_samples // input buffer size
			);
			if (ret < 0) {
			LOG(ERROR) << "Error resampling audio";
			break;
			}
			std::copy(resampled_buffer.begin(), resampled_buffer.end(), std::back_inserter(resampled_buffers));
			resampled_buffers.insert(resampled_buffers.end(), resampled_buffer.begin(), resampled_buffer.begin() + resampled_size);
			}
			}
			}
			av_packet_unref(packet);
			}

			avio_context_free(&avio_ctx);
			//avio_context_free(&avio_ctx);
			av_freep(&avio_ctx ->buffer);
			av_freep(&avio_ctx);
			avformat_close_input(&formatContext);
			avformat_free_context(formatContext);
			avcodec_free_context(&codecContext);
			@@ -570,41 +585,32 @@
			if (speech_data != NULL) {
			free(speech_data);
			}
			if (speech_buff != NULL) {
			free(speech_buff);
			}
			offset = 0;

			speech_len = (resampled_buffers.size()) / 2;
			speech_buff = (int16_t)malloc(sizeof(int16_t) speech_len);
			if (speech_buff)
			{
			memset(speech_buff, 0, sizeof(int16_t) * speech_len);
			memcpy((void)speech_buff, (const void)resampled_buffers.data(), speech_len * sizeof(int16_t));

			speech_data = (float)malloc(sizeof(float) speech_len);
			speech_data = (float)malloc(sizeof(float) speech_len);
			if(speech_data){
			memset(speech_data, 0, sizeof(float) * speech_len);

			float scale = 1;
			if (data_type == 1) {
			scale = 32768;
			scale = 32768.0f;
			}
			for (int32_t i = 0; i != speech_len; ++i) {
			speech_data[i] = (float)speech_buff[i] / scale;
			for (int32_t i = 0; i < speech_len; ++i) {
			int16_t val = (int16_t)((resampled_buffers[2 * i + 1] << 8) \| resampled_buffers[2 * i]);
			speech_data[i] = (float)val / scale;
			}

			AudioFrame* frame = new AudioFrame(speech_len);
			frame_queue.push(frame);

			return true;
			}
			else
			}else{
			return false;
			}

			#endif
			}


			bool Audio::LoadWav(const char filename, int32_t sampling_rate)
			bool Audio::LoadWav(const char filename, int32_t sampling_rate, bool resample)
			{
			WaveHeader header;
			if (speech_data != NULL) {
			@@ -666,7 +672,7 @@
			}

			//resample
			if(*sampling_rate != MODEL_SAMPLE_RATE){
			if(resample && *sampling_rate != dest_sample_rate){
			WavResample(*sampling_rate, speech_data, speech_len);
			}

			@@ -727,7 +733,6 @@
			if (speech_buff != NULL) {
			free(speech_buff);
			}
			offset = 0;

			std::memcpy(&header, buf, sizeof(header));

			@@ -752,7 +757,7 @@
			}

			//resample
			if(*sampling_rate != MODEL_SAMPLE_RATE){
			if(*sampling_rate != dest_sample_rate){
			WavResample(*sampling_rate, speech_data, speech_len);
			}

			@@ -770,42 +775,32 @@
			if (speech_data != NULL) {
			free(speech_data);
			}
			if (speech_buff != NULL) {
			free(speech_buff);
			}
			offset = 0;

			speech_len = n_buf_len / 2;
			speech_buff = (int16_t)malloc(sizeof(int16_t) speech_len);
			if (speech_buff)
			{
			memset(speech_buff, 0, sizeof(int16_t) * speech_len);
			memcpy((void)speech_buff, (const void)buf, speech_len * sizeof(int16_t));

			speech_data = (float)malloc(sizeof(float) speech_len);
			memset(speech_data, 0, sizeof(float) * speech_len);

			speech_data = (float)malloc(sizeof(float) speech_len);
			if(speech_data){
			float scale = 1;
			if (data_type == 1) {
			scale = 32768;
			scale = 32768.0f;
			}
			const uint8_t* byte_buf = reinterpret_cast<const uint8_t*>(buf);
			for (int32_t i = 0; i < speech_len; ++i) {
			int16_t val = (int16_t)((byte_buf[2 * i + 1] << 8) \| byte_buf[2 * i]);
			speech_data[i] = (float)val / scale;
			}

			for (int32_t i = 0; i != speech_len; ++i) {
			speech_data[i] = (float)speech_buff[i] / scale;
			}

			//resample
			if(*sampling_rate != MODEL_SAMPLE_RATE){
			if(*sampling_rate != dest_sample_rate){
			WavResample(*sampling_rate, speech_data, speech_len);
			}

			AudioFrame* frame = new AudioFrame(speech_len);
			frame_queue.push(frame);

			return true;

			}
			else
			}else{
			return false;
			}
			}

			bool Audio::LoadPcmwavOnline(const char* buf, int n_buf_len, int32_t* sampling_rate)
			@@ -813,34 +808,22 @@
			if (speech_data != NULL) {
			free(speech_data);
			}
			if (speech_buff != NULL) {
			free(speech_buff);
			}
			if (speech_char != NULL) {
			free(speech_char);
			}

			speech_len = n_buf_len / 2;
			speech_buff = (int16_t)malloc(sizeof(int16_t) speech_len);
			if (speech_buff)
			{
			memset(speech_buff, 0, sizeof(int16_t) * speech_len);
			memcpy((void)speech_buff, (const void)buf, speech_len * sizeof(int16_t));

			speech_data = (float)malloc(sizeof(float) speech_len);
			memset(speech_data, 0, sizeof(float) * speech_len);

			speech_data = (float)malloc(sizeof(float) speech_len);
			if(speech_data){
			float scale = 1;
			if (data_type == 1) {
			scale = 32768;
			scale = 32768.0f;
			}
			const uint8_t* byte_buf = reinterpret_cast<const uint8_t*>(buf);
			for (int32_t i = 0; i < speech_len; ++i) {
			int16_t val = (int16_t)((byte_buf[2 * i + 1] << 8) \| byte_buf[2 * i]);
			speech_data[i] = (float)val / scale;
			}

			for (int32_t i = 0; i != speech_len; ++i) {
			speech_data[i] = (float)speech_buff[i] / scale;
			}

			//resample
			if(*sampling_rate != MODEL_SAMPLE_RATE){
			if(*sampling_rate != dest_sample_rate){
			WavResample(*sampling_rate, speech_data, speech_len);
			}

			@@ -850,14 +833,14 @@

			AudioFrame* frame = new AudioFrame(speech_len);
			frame_queue.push(frame);

			return true;

			}
			else
			}else{
			return false;
			}
			}

			bool Audio::LoadPcmwav(const char* filename, int32_t* sampling_rate)
			bool Audio::LoadPcmwav(const char* filename, int32_t* sampling_rate, bool resample)
			{
			if (speech_data != NULL) {
			free(speech_data);
			@@ -898,7 +881,7 @@
			}

			//resample
			if(*sampling_rate != MODEL_SAMPLE_RATE){
			if(resample && *sampling_rate != dest_sample_rate){
			WavResample(*sampling_rate, speech_data, speech_len);
			}

			@@ -1009,7 +992,7 @@
			AudioFrame *frame = frame_queue.front();
			frame_queue.pop();

			start_time = (float)(frame->GetStart())/MODEL_SAMPLE_RATE;
			start_time = (float)(frame->GetStart())/ dest_sample_rate;
			dout = speech_data + frame->GetStart();
			len = frame->GetLen();
			delete frame;
			@@ -1193,7 +1176,7 @@
			}

			}else if(speech_end_i != -1){ // [-1,100]
			if(speech_start == -1 or speech_offline_start == -1){
			if(speech_start == -1 \|\| speech_offline_start == -1){
			LOG(ERROR) <<"Vad start is null while vad end is available. Set vad start 0" ;
			speech_start = 0;
			}
			@@ -1248,7 +1231,7 @@
			}

			// erase all_samples
			int vector_cache = MODEL_SAMPLE_RATE*2;
			int vector_cache = dest_sample_rate*2;
			if(speech_offline_start == -1){
			if(all_samples.size() > vector_cache){
			int erase_num = all_samples.size() - vector_cache;