From 1358e8a5fcba9d9da4ed1478a91f217e8c211fc0 Mon Sep 17 00:00:00 2001
From: Yabin Li <wucong.lyb@alibaba-inc.com>
Date: 星期一, 08 五月 2023 19:49:51 +0800
Subject: [PATCH] Merge pull request #476 from alibaba-damo-academy/dev_grpc
---
funasr/runtime/grpc/Readme.md | 49 +++++-------
funasr/runtime/onnxruntime/src/funasr-onnx-offline-rtf.cpp | 2
funasr/runtime/onnxruntime/src/funasr-onnx-offline.cpp | 4
funasr/runtime/onnxruntime/src/funasrruntime.cpp | 49 ++++++++++-
funasr/runtime/onnxruntime/include/funasrruntime.h | 15 ++-
funasr/runtime/onnxruntime/src/funasr-onnx-offline-vad.cpp | 14 +-
funasr/runtime/grpc/paraformer-server.cc | 50 +++++-------
funasr/runtime/onnxruntime/src/precomp.h | 2
funasr/runtime/onnxruntime/src/funasr-onnx-offline-punc.cpp | 2
funasr/runtime/grpc/paraformer-server.h | 2
funasr/runtime/onnxruntime/src/resample.cpp | 0
11 files changed, 105 insertions(+), 84 deletions(-)
diff --git a/funasr/runtime/grpc/Readme.md b/funasr/runtime/grpc/Readme.md
index 4499441..71bb035 100644
--- a/funasr/runtime/grpc/Readme.md
+++ b/funasr/runtime/grpc/Readme.md
@@ -37,39 +37,32 @@
### Start grpc paraformer server
```
-./cmake/build/paraformer-server --port-id <string> [--punc-config
- <string>] [--punc-model <string>]
- --am-config <string> --am-cmvn <string>
- --am-model <string> [--vad-config
- <string>] [--vad-cmvn <string>]
- [--vad-model <string>] [--] [--version]
- [-h]
+
+./cmake/build/paraformer-server --port-id <string> [--punc-quant <string>]
+ [--punc-dir <string>] [--vad-quant <string>]
+ [--vad-dir <string>] [--quantize <string>]
+ --model-dir <string> [--] [--version] [-h]
Where:
--port-id <string>
(required) port id
+ --model-dir <string>
+ (required) the asr model path, which contains model.onnx, config.yaml, am.mvn
+ --quantize <string>
+ false (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir
- --am-config <string>
- (required) am config path
- --am-cmvn <string>
- (required) am cmvn path
- --am-model <string>
- (required) am model path
+ --vad-dir <string>
+ the vad model path, which contains model.onnx, vad.yaml, vad.mvn
+ --vad-quant <string>
+ false (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir
- --punc-config <string>
- punc config path
- --punc-model <string>
- punc model path
-
- --vad-config <string>
- vad config path
- --vad-cmvn <string>
- vad cmvn path
- --vad-model <string>
- vad model path
-
- Required: --port-id <string> --am-config <string> --am-cmvn <string> --am-model <string>
- If use vad, please add: [--vad-config <string>] [--vad-cmvn <string>] [--vad-model <string>]
- If use punc, please add: [--punc-config <string>] [--punc-model <string>]
+ --punc-dir <string>
+ the punc model path, which contains model.onnx, punc.yaml
+ --punc-quant <string>
+ false (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir
+
+ Required: --port-id <string> --model-dir <string>
+ If use vad, please add: --vad-dir <string>
+ If use punc, please add: --punc-dir <string>
```
## For the client
diff --git a/funasr/runtime/grpc/paraformer-server.cc b/funasr/runtime/grpc/paraformer-server.cc
index 31333c9..3bc011a 100644
--- a/funasr/runtime/grpc/paraformer-server.cc
+++ b/funasr/runtime/grpc/paraformer-server.cc
@@ -31,7 +31,7 @@
using paraformer::ASR;
ASRServicer::ASRServicer(std::map<std::string, std::string>& model_path) {
- AsrHanlde=FunASRInit(model_path, 1);
+ AsrHanlde=FunOfflineInit(model_path, 1);
std::cout << "ASRServicer init" << std::endl;
init_flag = 0;
}
@@ -137,7 +137,7 @@
stream->Write(res);
}
else {
- FUNASR_RESULT Result= FunASRRecogPCMBuffer(AsrHanlde, tmp_data.c_str(), data_len_int, 16000, RASR_NONE, NULL);
+ FUNASR_RESULT Result= FunOfflineRecogPCMBuffer(AsrHanlde, tmp_data.c_str(), data_len_int, 16000, RASR_NONE, NULL);
std::string asr_result = ((FUNASR_RECOG_RESULT*)Result)->msg;
auto end_time = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
@@ -204,38 +204,30 @@
FLAGS_logtostderr = true;
TCLAP::CmdLine cmd("paraformer-server", ' ', "1.0");
- TCLAP::ValueArg<std::string> vad_model("", VAD_MODEL_PATH, "vad model path", false, "", "string");
- TCLAP::ValueArg<std::string> vad_cmvn("", VAD_CMVN_PATH, "vad cmvn path", false, "", "string");
- TCLAP::ValueArg<std::string> vad_config("", VAD_CONFIG_PATH, "vad config path", false, "", "string");
+ TCLAP::ValueArg<std::string> model_dir("", MODEL_DIR, "the asr model path, which contains model.onnx, config.yaml, am.mvn", true, "", "string");
+ TCLAP::ValueArg<std::string> quantize("", QUANTIZE, "false (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "false", "string");
+ TCLAP::ValueArg<std::string> vad_dir("", VAD_DIR, "the vad model path, which contains model.onnx, vad.yaml, vad.mvn", false, "", "string");
+ TCLAP::ValueArg<std::string> vad_quant("", VAD_QUANT, "false (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir", false, "false", "string");
+ TCLAP::ValueArg<std::string> punc_dir("", PUNC_DIR, "the punc model path, which contains model.onnx, punc.yaml", false, "", "string");
+ TCLAP::ValueArg<std::string> punc_quant("", PUNC_QUANT, "false (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir", false, "false", "string");
+ TCLAP::ValueArg<std::string> port_id("", PORT_ID, "port id", true, "", "string");
- TCLAP::ValueArg<std::string> am_model("", AM_MODEL_PATH, "am model path", true, "", "string");
- TCLAP::ValueArg<std::string> am_cmvn("", AM_CMVN_PATH, "am cmvn path", true, "", "string");
- TCLAP::ValueArg<std::string> am_config("", AM_CONFIG_PATH, "am config path", true, "", "string");
-
- TCLAP::ValueArg<std::string> punc_model("", PUNC_MODEL_PATH, "punc model path", false, "", "string");
- TCLAP::ValueArg<std::string> punc_config("", PUNC_CONFIG_PATH, "punc config path", false, "", "string");
- TCLAP::ValueArg<std::string> port_id("", PORT_ID, "port id", true, "", "string");
-
- cmd.add(vad_model);
- cmd.add(vad_cmvn);
- cmd.add(vad_config);
- cmd.add(am_model);
- cmd.add(am_cmvn);
- cmd.add(am_config);
- cmd.add(punc_model);
- cmd.add(punc_config);
+ cmd.add(model_dir);
+ cmd.add(quantize);
+ cmd.add(vad_dir);
+ cmd.add(vad_quant);
+ cmd.add(punc_dir);
+ cmd.add(punc_quant);
cmd.add(port_id);
cmd.parse(argc, argv);
std::map<std::string, std::string> model_path;
- GetValue(vad_model, VAD_MODEL_PATH, model_path);
- GetValue(vad_cmvn, VAD_CMVN_PATH, model_path);
- GetValue(vad_config, VAD_CONFIG_PATH, model_path);
- GetValue(am_model, AM_MODEL_PATH, model_path);
- GetValue(am_cmvn, AM_CMVN_PATH, model_path);
- GetValue(am_config, AM_CONFIG_PATH, model_path);
- GetValue(punc_model, PUNC_MODEL_PATH, model_path);
- GetValue(punc_config, PUNC_CONFIG_PATH, model_path);
+ GetValue(model_dir, MODEL_DIR, model_path);
+ GetValue(quantize, QUANTIZE, model_path);
+ GetValue(vad_dir, VAD_DIR, model_path);
+ GetValue(vad_quant, VAD_QUANT, model_path);
+ GetValue(punc_dir, PUNC_DIR, model_path);
+ GetValue(punc_quant, PUNC_QUANT, model_path);
GetValue(port_id, PORT_ID, model_path);
RunServer(model_path);
diff --git a/funasr/runtime/grpc/paraformer-server.h b/funasr/runtime/grpc/paraformer-server.h
index 108e3b6..760ea2a 100644
--- a/funasr/runtime/grpc/paraformer-server.h
+++ b/funasr/runtime/grpc/paraformer-server.h
@@ -15,7 +15,7 @@
#include <chrono>
#include "paraformer.grpc.pb.h"
-#include "libfunasrapi.h"
+#include "funasrruntime.h"
using grpc::Server;
diff --git a/funasr/runtime/onnxruntime/include/libfunasrapi.h b/funasr/runtime/onnxruntime/include/funasrruntime.h
similarity index 75%
rename from funasr/runtime/onnxruntime/include/libfunasrapi.h
rename to funasr/runtime/onnxruntime/include/funasrruntime.h
index 152db61..7d290aa 100644
--- a/funasr/runtime/onnxruntime/include/libfunasrapi.h
+++ b/funasr/runtime/onnxruntime/include/funasrruntime.h
@@ -63,13 +63,13 @@
_FUNASRAPI const float FunASRGetRetSnippetTime(FUNASR_RESULT result);
// VAD
-_FUNASRAPI FUNASR_HANDLE FunVadInit(std::map<std::string, std::string>& model_path, int thread_num);
+_FUNASRAPI FUNASR_HANDLE FsmnVadInit(std::map<std::string, std::string>& model_path, int thread_num);
-_FUNASRAPI FUNASR_RESULT FunVadWavFile(FUNASR_HANDLE handle, const char* sz_wavfile, FUNASR_MODE mode, QM_CALLBACK fn_callback);
-_FUNASRAPI std::vector<std::vector<int>>* FunVadGetResult(FUNASR_RESULT result,int n_index);
-_FUNASRAPI void FunVadFreeResult(FUNASR_RESULT result);
-_FUNASRAPI void FunVadUninit(FUNASR_HANDLE handle);
-_FUNASRAPI const float FunVadGetRetSnippetTime(FUNASR_RESULT result);
+_FUNASRAPI FUNASR_RESULT FsmnVadWavFile(FUNASR_HANDLE handle, const char* sz_wavfile, FUNASR_MODE mode, QM_CALLBACK fn_callback);
+_FUNASRAPI std::vector<std::vector<int>>* FsmnVadGetResult(FUNASR_RESULT result,int n_index);
+_FUNASRAPI void FsmnVadFreeResult(FUNASR_RESULT result);
+_FUNASRAPI void FsmnVadUninit(FUNASR_HANDLE handle);
+_FUNASRAPI const float FsmnVadGetRetSnippetTime(FUNASR_RESULT result);
// PUNC
_FUNASRAPI FUNASR_HANDLE FunPuncInit(std::map<std::string, std::string>& model_path, int thread_num);
@@ -78,7 +78,8 @@
//OfflineStream
_FUNASRAPI FUNASR_HANDLE FunOfflineInit(std::map<std::string, std::string>& model_path, int thread_num);
-_FUNASRAPI FUNASR_RESULT FunOfflineStream(FUNASR_HANDLE handle, const char* sz_wavfile, FUNASR_MODE mode, QM_CALLBACK fn_callback);
+_FUNASRAPI FUNASR_RESULT FunOfflineRecogFile(FUNASR_HANDLE handle, const char* sz_wavfile, FUNASR_MODE mode, QM_CALLBACK fn_callback);
+_FUNASRAPI FUNASR_RESULT FunOfflineRecogPCMBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, int sampling_rate, FUNASR_MODE mode, QM_CALLBACK fn_callback);
_FUNASRAPI void FunOfflineUninit(FUNASR_HANDLE handle);
#ifdef __cplusplus
diff --git a/funasr/runtime/onnxruntime/src/funasr-onnx-offline-punc.cpp b/funasr/runtime/onnxruntime/src/funasr-onnx-offline-punc.cpp
index e8f221f..4354ad4 100644
--- a/funasr/runtime/onnxruntime/src/funasr-onnx-offline-punc.cpp
+++ b/funasr/runtime/onnxruntime/src/funasr-onnx-offline-punc.cpp
@@ -14,7 +14,7 @@
#include <sstream>
#include <map>
#include <glog/logging.h>
-#include "libfunasrapi.h"
+#include "funasrruntime.h"
#include "tclap/CmdLine.h"
#include "com-define.h"
diff --git a/funasr/runtime/onnxruntime/src/funasr-onnx-offline-rtf.cpp b/funasr/runtime/onnxruntime/src/funasr-onnx-offline-rtf.cpp
index e2f8ee5..76624e7 100644
--- a/funasr/runtime/onnxruntime/src/funasr-onnx-offline-rtf.cpp
+++ b/funasr/runtime/onnxruntime/src/funasr-onnx-offline-rtf.cpp
@@ -10,7 +10,7 @@
#endif
#include <glog/logging.h>
-#include "libfunasrapi.h"
+#include "funasrruntime.h"
#include "tclap/CmdLine.h"
#include "com-define.h"
diff --git a/funasr/runtime/onnxruntime/src/funasr-onnx-offline-vad.cpp b/funasr/runtime/onnxruntime/src/funasr-onnx-offline-vad.cpp
index 2787534..37513ae 100644
--- a/funasr/runtime/onnxruntime/src/funasr-onnx-offline-vad.cpp
+++ b/funasr/runtime/onnxruntime/src/funasr-onnx-offline-vad.cpp
@@ -15,7 +15,7 @@
#include <map>
#include <vector>
#include <glog/logging.h>
-#include "libfunasrapi.h"
+#include "funasrruntime.h"
#include "tclap/CmdLine.h"
#include "com-define.h"
@@ -76,7 +76,7 @@
struct timeval start, end;
gettimeofday(&start, NULL);
int thread_num = 1;
- FUNASR_HANDLE vad_hanlde=FunVadInit(model_path, thread_num);
+ FUNASR_HANDLE vad_hanlde=FsmnVadInit(model_path, thread_num);
if (!vad_hanlde)
{
@@ -116,17 +116,17 @@
long taking_micros = 0;
for(auto& wav_file : wav_list){
gettimeofday(&start, NULL);
- FUNASR_RESULT result=FunVadWavFile(vad_hanlde, wav_file.c_str(), RASR_NONE, NULL);
+ FUNASR_RESULT result=FsmnVadWavFile(vad_hanlde, wav_file.c_str(), RASR_NONE, NULL);
gettimeofday(&end, NULL);
seconds = (end.tv_sec - start.tv_sec);
taking_micros += ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
if (result)
{
- vector<std::vector<int>>* vad_segments = FunVadGetResult(result, 0);
+ vector<std::vector<int>>* vad_segments = FsmnVadGetResult(result, 0);
print_segs(vad_segments);
- snippet_time += FunVadGetRetSnippetTime(result);
- FunVadFreeResult(result);
+ snippet_time += FsmnVadGetRetSnippetTime(result);
+ FsmnVadFreeResult(result);
}
else
{
@@ -137,7 +137,7 @@
LOG(INFO) << "Audio length: " << (double)snippet_time << " s";
LOG(INFO) << "Model inference takes: " << (double)taking_micros / 1000000 <<" s";
LOG(INFO) << "Model inference RTF: " << (double)taking_micros/ (snippet_time*1000000);
- FunVadUninit(vad_hanlde);
+ FsmnVadUninit(vad_hanlde);
return 0;
}
diff --git a/funasr/runtime/onnxruntime/src/funasr-onnx-offline.cpp b/funasr/runtime/onnxruntime/src/funasr-onnx-offline.cpp
index 51758a7..343039d 100644
--- a/funasr/runtime/onnxruntime/src/funasr-onnx-offline.cpp
+++ b/funasr/runtime/onnxruntime/src/funasr-onnx-offline.cpp
@@ -14,7 +14,7 @@
#include <sstream>
#include <map>
#include <glog/logging.h>
-#include "libfunasrapi.h"
+#include "funasrruntime.h"
#include "tclap/CmdLine.h"
#include "com-define.h"
@@ -107,7 +107,7 @@
long taking_micros = 0;
for(auto& wav_file : wav_list){
gettimeofday(&start, NULL);
- FUNASR_RESULT result=FunOfflineStream(asr_hanlde, wav_file.c_str(), RASR_NONE, NULL);
+ FUNASR_RESULT result=FunOfflineRecogFile(asr_hanlde, wav_file.c_str(), RASR_NONE, NULL);
gettimeofday(&end, NULL);
seconds = (end.tv_sec - start.tv_sec);
taking_micros += ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
diff --git a/funasr/runtime/onnxruntime/src/libfunasrapi.cpp b/funasr/runtime/onnxruntime/src/funasrruntime.cpp
similarity index 81%
rename from funasr/runtime/onnxruntime/src/libfunasrapi.cpp
rename to funasr/runtime/onnxruntime/src/funasrruntime.cpp
index 439e8da..ee6143e 100644
--- a/funasr/runtime/onnxruntime/src/libfunasrapi.cpp
+++ b/funasr/runtime/onnxruntime/src/funasrruntime.cpp
@@ -11,7 +11,7 @@
return mm;
}
- _FUNASRAPI FUNASR_HANDLE FunVadInit(std::map<std::string, std::string>& model_path, int thread_num)
+ _FUNASRAPI FUNASR_HANDLE FsmnVadInit(std::map<std::string, std::string>& model_path, int thread_num)
{
VadModel* mm = CreateVadModel(model_path, thread_num);
return mm;
@@ -145,7 +145,7 @@
}
// APIs for VAD Infer
- _FUNASRAPI FUNASR_RESULT FunVadWavFile(FUNASR_HANDLE handle, const char* sz_wavfile, FUNASR_MODE mode, QM_CALLBACK fn_callback)
+ _FUNASRAPI FUNASR_RESULT FsmnVadWavFile(FUNASR_HANDLE handle, const char* sz_wavfile, FUNASR_MODE mode, QM_CALLBACK fn_callback)
{
VadModel* vad_obj = (VadModel*)handle;
if (!vad_obj)
@@ -178,7 +178,7 @@
}
// APIs for Offline-stream Infer
- _FUNASRAPI FUNASR_RESULT FunOfflineStream(FUNASR_HANDLE handle, const char* sz_wavfile, FUNASR_MODE mode, QM_CALLBACK fn_callback)
+ _FUNASRAPI FUNASR_RESULT FunOfflineRecogFile(FUNASR_HANDLE handle, const char* sz_wavfile, FUNASR_MODE mode, QM_CALLBACK fn_callback)
{
OfflineStream* offline_stream = (OfflineStream*)handle;
if (!offline_stream)
@@ -214,6 +214,41 @@
return p_result;
}
+ _FUNASRAPI FUNASR_RESULT FunOfflineRecogPCMBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, int sampling_rate, FUNASR_MODE mode, QM_CALLBACK fn_callback)
+ {
+ OfflineStream* offline_stream = (OfflineStream*)handle;
+ if (!offline_stream)
+ return nullptr;
+
+ Audio audio(1);
+ if (!audio.LoadPcmwav(sz_buf, n_len, &sampling_rate))
+ return nullptr;
+ if(offline_stream->UseVad()){
+ audio.Split(offline_stream);
+ }
+
+ float* buff;
+ int len;
+ int flag = 0;
+ FUNASR_RECOG_RESULT* p_result = new FUNASR_RECOG_RESULT;
+ p_result->snippet_time = audio.GetTimeLen();
+ int n_step = 0;
+ int n_total = audio.GetQueueSize();
+ while (audio.Fetch(buff, len, flag) > 0) {
+ string msg = (offline_stream->asr_handle)->Forward(buff, len, flag);
+ p_result->msg += msg;
+ n_step++;
+ if (fn_callback)
+ fn_callback(n_step, n_total);
+ }
+ if(offline_stream->UsePunc()){
+ string punc_res = (offline_stream->punc_handle)->AddPunc((p_result->msg).c_str());
+ p_result->msg = punc_res;
+ }
+
+ return p_result;
+ }
+
_FUNASRAPI const int FunASRGetRetNumber(FUNASR_RESULT result)
{
if (!result)
@@ -231,7 +266,7 @@
return ((FUNASR_RECOG_RESULT*)result)->snippet_time;
}
- _FUNASRAPI const float FunVadGetRetSnippetTime(FUNASR_RESULT result)
+ _FUNASRAPI const float FsmnVadGetRetSnippetTime(FUNASR_RESULT result)
{
if (!result)
return 0.0f;
@@ -249,7 +284,7 @@
return p_result->msg.c_str();
}
- _FUNASRAPI vector<std::vector<int>>* FunVadGetResult(FUNASR_RESULT result,int n_index)
+ _FUNASRAPI vector<std::vector<int>>* FsmnVadGetResult(FUNASR_RESULT result,int n_index)
{
FUNASR_VAD_RESULT * p_result = (FUNASR_VAD_RESULT*)result;
if(!p_result)
@@ -267,7 +302,7 @@
}
}
- _FUNASRAPI void FunVadFreeResult(FUNASR_RESULT result)
+ _FUNASRAPI void FsmnVadFreeResult(FUNASR_RESULT result)
{
FUNASR_VAD_RESULT * p_result = (FUNASR_VAD_RESULT*)result;
if (p_result)
@@ -290,7 +325,7 @@
delete recog_obj;
}
- _FUNASRAPI void FunVadUninit(FUNASR_HANDLE handle)
+ _FUNASRAPI void FsmnVadUninit(FUNASR_HANDLE handle)
{
VadModel* recog_obj = (VadModel*)handle;
diff --git a/funasr/runtime/onnxruntime/src/precomp.h b/funasr/runtime/onnxruntime/src/precomp.h
index 0d3199e..9a8b17d 100644
--- a/funasr/runtime/onnxruntime/src/precomp.h
+++ b/funasr/runtime/onnxruntime/src/precomp.h
@@ -45,6 +45,6 @@
#include "resample.h"
#include "paraformer.h"
#include "offline-stream.h"
-#include "libfunasrapi.h"
+#include "funasrruntime.h"
using namespace paraformer;
diff --git a/funasr/runtime/onnxruntime/src/resample.cc b/funasr/runtime/onnxruntime/src/resample.cpp
similarity index 100%
rename from funasr/runtime/onnxruntime/src/resample.cc
rename to funasr/runtime/onnxruntime/src/resample.cpp
--
Gitblit v1.9.1