From c8f3af885543aff5018a4897909ade8517d820ca Mon Sep 17 00:00:00 2001
From: lyblsgo <lyblsgo@163.com>
Date: 星期二, 21 三月 2023 16:50:30 +0800
Subject: [PATCH] add onnx quantize model for onnx; add tester_rtf
---
funasr/runtime/onnxruntime/include/Model.h | 2
funasr/runtime/onnxruntime/src/Model.cpp | 5 -
funasr/runtime/onnxruntime/tester/tester_rtf.cpp | 12 ++-
funasr/runtime/onnxruntime/src/librapidasrapi.cpp | 35 -----------
funasr/runtime/onnxruntime/tester/CMakeLists.txt | 3 +
funasr/runtime/onnxruntime/include/librapidasrapi.h | 29 +--------
funasr/runtime/onnxruntime/src/paraformer_onnx.h | 7 --
funasr/runtime/onnxruntime/src/paraformer_onnx.cpp | 13 +++-
funasr/runtime/onnxruntime/tester/tester.cpp | 18 ++---
9 files changed, 39 insertions(+), 85 deletions(-)
diff --git a/funasr/runtime/onnxruntime/include/Model.h b/funasr/runtime/onnxruntime/include/Model.h
index 06267cb..6f45c38 100644
--- a/funasr/runtime/onnxruntime/include/Model.h
+++ b/funasr/runtime/onnxruntime/include/Model.h
@@ -13,5 +13,5 @@
virtual std::string rescoring() = 0;
};
-Model *create_model(const char *path,int nThread=0);
+Model *create_model(const char *path,int nThread=0,bool quantize=false);
#endif
diff --git a/funasr/runtime/onnxruntime/include/librapidasrapi.h b/funasr/runtime/onnxruntime/include/librapidasrapi.h
index a83098f..918e574 100644
--- a/funasr/runtime/onnxruntime/include/librapidasrapi.h
+++ b/funasr/runtime/onnxruntime/include/librapidasrapi.h
@@ -1,33 +1,20 @@
#pragma once
-
#ifdef WIN32
-
-
#ifdef _RPASR_API_EXPORT
-
#define _RAPIDASRAPI __declspec(dllexport)
#else
#define _RAPIDASRAPI __declspec(dllimport)
#endif
-
-
#else
-#define _RAPIDASRAPI
+#define _RAPIDASRAPI
#endif
-
-
-
-
#ifndef _WIN32
-
#define RPASR_CALLBCK_PREFIX __attribute__((__stdcall__))
-
#else
#define RPASR_CALLBCK_PREFIX __stdcall
#endif
-
#ifdef __cplusplus
@@ -35,15 +22,12 @@
#endif
typedef void* RPASR_HANDLE;
-
typedef void* RPASR_RESULT;
-
typedef unsigned char RPASR_BOOL;
#define RPASR_TRUE 1
#define RPASR_FALSE 0
#define QM_DEFAULT_THREAD_NUM 4
-
typedef enum
{
@@ -55,7 +39,6 @@
}RPASR_MODE;
typedef enum {
-
RPASR_MODEL_PADDLE = 0,
RPASR_MODEL_PADDLE_2 = 1,
RPASR_MODEL_K2 = 2,
@@ -63,17 +46,15 @@
}RPASR_MODEL_TYPE;
-
typedef void (* QM_CALLBACK)(int nCurStep, int nTotal); // nTotal: total steps; nCurStep: Current Step.
- // APIs for qmasr
-
-_RAPIDASRAPI RPASR_HANDLE RapidAsrInit(const char* szModelDir, int nThread);
-
+// APIs for qmasr
+_RAPIDASRAPI RPASR_HANDLE RapidAsrInit(const char* szModelDir, int nThread, bool quantize);
// if not give a fnCallback ,it should be NULL
_RAPIDASRAPI RPASR_RESULT RapidAsrRecogBuffer(RPASR_HANDLE handle, const char* szBuf, int nLen, RPASR_MODE Mode, QM_CALLBACK fnCallback);
+
_RAPIDASRAPI RPASR_RESULT RapidAsrRecogPCMBuffer(RPASR_HANDLE handle, const char* szBuf, int nLen, RPASR_MODE Mode, QM_CALLBACK fnCallback);
_RAPIDASRAPI RPASR_RESULT RapidAsrRecogPCMFile(RPASR_HANDLE handle, const char* szFileName, RPASR_MODE Mode, QM_CALLBACK fnCallback);
@@ -83,8 +64,8 @@
_RAPIDASRAPI const char* RapidAsrGetResult(RPASR_RESULT Result,int nIndex);
_RAPIDASRAPI const int RapidAsrGetRetNumber(RPASR_RESULT Result);
-_RAPIDASRAPI void RapidAsrFreeResult(RPASR_RESULT Result);
+_RAPIDASRAPI void RapidAsrFreeResult(RPASR_RESULT Result);
_RAPIDASRAPI void RapidAsrUninit(RPASR_HANDLE Handle);
diff --git a/funasr/runtime/onnxruntime/src/Model.cpp b/funasr/runtime/onnxruntime/src/Model.cpp
index ddd4fd0..7ddb635 100644
--- a/funasr/runtime/onnxruntime/src/Model.cpp
+++ b/funasr/runtime/onnxruntime/src/Model.cpp
@@ -1,11 +1,10 @@
#include "precomp.h"
-Model *create_model(const char *path,int nThread)
+Model *create_model(const char *path, int nThread, bool quantize)
{
Model *mm;
-
- mm = new paraformer::ModelImp(path, nThread);
+ mm = new paraformer::ModelImp(path, nThread, quantize);
return mm;
}
diff --git a/funasr/runtime/onnxruntime/src/librapidasrapi.cpp b/funasr/runtime/onnxruntime/src/librapidasrapi.cpp
index f5f9d66..62f47a5 100644
--- a/funasr/runtime/onnxruntime/src/librapidasrapi.cpp
+++ b/funasr/runtime/onnxruntime/src/librapidasrapi.cpp
@@ -4,24 +4,16 @@
extern "C" {
#endif
-
// APIs for qmasr
- _RAPIDASRAPI RPASR_HANDLE RapidAsrInit(const char* szModelDir, int nThreadNum)
+ _RAPIDASRAPI RPASR_HANDLE RapidAsrInit(const char* szModelDir, int nThreadNum, bool quantize)
{
-
-
- Model* mm = create_model(szModelDir, nThreadNum);
-
+ Model* mm = create_model(szModelDir, nThreadNum, quantize);
return mm;
}
-
_RAPIDASRAPI RPASR_RESULT RapidAsrRecogBuffer(RPASR_HANDLE handle, const char* szBuf, int nLen, RPASR_MODE Mode, QM_CALLBACK fnCallback)
{
-
-
Model* pRecogObj = (Model*)handle;
-
if (!pRecogObj)
return nullptr;
@@ -46,15 +38,12 @@
fnCallback(nStep, nTotal);
}
-
return pResult;
}
_RAPIDASRAPI RPASR_RESULT RapidAsrRecogPCMBuffer(RPASR_HANDLE handle, const char* szBuf, int nLen, RPASR_MODE Mode, QM_CALLBACK fnCallback)
{
-
Model* pRecogObj = (Model*)handle;
-
if (!pRecogObj)
return nullptr;
@@ -79,16 +68,12 @@
fnCallback(nStep, nTotal);
}
-
return pResult;
-
}
_RAPIDASRAPI RPASR_RESULT RapidAsrRecogPCMFile(RPASR_HANDLE handle, const char* szFileName, RPASR_MODE Mode, QM_CALLBACK fnCallback)
{
-
Model* pRecogObj = (Model*)handle;
-
if (!pRecogObj)
return nullptr;
@@ -113,15 +98,12 @@
fnCallback(nStep, nTotal);
}
-
return pResult;
-
}
_RAPIDASRAPI RPASR_RESULT RapidAsrRecogFile(RPASR_HANDLE handle, const char* szWavfile, RPASR_MODE Mode, QM_CALLBACK fnCallback)
{
Model* pRecogObj = (Model*)handle;
-
if (!pRecogObj)
return nullptr;
@@ -146,9 +128,6 @@
fnCallback(nStep, nTotal);
}
-
-
-
return pResult;
}
@@ -158,7 +137,6 @@
return 0;
return 1;
-
}
@@ -168,7 +146,6 @@
return 0.0f;
return ((RPASR_RECOG_RESULT*)Result)->snippet_time;
-
}
_RAPIDASRAPI const char* RapidAsrGetResult(RPASR_RESULT Result,int nIndex)
@@ -178,33 +155,25 @@
return nullptr;
return pResult->msg.c_str();
-
}
_RAPIDASRAPI void RapidAsrFreeResult(RPASR_RESULT Result)
{
-
if (Result)
{
delete (RPASR_RECOG_RESULT*)Result;
-
}
}
_RAPIDASRAPI void RapidAsrUninit(RPASR_HANDLE handle)
{
-
Model* pRecogObj = (Model*)handle;
-
if (!pRecogObj)
return;
delete pRecogObj;
-
}
-
-
#ifdef __cplusplus
diff --git a/funasr/runtime/onnxruntime/src/paraformer_onnx.cpp b/funasr/runtime/onnxruntime/src/paraformer_onnx.cpp
index cca8d48..8eb0e89 100644
--- a/funasr/runtime/onnxruntime/src/paraformer_onnx.cpp
+++ b/funasr/runtime/onnxruntime/src/paraformer_onnx.cpp
@@ -3,10 +3,17 @@
using namespace std;
using namespace paraformer;
-ModelImp::ModelImp(const char* path,int nNumThread)
+ModelImp::ModelImp(const char* path,int nNumThread, bool quantize)
{
- string model_path = pathAppend(path, "model.onnx");
- string vocab_path = pathAppend(path, "vocab.txt");
+ string model_path;
+ string vocab_path;
+ if(quantize)
+ {
+ model_path = pathAppend(path, "model_quant.onnx");
+ }else{
+ model_path = pathAppend(path, "model.onnx");
+ }
+ vocab_path = pathAppend(path, "vocab.txt");
fe = new FeatureExtract(3);
diff --git a/funasr/runtime/onnxruntime/src/paraformer_onnx.h b/funasr/runtime/onnxruntime/src/paraformer_onnx.h
index ebbbb51..db00842 100644
--- a/funasr/runtime/onnxruntime/src/paraformer_onnx.h
+++ b/funasr/runtime/onnxruntime/src/paraformer_onnx.h
@@ -4,10 +4,6 @@
#ifndef PARAFORMER_MODELIMP_H
#define PARAFORMER_MODELIMP_H
-
-
-
-
namespace paraformer {
class ModelImp : public Model {
@@ -19,7 +15,6 @@
void apply_lfr(Tensor<float>*& din);
void apply_cmvn(Tensor<float>* din);
-
string greedy_search( float* in, int nLen);
#ifdef _WIN_X86
@@ -39,7 +34,7 @@
//string m_strOutputName, m_strOutputNameLen;
public:
- ModelImp(const char* path, int nNumThread=0);
+ ModelImp(const char* path, int nNumThread=0, bool quantize=false);
~ModelImp();
void reset();
string forward_chunk(float* din, int len, int flag);
diff --git a/funasr/runtime/onnxruntime/tester/CMakeLists.txt b/funasr/runtime/onnxruntime/tester/CMakeLists.txt
index d794271..f66319d 100644
--- a/funasr/runtime/onnxruntime/tester/CMakeLists.txt
+++ b/funasr/runtime/onnxruntime/tester/CMakeLists.txt
@@ -13,8 +13,11 @@
include_directories(${CMAKE_SOURCE_DIR}/include)
set(EXECNAME "tester")
+set(EXECNAMERTF "tester_rtf")
add_executable(${EXECNAME} "tester.cpp")
target_link_libraries(${EXECNAME} PUBLIC ${EXTRA_LIBS})
+add_executable(${EXECNAMERTF} "tester_rtf.cpp")
+target_link_libraries(${EXECNAMERTF} PUBLIC ${EXTRA_LIBS})
diff --git a/funasr/runtime/onnxruntime/tester/tester.cpp b/funasr/runtime/onnxruntime/tester/tester.cpp
index f2491a9..1739f0b 100644
--- a/funasr/runtime/onnxruntime/tester/tester.cpp
+++ b/funasr/runtime/onnxruntime/tester/tester.cpp
@@ -9,39 +9,38 @@
#include <iostream>
#include <fstream>
+#include <sstream>
using namespace std;
int main(int argc, char *argv[])
{
- if (argc < 2)
+ if (argc < 3)
{
- printf("Usage: %s /path/to/model_dir /path/to/wav/file", argv[0]);
+ printf("Usage: %s /path/to/model_dir /path/to/wav/file quantize(true or false)", argv[0]);
exit(-1);
}
struct timeval start, end;
gettimeofday(&start, NULL);
int nThreadNum = 4;
- RPASR_HANDLE AsrHanlde=RapidAsrInit(argv[1], nThreadNum);
+ // is quantize
+ bool quantize = false;
+ istringstream(argv[3]) >> boolalpha >> quantize;
+ RPASR_HANDLE AsrHanlde=RapidAsrInit(argv[1], nThreadNum, quantize);
if (!AsrHanlde)
{
printf("Cannot load ASR Model from: %s, there must be files model.onnx and vocab.txt", argv[1]);
exit(-1);
}
-
-
gettimeofday(&end, NULL);
long seconds = (end.tv_sec - start.tv_sec);
long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
printf("Model initialization takes %lfs.\n", (double)modle_init_micros / 1000000);
-
-
gettimeofday(&start, NULL);
float snippet_time = 0.0f;
-
RPASR_RESULT Result=RapidAsrRecogFile(AsrHanlde, argv[2], RASR_NONE, NULL);
@@ -61,7 +60,6 @@
{
cout <<"no return data!";
}
-
//char* buff = nullptr;
//int len = 0;
@@ -101,13 +99,11 @@
//
//delete[]buff;
//}
-
printf("Audio length %lfs.\n", (double)snippet_time);
seconds = (end.tv_sec - start.tv_sec);
long taking_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
printf("Model inference takes %lfs.\n", (double)taking_micros / 1000000);
-
printf("Model inference RTF: %04lf.\n", (double)taking_micros/ (snippet_time*1000000));
RapidAsrUninit(AsrHanlde);
diff --git a/funasr/runtime/onnxruntime/tester/tester_rtf.cpp b/funasr/runtime/onnxruntime/tester/tester_rtf.cpp
index 19c3ec1..9651900 100644
--- a/funasr/runtime/onnxruntime/tester/tester_rtf.cpp
+++ b/funasr/runtime/onnxruntime/tester/tester_rtf.cpp
@@ -16,9 +16,9 @@
int main(int argc, char *argv[])
{
- if (argc < 2)
+ if (argc < 4)
{
- printf("Usage: %s /path/to/model_dir /path/to/wav.scp", argv[0]);
+ printf("Usage: %s /path/to/model_dir /path/to/wav.scp quantize(true or false) \n", argv[0]);
exit(-1);
}
@@ -43,7 +43,11 @@
struct timeval start, end;
gettimeofday(&start, NULL);
int nThreadNum = 1;
- RPASR_HANDLE AsrHanlde=RapidAsrInit(argv[1], nThreadNum);
+ // is quantize
+ bool quantize = false;
+ istringstream(argv[3]) >> boolalpha >> quantize;
+
+ RPASR_HANDLE AsrHanlde=RapidAsrInit(argv[1], nThreadNum, quantize);
if (!AsrHanlde)
{
printf("Cannot load ASR Model from: %s, there must be files model.onnx and vocab.txt", argv[1]);
@@ -88,7 +92,7 @@
printf("total_time_wav %ld ms.\n", (long)(total_length * 1000));
printf("total_time_comput %ld ms.\n", total_time / 1000);
- printf("Model inference RTF: %05lf.\n", (double)total_time/ (total_length*1000000));
+ printf("total_rtf %05lf .\n", (double)total_time/ (total_length*1000000));
RapidAsrUninit(AsrHanlde);
return 0;
--
Gitblit v1.9.1