From ee2d96ad2e193e35dc5b02f55e87be282126f35c Mon Sep 17 00:00:00 2001
From: zhifu gao <zhifu.gzf@alibaba-inc.com>
Date: 星期三, 17 五月 2023 18:30:36 +0800
Subject: [PATCH] Merge pull request #523 from alibaba-damo-academy/main
---
funasr/runtime/onnxruntime/readme.md | 13 ++
funasr/runtime/onnxruntime/src/funasr-onnx-offline-rtf.cpp | 22 ++
funasr/runtime/html5/static/wsconnecter.js | 2
docs/runtime/html5.md | 1
funasr/runtime/html5/readme.md | 54 ++++----
funasr/runtime/onnxruntime/src/paraformer.cpp | 6
docs/index.rst | 6
funasr/runtime/html5/readme_cn.md | 111 ++++++++++++++++++
funasr/runtime/python/websocket/ws_server_online.py | 65 ++++++++--
funasr/runtime/onnxruntime/src/fsmn-vad.cpp | 18 +-
funasr/runtime/onnxruntime/include/vad-model.h | 8 -
funasr/runtime/onnxruntime/src/fsmn-vad.h | 4
12 files changed, 243 insertions(+), 67 deletions(-)
diff --git a/docs/index.rst b/docs/index.rst
index c2656bd..cb98f35 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -68,10 +68,12 @@
./runtime/onnxruntime_python.md
./runtime/onnxruntime_cpp.md
./runtime/libtorch_python.md
- ./runtime/grpc_python.md
- ./runtime/grpc_cpp.md
+ ./runtime/html5.md
./runtime/websocket_python.md
./runtime/websocket_cpp.md
+ ./runtime/grpc_python.md
+ ./runtime/grpc_cpp.md
+
.. toctree::
:maxdepth: 1
diff --git a/docs/runtime/html5.md b/docs/runtime/html5.md
new file mode 120000
index 0000000..bf47840
--- /dev/null
+++ b/docs/runtime/html5.md
@@ -0,0 +1 @@
+../../funasr/runtime/html5/readme.md
\ No newline at end of file
diff --git a/funasr/runtime/html5/readme.md b/funasr/runtime/html5/readme.md
index 612dc20..5dd462b 100644
--- a/funasr/runtime/html5/readme.md
+++ b/funasr/runtime/html5/readme.md
@@ -9,70 +9,70 @@
```
### javascript
-[html5褰曢煶](https://github.com/xiangyuecn/Recorder)
+[html5 recorder.js](https://github.com/xiangyuecn/Recorder)
```shell
Recorder
```
-### demo椤甸潰濡備笅
-
+### demo
+
-## 涓ょws_server_online杩炴帴妯″紡
-### 1)鐩存帴杩炴帴妯″紡锛屾祻瑙堝櫒https楹﹀厠椋� --> html5 demo鏈嶅姟 --> js wss鎺ュ彛 --> wss asr online srv(璇佷功鐢熸垚璇峰線鍚庣湅)
+## wss or ws protocol for ws_server_online
+1) wss: browser microphone data --> html5 demo server --> js wss api --> wss asr online srv #for certificate generation just look back
-### 2)nginx涓浆锛屾祻瑙堝櫒https楹﹀厠椋� --> html5 demo鏈嶅姟 --> js wss鎺ュ彛 --> nginx鏈嶅姟 --> ws asr online srv
+2) ws: browser microphone data --> html5 demo server --> js wss api --> nginx wss server --> ws asr online srv
-## 1.html5 demo鏈嶅姟鍚姩
-### 鍚姩html5鏈嶅姟锛岄渶瑕乻sl璇佷功(鑷繁鐢熸垚璇峰線鍚庣湅)
+## 1.html5 demo start
+### ssl certificate is required
```shell
usage: h5Server.py [-h] [--host HOST] [--port PORT] [--certfile CERTFILE]
[--keyfile KEYFILE]
python h5Server.py --port 1337
```
-## 2.鍚姩ws or wss asr online srv
-[鍏蜂綋璇风湅online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket)
-online asr鎻愪緵涓ょws鍜寃ss妯″紡锛寃ss妯″紡鍙互鐩存帴鍚姩锛屾棤闇�nginx涓浆銆傚惁鍒欓渶瑕侀�氳繃nginx灏唚ss杞彂鍒拌online asr鐨剋s绔彛涓�
-### wss鏂瑰紡
+## 2.asr online srv start
+[detail for online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket)
+Online asr provides wss or ws way. if started in ws way, nginx is required for relay.
+### wss way, ssl certificate is required
```shell
python ws_server_online.py --certfile server.crt --keyfile server.key --port 5921
```
-### ws鏂瑰紡
+### ws way
```shell
python ws_server_online.py --port 5921
```
-## 3.淇敼wsconnecter.js閲宎sr鎺ュ彛鍦板潃
-wsconnecter.js閲岄厤缃畂nline asr鏈嶅姟鍦板潃璺緞锛岃繖閲岄厤缃殑鏄痺ss绔彛
+## 3.modify asr address in wsconnecter.js according to your environment
+asr address in wsconnecter.js must be wss, just like
var Uri = "wss://xxx:xxx/"
-## 4.娴忚鍣ㄦ墦寮�鍦板潃娴嬭瘯
-https://127.0.0.1:1337/static/index.html
+## 4.open browser to access html5 demo
+https://youraddress:port/static/index.html
-## 鑷鐢熸垚璇佷功
-鐢熸垚璇佷功(娉ㄦ剰杩欑璇佷功骞朵笉鑳借鎵�鏈夋祻瑙堝櫒璁ゅ彲锛岄儴鍒嗘墜鍔ㄦ巿鏉冨彲浠ヨ闂�,鏈�濂戒娇鐢ㄥ叾浠栬璇佺殑瀹樻柟ssl璇佷功)
+## certificate generation by yourself
+generated certificate may not suitable for all browsers due to security concerns. you'd better buy or download an authenticated ssl certificate from authorized agency.
```shell
-### 1)鐢熸垚绉侀挜锛屾寜鐓ф彁绀哄～鍐欏唴瀹�
+### 1) Generate a private key
openssl genrsa -des3 -out server.key 1024
-### 2)鐢熸垚csr鏂囦欢 锛屾寜鐓ф彁绀哄～鍐欏唴瀹�
+### 2) Generate a csr file
openssl req -new -key server.key -out server.csr
-### 鍘绘帀pass
+### 3) Remove pass
cp server.key server.key.org
openssl rsa -in server.key.org -out server.key
-### 鐢熸垚crt鏂囦欢锛屾湁鏁堟湡1骞达紙365澶╋級
+### 4) Generated a crt file, valid for 1 year
openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt
```
-## nginx閰嶇疆璇存槑(浜嗚В鐨勫彲浠ヨ烦杩�)
-h5鎵撳紑楹﹀厠椋庨渶瑕乭ttps鍗忚锛屽悓鏃跺悗绔殑asr websocket涔熷繀椤绘槸wss鍗忚锛屽鏋淸online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket)浠s鏂瑰紡杩愯锛屾垜浠彲浠ラ�氳繃nginx閰嶇疆瀹炵幇wss鍗忚鍒皐s鍗忚鐨勮浆鎹€��
-
-### nginx杞彂閰嶇疆绀轰緥
+## nginx configuration (you can skip it if you known)
+https and wss protocol are required by browsers when want to open microphone and websocket.
+if [online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket) run in ws way, you should use nginx to convert wss to ws.
+### nginx wss->ws configuration example
```shell
events { [0/1548]
worker_connections 1024;
diff --git a/funasr/runtime/html5/readme_cn.md b/funasr/runtime/html5/readme_cn.md
new file mode 100644
index 0000000..d1a56eb
--- /dev/null
+++ b/funasr/runtime/html5/readme_cn.md
@@ -0,0 +1,111 @@
+# online asr demo for html5
+
+## requirement
+### python
+```shell
+flask
+gevent
+pyOpenSSL
+```
+
+### javascript
+[html5褰曢煶](https://github.com/xiangyuecn/Recorder)
+```shell
+Recorder
+```
+
+### demo椤甸潰濡備笅
+
+
+## 涓ょws_server_online杩炴帴妯″紡
+### 1)鐩存帴杩炴帴妯″紡锛屾祻瑙堝櫒https楹﹀厠椋� --> html5 demo鏈嶅姟 --> js wss鎺ュ彛 --> wss asr online srv(璇佷功鐢熸垚璇峰線鍚庣湅)
+
+### 2)nginx涓浆锛屾祻瑙堝櫒https楹﹀厠椋� --> html5 demo鏈嶅姟 --> js wss鎺ュ彛 --> nginx鏈嶅姟 --> ws asr online srv
+
+## 1.html5 demo鏈嶅姟鍚姩
+### 鍚姩html5鏈嶅姟锛岄渶瑕乻sl璇佷功(鑷繁鐢熸垚璇峰線鍚庣湅)
+
+```shell
+usage: h5Server.py [-h] [--host HOST] [--port PORT] [--certfile CERTFILE]
+ [--keyfile KEYFILE]
+python h5Server.py --port 1337
+```
+## 2.鍚姩ws or wss asr online srv
+[鍏蜂綋璇风湅online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket)
+online asr鎻愪緵涓ょws鍜寃ss妯″紡锛寃ss妯″紡鍙互鐩存帴鍚姩锛屾棤闇�nginx涓浆銆傚惁鍒欓渶瑕侀�氳繃nginx灏唚ss杞彂鍒拌online asr鐨剋s绔彛涓�
+### wss鏂瑰紡
+```shell
+python ws_server_online.py --certfile server.crt --keyfile server.key --port 5921
+```
+### ws鏂瑰紡
+```shell
+python ws_server_online.py --port 5921
+```
+## 3.淇敼wsconnecter.js閲宎sr鎺ュ彛鍦板潃
+wsconnecter.js閲岄厤缃畂nline asr鏈嶅姟鍦板潃璺緞锛岃繖閲岄厤缃殑鏄痺ss绔彛
+var Uri = "wss://xxx:xxx/"
+
+## 4.娴忚鍣ㄦ墦寮�鍦板潃娴嬭瘯
+https://127.0.0.1:1337/static/index.html
+
+
+
+
+## 鑷鐢熸垚璇佷功
+鐢熸垚璇佷功(娉ㄦ剰杩欑璇佷功骞朵笉鑳借鎵�鏈夋祻瑙堝櫒璁ゅ彲锛岄儴鍒嗘墜鍔ㄦ巿鏉冨彲浠ヨ闂�,鏈�濂戒娇鐢ㄥ叾浠栬璇佺殑瀹樻柟ssl璇佷功)
+
+```shell
+### 1)鐢熸垚绉侀挜锛屾寜鐓ф彁绀哄～鍐欏唴瀹�
+openssl genrsa -des3 -out server.key 1024
+
+### 2)鐢熸垚csr鏂囦欢 锛屾寜鐓ф彁绀哄～鍐欏唴瀹�
+openssl req -new -key server.key -out server.csr
+
+### 鍘绘帀pass
+cp server.key server.key.org
+openssl rsa -in server.key.org -out server.key
+
+### 鐢熸垚crt鏂囦欢锛屾湁鏁堟湡1骞达紙365澶╋級
+openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt
+```
+
+## nginx閰嶇疆璇存槑(浜嗚В鐨勫彲浠ヨ烦杩�)
+h5鎵撳紑楹﹀厠椋庨渶瑕乭ttps鍗忚锛屽悓鏃跺悗绔殑asr websocket涔熷繀椤绘槸wss鍗忚锛屽鏋淸online asr](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket)浠s鏂瑰紡杩愯锛屾垜浠彲浠ラ�氳繃nginx閰嶇疆瀹炵幇wss鍗忚鍒皐s鍗忚鐨勮浆鎹€��
+
+### nginx杞彂閰嶇疆绀轰緥
+```shell
+events { [0/1548]
+ worker_connections 1024;
+ accept_mutex on;
+ }
+http {
+ error_log error.log;
+ access_log access.log;
+ server {
+
+ listen 5921 ssl http2; # nginx listen port for wss
+ server_name www.test.com;
+
+ ssl_certificate /funasr/server.crt;
+ ssl_certificate_key /funasr/server.key;
+ ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+ ssl_ciphers HIGH:!aNULL:!MD5;
+
+ location /wss/ {
+
+
+ proxy_pass http://127.0.0.1:1111/; # asr online model ws address and port
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ proxy_read_timeout 600s;
+
+ }
+ }
+```
+### 淇敼wsconnecter.js閲宎sr鎺ュ彛鍦板潃
+wsconnecter.js閲岄厤缃畂nline asr鏈嶅姟鍦板潃璺緞锛岃繖閲岄厤缃殑鏄痺ss绔彛
+var Uri = "wss://xxx:xxx/wss/"
+## Acknowledge
+1. This project is maintained by [FunASR community](https://github.com/alibaba-damo-academy/FunASR).
+2. We acknowledge [AiHealthx](http://www.aihealthx.com/) for contributing the html5 demo.
\ No newline at end of file
diff --git a/funasr/runtime/html5/static/wsconnecter.js b/funasr/runtime/html5/static/wsconnecter.js
index 4b22e8f..594652d 100644
--- a/funasr/runtime/html5/static/wsconnecter.js
+++ b/funasr/runtime/html5/static/wsconnecter.js
@@ -5,7 +5,7 @@
/* 2021-2023 by zhaoming,mali aihealthx.com */
function WebSocketConnectMethod( config ) { //瀹氫箟socket杩炴帴鏂规硶绫�
- var Uri = "wss://111.205.137.58:5821/wss/" //璁剧疆wss asr online鎺ュ彛鍦板潃 濡� wss://X.X.X.X:port/wss/
+ var Uri = "wss://30.220.136.139:5921/" // var Uri = "wss://30.221.177.46:5921/" //璁剧疆wss asr online鎺ュ彛鍦板潃 濡� wss://X.X.X.X:port/wss/
var speechSokt;
var connKeeperID;
diff --git a/funasr/runtime/onnxruntime/include/vad-model.h b/funasr/runtime/onnxruntime/include/vad-model.h
index e37bd97..b1b1e9d 100644
--- a/funasr/runtime/onnxruntime/include/vad-model.h
+++ b/funasr/runtime/onnxruntime/include/vad-model.h
@@ -11,15 +11,11 @@
public:
virtual ~VadModel(){};
virtual void InitVad(const std::string &vad_model, const std::string &vad_cmvn, const std::string &vad_config, int thread_num)=0;
- virtual std::vector<std::vector<int>> Infer(const std::vector<float> &waves)=0;
+ virtual std::vector<std::vector<int>> Infer(std::vector<float> &waves, bool input_finished=true)=0;
virtual void ReadModel(const char* vad_model)=0;
virtual void LoadConfigFromYaml(const char* filename)=0;
virtual void FbankKaldi(float sample_rate, std::vector<std::vector<float>> &vad_feats,
- const std::vector<float> &waves)=0;
- virtual void LfrCmvn(std::vector<std::vector<float>> &vad_feats)=0;
- virtual void Forward(
- const std::vector<std::vector<float>> &chunk_feats,
- std::vector<std::vector<float>> *out_prob)=0;
+ std::vector<float> &waves)=0;
virtual void LoadCmvn(const char *filename)=0;
virtual void InitCache()=0;
};
diff --git a/funasr/runtime/onnxruntime/readme.md b/funasr/runtime/onnxruntime/readme.md
index 3e34a67..4ed184f 100644
--- a/funasr/runtime/onnxruntime/readme.md
+++ b/funasr/runtime/onnxruntime/readme.md
@@ -127,6 +127,8 @@
### funasr-onnx-offline-rtf
```shell
./funasr-onnx-offline-rtf --model-dir <string> [--quantize <string>]
+ [--vad-dir <string>] [--vad-quant <string>]
+ [--punc-dir <string>] [--punc-quant <string>]
--wav-path <string> --thread-num <int32_t>
[--] [--version] [-h]
Where:
@@ -136,6 +138,17 @@
(required) the model path, which contains model.onnx, config.yaml, am.mvn
--quantize <string>
false (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir
+
+ --vad-dir <string>
+ the vad model path, which contains model.onnx, vad.yaml, vad.mvn
+ --vad-quant <string>
+ false (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir
+
+ --punc-dir <string>
+ the punc model path, which contains model.onnx, punc.yaml
+ --punc-quant <string>
+ false (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir
+
--wav-path <string>
(required) the input could be:
wav_path, e.g.: asr_example.wav;
diff --git a/funasr/runtime/onnxruntime/src/fsmn-vad.cpp b/funasr/runtime/onnxruntime/src/fsmn-vad.cpp
index f061534..516dc88 100644
--- a/funasr/runtime/onnxruntime/src/fsmn-vad.cpp
+++ b/funasr/runtime/onnxruntime/src/fsmn-vad.cpp
@@ -162,17 +162,21 @@
}
// get 4 caches outputs,each size is 128*19
- for (int i = 1; i < 5; i++) {
- float* data = vad_ort_outputs[i].GetTensorMutableData<float>();
- memcpy(in_cache_[i-1].data(), data, sizeof(float) * 128*19);
- }
+ // for (int i = 1; i < 5; i++) {
+ // float* data = vad_ort_outputs[i].GetTensorMutableData<float>();
+ // memcpy(in_cache_[i-1].data(), data, sizeof(float) * 128*19);
+ // }
}
void FsmnVad::FbankKaldi(float sample_rate, std::vector<std::vector<float>> &vad_feats,
- const std::vector<float> &waves) {
+ std::vector<float> &waves) {
knf::OnlineFbank fbank(fbank_opts);
- fbank.AcceptWaveform(sample_rate, &waves[0], waves.size());
+ std::vector<float> buf(waves.size());
+ for (int32_t i = 0; i != waves.size(); ++i) {
+ buf[i] = waves[i] * 32768;
+ }
+ fbank.AcceptWaveform(sample_rate, buf.data(), buf.size());
int32_t frames = fbank.NumFramesReady();
for (int32_t i = 0; i != frames; ++i) {
const float *frame = fbank.GetFrame(i);
@@ -267,7 +271,7 @@
}
std::vector<std::vector<int>>
-FsmnVad::Infer(const std::vector<float> &waves) {
+FsmnVad::Infer(std::vector<float> &waves, bool input_finished) {
std::vector<std::vector<float>> vad_feats;
std::vector<std::vector<float>> vad_probs;
FbankKaldi(vad_sample_rate_, vad_feats, waves);
diff --git a/funasr/runtime/onnxruntime/src/fsmn-vad.h b/funasr/runtime/onnxruntime/src/fsmn-vad.h
index 3d183f8..a8ec4ce 100644
--- a/funasr/runtime/onnxruntime/src/fsmn-vad.h
+++ b/funasr/runtime/onnxruntime/src/fsmn-vad.h
@@ -21,7 +21,7 @@
~FsmnVad();
void Test();
void InitVad(const std::string &vad_model, const std::string &vad_cmvn, const std::string &vad_config, int thread_num);
- std::vector<std::vector<int>> Infer(const std::vector<float> &waves);
+ std::vector<std::vector<int>> Infer(std::vector<float> &waves, bool input_finished=true);
void Reset();
private:
@@ -34,7 +34,7 @@
std::vector<const char *> *in_names, std::vector<const char *> *out_names);
void FbankKaldi(float sample_rate, std::vector<std::vector<float>> &vad_feats,
- const std::vector<float> &waves);
+ std::vector<float> &waves);
void LfrCmvn(std::vector<std::vector<float>> &vad_feats);
diff --git a/funasr/runtime/onnxruntime/src/funasr-onnx-offline-rtf.cpp b/funasr/runtime/onnxruntime/src/funasr-onnx-offline-rtf.cpp
index 6ba65c6..2d182e0 100644
--- a/funasr/runtime/onnxruntime/src/funasr-onnx-offline-rtf.cpp
+++ b/funasr/runtime/onnxruntime/src/funasr-onnx-offline-rtf.cpp
@@ -39,7 +39,7 @@
// warm up
for (size_t i = 0; i < 1; i++)
{
- FUNASR_RESULT result=FunASRInfer(asr_handle, wav_list[0].c_str(), RASR_NONE, NULL, 16000);
+ FUNASR_RESULT result=FunOfflineInfer(asr_handle, wav_list[0].c_str(), RASR_NONE, NULL, 16000);
}
while (true) {
@@ -50,7 +50,7 @@
}
gettimeofday(&start, NULL);
- FUNASR_RESULT result=FunASRInfer(asr_handle, wav_list[i].c_str(), RASR_NONE, NULL, 16000);
+ FUNASR_RESULT result=FunOfflineInfer(asr_handle, wav_list[i].c_str(), RASR_NONE, NULL, 16000);
gettimeofday(&end, NULL);
seconds = (end.tv_sec - start.tv_sec);
@@ -102,12 +102,20 @@
TCLAP::CmdLine cmd("funasr-onnx-offline-rtf", ' ', "1.0");
TCLAP::ValueArg<std::string> model_dir("", MODEL_DIR, "the model path, which contains model.onnx, config.yaml, am.mvn", true, "", "string");
TCLAP::ValueArg<std::string> quantize("", QUANTIZE, "false (Default), load the model of model.onnx in model_dir. If set true, load the model of model_quant.onnx in model_dir", false, "false", "string");
+ TCLAP::ValueArg<std::string> vad_dir("", VAD_DIR, "the vad model path, which contains model.onnx, vad.yaml, vad.mvn", false, "", "string");
+ TCLAP::ValueArg<std::string> vad_quant("", VAD_QUANT, "false (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir", false, "false", "string");
+ TCLAP::ValueArg<std::string> punc_dir("", PUNC_DIR, "the punc model path, which contains model.onnx, punc.yaml", false, "", "string");
+ TCLAP::ValueArg<std::string> punc_quant("", PUNC_QUANT, "false (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir", false, "false", "string");
TCLAP::ValueArg<std::string> wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string");
TCLAP::ValueArg<std::int32_t> thread_num("", THREAD_NUM, "multi-thread num for rtf", true, 0, "int32_t");
cmd.add(model_dir);
cmd.add(quantize);
+ cmd.add(vad_dir);
+ cmd.add(vad_quant);
+ cmd.add(punc_dir);
+ cmd.add(punc_quant);
cmd.add(wav_path);
cmd.add(thread_num);
cmd.parse(argc, argv);
@@ -115,11 +123,15 @@
std::map<std::string, std::string> model_path;
GetValue(model_dir, MODEL_DIR, model_path);
GetValue(quantize, QUANTIZE, model_path);
+ GetValue(vad_dir, VAD_DIR, model_path);
+ GetValue(vad_quant, VAD_QUANT, model_path);
+ GetValue(punc_dir, PUNC_DIR, model_path);
+ GetValue(punc_quant, PUNC_QUANT, model_path);
GetValue(wav_path, WAV_PATH, model_path);
struct timeval start, end;
gettimeofday(&start, NULL);
- FUNASR_HANDLE asr_handle=FunASRInit(model_path, 1);
+ FUNASR_HANDLE asr_handle=FunOfflineInit(model_path, 1);
if (!asr_handle)
{
@@ -132,7 +144,7 @@
long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec);
LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s";
- // read wav_scp
+ // read wav_path
vector<string> wav_list;
string wav_path_ = model_path.at(WAV_PATH);
if(is_target_file(wav_path_, "wav") || is_target_file(wav_path_, "pcm")){
@@ -179,6 +191,6 @@
LOG(INFO) << "total_rtf " << (double)total_time/ (total_length*1000000);
LOG(INFO) << "speedup " << 1.0/((double)total_time/ (total_length*1000000));
- FunASRUninit(asr_handle);
+ FunOfflineUninit(asr_handle);
return 0;
}
diff --git a/funasr/runtime/onnxruntime/src/paraformer.cpp b/funasr/runtime/onnxruntime/src/paraformer.cpp
index 74366a0..1957a12 100644
--- a/funasr/runtime/onnxruntime/src/paraformer.cpp
+++ b/funasr/runtime/onnxruntime/src/paraformer.cpp
@@ -69,7 +69,11 @@
vector<float> Paraformer::FbankKaldi(float sample_rate, const float* waves, int len) {
knf::OnlineFbank fbank_(fbank_opts);
- fbank_.AcceptWaveform(sample_rate, waves, len);
+ std::vector<float> buf(len);
+ for (int32_t i = 0; i != len; ++i) {
+ buf[i] = waves[i] * 32768;
+ }
+ fbank_.AcceptWaveform(sample_rate, buf.data(), buf.size());
//fbank_->InputFinished();
int32_t frames = fbank_.NumFramesReady();
int32_t feature_dim = fbank_opts.mel_opts.num_bins;
diff --git a/funasr/runtime/python/websocket/ws_server_online.py b/funasr/runtime/python/websocket/ws_server_online.py
index 16a3abe..2255688 100644
--- a/funasr/runtime/python/websocket/ws_server_online.py
+++ b/funasr/runtime/python/websocket/ws_server_online.py
@@ -32,15 +32,29 @@
ncpu=args.ncpu,
model_revision='v1.0.4')
+# vad
+inference_pipeline_vad = pipeline(
+ task=Tasks.voice_activity_detection,
+ model=args.vad_model,
+ model_revision=None,
+ output_dir=None,
+ batch_size=1,
+ mode='online',
+ ngpu=args.ngpu,
+ ncpu=1,
+)
+
print("model loaded")
async def ws_serve(websocket, path):
+ frames = []
frames_asr_online = []
global websocket_users
websocket_users.add(websocket)
websocket.param_dict_asr_online = {"cache": dict()}
+ websocket.param_dict_vad = {'in_cache': dict()}
websocket.wav_name = "microphone"
print("new user connected",flush=True)
try:
@@ -53,9 +67,10 @@
if "is_speaking" in messagejson:
websocket.is_speaking = messagejson["is_speaking"]
websocket.param_dict_asr_online["is_final"] = not websocket.is_speaking
+ websocket.param_dict_vad["is_final"] = not websocket.is_speaking
# need to fire engine manually if no data received any more
if not websocket.is_speaking:
- await async_asr_online(websocket,b"")
+ await async_asr_online(websocket, b"")
if "chunk_interval" in messagejson:
websocket.chunk_interval=messagejson["chunk_interval"]
if "wav_name" in messagejson:
@@ -64,14 +79,18 @@
websocket.param_dict_asr_online["chunk_size"] = messagejson["chunk_size"]
# if has bytes in buffer or message is bytes
if len(frames_asr_online) > 0 or not isinstance(message, str):
- if not isinstance(message,str):
+ if not isinstance(message, str):
frames_asr_online.append(message)
+ # frames.append(message)
+ # duration_ms = len(message) // 32
+ # websocket.vad_pre_idx += duration_ms
+ speech_start_i, speech_end_i = await async_vad(websocket, message)
+ websocket.is_speaking = not speech_end_i
+
if len(frames_asr_online) % websocket.chunk_interval == 0 or not websocket.is_speaking:
+ websocket.param_dict_asr_online["is_final"] = not websocket.is_speaking
audio_in = b"".join(frames_asr_online)
- # if not websocket.is_speaking:
- #padding 0.5s at end gurantee that asr engine can fire out last word
- # audio_in=audio_in+b''.join(np.zeros(int(16000*0.5),dtype=np.int16))
- await async_asr_online(websocket,audio_in)
+ await async_asr_online(websocket, audio_in)
frames_asr_online = []
@@ -85,7 +104,7 @@
async def async_asr_online(websocket,audio_in):
- if len(audio_in) >=0:
+ if len(audio_in) >= 0:
audio_in = load_bytes(audio_in)
rec_result = inference_pipeline_asr_online(audio_in=audio_in,
param_dict=websocket.param_dict_asr_online)
@@ -97,16 +116,30 @@
await websocket.send(message)
+async def async_vad(websocket, audio_in):
+ segments_result = inference_pipeline_vad(audio_in=audio_in, param_dict=websocket.param_dict_vad)
+
+ speech_start = False
+ speech_end = False
+
+ if len(segments_result) == 0 or len(segments_result["text"]) > 1:
+ return speech_start, speech_end
+ if segments_result["text"][0][0] != -1:
+ speech_start = segments_result["text"][0][0]
+ if segments_result["text"][0][1] != -1:
+ speech_end = True
+ return speech_start, speech_end
+
if len(args.certfile)>0:
- ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
-
- # Generate with Lets Encrypt, copied to this location, chown to current user and 400 permissions
- ssl_cert = args.certfile
- ssl_key = args.keyfile
-
- ssl_context.load_cert_chain(ssl_cert, keyfile=ssl_key)
- start_server = websockets.serve(ws_serve, args.host, args.port, subprotocols=["binary"], ping_interval=None,ssl=ssl_context)
+ ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+
+ # Generate with Lets Encrypt, copied to this location, chown to current user and 400 permissions
+ ssl_cert = args.certfile
+ ssl_key = args.keyfile
+
+ ssl_context.load_cert_chain(ssl_cert, keyfile=ssl_key)
+ start_server = websockets.serve(ws_serve, args.host, args.port, subprotocols=["binary"], ping_interval=None,ssl=ssl_context)
else:
- start_server = websockets.serve(ws_serve, args.host, args.port, subprotocols=["binary"], ping_interval=None)
+ start_server = websockets.serve(ws_serve, args.host, args.port, subprotocols=["binary"], ping_interval=None)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
\ No newline at end of file
--
Gitblit v1.9.1