From 828b5f81afd5173cda6eec980017b28d0b298069 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期二, 18 四月 2023 13:04:51 +0800
Subject: [PATCH] docs
---
funasr/runtime/python/libtorch/README.md | 86 +++++-----
docs/websocket_python.md | 1
funasr/runtime/onnxruntime/readme.md | 122 ++++++++-------
funasr/runtime/python/onnxruntime/README.md | 63 +++----
funasr/runtime/grpc/Readme.md | 198 ++++++++++++++++--------
funasr/export/README.md | 1
funasr/runtime/python/grpc/Readme.md | 4
docs/index.rst | 1
8 files changed, 272 insertions(+), 204 deletions(-)
diff --git a/docs/index.rst b/docs/index.rst
index d3d629c..7bf22b5 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -32,6 +32,7 @@
./libtorch_python.md
./grpc_python.md
./grpc_cpp.md
+ ./websocket_python.md
diff --git a/docs/websocket_python.md b/docs/websocket_python.md
new file mode 120000
index 0000000..bcf239c
--- /dev/null
+++ b/docs/websocket_python.md
@@ -0,0 +1 @@
+../funasr/runtime/python/websocket/README.md
\ No newline at end of file
diff --git a/funasr/export/README.md b/funasr/export/README.md
index 4d09ff8..8f57673 100644
--- a/funasr/export/README.md
+++ b/funasr/export/README.md
@@ -1,3 +1,4 @@
+# Export models
## Environments
torch >= 1.11.0
diff --git a/funasr/runtime/grpc/Readme.md b/funasr/runtime/grpc/Readme.md
index 82347be..23e618c 100644
--- a/funasr/runtime/grpc/Readme.md
+++ b/funasr/runtime/grpc/Readme.md
@@ -1,6 +1,9 @@
-## paraformer grpc onnx server in c++
+# Using funasr with grpc-cpp
-#### Step 1. Build ../onnxruntime as it's document
+## For the Server
+
+### Build [onnxruntime](./onnxruntime_cpp.md) as it's document
+
```
#put onnx-lib & onnx-asr-model into /path/to/asrmodel(eg: /data/asrmodel)
ls /data/asrmodel/
@@ -10,7 +13,7 @@
```
-#### Step 2. Compile and install grpc v1.52.0 in case of grpc bugs
+### Compile and install grpc v1.52.0 in case of grpc bugs
```
export GRPC_INSTALL_DIR=/data/soft/grpc
export PKG_CONFIG_PATH=$GRPC_INSTALL_DIR/lib/pkgconfig
@@ -35,84 +38,149 @@
source ~/.bashrc
```
-#### Step 3. Compile and start grpc onnx paraformer server
+### Compile and start grpc onnx paraformer server
```
# set -DONNXRUNTIME_DIR=/path/to/asrmodel/onnxruntime-linux-x64-1.14.0
./rebuild.sh
```
-#### Step 4. Start grpc paraformer server
+### Start grpc paraformer server
```
Usage: ./cmake/build/paraformer_server port thread_num /path/to/model_file quantize(true or false)
./cmake/build/paraformer_server 10108 4 /data/asrmodel/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch false
```
-#### Step 5. Start grpc python paraformer client on PC with MIC
-```
-cd ../python/grpc
-python grpc_main_client_mic.py --host $server_ip --port 10108
+## For the client
+
+### Install the requirements as in [grpc-python](./docs/grpc_python.md)
+
+```shell
+git clone https://github.com/alibaba/FunASR.git && cd FunASR
+cd funasr/runtime/python/grpc
+pip install -r requirements_client.txt
```
-The `grpc_main_client_mic.py` follows the [original design] (https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/grpc#workflow-in-desgin) by sending audio_data with chunks. If you want to send audio_data in one request, here is an example:
+### Generate protobuf file
+Run on server, the two generated pb files are both used for server and client
+```shell
+# paraformer_pb2.py and paraformer_pb2_grpc.py are already generated,
+# regenerate it only when you make changes to ./proto/paraformer.proto file.
+python -m grpc_tools.protoc --proto_path=./proto -I ./proto --python_out=. --grpc_python_out=./ ./proto/paraformer.proto
```
-# go to ../python/grpc to find this package
-import paraformer_pb2
-
-class RecognizeStub:
- def __init__(self, channel):
- self.Recognize = channel.stream_stream(
- '/paraformer.ASR/Recognize',
- request_serializer=paraformer_pb2.Request.SerializeToString,
- response_deserializer=paraformer_pb2.Response.FromString,
- )
-
-
-async def send(channel, data, speaking, isEnd):
- stub = RecognizeStub(channel)
- req = paraformer_pb2.Request()
- if data:
- req.audio_data = data
- req.user = 'zz'
- req.language = 'zh-CN'
- req.speaking = speaking
- req.isEnd = isEnd
- q = queue.SimpleQueue()
- q.put(req)
- return stub.Recognize(iter(q.get, None))
-
-# send the audio data once
-async def grpc_rec(data, grpc_uri):
- with grpc.insecure_channel(grpc_uri) as channel:
- b = time.time()
- response = await send(channel, data, False, False)
- resp = response.next()
- text = ''
- if 'decoding' == resp.action:
- resp = response.next()
- if 'finish' == resp.action:
- text = json.loads(resp.sentence)['text']
- response = await send(channel, None, False, True)
- return {
- 'text': text,
- 'time': time.time() - b,
- }
-
-async def test():
- # fc = FunAsrGrpcClient('127.0.0.1', 9900)
- # t = await fc.rec(wav.tobytes())
- # print(t)
- wav, _ = sf.read('z-10s.wav', dtype='int16')
- uri = '127.0.0.1:9900'
- res = await grpc_rec(wav.tobytes(), uri)
- print(res)
-
-
-if __name__ == '__main__':
- asyncio.run(test())
-
+### Start grpc client
```
+# Start client.
+python grpc_main_client_mic.py --host 127.0.0.1 --port 10095
+```
+
+[//]: # (```)
+
+[//]: # (# go to ../python/grpc to find this package)
+
+[//]: # (import paraformer_pb2)
+
+[//]: # ()
+[//]: # ()
+[//]: # (class RecognizeStub:)
+
+[//]: # ( def __init__(self, channel):)
+
+[//]: # ( self.Recognize = channel.stream_stream()
+
+[//]: # ( '/paraformer.ASR/Recognize',)
+
+[//]: # ( request_serializer=paraformer_pb2.Request.SerializeToString,)
+
+[//]: # ( response_deserializer=paraformer_pb2.Response.FromString,)
+
+[//]: # ( ))
+
+[//]: # ()
+[//]: # ()
+[//]: # (async def send(channel, data, speaking, isEnd):)
+
+[//]: # ( stub = RecognizeStub(channel))
+
+[//]: # ( req = paraformer_pb2.Request())
+
+[//]: # ( if data:)
+
+[//]: # ( req.audio_data = data)
+
+[//]: # ( req.user = 'zz')
+
+[//]: # ( req.language = 'zh-CN')
+
+[//]: # ( req.speaking = speaking)
+
+[//]: # ( req.isEnd = isEnd)
+
+[//]: # ( q = queue.SimpleQueue())
+
+[//]: # ( q.put(req))
+
+[//]: # ( return stub.Recognize(iter(q.get, None)))
+
+[//]: # ()
+[//]: # (# send the audio data once)
+
+[//]: # (async def grpc_rec(data, grpc_uri):)
+
+[//]: # ( with grpc.insecure_channel(grpc_uri) as channel:)
+
+[//]: # ( b = time.time())
+
+[//]: # ( response = await send(channel, data, False, False))
+
+[//]: # ( resp = response.next())
+
+[//]: # ( text = '')
+
+[//]: # ( if 'decoding' == resp.action:)
+
+[//]: # ( resp = response.next())
+
+[//]: # ( if 'finish' == resp.action:)
+
+[//]: # ( text = json.loads(resp.sentence)['text'])
+
+[//]: # ( response = await send(channel, None, False, True))
+
+[//]: # ( return {)
+
+[//]: # ( 'text': text,)
+
+[//]: # ( 'time': time.time() - b,)
+
+[//]: # ( })
+
+[//]: # ()
+[//]: # (async def test():)
+
+[//]: # ( # fc = FunAsrGrpcClient('127.0.0.1', 9900))
+
+[//]: # ( # t = await fc.rec(wav.tobytes()))
+
+[//]: # ( # print(t))
+
+[//]: # ( wav, _ = sf.read('z-10s.wav', dtype='int16'))
+
+[//]: # ( uri = '127.0.0.1:9900')
+
+[//]: # ( res = await grpc_rec(wav.tobytes(), uri))
+
+[//]: # ( print(res))
+
+[//]: # ()
+[//]: # ()
+[//]: # (if __name__ == '__main__':)
+
+[//]: # ( asyncio.run(test()))
+
+[//]: # ()
+[//]: # (```)
## Acknowledge
diff --git a/funasr/runtime/onnxruntime/readme.md b/funasr/runtime/onnxruntime/readme.md
index dddb46a..f7be2e0 100644
--- a/funasr/runtime/onnxruntime/readme.md
+++ b/funasr/runtime/onnxruntime/readme.md
@@ -1,5 +1,69 @@
+# ONNXRuntime-cpp
-## Demo
+## Export the model
+### Install [modelscope and funasr](https://github.com/alibaba-damo-academy/FunASR#installation)
+
+```shell
+pip3 install torch torchaudio
+pip install -U modelscope
+pip install -U funasr
+```
+
+### Export [onnx model](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/export)
+
+```shell
+python -m funasr.export.export_model --model-name damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize True
+```
+
+## Building for Linux/Unix
+
+### Download onnxruntime
+```shell
+# download an appropriate onnxruntime from https://github.com/microsoft/onnxruntime/releases/tag/v1.14.0
+# here we get a copy of onnxruntime for linux 64
+wget https://github.com/microsoft/onnxruntime/releases/download/v1.14.0/onnxruntime-linux-x64-1.14.0.tgz
+tar -zxvf onnxruntime-linux-x64-1.14.0.tgz
+```
+
+### Install fftw3
+```shell
+sudo apt install libfftw3-dev #ubuntu
+# sudo yum install fftw fftw-devel #centos
+```
+
+### Install openblas
+```shell
+sudo apt-get install libopenblas-dev #ubuntu
+# sudo yum -y install openblas-devel #centos
+```
+
+### Build runtime
+```shell
+git clone https://github.com/alibaba-damo-academy/FunASR.git && cd funasr/runtime/onnxruntime
+mkdir build && cd build
+cmake -DCMAKE_BUILD_TYPE=release .. -DONNXRUNTIME_DIR=/path/to/onnxruntime-linux-x64-1.14.0
+make
+```
+
+[//]: # (### The structure of a qualified onnxruntime package.)
+
+[//]: # (```)
+
+[//]: # (onnxruntime_xxx)
+
+[//]: # (鈹溾攢鈹�鈹�include)
+
+[//]: # (鈹斺攢鈹�鈹�lib)
+
+[//]: # (```)
+
+## Building for Windows
+
+Ref to win/
+
+
+## Run the demo
+
```shell
tester /path/models_dir /path/wave_file quantize(true or false)
```
@@ -9,62 +73,6 @@
config.yaml, am.mvn, model.onnx(or model_quant.onnx)
```
-## Steps
-
-### Export onnx
-#### Install [modelscope and funasr](https://github.com/alibaba-damo-academy/FunASR#installation)
-
-```shell
-pip3 install torch torchaudio
-pip install -U modelscope
-pip install -U funasr
-```
-#### Export [onnx model](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/export)
-
-```shell
-python -m funasr.export.export_model --model-name damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize True
-```
-
-### Building for Linux/Unix
-
-#### Download onnxruntime
-```shell
-# download an appropriate onnxruntime from https://github.com/microsoft/onnxruntime/releases/tag/v1.14.0
-# here we get a copy of onnxruntime for linux 64
-wget https://github.com/microsoft/onnxruntime/releases/download/v1.14.0/onnxruntime-linux-x64-1.14.0.tgz
-tar -zxvf onnxruntime-linux-x64-1.14.0.tgz
-```
-
-#### Install fftw3
-```shell
-sudo apt install libfftw3-dev #ubuntu
-# sudo yum install fftw fftw-devel #centos
-```
-
-#### Install openblas
-```shell
-sudo apt-get install libopenblas-dev #ubuntu
-# sudo yum -y install openblas-devel #centos
-```
-
-#### Build runtime
-```shell
-git clone https://github.com/alibaba-damo-academy/FunASR.git && cd funasr/runtime/onnxruntime
-mkdir build && cd build
-cmake -DCMAKE_BUILD_TYPE=release .. -DONNXRUNTIME_DIR=/path/to/onnxruntime-linux-x64-1.14.0
-make
-```
-
-#### The structure of a qualified onnxruntime package.
-```
-onnxruntime_xxx
-鈹溾攢鈹�鈹�include
-鈹斺攢鈹�鈹�lib
-```
-
-### Building for Windows
-
-Ref to win/
## Acknowledge
1. This project is maintained by [FunASR community](https://github.com/alibaba-damo-academy/FunASR).
diff --git a/funasr/runtime/python/grpc/Readme.md b/funasr/runtime/python/grpc/Readme.md
index 8c14985..822cb52 100644
--- a/funasr/runtime/python/grpc/Readme.md
+++ b/funasr/runtime/python/grpc/Readme.md
@@ -1,8 +1,6 @@
-# Using paraformer with grpc
+# Using funasr with grpc-python
We can send streaming audio data to server in real-time with grpc client every 10 ms e.g., and get transcribed text when stop speaking.
The audio data is in streaming, the asr inference process is in offline.
-
-
## For the Server
diff --git a/funasr/runtime/python/libtorch/README.md b/funasr/runtime/python/libtorch/README.md
index 27b5f86..fd64cc6 100644
--- a/funasr/runtime/python/libtorch/README.md
+++ b/funasr/runtime/python/libtorch/README.md
@@ -1,60 +1,54 @@
-## Using funasr with libtorch
+# Libtorch-python
-[FunASR](https://github.com/alibaba-damo-academy/FunASR) hopes to build a bridge between academic research and industrial applications on speech recognition. By supporting the training & finetuning of the industrial-grade speech recognition model released on ModelScope, researchers and developers can conduct research and production of speech recognition models more conveniently, and promote the development of speech recognition ecology. ASR for Fun锛�
+## Export the model
+### Install [modelscope and funasr](https://github.com/alibaba-damo-academy/FunASR#installation)
+```shell
+pip3 install torch torchaudio
+pip install -U modelscope
+pip install -U funasr
+```
-### Steps:
-1. Export the model.
- - Command: (`Tips`: torch >= 1.11.0 is required.)
+### Export [onnx model](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/export)
- More details ref to ([export docs](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/export))
+```shell
+python -m funasr.export.export_model --model-name damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type torch --quantize True
+```
- - `e.g.`, Export model from modelscope
- ```shell
- python -m funasr.export.export_model --model-name damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type torch --quantize False
- ```
- - `e.g.`, Export model from local path, the model'name must be `model.pb`.
- ```shell
- python -m funasr.export.export_model --model-name ./damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type torch --quantize False
- ```
-
-
-2. Install the `funasr_torch`.
+## Install the `funasr_torch`.
- install from pip
- ```shell
- pip install -U funasr_torch
- # For the users in China, you could install with the command:
- # pip install -U funasr_torch -i https://mirror.sjtu.edu.cn/pypi/web/simple
+install from pip
+```shell
+pip install -U funasr_torch
+# For the users in China, you could install with the command:
+# pip install -U funasr_torch -i https://mirror.sjtu.edu.cn/pypi/web/simple
+```
+or install from source code
- ```
- or install from source code
+```shell
+git clone https://github.com/alibaba/FunASR.git && cd FunASR
+cd funasr/runtime/python/libtorch
+pip install -e ./
+# For the users in China, you could install with the command:
+# pip install -e ./ -i https://mirror.sjtu.edu.cn/pypi/web/simple
+```
- ```shell
- git clone https://github.com/alibaba/FunASR.git && cd FunASR
- cd funasr/runtime/python/libtorch
- pip install -e ./
- # For the users in China, you could install with the command:
- # pip install -e ./ -i https://mirror.sjtu.edu.cn/pypi/web/simple
+## Run the demo.
+- Model_dir: the model path, which contains `model.torchscripts`, `config.yaml`, `am.mvn`.
+- Input: wav formt file, support formats: `str, np.ndarray, List[str]`
+- Output: `List[str]`: recognition result.
+- Example:
+ ```python
+ from funasr_torch import Paraformer
- ```
+ model_dir = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
+ model = Paraformer(model_dir, batch_size=1)
-3. Run the demo.
- - Model_dir: the model path, which contains `model.torchscripts`, `config.yaml`, `am.mvn`.
- - Input: wav formt file, support formats: `str, np.ndarray, List[str]`
- - Output: `List[str]`: recognition result.
- - Example:
- ```python
- from funasr_torch import Paraformer
+ wav_path = ['/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav']
- model_dir = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
- model = Paraformer(model_dir, batch_size=1)
-
- wav_path = ['/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav']
-
- result = model(wav_path)
- print(result)
- ```
+ result = model(wav_path)
+ print(result)
+ ```
## Performance benchmark
diff --git a/funasr/runtime/python/onnxruntime/README.md b/funasr/runtime/python/onnxruntime/README.md
index 87510fa..e85e08a 100644
--- a/funasr/runtime/python/onnxruntime/README.md
+++ b/funasr/runtime/python/onnxruntime/README.md
@@ -1,30 +1,28 @@
-## Using funasr with ONNXRuntime
+# ONNXRuntime-python
+
+## Export the model
+### Install [modelscope and funasr](https://github.com/alibaba-damo-academy/FunASR#installation)
+
+```shell
+pip3 install torch torchaudio
+pip install -U modelscope
+pip install -U funasr
+```
+
+### Export [onnx model](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/export)
+
+```shell
+python -m funasr.export.export_model --model-name damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize True
+```
-### Steps:
-1. Export the model.
- - Command: (`Tips`: torch >= 1.11.0 is required.)
-
- More details ref to ([export docs](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/export))
-
- - `e.g.`, Export model from modelscope
- ```shell
- python -m funasr.export.export_model --model-name damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize False
- ```
- - `e.g.`, Export model from local path, the model'name must be `model.pb`.
- ```shell
- python -m funasr.export.export_model --model-name ./damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize False
- ```
-
-
-2. Install the `funasr_onnx`
+## Install the `funasr_onnx`
install from pip
```shell
pip install -U funasr_onnx
# For the users in China, you could install with the command:
# pip install -U funasr_onnx -i https://mirror.sjtu.edu.cn/pypi/web/simple
-
```
or install from source code
@@ -35,25 +33,24 @@
pip install -e ./
# For the users in China, you could install with the command:
# pip install -e ./ -i https://mirror.sjtu.edu.cn/pypi/web/simple
-
```
-3. Run the demo.
- - Model_dir: the model path, which contains `model.onnx`, `config.yaml`, `am.mvn`.
- - Input: wav formt file, support formats: `str, np.ndarray, List[str]`
- - Output: `List[str]`: recognition result.
- - Example:
- ```python
- from funasr_onnx import Paraformer
+## Run the demo
+- Model_dir: the model path, which contains `model.onnx`, `config.yaml`, `am.mvn`.
+- Input: wav formt file, support formats: `str, np.ndarray, List[str]`
+- Output: `List[str]`: recognition result.
+- Example:
+ ```python
+ from funasr_onnx import Paraformer
- model_dir = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
- model = Paraformer(model_dir, batch_size=1)
+ model_dir = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
+ model = Paraformer(model_dir, batch_size=1)
- wav_path = ['/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav']
+ wav_path = ['/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav']
- result = model(wav_path)
- print(result)
- ```
+ result = model(wav_path)
+ print(result)
+ ```
## Performance benchmark
--
Gitblit v1.9.1