From 1eb7d6a3db0ecdb56e7b7ecde33a3b160b0046d4 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期四, 16 三月 2023 19:26:37 +0800
Subject: [PATCH] Merge branch 'main' of github.com:alibaba-damo-academy/FunASR add
---
funasr/runtime/grpc/paraformer_server.cc | 187 ++++++++++++++++++++
funasr/runtime/grpc/CMakeLists.txt | 83 +++++++++
funasr/runtime/grpc/Readme.md | 49 +++++
funasr/runtime/grpc/common.cmake | 125 +++++++++++++
funasr/runtime/grpc/rebuild.sh | 16 +
funasr/runtime/grpc/paraformer_server.h | 51 +++++
6 files changed, 511 insertions(+), 0 deletions(-)
diff --git a/funasr/runtime/grpc/CMakeLists.txt b/funasr/runtime/grpc/CMakeLists.txt
new file mode 100644
index 0000000..56e3074
--- /dev/null
+++ b/funasr/runtime/grpc/CMakeLists.txt
@@ -0,0 +1,83 @@
+# Copyright 2018 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# cmake build file for C++ paraformer example.
+# Assumes protobuf and gRPC have been installed using cmake.
+# See cmake_externalproject/CMakeLists.txt for all-in-one cmake build
+# that automatically builds all the dependencies before building paraformer.
+
+cmake_minimum_required(VERSION 3.10)
+
+project(ASR C CXX)
+
+include(common.cmake)
+
+# Proto file
+get_filename_component(rg_proto "../python/grpc/proto/paraformer.proto" ABSOLUTE)
+get_filename_component(rg_proto_path "${rg_proto}" PATH)
+
+# Generated sources
+set(rg_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/paraformer.pb.cc")
+set(rg_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/paraformer.pb.h")
+set(rg_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/paraformer.grpc.pb.cc")
+set(rg_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/paraformer.grpc.pb.h")
+add_custom_command(
+ OUTPUT "${rg_proto_srcs}" "${rg_proto_hdrs}" "${rg_grpc_srcs}" "${rg_grpc_hdrs}"
+ COMMAND ${_PROTOBUF_PROTOC}
+ ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}"
+ --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
+ -I "${rg_proto_path}"
+ --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
+ "${rg_proto}"
+ DEPENDS "${rg_proto}")
+
+
+# Include generated *.pb.h files
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+
+include_directories(../onnxruntime/include/)
+link_directories(../onnxruntime/build/src/)
+link_directories(../onnxruntime/build/third_party/webrtc/)
+
+link_directories(${ONNXRUNTIME_DIR}/lib)
+add_subdirectory("../onnxruntime/src" onnx_src)
+
+# rg_grpc_proto
+add_library(rg_grpc_proto
+ ${rg_grpc_srcs}
+ ${rg_grpc_hdrs}
+ ${rg_proto_srcs}
+ ${rg_proto_hdrs})
+
+
+
+target_link_libraries(rg_grpc_proto
+ ${_REFLECTION}
+ ${_GRPC_GRPCPP}
+ ${_PROTOBUF_LIBPROTOBUF})
+
+# Targets paraformer_(server)
+foreach(_target
+ paraformer_server)
+ add_executable(${_target}
+ "${_target}.cc")
+ target_link_libraries(${_target}
+ rg_grpc_proto
+ rapidasr
+ webrtcvad
+ ${EXTRA_LIBS}
+ ${_REFLECTION}
+ ${_GRPC_GRPCPP}
+ ${_PROTOBUF_LIBPROTOBUF})
+endforeach()
diff --git a/funasr/runtime/grpc/Readme.md b/funasr/runtime/grpc/Readme.md
new file mode 100644
index 0000000..b8722c7
--- /dev/null
+++ b/funasr/runtime/grpc/Readme.md
@@ -0,0 +1,49 @@
+## paraformer grpc onnx server in c++
+
+
+#### Step 1. Build ../onnxruntime as it's document
+```
+#put onnx-lib & onnx-asr-model & vocab.txt into /data/asrmodel
+ls /data/asrmodel/
+onnxruntime-linux-x64-1.14.0 speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch
+
+file /data/asrmodel/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/vocab.txt
+UTF-8 Unicode text
+```
+
+#### Step 2. Compile and install grpc v1.52.0 in case of grpc bugs
+```
+export GRPC_INSTALL_DIR=/data/soft/grpc
+export PKG_CONFIG_PATH=$GRPC_INSTALL_DIR/lib/pkgconfig
+
+git clone -b v1.52.0 --depth=1 https://github.com/grpc/grpc.git
+cd grpc
+git submodule update --init --recursive
+
+mkdir -p cmake/build
+pushd cmake/build
+cmake -DgRPC_INSTALL=ON \
+ -DgRPC_BUILD_TESTS=OFF \
+ -DCMAKE_INSTALL_PREFIX=$GRPC_INSTALL_DIR \
+ ../..
+make
+popd
+
+echo "export GRPC_INSTALL_DIR=/data/soft/grpc" >> ~/.bashrc
+echo "export PKG_CONFIG_PATH=\$GRPC_INSTALL_DIR/lib/pkgconfig" >> ~/.bashrc
+echo "export PATH=\$GRPC_INSTALL_DIR/bin/:\$PKG_CONFIG_PATH:\$PATH" >> ~/.bashrc
+source ~/.bashrc
+```
+
+#### Step 3. Compile and start grpc onnx paraformer server
+```
+./rebuild.sh
+```
+
+
+
+#### Step 4. Start grpc python paraformer client on PC with MIC
+```
+cd ../python/grpc
+python grpc_main_client_mic.py --host $server_ip --port 10108
+```
diff --git a/funasr/runtime/grpc/common.cmake b/funasr/runtime/grpc/common.cmake
new file mode 100644
index 0000000..1326a5b
--- /dev/null
+++ b/funasr/runtime/grpc/common.cmake
@@ -0,0 +1,125 @@
+# Copyright 2018 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# cmake build file for C++ route_guide example.
+# Assumes protobuf and gRPC have been installed using cmake.
+# See cmake_externalproject/CMakeLists.txt for all-in-one cmake build
+# that automatically builds all the dependencies before building route_guide.
+
+cmake_minimum_required(VERSION 3.5.1)
+
+if (NOT DEFINED CMAKE_CXX_STANDARD)
+ set (CMAKE_CXX_STANDARD 14)
+endif()
+
+if(MSVC)
+ add_definitions(-D_WIN32_WINNT=0x600)
+endif()
+
+find_package(Threads REQUIRED)
+
+if(GRPC_AS_SUBMODULE)
+ # One way to build a projects that uses gRPC is to just include the
+ # entire gRPC project tree via "add_subdirectory".
+ # This approach is very simple to use, but the are some potential
+ # disadvantages:
+ # * it includes gRPC's CMakeLists.txt directly into your build script
+ # without and that can make gRPC's internal setting interfere with your
+ # own build.
+ # * depending on what's installed on your system, the contents of submodules
+ # in gRPC's third_party/* might need to be available (and there might be
+ # additional prerequisites required to build them). Consider using
+ # the gRPC_*_PROVIDER options to fine-tune the expected behavior.
+ #
+ # A more robust approach to add dependency on gRPC is using
+ # cmake's ExternalProject_Add (see cmake_externalproject/CMakeLists.txt).
+
+ # Include the gRPC's cmake build (normally grpc source code would live
+ # in a git submodule called "third_party/grpc", but this example lives in
+ # the same repository as gRPC sources, so we just look a few directories up)
+ add_subdirectory(../../.. ${CMAKE_CURRENT_BINARY_DIR}/grpc EXCLUDE_FROM_ALL)
+ message(STATUS "Using gRPC via add_subdirectory.")
+
+ # After using add_subdirectory, we can now use the grpc targets directly from
+ # this build.
+ set(_PROTOBUF_LIBPROTOBUF libprotobuf)
+ set(_REFLECTION grpc++_reflection)
+ if(CMAKE_CROSSCOMPILING)
+ find_program(_PROTOBUF_PROTOC protoc)
+ else()
+ set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
+ endif()
+ set(_GRPC_GRPCPP grpc++)
+ if(CMAKE_CROSSCOMPILING)
+ find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
+ else()
+ set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:grpc_cpp_plugin>)
+ endif()
+elseif(GRPC_FETCHCONTENT)
+ # Another way is to use CMake's FetchContent module to clone gRPC at
+ # configure time. This makes gRPC's source code available to your project,
+ # similar to a git submodule.
+ message(STATUS "Using gRPC via add_subdirectory (FetchContent).")
+ include(FetchContent)
+ FetchContent_Declare(
+ grpc
+ GIT_REPOSITORY https://github.com/grpc/grpc.git
+ # when using gRPC, you will actually set this to an existing tag, such as
+ # v1.25.0, v1.26.0 etc..
+ # For the purpose of testing, we override the tag used to the commit
+ # that's currently under test.
+ GIT_TAG vGRPC_TAG_VERSION_OF_YOUR_CHOICE)
+ FetchContent_MakeAvailable(grpc)
+
+ # Since FetchContent uses add_subdirectory under the hood, we can use
+ # the grpc targets directly from this build.
+ set(_PROTOBUF_LIBPROTOBUF libprotobuf)
+ set(_REFLECTION grpc++_reflection)
+ set(_PROTOBUF_PROTOC $<TARGET_FILE:protoc>)
+ set(_GRPC_GRPCPP grpc++)
+ if(CMAKE_CROSSCOMPILING)
+ find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
+ else()
+ set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:grpc_cpp_plugin>)
+ endif()
+else()
+ # This branch assumes that gRPC and all its dependencies are already installed
+ # on this system, so they can be located by find_package().
+
+ # Find Protobuf installation
+ # Looks for protobuf-config.cmake file installed by Protobuf's cmake installation.
+ set(protobuf_MODULE_COMPATIBLE TRUE)
+ find_package(Protobuf CONFIG REQUIRED)
+ message(STATUS "Using protobuf ${Protobuf_VERSION}")
+
+ set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf)
+ set(_REFLECTION gRPC::grpc++_reflection)
+ if(CMAKE_CROSSCOMPILING)
+ find_program(_PROTOBUF_PROTOC protoc)
+ else()
+ set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
+ endif()
+
+ # Find gRPC installation
+ # Looks for gRPCConfig.cmake file installed by gRPC's cmake installation.
+ find_package(gRPC CONFIG REQUIRED)
+ message(STATUS "Using gRPC ${gRPC_VERSION}")
+
+ set(_GRPC_GRPCPP gRPC::grpc++)
+ if(CMAKE_CROSSCOMPILING)
+ find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin)
+ else()
+ set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
+ endif()
+endif()
diff --git a/funasr/runtime/grpc/paraformer_server.cc b/funasr/runtime/grpc/paraformer_server.cc
new file mode 100644
index 0000000..b721f05
--- /dev/null
+++ b/funasr/runtime/grpc/paraformer_server.cc
@@ -0,0 +1,187 @@
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <iostream>
+#include <sstream>
+#include <memory>
+#include <string>
+
+#include <grpc/grpc.h>
+#include <grpcpp/server.h>
+#include <grpcpp/server_builder.h>
+#include <grpcpp/server_context.h>
+#include <grpcpp/security/server_credentials.h>
+
+#include "paraformer.grpc.pb.h"
+#include "paraformer_server.h"
+
+
+using grpc::Server;
+using grpc::ServerBuilder;
+using grpc::ServerContext;
+using grpc::ServerReader;
+using grpc::ServerReaderWriter;
+using grpc::ServerWriter;
+using grpc::Status;
+
+
+using paraformer::Request;
+using paraformer::Response;
+using paraformer::ASR;
+
+ASRServicer::ASRServicer() {
+ std::cout << "ASRServicer init" << std::endl;
+ init_flag = 0;
+}
+
+void ASRServicer::clear_states(const std::string& user) {
+ clear_buffers(user);
+ clear_transcriptions(user);
+}
+
+void ASRServicer::clear_buffers(const std::string& user) {
+ if (client_buffers.count(user)) {
+ client_buffers.erase(user);
+ }
+}
+
+void ASRServicer::clear_transcriptions(const std::string& user) {
+ if (client_transcription.count(user)) {
+ client_transcription.erase(user);
+ }
+}
+
+void ASRServicer::disconnect(const std::string& user) {
+ clear_states(user);
+ std::cout << "Disconnecting user: " << user << std::endl;
+}
+
+grpc::Status ASRServicer::Recognize(
+ grpc::ServerContext* context,
+ grpc::ServerReaderWriter<Response, Request>* stream) {
+
+ Request req;
+ while (stream->Read(&req)) {
+ if (req.isend()) {
+ std::cout << "asr end" << std::endl;
+ disconnect(req.user());
+ Response res;
+ res.set_sentence(
+ R"({"success": true, "detail": "asr end"})"
+ );
+ res.set_user(req.user());
+ res.set_action("terminate");
+ res.set_language(req.language());
+ stream->Write(res);
+ } else if (req.speaking()) {
+ if (req.audio_data().size() > 0) {
+ auto& buf = client_buffers[req.user()];
+ buf.insert(buf.end(), req.audio_data().begin(), req.audio_data().end());
+ }
+ Response res;
+ res.set_sentence(
+ R"({"success": true, "detail": "speaking"})"
+ );
+ res.set_user(req.user());
+ res.set_action("speaking");
+ res.set_language(req.language());
+ stream->Write(res);
+ } else if (!req.speaking()) {
+ if (client_buffers.count(req.user()) == 0) {
+ Response res;
+ res.set_sentence(
+ R"({"success": true, "detail": "waiting_for_voice"})"
+ );
+ res.set_user(req.user());
+ res.set_action("waiting");
+ res.set_language(req.language());
+ stream->Write(res);
+ }else {
+ auto begin_time = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+ std::string tmp_data = this->client_buffers[req.user()];
+ this->clear_states(req.user());
+
+ Response res;
+ res.set_sentence(
+ R"({"success": true, "detail": "decoding data: " + std::to_string(tmp_data.length()) + " bytes"})"
+ );
+ int data_len_int = tmp_data.length();
+ std::string data_len = std::to_string(data_len_int);
+ std::stringstream ss;
+ ss << R"({"success": true, "detail": "decoding data: )" << data_len << R"( bytes")" << R"("})";
+ std::string result = ss.str();
+ res.set_sentence(result);
+ res.set_user(req.user());
+ res.set_action("decoding");
+ res.set_language(req.language());
+ stream->Write(res);
+ if (tmp_data.length() < 800) { //min input_len for asr model
+ auto end_time = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+ std::string delay_str = std::to_string(end_time - begin_time);
+ std::cout << "user: " << req.user() << " , delay(ms): " << delay_str << ", error: data_is_not_long_enough" << std::endl;
+ Response res;
+ std::stringstream ss;
+ std::string asr_result = "";
+ ss << R"({"success": true, "detail": "finish_sentence","server_delay_ms":)" << delay_str << R"(,"text":")" << asr_result << R"("})";
+ std::string result = ss.str();
+ res.set_sentence(result);
+ res.set_user(req.user());
+ res.set_action("finish");
+ res.set_language(req.language());
+
+
+
+ stream->Write(res);
+ }
+ else {
+ RPASR_RESULT Result= RapidAsrRecogPCMBuffer(AsrHanlde, tmp_data.c_str(), data_len_int, RASR_NONE, NULL);
+ std::string asr_result = "浣犲ソ浣犲ソ锛屾垜鏄痑sr璇嗗埆缁撴灉銆俿tatic";
+
+ auto end_time = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count();
+ std::string delay_str = std::to_string(end_time - begin_time);
+
+ std::cout << "user: " << req.user() << " , delay(ms): " << delay_str << ", text: " << asr_result << std::endl;
+ Response res;
+ std::stringstream ss;
+ ss << R"({"success": true, "detail": "finish_sentence","server_delay_ms":)" << delay_str << R"(,"text":")" << asr_result << R"("})";
+ std::string result = ss.str();
+ res.set_sentence(result);
+ res.set_user(req.user());
+ res.set_action("finish");
+ res.set_language(req.language());
+
+
+ stream->Write(res);
+ }
+ }
+ }else {
+ Response res;
+ res.set_sentence(
+ R"({"success": false, "detail": "error, no condition matched! Unknown reason."})"
+ );
+ res.set_user(req.user());
+ res.set_action("terminate");
+ res.set_language(req.language());
+ stream->Write(res);
+ }
+ }
+ return Status::OK;
+}
+
+
+void RunServer() {
+ std::string server_address("0.0.0.0:10108");
+ ASRServicer service;
+
+ ServerBuilder builder;
+ builder.AddListeningPort(server_address, grpc::InsecureServerCredentials());
+ builder.RegisterService(&service);
+ std::unique_ptr<Server> server(builder.BuildAndStart());
+ std::cout << "Server listening on " << server_address << std::endl;
+ server->Wait();
+}
+
+int main(int argc, char** argv) {
+ RunServer();
+ return 0;
+}
diff --git a/funasr/runtime/grpc/paraformer_server.h b/funasr/runtime/grpc/paraformer_server.h
new file mode 100644
index 0000000..f7cc7a6
--- /dev/null
+++ b/funasr/runtime/grpc/paraformer_server.h
@@ -0,0 +1,51 @@
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <iostream>
+#include <memory>
+#include <string>
+
+#include <grpc/grpc.h>
+#include <grpcpp/server.h>
+#include <grpcpp/server_builder.h>
+#include <grpcpp/server_context.h>
+#include <grpcpp/security/server_credentials.h>
+
+#include <unordered_map>
+#include <chrono>
+
+#include "paraformer.grpc.pb.h"
+#include "librapidasrapi.h"
+
+
+using grpc::Server;
+using grpc::ServerBuilder;
+using grpc::ServerContext;
+using grpc::ServerReader;
+using grpc::ServerReaderWriter;
+using grpc::ServerWriter;
+using grpc::Status;
+
+
+using paraformer::Request;
+using paraformer::Response;
+using paraformer::ASR;
+
+
+class ASRServicer final : public ASR::Service {
+ private:
+ int init_flag;
+ std::unordered_map<std::string, std::string> client_buffers;
+ std::unordered_map<std::string, std::string> client_transcription;
+
+ public:
+ ASRServicer();
+ void clear_states(const std::string& user);
+ void clear_buffers(const std::string& user);
+ void clear_transcriptions(const std::string& user);
+ void disconnect(const std::string& user);
+ grpc::Status Recognize(grpc::ServerContext* context, grpc::ServerReaderWriter<Response, Request>* stream);
+ int nThreadNum = 4;
+ RPASR_HANDLE AsrHanlde=RapidAsrInit("/data/asrmodel/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/", nThreadNum);
+
+};
diff --git a/funasr/runtime/grpc/rebuild.sh b/funasr/runtime/grpc/rebuild.sh
new file mode 100644
index 0000000..4b76012
--- /dev/null
+++ b/funasr/runtime/grpc/rebuild.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+rm cmake -rf
+mkdir -p cmake/build
+
+cd cmake/build
+
+cmake -DCMAKE_BUILD_TYPE=release ../.. -DONNXRUNTIME_DIR=/data/asrmodel/onnxruntime-linux-x64-1.14.0
+make
+
+
+echo "Build cmake/build/paraformer_server successfully!"
+
+echo "Start server"
+
+./paraformer_server
\ No newline at end of file
--
Gitblit v1.9.1