Merge pull request #510 from zhaomingwork/cpp-python-websocket-compatible

zhifu gao

2023-05-13 9c056701a5fcfedffe439cd206093bb6b3785637

Merge pull request #510 from zhaomingwork/cpp-python-websocket-compatible

fix ws_client.py NoneType for microphone and cpp not return right wav_name

 funasr/runtime/python/websocket/ws_client.py

@@ -237,48 +237,43 @@


if __name__ == '__main__':
    process_list = []
    for i in range(args.test_thread_num):
        p = Process(target=one_thread,args=(i, 0, 0))
        p.start()
        process_list.append(p)
   # for microphone 
   if  args.audio_in is  None:
     p = Process(target=one_thread,args=(0, 0, 0))
     p.start()
     p.join()
     print('end')
   else:
     # calculate the number of wavs for each preocess
     if args.audio_in.endswith(".scp"):
         f_scp = open(args.audio_in)
         wavs = f_scp.readlines()
     else:
         wavs = [args.audio_in]
     total_len=len(wavs)
     if total_len>=args.test_thread_num:
          chunk_size=int((total_len)/args.test_thread_num)
          remain_wavs=total_len-chunk_size*args.test_thread_num
     else:
          chunk_size=1
          remain_wavs=0

    for i in process_list:
        p.join()
     process_list = []
     chunk_begin=0
     for i in range(args.test_thread_num):
         now_chunk_size= chunk_size
         if remain_wavs>0:
             now_chunk_size=chunk_size+1
             remain_wavs=remain_wavs-1
         # process i handle wavs at chunk_begin and size of now_chunk_size
         p = Process(target=one_thread,args=(i,chunk_begin,now_chunk_size))
         chunk_begin=chunk_begin+now_chunk_size
         p.start()
         process_list.append(p)

    print('end')
     for i in process_list:
         p.join()

#
# if __name__ == '__main__':
#     # calculate the number of wavs for each preocess
#     if args.audio_in.endswith(".scp"):
#         f_scp = open(args.audio_in)
#         wavs = f_scp.readlines()
#     else:
#         wavs = [args.audio_in]
#     total_len=len(wavs)
#     if total_len>=args.test_thread_num:
#          chunk_size=int((total_len)/args.test_thread_num)
#          remain_wavs=total_len-chunk_size*args.test_thread_num
#     else:
#          chunk_size=0
#
#     process_list = []
#     chunk_begin=0
#     for i in range(args.test_thread_num):
#         now_chunk_size= chunk_size
#         if remain_wavs>0:
#             now_chunk_size=chunk_size+1
#             remain_wavs=remain_wavs-1
#         # process i handle wavs at chunk_begin and size of now_chunk_size
#         p = Process(target=one_thread,args=(i,chunk_begin,now_chunk_size))
#         chunk_begin=chunk_begin+now_chunk_size
#         p.start()
#         process_list.append(p)
#
#     for i in process_list:
#         p.join()
#
#     print('end')
#
     print('end')



 funasr/runtime/websocket/websocketsrv.cpp

@@ -18,7 +18,8 @@


// feed buffer to asr engine for decoder

void WebSocketServer::do_decoder(const std::vector<char>& buffer,

                                 websocketpp::connection_hdl& hdl) {

                                 websocketpp::connection_hdl& hdl,

                                 const nlohmann::json& msg) {

  try {

    int num_samples = buffer.size();  // the size of the buf



@@ -35,13 +36,8 @@
      nlohmann::json jsonresult;        // result json

      jsonresult["text"] = asr_result;  // put result in 'text'

      jsonresult["mode"] = "offline";

      std::shared_ptr<FUNASR_MESSAGE> msg_data = nullptr;

      auto it_data = data_map.find(hdl);

      if (it_data != data_map.end()) {

        msg_data = it_data->second;

      }



      jsonresult["wav_name"] = msg_data->msg["wav_name"];

      jsonresult["wav_name"] = msg["wav_name"];



      // send the json to client

      server_->send(hdl, jsonresult.dump(), websocketpp::frame::opcode::text,

@@ -125,6 +121,7 @@
      if (jsonresult["wav_name"] != nullptr) {

        msg_data->msg["wav_name"] = jsonresult["wav_name"];

      }



      if (jsonresult["is_speaking"] == false ||

          jsonresult["is_finished"] == true) {

        std::cout << "client done" << std::endl;

@@ -137,9 +134,10 @@
          sample_data_p->insert(sample_data_p->end(), padding.data(),

                                padding.data() + padding.size());

          // for offline, send all receive data to decoder engine

          asio::post(io_decoder_, std::bind(&WebSocketServer::do_decoder, this,

                                            std::move(*(sample_data_p.get())),

                                            std::move(hdl)));

          asio::post(io_decoder_,

                     std::bind(&WebSocketServer::do_decoder, this,

                               std::move(*(sample_data_p.get())),

                               std::move(hdl), std::move(msg_data->msg)));

        }

      }

      break;

@@ -152,8 +150,9 @@
      if (isonline) {

        // if online TODO(zhaoming) still not done

        std::vector<char> s(pcm_data, pcm_data + num_samples);

        asio::post(io_decoder_, std::bind(&WebSocketServer::do_decoder, this,

                                          std::move(s), std::move(hdl)));

        asio::post(io_decoder_,

                   std::bind(&WebSocketServer::do_decoder, this, std::move(s),

                             std::move(hdl), std::move(msg_data->msg)));

      } else {

        // for offline, we add receive data to end of the sample data vector

        sample_data_p->insert(sample_data_p->end(), pcm_data,


 funasr/runtime/websocket/websocketsrv.h

@@ -72,7 +72,7 @@
    server_->clear_access_channels(websocketpp::log::alevel::all);
  }
  void do_decoder(const std::vector<char>& buffer,
                  websocketpp::connection_hdl& hdl);
                  websocketpp::connection_hdl& hdl, const nlohmann::json& msg);

  void initAsr(std::map<std::string, std::string>& model_path, int thread_num);
  void on_message(websocketpp::connection_hdl hdl, message_ptr msg);
@@ -89,7 +89,6 @@

  // use map to keep the received samples data from one connection in offline
  // engine. if for online engline, a data struct is needed(TODO)
 

  std::map<websocketpp::connection_hdl, std::shared_ptr<FUNASR_MESSAGE>,
           std::owner_less<websocketpp::connection_hdl>>