add sentence timestamp

雾聪

2023-12-18 f72914003a8c4ab7ae72d52dbd7c5f70ea22313a

 runtime/websocket/bin/websocket-server-2pass.cpp

@@ -80,6 +80,12 @@
    jsonresult["timestamp"] = tmp_stamp_msg;
  }

  std::string tmp_stamp_sents = FunASRGetStampSents(result);
  if (tmp_stamp_sents != "") {
    LOG(INFO) << "offline stamp_sents : " << tmp_stamp_sents;
    jsonresult["stamp_sents"] = tmp_stamp_sents;
  }

  return jsonresult;
}
// feed buffer to asr engine for decoder
@@ -98,8 +104,8 @@
    std::string wav_format,
    FUNASR_HANDLE& tpass_online_handle) {
  // lock for each connection
  scoped_lock guard(thread_lock);
  if(!tpass_online_handle){
    scoped_lock guard(thread_lock);
     LOG(INFO) << "tpass_online_handle  is free, return";
     msg["access_num"]=(int)msg["access_num"]-1;
     return;
@@ -128,10 +134,12 @@
                                       hotwords_embedding, itn);

        } else {
          scoped_lock guard(thread_lock);
          msg["access_num"]=(int)msg["access_num"]-1;
          return;
        }
      } catch (std::exception const& e) {
        scoped_lock guard(thread_lock);
        LOG(ERROR) << e.what();
        msg["access_num"]=(int)msg["access_num"]-1;
        return;
@@ -162,10 +170,12 @@
                                       wav_format, (ASR_TYPE)asr_mode_,
                                       hotwords_embedding, itn);
        } else {
          scoped_lock guard(thread_lock);
          msg["access_num"]=(int)msg["access_num"]-1;    
          return;
        }
      } catch (std::exception const& e) {
        scoped_lock guard(thread_lock);
        LOG(ERROR) << e.what();
        msg["access_num"]=(int)msg["access_num"]-1;
        return;
@@ -209,6 +219,7 @@
  } catch (std::exception const& e) {
    std::cerr << "Error: " << e.what() << std::endl;
  }
  scoped_lock guard(thread_lock);
  msg["access_num"]=(int)msg["access_num"]-1;
 
}
@@ -227,7 +238,7 @@
    data_msg->msg["wav_name"] = "wav-default-id";
    data_msg->msg["mode"] = "2pass";
    data_msg->msg["itn"] = true;
    data_msg->msg["audio_fs"] = 16000;
    data_msg->msg["audio_fs"] = 16000; // default is 16k
    data_msg->msg["access_num"] = 0; // the number of access for this object, when it is 0, we can free it saftly
    data_msg->msg["is_eof"]=false; // if this connection is closed
    data_msg->punc_cache =
@@ -313,7 +324,7 @@
        data_msg->msg["is_eof"]=true;
        guard_decoder.unlock();
        to_remove.push_back(hdl);
        LOG(INFO)<<"connection is closed: "<<e.what();
        LOG(INFO)<<"connection is closed.";
        
      }
      iter++;
@@ -336,6 +347,10 @@
  auto it_data = data_map.find(hdl);
  if (it_data != data_map.end()) {
    msg_data = it_data->second;
    if(msg_data->msg["is_eof"]){
      lock.unlock();
      return;
    }
  } else {
    lock.unlock();
    return;
@@ -363,7 +378,9 @@
      }catch (std::exception const &e)
      {
        LOG(ERROR)<<e.what();
        break;
        msg_data->msg["is_eof"]=true;
        guard_decoder.unlock();
        return;
      }

      if (jsonresult.contains("wav_name")) {
@@ -387,25 +404,22 @@
            nlohmann::json json_fst_hws;
            try{
              json_fst_hws = nlohmann::json::parse(json_string);
              if(json_fst_hws.type() == nlohmann::json::value_t::object){
                // fst
                try{
                  std::unordered_map<std::string, int> client_hws_map = json_fst_hws;
                  merged_hws_map.insert(client_hws_map.begin(), client_hws_map.end());
                } catch (const std::exception& e) {
                  LOG(INFO) << e.what();
                }
              }
            } catch (std::exception const &e)
            {
              LOG(ERROR)<<e.what();
              break;
            }
            
            if(json_fst_hws.type() == nlohmann::json::value_t::object){
              // fst
              try{
                std::unordered_map<std::string, int> client_hws_map = json_fst_hws;
                merged_hws_map.insert(client_hws_map.begin(), client_hws_map.end());
              } catch (const std::exception& e) {
                LOG(INFO) << e.what();
              }
            }else{
              // nn
              std::string client_nn_hws = jsonresult["hotwords"];
              nn_hotwords += " " + client_nn_hws;
              LOG(INFO) << "nn hotwords: " << client_nn_hws;
              // LOG(INFO) << "nn hotwords: " << client_nn_hws;
            }
          }
        }
@@ -448,8 +462,10 @@
      }
      LOG(INFO) << "jsonresult=" << jsonresult
                << ", msg_data->msg=" << msg_data->msg;
      if (jsonresult["is_speaking"] == false ||
          jsonresult["is_finished"] == true) {
      if ((jsonresult["is_speaking"] == false ||
          jsonresult["is_finished"] == true) && 
          msg_data->msg["is_eof"] != true &&
          msg_data->hotwords_embedding != NULL) {
        LOG(INFO) << "client done";

        // if it is in final message, post the sample_data to decode
@@ -500,21 +516,23 @@

          try{
            // post to decode
            std::vector<std::vector<float>> hotwords_embedding_(*(msg_data->hotwords_embedding));
            msg_data->strand_->post( 
                      std::bind(&WebSocketServer::do_decoder, this,
                                std::move(subvector), std::move(hdl),
                                std::ref(msg_data->msg),
                                std::ref(*(punc_cache_p.get())),
                                std::move(hotwords_embedding_),
                                std::ref(*thread_lock_p), std::move(false),
                                msg_data->msg["wav_name"],
                                msg_data->msg["mode"],
                                msg_data->msg["itn"],
                                msg_data->msg["audio_fs"],
                                msg_data->msg["wav_format"],
                                std::ref(msg_data->tpass_online_handle)));
            msg_data->msg["access_num"]=(int)(msg_data->msg["access_num"])+1;
            if (msg_data->msg["is_eof"] != true && msg_data->hotwords_embedding != NULL) {
              std::vector<std::vector<float>> hotwords_embedding_(*(msg_data->hotwords_embedding));
              msg_data->strand_->post(
                        std::bind(&WebSocketServer::do_decoder, this,
                                  std::move(subvector), std::move(hdl),
                                  std::ref(msg_data->msg),
                                  std::ref(*(punc_cache_p.get())),
                                  std::move(hotwords_embedding_),
                                  std::ref(*thread_lock_p), std::move(false),
                                  msg_data->msg["wav_name"],
                                  msg_data->msg["mode"],
                                  msg_data->msg["itn"],
                                  msg_data->msg["audio_fs"],
                                  msg_data->msg["wav_format"],
                                  std::ref(msg_data->tpass_online_handle)));
              msg_data->msg["access_num"]=(int)(msg_data->msg["access_num"])+1;
            }
          }
          catch (std::exception const &e)
          {