Dev gzf exp (#1654)

zhifu gao

2024-04-24 861147c7308b91068ffa02724fdf74ee623a909e

 funasr/models/fsmn_vad_streaming/model.py

@@ -162,10 +162,13 @@
   https://arxiv.org/abs/1803.05030
   """
   
   def __init__(self, window_size_ms: int,
    def __init__(
        self,
        window_size_ms: int,
                sil_to_speech_time: int,
                speech_to_sil_time: int,
                frame_size_ms: int):
        frame_size_ms: int,
    ):
      self.window_size_ms = window_size_ms
      self.sil_to_speech_time = sil_to_speech_time
      self.speech_to_sil_time = speech_to_sil_time
@@ -198,7 +201,9 @@
   def GetWinSize(self) -> int:
      return int(self.win_size_frame)
   
   def DetectOneFrame(self, frameState: FrameState, frame_count: int, cache: dict = {}) -> AudioChangeState:
    def DetectOneFrame(
        self, frameState: FrameState, frame_count: int, cache: dict = {}
    ) -> AudioChangeState:
      cur_frame_state = FrameState.kFrameStateSil
      if frameState == FrameState.kFrameStateSpeech:
         cur_frame_state = 1
@@ -211,11 +216,17 @@
      self.win_state[self.cur_win_pos] = cur_frame_state
      self.cur_win_pos = (self.cur_win_pos + 1) % self.win_size_frame
      
      if self.pre_frame_state == FrameState.kFrameStateSil and self.win_sum >= self.sil_to_speech_frmcnt_thres:
        if (
            self.pre_frame_state == FrameState.kFrameStateSil
            and self.win_sum >= self.sil_to_speech_frmcnt_thres
        ):
         self.pre_frame_state = FrameState.kFrameStateSpeech
         return AudioChangeState.kChangeStateSil2Speech
      
      if self.pre_frame_state == FrameState.kFrameStateSpeech and self.win_sum <= self.speech_to_sil_frmcnt_thres:
        if (
            self.pre_frame_state == FrameState.kFrameStateSpeech
            and self.win_sum <= self.speech_to_sil_frmcnt_thres
        ):
         self.pre_frame_state = FrameState.kFrameStateSil
         return AudioChangeState.kChangeStateSpeech2Sil
      
@@ -230,7 +241,8 @@


class Stats(object):
   def __init__(self,
    def __init__(
        self,
                sil_pdf_ids,
                max_end_sil_frame_cnt_thresh,
                speech_noise_thres,
@@ -272,7 +284,8 @@
   https://arxiv.org/abs/1803.05030
   """
   
   def __init__(self,
    def __init__(
        self,
                encoder: str = None,
                encoder_conf: Optional[Dict] = None,
                vad_post_args: Dict[str, Any] = None,
@@ -302,8 +315,10 @@
         drop_frames = int(cache["stats"].output_data_buf[-1].end_ms / self.vad_opts.frame_in_ms)
         real_drop_frames = drop_frames - cache["stats"].last_drop_frames
         cache["stats"].last_drop_frames = drop_frames
         cache["stats"].data_buf_all = cache["stats"].data_buf_all[real_drop_frames * int(
            self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000):]
            cache["stats"].data_buf_all = cache["stats"].data_buf_all[
                real_drop_frames
                * int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) :
            ]
         cache["stats"].decibel = cache["stats"].decibel[real_drop_frames:]
         cache["stats"].scores = cache["stats"].scores[:, real_drop_frames:, :]
   
@@ -312,18 +327,31 @@
      frame_shift_length = int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000)
      if cache["stats"].data_buf_all is None:
         cache["stats"].data_buf_all = cache["stats"].waveform[
            0]  # cache["stats"].data_buf is pointed to cache["stats"].waveform[0]
                0
            ]  # cache["stats"].data_buf is pointed to cache["stats"].waveform[0]
         cache["stats"].data_buf = cache["stats"].data_buf_all
      else:
         cache["stats"].data_buf_all = torch.cat((cache["stats"].data_buf_all, cache["stats"].waveform[0]))
      for offset in range(0, cache["stats"].waveform.shape[1] - frame_sample_length + 1, frame_shift_length):
            cache["stats"].data_buf_all = torch.cat(
                (cache["stats"].data_buf_all, cache["stats"].waveform[0])
            )
        for offset in range(
            0, cache["stats"].waveform.shape[1] - frame_sample_length + 1, frame_shift_length
        ):
         cache["stats"].decibel.append(
            10 * math.log10((cache["stats"].waveform[0][offset: offset + frame_sample_length]).square().sum() + \
                            0.000001))
                10
                * math.log10(
                    (cache["stats"].waveform[0][offset : offset + frame_sample_length])
                    .square()
                    .sum()
                    + 0.000001
                )
            )
   
   def ComputeScores(self, feats: torch.Tensor, cache: dict = {}) -> None:
      scores = self.encoder(feats, cache=cache["encoder"]).to('cpu')  # return B * T * D
      assert scores.shape[1] == feats.shape[1], "The shape between feats and scores does not match"
        scores = self.encoder(feats, cache=cache["encoder"]).to("cpu")  # return B * T * D
        assert (
            scores.shape[1] == feats.shape[1]
        ), "The shape between feats and scores does not match"
      self.vad_opts.nn_eval_block_size = scores.shape[1]
      cache["stats"].frm_cnt += scores.shape[1]  # count total frames
      if cache["stats"].scores is None:
@@ -333,24 +361,41 @@
   
   def PopDataBufTillFrame(self, frame_idx: int, cache: dict = {}) -> None:  # need check again
      while cache["stats"].data_buf_start_frame < frame_idx:
         if len(cache["stats"].data_buf) >= int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000):
            if len(cache["stats"].data_buf) >= int(
                self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000
            ):
            cache["stats"].data_buf_start_frame += 1
            cache["stats"].data_buf = cache["stats"].data_buf_all[
                                      (cache["stats"].data_buf_start_frame - cache["stats"].last_drop_frames) * int(
                                         self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000):]
                    (cache["stats"].data_buf_start_frame - cache["stats"].last_drop_frames)
                    * int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) :
                ]
   
   def PopDataToOutputBuf(self, start_frm: int, frm_cnt: int, first_frm_is_start_point: bool,
                          last_frm_is_end_point: bool, end_point_is_sent_end: bool, cache: dict = {}) -> None:
    def PopDataToOutputBuf(
        self,
        start_frm: int,
        frm_cnt: int,
        first_frm_is_start_point: bool,
        last_frm_is_end_point: bool,
        end_point_is_sent_end: bool,
        cache: dict = {},
    ) -> None:
      self.PopDataBufTillFrame(start_frm, cache=cache)
      expected_sample_number = int(frm_cnt * self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000)
        expected_sample_number = int(
            frm_cnt * self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000
        )
      if last_frm_is_end_point:
         extra_sample = max(0, int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000 - \
                                   self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000))
            extra_sample = max(
                0,
                int(
                    self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000
                    - self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000
                ),
            )
         expected_sample_number += int(extra_sample)
      if end_point_is_sent_end:
         expected_sample_number = max(expected_sample_number, len(cache["stats"].data_buf))
      if len(cache["stats"].data_buf) < expected_sample_number:
         print('error in calling pop data_buf\n')
            print("error in calling pop data_buf\n")
      
      if len(cache["stats"].output_data_buf) == 0 or first_frm_is_start_point:
         cache["stats"].output_data_buf.append(E2EVadSpeechBufWithDoa())
@@ -360,13 +405,15 @@
         cache["stats"].output_data_buf[-1].doa = 0
      cur_seg = cache["stats"].output_data_buf[-1]
      if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms:
         print('warning\n')
            print("warning\n")
      out_pos = len(cur_seg.buffer)  # cur_seg.buff现在没做任何操作
      data_to_pop = 0
      if end_point_is_sent_end:
         data_to_pop = expected_sample_number
      else:
         data_to_pop = int(frm_cnt * self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000)
            data_to_pop = int(
                frm_cnt * self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000
            )
      if data_to_pop > len(cache["stats"].data_buf):
         print('VAD data_to_pop is bigger than cache["stats"].data_buf.size()!!!\n')
         data_to_pop = len(cache["stats"].data_buf)
@@ -380,7 +427,7 @@
         # cur_seg.buffer[out_pos++] = data_buf_.back()
         out_pos += 1
      if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms:
         print('Something wrong with the VAD algorithm\n')
            print("Something wrong with the VAD algorithm\n")
      cache["stats"].data_buf_start_frame += frm_cnt
      cur_seg.end_ms = (start_frm + frm_cnt) * self.vad_opts.frame_in_ms
      if first_frm_is_start_point:
@@ -404,28 +451,39 @@
      if self.vad_opts.do_start_point_detection:
         pass
      if cache["stats"].confirmed_start_frame != -1:
         print('not reset vad properly\n')
            print("not reset vad properly\n")
      else:
         cache["stats"].confirmed_start_frame = start_frame
      
      if not fake_result and cache["stats"].vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected:
         self.PopDataToOutputBuf(cache["stats"].confirmed_start_frame, 1, True, False, False, cache=cache)
        if (
            not fake_result
            and cache["stats"].vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected
        ):
            self.PopDataToOutputBuf(
                cache["stats"].confirmed_start_frame, 1, True, False, False, cache=cache
            )
   
   def OnVoiceEnd(self, end_frame: int, fake_result: bool, is_last_frame: bool, cache: dict = {}) -> None:
    def OnVoiceEnd(
        self, end_frame: int, fake_result: bool, is_last_frame: bool, cache: dict = {}
    ) -> None:
      for t in range(cache["stats"].latest_confirmed_speech_frame + 1, end_frame):
         self.OnVoiceDetected(t, cache=cache)
      if self.vad_opts.do_end_point_detection:
         pass
      if cache["stats"].confirmed_end_frame != -1:
         print('not reset vad properly\n')
            print("not reset vad properly\n")
      else:
         cache["stats"].confirmed_end_frame = end_frame
      if not fake_result:
         cache["stats"].sil_frame = 0
         self.PopDataToOutputBuf(cache["stats"].confirmed_end_frame, 1, False, True, is_last_frame, cache=cache)
            self.PopDataToOutputBuf(
                cache["stats"].confirmed_end_frame, 1, False, True, is_last_frame, cache=cache
            )
      cache["stats"].number_end_time_detected += 1
   
   def MaybeOnVoiceEndIfLastFrame(self, is_final_frame: bool, cur_frm_idx: int, cache: dict = {}) -> None:
    def MaybeOnVoiceEndIfLastFrame(
        self, is_final_frame: bool, cur_frm_idx: int, cache: dict = {}
    ) -> None:
      if is_final_frame:
         self.OnVoiceEnd(cur_frm_idx, False, True, cache=cache)
         cache["stats"].vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
@@ -454,7 +512,9 @@
      assert len(cache["stats"].sil_pdf_ids) == self.vad_opts.silence_pdf_num
      if len(cache["stats"].sil_pdf_ids) > 0:
         assert len(cache["stats"].scores) == 1  # 只支持batch_size = 1的测试
         sil_pdf_scores = [cache["stats"].scores[0][t][sil_pdf_id] for sil_pdf_id in cache["stats"].sil_pdf_ids]
            sil_pdf_scores = [
                cache["stats"].scores[0][t][sil_pdf_id] for sil_pdf_id in cache["stats"].sil_pdf_ids
            ]
         sum_score = sum(sil_pdf_scores)
         noise_prob = math.log(sum_score) * self.vad_opts.speech_2_noise_ratio
         total_score = 1.0
@@ -477,13 +537,17 @@
         if cache["stats"].noise_average_decibel < -99.9:
            cache["stats"].noise_average_decibel = cur_decibel
         else:
            cache["stats"].noise_average_decibel = (cur_decibel + cache["stats"].noise_average_decibel * (
               self.vad_opts.noise_frame_num_used_for_snr
               - 1)) / self.vad_opts.noise_frame_num_used_for_snr
                cache["stats"].noise_average_decibel = (
                    cur_decibel
                    + cache["stats"].noise_average_decibel
                    * (self.vad_opts.noise_frame_num_used_for_snr - 1)
                ) / self.vad_opts.noise_frame_num_used_for_snr
      
      return frame_state
   
   def forward(self, feats: torch.Tensor,
    def forward(
        self,
        feats: torch.Tensor,
               waveform: torch.tensor,
               cache: dict = {},
               is_final: bool = False,
@@ -504,13 +568,24 @@
      for batch_num in range(0, feats.shape[0]):  # only support batch_size = 1 now
         segment_batch = []
         if len(cache["stats"].output_data_buf) > 0:
            for i in range(cache["stats"].output_data_buf_offset, len(cache["stats"].output_data_buf)):
               if is_streaming_input: # in this case, return [beg, -1], [], [-1, end], [beg, end]
                for i in range(
                    cache["stats"].output_data_buf_offset, len(cache["stats"].output_data_buf)
                ):
                    if (
                        is_streaming_input
                    ):  # in this case, return [beg, -1], [], [-1, end], [beg, end]
                  if not cache["stats"].output_data_buf[i].contain_seg_start_point:
                     continue
                  if not cache["stats"].next_seg and not cache["stats"].output_data_buf[i].contain_seg_end_point:
                        if (
                            not cache["stats"].next_seg
                            and not cache["stats"].output_data_buf[i].contain_seg_end_point
                        ):
                     continue
                  start_ms = cache["stats"].output_data_buf[i].start_ms if cache["stats"].next_seg else -1
                        start_ms = (
                            cache["stats"].output_data_buf[i].start_ms
                            if cache["stats"].next_seg
                            else -1
                        )
                  if cache["stats"].output_data_buf[i].contain_seg_end_point:
                     end_ms = cache["stats"].output_data_buf[i].end_ms
                     cache["stats"].next_seg = True
@@ -522,11 +597,15 @@
                  
               else: # in this case, return [beg, end]
                  
                  if not is_final and (not cache["stats"].output_data_buf[i].contain_seg_start_point or not
                  cache["stats"].output_data_buf[
                     i].contain_seg_end_point):
                        if not is_final and (
                            not cache["stats"].output_data_buf[i].contain_seg_start_point
                            or not cache["stats"].output_data_buf[i].contain_seg_end_point
                        ):
                     continue
                  segment = [cache["stats"].output_data_buf[i].start_ms, cache["stats"].output_data_buf[i].end_ms]
                        segment = [
                            cache["stats"].output_data_buf[i].start_ms,
                            cache["stats"].output_data_buf[i].end_ms,
                        ]
                  cache["stats"].output_data_buf_offset += 1  # need update this parameter
               
               segment_batch.append(segment)
@@ -548,21 +627,26 @@
         # update the max_end_silence_time
         self.vad_opts.max_end_silence_time = kwargs.get("max_end_silence_time")

      windows_detector = WindowDetector(self.vad_opts.window_size_ms,
        windows_detector = WindowDetector(
            self.vad_opts.window_size_ms,
                                        self.vad_opts.sil_to_speech_time_thres,
                                        self.vad_opts.speech_to_sil_time_thres,
                                        self.vad_opts.frame_in_ms)
            self.vad_opts.frame_in_ms,
        )
      windows_detector.Reset()
      
      stats = Stats(sil_pdf_ids=self.vad_opts.sil_pdf_ids,
                    max_end_sil_frame_cnt_thresh=self.vad_opts.max_end_silence_time - self.vad_opts.speech_to_sil_time_thres,
                    speech_noise_thres=self.vad_opts.speech_noise_thres
        stats = Stats(
            sil_pdf_ids=self.vad_opts.sil_pdf_ids,
            max_end_sil_frame_cnt_thresh=self.vad_opts.max_end_silence_time
            - self.vad_opts.speech_to_sil_time_thres,
            speech_noise_thres=self.vad_opts.speech_noise_thres,
                    )
      cache["windows_detector"] = windows_detector
      cache["stats"] = stats
      return cache
   
   def inference(self,
    def inference(
        self,
                 data_in,
                 data_lengths=None,
                 key: list = None,
@@ -580,10 +664,17 @@
      chunk_stride_samples = int(chunk_size * frontend.fs / 1000)
      
      time1 = time.perf_counter()
      is_streaming_input = kwargs.get("is_streaming_input", False) if chunk_size >= 15000 else kwargs.get("is_streaming_input", True)
      is_final = kwargs.get("is_final", False) if is_streaming_input else kwargs.get("is_final", True)
        is_streaming_input = (
            kwargs.get("is_streaming_input", False)
            if chunk_size >= 15000
            else kwargs.get("is_streaming_input", True)
        )
        is_final = (
            kwargs.get("is_final", False) if is_streaming_input else kwargs.get("is_final", True)
        )
      cfg = {"is_final": is_final, "is_streaming_input": is_streaming_input}
      audio_sample_list = load_audio_text_image_video(data_in,
        audio_sample_list = load_audio_text_image_video(
            data_in,
                                                      fs=frontend.fs,
                                                      audio_fs=kwargs.get("fs", 16000),
                                                      data_type=kwargs.get("data_type", "sound"),
@@ -606,12 +697,18 @@
         audio_sample_i = audio_sample[i * chunk_stride_samples:(i + 1) * chunk_stride_samples]
         
         # extract fbank feats
         speech, speech_lengths = extract_fbank([audio_sample_i], data_type=kwargs.get("data_type", "sound"),
                                                frontend=frontend, cache=cache["frontend"],
                                                is_final=kwargs["is_final"])
            speech, speech_lengths = extract_fbank(
                [audio_sample_i],
                data_type=kwargs.get("data_type", "sound"),
                frontend=frontend,
                cache=cache["frontend"],
                is_final=kwargs["is_final"],
            )
         time3 = time.perf_counter()
         meta_data["extract_feat"] = f"{time3 - time2:0.3f}"
         meta_data["batch_data_time"] = speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000
            meta_data["batch_data_time"] = (
                speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000
            )
         speech = speech.to(device=kwargs["device"])
         speech_lengths = speech_lengths.to(device=kwargs["device"])
         
@@ -620,7 +717,7 @@
            "waveform": cache["frontend"]["waveforms"],
            "is_final": kwargs["is_final"],
            "cache": cache,
            "is_streaming_input": is_streaming_input
                "is_streaming_input": is_streaming_input,
         }
         segments_i = self.forward(**batch)
         if len(segments_i) > 0:
@@ -651,6 +748,7 @@
   def export(self, **kwargs):

      from .export_meta import export_rebuild_model

      models = export_rebuild_model(model=self, **kwargs)
      return models

@@ -659,8 +757,9 @@
         return 0
      for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1):
         frame_state = FrameState.kFrameStateInvalid
         frame_state = self.GetFrameState(cache["stats"].frm_cnt - 1 - i - cache["stats"].last_drop_frames,
                                          cache=cache)
            frame_state = self.GetFrameState(
                cache["stats"].frm_cnt - 1 - i - cache["stats"].last_drop_frames, cache=cache
            )
         self.DetectOneFrame(frame_state, cache["stats"].frm_cnt - 1 - i, False, cache=cache)
      
      return 0
@@ -670,8 +769,9 @@
         return 0
      for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1):
         frame_state = FrameState.kFrameStateInvalid
         frame_state = self.GetFrameState(cache["stats"].frm_cnt - 1 - i - cache["stats"].last_drop_frames,
                                          cache=cache)
            frame_state = self.GetFrameState(
                cache["stats"].frm_cnt - 1 - i - cache["stats"].last_drop_frames, cache=cache
            )
         if i != 0:
            self.DetectOneFrame(frame_state, cache["stats"].frm_cnt - 1 - i, False, cache=cache)
         else:
@@ -679,8 +779,9 @@
      
      return 0
   
   def DetectOneFrame(self, cur_frm_state: FrameState, cur_frm_idx: int, is_final_frame: bool,
                      cache: dict = {}) -> None:
    def DetectOneFrame(
        self, cur_frm_state: FrameState, cur_frm_idx: int, is_final_frame: bool, cache: dict = {}
    ) -> None:
      tmp_cur_frm_state = FrameState.kFrameStateInvalid
      if cur_frm_state == FrameState.kFrameStateSpeech:
         if math.fabs(1.0) > self.vad_opts.fe_prior_thres:
@@ -689,7 +790,9 @@
            tmp_cur_frm_state = FrameState.kFrameStateSil
      elif cur_frm_state == FrameState.kFrameStateSil:
         tmp_cur_frm_state = FrameState.kFrameStateSil
      state_change = cache["windows_detector"].DetectOneFrame(tmp_cur_frm_state, cur_frm_idx, cache=cache)
        state_change = cache["windows_detector"].DetectOneFrame(
            tmp_cur_frm_state, cur_frm_idx, cache=cache
        )
      frm_shift_in_ms = self.vad_opts.frame_in_ms
      if AudioChangeState.kChangeStateSil2Speech == state_change:
         silence_frame_count = cache["stats"].continous_silence_frame_count
@@ -697,8 +800,10 @@
         cache["stats"].pre_end_silence_detected = False
         start_frame = 0
         if cache["stats"].vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected:
            start_frame = max(cache["stats"].data_buf_start_frame,
                              cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache))
                start_frame = max(
                    cache["stats"].data_buf_start_frame,
                    cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache),
                )
            self.OnVoiceStart(start_frame, cache=cache)
            cache["stats"].vad_state_machine = VadStateMachine.kVadInStateInSpeechSegment
            for t in range(start_frame + 1, cur_frm_idx + 1):
@@ -706,8 +811,10 @@
         elif cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment:
            for t in range(cache["stats"].latest_confirmed_speech_frame + 1, cur_frm_idx):
               self.OnVoiceDetected(t, cache=cache)
            if cur_frm_idx - cache["stats"].confirmed_start_frame + 1 > \
               self.vad_opts.max_single_segment_time / frm_shift_in_ms:
                if (
                    cur_frm_idx - cache["stats"].confirmed_start_frame + 1
                    > self.vad_opts.max_single_segment_time / frm_shift_in_ms
                ):
               self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache)
               cache["stats"].vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
            elif not is_final_frame:
@@ -721,8 +828,10 @@
         if cache["stats"].vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected:
            pass
         elif cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment:
            if cur_frm_idx - cache["stats"].confirmed_start_frame + 1 > \
               self.vad_opts.max_single_segment_time / frm_shift_in_ms:
                if (
                    cur_frm_idx - cache["stats"].confirmed_start_frame + 1
                    > self.vad_opts.max_single_segment_time / frm_shift_in_ms
                ):
               self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache)
               cache["stats"].vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
            elif not is_final_frame:
@@ -734,8 +843,10 @@
      elif AudioChangeState.kChangeStateSpeech2Speech == state_change:
         cache["stats"].continous_silence_frame_count = 0
         if cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment:
            if cur_frm_idx - cache["stats"].confirmed_start_frame + 1 > \
               self.vad_opts.max_single_segment_time / frm_shift_in_ms:
                if (
                    cur_frm_idx - cache["stats"].confirmed_start_frame + 1
                    > self.vad_opts.max_single_segment_time / frm_shift_in_ms
                ):
               cache["stats"].max_time_out = True
               self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache)
               cache["stats"].vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
@@ -749,10 +860,13 @@
         cache["stats"].continous_silence_frame_count += 1
         if cache["stats"].vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected:
            # silence timeout, return zero length decision
            if ((self.vad_opts.detect_mode == VadDetectMode.kVadSingleUtteranceDetectMode.value) and (
               cache[
                  "stats"].continous_silence_frame_count * frm_shift_in_ms > self.vad_opts.max_start_silence_time)) \
               or (is_final_frame and cache["stats"].number_end_time_detected == 0):
                if (
                    (self.vad_opts.detect_mode == VadDetectMode.kVadSingleUtteranceDetectMode.value)
                    and (
                        cache["stats"].continous_silence_frame_count * frm_shift_in_ms
                        > self.vad_opts.max_start_silence_time
                    )
                ) or (is_final_frame and cache["stats"].number_end_time_detected == 0):
               for t in range(cache["stats"].lastest_confirmed_silence_frame + 1, cur_frm_idx):
                  self.OnSilenceDetected(t, cache=cache)
               self.OnVoiceStart(0, True, cache=cache)
@@ -760,32 +874,43 @@
               cache["stats"].vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
            else:
               if cur_frm_idx >= self.LatencyFrmNumAtStartPoint(cache=cache):
                  self.OnSilenceDetected(cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), cache=cache)
                        self.OnSilenceDetected(
                            cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), cache=cache
                        )
         elif cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment:
            if cache["stats"].continous_silence_frame_count * frm_shift_in_ms >= cache[
               "stats"].max_end_sil_frame_cnt_thresh:
               lookback_frame = int(cache["stats"].max_end_sil_frame_cnt_thresh / frm_shift_in_ms)
                if (
                    cache["stats"].continous_silence_frame_count * frm_shift_in_ms
                    >= cache["stats"].max_end_sil_frame_cnt_thresh
                ):
                    lookback_frame = int(
                        cache["stats"].max_end_sil_frame_cnt_thresh / frm_shift_in_ms
                    )
               if self.vad_opts.do_extend:
                  lookback_frame -= int(self.vad_opts.lookahead_time_end_point / frm_shift_in_ms)
                        lookback_frame -= int(
                            self.vad_opts.lookahead_time_end_point / frm_shift_in_ms
                        )
                  lookback_frame -= 1
                  lookback_frame = max(0, lookback_frame)
               self.OnVoiceEnd(cur_frm_idx - lookback_frame, False, False, cache=cache)
               cache["stats"].vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
            elif cur_frm_idx - cache["stats"].confirmed_start_frame + 1 > \
               self.vad_opts.max_single_segment_time / frm_shift_in_ms:
                elif (
                    cur_frm_idx - cache["stats"].confirmed_start_frame + 1
                    > self.vad_opts.max_single_segment_time / frm_shift_in_ms
                ):
               self.OnVoiceEnd(cur_frm_idx, False, False, cache=cache)
               cache["stats"].vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
            elif self.vad_opts.do_extend and not is_final_frame:
               if cache["stats"].continous_silence_frame_count <= int(
                  self.vad_opts.lookahead_time_end_point / frm_shift_in_ms):
                        self.vad_opts.lookahead_time_end_point / frm_shift_in_ms
                    ):
                  self.OnVoiceDetected(cur_frm_idx, cache=cache)
            else:
               self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx, cache=cache)
         else:
            pass
      
      if cache["stats"].vad_state_machine == VadStateMachine.kVadInStateEndPointDetected and \
         self.vad_opts.detect_mode == VadDetectMode.kVadMutipleUtteranceDetectMode.value:
        if (
            cache["stats"].vad_state_machine == VadStateMachine.kVadInStateEndPointDetected
            and self.vad_opts.detect_mode == VadDetectMode.kVadMutipleUtteranceDetectMode.value
        ):
         self.ResetDetection(cache=cache)