python/FunASR-XL.git

			@@ -42,6 +42,8 @@
			from funasr.models.frontend.wav_frontend import WavFrontend
			from funasr.models.e2e_asr_paraformer import BiCifParaformer, ContextualParaformer
			from funasr.export.models.e2e_asr_paraformer import Paraformer as Paraformer_export
			from funasr.utils.timestamp_tools import ts_prediction_lfr6_standard
			from funasr.bin.tp_inference import SpeechText2Timestamp


			class Speech2Text:
			@@ -49,7 +51,7 @@

			Examples:
			>>> import soundfile
			>>> speech2text = Speech2Text("asr_config.yml", "asr.pth")
			>>> speech2text = Speech2Text("asr_config.yml", "asr.pb")
			>>> audio, rate = soundfile.read("speech.wav")
			>>> speech2text(audio)
			[(text, token, token_int, hypothesis object), ...]
			@@ -190,7 +192,8 @@

			@torch.no_grad()
			def __call__(
			self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None
			self, speech: Union[torch.Tensor, np.ndarray], speech_lengths: Union[torch.Tensor, np.ndarray] = None,
			begin_time: int = 0, end_time: int = None,
			):
			"""Inference

			@@ -242,6 +245,10 @@
			decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds, pre_token_length, hw_list=self.hotword_list)
			decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]

			if isinstance(self.asr_model, BiCifParaformer):
			_, _, us_alphas, us_peaks = self.asr_model.calc_predictor_timestamp(enc, enc_len,
			pre_token_length) # test no bias cif2

			results = []
			b, n, d = decoder_out.size()
			for i in range(b):
			@@ -284,7 +291,14 @@
			else:
			text = None

			results.append((text, token, token_int, hyp, enc_len_batch_total, lfr_factor))
			if isinstance(self.asr_model, BiCifParaformer):
			_, timestamp = ts_prediction_lfr6_standard(us_alphas[i],
			us_peaks[i],
			copy.copy(token),
			vad_offset=begin_time)
			results.append((text, token, token_int, hyp, timestamp, enc_len_batch_total, lfr_factor))
			else:
			results.append((text, token, token_int, hyp, enc_len_batch_total, lfr_factor))

			# assert check_return_type(results)
			return results
			@@ -527,7 +541,8 @@
			ngram_weight: float = 0.9,
			nbest: int = 1,
			num_workers: int = 1,

			timestamp_infer_config: Union[Path, str] = None,
			timestamp_model_file: Union[Path, str] = None,
			**kwargs,
			):
			inference_pipeline = inference_modelscope(
			@@ -591,6 +606,8 @@
			nbest: int = 1,
			num_workers: int = 1,
			output_dir: Optional[str] = None,
			timestamp_infer_config: Union[Path, str] = None,
			timestamp_model_file: Union[Path, str] = None,
			param_dict: dict = None,
			**kwargs,
			):
			@@ -648,6 +665,15 @@
			else:
			speech2text = Speech2Text(**speech2text_kwargs)

			if timestamp_model_file is not None:
			speechtext2timestamp = SpeechText2Timestamp(
			timestamp_cmvn_file=cmvn_file,
			timestamp_model_file=timestamp_model_file,
			timestamp_infer_config=timestamp_infer_config,
			)
			else:
			speechtext2timestamp = None

			def _forward(
			data_path_and_name_and_type,
			raw_inputs: Union[np.ndarray, torch.Tensor] = None,
			@@ -660,11 +686,9 @@
			hotword_list_or_file = None
			if param_dict is not None:
			hotword_list_or_file = param_dict.get('hotword')

			if 'hotword' in kwargs:
			hotword_list_or_file = kwargs['hotword']

			if speech2text.hotword_list is None:
			if hotword_list_or_file is not None or 'hotword' in kwargs:
			speech2text.hotword_list = speech2text.generate_hotwords_list(hotword_list_or_file)

			# 3. Build data-iterator
			@@ -684,6 +708,11 @@
			allow_variable_data_keys=allow_variable_data_keys,
			inference=True,
			)

			if param_dict is not None:
			use_timestamp = param_dict.get('use_timestamp', True)
			else:
			use_timestamp = True

			forward_time_total = 0.0
			length_total = 0.0
			@@ -726,7 +755,19 @@
			result = [results[batch_id][:-2]]

			key = keys[batch_id]
			for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), result):
			for n, result in zip(range(1, nbest + 1), result):
			text, token, token_int, hyp = result[0], result[1], result[2], result[3]
			timestamp = None if len(result) < 5 else result[4]
			# conduct timestamp prediction here
			# timestamp inference requires token length
			# thus following inference cannot be conducted in batch
			if timestamp is None and speechtext2timestamp:
			ts_batch = {}
			ts_batch['speech'] = batch['speech'][batch_id].unsqueeze(0)
			ts_batch['speech_lengths'] = torch.tensor([batch['speech_lengths'][batch_id]])
			ts_batch['text_lengths'] = torch.tensor([len(token)])
			us_alphas, us_peaks = speechtext2timestamp(**ts_batch)
			ts_str, timestamp = ts_prediction_lfr6_standard(us_alphas[0], us_peaks[0], token, force_time_shift=-3.0)
			# Create a directory: outdir/{n}best_recog
			if writer is not None:
			ibest_writer = writer[f"{n}best_recog"]
			@@ -738,13 +779,25 @@
			ibest_writer["rtf"][key] = rtf_cur

			if text is not None:
			text_postprocessed, _ = postprocess_utils.sentence_postprocess(token)
			if use_timestamp and timestamp is not None:
			postprocessed_result = postprocess_utils.sentence_postprocess(token, timestamp)
			else:
			postprocessed_result = postprocess_utils.sentence_postprocess(token)
			timestamp_postprocessed = ""
			if len(postprocessed_result) == 3:
			text_postprocessed, timestamp_postprocessed, word_lists = postprocessed_result[0], \
			postprocessed_result[1], \
			postprocessed_result[2]
			else:
			text_postprocessed, word_lists = postprocessed_result[0], postprocessed_result[1]
			item = {'key': key, 'value': text_postprocessed}
			if timestamp_postprocessed != "":
			item['timestamp'] = timestamp_postprocessed
			asr_result_list.append(item)
			finish_count += 1
			# asr_utils.print_progress(finish_count / file_count)
			if writer is not None:
			ibest_writer["text"][key] = text_postprocessed
			ibest_writer["text"][key] = " ".join(word_lists)

			logging.info("decoding, utt: {}, predictions: {}".format(key, text))
			rtf_avg = "decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}".format(length_total, forward_time_total, 100 * forward_time_total / (length_total * lfr_factor))