python/FunASR-XL.git

			@@ -35,6 +35,7 @@
			def autocast(enabled=True):
			yield


			@tables.register("model_classes", "SCAMA")
			class SCAMA(nn.Module):
			"""
			@@ -97,9 +98,7 @@
			if ctc_conf is None:
			ctc_conf = {}

			ctc = CTC(
			odim=vocab_size, encoder_output_size=encoder_output_size, **ctc_conf
			)
			ctc = CTC(odim=vocab_size, encoder_output_size=encoder_output_size, **ctc_conf)

			predictor_class = tables.predictor_classes.get(predictor)
			predictor = predictor_class(**predictor_conf)
			@@ -116,7 +115,6 @@
			self.normalize = normalize

			self.encoder = encoder


			if ctc_weight == 1.0:
			self.decoder = None
			@@ -150,8 +148,13 @@
			self.error_calculator = None

			if self.encoder.overlap_chunk_cls is not None:
			from funasr.models.scama.chunk_utilis import build_scama_mask_for_cross_attention_decoder
			self.build_scama_mask_for_cross_attention_decoder_fn = build_scama_mask_for_cross_attention_decoder
			from funasr.models.scama.chunk_utilis import (
			build_scama_mask_for_cross_attention_decoder,
			)

			self.build_scama_mask_for_cross_attention_decoder_fn = (
			build_scama_mask_for_cross_attention_decoder
			)
			self.decoder_attention_chunk_type = kwargs.get("decoder_attention_chunk_type", "chunk")

			def forward(
			@@ -182,7 +185,6 @@
			ind = self.encoder.overlap_chunk_cls.random_choice(self.training, decoding_ind)
			encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind)


			loss_ctc, cer_ctc = None, None
			loss_pre = None
			stats = dict()
			@@ -191,10 +193,9 @@

			if self.ctc_weight > 0.0:

			encoder_out_ctc, encoder_out_lens_ctc = self.encoder.overlap_chunk_cls.remove_chunk(encoder_out,
			encoder_out_lens,
			chunk_outs=None)

			encoder_out_ctc, encoder_out_lens_ctc = self.encoder.overlap_chunk_cls.remove_chunk(
			encoder_out, encoder_out_lens, chunk_outs=None
			)

			loss_ctc, cer_ctc = self._calc_ctc_loss(
			encoder_out_ctc, encoder_out_lens_ctc, text, text_lengths
			@@ -212,8 +213,11 @@
			if self.ctc_weight == 0.0:
			loss = loss_att + loss_pre * self.predictor_weight
			else:
			loss = self.ctc_weight * loss_ctc + (
			1 - self.ctc_weight) * loss_att + loss_pre * self.predictor_weight
			loss = (
			self.ctc_weight * loss_ctc
			+ (1 - self.ctc_weight) * loss_att
			+ loss_pre * self.predictor_weight
			)

			# Collect Attn branch stats
			stats["loss_att"] = loss_att.detach() if loss_att is not None else None
			@@ -231,7 +235,10 @@
			return loss, stats, weight

			def encode(
			self, speech: torch.Tensor, speech_lengths: torch.Tensor, **kwargs,
			self,
			speech: torch.Tensor,
			speech_lengths: torch.Tensor,
			**kwargs,
			) -> Tuple[torch.Tensor, torch.Tensor]:
			"""Encoder. Note that this method is used by asr_inference.py
			Args:
			@@ -257,7 +264,11 @@
			return encoder_out, encoder_out_lens

			def encode_chunk(
			self, speech: torch.Tensor, speech_lengths: torch.Tensor, cache: dict = None, **kwargs,
			self,
			speech: torch.Tensor,
			speech_lengths: torch.Tensor,
			cache: dict = None,
			**kwargs,
			) -> Tuple[torch.Tensor, torch.Tensor]:
			"""Frontend + Encoder. Note that this method is used by asr_inference.py
			Args:
			@@ -276,7 +287,9 @@
			speech, speech_lengths = self.normalize(speech, speech_lengths)

			# Forward encoder
			encoder_out, encoder_out_lens, _ = self.encoder.forward_chunk(speech, speech_lengths, cache=cache["encoder"])
			encoder_out, encoder_out_lens, _ = self.encoder.forward_chunk(
			speech, speech_lengths, cache=cache["encoder"]
			)
			if isinstance(encoder_out, tuple):
			encoder_out = encoder_out[0]

			@@ -297,36 +310,44 @@
			ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
			ys_in_lens = ys_pad_lens + 1

			encoder_out_mask = sequence_mask(encoder_out_lens, maxlen=encoder_out.size(1), dtype=encoder_out.dtype,
			device=encoder_out.device)[:, None, :]
			encoder_out_mask = sequence_mask(
			encoder_out_lens,
			maxlen=encoder_out.size(1),
			dtype=encoder_out.dtype,
			device=encoder_out.device,
			)[:, None, :]
			mask_chunk_predictor = None
			if self.encoder.overlap_chunk_cls is not None:
			mask_chunk_predictor = self.encoder.overlap_chunk_cls.get_mask_chunk_predictor(None,
			device=encoder_out.device,
			batch_size=encoder_out.size(
			0))
			mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk(None, device=encoder_out.device,
			batch_size=encoder_out.size(0))
			mask_chunk_predictor = self.encoder.overlap_chunk_cls.get_mask_chunk_predictor(
			None, device=encoder_out.device, batch_size=encoder_out.size(0)
			)
			mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk(
			None, device=encoder_out.device, batch_size=encoder_out.size(0)
			)
			encoder_out = encoder_out * mask_shfit_chunk
			pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor(encoder_out,
			pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor(
			encoder_out,
			ys_out_pad,
			encoder_out_mask,
			ignore_id=self.ignore_id,
			mask_chunk_predictor=mask_chunk_predictor,
			target_label_length=ys_in_lens,
			)
			predictor_alignments, predictor_alignments_len = self.predictor.gen_frame_alignments(pre_alphas,
			encoder_out_lens)

			predictor_alignments, predictor_alignments_len = self.predictor.gen_frame_alignments(
			pre_alphas, encoder_out_lens
			)

			encoder_chunk_size = self.encoder.overlap_chunk_cls.chunk_size_pad_shift_cur
			attention_chunk_center_bias = 0
			attention_chunk_size = encoder_chunk_size
			decoder_att_look_back_factor = self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur
			mask_shift_att_chunk_decoder = self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder(None,
			device=encoder_out.device,
			batch_size=encoder_out.size(
			0))
			decoder_att_look_back_factor = (
			self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur
			)
			mask_shift_att_chunk_decoder = (
			self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder(
			None, device=encoder_out.device, batch_size=encoder_out.size(0)
			)
			)
			scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn(
			predictor_alignments=predictor_alignments,
			encoder_sequence_length=encoder_out_lens,
			@@ -343,7 +364,6 @@
			is_training=self.training,
			)


			# try:
			# 1. Forward decoder
			decoder_out, _ = self.decoder(
			@@ -353,7 +373,6 @@
			ys_in_lens,
			chunk_mask=scama_mask,
			pre_acoustic_embeds=pre_acoustic_embeds,

			)

			# 2. Compute attention loss
			@@ -385,36 +404,44 @@
			# ys_in_lens = ys_pad_lens + 1
			ys_out_pad, ys_in_lens = None, None

			encoder_out_mask = sequence_mask(encoder_out_lens, maxlen=encoder_out.size(1), dtype=encoder_out.dtype,
			device=encoder_out.device)[:, None, :]
			encoder_out_mask = sequence_mask(
			encoder_out_lens,
			maxlen=encoder_out.size(1),
			dtype=encoder_out.dtype,
			device=encoder_out.device,
			)[:, None, :]
			mask_chunk_predictor = None

			mask_chunk_predictor = self.encoder.overlap_chunk_cls.get_mask_chunk_predictor(None,
			device=encoder_out.device,
			batch_size=encoder_out.size(
			0))
			mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk(None, device=encoder_out.device,
			batch_size=encoder_out.size(0))
			mask_chunk_predictor = self.encoder.overlap_chunk_cls.get_mask_chunk_predictor(
			None, device=encoder_out.device, batch_size=encoder_out.size(0)
			)
			mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk(
			None, device=encoder_out.device, batch_size=encoder_out.size(0)
			)
			encoder_out = encoder_out * mask_shfit_chunk
			pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor(encoder_out,
			pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor(
			encoder_out,
			ys_out_pad,
			encoder_out_mask,
			ignore_id=self.ignore_id,
			mask_chunk_predictor=mask_chunk_predictor,
			target_label_length=ys_in_lens,
			)
			predictor_alignments, predictor_alignments_len = self.predictor.gen_frame_alignments(pre_alphas,
			encoder_out_lens)

			predictor_alignments, predictor_alignments_len = self.predictor.gen_frame_alignments(
			pre_alphas, encoder_out_lens
			)

			encoder_chunk_size = self.encoder.overlap_chunk_cls.chunk_size_pad_shift_cur
			attention_chunk_center_bias = 0
			attention_chunk_size = encoder_chunk_size
			decoder_att_look_back_factor = self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur
			mask_shift_att_chunk_decoder = self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder(None,
			device=encoder_out.device,
			batch_size=encoder_out.size(
			0))
			decoder_att_look_back_factor = (
			self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur
			)
			mask_shift_att_chunk_decoder = (
			self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder(
			None, device=encoder_out.device, batch_size=encoder_out.size(0)
			)
			)
			scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn(
			predictor_alignments=predictor_alignments,
			encoder_sequence_length=encoder_out_lens,
			@@ -431,14 +458,20 @@
			is_training=self.training,
			)

			return pre_acoustic_embeds, pre_token_length, predictor_alignments, predictor_alignments_len, scama_mask
			return (
			pre_acoustic_embeds,
			pre_token_length,
			predictor_alignments,
			predictor_alignments_len,
			scama_mask,
			)

			def init_beam_search(self,
			def init_beam_search(
			self,
			**kwargs,
			):

			from funasr.models.scama.beam_search import BeamSearchScamaStreaming


			from funasr.models.transformer.scorers.ctc import CTCPrefixScorer
			from funasr.models.transformer.scorers.length_bonus import LengthBonus
			@@ -448,9 +481,7 @@

			if self.ctc != None:
			ctc = CTCPrefixScorer(ctc=self.ctc, eos=self.eos)
			scorers.update(
			ctc=ctc
			)
			scorers.update(ctc=ctc)
			token_list = kwargs.get("token_list")
			scorers.update(
			decoder=self.decoder,
			@@ -486,7 +517,8 @@
			# scorer.to(device=kwargs.get("device", "cpu"), dtype=getattr(torch, kwargs.get("dtype", "float32"))).eval()
			self.beam_search = beam_search

			def generate_chunk(self,
			def generate_chunk(
			self,
			speech,
			speech_lengths=None,
			key: list = None,
			@@ -499,25 +531,29 @@
			speech_lengths = speech_lengths.to(device=kwargs["device"])

			# Encoder
			encoder_out, encoder_out_lens = self.encode_chunk(speech, speech_lengths, cache=cache,
			is_final=kwargs.get("is_final", False))
			encoder_out, encoder_out_lens = self.encode_chunk(
			speech, speech_lengths, cache=cache, is_final=kwargs.get("is_final", False)
			)
			if isinstance(encoder_out, tuple):
			encoder_out = encoder_out[0]
			if "running_hyps" not in cache:
			running_hyps = self.beam_search.init_hyp(encoder_out)
			cache["running_hyps"] = running_hyps


			# predictor
			predictor_outs = self.calc_predictor_chunk(encoder_out,
			predictor_outs = self.calc_predictor_chunk(
			encoder_out,
			encoder_out_lens,
			cache=cache,
			is_final=kwargs.get("is_final", False),
			)
			pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = predictor_outs[0], predictor_outs[1], \
			predictor_outs[2], predictor_outs[3]
			pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = (
			predictor_outs[0],
			predictor_outs[1],
			predictor_outs[2],
			predictor_outs[3],
			)
			pre_token_length = pre_token_length.round().long()


			if torch.max(pre_token_length) < 1:
			return []
			@@ -527,7 +563,12 @@
			minlen = max(0, minlen - kwargs.get("token_num_relax", 5))
			# c. Passed the encoder result and the beam search
			nbest_hyps = self.beam_search(
			x=encoder_out[0], scama_mask=None, pre_acoustic_embeds=pre_acoustic_embeds, maxlen=int(maxlen), minlen=int(minlen), cache=cache,
			x=encoder_out[0],
			scama_mask=None,
			pre_acoustic_embeds=pre_acoustic_embeds,
			maxlen=int(maxlen),
			minlen=int(minlen),
			cache=cache,
			)

			cache["running_hyps"] = nbest_hyps
			@@ -544,9 +585,12 @@
			else:
			token_int = hyp.yseq[1:last_pos].tolist()


			# remove blank symbol id, which is assumed to be 0
			token_int = list(filter(lambda x: x != self.eos and x != self.sos and x != self.blank_id, token_int))
			token_int = list(
			filter(
			lambda x: x != self.eos and x != self.sos and x != self.blank_id, token_int
			)
			)

			# Change integer-ids to tokens
			token = tokenizer.ids2tokens(token_int)
			@@ -569,24 +613,36 @@
			enc_output_size = kwargs["encoder_conf"]["output_size"]
			feats_dims = kwargs["frontend_conf"]["n_mels"] * kwargs["frontend_conf"]["lfr_m"]

			cache_encoder = {"start_idx": 0, "cif_hidden": torch.zeros((batch_size, 1, enc_output_size)).to(device=device),
			"cif_alphas": torch.zeros((batch_size, 1)).to(device=device), "chunk_size": chunk_size,
			"encoder_chunk_look_back": encoder_chunk_look_back, "last_chunk": False, "opt": None,
			"feats": torch.zeros((batch_size, chunk_size[0] + chunk_size[2], feats_dims)).to(device=device),
			"tail_chunk": False}
			cache_encoder = {
			"start_idx": 0,
			"cif_hidden": torch.zeros((batch_size, 1, enc_output_size)).to(device=device),
			"cif_alphas": torch.zeros((batch_size, 1)).to(device=device),
			"chunk_size": chunk_size,
			"encoder_chunk_look_back": encoder_chunk_look_back,
			"last_chunk": False,
			"opt": None,
			"feats": torch.zeros((batch_size, chunk_size[0] + chunk_size[2], feats_dims)).to(
			device=device
			),
			"tail_chunk": False,
			}
			cache["encoder"] = cache_encoder

			cache_decoder = {"decode_fsmn": None, "decoder_chunk_look_back": decoder_chunk_look_back, "opt": None,
			"chunk_size": chunk_size}
			cache_decoder = {
			"decode_fsmn": None,
			"decoder_chunk_look_back": decoder_chunk_look_back,
			"opt": None,
			"chunk_size": chunk_size,
			}
			cache["decoder"] = cache_decoder
			cache["frontend"] = {}


			cache["prev_samples"] = torch.empty(0).to(device=device)

			return cache

			def inference(self,
			def inference(
			self,
			data_in,
			data_lengths=None,
			key: list = None,
			@@ -598,10 +654,11 @@

			# init beamsearch
			is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None
			is_use_lm = kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None
			is_use_lm = (
			kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None
			)

			if self.beam_search is None:


			logging.info("enable beam_search")
			self.init_beam_search(**kwargs)
			@@ -616,7 +673,8 @@

			time1 = time.perf_counter()
			cfg = {"is_final": kwargs.get("is_final", False)}
			audio_sample_list = load_audio_text_image_video(data_in,
			audio_sample_list = load_audio_text_image_video(
			data_in,
			fs=frontend.fs,
			audio_fs=kwargs.get("fs", 16000),
			data_type=kwargs.get("data_type", "sound"),
			@@ -639,15 +697,28 @@
			audio_sample_i = audio_sample[i * chunk_stride_samples:(i + 1) * chunk_stride_samples]

			# extract fbank feats
			speech, speech_lengths = extract_fbank([audio_sample_i], data_type=kwargs.get("data_type", "sound"),
			frontend=frontend, cache=cache["frontend"],
			is_final=kwargs["is_final"])
			speech, speech_lengths = extract_fbank(
			[audio_sample_i],
			data_type=kwargs.get("data_type", "sound"),
			frontend=frontend,
			cache=cache["frontend"],
			is_final=kwargs["is_final"],
			)
			time3 = time.perf_counter()
			meta_data["extract_feat"] = f"{time3 - time2:0.3f}"
			meta_data["batch_data_time"] = speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000
			meta_data["batch_data_time"] = (
			speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000
			)

			tokens_i = self.generate_chunk(speech, speech_lengths, key=key, tokenizer=tokenizer, cache=cache,
			frontend=frontend, **kwargs)
			tokens_i = self.generate_chunk(
			speech,
			speech_lengths,
			key=key,
			tokenizer=tokenizer,
			cache=cache,
			frontend=frontend,
			**kwargs,
			)
			tokens.extend(tokens_i)

			text_postprocessed, _ = postprocess_utils.sentence_postprocess(tokens)