python/FunASR-XL.git

			@@ -325,12 +325,67 @@

			return encoder_out, encoder_out_lens

			def encode_chunk(
			self, speech: torch.Tensor, speech_lengths: torch.Tensor, cache: dict = None
			) -> Tuple[torch.Tensor, torch.Tensor]:
			"""Frontend + Encoder. Note that this method is used by asr_inference.py

			Args:
			speech: (Batch, Length, ...)
			speech_lengths: (Batch, )
			"""
			with autocast(False):
			# 1. Extract feats
			feats, feats_lengths = self._extract_feats(speech, speech_lengths)

			# 2. Data augmentation
			if self.specaug is not None and self.training:
			feats, feats_lengths = self.specaug(feats, feats_lengths)

			# 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN
			if self.normalize is not None:
			feats, feats_lengths = self.normalize(feats, feats_lengths)

			# Pre-encoder, e.g. used for raw input data
			if self.preencoder is not None:
			feats, feats_lengths = self.preencoder(feats, feats_lengths)

			# 4. Forward encoder
			# feats: (Batch, Length, Dim)
			# -> encoder_out: (Batch, Length2, Dim2)
			if self.encoder.interctc_use_conditioning:
			encoder_out, encoder_out_lens, _ = self.encoder.forward_chunk(
			feats, feats_lengths, cache=cache["encoder"], ctc=self.ctc
			)
			else:
			encoder_out, encoder_out_lens, _ = self.encoder.forward_chunk(feats, feats_lengths, cache=cache["encoder"])
			intermediate_outs = None
			if isinstance(encoder_out, tuple):
			intermediate_outs = encoder_out[1]
			encoder_out = encoder_out[0]

			# Post-encoder, e.g. NLU
			if self.postencoder is not None:
			encoder_out, encoder_out_lens = self.postencoder(
			encoder_out, encoder_out_lens
			)

			if intermediate_outs is not None:
			return (encoder_out, intermediate_outs), encoder_out_lens

			return encoder_out, torch.tensor([encoder_out.size(1)])

			def calc_predictor(self, encoder_out, encoder_out_lens):

			encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(
			encoder_out.device)
			pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = self.predictor(encoder_out, None, encoder_out_mask,
			ignore_id=self.ignore_id)
			return pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index

			def calc_predictor_chunk(self, encoder_out, cache=None):

			pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index = self.predictor.forward_chunk(encoder_out, cache["encoder"])
			return pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index

			def cal_decoder_with_predictor(self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens):
			@@ -341,6 +396,14 @@
			decoder_out = decoder_outs[0]
			decoder_out = torch.log_softmax(decoder_out, dim=-1)
			return decoder_out, ys_pad_lens

			def cal_decoder_with_predictor_chunk(self, encoder_out, sematic_embeds, cache=None):
			decoder_outs = self.decoder.forward_chunk(
			encoder_out, sematic_embeds, cache["decoder"]
			)
			decoder_out = decoder_outs
			decoder_out = torch.log_softmax(decoder_out, dim=-1)
			return decoder_out

			def _extract_feats(
			self, speech: torch.Tensor, speech_lengths: torch.Tensor
			@@ -926,10 +989,10 @@
			def calc_predictor_timestamp(self, encoder_out, encoder_out_lens, token_num):
			encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(
			encoder_out.device)
			ds_alphas, ds_cif_peak, us_alphas, us_cif_peak = self.predictor.get_upsample_timestamp(encoder_out,
			ds_alphas, ds_cif_peak, us_alphas, us_peaks = self.predictor.get_upsample_timestamp(encoder_out,
			encoder_out_mask,
			token_num)
			return ds_alphas, ds_cif_peak, us_alphas, us_cif_peak
			return ds_alphas, ds_cif_peak, us_alphas, us_peaks

			def forward(
			self,
			@@ -1022,6 +1085,7 @@
			inner_dim: int = 256,
			bias_encoder_type: str = 'lstm',
			label_bracket: bool = False,
			use_decoder_embedding: bool = False,
			):
			assert check_argument_types()
			assert 0.0 <= ctc_weight <= 1.0, ctc_weight
			@@ -1075,6 +1139,7 @@
			self.hotword_buffer = None
			self.length_record = []
			self.current_buffer_length = 0
			self.use_decoder_embedding = use_decoder_embedding

			def forward(
			self,
			@@ -1216,7 +1281,10 @@
			hw_list.append(hw_tokens)
			# padding
			hw_list_pad = pad_list(hw_list, 0)
			hw_embed = self.decoder.embed(hw_list_pad)
			if self.use_decoder_embedding:
			hw_embed = self.decoder.embed(hw_list_pad)
			else:
			hw_embed = self.bias_embed(hw_list_pad)
			hw_embed, (_, _) = self.bias_encoder(hw_embed)
			_ind = np.arange(0, len(hw_list)).tolist()
			# update self.hotword_buffer, throw a part if oversize
			@@ -1332,13 +1400,19 @@
			# default hotword list
			hw_list = [torch.Tensor([self.sos]).long().to(encoder_out.device)] # empty hotword list
			hw_list_pad = pad_list(hw_list, 0)
			hw_embed = self.bias_embed(hw_list_pad)
			if self.use_decoder_embedding:
			hw_embed = self.decoder.embed(hw_list_pad)
			else:
			hw_embed = self.bias_embed(hw_list_pad)
			_, (h_n, _) = self.bias_encoder(hw_embed)
			contextual_info = h_n.squeeze(0).repeat(encoder_out.shape[0], 1, 1)
			else:
			hw_lengths = [len(i) for i in hw_list]
			hw_list_pad = pad_list([torch.Tensor(i).long() for i in hw_list], 0).to(encoder_out.device)
			hw_embed = self.bias_embed(hw_list_pad)
			if self.use_decoder_embedding:
			hw_embed = self.decoder.embed(hw_list_pad)
			else:
			hw_embed = self.bias_embed(hw_list_pad)
			hw_embed = torch.nn.utils.rnn.pack_padded_sequence(hw_embed, hw_lengths, batch_first=True,
			enforce_sorted=False)
			_, (h_n, _) = self.bias_encoder(hw_embed)
			@@ -1459,4 +1533,4 @@
			"torch tensor: {}, {}, loading from tf tensor: {}, {}".format(name, data_tf.size(), name_tf,
			var_dict_tf[name_tf].shape))

			return var_dict_torch_update
			return var_dict_torch_update