python/FunASR-XL.git

			@@ -63,19 +63,21 @@
			seaco_length_normalized_loss = kwargs.get("seaco_length_normalized_loss", True)

			# bias encoder
			if self.bias_encoder_type == 'lstm':
			self.bias_encoder = torch.nn.LSTM(self.inner_dim,
			if self.bias_encoder_type == "lstm":
			self.bias_encoder = torch.nn.LSTM(
			self.inner_dim,
			self.inner_dim,
			2,
			batch_first=True,
			dropout=bias_encoder_dropout_rate,
			bidirectional=bias_encoder_bid)
			bidirectional=bias_encoder_bid,
			)
			if bias_encoder_bid:
			self.lstm_proj = torch.nn.Linear(self.inner_dim*2, self.inner_dim)
			else:
			self.lstm_proj = None
			# self.bias_embed = torch.nn.Embedding(self.vocab_size, self.inner_dim)
			elif self.bias_encoder_type == 'mean':
			elif self.bias_encoder_type == "mean":
			self.bias_embed = torch.nn.Embedding(self.vocab_size, self.inner_dim)
			else:
			logging.error("Unsupport bias encoder type: {}".format(self.bias_encoder_type))
			@@ -124,10 +126,7 @@
			speech_lengths = speech_lengths[:, 0]
			# Check that batch_size is unified
			assert (
			speech.shape[0]
			== speech_lengths.shape[0]
			== text.shape[0]
			== text_lengths.shape[0]
			speech.shape[0] == speech_lengths.shape[0] == text.shape[0] == text_lengths.shape[0]
			), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape)

			hotword_pad = kwargs.get("hotword_pad")
			@@ -148,7 +147,8 @@
			ys_lengths = text_lengths + self.predictor_bias

			stats = dict()
			loss_seaco = self._calc_seaco_loss(encoder_out,
			loss_seaco = self._calc_seaco_loss(
			encoder_out,
			encoder_out_lens,
			ys_pad,
			ys_lengths,
			@@ -179,9 +179,12 @@
			return cif_attended + dec_attended

			def calc_predictor(self, encoder_out, encoder_out_lens):
			encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(
			encoder_out.device)
			predictor_outs = self.predictor(encoder_out, None, encoder_out_mask, ignore_id=self.ignore_id)
			encoder_out_mask = (
			~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
			).to(encoder_out.device)
			predictor_outs = self.predictor(
			encoder_out, None, encoder_out_mask, ignore_id=self.ignore_id
			)
			return predictor_outs[:4]

			def _calc_seaco_loss(
			@@ -195,50 +198,84 @@
			seaco_label_pad: torch.Tensor,
			):
			# predictor forward
			encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(
			encoder_out.device)
			pre_acoustic_embeds = self.predictor(encoder_out, ys_pad, encoder_out_mask, ignore_id=self.ignore_id)[0]
			encoder_out_mask = (
			~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
			).to(encoder_out.device)
			pre_acoustic_embeds = self.predictor(
			encoder_out, ys_pad, encoder_out_mask, ignore_id=self.ignore_id
			)[0]
			# decoder forward
			decoder_out, _ = self.decoder(encoder_out, encoder_out_lens, pre_acoustic_embeds, ys_lengths, return_hidden=True)
			selected = self._hotword_representation(hotword_pad,
			hotword_lengths)
			contextual_info = selected.squeeze(0).repeat(encoder_out.shape[0], 1, 1).to(encoder_out.device)
			decoder_out, _ = self.decoder(
			encoder_out, encoder_out_lens, pre_acoustic_embeds, ys_lengths, return_hidden=True
			)
			selected = self._hotword_representation(hotword_pad, hotword_lengths)
			contextual_info = (
			selected.squeeze(0).repeat(encoder_out.shape[0], 1, 1).to(encoder_out.device)
			)
			num_hot_word = contextual_info.shape[1]
			_contextual_length = torch.Tensor([num_hot_word]).int().repeat(encoder_out.shape[0]).to(encoder_out.device)
			_contextual_length = (
			torch.Tensor([num_hot_word]).int().repeat(encoder_out.shape[0]).to(encoder_out.device)
			)
			# dha core
			cif_attended, _ = self.seaco_decoder(contextual_info, _contextual_length, pre_acoustic_embeds, ys_lengths)
			dec_attended, _ = self.seaco_decoder(contextual_info, _contextual_length, decoder_out, ys_lengths)
			cif_attended, _ = self.seaco_decoder(
			contextual_info, _contextual_length, pre_acoustic_embeds, ys_lengths
			)
			dec_attended, _ = self.seaco_decoder(
			contextual_info, _contextual_length, decoder_out, ys_lengths
			)
			merged = self._merge(cif_attended, dec_attended)
			dha_output = self.hotword_output_layer(merged[:, :-1]) # remove the last token in loss calculation
			dha_output = self.hotword_output_layer(
			merged[:, :-1]
			) # remove the last token in loss calculation
			loss_att = self.criterion_seaco(dha_output, seaco_label_pad)
			return loss_att

			def _seaco_decode_with_ASF(self,
			def _seaco_decode_with_ASF(
			self,
			encoder_out,
			encoder_out_lens,
			sematic_embeds,
			ys_pad_lens,
			hw_list,
			nfilter=50,
			seaco_weight=1.0):
			seaco_weight=1.0,
			):
			# decoder forward

			decoder_out, decoder_hidden, _ = self.decoder(encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens, return_hidden=True, return_both=True)
			decoder_out, decoder_hidden, _ = self.decoder(
			encoder_out,
			encoder_out_lens,
			sematic_embeds,
			ys_pad_lens,
			return_hidden=True,
			return_both=True,
			)

			decoder_pred = torch.log_softmax(decoder_out, dim=-1)
			if hw_list is not None:
			hw_lengths = [len(i) for i in hw_list]
			hw_list_ = [torch.Tensor(i).long() for i in hw_list]
			hw_list_pad = pad_list(hw_list_, 0).to(encoder_out.device)
			selected = self._hotword_representation(hw_list_pad, torch.Tensor(hw_lengths).int().to(encoder_out.device))
			selected = self._hotword_representation(
			hw_list_pad, torch.Tensor(hw_lengths).int().to(encoder_out.device)
			)

			contextual_info = selected.squeeze(0).repeat(encoder_out.shape[0], 1, 1).to(encoder_out.device)
			contextual_info = (
			selected.squeeze(0).repeat(encoder_out.shape[0], 1, 1).to(encoder_out.device)
			)
			num_hot_word = contextual_info.shape[1]
			_contextual_length = torch.Tensor([num_hot_word]).int().repeat(encoder_out.shape[0]).to(encoder_out.device)
			_contextual_length = (
			torch.Tensor([num_hot_word])
			.int()
			.repeat(encoder_out.shape[0])
			.to(encoder_out.device)
			)

			# ASF Core
			if nfilter > 0 and nfilter < num_hot_word:
			hotword_scores = self.seaco_decoder.forward_asf6(contextual_info, _contextual_length, decoder_hidden, ys_pad_lens)
			hotword_scores = self.seaco_decoder.forward_asf6(
			contextual_info, _contextual_length, decoder_hidden, ys_pad_lens
			)
			hotword_scores = hotword_scores[0].sum(0).sum(0)
			# hotword_scores /= torch.sqrt(torch.tensor(hw_lengths)[:-1].float()).to(hotword_scores.device)
			dec_filter = torch.topk(hotword_scores, min(nfilter, num_hot_word-1))[1].tolist()
			@@ -247,17 +284,31 @@
			# filter hotword embedding
			selected = selected[add_filter]
			# again
			contextual_info = selected.squeeze(0).repeat(encoder_out.shape[0], 1, 1).to(encoder_out.device)
			contextual_info = (
			selected.squeeze(0).repeat(encoder_out.shape[0], 1, 1).to(encoder_out.device)
			)
			num_hot_word = contextual_info.shape[1]
			_contextual_length = torch.Tensor([num_hot_word]).int().repeat(encoder_out.shape[0]).to(encoder_out.device)
			_contextual_length = (
			torch.Tensor([num_hot_word])
			.int()
			.repeat(encoder_out.shape[0])
			.to(encoder_out.device)
			)

			# SeACo Core
			cif_attended, _ = self.seaco_decoder(contextual_info, _contextual_length, sematic_embeds, ys_pad_lens)
			dec_attended, _ = self.seaco_decoder(contextual_info, _contextual_length, decoder_hidden, ys_pad_lens)
			cif_attended, _ = self.seaco_decoder(
			contextual_info, _contextual_length, sematic_embeds, ys_pad_lens
			)
			dec_attended, _ = self.seaco_decoder(
			contextual_info, _contextual_length, decoder_hidden, ys_pad_lens
			)
			merged = self._merge(cif_attended, dec_attended)

			dha_output = self.hotword_output_layer(merged) # remove the last token in loss calculation
			dha_output = self.hotword_output_layer(
			merged
			) # remove the last token in loss calculation
			dha_pred = torch.log_softmax(dha_output, dim=-1)

			def _merge_res(dec_output, dha_output):
			lmbd = torch.Tensor([seaco_weight] * dha_output.shape[0])
			dha_ids = dha_output.max(-1)[-1]# [0]
			@@ -275,13 +326,11 @@
			else:
			return decoder_pred

			def _hotword_representation(self,
			hotword_pad,
			hotword_lengths):
			if self.bias_encoder_type != 'lstm':
			def _hotword_representation(self, hotword_pad, hotword_lengths):
			if self.bias_encoder_type != "lstm":
			logging.error("Unsupported bias encoder type")

			'''
			"""
			hw_embed = self.decoder.embed(hotword_pad)
			hw_embed, (_, _) = self.bias_encoder(hw_embed)
			if self.lstm_proj is not None:
			@@ -289,11 +338,16 @@
			_ind = np.arange(0, hw_embed.shape[0]).tolist()
			selected = hw_embed[_ind, [i-1 for i in hotword_lengths.detach().cpu().tolist()]]
			return selected
			'''
			"""

			# hw_embed = self.sac_embedding(hotword_pad)
			hw_embed = self.decoder.embed(hotword_pad)
			hw_embed = torch.nn.utils.rnn.pack_padded_sequence(hw_embed, hotword_lengths.cpu().type(torch.int64), batch_first=True, enforce_sorted=False)
			hw_embed = torch.nn.utils.rnn.pack_padded_sequence(
			hw_embed,
			hotword_lengths.cpu().type(torch.int64),
			batch_first=True,
			enforce_sorted=False,
			)
			packed_rnn_output, _ = self.bias_encoder(hw_embed)
			rnn_output = torch.nn.utils.rnn.pad_packed_sequence(packed_rnn_output, batch_first=True)[0]
			if self.lstm_proj is not None:
			@@ -304,7 +358,8 @@
			selected = hw_hidden[_ind, [i-1 for i in hotword_lengths.detach().cpu().tolist()]]
			return selected

			def inference(self,
			def inference(
			self,
			data_in,
			data_lengths=None,
			key: list = None,
			@@ -315,7 +370,9 @@

			# init beamsearch
			is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None
			is_use_lm = kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None
			is_use_lm = (
			kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None
			)
			if self.beam_search is None and (is_use_lm or is_use_ctc):
			logging.info("enable beam_search")
			self.init_beam_search(**kwargs)
			@@ -324,21 +381,27 @@

			# extract fbank feats
			time1 = time.perf_counter()
			audio_sample_list = load_audio_text_image_video(data_in, fs=frontend.fs, audio_fs=kwargs.get("fs", 16000))
			audio_sample_list = load_audio_text_image_video(
			data_in, fs=frontend.fs, audio_fs=kwargs.get("fs", 16000)
			)
			time2 = time.perf_counter()
			meta_data["load_data"] = f"{time2 - time1:0.3f}"
			speech, speech_lengths = extract_fbank(audio_sample_list, data_type=kwargs.get("data_type", "sound"),
			frontend=frontend)
			speech, speech_lengths = extract_fbank(
			audio_sample_list, data_type=kwargs.get("data_type", "sound"), frontend=frontend
			)
			time3 = time.perf_counter()
			meta_data["extract_feat"] = f"{time3 - time2:0.3f}"
			meta_data[
			"batch_data_time"] = speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000
			meta_data["batch_data_time"] = (
			speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000
			)

			speech = speech.to(device=kwargs["device"])
			speech_lengths = speech_lengths.to(device=kwargs["device"])

			# hotword
			self.hotword_list = self.generate_hotwords_list(kwargs.get("hotword", None), tokenizer=tokenizer, frontend=frontend)
			self.hotword_list = self.generate_hotwords_list(
			kwargs.get("hotword", None), tokenizer=tokenizer, frontend=frontend
			)

			# Encoder
			encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)
			@@ -352,18 +415,19 @@
			if torch.max(pre_token_length) < 1:
			return ([],)

			decoder_out = self._seaco_decode_with_ASF(encoder_out,
			decoder_out = self._seaco_decode_with_ASF(
			encoder_out,
			encoder_out_lens,
			pre_acoustic_embeds,
			pre_token_length,
			hw_list=self.hotword_list
			hw_list=self.hotword_list,
			)

			# decoder_out, _ = decoder_outs[0], decoder_outs[1]
			if self.predictor_name == "CifPredictorV3":
			_, _, us_alphas, us_peaks = self.calc_predictor_timestamp(encoder_out,
			encoder_out_lens,
			pre_token_length)
			_, _, us_alphas, us_peaks = self.calc_predictor_timestamp(
			encoder_out, encoder_out_lens, pre_token_length
			)
			else:
			us_alphas = None

			@@ -374,8 +438,10 @@
			am_scores = decoder_out[i, :pre_token_length[i], :]
			if self.beam_search is not None:
			nbest_hyps = self.beam_search(
			x=x, am_scores=am_scores, maxlenratio=kwargs.get("maxlenratio", 0.0),
			minlenratio=kwargs.get("minlenratio", 0.0)
			x=x,
			am_scores=am_scores,
			maxlenratio=kwargs.get("maxlenratio", 0.0),
			minlenratio=kwargs.get("minlenratio", 0.0),
			)

			nbest_hyps = nbest_hyps[: self.nbest]
			@@ -385,9 +451,7 @@
			score = am_scores.max(dim=-1)[0]
			score = torch.sum(score, dim=-1)
			# pad with mask tokens to ensure compatibility with sos/eos tokens
			yseq = torch.tensor(
			[self.sos] + yseq.tolist() + [self.eos], device=yseq.device
			)
			yseq = torch.tensor([self.sos] + yseq.tolist() + [self.eos], device=yseq.device)
			nbest_hyps = [Hypothesis(yseq=yseq, score=score)]
			for nbest_idx, hyp in enumerate(nbest_hyps):
			ibest_writer = None
			@@ -405,21 +469,30 @@

			# remove blank symbol id, which is assumed to be 0
			token_int = list(
			filter(lambda x: x != self.eos and x != self.sos and x != self.blank_id, token_int))
			filter(
			lambda x: x != self.eos and x != self.sos and x != self.blank_id, token_int
			)
			)

			if tokenizer is not None:
			# Change integer-ids to tokens
			token = tokenizer.ids2tokens(token_int)
			text = tokenizer.tokens2text(token)
			if us_alphas is not None:
			_, timestamp = ts_prediction_lfr6_standard(us_alphas[i][:encoder_out_lens[i] * 3],
			_, timestamp = ts_prediction_lfr6_standard(
			us_alphas[i][: encoder_out_lens[i] * 3],
			us_peaks[i][:encoder_out_lens[i] * 3],
			copy.copy(token),
			vad_offset=kwargs.get("begin_time", 0))
			text_postprocessed, time_stamp_postprocessed, _ = \
			vad_offset=kwargs.get("begin_time", 0),
			)
			text_postprocessed, time_stamp_postprocessed, _ = (
			postprocess_utils.sentence_postprocess(token, timestamp)
			result_i = {"key": key[i], "text": text_postprocessed,
			"timestamp": time_stamp_postprocessed}
			)
			result_i = {
			"key": key[i],
			"text": text_postprocessed,
			"timestamp": time_stamp_postprocessed,
			}
			if ibest_writer is not None:
			ibest_writer["token"][key[i]] = " ".join(token)
			ibest_writer["timestamp"][key[i]] = time_stamp_postprocessed
			@@ -450,7 +523,7 @@
			return seg_dict

			def seg_tokenize(txt, seg_dict):
			pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
			pattern = re.compile(r"^[\u4E00-\u9FA50-9]+$")
			out_txt = ""
			for word in txt:
			word = word.lower()
			@@ -470,7 +543,7 @@
			seg_dict = None
			if frontend.cmvn_file is not None:
			model_dir = os.path.dirname(frontend.cmvn_file)
			seg_dict_file = os.path.join(model_dir, 'seg_dict')
			seg_dict_file = os.path.join(model_dir, "seg_dict")
			if os.path.exists(seg_dict_file):
			seg_dict = load_seg_dict(seg_dict_file)
			else:
			@@ -479,11 +552,11 @@
			if hotword_list_or_file is None:
			hotword_list = None
			# for local txt inputs
			elif os.path.exists(hotword_list_or_file) and hotword_list_or_file.endswith('.txt'):
			elif os.path.exists(hotword_list_or_file) and hotword_list_or_file.endswith(".txt"):
			logging.info("Attempting to parse hotwords from local txt...")
			hotword_list = []
			hotword_str_list = []
			with codecs.open(hotword_list_or_file, 'r') as fin:
			with codecs.open(hotword_list_or_file, "r") as fin:
			for line in fin.readlines():
			hw = line.strip()
			hw_list = hw.split()
			@@ -492,11 +565,14 @@
			hotword_str_list.append(hw)
			hotword_list.append(tokenizer.tokens2ids(hw_list))
			hotword_list.append([self.sos])
			hotword_str_list.append('<s>')
			logging.info("Initialized hotword list from file: {}, hotword list: {}."
			.format(hotword_list_or_file, hotword_str_list))
			hotword_str_list.append("<s>")
			logging.info(
			"Initialized hotword list from file: {}, hotword list: {}.".format(
			hotword_list_or_file, hotword_str_list
			)
			)
			# for url, download and generate txt
			elif hotword_list_or_file.startswith('http'):
			elif hotword_list_or_file.startswith("http"):
			logging.info("Attempting to parse hotwords from url...")
			work_dir = tempfile.TemporaryDirectory().name
			if not os.path.exists(work_dir):
			@@ -507,7 +583,7 @@
			hotword_list_or_file = text_file_path
			hotword_list = []
			hotword_str_list = []
			with codecs.open(hotword_list_or_file, 'r') as fin:
			with codecs.open(hotword_list_or_file, "r") as fin:
			for line in fin.readlines():
			hw = line.strip()
			hw_list = hw.split()
			@@ -516,11 +592,14 @@
			hotword_str_list.append(hw)
			hotword_list.append(tokenizer.tokens2ids(hw_list))
			hotword_list.append([self.sos])
			hotword_str_list.append('<s>')
			logging.info("Initialized hotword list from file: {}, hotword list: {}."
			.format(hotword_list_or_file, hotword_str_list))
			hotword_str_list.append("<s>")
			logging.info(
			"Initialized hotword list from file: {}, hotword list: {}.".format(
			hotword_list_or_file, hotword_str_list
			)
			)
			# for text str input
			elif not hotword_list_or_file.endswith('.txt'):
			elif not hotword_list_or_file.endswith(".txt"):
			logging.info("Attempting to parse hotwords as str...")
			hotword_list = []
			hotword_str_list = []
			@@ -531,7 +610,7 @@
			hw_list = seg_tokenize(hw_list, seg_dict)
			hotword_list.append(tokenizer.tokens2ids(hw_list))
			hotword_list.append([self.sos])
			hotword_str_list.append('<s>')
			hotword_str_list.append("<s>")
			logging.info("Hotword list: {}.".format(hotword_str_list))
			else:
			hotword_list = None
			@@ -541,9 +620,9 @@
			self,
			**kwargs,
			):
			if 'max_seq_len' not in kwargs:
			kwargs['max_seq_len'] = 512
			if "max_seq_len" not in kwargs:
			kwargs["max_seq_len"] = 512
			from .export_meta import export_rebuild_model

			models = export_rebuild_model(model=self, **kwargs)
			return models