| | |
|---|
| | | ibest_writer["text"][key[i]] = text |
|---|
| | | |
|---|
| | | return results, meta_data |
|---|
| | | |
|---|
| | | |
|---|
| | | @tables.register("model_classes", "SenseVoiceSANM") |
|---|
| | | class SenseVoiceSANM(nn.Module): |
|---|
| | | |
|---|
| | | def __init__( |
|---|
| | | self, |
|---|
| | | specaug: str = None, |
|---|
| | | specaug_conf: dict = None, |
|---|
| | | normalize: str = None, |
|---|
| | | normalize_conf: dict = None, |
|---|
| | | encoder: str = None, |
|---|
| | | encoder_conf: dict = None, |
|---|
| | | decoder: str = None, |
|---|
| | | decoder_conf: dict = None, |
|---|
| | | input_size: int = 80, |
|---|
| | | vocab_size: int = -1, |
|---|
| | | ignore_id: int = -1, |
|---|
| | | blank_id: int = 0, |
|---|
| | | sos: int = 1, |
|---|
| | | eos: int = 2, |
|---|
| | | lsm_weight: float = 0.0, |
|---|
| | | length_normalized_loss: bool = False, |
|---|
| | | report_cer: bool = True, |
|---|
| | | report_wer: bool = True, |
|---|
| | | sym_space: str = "<space>", |
|---|
| | | sym_blank: str = "<blank>", |
|---|
| | | # extract_feats_in_collect_stats: bool = True, |
|---|
| | | share_embedding: bool = False, |
|---|
| | | # preencoder: Optional[AbsPreEncoder] = None, |
|---|
| | | # postencoder: Optional[AbsPostEncoder] = None, |
|---|
| | | **kwargs, |
|---|
| | | ): |
|---|
| | | |
|---|
| | | super().__init__() |
|---|
| | | |
|---|
| | | if specaug is not None: |
|---|
| | | specaug_class = tables.specaug_classes.get(specaug) |
|---|
| | | specaug = specaug_class(**specaug_conf) |
|---|
| | | |
|---|
| | | encoder_class = tables.encoder_classes.get(encoder) |
|---|
| | | encoder = encoder_class(input_size=input_size, **encoder_conf) |
|---|
| | | encoder_output_size = encoder.output_size() |
|---|
| | | |
|---|
| | | decoder_class = tables.decoder_classes.get(decoder) |
|---|
| | | decoder = decoder_class( |
|---|
| | | vocab_size=vocab_size, |
|---|
| | | encoder_output_size=encoder_output_size, |
|---|
| | | **decoder_conf, |
|---|
| | | ) |
|---|
| | | |
|---|
| | | self.blank_id = blank_id |
|---|
| | | self.sos = sos if sos is not None else vocab_size - 1 |
|---|
| | | self.eos = eos if eos is not None else vocab_size - 1 |
|---|
| | | self.vocab_size = vocab_size |
|---|
| | | self.ignore_id = ignore_id |
|---|
| | | |
|---|
| | | self.specaug = specaug |
|---|
| | | |
|---|
| | | self.encoder = encoder |
|---|
| | | |
|---|
| | | self.decoder = decoder |
|---|
| | | |
|---|
| | | self.criterion_att = LabelSmoothingLoss( |
|---|
| | | size=vocab_size, |
|---|
| | | padding_idx=ignore_id, |
|---|
| | | smoothing=lsm_weight, |
|---|
| | | normalize_length=length_normalized_loss, |
|---|
| | | ) |
|---|
| | | |
|---|
| | | self.error_calculator = None |
|---|
| | | |
|---|
| | | self.length_normalized_loss = length_normalized_loss |
|---|
| | | self.beam_search = None |
|---|
| | | self.activation_checkpoint = kwargs.get("activation_checkpoint", False) |
|---|
| | | |
|---|
| | | def forward( |
|---|
| | | self, |
|---|
| | | speech: torch.Tensor, |
|---|
| | | speech_lengths: torch.Tensor, |
|---|
| | | text: torch.Tensor, |
|---|
| | | text_lengths: torch.Tensor, |
|---|
| | | **kwargs, |
|---|
| | | ): |
|---|
| | | target_mask = kwargs.get("target_mask", None) |
|---|
| | | |
|---|
| | | # import pdb; |
|---|
| | | # pdb.set_trace() |
|---|
| | | if len(text_lengths.size()) > 1: |
|---|
| | | text_lengths = text_lengths[:, 0] |
|---|
| | | if len(speech_lengths.size()) > 1: |
|---|
| | | speech_lengths = speech_lengths[:, 0] |
|---|
| | | |
|---|
| | | batch_size, frames, _ = speech.shape |
|---|
| | | _, text_tokens = text.shape |
|---|
| | | |
|---|
| | | if self.activation_checkpoint: |
|---|
| | | from torch.utils.checkpoint import checkpoint |
|---|
| | | |
|---|
| | | encoder_out, encoder_out_lens = checkpoint( |
|---|
| | | self.encode, speech, speech_lengths, use_reentrant=False |
|---|
| | | ) |
|---|
| | | else: |
|---|
| | | encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) |
|---|
| | | |
|---|
| | | loss_att, acc_att, cer_att, wer_att = self._calc_att_loss( |
|---|
| | | encoder_out, encoder_out_lens, text, text_lengths, target_mask=target_mask |
|---|
| | | ) |
|---|
| | | |
|---|
| | | loss = loss_att |
|---|
| | | stats = {} |
|---|
| | | stats["acc"] = acc_att |
|---|
| | | stats["loss"] = torch.clone(loss.detach()) |
|---|
| | | stats["batch_size"] = batch_size |
|---|
| | | stats["batch_size_x_frames"] = frames * batch_size |
|---|
| | | stats["batch_size_real_frames"] = speech_lengths.sum().item() |
|---|
| | | stats["padding_frames"] = stats["batch_size_x_frames"] - stats["batch_size_real_frames"] |
|---|
| | | stats["batch_size_x_tokens"] = text_tokens * batch_size |
|---|
| | | stats["batch_size_real_tokens"] = text_lengths.sum().item() |
|---|
| | | stats["padding_tokens"] = stats["batch_size_x_tokens"] - stats["batch_size_real_tokens"] |
|---|
| | | stats["batch_size_x_frames_plus_tokens"] = (text_tokens + frames) * batch_size |
|---|
| | | |
|---|
| | | # force_gatherable: to-device and to-tensor if scalar for DataParallel |
|---|
| | | if self.length_normalized_loss: |
|---|
| | | batch_size = int((text_lengths + 1).sum()) |
|---|
| | | loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) |
|---|
| | | return loss, stats, weight |
|---|
| | | |
|---|
| | | def encode( |
|---|
| | | self, |
|---|
| | | speech: torch.Tensor, |
|---|
| | | speech_lengths: torch.Tensor, |
|---|
| | | **kwargs, |
|---|
| | | ): |
|---|
| | | """Frontend + Encoder. Note that this method is used by asr_inference.py |
|---|
| | | Args: |
|---|
| | | speech: (Batch, Length, ...) |
|---|
| | | speech_lengths: (Batch, ) |
|---|
| | | ind: int |
|---|
| | | """ |
|---|
| | | with autocast(False): |
|---|
| | | |
|---|
| | | # Data augmentation |
|---|
| | | if self.specaug is not None and self.training: |
|---|
| | | speech, speech_lengths = self.specaug(speech, speech_lengths) |
|---|
| | | |
|---|
| | | # Forward encoder |
|---|
| | | # feats: (Batch, Length, Dim) |
|---|
| | | # -> encoder_out: (Batch, Length2, Dim2) |
|---|
| | | |
|---|
| | | encoder_out, encoder_out_lens, _ = self.encoder(speech, speech_lengths) |
|---|
| | | if isinstance(encoder_out, (tuple, list)): |
|---|
| | | encoder_out = encoder_out[0] |
|---|
| | | |
|---|
| | | return encoder_out, encoder_out_lens |
|---|
| | | |
|---|
| | | def _calc_att_loss( |
|---|
| | | self, |
|---|
| | | encoder_out: torch.Tensor, |
|---|
| | | encoder_out_lens: torch.Tensor, |
|---|
| | | ys_pad: torch.Tensor, |
|---|
| | | ys_pad_lens: torch.Tensor, |
|---|
| | | **kwargs, |
|---|
| | | ): |
|---|
| | | target_mask = kwargs.get("target_mask", None) |
|---|
| | | stats = {} |
|---|
| | | |
|---|
| | | # 1. Forward decoder |
|---|
| | | ys_pad[ys_pad == -1] = 0 |
|---|
| | | decoder_out = self.decoder(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) |
|---|
| | | if isinstance(decoder_out, (list, tuple)): |
|---|
| | | decoder_out = decoder_out[0] |
|---|
| | | |
|---|
| | | # 2. Compute attention loss |
|---|
| | | mask = torch.ones_like(ys_pad) * (-1) |
|---|
| | | ys_pad_mask = (ys_pad * target_mask + mask * (1 - target_mask)).to(torch.int64) |
|---|
| | | ys_pad_mask[ys_pad_mask == 0] = -1 |
|---|
| | | loss_att = self.criterion_att(decoder_out[:, :-1, :], ys_pad_mask[:, 1:]) |
|---|
| | | |
|---|
| | | with torch.no_grad(): |
|---|
| | | preds = torch.argmax(decoder_out, -1) |
|---|
| | | acc_att = compute_accuracy( |
|---|
| | | preds[:, :-1], ys_pad_mask[:, 1:], ignore_label=self.ignore_id |
|---|
| | | ) |
|---|
| | | |
|---|
| | | return loss_att, acc_att, None, None |
|---|
| | | |
|---|
| | | def init_beam_search( |
|---|
| | | self, |
|---|
| | | **kwargs, |
|---|
| | | ): |
|---|
| | | from .search import BeamSearch |
|---|
| | | |
|---|
| | | from funasr.models.transformer.scorers.length_bonus import LengthBonus |
|---|
| | | |
|---|
| | | # 1. Build ASR model |
|---|
| | | scorers = {} |
|---|
| | | |
|---|
| | | scorers.update( |
|---|
| | | decoder=self.decoder, |
|---|
| | | length_bonus=LengthBonus(self.vocab_size), |
|---|
| | | ) |
|---|
| | | |
|---|
| | | weights = dict( |
|---|
| | | decoder=1.0, |
|---|
| | | ctc=0.0, |
|---|
| | | lm=0.0, |
|---|
| | | ngram=0.0, |
|---|
| | | length_bonus=kwargs.get("penalty", 0.0), |
|---|
| | | ) |
|---|
| | | beam_search = BeamSearch( |
|---|
| | | beam_size=kwargs.get("beam_size", 5), |
|---|
| | | weights=weights, |
|---|
| | | scorers=scorers, |
|---|
| | | sos=None, |
|---|
| | | eos=None, |
|---|
| | | vocab_size=self.vocab_size, |
|---|
| | | token_list=None, |
|---|
| | | pre_beam_score_key="full", |
|---|
| | | ) |
|---|
| | | |
|---|
| | | self.beam_search = beam_search |
|---|
| | | |
|---|
| | | def inference( |
|---|
| | | self, |
|---|
| | | data_in, |
|---|
| | | data_lengths=None, |
|---|
| | | key: list = None, |
|---|
| | | tokenizer=None, |
|---|
| | | frontend=None, |
|---|
| | | **kwargs, |
|---|
| | | ): |
|---|
| | | if kwargs.get("batch_size", 1) > 1: |
|---|
| | | raise NotImplementedError("batch decoding is not implemented") |
|---|
| | | |
|---|
| | | # init beamsearch |
|---|
| | | if not hasattr(self, "beam_search") or self.beam_search is None: |
|---|
| | | logging.info("enable beam_search") |
|---|
| | | self.init_beam_search(**kwargs) |
|---|
| | | self.nbest = kwargs.get("nbest", 1) |
|---|
| | | |
|---|
| | | if frontend is None and not hasattr(self, "frontend"): |
|---|
| | | frontend_class = tables.frontend_classes.get("WhisperFrontend") |
|---|
| | | frontend = frontend_class( |
|---|
| | | n_mels=self.model.dims.n_mels, do_pad_trim=kwargs.get("do_pad_trim", True) |
|---|
| | | ) |
|---|
| | | self.frontend = frontend |
|---|
| | | else: |
|---|
| | | frontend = frontend if frontend is not None else self.frontend |
|---|
| | | |
|---|
| | | meta_data = {} |
|---|
| | | if ( |
|---|
| | | isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank" |
|---|
| | | ): # fbank |
|---|
| | | speech, speech_lengths = data_in, data_lengths |
|---|
| | | if len(speech.shape) < 3: |
|---|
| | | speech = speech[None, :, :] |
|---|
| | | if speech_lengths is None: |
|---|
| | | speech_lengths = speech.shape[1] |
|---|
| | | else: |
|---|
| | | # extract fbank feats |
|---|
| | | time1 = time.perf_counter() |
|---|
| | | audio_sample_list = load_audio_text_image_video( |
|---|
| | | data_in, |
|---|
| | | fs=frontend.fs if hasattr(frontend, "fs") else 16000, |
|---|
| | | audio_fs=kwargs.get("fs", 16000), |
|---|
| | | data_type=kwargs.get("data_type", "sound"), |
|---|
| | | tokenizer=tokenizer, |
|---|
| | | ) |
|---|
| | | |
|---|
| | | if ( |
|---|
| | | isinstance(kwargs.get("data_type", None), (list, tuple)) |
|---|
| | | and len(kwargs.get("data_type", [])) > 1 |
|---|
| | | ): |
|---|
| | | audio_sample_list, text_token_int_list = audio_sample_list |
|---|
| | | text_token_int = text_token_int_list[0] |
|---|
| | | else: |
|---|
| | | text_token_int = None |
|---|
| | | |
|---|
| | | time2 = time.perf_counter() |
|---|
| | | meta_data["load_data"] = f"{time2 - time1:0.3f}" |
|---|
| | | speech, speech_lengths = extract_fbank( |
|---|
| | | audio_sample_list, data_type=kwargs.get("data_type", "sound"), frontend=frontend |
|---|
| | | ) |
|---|
| | | time3 = time.perf_counter() |
|---|
| | | meta_data["extract_feat"] = f"{time3 - time2:0.3f}" |
|---|
| | | frame_shift = frontend.frame_shift if hasattr(frontend, "frame_shift") else 10 |
|---|
| | | lfr_n = frontend.lfr_n if hasattr(frontend, "lfr_n") else 1 |
|---|
| | | meta_data["batch_data_time"] = speech_lengths.sum().item() * frame_shift * lfr_n / 1000 |
|---|
| | | |
|---|
| | | speech = speech.to(device=kwargs["device"])[0, :, :] |
|---|
| | | speech_lengths = speech_lengths.to(device=kwargs["device"]) |
|---|
| | | |
|---|
| | | DecodingOptions = kwargs.get("DecodingOptions", {}) |
|---|
| | | task = DecodingOptions.get("task", "ASR") |
|---|
| | | if isinstance(task, str): |
|---|
| | | task = [task] |
|---|
| | | task = "".join([f"<|{x}|>" for x in task]) |
|---|
| | | initial_prompt = kwargs.get("initial_prompt", f"<|startoftranscript|>{task}") |
|---|
| | | |
|---|
| | | language = DecodingOptions.get("language", None) |
|---|
| | | language = None if language == "auto" else language |
|---|
| | | |
|---|
| | | sos = f"{initial_prompt}<|{language}|>" if language is not None else initial_prompt |
|---|
| | | sos_int = tokenizer.encode(sos, allowed_special="all") |
|---|
| | | eos = kwargs.get("model_conf").get("eos") |
|---|
| | | eos_int = tokenizer.encode(eos, allowed_special="all") |
|---|
| | | self.beam_search.sos = sos_int |
|---|
| | | self.beam_search.eos = eos_int[0] |
|---|
| | | |
|---|
| | | # Paramterts for rich decoding |
|---|
| | | self.beam_search.emo_unk = tokenizer.encode( |
|---|
| | | DecodingOptions.get("emo_unk_token", "<|SPECIAL_TOKEN_1|>"), allowed_special="all" |
|---|
| | | )[0] |
|---|
| | | self.beam_search.emo_unk_score = 1 |
|---|
| | | self.beam_search.emo_tokens = tokenizer.encode( |
|---|
| | | DecodingOptions.get("emo_target_tokens", "<|HAPPY|><|SAD|><|ANGRY|>"), |
|---|
| | | allowed_special="all", |
|---|
| | | ) |
|---|
| | | self.beam_search.emo_scores = DecodingOptions.get("emo_target_threshold", [0.1, 0.1, 0.1]) |
|---|
| | | |
|---|
| | | self.beam_search.event_bg_token = tokenizer.encode( |
|---|
| | | DecodingOptions.get("gain_tokens_bg", "<|Speech|><|BGM|><|Applause|><|Laughter|>"), |
|---|
| | | allowed_special="all", |
|---|
| | | ) |
|---|
| | | self.beam_search.event_ed_token = tokenizer.encode( |
|---|
| | | DecodingOptions.get("gain_tokens_ed", "<|/Speech|><|/BGM|><|/Applause|><|/Laughter|>"), |
|---|
| | | allowed_special="all", |
|---|
| | | ) |
|---|
| | | self.beam_search.event_score_ga = DecodingOptions.get("gain_tokens_score", [1, 1, 1, 1]) |
|---|
| | | |
|---|
| | | encoder_out, encoder_out_lens = self.encode( |
|---|
| | | speech[None, :, :].permute(0, 2, 1), speech_lengths |
|---|
| | | ) |
|---|
| | | |
|---|
| | | if text_token_int is not None: |
|---|
| | | i = 0 |
|---|
| | | results = [] |
|---|
| | | ibest_writer = None |
|---|
| | | if kwargs.get("output_dir") is not None: |
|---|
| | | if not hasattr(self, "writer"): |
|---|
| | | self.writer = DatadirWriter(kwargs.get("output_dir")) |
|---|
| | | ibest_writer = self.writer[f"1best_recog"] |
|---|
| | | |
|---|
| | | # 1. Forward decoder |
|---|
| | | ys_pad = torch.tensor(sos_int + text_token_int, dtype=torch.int64).to(kwargs["device"])[ |
|---|
| | | None, : |
|---|
| | | ] |
|---|
| | | ys_pad_lens = torch.tensor([len(sos_int + text_token_int)], dtype=torch.int64).to( |
|---|
| | | kwargs["device"] |
|---|
| | | )[None, :] |
|---|
| | | decoder_out = self.model.decoder( |
|---|
| | | x=ys_pad, xa=encoder_out, hlens=encoder_out_lens, ys_in_lens=ys_pad_lens |
|---|
| | | ) |
|---|
| | | |
|---|
| | | token_int = decoder_out.argmax(-1)[0, :].tolist() |
|---|
| | | text = tokenizer.decode(token_int) |
|---|
| | | |
|---|
| | | result_i = {"key": key[i], "text": text} |
|---|
| | | results.append(result_i) |
|---|
| | | |
|---|
| | | if ibest_writer is not None: |
|---|
| | | # ibest_writer["token"][key[i]] = " ".join(token) |
|---|
| | | ibest_writer["text"][key[i]] = text |
|---|
| | | return results, meta_data |
|---|
| | | |
|---|
| | | # c. Passed the encoder result and the beam search |
|---|
| | | nbest_hyps = self.beam_search( |
|---|
| | | x=encoder_out[0], |
|---|
| | | maxlenratio=kwargs.get("maxlenratio", 0.0), |
|---|
| | | minlenratio=kwargs.get("minlenratio", 0.0), |
|---|
| | | ) |
|---|
| | | |
|---|
| | | nbest_hyps = nbest_hyps[: self.nbest] |
|---|
| | | |
|---|
| | | results = [] |
|---|
| | | b, n, d = encoder_out.size() |
|---|
| | | for i in range(b): |
|---|
| | | |
|---|
| | | for nbest_idx, hyp in enumerate(nbest_hyps): |
|---|
| | | ibest_writer = None |
|---|
| | | if kwargs.get("output_dir") is not None: |
|---|
| | | if not hasattr(self, "writer"): |
|---|
| | | self.writer = DatadirWriter(kwargs.get("output_dir")) |
|---|
| | | ibest_writer = self.writer[f"{nbest_idx + 1}best_recog"] |
|---|
| | | |
|---|
| | | # remove sos/eos and get results |
|---|
| | | last_pos = -1 |
|---|
| | | if isinstance(hyp.yseq, list): |
|---|
| | | token_int = hyp.yseq[1:last_pos] |
|---|
| | | else: |
|---|
| | | token_int = hyp.yseq[1:last_pos].tolist() |
|---|
| | | |
|---|
| | | # # remove blank symbol id, which is assumed to be 0 |
|---|
| | | # token_int = list( |
|---|
| | | # filter( |
|---|
| | | # lambda x: x != self.eos and x != self.sos and x != self.blank_id, token_int |
|---|
| | | # ) |
|---|
| | | # ) |
|---|
| | | |
|---|
| | | # Change integer-ids to tokens |
|---|
| | | # token = tokenizer.ids2tokens(token_int) |
|---|
| | | text = tokenizer.decode(token_int) |
|---|
| | | |
|---|
| | | result_i = {"key": key[i], "text": text} |
|---|
| | | results.append(result_i) |
|---|
| | | |
|---|
| | | if ibest_writer is not None: |
|---|
| | | # ibest_writer["token"][key[i]] = " ".join(token) |
|---|
| | | ibest_writer["text"][key[i]] = text |
|---|
| | | |
|---|
| | | return results, meta_data |
|---|
