websocket 2pass bugfix

游雁

2023-05-19 219c2482ab755fbd4e49dfbdee91bf1a8a4ec49a

 funasr/models/e2e_uni_asr.py

@@ -25,7 +25,7 @@
from funasr.models.specaug.abs_specaug import AbsSpecAug
from funasr.layers.abs_normalize import AbsNormalize
from funasr.torch_utils.device_funcs import force_gatherable
from funasr.train.abs_espnet_model import AbsESPnetModel
from funasr.models.base_model import FunASRModel
from funasr.modules.streaming_utils.chunk_utilis import sequence_mask
from funasr.models.predictor.cif import mae_loss

@@ -38,9 +38,9 @@
        yield


class UniASR(AbsESPnetModel):
class UniASR(FunASRModel):
    """
    Author: Speech Lab, Alibaba Group, China
    Author: Speech Lab of DAMO Academy, Alibaba Group
    """

    def __init__(
@@ -179,7 +179,6 @@
        decoding_ind: int = None,
    ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:
        """Frontend + Encoder + Decoder + Calc loss

        Args:
                        speech: (Batch, Length, ...)
                        speech_lengths: (Batch, )
@@ -198,16 +197,15 @@

        # for data-parallel
        text = text[:, : text_lengths.max()]
        speech = speech[:, :speech_lengths.max(), :]
        speech = speech[:, :speech_lengths.max()]

        ind = self.encoder.overlap_chunk_cls.random_choice(self.training, decoding_ind)
        speech_raw = speech.clone().to(speech.device)
        # 1. Encoder
        if self.enable_maas_finetune:
            with torch.no_grad():
                encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind)
                speech_raw, encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind)
        else:
            encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind)
            speech_raw, encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind)

        intermediate_outs = None
        if isinstance(encoder_out, tuple):
@@ -470,7 +468,6 @@
        self, speech: torch.Tensor, speech_lengths: torch.Tensor, ind: int = 0,
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        """Frontend + Encoder. Note that this method is used by asr_inference.py

        Args:
                        speech: (Batch, Length, ...)
                        speech_lengths: (Batch, )
@@ -486,7 +483,7 @@
            # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN
            if self.normalize is not None:
                feats, feats_lengths = self.normalize(feats, feats_lengths)

        speech_raw = feats.clone().to(feats.device)
        # Pre-encoder, e.g. used for raw input data
        if self.preencoder is not None:
            feats, feats_lengths = self.preencoder(feats, feats_lengths)
@@ -523,7 +520,7 @@
        if intermediate_outs is not None:
            return (encoder_out, intermediate_outs), encoder_out_lens

        return encoder_out, encoder_out_lens
        return speech_raw, encoder_out, encoder_out_lens

    def encode2(
        self,
@@ -534,7 +531,6 @@
        ind: int = 0,
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        """Frontend + Encoder. Note that this method is used by asr_inference.py

        Args:
                        speech: (Batch, Length, ...)
                        speech_lengths: (Batch, )
@@ -628,9 +624,7 @@
        ys_pad_lens: torch.Tensor,
    ) -> torch.Tensor:
        """Compute negative log likelihood(nll) from transformer-decoder

        Normally, this function is called in batchify_nll.

        Args:
                        encoder_out: (Batch, Length, Dim)
                        encoder_out_lens: (Batch,)
@@ -667,7 +661,6 @@
        batch_size: int = 100,
    ):
        """Compute negative log likelihood(nll) from transformer-decoder

        To avoid OOM, this fuction seperate the input into batches.
        Then call nll for each batch and combine and return results.
        Args:
@@ -1073,4 +1066,3 @@
            ys_hat = self.ctc2.argmax(encoder_out).data
            cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True)
        return loss_ctc, cer_ctc