From 33d3d2084403fd34b79c835d2f2fe04f6cd8f738 Mon Sep 17 00:00:00 2001
From: 游雁 <zhifu.gzf@alibaba-inc.com>
Date: 星期三, 13 九月 2023 09:33:54 +0800
Subject: [PATCH] Merge branch 'main' of github.com:alibaba-damo-academy/FunASR add
---
funasr/models/e2e_asr_transducer.py | 52 +++++++++++++++-------------------------------------
1 files changed, 15 insertions(+), 37 deletions(-)
diff --git a/funasr/models/e2e_asr_transducer.py b/funasr/models/e2e_asr_transducer.py
index a5aaa6c..bffd54d 100644
--- a/funasr/models/e2e_asr_transducer.py
+++ b/funasr/models/e2e_asr_transducer.py
@@ -6,8 +6,9 @@
import torch
from packaging.version import parse as V
-from typeguard import check_argument_types
-
+from funasr.losses.label_smoothing_loss import (
+ LabelSmoothingLoss, # noqa: H301
+)
from funasr.models.frontend.abs_frontend import AbsFrontend
from funasr.models.specaug.abs_specaug import AbsSpecAug
from funasr.models.decoder.rnnt_decoder import RNNTDecoder
@@ -15,9 +16,11 @@
from funasr.models.encoder.abs_encoder import AbsEncoder
from funasr.models.joint_net.joint_network import JointNetwork
from funasr.modules.nets_utils import get_transducer_task_io
+from funasr.modules.nets_utils import th_accuracy
+from funasr.modules.add_sos_eos import add_sos_eos
from funasr.layers.abs_normalize import AbsNormalize
from funasr.torch_utils.device_funcs import force_gatherable
-from funasr.train.abs_espnet_model import AbsESPnetModel
+from funasr.models.base_model import FunASRModel
if V(torch.__version__) >= V("1.6.0"):
from torch.cuda.amp import autocast
@@ -28,7 +31,7 @@
yield
-class TransducerModel(AbsESPnetModel):
+class TransducerModel(FunASRModel):
"""ESPnet2ASRTransducerModel module definition.
Args:
@@ -82,8 +85,6 @@
"""Construct an ESPnetASRTransducerModel object."""
super().__init__()
- assert check_argument_types()
-
# The following labels ID are reserved: 0 (blank) and vocab_size - 1 (sos/eos)
self.blank_id = 0
self.vocab_size = vocab_size
@@ -108,7 +109,7 @@
self.use_auxiliary_lm_loss = auxiliary_lm_loss_weight > 0
if self.use_auxiliary_ctc:
- self.ctc_lin = torch.nn.Linear(encoder.output_size, vocab_size)
+ self.ctc_lin = torch.nn.Linear(encoder.output_size(), vocab_size)
self.ctc_dropout_rate = auxiliary_ctc_dropout_rate
if self.use_auxiliary_lm_loss:
@@ -162,7 +163,9 @@
# 1. Encoder
encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)
-
+ if hasattr(self.encoder, 'overlap_chunk_cls') and self.encoder.overlap_chunk_cls is not None:
+ encoder_out, encoder_out_lens = self.encoder.overlap_chunk_cls.remove_chunk(encoder_out, encoder_out_lens,
+ chunk_outs=None)
# 2. Transducer-related I/O preparation
decoder_in, target, t_len, u_len = get_transducer_task_io(
text,
@@ -350,11 +353,6 @@
"""
if self.criterion_transducer is None:
try:
- # from warprnnt_pytorch import RNNTLoss
- # self.criterion_transducer = RNNTLoss(
- # reduction="mean",
- # fastemit_lambda=self.fastemit_lambda,
- # )
from warp_rnnt import rnnt_loss as RNNTLoss
self.criterion_transducer = RNNTLoss
@@ -365,12 +363,6 @@
)
exit(1)
- # loss_transducer = self.criterion_transducer(
- # joint_out,
- # target,
- # t_len,
- # u_len,
- # )
log_probs = torch.log_softmax(joint_out, dim=-1)
loss_transducer = self.criterion_transducer(
@@ -483,7 +475,7 @@
return loss_lm
-class UnifiedTransducerModel(AbsESPnetModel):
+class UnifiedTransducerModel(FunASRModel):
"""ESPnet2ASRTransducerModel module definition.
Args:
vocab_size: Size of complete vocabulary (w/ EOS and blank included).
@@ -540,8 +532,6 @@
"""Construct an ESPnetASRTransducerModel object."""
super().__init__()
- assert check_argument_types()
-
# The following labels ID are reserved: 0 (blank) and vocab_size - 1 (sos/eos)
self.blank_id = 0
@@ -577,7 +567,7 @@
self.use_auxiliary_lm_loss = auxiliary_lm_loss_weight > 0
if self.use_auxiliary_ctc:
- self.ctc_lin = torch.nn.Linear(encoder.output_size, vocab_size)
+ self.ctc_lin = torch.nn.Linear(encoder.output_size(), vocab_size)
self.ctc_dropout_rate = auxiliary_ctc_dropout_rate
if self.use_auxiliary_att:
@@ -636,7 +626,6 @@
batch_size = speech.shape[0]
text = text[:, : text_lengths.max()]
- #print(speech.shape)
# 1. Encoder
encoder_out, encoder_out_chunk, encoder_out_lens = self.encode(speech, speech_lengths)
@@ -707,8 +696,8 @@
loss_lm = self._calc_lm_loss(decoder_out, target)
loss_trans = loss_trans_utt + loss_trans_chunk
- loss_ctc = loss_ctc + loss_ctc_chunk
- loss_ctc = loss_att + loss_att_chunk
+ loss_ctc = loss_ctc + loss_ctc_chunk
+ loss_att = loss_att + loss_att_chunk
loss = (
self.transducer_weight * loss_trans
@@ -853,11 +842,6 @@
"""
if self.criterion_transducer is None:
try:
- # from warprnnt_pytorch import RNNTLoss
- # self.criterion_transducer = RNNTLoss(
- # reduction="mean",
- # fastemit_lambda=self.fastemit_lambda,
- # )
from warp_rnnt import rnnt_loss as RNNTLoss
self.criterion_transducer = RNNTLoss
@@ -868,12 +852,6 @@
)
exit(1)
- # loss_transducer = self.criterion_transducer(
- # joint_out,
- # target,
- # t_len,
- # u_len,
- # )
log_probs = torch.log_softmax(joint_out, dim=-1)
loss_transducer = self.criterion_transducer(
--
Gitblit v1.9.1