Dev gzf exp (#1654)

zhifu gao

2024-04-24 861147c7308b91068ffa02724fdf74ee623a909e

 funasr/models/sond/encoder/resnet34_encoder.py

@@ -64,8 +64,9 @@
        self.num_layer = num_layer

        for i in range(num_layer):
            layer = BasicLayer(in_filters if i == 0 else filters, filters,
                               stride if i == 0 else 1, bn_momentum)
            layer = BasicLayer(
                in_filters if i == 0 else filters, filters, stride if i == 0 else 1, bn_momentum
            )
            self.add_module("layer_{}".format(i), layer)

    def forward(self, xs_pad, ilens):
@@ -98,8 +99,12 @@

        pre_filters = filters_in_block[0]
        if use_head_conv:
            self.pre_conv = torch.nn.Conv2d(1, pre_filters, 3, 1, 1, bias=False, padding_mode="zeros")
            self.pre_conv_bn = torch.nn.BatchNorm2d(pre_filters, eps=1e-3, momentum=batchnorm_momentum)
            self.pre_conv = torch.nn.Conv2d(
                1, pre_filters, 3, 1, 1, bias=False, padding_mode="zeros"
            )
            self.pre_conv_bn = torch.nn.BatchNorm2d(
                pre_filters, eps=1e-3, momentum=batchnorm_momentum
            )

        if use_head_maxpool:
            self.head_maxpool = torch.nn.MaxPool2d(3, 1, padding=1)
@@ -110,15 +115,19 @@
            else:
                in_filters = filters_in_block[i-1]

            block = BasicBlock(in_filters,
            block = BasicBlock(
                in_filters,
                               filters=filters_in_block[i],
                               num_layer=layers_in_block[i],
                               stride=1 if i == 0 else 2,
                               bn_momentum=batchnorm_momentum)
                bn_momentum=batchnorm_momentum,
            )
            self.add_module("block_{}".format(i), block)

        self.resnet0_dense = torch.nn.Conv2d(filters_in_block[-1], num_nodes_pooling_layer, 1)
        self.resnet0_bn = torch.nn.BatchNorm2d(num_nodes_pooling_layer, eps=1e-3, momentum=batchnorm_momentum)
        self.resnet0_bn = torch.nn.BatchNorm2d(
            num_nodes_pooling_layer, eps=1e-3, momentum=batchnorm_momentum
        )

        self.time_ds_ratio = 8

@@ -126,15 +135,15 @@
        return self.num_nodes_pooling_layer

    def forward(
            self,
            xs_pad: torch.Tensor,
            ilens: torch.Tensor,
            prev_states: torch.Tensor = None
        self, xs_pad: torch.Tensor, ilens: torch.Tensor, prev_states: torch.Tensor = None
    ) -> Tuple[torch.Tensor, torch.Tensor]:

        features = xs_pad
        assert features.size(-1) == self.input_size, \
            "Dimension of features {} doesn't match the input_size {}.".format(features.size(-1), self.input_size)
        assert (
            features.size(-1) == self.input_size
        ), "Dimension of features {} doesn't match the input_size {}.".format(
            features.size(-1), self.input_size
        )
        features = torch.unsqueeze(features, dim=1)
        if self.use_head_conv:
            features = self.pre_conv(features)
@@ -154,6 +163,7 @@
        features = self.resnet0_bn(features)

        return features, resnet_out_lens


# Note: For training, this implement is not equivalent to tf because of the kernel_regularizer in tf.layers.
# TODO: implement kernel_regularizer in torch with munal loss addition or weigth_decay in the optimizer
@@ -185,8 +195,12 @@

        pre_filters = filters_in_block[0]
        if use_head_conv:
            self.pre_conv = torch.nn.Conv2d(1, pre_filters, 3, 1, 1, bias=False, padding_mode="zeros")
            self.pre_conv_bn = torch.nn.BatchNorm2d(pre_filters, eps=1e-3, momentum=batchnorm_momentum)
            self.pre_conv = torch.nn.Conv2d(
                1, pre_filters, 3, 1, 1, bias=False, padding_mode="zeros"
            )
            self.pre_conv_bn = torch.nn.BatchNorm2d(
                pre_filters, eps=1e-3, momentum=batchnorm_momentum
            )

        if use_head_maxpool:
            self.head_maxpool = torch.nn.MaxPool2d(3, 1, padding=1)
@@ -197,15 +211,21 @@
            else:
                in_filters = filters_in_block[i-1]

            block = BasicBlock(in_filters,
            block = BasicBlock(
                in_filters,
                               filters=filters_in_block[i],
                               num_layer=layers_in_block[i],
                               stride=1 if i == 0 else 2,
                               bn_momentum=batchnorm_momentum)
                bn_momentum=batchnorm_momentum,
            )
            self.add_module("block_{}".format(i), block)

        self.resnet0_dense = torch.nn.Conv1d(filters_in_block[-1] * input_size // 8, num_nodes_pooling_layer, 1)
        self.resnet0_bn = torch.nn.BatchNorm1d(num_nodes_pooling_layer, eps=1e-3, momentum=batchnorm_momentum)
        self.resnet0_dense = torch.nn.Conv1d(
            filters_in_block[-1] * input_size // 8, num_nodes_pooling_layer, 1
        )
        self.resnet0_bn = torch.nn.BatchNorm1d(
            num_nodes_pooling_layer, eps=1e-3, momentum=batchnorm_momentum
        )

        self.time_ds_ratio = 8

@@ -213,15 +233,15 @@
        return self.num_nodes_pooling_layer

    def forward(
            self,
            xs_pad: torch.Tensor,
            ilens: torch.Tensor,
            prev_states: torch.Tensor = None
        self, xs_pad: torch.Tensor, ilens: torch.Tensor, prev_states: torch.Tensor = None
    ) -> Tuple[torch.Tensor, torch.Tensor]:

        features = xs_pad
        assert features.size(-1) == self.input_size, \
            "Dimension of features {} doesn't match the input_size {}.".format(features.size(-1), self.input_size)
        assert (
            features.size(-1) == self.input_size
        ), "Dimension of features {} doesn't match the input_size {}.".format(
            features.size(-1), self.input_size
        )
        features = torch.unsqueeze(features, dim=1)
        if self.use_head_conv:
            features = self.pre_conv(features)
@@ -263,7 +283,7 @@
            pool_size=20,
            stride=1,
            tf2torch_tensor_name_prefix_torch="encoder",
            tf2torch_tensor_name_prefix_tf="seq2seq/speech_encoder"
        tf2torch_tensor_name_prefix_tf="seq2seq/speech_encoder",
    ):
        """
        Author: Speech Lab, Alibaba Group, China
@@ -291,10 +311,14 @@
        self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf

        self.resnet1_dense = torch.nn.Linear(num_nodes_pooling_layer * 2, num_nodes_resnet1)
        self.resnet1_bn = torch.nn.BatchNorm1d(num_nodes_resnet1, eps=1e-3, momentum=batchnorm_momentum)
        self.resnet1_bn = torch.nn.BatchNorm1d(
            num_nodes_resnet1, eps=1e-3, momentum=batchnorm_momentum
        )

        self.resnet2_dense = torch.nn.Linear(num_nodes_resnet1, num_nodes_last_layer)
        self.resnet2_bn = torch.nn.BatchNorm1d(num_nodes_last_layer, eps=1e-3, momentum=batchnorm_momentum)
        self.resnet2_bn = torch.nn.BatchNorm1d(
            num_nodes_last_layer, eps=1e-3, momentum=batchnorm_momentum
        )

    def output_size(self) -> int:
        if self.embedding_node.startswith("resnet1"):
@@ -317,7 +341,9 @@
        if self.pooling_type == "frame_gsp":
            features = statistic_pooling(res_out, ilens, (3, ))
        else:
            features, ilens = windowed_statistic_pooling(res_out, ilens, (2, 3), self.pool_size, self.stride)
            features, ilens = windowed_statistic_pooling(
                res_out, ilens, (2, 3), self.pool_size, self.stride
            )
        features = features.transpose(1, 2)
        endpoints["pooling"] = features

@@ -355,7 +381,7 @@
            pool_size=20,
            stride=1,
            tf2torch_tensor_name_prefix_torch="encoder",
            tf2torch_tensor_name_prefix_tf="seq2seq/speech_encoder"
        tf2torch_tensor_name_prefix_tf="seq2seq/speech_encoder",
    ):
        """
        Author: Speech Lab, Alibaba Group, China
@@ -383,10 +409,14 @@
        self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf

        self.resnet1_dense = torch.nn.Linear(num_nodes_pooling_layer * 2, num_nodes_resnet1)
        self.resnet1_bn = torch.nn.BatchNorm1d(num_nodes_resnet1, eps=1e-3, momentum=batchnorm_momentum)
        self.resnet1_bn = torch.nn.BatchNorm1d(
            num_nodes_resnet1, eps=1e-3, momentum=batchnorm_momentum
        )

        self.resnet2_dense = torch.nn.Linear(num_nodes_resnet1, num_nodes_last_layer)
        self.resnet2_bn = torch.nn.BatchNorm1d(num_nodes_last_layer, eps=1e-3, momentum=batchnorm_momentum)
        self.resnet2_bn = torch.nn.BatchNorm1d(
            num_nodes_last_layer, eps=1e-3, momentum=batchnorm_momentum
        )

    def output_size(self) -> int:
        if self.embedding_node.startswith("resnet1"):
@@ -409,7 +439,9 @@
        if self.pooling_type == "frame_gsp":
            features = statistic_pooling(res_out, ilens, (2, ))
        else:
            features, ilens = windowed_statistic_pooling(res_out, ilens, (2, ), self.pool_size, self.stride)
            features, ilens = windowed_statistic_pooling(
                res_out, ilens, (2,), self.pool_size, self.stride
            )
        features = features.transpose(1, 2)
        endpoints["pooling"] = features

@@ -428,4 +460,3 @@
        endpoints["resnet2_bn"] = features

        return endpoints[self.embedding_node], ilens, None