contextual&seaco ONNX export (#1481)

Shi Xian

2024-03-13 e04489ce4c0fd0095d0c79ef8f504f425e0435a8

 funasr/models/sond/encoder/resnet34_encoder.py

@@ -245,147 +245,6 @@

        return features, resnet_out_lens

    def gen_tf2torch_map_dict(self):
        tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch
        tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf
        train_steps = self.tf_train_steps
        map_dict_local = {
            # torch: conv1d.weight in "out_channel in_channel kernel_size"
            # tf   : conv1d.weight in "kernel_size in_channel out_channel"
            # torch: linear.weight in "out_channel in_channel"
            # tf   :  dense.weight in "in_channel out_channel"
            "{}.pre_conv.weight".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv/kernel".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": (3, 2, 0, 1),
                 },
            "{}.pre_conv_bn.bias".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/beta".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.weight".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/gamma".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.running_mean".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/moving_mean".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.running_var".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/moving_variance".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.num_batches_tracked".format(tensor_name_prefix_torch): train_steps
        }
        for layer_idx in range(3):
            map_dict_local.update({
                "{}.resnet{}_dense.weight".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_dense/kernel".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": (2, 1, 0) if layer_idx == 0 else (1, 0),
                     },
                "{}.resnet{}_dense.bias".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_dense/bias".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.weight".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/gamma".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.bias".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/beta".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.running_mean".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/moving_mean".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.running_var".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/moving_variance".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.num_batches_tracked".format(tensor_name_prefix_torch, layer_idx): train_steps
            })

        for block_idx in range(len(self.layers_in_block)):
            for layer_idx in range(self.layers_in_block[block_idx]):
                for i in ["1", "2", "_sc"]:
                    map_dict_local.update({
                        "{}.block_{}.layer_{}.conv{}.weight".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/conv{}/kernel".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": (3, 2, 0, 1),
                             },
                        "{}.block_{}.layer_{}.bn{}.weight".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/gamma".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.bias".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/beta".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.running_mean".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/moving_mean".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.running_var".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/moving_variance".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.num_batches_tracked".format(tensor_name_prefix_torch, block_idx, layer_idx, i): train_steps,
                    })

        return map_dict_local

    def convert_tf2torch(self,
                         var_dict_tf,
                         var_dict_torch,
                         ):

        map_dict = self.gen_tf2torch_map_dict()

        var_dict_torch_update = dict()
        for name in sorted(var_dict_torch.keys(), reverse=False):
            if name.startswith(self.tf2torch_tensor_name_prefix_torch):
                if name in map_dict:
                    if "num_batches_tracked" not in name:
                        name_tf = map_dict[name]["name"]
                        data_tf = var_dict_tf[name_tf]
                        if map_dict[name]["squeeze"] is not None:
                            data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"])
                        if map_dict[name]["transpose"] is not None:
                            data_tf = np.transpose(data_tf, map_dict[name]["transpose"])
                        data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
                        assert var_dict_torch[name].size() == data_tf.size(), \
                            "{}, {}, {} != {}".format(name, name_tf,
                                                      var_dict_torch[name].size(), data_tf.size())
                        var_dict_torch_update[name] = data_tf
                        logging.info("torch tensor: {}, {}, loading from tf tensor: {}, {}".format(
                            name, data_tf.size(), name_tf, var_dict_tf[name_tf].shape
                        ))
                    else:
                        var_dict_torch_update[name] = torch.Tensor(map_dict[name]).type(torch.int64).to("cpu")
                        logging.info("torch tensor: {}, manually assigning to: {}".format(
                            name, map_dict[name]
                        ))
                else:
                    logging.warning("{} is missed from tf checkpoint".format(name))

        return var_dict_torch_update


class ResNet34Diar(ResNet34):
    def __init__(
@@ -477,147 +336,6 @@
        endpoints["resnet2_bn"] = features

        return endpoints[self.embedding_node], ilens, None

    def gen_tf2torch_map_dict(self):
        tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch
        tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf
        train_steps = 300000
        map_dict_local = {
            # torch: conv1d.weight in "out_channel in_channel kernel_size"
            # tf   : conv1d.weight in "kernel_size in_channel out_channel"
            # torch: linear.weight in "out_channel in_channel"
            # tf   :  dense.weight in "in_channel out_channel"
            "{}.pre_conv.weight".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv/kernel".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": (3, 2, 0, 1),
                 },
            "{}.pre_conv_bn.bias".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/beta".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.weight".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/gamma".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.running_mean".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/moving_mean".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.running_var".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/moving_variance".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.num_batches_tracked".format(tensor_name_prefix_torch): train_steps
        }
        for layer_idx in range(3):
            map_dict_local.update({
                "{}.resnet{}_dense.weight".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_dense/kernel".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": (3, 2, 0, 1) if layer_idx == 0 else (1, 0),
                     },
                "{}.resnet{}_dense.bias".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_dense/bias".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.weight".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/gamma".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.bias".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/beta".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.running_mean".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/moving_mean".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.running_var".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/moving_variance".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.num_batches_tracked".format(tensor_name_prefix_torch, layer_idx): train_steps
            })

        for block_idx in range(len(self.layers_in_block)):
            for layer_idx in range(self.layers_in_block[block_idx]):
                for i in ["1", "2", "_sc"]:
                    map_dict_local.update({
                        "{}.block_{}.layer_{}.conv{}.weight".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/conv{}/kernel".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": (3, 2, 0, 1),
                             },
                        "{}.block_{}.layer_{}.bn{}.weight".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/gamma".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.bias".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/beta".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.running_mean".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/moving_mean".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.running_var".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/moving_variance".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.num_batches_tracked".format(tensor_name_prefix_torch, block_idx, layer_idx, i): train_steps,
                    })

        return map_dict_local

    def convert_tf2torch(self,
                         var_dict_tf,
                         var_dict_torch,
                         ):

        map_dict = self.gen_tf2torch_map_dict()

        var_dict_torch_update = dict()
        for name in sorted(var_dict_torch.keys(), reverse=False):
            if name.startswith(self.tf2torch_tensor_name_prefix_torch):
                if name in map_dict:
                    if "num_batches_tracked" not in name:
                        name_tf = map_dict[name]["name"]
                        data_tf = var_dict_tf[name_tf]
                        if map_dict[name]["squeeze"] is not None:
                            data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"])
                        if map_dict[name]["transpose"] is not None:
                            data_tf = np.transpose(data_tf, map_dict[name]["transpose"])
                        data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
                        assert var_dict_torch[name].size() == data_tf.size(), \
                            "{}, {}, {} != {}".format(name, name_tf,
                                                      var_dict_torch[name].size(), data_tf.size())
                        var_dict_torch_update[name] = data_tf
                        logging.info("torch tensor: {}, {}, loading from tf tensor: {}, {}".format(
                            name, data_tf.size(), name_tf, var_dict_tf[name_tf].shape
                        ))
                    else:
                        var_dict_torch_update[name] = torch.Tensor(map_dict[name]).type(torch.int64).to("cpu")
                        logging.info("torch tensor: {}, manually assigning to: {}".format(
                            name, map_dict[name]
                        ))
                else:
                    logging.warning("{} is missed from tf checkpoint".format(name))

        return var_dict_torch_update


class ResNet34SpL2RegDiar(ResNet34_SP_L2Reg):
@@ -711,143 +429,3 @@

        return endpoints[self.embedding_node], ilens, None

    def gen_tf2torch_map_dict(self):
        tensor_name_prefix_torch = self.tf2torch_tensor_name_prefix_torch
        tensor_name_prefix_tf = self.tf2torch_tensor_name_prefix_tf
        train_steps = 720000
        map_dict_local = {
            # torch: conv1d.weight in "out_channel in_channel kernel_size"
            # tf   : conv1d.weight in "kernel_size in_channel out_channel"
            # torch: linear.weight in "out_channel in_channel"
            # tf   :  dense.weight in "in_channel out_channel"
            "{}.pre_conv.weight".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv/kernel".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": (3, 2, 0, 1),
                 },
            "{}.pre_conv_bn.bias".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/beta".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.weight".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/gamma".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.running_mean".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/moving_mean".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.running_var".format(tensor_name_prefix_torch):
                {"name": "{}/pre_conv_bn/moving_variance".format(tensor_name_prefix_tf),
                 "squeeze": None,
                 "transpose": None,
                 },
            "{}.pre_conv_bn.num_batches_tracked".format(tensor_name_prefix_torch): train_steps
        }
        for layer_idx in range(3):
            map_dict_local.update({
                "{}.resnet{}_dense.weight".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_dense/kernel".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": (2, 1, 0) if layer_idx == 0 else (1, 0),
                     },
                "{}.resnet{}_dense.bias".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_dense/bias".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.weight".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/gamma".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.bias".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/beta".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.running_mean".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/moving_mean".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.running_var".format(tensor_name_prefix_torch, layer_idx):
                    {"name": "{}/resnet{}_bn/moving_variance".format(tensor_name_prefix_tf, layer_idx),
                     "squeeze": None,
                     "transpose": None,
                     },
                "{}.resnet{}_bn.num_batches_tracked".format(tensor_name_prefix_torch, layer_idx): train_steps
            })

        for block_idx in range(len(self.layers_in_block)):
            for layer_idx in range(self.layers_in_block[block_idx]):
                for i in ["1", "2", "_sc"]:
                    map_dict_local.update({
                        "{}.block_{}.layer_{}.conv{}.weight".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/conv{}/kernel".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": (3, 2, 0, 1),
                             },
                        "{}.block_{}.layer_{}.bn{}.weight".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/gamma".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.bias".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/beta".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.running_mean".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/moving_mean".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.running_var".format(tensor_name_prefix_torch, block_idx, layer_idx, i):
                            {"name": "{}/block_{}/layer_{}/bn{}/moving_variance".format(tensor_name_prefix_tf, block_idx, layer_idx, i),
                             "squeeze": None,
                             "transpose": None,
                             },
                        "{}.block_{}.layer_{}.bn{}.num_batches_tracked".format(tensor_name_prefix_torch, block_idx, layer_idx, i): train_steps,
                    })

        return map_dict_local

    def convert_tf2torch(self,
                         var_dict_tf,
                         var_dict_torch,
                         ):

        map_dict = self.gen_tf2torch_map_dict()

        var_dict_torch_update = dict()
        for name in sorted(var_dict_torch.keys(), reverse=False):
            if name.startswith(self.tf2torch_tensor_name_prefix_torch):
                if name in map_dict:
                    if "num_batches_tracked" not in name:
                        name_tf = map_dict[name]["name"]
                        data_tf = var_dict_tf[name_tf]
                        if map_dict[name]["squeeze"] is not None:
                            data_tf = np.squeeze(data_tf, axis=map_dict[name]["squeeze"])
                        if map_dict[name]["transpose"] is not None:
                            data_tf = np.transpose(data_tf, map_dict[name]["transpose"])
                        data_tf = torch.from_numpy(data_tf).type(torch.float32).to("cpu")
                        assert var_dict_torch[name].size() == data_tf.size(), \
                            "{}, {}, {} != {}".format(name, name_tf,
                                                      var_dict_torch[name].size(), data_tf.size())
                        var_dict_torch_update[name] = data_tf
                        logging.info("torch tensor: {}, {}, loading from tf tensor: {}, {}".format(
                            name, data_tf.size(), name_tf, var_dict_tf[name_tf].shape
                        ))
                    else:
                        var_dict_torch_update[name] = torch.from_numpy(np.array(map_dict[name])).type(torch.int64).to("cpu")
                        logging.info("torch tensor: {}, manually assigning to: {}".format(
                            name, map_dict[name]
                        ))
                else:
                    logging.warning("{} is missed from tf checkpoint".format(name))

        return var_dict_torch_update