模型拆解（三）：EGNet、FMFINet、MJRBM

文章目录

一、EGNet
- 1.1编码器：VGG16的扩展网络
二、EMFINet
- 2.1编码器：三分支并行卷积编码器
- 2.2CFFM：级联特征融合模块
- 2.3Edge Module：突出边缘提取模块
- 2.4Bridge Module：桥接器
- 2.5解码器：深度特征融合解码器
三、MJRBM
- 3.1编码器：VGG16双分支编码器
- 3.2SBFEM：显著边界特征提取模块
- 3.4分层注意力模块：HAM
- 3.4Aggregation：聚合模块
- 3.5JRBM：联合区域和边界模块
- 3.6解码器：双分支解码器

一、EGNet

论文：EGNet: Edge Guidance Network for Salient Object Detection
论文链接：EGNet: Edge Guidance Network for Salient Object Detection
代码链接：Github
博客链接：CSDN

1.1编码器：VGG16的扩展网络

在这里插入图片描述

vgg.py

#创建网络基础部分
def vgg(cfg, i, batch_norm=False):layers = []in_channels = istage = 1for v in cfg:if v == 'M':stage += 1if stage == 6:layers += [nn.MaxPool2d(kernel_size=3, stride=2, padding=1)]else:layers += [nn.MaxPool2d(kernel_size=3, stride=2, padding=1)]else:if stage == 6:# conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=4, dilation=4, bias=False)conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)else:conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)if batch_norm:layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]else:layers += [conv2d, nn.ReLU(inplace=True)]in_channels = vreturn layers#创建网络扩展部分
class vgg_ex(nn.Module):def __init__(self, cfg, incs=512, padding=1, dilation=1):super(vgg_ex, self).__init__()self.cfg = cfglayers = []for v in self.cfg:# conv2d = nn.Conv2d(incs, v, kernel_size=3, padding=4, dilation=4, bias=False)conv2d = nn.Conv2d(incs, v, kernel_size=3, padding=padding, dilation=dilation, bias=False)layers += [conv2d, nn.ReLU(inplace=True)]incs = vself.ex = nn.Sequential(*layers)for m in self.modules():if isinstance(m, nn.Conv2d):n = m.kernel_size[0] * m.kernel_size[1] * m.out_channelsm.weight.data.normal_(0, 0.01)elif isinstance(m, nn.BatchNorm2d):m.weight.data.fill_(1)m.bias.data.zero_()def forward(self, x):x = self.ex(x)return xclass vgg16(nn.Module):def __init__(self):super(vgg16, self).__init__()#定义VGG16网络参数,数字代表卷积核数,'M'代表平均池化#tun:主网络的层配置#tun_ex:额外的层配置self.cfg = {'tun': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], 'tun_ex': [512, 512, 512]}#需要提取特征的层索引self.extract = [8, 15, 22, 29] # [3, 8, 15, 22, 29]self.extract_ex = [5]#创建网络基础部分base,参数为self.cfg['tun'],输入通道数为3self.base = nn.ModuleList(vgg(self.cfg['tun'], 3))#创建网络扩展部分base_ex,参数为self.cfg['tun_ex'],输入通道数为512self.base_ex = vgg_ex(self.cfg['tun_ex'], 512)#权重初始化for m in self.modules():if isinstance(m, nn.Conv2d):n = m.kernel_size[0] * m.kernel_size[1] * m.out_channelsm.weight.data.normal_(0, 0.01)elif isinstance(m, nn.BatchNorm2d):m.weight.data.fill_(1)m.bias.data.zero_()def load_pretrained_model(self, model):self.base.load_state_dict(model)def forward(self, x, multi=0):#保存提取的特征tmp_x = []for k in range(len(self.base)):x = self.base[k](x)#若层索引在extract列表中,则将该层的输出特征图添加到tmp_x列表中if k in self.extract:tmp_x.append(x)#将主干网络输出x传递给扩展部分base_ex,并将其输出添加到tmp_x中x = self.base_ex(x)tmp_x.append(x)#multi == 1时返回仅包含第一个特征图的列表tmp_y;否则返回tmp_x中的所有提取特征图if multi == 1:tmp_y = []tmp_y.append(tmp_x[0])return tmp_yelse:return tmp_x

之后初始化模型时通过以下方式初始化：

config_vgg = {'convert': [[128,256,512,512,512],[64,128,256,512,512]], 'merge1': [[128, 256, 128, 3,1], [256, 512, 256, 3, 1], [512, 0, 512, 5, 2], [512, 0, 512, 5, 2],[512, 0, 512, 7, 3]], 'merge2': [[128], [256, 512, 512, 512]]}  # no convert layer, no conv6config_resnet = {'convert': [[64,256,512,1024,2048],[128,256,512,512,512]], 'deep_pool': [[512, 512, 256, 256, 128], [512, 256, 256, 128, 128], [False, True, True, True, False], [True, True, True, True, False]], 'score': 256, 'edgeinfo':[[16, 16, 16, 16], 128, [16,8,4,2]],'edgeinfoc':[64,128], 'block': [[512, [16]], [256, [16]], [256, [16]], [128, [16]]], 'fuse': [[16, 16, 16, 16], True], 'fuse_ratio': [[16,1], [8,1], [4,1], [2,1]],  'merge1': [[128, 256, 128, 3,1], [256, 512, 256, 3, 1], [512, 0, 512, 5, 2], [512, 0, 512, 5, 2],[512, 0, 512, 7, 3]], 'merge2': [[128], [256, 512, 512, 512]]}#接受参数base_model_cfg(网络配置), vgg(模型实例)创建编码器及两个合并层,返回vgg, merge1_layers, merge2_layers
def extra_layer(base_model_cfg, vgg):if base_model_cfg == 'vgg':config = config_vggelif base_model_cfg == 'resnet':config = config_resnetmerge1_layers = MergeLayer1(config['merge1'])merge2_layers = MergeLayer2(config['merge2'])return vgg, merge1_layers, merge2_layersif __name__ == '__main__':from torch.autograd import Variable#通过extra_layer()创建包含编码器架构及两个合并层的列表,列表内容用于初始化模型TUNnet = TUN(*extra_layer(vgg(base['tun'], 3), vgg(base['tun_ex'], 512), config['merge_block'], config['fuse'])).cuda()...

模型代码：

# TUN network
class TUN_bone(nn.Module):def __init__(self, base_model_cfg, base, merge1_layers, merge2_layers):super(TUN_bone, self).__init__()self.base_model_cfg = base_model_cfg#VGG版本的模型初始化if self.base_model_cfg == 'vgg':self.base = base# self.base_ex = nn.ModuleList(base_ex)self.merge1 = merge1_layersself.merge2 = merge2_layerselif self.base_model_cfg == 'resnet':self.convert = ConvertLayer(config_resnet['convert'])self.base = baseself.merge1 = merge1_layersself.merge2 = merge2_layersdef forward(self, x):x_size = x.size()[2:]conv2merge = self.base(x)        if self.base_model_cfg == 'resnet':            conv2merge = self.convert(conv2merge)up_edge, edge_feature, up_sal, sal_feature = self.merge1(conv2merge, x_size)up_sal_final = self.merge2(edge_feature, sal_feature, x_size)return up_edge, up_sal, up_sal_final

class MergeLayer1(nn.Module): # list_k: [[64, 512, 64], [128, 512, 128], [256, 0, 256] ... ]def __init__(self, list_k):super(MergeLayer1, self).__init__()self.list_k = list_ktrans, up, score = [], [], []for ik in list_k:if ik[1] > 0:trans.append(nn.Sequential(nn.Conv2d(ik[1], ik[0], 1, 1, bias=False), nn.ReLU(inplace=True)))up.append(nn.Sequential(nn.Conv2d(ik[0], ik[2], ik[3], 1, ik[4]), nn.ReLU(inplace=True), nn.Conv2d(ik[2], ik[2], ik[3], 1, ik[4]), nn.ReLU(inplace=True), nn.Conv2d(ik[2], ik[2], ik[3], 1, ik[4]), nn.ReLU(inplace=True)))score.append(nn.Conv2d(ik[2], 1, 3, 1, 1))trans.append(nn.Sequential(nn.Conv2d(512, 128, 1, 1, bias=False), nn.ReLU(inplace=True)))self.trans, self.up, self.score = nn.ModuleList(trans), nn.ModuleList(up), nn.ModuleList(score)self.relu =nn.ReLU()def forward(self, list_x, x_size):up_edge, up_sal, edge_feature, sal_feature = [], [], [], []num_f = len(list_x)tmp = self.up[num_f - 1](list_x[num_f-1])sal_feature.append(tmp)U_tmp = tmpup_sal.append(F.interpolate(self.score[num_f - 1](tmp), x_size, mode='bilinear', align_corners=True))for j in range(2, num_f ):i = num_f - jif list_x[i].size()[1] < U_tmp.size()[1]:U_tmp = list_x[i] + F.interpolate((self.trans[i](U_tmp)), list_x[i].size()[2:], mode='bilinear', align_corners=True)else:U_tmp = list_x[i] + F.interpolate((U_tmp), list_x[i].size()[2:], mode='bilinear', align_corners=True)tmp = self.up[i](U_tmp)U_tmp = tmpsal_feature.append(tmp)up_sal.append(F.interpolate(self.score[i](tmp), x_size, mode='bilinear', align_corners=True))U_tmp = list_x[0] + F.interpolate((self.trans[-1](sal_feature[0])), list_x[0].size()[2:], mode='bilinear', align_corners=True)tmp = self.up[0](U_tmp)edge_feature.append(tmp)up_edge.append(F.interpolate(self.score[0](tmp), x_size, mode='bilinear', align_corners=True)) return up_edge, edge_feature, up_sal, sal_feature

https://github.com/JXingZhao/EGNet/blob/master/model.py
抽空再看，屎一样的代码！

二、EMFINet

论文：Edge-Aware Multiscale Feature Integration Network for Salient Object Detection in Optical Remote Sensing Images（用于光学遥感图像中显著目标检测的边缘感知多尺度特征集成网络）
论文链接：Edge-Aware Multiscale Feature Integration Network for Salient Object Detection in Optical Remote Sensing Images
代码链接：Github
博客链接：CSDN

2.1编码器：三分支并行卷积编码器

在这里插入图片描述
三分支并行卷积编码器由 $R es N e t - 34$ 改造而来，以三幅不同分辨率的图像作为输入。原始 $R es N e t - 34$ 结构为：

EMFINet使用的卷积分支在此基础上进行了修改：

1.将 $co n v 1$ 修改为 $3 \times 3$ 大小、步幅为1。
2.将 $co n v 1$ 后的最大池化层取消。

EMFINet卷积分支中各个模块定义如下：

1.Conv-B1：conv1+conv2_x
2.Conv-B2：conv3_x
3.Conv-B3：conv4_x
4.Conv-B4：conv5_x
5.Conv-B5：最大池化（ $2 \times 2$ ，stride=2，padding=0）+conv5中的三个基本残差块。

#model/EMFINet.py
class EMFINet(nn.Module):def __init__(self):super(EMFINet,self).__init__()#初始化ResNet34模型resnet = models.resnet34(pretrained=True)#-------------Multiscale Feature Extraction--------------#self.poola = nn.MaxPool2d(2,2,ceil_mode=True)self.poolb = nn.MaxPool2d(4,4,ceil_mode=True)#Conv-B1#conv1(修改后)self.preconv = nn.Conv2d(3,64,3,padding=1)self.prebn = nn.BatchNorm2d(64)self.prerelu = nn.ReLU(inplace=True)#conv2_xself.encoder1 = resnet.layer1#Conv-B2:conv3_xself.encoder2 = resnet.layer2#Conv-B3:conv4_xself.encoder3 = resnet.layer3#Conv-B4:conv5_xself.encoder4 = resnet.layer4#Conv-B5self.poolc = nn.MaxPool2d(2,2,ceil_mode=True)self.encoder5_1 = BasicBlock(512,512)self.encoder5_2 = BasicBlock(512,512)self.encoder5_3 = BasicBlock(512,512)...def forward(self,x):#最大分辨率,对应Convs-1score1_1 = x#使用最大池化下采样得到不同分辨率的图像作为输入#中间分辨率,对应Convs-2score2 = self.poola(score1_1)#最小分辨率,对应Convs-3score3 = self.poolb(score1_1)#-------------Multiscale Feature Extraction--------------##Convs-1#Conv-B1:conv1(修改后)+conv2_xscore1_1 = self.encoder1(self.prerelu(self.prebn(self.preconv(score1_1))))#Conv-B2:conv3_xscore1_2 = self.encoder2(score1_1)#Conv-B3:conv4_xscore1_3 = self.encoder3(score1_2)#Conv-B4:conv5_xscore1_4 = self.encoder4(score1_3)#Conv-B5score1_5 = self.poolc(score1_4)score1_5 = self.encoder5_1(score1_5)score1_5 = self.encoder5_2(score1_5)score1_5 = self.encoder5_3(score1_5)#Convs-2#Conv-B1:conv1(修改后)+conv2_xscore2 = self.encoder1(self.prerelu(self.prebn(self.preconv(score2))))#Conv-B2:conv3_xscore2 = self.encoder2(score2)#Conv-B3:conv4_xscore2 = self.encoder3(score2)#Conv-B4:conv5_xscore2 = self.encoder4(score2)#Conv-B5score2 = self.poolc(score2)score2 = self.encoder5_1(score2)score2 = self.encoder5_2(score2)score2 = self.encoder5_3(score2)#Convs-2score3 = self.encoder1(self.prerelu(self.prebn(self.preconv(score3))))#Conv-B1:conv1(修改后)+conv2_xscore3 = self.encoder2(score3)#Conv-B2:conv3_xscore3 = self.encoder3(score3)#Conv-B3:conv4_xscore3 = self.encoder4(score3)#Conv-B5score3 = self.poolc(score3)score3 = self.encoder5_1(score3)score3 = self.encoder5_2(score3)score3 = self.encoder5_3(score3)...

从三张遥感图像中提取了高级深度语义特征 $\{F^1_5,F^2_5,F^3_5\}$ ，其中，分辨率最大的遥感图像中还提取了多级深度语义特征 $\{F^1_1,F^1_2,F^1_3,F^1_4\}$ 。
代码中的score1_1、score1_2、score1_3、score1_4、score1_5代表 $\{F^1_1,F^1_2,F^1_3,F^1_4,F^1_5\}$ ，score2、score3代表 $\{F^2_5,F^3_5\}$ 。

2.2CFFM：级联特征融合模块

在这里插入图片描述
用于将不同分辨率遥感图像提取的高级深度语义特征 $\{F^1_5,F^2_5,F^3_5\}$ 融合，得到增强后的深度语义特征：

其中， $C o n v$ 包含三个卷积层，每个都带有BN、ReLU。

#model/EMFINet.py
class EMFINet(nn.Module):def __init__(self):...#-------------Cascaded feature fusion module--------------##part 1self.cat_conv11 = nn.Conv2d(512,512,3,padding=1)self.cat_bn11 = nn.BatchNorm2d(512)self.cat_relu11 = nn.ReLU(inplace=True)self.cat_conv12 = nn.Conv2d(512,512,3,padding=1)self.cat_bn12 = nn.BatchNorm2d(512)self.cat_relu12 = nn.ReLU(inplace=True)self.cat_conv13 = nn.Conv2d(512,512,3,padding=1)self.cat_bn13 = nn.BatchNorm2d(512)self.cat_relu13 = nn.ReLU(inplace=True)#part 2self.cat_conv21 = nn.Conv2d(512,512,3,padding=1)self.cat_bn21 = nn.BatchNorm2d(512)self.cat_relu21 = nn.ReLU(inplace=True)self.cat_conv22 = nn.Conv2d(512,512,3,padding=1)self.cat_bn22 = nn.BatchNorm2d(512)self.cat_relu22 = nn.ReLU(inplace=True)self.cat_conv23 = nn.Conv2d(512,512,3,padding=1)self.cat_bn23 = nn.BatchNorm2d(512)self.cat_relu23 = nn.ReLU(inplace=True)#part 3self.cat_conv31 = nn.Conv2d(512,512,3,padding=1)self.cat_bn31 = nn.BatchNorm2d(512)self.cat_relu31 = nn.ReLU(inplace=True)self.cat_conv32 = nn.Conv2d(512,512,3,padding=1)self.cat_bn32 = nn.BatchNorm2d(512)self.cat_relu32 = nn.ReLU(inplace=True)self.cat_conv33 = nn.Conv2d(512,512,3,padding=1)self.cat_bn33 = nn.BatchNorm2d(512)self.cat_relu33 = nn.ReLU(inplace=True)...#-------------Label output--------------#self.label_outconv1 = nn.Conv2d(512,1,1,padding=0)self.label_outconv2 = nn.Conv2d(512,1,1,padding=0)self.label_outconv3 = nn.Conv2d(512,1,1,padding=0)self.label_outconv4 = nn.Conv2d(512,1,1,padding=0)self.label_outconv5 = nn.Conv2d(256,1,1,padding=0)self.label_outconv6 = nn.Conv2d(128,1,1,padding=0)self.label_outconv7 = nn.Conv2d(64,1,1,padding=0)self.label_outconv8 = nn.Conv2d(64,1,1,padding=0)self.label_outconv9 = nn.Conv2d(1024,1,1,padding=0)def forward(self,x):...#-------------Cascaded feature fusion module--------------##将F35输入Conv得到F'35score3 = self.cat_relu11(self.cat_bn11(self.cat_conv11(score3)))score3 = self.cat_relu12(self.cat_bn12(self.cat_conv12(score3)))score3 = self.cat_relu13(self.cat_bn13(self.cat_conv13(score3)))#将F'35上采样到F25大小并相加score3_up1 = self.upscore2(score3)score2 = score3_up1 + score2#得到F'25score2 = self.cat_relu21(self.cat_bn21(self.cat_conv21(score2)))score2 = self.cat_relu22(self.cat_bn22(self.cat_conv22(score2)))score2 = self.cat_relu23(self.cat_bn23(self.cat_conv23(score2)))#将F35、F'25上采样到F15大小并相加score2_up = self.upscore2(score2)score3_up2 = self.upscore4(score3)score1_5 = score2_up + score3_up2 + score1_5#得到F'15score1_5 = self.cat_relu31(self.cat_bn31(self.cat_conv31(score1_5)))score1_5 = self.cat_relu32(self.cat_bn32(self.cat_conv32(score1_5)))score1_5 = self.cat_relu33(self.cat_bn33(self.cat_conv33(score1_5)))....#-------------Label output--------------##深度监督F'35label1_out = self.label_outconv1(score3)label1_out = self.upscore64(label1_out)#深度监督F'25label2_out = self.label_outconv2(score2)label2_out = self.upscore32(label2_out)#深度监督F'15label3_out = self.label_outconv3(score1_5)label3_out = self.upscore16(label3_out)...#-------------Return--------------#return torch.sigmoid(edge_out), torch.sigmoid(label1_out), torch.sigmoid(label2_out), torch.sigmoid(label3_out), torch.sigmoid(labelbdg_out), \torch.sigmoid(label4_out), torch.sigmoid(label5_out), torch.sigmoid(label6_out), torch.sigmoid(label7_out), torch.sigmoid(label8_out)

2.3Edge Module：突出边缘提取模块

在这里插入图片描述

边缘提取模块用于从低级深度特征 $F^1_2$ 和高级深度特征 $\hat{F}^1_5$ 中生成显著边缘图 $E$ 。计算公式：

之后将突出边缘特征图E调整为与多尺度深度特征相同大小，得到 $E_j,j∈\{1,2,3,4,5\}$ ，并将其与多尺度深度特征进行融合。公式：
在这里插入图片描述
其中。 $[,]$ 表Concat操作。为准确生成突出边缘特征图E，模型采用深度监督的方式来指导突出边缘的提取。即，将突出边缘特征图E卷积、上采样、sigmoid处理来获取突出边缘图，再使用交叉熵损失函数来监督。

#model/EMFINet.py
class EMFINet(nn.Module):def __init__(self):...#-------------Edge--------------##Edge Moduleself.up_conv = nn.Conv2d(512,128,3,dilation=8, padding=8)self.up_bn = nn.BatchNorm2d(128)self.edge_conv = nn.Conv2d(128,128,3,padding=1)self.edge_bn = nn.BatchNorm2d(128)self.edge_relu = nn.ReLU(inplace=True)...#-------------Edge output--------------##128通道转为1通道,生成显著边缘图self.edge_outconv1 = nn.Conv2d(128,1,1,padding=0)def forward(self,x):...#-------------Edge--------------##8倍上采样F'1_5score1_5_up = self.upscore8(score1_5)score1_5_up = self.up_bn(self.up_conv(score1_5_up))#元素求和、Conv、BN、ReLUscore_edge = score1_2 +score1_5_upscore_edge = self.edge_relu(self.edge_bn(self.edge_conv(score_edge)))...#-------------Edge output--------------##生成显著边缘图并两倍上采样edge_out = self.edge_outconv1(score_edge)edge_out = self.upscore2(edge_out)#-------------Return--------------##返回显著边缘图用于深度监督return torch.sigmoid(edge_out), torch.sigmoid(label1_out), torch.sigmoid(label2_out), torch.sigmoid(label3_out), torch.sigmoid(labelbdg_out), \torch.sigmoid(label4_out), torch.sigmoid(label5_out), torch.sigmoid(label6_out), torch.sigmoid(label7_out), torch.sigmoid(label8_out)

2.4Bridge Module：桥接器

在这里插入图片描述
桥接器包含了四个膨胀卷积，膨胀率 $r∈\{2,4,8,16\}$ ，为扩大感受野并捕获全局上下文特征，从输入的高级深度语义特征 $\hat{F}^1_5$ 中提取了四个深度特征 $F e a 2 、 F e a 4 、 F e a 8 、 F e a 16$ 并进行串联（用 $c$ 表示）。

class EMFINet(nn.Module):def __init__(self):super(EMFINet,self).__init__()...#-------------Bridge--------------##四个膨胀卷积+BNself.bdg_conv1 = nn.Conv2d(512,128,3,dilation=2, padding=2)self.bdg_bn1 = nn.BatchNorm2d(128)self.bdg_conv2 = nn.Conv2d(512,128,3,dilation=4, padding=4)self.bdg_bn2 = nn.BatchNorm2d(128)self.bdg_conv3 = nn.Conv2d(512,128,3,dilation=8, padding=8)self.bdg_bn3 = nn.BatchNorm2d(128)self.bdg_conv4 = nn.Conv2d(512,128,3,dilation=16, padding=16)self.bdg_bn4 = nn.BatchNorm2d(128)...def forward(self,x):...#-------------Bridge--------------##提取多尺度上下文信息Fea2 = self.bdg_bn1(self.bdg_conv1(score1_5))Fea4 = self.bdg_bn2(self.bdg_conv2(score1_5))Fea8 = self.bdg_bn3(self.bdg_conv3(score1_5))Fea16 = self.bdg_bn4(self.bdg_conv4(score1_5))#多尺度融合score_bdg = torch.cat((Fea2,Fea4,Fea8,Fea16,score1_5),1)...#深度监督labelbdg_out = self.label_outconv9(score_bdg)labelbdg_out = self.upscore16(labelbdg_out)...#-------------Return--------------#return torch.sigmoid(edge_out), torch.sigmoid(label1_out), torch.sigmoid(label2_out), torch.sigmoid(label3_out), torch.sigmoid(labelbdg_out), \torch.sigmoid(label4_out), torch.sigmoid(label5_out), torch.sigmoid(label6_out), torch.sigmoid(label7_out), torch.sigmoid(label8_out)

2.5解码器：深度特征融合解码器

在这里插入图片描述
深度特征聚合模块（Deep Feature Aggregation Module）将多尺度深度特征图 $F^E_i,i∈\{1,2,3,4,5\}$ 转换为显著性图。共包含了五个解码器 $Decoder-Bi,i∈\{1,2,3,4,5\}$ ，每个解码器都由三个卷积模块 $DBi-j,j∈\{1,2,3\}$ 组成，每个卷积模块都包含卷积层、BN、ReLU。解码过程为：
在这里插入图片描述

class EMFINet(nn.Module):def __init__(self):super(EMFINet,self).__init__()...#-------------deep feature aggregation module--------------##self.pool5 = nn.MaxPool2d(8,8,ceil_mode=True)self.mconv5 = nn.Conv2d(1024 + 128,1024,3,padding=1)self.mbn5 = nn.BatchNorm2d(1024)self.mrelu5 = nn.ReLU(inplace=True)self.convd5_1 = nn.Conv2d(1024,512,3,padding=1)self.bnd5_1 = nn.BatchNorm2d(512)self.relud5_1 = nn.ReLU(inplace=True)self.convd5_2 = nn.Conv2d(512,512,3,padding=1)self.bnd5_2 = nn.BatchNorm2d(512)self.relud5_2 = nn.ReLU(inplace=True)self.convd5_3 = nn.Conv2d(512,512,3,padding=1)self.bnd5_3 = nn.BatchNorm2d(512)self.relud5_3 = nn.ReLU(inplace=True)#stage 4self.pool4 = nn.MaxPool2d(4,4,ceil_mode=True)self.mconv4 = nn.Conv2d(512 + 128,512,3,padding=1)self.mbn4 = nn.BatchNorm2d(512)self.mrelu4 = nn.ReLU(inplace=True)self.convd4_1 = nn.Conv2d(1024,512,3,padding=1)self.bnd4_1 = nn.BatchNorm2d(512)self.relud4_1 = nn.ReLU(inplace=True)self.convd4_2 = nn.Conv2d(512,512,3,padding=1)self.bnd4_2 = nn.BatchNorm2d(512)self.relud4_2 = nn.ReLU(inplace=True)self.convd4_3 = nn.Conv2d(512,256,3,padding=1)self.bnd4_3 = nn.BatchNorm2d(256)self.relud4_3 = nn.ReLU(inplace=True)#stage 3self.pool3 = nn.MaxPool2d(2,2,ceil_mode=True)self.mconv3 = nn.Conv2d(256 + 128,256,3,padding=1)self.mbn3 = nn.BatchNorm2d(256)self.mrelu3 = nn.ReLU(inplace=True)self.convd3_1 = nn.Conv2d(512,256,3,padding=1)self.bnd3_1 = nn.BatchNorm2d(256)self.relud3_1 = nn.ReLU(inplace=True)self.convd3_2 = nn.Conv2d(256,256,3,padding=1)self.bnd3_2 = nn.BatchNorm2d(256)self.relud3_2 = nn.ReLU(inplace=True)self.convd3_3 = nn.Conv2d(256,128,3,padding=1)self.bnd3_3 = nn.BatchNorm2d(128)self.relud3_3 = nn.ReLU(inplace=True)#stage 2self.mconv2 = nn.Conv2d(128 + 128,128,3,padding=1)self.mbn2 = nn.BatchNorm2d(128)self.mrelu2 = nn.ReLU(inplace=True)self.convd2_1 = nn.Conv2d(256,128,3,padding=1)self.bnd2_1 = nn.BatchNorm2d(128)self.relud2_1 = nn.ReLU(inplace=True)self.convd2_2 = nn.Conv2d(128,128,3,padding=1)self.bnd2_2 = nn.BatchNorm2d(128)self.relud2_2 = nn.ReLU(inplace=True)self.convd2_3 = nn.Conv2d(128,64,3,padding=1)self.bnd2_3 = nn.BatchNorm2d(64)self.relud2_3 = nn.ReLU(inplace=True)#stage 1self.up1 = nn.Upsample(scale_factor=2, mode='bilinear',align_corners=True)self.mconv1 = nn.Conv2d(64 + 128,64,3,padding=1)self.mbn1 = nn.BatchNorm2d(64)self.mrelu1 = nn.ReLU(inplace=True)self.convd1_1 = nn.Conv2d(128,64,3,padding=1)self.bnd1_1 = nn.BatchNorm2d(64)self.relud1_1 = nn.ReLU(inplace=True)self.convd1_2 = nn.Conv2d(64,64,3,padding=1)self.bnd1_2 = nn.BatchNorm2d(64)self.relud1_2 = nn.ReLU(inplace=True)self.convd1_3 = nn.Conv2d(64,64,3,padding=1)self.bnd1_3 = nn.BatchNorm2d(64)self.relud1_3 = nn.ReLU(inplace=True)...def forward(self,x):#-------------deep feature aggregation module--------------##Decoder-B5:融合桥接器输出score_bdg与边缘提取模块输出score_edge##concate and convolutiont = self.pool5(score_edge)t = torch.cat((t,score_bdg),1)t = self.mrelu5(self.mbn5(self.mconv5(t)))##Decoder-B5scored5 = self.relud5_1(self.bnd5_1(self.convd5_1(t)))scored5 = self.relud5_2(self.bnd5_2(self.convd5_2(scored5)))scored5 = self.relud5_3(self.bnd5_3(self.convd5_3(scored5)))##上采样,作为Decoder-B4的输入scored5_up = self.upscore2(scored5)#Decoder-B4:融合F14与边缘提取模块输出score_edget = self.pool4(score_edge)t = torch.cat((t,score1_4),1)t = self.mrelu4(self.mbn4(self.mconv4(t)))##Decoder-B4scored4 = torch.cat((scored5_up,t),1)scored4 = self.relud4_1(self.bnd4_1(self.convd4_1(scored4)))scored4 = self.relud4_2(self.bnd4_2(self.convd4_2(scored4)))scored4 = self.relud4_3(self.bnd4_3(self.convd4_3(scored4)))##上采样,作为Decoder-B3的输入scored4_up = self.upscore2(scored4)#Decoder-B3:融合F13与边缘提取模块输出score_edget = self.pool3(score_edge)t = torch.cat((t,score1_3),1)t = self.mrelu3(self.mbn3(self.mconv3(t)))##Decoder-B3scored3 = torch.cat((scored4_up,t),1)scored3 = self.relud3_1(self.bnd3_1(self.convd3_1(scored3)))scored3 = self.relud3_2(self.bnd3_2(self.convd3_2(scored3)))scored3 = self.relud3_3(self.bnd3_3(self.convd3_3(scored3)))##上采样,作为Decoder-B2的输入scored3_up = self.upscore2(scored3)#Decoder-B2:融合F12与边缘提取模块输出score_edget = torch.cat((score1_2,score_edge),1)t = self.mrelu2(self.mbn2(self.mconv2(t)))##Decoder-B2scored2 = torch.cat((scored3_up,t),1)scored2 = self.relud2_1(self.bnd2_1(self.convd2_1(scored2)))scored2 = self.relud2_2(self.bnd2_2(self.convd2_2(scored2)))scored2 = self.relud2_3(self.bnd2_3(self.convd2_3(scored2)))##上采样,作为Decoder-B1的输入scored2_up = self.upscore2(scored2)#Decoder-B3:融合F11与边缘提取模块输出score_edget = self.up1(score_edge)t = torch.cat((score1_1,t),1)t = self.mrelu1(self.mbn1(self.mconv1(t)))##Decoder-B1scored1 = torch.cat((scored2_up,t),1)scored1 = self.relud1_1(self.bnd1_1(self.convd1_1(scored1)))scored1 = self.relud1_2(self.bnd1_2(self.convd1_2(scored1)))scored1 = self.relud1_3(self.bnd1_3(self.convd1_3(scored1)))...#-------------Label output--------------##深度监督编码器5的输出label4_out = self.label_outconv4(scored5)label4_out = self.upscore16(label4_out)#深度监督编码器4的输出label5_out = self.label_outconv5(scored4)label5_out = self.upscore8(label5_out)#深度监督编码器3的输出label6_out = self.label_outconv6(scored3)label6_out = self.upscore4(label6_out)#深度监督编码器2的输出label7_out = self.label_outconv7(scored2)label7_out = self.upscore2(label7_out)#深度监督编码器1的输出(无需上采样)label8_out = self.label_outconv8(scored1)#-------------Return--------------#return torch.sigmoid(edge_out), torch.sigmoid(label1_out), torch.sigmoid(label2_out), torch.sigmoid(label3_out), torch.sigmoid(labelbdg_out), \torch.sigmoid(label4_out), torch.sigmoid(label5_out), torch.sigmoid(label6_out), torch.sigmoid(label7_out), torch.sigmoid(label8_out)

三、MJRBM

论文：ORSI Salient Object Detection via Multiscale Joint Region and Boundary Model（用于光学遥感影像显著性目标检测的多尺度联合区域和边界模型）
论文链接：ORSI Salient Object Detection via Multiscale Joint Region and Boundary Model
代码链接：Github
博客链接：CSDN

3.1编码器：VGG16双分支编码器

在这里插入图片描述
双分支结构的设计思想参考了级联部分解码器，两个分支的作用如下：

1.第一个分支：通过PPM模块（见论文Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition）提取语义信息，并将其与VGG-16第二个模块的输出结合，以提取边界特征。
2.第二个分支：通过分层注意力模块（HAM，Hierarchical Attention Module）提取多尺度特征，并通过联合区域和边界模块（JRBM，Joint Region and Boundary Model）来联合优化边界和区域特征。

在这里插入图片描述
上图中D即为VGG-16架构。在代码中并未包含模型中 $V GG 3 - 2$ 模块的定义，而 $V GG 4 - 2 、 V GG 5 - 2$ 实则是VGG4-1、VGG5-1$的重复。

#model/vgg.py
class B2_VGG(nn.Module):# VGG16 with two branches# pooling layer at the front of blockdef __init__(self):super(B2_VGG, self).__init__()#VGG1conv1 = nn.Sequential()#conv3- 64conv1.add_module('conv1_1', nn.Conv2d(3, 64, 3, 1, 1))conv1.add_module('relu1_1', nn.ReLU(inplace=True))#conv3- 64conv1.add_module('conv1_2', nn.Conv2d(64, 64, 3, 1, 1))conv1.add_module('relu1_2', nn.ReLU(inplace=True))self.conv1 = conv1#VGG2conv2 = nn.Sequential()#maxpoolconv2.add_module('pool1', nn.AvgPool2d(2, stride=2))#conv3- 128conv2.add_module('conv2_1', nn.Conv2d(64, 128, 3, 1, 1))conv2.add_module('relu2_1', nn.ReLU())#conv3- 128conv2.add_module('conv2_2', nn.Conv2d(128, 128, 3, 1, 1))conv2.add_module('relu2_2', nn.ReLU())self.conv2 = conv2#VGG3_1conv3 = nn.Sequential()#maxpoolconv3.add_module('pool2', nn.AvgPool2d(2, stride=2))#conv3- 256conv3.add_module('conv3_1', nn.Conv2d(128, 256, 3, 1, 1))conv3.add_module('relu3_1', nn.ReLU())#conv3- 256conv3.add_module('conv3_2', nn.Conv2d(256, 256, 3, 1, 1))conv3.add_module('relu3_2', nn.ReLU())#conv3- 256conv3.add_module('conv3_3', nn.Conv2d(256, 256, 3, 1, 1))conv3.add_module('relu3_3', nn.ReLU())self.conv3 = conv3#VGG4_1conv4_1 = nn.Sequential()#maxpoolconv4_1.add_module('pool3_1', nn.AvgPool2d(2, stride=2))#conv3- 512conv4_1.add_module('conv4_1_1', nn.Conv2d(256, 512, 3, 1, 1))conv4_1.add_module('relu4_1_1', nn.ReLU())#conv3- 512conv4_1.add_module('conv4_2_1', nn.Conv2d(512, 512, 3, 1, 1))conv4_1.add_module('relu4_2_1', nn.ReLU())#conv3- 512conv4_1.add_module('conv4_3_1', nn.Conv2d(512, 512, 3, 1, 1))conv4_1.add_module('relu4_3_1', nn.ReLU())self.conv4_1 = conv4_1#VGG5_1conv5_1 = nn.Sequential()#maxpoolconv5_1.add_module('pool4_1', nn.AvgPool2d(2, stride=2))#conv3- 512conv5_1.add_module('conv5_1_1', nn.Conv2d(512, 512, 3, 1, 1))conv5_1.add_module('relu5_1_1', nn.ReLU())#conv3- 512conv5_1.add_module('conv5_2_1', nn.Conv2d(512, 512, 3, 1, 1))conv5_1.add_module('relu5_2_1', nn.ReLU())#conv3- 512conv5_1.add_module('conv5_3_1', nn.Conv2d(512, 512, 3, 1, 1))conv5_1.add_module('relu5_3_1', nn.ReLU())self.conv5_1 = conv5_1#VGG4_2conv4_2 = nn.Sequential()conv4_2.add_module('pool3_2', nn.AvgPool2d(2, stride=2))#conv3- 512conv4_2.add_module('conv4_1_2', nn.Conv2d(256, 512, 3, 1, 1))conv4_2.add_module('relu4_1_2', nn.ReLU())#conv3- 512conv4_2.add_module('conv4_2_2', nn.Conv2d(512, 512, 3, 1, 1))conv4_2.add_module('relu4_2_2', nn.ReLU())#conv3- 512conv4_2.add_module('conv4_3_2', nn.Conv2d(512, 512, 3, 1, 1))conv4_2.add_module('relu4_3_2', nn.ReLU())self.conv4_2 = conv4_2#VGG5_2conv5_2 = nn.Sequential()conv5_2.add_module('pool4_2', nn.AvgPool2d(2, stride=2))#conv3- 512conv5_2.add_module('conv5_1_2', nn.Conv2d(512, 512, 3, 1, 1))conv5_2.add_module('relu5_1_2', nn.ReLU())#conv3- 512conv5_2.add_module('conv5_2_2', nn.Conv2d(512, 512, 3, 1, 1))conv5_2.add_module('relu5_2_2', nn.ReLU())#conv3- 512conv5_2.add_module('conv5_3_2', nn.Conv2d(512, 512, 3, 1, 1))conv5_2.add_module('relu5_3_2', nn.ReLU())self.conv5_2 = conv5_2#加载预训练权重pre_train = torch.load('./model/CPD.pth')self._initialize_weights(pre_train)def forward(self, x):x = self.conv1(x)x = self.conv2(x)x = self.conv3(x)x1 = self.conv4_1(x)x1 = self.conv5_1(x1)x2 = self.conv4_2(x)x2 = self.conv5_2(x2)#返回VGG5_1、VGG5_2的输出return x1, x2#初始化权重def _initialize_weights(self, pre_train):...

在模型代码中：

#model/CPD_models.py
class JRBM(nn.Module):def __init__(self, channel=32):super(JRBM, self).__init__()#加载编码器架构self.vgg = B2_VGG()...def forward(self, x):#VGG1x1 = self.vgg.conv1(x)#VGG2x2 = self.vgg.conv2(x1)#VGG3_1x3 = self.vgg.conv3(x2)x3_1 = x3#VGG4_1x4_1 = self.vgg.conv4_1(x3_1)#VGG5_1x5_1 = self.vgg.conv5_1(x4_1)...#VGG3_2x3_2 = self.HA(attention.sigmoid(), x3)#VGG4_2x4_2 = self.vgg.conv4_2(x3_2)#VGG5_2x5_2 = self.vgg.conv5_2(x4_2)...

3.2SBFEM：显著边界特征提取模块

在这里插入图片描述
显著边界特征提取模块（Salient Boundary Feature Extraction Module）由PPM模块（用于获取全局信息的金字塔池模块，见论文Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition，在代码中定义为GGM模块）和两个卷积模块组成。用于从高级特征图中捕获全局信息来定位显著对象、从低级特征图中获取显著边界信息，来指导显著目标边界特征的生成。PPM模块代码实现：

#model/BGM_PPM.py
class GGM(nn.Module):  # get global featuredef __init__(self, in_channels):super(GGM, self).__init__()self.branch0 = nn.Sequential(nn.Conv2d(in_channels, in_channels, 3, 1, 1),nn.ReLU())self.branch1 = nn.Sequential(nn.AdaptiveAvgPool2d((3, 3)),nn.Conv2d(in_channels, in_channels, 1, 1),nn.ReLU())self.branch2 = nn.Sequential(nn.AdaptiveAvgPool2d((5, 5)),nn.Conv2d(in_channels, in_channels, 1, 1),nn.ReLU())self.branch3 = nn.Sequential(nn.AdaptiveAvgPool2d((1, 1)),nn.Conv2d(in_channels, in_channels, 1, 1),nn.ReLU())self.con = nn.Conv2d(in_channels * 4, in_channels, 3, 1, 1)for m in self.modules():if isinstance(m, nn.Conv2d):m.weight.data.normal_(std=0.01)m.bias.data.fill_(0)def forward(self, x):x0 = self.branch0(x)x1 = self.branch1(x)x1 = F.upsample(x1, x0.size()[2:], mode='bilinear', align_corners=True)x2 = self.branch2(x)x2 = F.upsample(x2, x0.size()[2:], mode='bilinear', align_corners=True)x3 = self.branch3(x)x3 = F.upsample(x3, x0.size()[2:], mode='bilinear', align_corners=True)x = self.con(torch.cat((x0, x1, x2, x3), 1))return x

模型代码：

class JRBM(nn.Module):def __init__(self, channel=32):super(JRBM, self).__init__()...self.HA = HA()self.glob = GGM(512)self.upsample2 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=False)self.upsample4 = nn.Upsample(scale_factor=4, mode='bilinear', align_corners=False)self.upsample8 = nn.Upsample(scale_factor=8, mode='bilinear', align_corners=False)self.glob_vgg2 = nn.Sequential(nn.Conv2d(512 + 128, 128, 3, 1, 1),nn.ReLU(),nn.Conv2d(128, 128, 3, 1, 1),nn.ReLU(),nn.Conv2d(128, channel, 3, 1, 1))self.conv3 = nn.Conv2d(channel, 1, 1)def forward(self, x):...#输入PPM模块提取全局信息x_glob = self.glob(x5_1)        #out_channel=512#融合语义信息、细节信息,通过glob_vgg2模块改变通道数(以作为两个解码器JRBM的输入)x_edge = self.glob_vgg2(torch.cat((self.upsample8(x_glob),x2),1))   #out_channel=channel...#将通道数降为1,生成显著边缘图x_edge_pre = self.conv3(x_edge)...#返回显著边缘图作损失函数return self.upsample4(attention),self.upsample2(x_edge_pre), self.upsample4(detection)

3.4分层注意力模块：HAM

在这里插入图片描述
分层注意力模块（HAM，Hierarchical Attention Module）使用不同膨胀率的卷积核获取多尺度信息，并将结果直接串联。由于不同的感受野获得的特征无法区分细微的特征，因此突出的物体具有空间不一致。故将特征图输入到通道注意力模块（见Dual Attention Network for Scene Segmentation）中以解决此问题。

#model/CPD_models.py
class HAM(nn.Module):def __init__(self, in_channel, out_channel):super(HAM, self).__init__()self.relu = nn.ReLU(True)#普通卷积self.branch0 = nn.Sequential(nn.Conv2d(in_channel, out_channel, 1),)#膨胀卷积(r=3)self.branch1 = nn.Sequential(nn.Conv2d(in_channel, out_channel, 1),nn.Conv2d(out_channel, out_channel, kernel_size=(1, 3), padding=(0, 1)),nn.Conv2d(out_channel, out_channel, kernel_size=(3, 1), padding=(1, 0)),nn.Conv2d(out_channel, out_channel, 3, padding=3, dilation=3))#膨胀卷积(r=5)self.branch2 = nn.Sequential(nn.Conv2d(in_channel, out_channel, 1),nn.Conv2d(out_channel, out_channel, kernel_size=(1, 5), padding=(0, 2)),nn.Conv2d(out_channel, out_channel, kernel_size=(5, 1), padding=(2, 0)),nn.Conv2d(out_channel, out_channel, 3, padding=5, dilation=5))#膨胀卷积(r=7)self.branch3 = nn.Sequential(nn.Conv2d(in_channel, out_channel, 1),nn.Conv2d(out_channel, out_channel, kernel_size=(1, 7), padding=(0, 3)),nn.Conv2d(out_channel, out_channel, kernel_size=(7, 1), padding=(3, 0)),nn.Conv2d(out_channel, out_channel, 3, padding=7, dilation=7))#膨胀卷积(r=9)self.branch4 = nn.Sequential(nn.Conv2d(in_channel, out_channel, 1),nn.Conv2d(out_channel, out_channel, kernel_size=(1, 9), padding=(0, 4)),nn.Conv2d(out_channel, out_channel, kernel_size=(9, 1), padding=(4, 0)),nn.Conv2d(out_channel, out_channel, 3, padding=9, dilation=9))self.conv_cat = nn.Conv2d(5*out_channel, out_channel, 3, padding=1)self.conv_res = nn.Conv2d(in_channel, out_channel, 1)#通道注意力模块self.channel_attention = CAM_Module(out_channel)self.conv1 = nn.Conv2d(out_channel,out_channel,3,1,1)for m in self.modules():if isinstance(m, nn.Conv2d):m.weight.data.normal_(std=0.01)m.bias.data.fill_(0)def forward(self, x):#依次进行膨胀卷积x0 = self.branch0(x)x1 = self.branch1(x)x2 = self.branch2(x)x3 = self.branch3(x)x4 = self.branch4(x)#融合膨胀结果x_cat = torch.cat((x0, x1, x2, x3, x4), 1)#卷积后得到特征图Ax_cat = self.conv_cat(x_cat)#通道注意力增强x_cat = self.channel_attention(x_cat)x_cat = self.conv1(x_cat)#残差连接x = self.relu(x_cat + self.conv_res(x))return x

3.4Aggregation：聚合模块

在这里插入图片描述

MJRBM模型中的Aggregation模块用于聚合三个JRBM模块的输出，其先使用高层特征来优化低层特征，然后将多层特征串联起来做出最终预测。

#model/CPD_models.py
class BasicConv2d(nn.Module):def __init__(self, in_planes, out_planes, kernel_size,stride=1, padding=0, dilation=1, groups=1, bias=False):super(BasicConv2d, self).__init__()self.basicconv = nn.Sequential(nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride,padding=padding, dilation=dilation, groups=groups, bias=bias),nn.BatchNorm2d(out_planes),nn.ReLU(inplace=True))def forward(self, x):return self.basicconv(x)class aggregation_add(nn.Module):def __init__(self, channel):super(aggregation_add, self).__init__()self.relu = nn.ReLU(True)self.upsample = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)self.conv_upsample1 = BasicConv2d(channel, channel, 3, padding=1)self.conv_upsample2 = BasicConv2d(channel, channel, 3, padding=1)self.conv_upsample3 = BasicConv2d(channel, channel, 3, padding=1)self.conv_upsample4 = BasicConv2d(channel, channel, 3, padding=1)self.conv_upsample5 = BasicConv2d(channel, channel, 3, padding=1)self.conv_concat2 = BasicConv2d(2*channel, 2*channel, 3, padding=1)self.conv_concat3 = BasicConv2d(3*channel, 3*channel, 3, padding=1)self.conv4 = BasicConv2d(3*channel, 3*channel, 3, padding=1)self.conv5 = nn.Conv2d(3*channel, 1, 1)def forward(self, x1, x2, x3):#深层特征x1_1 = x1#使用x1_1优化x2_1x2_1 = self.conv_upsample1(self.upsample(x1)) * x2#使用x1_1、x2_1优化x3_1x3_1 = self.conv_upsample2(self.upsample(self.upsample(x1))) \+ self.conv_upsample3(self.upsample(x2)) * x3#特征融合x3_2 = torch.cat((x3_1, self.conv_upsample4(self.upsample(self.upsample(x1_1))), self.conv_upsample5(self.upsample(x2_1))), 1)x = self.conv4(x3_2)x = self.conv5(x)return x

3.5JRBM：联合区域和边界模块

在这里插入图片描述
联合区域和边界模块（JRBM，Joint Region and Boundary Module），在获得多尺度显著区域特征和边界特征后，共同优化边界特征和显著区域特征。最后将二者融合，输入到融合模块Aggregation中。在上图中， $β$ 为输入的边界特征， $γ$ 为输入的区域特征，由于二者当中均可能存在噪声，因此采用门机制实现自适应选择：
在这里插入图片描述
由于区域特征与对象边界特征之间存在如下逻辑关系：

因此在实际实现中使用乘法操作代表 $\land$ 、Concat操作代表 $\lor$ 来获取增强后的显著区域特征与边界特征，并将二者融合得到增强后的显著区域特征：
在这里插入图片描述

#JRBM
class BGModel(nn.Module):def __init__(self, channel,e1_stride, e2_stride):super(BGModel, self).__init__()self.relu = nn.ReLU()self.conv0 = nn.Conv2d(channel, channel, 5, e1_stride, 2)self.gamma = nn.Conv2d(channel, channel, 5, e2_stride, 2)self.conv1 = nn.Conv2d(channel, channel, 5, 1, 2)self.conv2 = nn.Conv2d(channel, channel, 3, 1, 1)self.conv3 = nn.Conv2d(channel*2, channel, 3, 1, 1)self.conv4 = nn.Conv2d(2*channel, channel, 3, 1, 1)self.edge_pre = nn.Conv2d(channel, 1, 1)self.fea_pre = nn.Conv2d(channel, 1, 1)#权重初始化for m in self.modules():...def forward(self, y, x):  # x= feature, y=edge#门函数,过滤显著区域噪声x = x * F.sigmoid(x)x = self.conv1(x)#门函数,过滤显著边界噪声y = y * F.sigmoid(y)y = self.relu(self.conv0(y))y = self.relu(self.gamma(y))#增强显著边界edge = self.relu(self.conv2( x * y))e_pre = self.edge_pre(edge)#增强显著区域fea = self.relu(self.conv3(torch.cat((x,y),1)))f_pre = self.fea_pre(fea)#显著边界与显著区域融合x = self.conv4(torch.cat((edge, fea),1))return x