1.单机多卡版本：代码中的DistributedDataParallel (DDP) 部分对应单机多卡的分布式训练方式
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import RandomHorizontalFlip, RandomVerticalFlip, RandomRotation, RandomResizedCrop, ToTensor
from torch.nn.parallel import DistributedDataParallel as DDP# 定义ResNet块
class ResNetBlock(nn.Module):def __init__(self, in_channels, out_channels):super(ResNetBlock, self).__init__()self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1)self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1)self.relu = nn.ReLU(inplace=True)def forward(self, x):residual = xout = self.conv1(x)out = self.relu(out)out = self.conv2(out)out += residualout = self.relu(out)return out# 定义UNet模型
class UNet(nn.Module):def __init__(self, in_channels, out_channels):super(UNet, self).__init__()self.conv1 = nn.Conv2d(in_channels, 64, kernel_size=3, padding=1)self.block1 = ResNetBlock(64, 64)self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)self.conv2 = nn.Conv2d(64, 128, kernel_size=3, padding=1)self.block2 = ResNetBlock(128, 128)self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)self.conv3 = nn.Conv2d(128, 256, kernel_size=3, padding=1)self.block3 = ResNetBlock(256, 256)self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2)self.conv4 = nn.Conv2d(256, 512, kernel_size=3, padding=1self.block4 = ResNetBlock(512, 512)self.upconv3 = nn.ConvTranspose2d(512, 256, kernel_size=2, stride=2)self.upconv2 = nn.ConvTranspose2d(256, 128, kernel_size=2, stride=2)self.upconv1 = nn.ConvTranspose2d(128, 64, kernel_size=2, stride=2)self.conv5 = nn.Conv2d(128, out_channels, kernel_size=1)def forward(self, x):x1 = self.conv1(x)x1 = self.block1(x1)x2 = self.pool1(x1)x2 = self.conv2(x2)x2 = self.block2(x2)x3 = self.pool2(x2)x3 = self.conv3(x3)x3 = self.block3(x3)x4 = self.pool3(x3)x4 = self.conv4(x4)x4 = self.block4(x4)x = self.upconv3(x4)x = torch.cat((x, x3), dim=1)x = self.conv5(x)x = self.upconv2(x)x = torch.cat((x, x2), dim=1)x = self.upconv1(x)x = torch.cat((x, x1), dim=1)x = self.conv5(x)return x# 定义数据集类
class CustomDataset(Dataset):def __init__(self, data_dir, transform=None):self.data = # Load data from data_dirself.transform = transformdef __len__(self):return len(self.data)def __getitem__(self, index):image, mask = self.data[index]if self.transform:image = self.transform(image)mask = self.transform(mask)return image, mask# 设置训练参数
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
num_epochs = 10
batch_size = 4# 创建UNet模型和优化器
model = UNet(in_channels=3, num_classes=2).to(device)
model = DDP(model)optimizer = optim.Adam(model.parameters(), lr=0.001)# 定义数据增强方法
transform = transforms.Compose([RandomHorizontalFlip(),RandomVerticalFlip(),RandomRotation(15),RandomResizedCrop(256, scale=(0.8, 1.0)),ToTensor(),
])# 加载数据集并进行数据增强
dataset = CustomDataset(data_dir="path_to_dataset", transform=transform)
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=4)# 训练循环
for epoch in range(num_epochs):model.train()total_loss = 0.0for images, masks in dataloader:images = images.to(device)masks = masks.to(device)optimizer.zero_grad()outputs = model(images)loss = nn.CrossEntropyLoss()(outputs, masks)loss.backward()optimizer.step()total_loss += loss.item()print(f"Epoch {epoch+1}/{num_epochs}, Loss: {total_loss/len(dataloader)}")
2.多机多卡版本：使用torch.utils.data.distributed.DistributedSampler和torch.distributed.init_process_group来实现多机多卡的分布式训练，确保在每个进程中都有不同的数据划分和完整的通信。
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.utils.data import DataLoader
from torch.nn.parallel import DistributedDataParallel
from torchvision.transforms import transforms
from torchvision.datasets import YourDataset
from torch.utils.data.distributed import DistributedSampler
import torch.distributed as dist# 定义ResNet块
class ResNetBlock(nn.Module):def __init__(self, in_channels, out_channels):super(ResNetBlock, self).__init__()self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1)self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1)self.relu = nn.ReLU(inplace=True)def forward(self, x):residual = xout = self.conv1(x)out = self.relu(out)out = self.conv2(out)out += residualout = self.relu(out)return out# 定义UNet模型
class UNet(nn.Module):def __init__(self, in_channels, out_channels):super(UNet, self).__init__()self.conv1 = nn.Conv2d(in_channels, 64, kernel_size=3, padding=1)self.block1 = ResNetBlock(64, 64)self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)self.conv2 = nn.Conv2d(64, 128, kernel_size=3, padding=1)self.block2 = ResNetBlock(128, 128)self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)self.conv3 = nn.Conv2d(128, 256, kernel_size=3, padding=1)self.block3 = ResNetBlock(256, 256)self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2)self.conv4 = nn.Conv2d(256, 512, kernel_size=3, padding=1self.block4 = ResNetBlock(512, 512)self.upconv3 = nn.ConvTranspose2d(512, 256, kernel_size=2, stride=2)self.upconv2 = nn.ConvTranspose2d(256, 128, kernel_size=2, stride=2)self.upconv1 = nn.ConvTranspose2d(128, 64, kernel_size=2, stride=2)self.conv5 = nn.Conv2d(128, out_channels, kernel_size=1)def forward(self, x):x1 = self.conv1(x)x1 = self.block1(x1)x2 = self.pool1(x1)x2 = self.conv2(x2)x2 = self.block2(x2)x3 = self.pool2(x2)x3 = self.conv3(x3)x3 = self.block3(x3)x4 = self.pool3(x3)x4 = self.conv4(x4)x4 = self.block4(x4)x = self.upconv3(x4)x = torch.cat((x, x3), dim=1)x = self.conv5(x)x = self.upconv2(x)x = torch.cat((x, x2), dim=1)x = self.upconv1(x)x = torch.cat((x, x1), dim=1)x = self.conv5(x)return x# 定义数据集类
class CustomDataset(Dataset):def __init__(self, data_dir, transform=None):self.data = # Load data from data_dirself.transform = transformdef __len__(self):return len(self.data)def __getitem__(self, index):image, mask = self.data[index]if self.transform:image = self.transform(image)mask = self.transform(mask)return image, maskdef main(rank, world_size):# 设置分布式训练参数torch.cuda.set_device(rank)torch.distributed.init_process_group(backend='nccl', init_method='env://', world_size=world_size, rank=rank)# 设置训练参数num_epochs = 10batch_size_per_gpu = 4# 创建UNet模型和优化器in_channels = 3model = UNet(in_channels=3, num_classes=2).cuda(rank)model = DistributedDataParallel(model, device_ids=[rank])optimizer = optim.Adam(model.parameters(), lr=0.001)# 数据增强方法transform = transforms.Compose([transforms.RandomHorizontalFlip(),transforms.RandomVerticalFlip(),transforms.RandomRotation(30),transforms.RandomResizedCrop(256, scale=(0.8, 1.2)),transforms.ToTensor()])# 加载训练集和验证集train_dataset = CustomDataset(transform=transform)train_sampler = DistributedSampler(train_dataset)train_loader = DataLoader(train_dataset, batch_size=batch_size_per_gpu, sampler=train_sampler)# 训练循环for epoch in range(num_epochs):model.train()total_loss = 0.0for images, masks in train_loader:images = images.cuda(rank)masks = masks.cuda(rank)# 执行前向传播和反向传播optimizer.zero_grad()outputs = model(images)loss = F.binary_cross_entropy_with_logits(outputs, masks)loss.backward()optimizer.step()total_loss += loss.item()if world_size > 1:torch.distributed.all_reduce(total_loss)total_loss /= len(train_sampler)print(f"Epoch {epoch + 1}/{num_epochs}, Loss: {total_loss:.4f}")def main_multi_gpu():world_size = torch.cuda.device_count()if world_size > 1:torch.multiprocessing.spawn(main, args=(world_size,), nprocs=world_size, join=True)else:main(0, 1)if __name__ == '__main__':main_multi_gpu()
