1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

%matplotlib inline
import numpy as np
import torch
from torch import nn
from d2l import torch as d2l

#@save
d2l.DATA_HUB['airfoil'] = (d2l.DATA_URL + 'airfoil_self_noise.dat',
                           '76e5be1548fd8222e5074cf0faae75edff8cf93f')

#@save
def get_data_ch11(batch_size=10, n=1500):
    data = np.genfromtxt(d2l.download('airfoil'),
                         dtype=np.float32, delimiter='\t')
    data = torch.from_numpy((data - data.mean(axis=0)) / data.std(axis=0))
    data_iter = d2l.load_array((data[:n, :-1], data[:n, -1]), #最后一列作为标签
                               batch_size, is_train=True)
    return data_iter, data.shape[1]-1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

#@save
def train_concise_ch11(trainer_fn, hyperparams, data_iter, num_epochs=4):
    # 初始化模型：只有一个隐藏层的MLP
    net = nn.Sequential(nn.Linear(5, 1))
    def init_weights(m):
        if type(m) == nn.Linear:
            torch.nn.init.normal_(m.weight, std=0.01)
    net.apply(init_weights)

    #优化器
    optimizer = trainer_fn(net.parameters(), **hyperparams)
    #损失函数
    loss = nn.MSELoss(reduction='none')
    animator = d2l.Animator(xlabel='epoch', ylabel='loss',
                            xlim=[0, num_epochs], ylim=[0.22, 0.35])
    n, timer = 0, d2l.Timer()
    for _ in range(num_epochs): #每个epoch
        for X, y in data_iter:  #每个batch
            optimizer.zero_grad()
            out = net(X)
            y = y.reshape(out.shape)
            l = loss(out, y)
            l.mean().backward()
            optimizer.step()
            n += X.shape[0]
            if n % 200 == 0:
                timer.stop()
                # MSELoss计算平方误差时不带系数1/2
                animator.add(n/X.shape[0]/len(data_iter),
                             (d2l.evaluate_loss(net, data_iter, loss) / 2,))
                timer.start()
    print(f'loss: {animator.Y[0][-1]:.3f}, {timer.avg():.3f} sec/epoch')

1
2
3
4
5

data_iter, _ = get_data_ch11(10)
# 定义优化器
trainer = torch.optim.SGD
# 训练
train_concise_ch11(trainer, {'lr': 0.01}, data_iter)

1
2
3
4
5

# 定义优化器
trainer = torch.optim.SGD

# 训练
d2l.train_concise_ch11(trainer, {'lr': 0.005, 'momentum': 0.9}, data_iter)

1
2
3
4
5

# 定义优化器
trainer = torch.optim.Adagrad

# 训练
d2l.train_concise_ch11(trainer, {'lr': 0.1}, data_iter) #初始学习率

1
2
3
4
5

# 定义优化器
trainer = torch.optim.RMSprop

# 训练
d2l.train_concise_ch11(trainer, {'lr': 0.01, 'alpha': 0.9}, data_iter)

1
2
3
4
5

# 定义优化器
trainer = torch.optim.Adam

# 训练
d2l.train_concise_ch11(trainer, {'lr': 0.01}, data_iter)

1
2
3
4
5

%matplotlib inline
import torch
from torch import nn
from torch.nn import functional as F
from d2l import torch as d2l

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

# 初始化模型参数
scale = 0.01
W1 = torch.randn(size=(20, 1, 3, 3)) * scale
b1 = torch.zeros(20)
W2 = torch.randn(size=(50, 20, 5, 5)) * scale
b2 = torch.zeros(50)
W3 = torch.randn(size=(800, 128)) * scale
b3 = torch.zeros(128)
W4 = torch.randn(size=(128, 10)) * scale
b4 = torch.zeros(10)
params = [W1, b1, W2, b2, W3, b3, W4, b4]

# 定义模型
def lenet(X, params):
    # 1个输入通道，20个输出通道
    h1_conv = F.conv2d(input=X, weight=params[0], bias=params[1])
    h1_activation = F.relu(h1_conv) # 激活层
    h1 = F.avg_pool2d(input=h1_activation, kernel_size=(2, 2), stride=(2, 2)) # 平均汇聚层
    # 20个输入通道，50个输出通道
    h2_conv = F.conv2d(input=h1, weight=params[2], bias=params[3])
    h2_activation = F.relu(h2_conv) # 激活层
    h2 = F.avg_pool2d(input=h2_activation, kernel_size=(2, 2), stride=(2, 2)) # 平均汇聚层
    h2 = h2.reshape(h2.shape[0], -1)
    h3_linear = torch.mm(h2, params[4]) + params[5] #全连接层
    h3 = F.relu(h3_linear)
    y_hat = torch.mm(h3, params[6]) + params[7]
    return y_hat

# 交叉熵损失函数
loss = nn.CrossEntropyLoss(reduction='none')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

def get_params(params, device):
    new_params = [p.to(device) for p in params]
    for p in new_params:
        p.requires_grad_()
    return new_params

# 示例
new_params = get_params(params, d2l.try_gpu(0))
print('b1 权重:', new_params[1])
# b1 权重: tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
#        		  device='cuda:0', requires_grad=True)
print('b1 梯度:', new_params[1].grad) #初始梯度为None
# b1 梯度: None

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

def allreduce(data):
    for i in range(1, len(data)):
        data[0][:] += data[i].to(data[0].device)
    for i in range(1, len(data)):
        data[i][:] = data[0].to(data[i].device)
        
# 示例
data = [torch.ones((1, 2), device=d2l.try_gpu(i)) * (i + 1) for i in range(2)]
print('allreduce之前：\n', data[0], '\n', data[1])
# allreduce之前：
#  tensor([[1., 1.]], device='cuda:0') 
#  tensor([[2., 2.]], device='cuda:1')

allreduce(data)
print('allreduce之后：\n', data[0], '\n', data[1])
# allreduce之后：
#  tensor([[3., 3.]], device='cuda:0') 
#  tensor([[3., 3.]], device='cuda:1')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

data = torch.arange(12).reshape(4, 3)
devices = [torch.device('cuda:0'), torch.device('cuda:1')]
print('input :', data)
# input : tensor([[ 0,  1,  2],
#                 [ 3,  4,  5],
#                 [ 6,  7,  8],
#                 [ 9, 10, 11]])
print('load into', devices)
# load into [device(type='cuda', index=0), device(type='cuda', index=1)]

split = nn.parallel.scatter(data, devices) #按行平均分给多个GPU
print('output:', split)
# output: (tensor([[0, 1, 2],
#                  [3, 4, 5]], device='cuda:0'), 
#         tensor([[ 6,  7,  8],
#                 [ 9, 10, 11]], device='cuda:1'))

#@save
def split_batch(X, y, devices):
    """将X和y拆分到多个设备上"""
    assert X.shape[0] == y.shape[0]
    return (nn.parallel.scatter(X, devices),
            nn.parallel.scatter(y, devices))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

def train_batch(X, y, device_params, devices, lr):
    X_shards, y_shards = split_batch(X, y, devices)
    # 在每个GPU上分别计算损失
    ls = [loss(lenet(X_shard, device_W), y_shard).sum()
          for X_shard, y_shard, device_W in zip(
              X_shards, y_shards, device_params)]
    for l in ls:  # 反向传播，计算每个GPU的当前梯度
        l.backward()
    # 将每个GPU的所有梯度相加，并将其广播到所有GPU
    with torch.no_grad():
        for i in range(len(device_params[0])): # device_params为list，为每个GPU的模型参数
            allreduce(
                [device_params[c][i].grad for c in range(len(devices))])
    # 在每个GPU上分别更新模型参数
    for param in device_params:
        d2l.sgd(param, lr, X.shape[0]) # 在这里，我们使用全尺寸的小批量

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

def train(num_gpus, batch_size, lr):
    train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)
    devices = [d2l.try_gpu(i) for i in range(num_gpus)]
    # 将模型参数复制到num_gpus个GPU
    device_params = [get_params(params, d) for d in devices]
    num_epochs = 10
    animator = d2l.Animator('epoch', 'test acc', xlim=[1, num_epochs])
    timer = d2l.Timer()
    for epoch in range(num_epochs):
        timer.start()
        for X, y in train_iter:
            # 为单个小批量执行多GPU训练
            train_batch(X, y, device_params, devices, lr)
            torch.cuda.synchronize() # 确保当前指定GPU已经训练完成
        timer.stop()
        # 在GPU0上评估模型
        animator.add(epoch + 1, (d2l.evaluate_accuracy_gpu(
            lambda x: lenet(x, device_params[0]), test_iter, devices[0]),))
    print(f'测试精度：{animator.Y[0][-1]:.2f}，{timer.avg():.1f}秒/轮，'
          f'在{str(devices)}')

1
2
3
4
5

train(num_gpus=1, batch_size=256, lr=0.2)
# 测试精度：0.76，4.9秒/轮，在[device(type='cuda', index=0)]

train(num_gpus=2, batch_size=256, lr=0.2)
# 测试精度：0.83，5.0秒/轮，在[device(type='cuda', index=0), device(type='cuda', index=1)]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

#@save
def resnet18(num_classes, in_channels=1):
    """稍加修改的ResNet-18模型"""
    def resnet_block(in_channels, out_channels, num_residuals,
                     first_block=False):
        blk = []
        for i in range(num_residuals):
            if i == 0 and not first_block:
                blk.append(d2l.Residual(in_channels, out_channels,
                                        use_1x1conv=True, strides=2))
            else:
                blk.append(d2l.Residual(out_channels, out_channels))
        return nn.Sequential(*blk)

    # 该模型使用了更小的卷积核、步长和填充，而且删除了最大汇聚层
    net = nn.Sequential(
        nn.Conv2d(in_channels, 64, kernel_size=3, stride=1, padding=1),
        nn.BatchNorm2d(64),
        nn.ReLU())
    net.add_module("resnet_block1", resnet_block(
        64, 64, 2, first_block=True))
    net.add_module("resnet_block2", resnet_block(64, 128, 2))
    net.add_module("resnet_block3", resnet_block(128, 256, 2))
    net.add_module("resnet_block4", resnet_block(256, 512, 2))
    net.add_module("resnet_block5", resnet_block(512, 1024, 2))
    net.add_module("resnet_block6", resnet_block(1024, 1024, 2))
    # net.add_module("resnet_block7", resnet_block(1024, 1024, 2))
    # net.add_module("resnet_block8", resnet_block(1024, 1024, 2))
    net.add_module("global_avg_pool", nn.AdaptiveAvgPool2d((1,1)))
    net.add_module("fc", nn.Sequential(nn.Flatten(),
                                       nn.Linear(1024, num_classes)))
    return net


# 实例化模型
net = resnet18(10)

# 获取所有的GPU列表
devices = d2l.try_all_gpus()
devices
# [device(type='cuda', index=0),
#  device(type='cuda', index=1),
#  device(type='cuda', index=2)]

# 我们将在训练代码实现中初始化网络

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

def train(net, num_gpus, batch_size, lr):
    train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)
    # 定义所使用的多个GPU
    devices = [d2l.try_gpu(i) for i in range(num_gpus)]
    def init_weights(m):
        if type(m) in [nn.Linear, nn.Conv2d]:
            nn.init.normal_(m.weight, std=0.01)
    net.apply(init_weights)
    # 在多个GPU上设置模型
    net = nn.DataParallel(net, device_ids=devices)
    trainer = torch.optim.SGD(net.parameters(), lr) #优化器
    loss = nn.CrossEntropyLoss() #损失函数
    timer, num_epochs = d2l.Timer(), 10
    animator = d2l.Animator('epoch', 'test acc', xlim=[1, num_epochs])
    for epoch in range(num_epochs):
        net.train()
        timer.start()
        for X, y in train_iter:
            trainer.zero_grad()
            X, y = X.to(devices[0]), y.to(devices[0]) # 先把批量样本数据都暂时放到cuda:0中
            l = loss(net(X), y) #模型会自动将样本分发给多个GPU
            l.backward()
            trainer.step()
        timer.stop()
        animator.add(epoch + 1, (d2l.evaluate_accuracy_gpu(net, test_iter),))
    print(f'测试精度：{animator.Y[0][-1]:.2f}，{timer.avg():.1f}秒/轮，'
          f'在{str(devices)}')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

## Batch size = 256
train(net, num_gpus=1,batch_size=256,lr=0.1)
# 测试精度：0.90，7.9秒/轮，在[device(type='cuda', index=0)]

train(net, num_gpus=2,batch_size=512,lr=0.2)
# 测试精度：0.75，13.8秒/轮，在[device(type='cuda', index=0), device(type='cuda', index=1)]


## Batch size = 512
train(net, num_gpus=1,batch_size=512,lr=0.1)
# 测试精度：0.87，7.6秒/轮，在[device(type='cuda', index=0)]

train(net, num_gpus=2,batch_size=1024,lr=0.2)
# 测试精度：0.90，8.6秒/轮，在[device(type='cuda', index=0), device(type='cuda', index=1)]


## Batch size = 1024
train(net, num_gpus=1,batch_size=1024,lr=0.1)
# 测试精度：0.69，7.7秒/轮，在[device(type='cuda', index=0)]

train(net, num_gpus=2,batch_size=1024*2,lr=0.1*2)
# 测试精度：0.79，6.4秒/轮，在[device(type='cuda', index=0), device(type='cuda', index=1)]


train(net, num_gpus=3,batch_size=1024*3,lr=0.1*3)
# 测试精度：0.79，5.4秒/轮，在[device(type='cuda', index=0), device(type='cuda', index=1), device(type='cuda', index=2)]