1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

class MLP(nn.Module):
    # 用模型参数声明层。这里，我们声明两个全连接的层
    def __init__(self):
        # 调用MLP的父类Module的构造函数来执行必要的初始化。
        super().__init__()
        self.hidden = nn.Linear(20, 256)  # 隐藏层
        self.out = nn.Linear(256, 10)  # 输出层

    # 定义模型的前向传播，即如何根据输入X返回所需的模型输出
    def forward(self, X):
        # 注意，这里我们使用ReLU的函数版本，其在nn.functional模块中定义。
        return self.out(F.relu(self.hidden(X)))

1
2

net = MLP()
net(X)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

class MySequential(nn.Module):
    def __init__(self, *args):
        super().__init__()
        # 按顺序插入args中的每一个类
        for idx, module in enumerate(args):
            self._modules[str(idx)] = module

    def forward(self, X):
        # 按添加顺序遍历
        for block in self._modules.values():
            X = block(X)
        return X

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

class FixedHiddenMLP(nn.Module):
    def __init__(self):
        super().__init__()
        self.rand_weight = torch.rand((20, 20), requires_grad=False)
        self.linear = nn.Linear(20, 20)

    def forward(self, X):
        X = self.linear(X)
        X = F.relu(torch.mm(X, self.rand_weight) + 1)
        X = self.linear(X)
        while X.abs().sum() > 1:
            X /= 2
        return X.sum()

1
2
3
4
5
6
7

import torch
from torch import nn

# 定义一个单隐藏层的多层感知机
net = nn.Sequential(nn.Linear(4, 8), nn.ReLU(), nn.Linear(8, 1))
X = torch.rand(size=(2, 4))
net(X)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# net[2]可以获取第2层
print(net[2].state_dict())
# OrderedDict([('weight', tensor([[-0.0427, -0.2939, -0.1894,  0.0220, -0.1709, -0.1522, -0.0334, -0.2263]])), 
# ('bias', tensor([0.0887]))])

# 访问目标参数
print(type(net[2].bias))
print(net[2].bias)
print(net[2].bias.data)

# <class 'torch.nn.parameter.Parameter'> # Parameter定义可优化的参数
# Parameter containing:
# tensor([0.0887], requires_grad=True)
# tensor([0.0887])

# 访问梯度
net[2].weight.grad == None # 尚未调用反向传播，因此梯度为None
# True

# 访问所有参数
print(*[(name, param.shape) for name, param in net[0].named_parameters()])
print(*[(name, param.shape) for name, param in net.named_parameters()])
# ('weight', torch.Size([8, 4])) ('bias', torch.Size([8]))
# ('0.weight', torch.Size([8, 4])) ('0.bias', torch.Size([8])) ('2.weight', torch.Size([1, 8])) ('2.bias', torch.Size([1]))

# 通过name进行访问
net.state_dict()['2.bias'].data
# tensor([0.0887])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

def block1():
    return nn.Sequential(nn.Linear(4, 8), nn.ReLU(),
                         nn.Linear(8, 4), nn.ReLU())

def block2():
    net = nn.Sequential()
    for i in range(4):
        # 在这里嵌套
        net.add_module(f'block {i}', block1())
    return net

rgnet = nn.Sequential(block2(), nn.Linear(4, 1))
rgnet(X)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

print(rgnet)
# Sequential(
#   (0): Sequential(
#     (block 0): Sequential(
#       (0): Linear(in_features=4, out_features=8, bias=True)
#       (1): ReLU()
#       (2): Linear(in_features=8, out_features=4, bias=True)
#       (3): ReLU()
#     )
#     (block 1): Sequential(
#       (0): Linear(in_features=4, out_features=8, bias=True)
#       (1): ReLU()
#       (2): Linear(in_features=8, out_features=4, bias=True)
#       (3): ReLU()
#     )
#     (block 2): Sequential(
#       (0): Linear(in_features=4, out_features=8, bias=True)
#       (1): ReLU()
#       (2): Linear(in_features=8, out_features=4, bias=True)
#       (3): ReLU()
#     )
#     (block 3): Sequential(
#       (0): Linear(in_features=4, out_features=8, bias=True)
#       (1): ReLU()
#       (2): Linear(in_features=8, out_features=4, bias=True)
#       (3): ReLU()
#     )
#   )
#   (1): Linear(in_features=4, out_features=1, bias=True)
# )

# 访问第一个主要的块中、第二个子块的第一层的偏置项
rgnet[0][1][0].bias.data
# tensor([ 0.1999, -0.4073, -0.1200, -0.2033, -0.1573,  0.3546, -0.2141, -0.2483])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# 初始为正态分布
def init_normal(m):
    if type(m) == nn.Linear:
        nn.init.normal_(m.weight, mean=0, std=0.01) # _结尾的函数代表替换函数
        nn.init.zeros_(m.bias)
net.apply(init_normal) # .apply对net中的所有module进行操作
net[0].weight.data[0], net[0].bias.data[0]

# 初始为常数（不建议）
def init_constant(m):
    if type(m) == nn.Linear:
        nn.init.constant_(m.weight, 1)
        nn.init.zeros_(m.bias)
net.apply(init_constant)
net[0].weight.data[0], net[0].bias.data[0]

# 对不同块应用不同初始化方法
def init_xavier(m):
    if type(m) == nn.Linear:
        nn.init.xavier_uniform_(m.weight)
def init_42(m):
    if type(m) == nn.Linear:
        nn.init.constant_(m.weight, 42)

net[0].apply(init_xavier)
net[2].apply(init_42)
print(net[0].weight.data[0])
print(net[2].weight.data)

1
2
3
4
5
6
7
8
9

def my_init(m):
    if type(m) == nn.Linear:
        print("Init", *[(name, param.shape)
                        for name, param in m.named_parameters()][0])
        nn.init.uniform_(m.weight, -10, 10)
        m.weight.data *= m.weight.data.abs() >= 5

net.apply(my_init)
net[0].weight[:2]

1
2
3

net[0].weight.data[:] += 1
net[0].weight.data[0, 0] = 42
net[0].weight.data[0]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 设置共享层的名称
shared = nn.Linear(8, 8)
net = nn.Sequential(nn.Linear(4, 8), nn.ReLU(),
                    shared, nn.ReLU(),
                    shared, nn.ReLU(),
                    nn.Linear(8, 1))
net(X)
# 检查参数是否相同
print(net[2].weight.data[0] == net[4].weight.data[0])
net[2].weight.data[0, 0] = 100
# 确保它们实际上是同一个对象，而不只是有相同的值
print(net[2].weight.data[0] == net[4].weight.data[0])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

import torch
import torch.nn.functional as F
from torch import nn


class CenteredLayer(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, X):
        return X - X.mean()

# 使用
net = nn.Sequential(nn.Linear(8, 128), CenteredLayer())

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 定义带参数的层
# in_units: 输入数
# units: 输出数
class MyLinear(nn.Module):
    def __init__(self, in_units, units):
        super().__init__()
        # 构造参数要放入nn.Parameter()
        self.weight = nn.Parameter(torch.randn(in_units, units))
        self.bias = nn.Parameter(torch.randn(units,))
    def forward(self, X):
        linear = torch.matmul(X, self.weight.data) + self.bias.data
        return F.relu(linear)

1
2
3
4
5
6
7
8
9

import torch
from torch import nn
from torch.nn import functional as F

x = torch.arange(4)
torch.save(x, 'x-file')

x2 = torch.load('x-file')
x2

1
2
3
4

mydict = {'x': x, 'y': y}
torch.save(mydict, 'mydict')
mydict2 = torch.load('mydict')
mydict2

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

class MLP(nn.Module):
    def __init__(self):
        super().__init__()
        self.hidden = nn.Linear(20, 256)
        self.output = nn.Linear(256, 10)

    def forward(self, x):
        return self.output(F.relu(self.hidden(x)))

net = MLP()
X = torch.randn(size=(2, 20))
Y = net(X)

1

torch.save(net.state_dict(), 'mlp.params')

1
2
3
4
5
6
7
8

clone = MLP()
clone.load_state_dict(torch.load('mlp.params'))
clone.eval() # 进入评估模式，不计算梯度

Y_clone = clone(X)
Y_clone == Y
# tensor([[True, True, True, True, True, True, True, True, True, True],
#         [True, True, True, True, True, True, True, True, True, True]])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# 查询可用gpu的数量
torch.cuda.device_count()

# 张量默认创建在CPU上
x = torch.tensor([1, 2, 3])
x.device
# device(type='cpu')

# 存储在GPU上
X = torch.ones(2, 3, device=try_gpu())
X
# tensor([[1., 1., 1.],
#         [1., 1., 1.]], device='cuda:0')