1
2

import torch
import torch.nn as nn

1
2
3
4
5

input = torch.randn(3, 4) #3个样本，每个样本4个特征

linear = nn.Linear(in_features=4, out_features=2)
linear(input).shape
# torch.Size([3, 2])

1
2
3
4
5
6
7

dropout = nn.Dropout(p=0.5)

input = torch.randn(3, 4)
dropout(input)
# tensor([[ 0.0000, -0.0000, -0.5670, -0.0000],
#         [ 4.7224, -4.0010,  0.0000, -0.0000],
#         [-0.9960,  0.0000,  0.6658, -0.0000]])

1
2
3
4
5
6
7

batchnorm = nn.BatchNorm1d(num_features=4)
batchnorm(input).shape       # (batch_size, emb_len)
# torch.Size([3, 4])

input2 = torch.randn(2, 4, 3) # (batch_size, emb_len, seq_len)
batchnorm(input2).shape
# torch.Size([2, 4, 3])

1
2
3
4
5
6
7
8

layer_norm = nn.LayerNorm(normalized_shape=4)

layer_norm(input).shape
# torch.Size([3, 4])

input2 = torch.randn(2, 3, 4) # (batch_size, seq_len, emb_len)
layer_norm(input2).shape
# torch.Size([2, 3, 4])

1
2
3

relu = nn.ReLU()
sigmoid = nn.Sigmoid()
softmax = nn.Softmax(dim=1) #对轴1进行softmax转换，使其和为1

1
2
3
4
5
6
7

embedding = nn.Embedding(num_embeddings=1000, embedding_dim=64)
# num_embeddings表示词汇表的大小，即共有多少个不同的词元
# embedding_dim表示嵌入向量的长度

input_indices = torch.tensor([1, 2, 3, 4])
embedding(input_indices).shape
# torch.Size([4, 64])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

d_model = 512  # 输入特征维度
nhead = 8      # 注意力头数

# d_model: 输入和输出的特征维度 
# nhead: 注意力头数
# dim_feedforward: FFN的神经元个数（MLP）
# dropout: 丢失率，默认0.1
# batch_first: 默认为False，即输入为(seq, batch, feature); 设置为True，则输入为(batch, seq, feature) 
# norm_first: 默认为False，即在Attention和FFN之前进行norm
# activation: 前馈网络中使用的激活函数，默认是 relu，可以选择其他激活函数如 gelu。

# 创建 Transformer 编码器层
encoder_layer = nn.TransformerEncoderLayer(d_model=d_model, 
                                           nhead=nhead, 
                                           dim_feedforward=d_model*2, 
                                           dropout=0.1, 
                                           batch_first=True)
 
input = torch.randn(2, 10, 512) # (批量大小, 序列长度, 特征维度)
encoder_layer(input).shape
# torch.Size([2, 10, 512])

1
2
3
4

num_layers = 12
transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
transformer_encoder(input).shape
# torch.Size([2, 10, 512])