Transformer模型是现代自然语言处理（NLP）中的一种重要架构，它在许多任务中表现出色。理解Transformer的原理可以从以下几个方面入手：
-
理论基础：
- Attention机制：Transformer的核心是Attention机制，特别是自注意力（Self-Attention）机制。它允许模型在处理输入序列的每个位置时，关注序列中的其他位置。
- 多头注意力（Multi-Head Attention）：通过并行计算多个注意力机制，模型可以捕捉到不同的特征。
- 位置编码（Positional Encoding）：因为Transformer没有内置的序列顺序信息，所以需要通过位置编码来引入位置信息。
- 前馈神经网络（Feed-Forward Neural Network）：在每个注意力层之后，使用前馈神经网络进行进一步的特征提取。
-
代码实现：
- 可以使用PyTorch或TensorFlow等深度学习框架来实现Transformer模型。以下是一个简化的Transformer编码示例，使用PyTorch实现：
import torch
import torch.nn as nn
import torch.nn.functional as F
class MultiHeadAttention(nn.Module):
def __init__(self, embed_size, heads):
super(MultiHeadAttention, self).__init__()
self.embed_size = embed_size
self.heads = heads
self.head_dim = embed_size // heads
assert (
self.head_dim * heads == embed_size
), "Embedding size needs to be divisible by heads"
self.values = nn.Linear(self.head_dim, embed_size, bias=False)
self.keys = nn.Linear(self.head_dim, embed_size, bias=False)
self.queries = nn.Linear(self.head_dim, embed_size, bias=False)
self.fc_out = nn.Linear(embed_size, embed_size)
def forward(self, values, keys, query, mask):
N = query.shape[0]
value_len, key_len, query_len = values.shape[1], keys.shape[1], query.shape[1]
# Split the embedding into self.heads different pieces
values = values.reshape(N, value_len, self.heads, self.head_dim)
keys = keys.reshape(N, key_len, self.heads, self.head_dim)
queries = query.reshape(N, query_len, self.heads, self.head_dim)
values = self.values(values)
keys = self.keys(keys)
queries = self.queries(queries)
# Scaled dot-product attention
energy = torch.einsum("nqhd,nkhd->nhqk", [queries, keys])
if mask is not None:
energy = energy.masked_fill(mask == 0, float("-1e20"))
attention = torch.softmax(energy / (self.embed_size ** (1 / 2)), dim=3)
out = torch.einsum("nhql,nlhd->nqhd", [attention, values]).reshape(
N, query_len, self.embed_size
)
out = self.fc_out(out)
return out
class TransformerBlock(nn.Module):
def __init__(self, embed_size, heads, dropout, forward_expansion):
super(TransformerBlock, self).__init__()
self.attention = MultiHeadAttention(embed_size, heads)
self.norm1 = nn.LayerNorm(embed_size)
self.norm2 = nn.LayerNorm(embed_size)
self.feed_forward = nn.Sequential(
nn.Linear(embed_size, forward_expansion * embed_size),
nn.ReLU(),
nn.Linear(forward_expansion * embed_size, embed_size)
)
self.dropout = nn.Dropout(dropout)
def forward(self, value, key, query, mask):
attention = self.attention(value, key, query, mask)
x = self.dropout(self.norm1(attention + query))
forward = self.feed_forward(x)
out = self.dropout(self.norm2(forward + x))
return out
class Encoder(nn.Module):
def __init__(
self,
src_vocab_size,
embed_size,
num_layers,
heads,
device,
forward_expansion,
dropout,
max_length,
):
super(Encoder, self).__init__()
self.embed_size = embed_size
self.device = device
self.word_embedding = nn.Embedding(src_vocab_size, embed_size)
self.position_embedding = nn.Embedding(max_length, embed_size)
self.layers = nn.ModuleList(
[
TransformerBlock(
embed_size,
heads,
dropout=dropout,
forward_expansion=forward_expansion,
)
for _ in range(num_layers)
]
)
self.dropout = nn.Dropout(dropout)
def forward(self, x, mask):
N, seq_length = x.shape
positions = torch.arange(0, seq_length).expand(N, seq_length).to(self.device)
out = self.dropout(self.word_embedding(x) + self.position_embedding(positions))
for layer in self.layers:
out = layer(out, out, out, mask)
return out
class Transformer(nn.Module):
def __init__(
self,
src_vocab_size,
trg_vocab_size,
src_pad_idx,
trg_pad_idx,
embed_size=256,
num_layers=6,
forward_expansion=4,
heads=8,
dropout=0,
device="cuda",
max_length=100,
):
super(Transformer, self).__init__()
self.encoder = Encoder(
src_vocab_size,
embed_size,
num_layers,
heads,
device,
forward_expansion,
dropout,
max_length,
)
self.src_pad_idx = src_pad_idx
self.trg_pad_idx = trg_pad_idx
self.device = device
def make_src_mask(self, src):
src_mask = (src != self.src_pad_idx).unsqueeze(1).unsqueeze(2)
return src_mask.to(self.device)
def forward(self, src, trg):
src_mask = self.make_src_mask(src)
enc_src = self.encoder(src, src_mask)
return enc_src
# Example usage
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
src_vocab_size = 10000
trg_vocab_size = 10000
src_pad_idx = 0
trg_pad_idx = 0
model = Transformer(src_vocab_size, trg_vocab_size, src_pad_idx, trg_pad_idx).to(device)
# Dummy input
src = torch.tensor([[1, 5, 6, 4, 3, 9, 5, 2, 0], [1, 8, 7, 3, 4, 5, 6, 7, 2]]).to(device)
trg = torch.tensor([[1, 7, 4, 3, 5, 9, 2, 0], [1, 5, 6, 2, 4, 7, 6, 2]]).to(device)
out = model(src, trg[:, :-1])
print(out.shape)
- 测试用例：
- 测试用例可以帮助验证模型的正确性。以下是一些测试用例：
# Test case 1: Simple input
src = torch.tensor([[1, 2, 3, 4, 0, 0, 0, 0, 0]]).to(device)
trg = torch.tensor([[1, 2, 3, 4, 0, 0, 0, 0]]).to(device)
out = model(src, trg[:, :-1])
print(out.shape) # Expected output shape: (1, 8, embed_size)
# Test case 2: Different lengths
src = torch.tensor([[1, 2, 3, 4, 5, 6, 7, 8, 9]]).to(device)
trg = torch.tensor([[1, 2, 3, 4, 5, 6, 7, 8]]).to(device)
out = model(src, trg[:, :-1])
print(out.shape) # Expected output shape: (1, 8, embed_size)
# Test case 3: Batch input
src = torch.tensor([[1, 2, 3, 4, 0, 0, 0, 0, 0], [1, 2, 3, 4, 5, 6, 7, 8, 9]]).to(device)
trg = torch.tensor([[1, 2, 3, 4, 0, 0, 0, 0], [1, 2, 3, 4, 5, 6, 7, 8]]).to(device)
out = model(src, trg[:, :-1])
print(out.shape) # Expected output shape: (2, 8, embed_size)
通过这些步骤和代码示例，你可以逐步理解Transformer模型的原理和实现。如果有任何具体问题或需要进一步的解释，请告诉我！
