这是第三次课。我们来构建ChatGPT 2.0。首先，我们直接上代码。
（借助已有的库，我们可以只用几百行代码就能实现一个GPT 2.0。我们不需要写那些非常基本的函数！这真是太棒了！）
# This file collects all the relevant code that we covered thus far
# throughout Chapters 2-4.
# This file can be run as a standalone script.
import tiktoken
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
#####################################
# Chapter 2
#####################################
class GPTDatasetV1(Dataset):
def __init__(self, txt, tokenizer, max_length, stride):
self.input_ids = []
self.target_ids = []
# Tokenize the entire text
token_ids = tokenizer.encode(txt, allowed_special={"<|endoftext|>"})
# Use a sliding window to chunk the book into overlapping sequences of max_length
for i in range(0, len(token_ids) - max_length, stride):
input_chunk = token_ids[i:i + max_length]
target_chunk = token_ids[i + 1: i + max_length + 1]
self.input_ids.append(torch.tensor(input_chunk))
self.target_ids.append(torch.tensor(target_chunk))
def __len__(self):
return len(self.input_ids)
def __getitem__(self, idx):
return self.input_ids[idx], self.target_ids[idx]
def create_dataloader_v1(txt, batch_size=4, max_length=256,
stride=128, shuffle=True, drop_last=True, num_workers=0):
# Initialize the tokenizer
tokenizer = tiktoken.get_encoding("gpt2")
# Create dataset
dataset = GPTDatasetV1(txt, tokenizer, max_length, stride)
# Create dataloader
dataloader = DataLoader(
dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=num_workers)
return dataloader
#####################################
# Chapter 3
#####################################
class MultiHeadAttention(nn.Module):
def __init__(self, d_in, d_out, context_length, dropout, num_heads, qkv_bias=False):
super().__init__()
assert d_out % num_heads == 0, "d_out must be divisible by num_heads"
self.d_out = d_out
self.num_heads = num_heads
self.head_dim = d_out // num_heads # Reduce the projection dim to match desired output dim
self.W_query = nn.Linear(d_in, d_out, bias=qkv_bias)
self.W_key = nn.Linear(d_in, d_out, bias=qkv_bias)
self.W_value = nn.Linear(d_in, d_out, bias=qkv_bias)
self.out_proj = nn.Linear(d_out, d_out) # Linear layer to combine head outputs
self.dropout = nn.Dropout(dropout)
self.register_buffer("mask", torch.triu(torch.ones(context_length, context_length), diagonal=1))
def forward(self, x):
b, num_tokens, d_in = x.shape
keys = self.W_key(x) # Shape: (b, num_tokens, d_out)
queries = self.W_query(x)
values = self.W_value(x)
# We implicitly split the matrix by adding a `num_heads` dimension
# Unroll last dim: (b, num_tokens, d_out) -> (b, num_tokens, num_heads, head_dim)
keys = keys.view(b, num_tokens, self.num_heads, self.head_dim)
values = values.view(b, num_tokens, self.num_heads, self.head_dim)
queries = queries.view(b, num_tokens, self.num_heads, self.head_dim)
# Transpose: (b, num_tokens, num_heads, head_dim) -> (b, num_heads, num_tokens, head_dim)
keys = keys.transpose(1, 2)
queries = queries.transpose(1, 2)
values = values.transpose(1, 2)
# Compute scaled dot-product attention (aka self-attention) with a causal mask
attn_scores = queries @ keys.transpose(2, 3) # Dot product for each head
# Original mask truncated to the number of tokens and converted to boolean
mask_bool = self.mask.bool()[:num_tokens, :num_tokens]
# Use the mask to fill attention scores
attn_scores.masked_fill_(mask_bool, -torch.inf)
attn_weights = torch.softmax(attn_scores / keys.shape[-1]**0.5, dim=-1)
attn_weights = self.dropout(attn_weights)
# Shape: (b, num_tokens, num_heads, head_dim)
context_vec = (attn_weights @ values).transpose(1, 2)
# Combine heads, where self.d_out = self.num_heads * self.head_dim
context_vec = context_vec.contiguous().view(b, num_tokens, self.d_out)
context_vec = self.out_proj(context_vec) # optional projection
return context_vec
#####################################
# Chapter 4
#####################################
class LayerNorm(nn.Module):
def __init__(self, emb_dim):
super().__init__()
self.eps = 1e-5
self.scale = nn.Parameter(torch.ones(emb_dim))
self.shift = nn.Parameter(torch.zeros(emb_dim))
def forward(self, x):
mean = x.mean(dim=-1, keepdim=True)
var = x.var(dim=-1, keepdim=True, unbiased=False)
norm_x = (x - mean) / torch.sqrt(var + self.eps)
return self.scale * norm_x + self.shift
class GELU(nn.Module):
def __init__(self):
super().__init__()
def forward(self, x):
return 0.5 * x * (1 + torch.tanh(
torch.sqrt(torch.tensor(2.0 / torch.pi)) *
(x + 0.044715 * torch.pow(x, 3))
))
class FeedForward(nn.Module):
def __init__(self, cfg):
super().__init__()
self.layers = nn.Sequential(
nn.Linear(cfg["emb_dim"], 4 * cfg["emb_dim"]),
GELU(),
nn.Linear(4 * cfg["emb_dim"], cfg["emb_dim"]),
)
def forward(self, x):
return self.layers(x)
class TransformerBlock(nn.Module):
def __init__(self, cfg):
super().__init__()
self.att = MultiHeadAttention(
d_in=cfg["emb_dim"],
d_out=cfg["emb_dim"],
context_length=cfg["context_length"],
num_heads=cfg["n_heads"],
dropout=cfg["drop_rate"],
qkv_bias=cfg["qkv_bias"])
self.ff = FeedForward(cfg)
self.norm1 = LayerNorm(cfg["emb_dim"])
self.norm2 = LayerNorm(cfg["emb_dim"])
self.drop_shortcut = nn.Dropout(cfg["drop_rate"])
def forward(self, x):
# Shortcut connection for attention block
shortcut = x
x = self.norm1(x)
x = self.att(x) # Shape [batch_size, num_tokens, emb_size]
x = self.drop_shortcut(x)
x = x + shortcut # Add the original input back
# Shortcut connection for feed-forward block
shortcut = x
x = self.norm2(x)
x = self.ff(x)
x = self.drop_shortcut(x)
x = x + shortcut # Add the original input back
return x
class GPTModel(nn.Module):
def __init__(self, cfg):
super().__init__()
self.tok_emb = nn.Embedding(cfg["vocab_size"], cfg["emb_dim"])
self.pos_emb = nn.Embedding(cfg["context_length"], cfg["emb_dim"])
self.drop_emb = nn.Dropout(cfg["drop_rate"])
self.trf_blocks = nn.Sequential(
*[TransformerBlock(cfg) for _ in range(cfg["n_layers"])])
self.final_norm = LayerNorm(cfg["emb_dim"])
self.out_head = nn.Linear(cfg["emb_dim"], cfg["vocab_size"], bias=False)
def forward(self, in_idx):
batch_size, seq_len = in_idx.shape
tok_embeds = self.tok_emb(in_idx)
pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device))
x = tok_embeds + pos_embeds # Shape [batch_size, num_tokens, emb_size]
x = self.drop_emb(x)
x = self.trf_blocks(x)
x = self.final_norm(x)
logits = self.out_head(x)
return logits
def generate_text_simple(model, idx, max_new_tokens, context_size):
# idx is (B, T) array of indices in the current context
for _ in range(max_new_tokens):
# Crop current context if it exceeds the supported context size
# E.g., if LLM supports only 5 tokens, and the context size is 10
# then only the last 5 tokens are used as context
idx_cond = idx[:, -context_size:]
# Get the predictions
with torch.no_grad():
logits = model(idx_cond)
# Focus only on the last time step
# (batch, n_token, vocab_size) becomes (batch, vocab_size)
logits = logits[:, -1, :]
# Get the idx of the vocab entry with the highest logits value
idx_next = torch.argmax(logits, dim=-1, keepdim=True) # (batch, 1)
# Append sampled index to the running sequence
idx = torch.cat((idx, idx_next), dim=1) # (batch, n_tokens+1)
return idx
def main():
GPT_CONFIG_124M = {
"vocab_size": 50257, # Vocabulary size
"context_length": 1024, # Context length
"emb_dim": 768, # Embedding dimension
"n_heads": 12, # Number of attention heads
"n_layers": 12, # Number of layers
"drop_rate": 0.1, # Dropout rate
"qkv_bias": False # Query-Key-Value bias
}
torch.manual_seed(123)
model = GPTModel(GPT_CONFIG_124M)
model.eval() # disable dropout
start_context = "Hello, My name is"
tokenizer = tiktoken.get_encoding("gpt2")
encoded = tokenizer.encode(start_context)
encoded_tensor = torch.tensor(encoded).unsqueeze(0)
print(f"\n{50*'='}\n{22*' '}IN\n{50*'='}")
print("\nInput text:", start_context)
print("Encoded input text:", encoded)
print("encoded_tensor.shape:", encoded_tensor.shape)
out = generate_text_simple(
model=model,
idx=encoded_tensor,
max_new_tokens=10,
context_size=GPT_CONFIG_124M["context_length"]
)
decoded_text = tokenizer.decode(out.squeeze(0).tolist())
print(f"\n\n{50*'='}\n{22*' '}OUT\n{50*'='}")
print("\nOutput:", out)
print("Output length:", len(out[0]))
print("Output text:", decoded_text)
if __name__ == "__main__":
main()
GPT 模型架构
输入 token ids
↓
Token Embedding
+
Positional Embedding
↓
Dropout
↓
TransformerBlock × N
↓
Final LayerNorm
↓
Linear (LM Head)
↓
Logits (预测下一个 token)
代码实现
我们已经明白架构。下面做的事情就是将输入通过不同函数的作用执行架构里面的流程。在这里这些流程通过forward函数来完成。
class GPTModel(nn.Module):
def __init__(self, cfg):
super().__init__()
self.tok_emb = nn.Embedding(cfg["vocab_size"], cfg["emb_dim"])
self.pos_emb = nn.Embedding(cfg["context_length"], cfg["emb_dim"])
self.drop_emb = nn.Dropout(cfg["drop_rate"])
self.trf_blocks = nn.Sequential(
*[TransformerBlock(cfg) for _ in range(cfg["n_layers"])])
self.final_norm = LayerNorm(cfg["emb_dim"])
self.out_head = nn.Linear(cfg["emb_dim"], cfg["vocab_size"], bias=False)
def forward(self, in_idx):
# in_idx: (B, T)
# 输入是 token 的整数索引（词表编号）
batch_size, seq_len = in_idx.shape
# 取出 batch 大小 B 和 序列长度 T
tok_embeds = self.tok_emb(in_idx)
# Token Embedding
# 把 token id 查表变成向量
# (B, T) → (B, T, C)
# C = emb_dim
pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device))
# Positional Embedding
# 生成位置 0...T-1 的向量
# (T) → (T, C)
x = tok_embeds + pos_embeds
# 把 token 向量 + 位置向量
# 得到带位置信息的输入表示
# 形状: (B, T, C)
x = self.drop_emb(x)
# Embedding 后做 Dropout（训练时防止过拟合）
# 形状不变 (B, T, C)
x = self.trf_blocks(x)
# 经过 N 层 TransformerBlock
# 每层包含：
# LayerNorm → Causal Multi-Head Attention → 残差
# LayerNorm → FeedForward(MLP) → 残差
# 形状保持 (B, T, C)
x = self.final_norm(x)
# 所有 Transformer 层之后再做一次 LayerNorm
# 提高数值稳定性
# 形状 (B, T, C)
logits = self.out
Transformer 架构如下
输入 x (B, T, C)
│
├───────────────────────────────┐
│ │
│ ┌───────────────┐ │
│ │ LayerNorm #1 │ norm1 │
│ └───────┬───────┘ │
│ │ │
│ ┌───────▼────────┐ │
│ │ Multi-Head │ att │
│ │ Causal SelfAttn │ │
│ └───────┬────────┘ │
│ │ │
│ ┌────▼─────┐ │
│ │ Dropout │ drop_shortcut
│ └────┬─────┘ │
│ │ │
└───────────┼────────── (+) ────┘ ← residual add
│
x1 (B, T, C)
│
├───────────────────────────────┐
│ │
│ ┌───────────────┐ │
│ │ LayerNorm #2 │ norm2 │
│ └───────┬───────┘ │
│ │ │
│ ┌───────▼────────┐ │
│ │ FeedForward │ ff │
│ │ (MLP: C→4C→C) │ │
│ └───────┬────────┘ │
│ │ │
│ ┌────▼─────┐ │
│ │ Dropout │ drop_shortcut
│ └────┬─────┘ │
│ │ │
└───────────┼────────── (+) ────┘ ← residual add
│
输出 x2 (B, T, C)
代码实现
class TransformerBlock(nn.Module):
def __init__(self, cfg):
super().__init__()
# ===== Multi-Head Causal Self-Attention =====
# 这是 GPT 的核心模块：
# 作用：让每个 token 根据“前面所有 token”更新自己的表示
# d_in = d_out = emb_dim（保持维度一致，方便做残差）
self.att = MultiHeadAttention(
d_in=cfg["emb_dim"],
d_out=cfg["emb_dim"],
context_length=cfg["context_length"], # 用来创建 causal mask
num_heads=cfg["n_heads"], # 多头数量
dropout=cfg["drop_rate"],
qkv_bias=cfg["qkv_bias"]
)
# ===== FeedForward (MLP) =====
# 结构：Linear(C,4C) → GELU → Linear(4C,C)
# 作用：做非线性特征变换
self.ff = FeedForward(cfg)
# ===== LayerNorm（Pre-LN结构）=====
# GPT-2 之后使用的是 Pre-LN：
# LN → 子层 → 残差
self.norm1 = LayerNorm(cfg["emb_dim"]) # Attention 前
self.norm2 = LayerNorm(cfg["emb_dim"]) # FFN 前
# ===== Dropout =====
# 用在子层输出后、残差相加前
self.drop_shortcut = nn.Dropout(cfg["drop_rate"])
def forward(self, x):
# x 的形状: (batch_size, seq_len, emb_dim)
# B = batch size
# T = token 数
# C = emb_dim
# ==================================================
# 1️. Attention 子层
# ==================================================
shortcut = x
# 保存原始输入，用于残差连接
x = self.norm1(x)
# Pre-LN：先做 LayerNorm
# 形状不变 (B, T, C)
x = self.att(x)
# 多头因果自注意力
# 每个 token 只能看到自己和前面的 token
# 输出形状仍为 (B, T, C)
x = self.drop_shortcut(x)
# 对 Attention 输出做 Dropout（训练时随机置零部分神经元）
x = x + shortcut
# 残差连接
# 防止梯度消失，让深层网络更稳定
# 输出形状仍 (B, T, C)
# ==================================================
# 2️. FeedForward 子层
# ==================================================
shortcut = x
# 再次保存当前输入，用于 FFN 的残差连接
x = self.norm2(x)
# Pre-LN：在 FFN 前做 LayerNorm
# 形状 (B, T, C)
x = self.ff(x)
# 前馈网络：
# Linear(C,4C) → GELU → Linear(4C,C)
# 输出形状仍 (B, T, C)
x = self.drop_shortcut(x)
# Dropout
x = x + shortcut
# 残差连接
# 输出形状仍 (B, T, C)
return x
# 返回这一层 TransformerBlock 的输出
附录
关于PyTorch里面nn的库（nn.embedding)
在 GPT 里：
Embedding 就是把“单词编号”变成“可以计算的向量”。
假设我们的词表只有 3 个词：
0 → "I"
1 → "love"
2 → "AI"
假设：
emb_dim = 2 （每个词用 2 维向量表示）
那 GPT 里的 embedding 层内部其实是一个矩阵：
权重矩阵 (vocab_size × emb_dim)
dim1 dim2
I [0.2 0.8]
love [0.9 0.1]
AI [0.4 0.6]
** 现在输入一句话**
"I love AI"
对应 token ids：
[0, 1, 2]
经过 Embedding
GPT 做的事情就是：
查表
变成：
[
[0.2, 0.8], ← "I"
[0.9, 0.1], ← "love"
[0.4, 0.6] ← "AI"
]
形状变成：
(3, 2)
如果 batch=1，就是：
(1, 3, 2)
然后 GPT 才开始工作
GPT 不能直接处理单词
它只能处理向量
所以 Embedding 是第一步
参考：
[1].《从零构建大模型》
[2]. 代码来自：
github.com/rasbt/LLMs-…
License: Apache 2.0
