transformer_多头注意力机制代码笔记

以GPT-2中多头注意力机制代码为例

class CausalSelfAttention(nn.Module):"""因果掩码+多头自注意力机制A vanilla multi-head masked self-attention layer with a projection at the end.It is possible to use torch.nn.MultiheadAttention here but I am including anexplicit implementation here to show that there is nothing too scary here."""def __init__(self, config):super().__init__()assert config.n_embd % config.n_head == 0# key, query, value projections for all heads, but in a batchself.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)  # Q/K/V的线性映射# output projectionself.c_proj = nn.Linear(config.n_embd, config.n_embd)  # 输出线性映射# regularizationself.attn_dropout = nn.Dropout(config.attn_pdrop)  # dropout正则化self.resid_dropout = nn.Dropout(config.resid_pdrop)# causal mask to ensure that attention is only applied to the left in the input sequenceself.register_buffer("bias", torch.tril(torch.ones(config.block_size, config.block_size)).view(1, 1, config.block_size, config.block_size))  # 构建casual mask矩阵self.n_head = config.n_headself.n_embd = config.n_embddef forward(self, x):B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)# calculate query, key, values for all heads in batch and move head forward to be the batch dimq, k ,v  = self.c_attn(x).split(self.n_embd, dim=2)  # 同时计算QKV# 多头切分, nh*hs=C# 一定要先view (B, T, nh, hs), 再transpose，不能直接view到(B, nh, T, hs)  k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)  q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)# causal self-attention; Self-attend: (B, nh, T, hs) x (B, nh, hs, T) -> (B, nh, T, T)# 除以根号下dk保持方差稳定att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))# 因果掩码att = att.masked_fill(self.bias[:,:,:T,:T] == 0, float('-inf'))att = F.softmax(att, dim=-1)att = self.attn_dropout(att)y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs)# contiguous:张量在底层存储时必须连续才可以使用viewy = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side# output projectiony = self.resid_dropout(self.c_proj(y))return y

以下为对多头注意力机制代码做分步笔记
上述代码中初始化中定义了构建多头注意力机制代码的组件（结构），在forward的方法中将使用初始化中的组件构建多头注意力机制。
从forward方法开始阅读，当使用到初始化方法中的代码时再进行阅读

# x是输入的数据
# B是批大小
# T是序列长度
# C是embedding的维度
B, T, C = x.size()

# 通过self.c_atten初始化q，k，v
q, k ,v  = self.c_attn(x).split(self.n_embd, dim=2)# 初始化q，k，v
# 验证数据维度与多头的数量是否一致
assert config.n_embd % config.n_head == 0
# 因为q，k，v是相同的大小的不同矩阵，通过线性映射获得初始化的q，k，v
self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)

# 多头切分, nh*hs=C，其中nh为头的个数，hs为头的维度
# 一定要先view (B, T, nh, hs), 再transpose，不能直接view到(B, nh, T, hs)  
k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)  
q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)# 其中self.n_head是配置文件中预先设置好的
self.n_head = config.n_head
# transpose是为了转换不同维度上的数据

# 此处是attention的公式
# causal self-attention; Self-attend: (B, nh, T, hs) x (B, nh, hs, T) -> (B, nh, T, T)
# 除以根号下dk保持方差稳定
att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1)))

# 因果掩码
att = att.masked_fill(self.bias[:,:,:T,:T] == 0, float('-inf'))
att = F.softmax(att, dim=-1)
att = self.attn_dropout(att)
y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs)

# contiguous:张量在底层存储时必须连续才可以使用view
y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side# 前面讲C拆解为两个矩阵相乘，现在讲得到的结果返回最初的数据形状
# contiguous()是深拷贝

# output projection
y = self.resid_dropout(self.c_proj(y))# output projection
self.c_proj = nn.Linear(config.n_embd, config.n_embd)  # 输出线性映射
# regularization
self.attn_dropout = nn.Dropout(config.attn_pdrop)  # dropout正则化
self.resid_dropout = nn.Dropout(config.resid_pdrop)