use std::{
cell::{Cell, RefCell},
rc::Rc,
};
use crate::{DType, Engine, Error, Matrix, Result, matrix as core_matrix};
use super::super::{
Module, ModuleRegistry,
matrix::{
bmm, bmm_nt, bmm_tn, flash_attention_causal, merge_heads, scaled_dot_product_attention,
softmax_scaled_masked, split_heads,
},
};
use super::Linear;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum AttentionBackend {
#[default]
Auto,
Standard,
Flash,
Linear,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum AttentionMode {
#[default]
Causal,
Bidirectional,
}
pub struct MultiHeadAttention {
model_width: usize,
num_heads: usize,
sequence_length: Cell<usize>,
query_projection: Rc<Linear>,
key_projection: Rc<Linear>,
value_projection: Rc<Linear>,
output_projection: Rc<Linear>,
dropout_probability: f32,
dropout_seed: u64,
backend: Cell<AttentionBackend>,
mode: Cell<AttentionMode>,
last_backend: Cell<AttentionBackend>,
mask_cache: RefCell<Vec<AttentionMask>>,
registry: ModuleRegistry,
}
struct AttentionMask {
batch: usize,
sequence_length: usize,
causal: bool,
value: Matrix,
}
impl MultiHeadAttention {
pub fn new_with_seed(
engine: &Engine,
model_width: usize,
num_heads: usize,
sequence_length: usize,
seed: u64,
) -> Result<Self> {
Self::new_with_seed_and_options(
engine,
model_width,
num_heads,
sequence_length,
0.0,
true,
seed,
)
}
pub fn new_with_seed_and_bias(
engine: &Engine,
model_width: usize,
num_heads: usize,
sequence_length: usize,
bias: bool,
seed: u64,
) -> Result<Self> {
Self::new_with_seed_and_options(
engine,
model_width,
num_heads,
sequence_length,
0.0,
bias,
seed,
)
}
#[allow(clippy::too_many_arguments)]
pub fn new_with_seed_and_options(
engine: &Engine,
model_width: usize,
num_heads: usize,
sequence_length: usize,
dropout_probability: f32,
bias: bool,
seed: u64,
) -> Result<Self> {
if model_width == 0
|| num_heads == 0
|| sequence_length == 0
|| !model_width.is_multiple_of(num_heads)
{
return Err(Error::invalid_argument(
"attention requires positive D/H/S and D divisible by H",
));
}
if !dropout_probability.is_finite() || !(0.0..1.0).contains(&dropout_probability) {
return Err(Error::invalid_argument(
"attention dropout probability must be finite and in [0, 1)",
));
}
let query_projection = Rc::new(Linear::new_with_seed_and_bias(
engine,
model_width,
model_width,
bias,
seed,
)?);
let key_projection = Rc::new(Linear::new_with_seed_and_bias(
engine,
model_width,
model_width,
bias,
seed.wrapping_add(1),
)?);
let value_projection = Rc::new(Linear::new_with_seed_and_bias(
engine,
model_width,
model_width,
bias,
seed.wrapping_add(2),
)?);
let output_projection = Rc::new(Linear::new_with_seed_and_bias(
engine,
model_width,
model_width,
bias,
seed.wrapping_add(3),
)?);
let mut registry = ModuleRegistry::new();
registry.register_module("q_proj", query_projection.clone())?;
registry.register_module("k_proj", key_projection.clone())?;
registry.register_module("v_proj", value_projection.clone())?;
registry.register_module("out_proj", output_projection.clone())?;
Ok(Self {
model_width,
num_heads,
sequence_length: Cell::new(sequence_length),
query_projection,
key_projection,
value_projection,
output_projection,
dropout_probability,
dropout_seed: seed.wrapping_add(4),
backend: Cell::new(AttentionBackend::Auto),
mode: Cell::new(AttentionMode::Causal),
last_backend: Cell::new(AttentionBackend::Standard),
mask_cache: RefCell::new(Vec::new()),
registry,
})
}
fn geometry(&self, input: &Matrix) -> Result<usize> {
let sequence_length = self.sequence_length.get();
let [rows, width] = input.shape() else {
return Err(Error::invalid_argument(
"MultiHeadAttention expects F32 [B*S, D] input",
));
};
if input.dtype() != DType::F32
|| *width != self.model_width
|| *rows == 0
|| !rows.is_multiple_of(sequence_length)
{
return Err(Error::invalid_argument(format!(
"MultiHeadAttention expects nonempty F32 [B*S, {}] with S={}; found {:?} {}",
self.model_width,
sequence_length,
input.shape(),
input.dtype().token(),
)));
}
Ok(rows / sequence_length)
}
fn project(&self, input: &Matrix) -> Result<(Matrix, Matrix, Matrix)> {
Ok((
self.query_projection.forward(input)?,
self.key_projection.forward(input)?,
self.value_projection.forward(input)?,
))
}
fn forward_standard(
&self,
input: &Matrix,
additive_mask: Option<&Matrix>,
causal: bool,
) -> Result<Matrix> {
let batch = self.geometry(input)?;
let sequence_length = self.sequence_length.get();
let (query, key, value) = self.project(input)?;
let query = split_heads(&query, batch, sequence_length, self.num_heads)?;
let key = split_heads(&key, batch, sequence_length, self.num_heads)?;
let value = split_heads(&value, batch, sequence_length, self.num_heads)?;
let scale = 1.0 / ((self.model_width / self.num_heads) as f32).sqrt();
let context = if self.is_training() && self.dropout_probability > 0.0 {
let batch_heads = batch
.checked_mul(self.num_heads)
.ok_or_else(|| Error::invalid_argument("attention batch by head count overflows usize"))?;
let score_rows = batch_heads
.checked_mul(sequence_length)
.ok_or_else(|| Error::invalid_argument("attention score rows overflow usize"))?;
let scores = bmm_nt(&query, &key)?.reshape([score_rows, sequence_length])?;
let owned_mask;
let mask = if let Some(mask) = additive_mask {
mask
} else {
owned_mask = self.cached_mask(input, batch, causal)?;
&owned_mask
};
let probability = softmax_scaled_masked(&scores, mask, scale)?;
let probability =
core_matrix::dropout(&probability, self.dropout_probability, self.dropout_seed)?;
bmm(
&probability.reshape([batch_heads, sequence_length, sequence_length])?,
&value,
)?
} else {
scaled_dot_product_attention(&query, &key, &value, additive_mask, scale, causal)?
};
let context = merge_heads(&context, batch, sequence_length, self.num_heads)?;
let output = self.output_projection.forward(&context)?;
self.last_backend.set(AttentionBackend::Standard);
Ok(output)
}
fn cached_mask(&self, input: &Matrix, batch: usize, causal: bool) -> Result<Matrix> {
let sequence_length = self.sequence_length.get();
if let Some(mask) = self.mask_cache.borrow().iter().find(|mask| {
mask.batch == batch && mask.sequence_length == sequence_length && mask.causal == causal
}) {
return Ok(mask.value.clone());
}
let batch_heads = batch
.checked_mul(self.num_heads)
.ok_or_else(|| Error::invalid_argument("attention batch by head count overflows usize"))?;
let score_rows = batch_heads
.checked_mul(sequence_length)
.ok_or_else(|| Error::invalid_argument("attention score rows overflow usize"))?;
let count = score_rows
.checked_mul(sequence_length)
.ok_or_else(|| Error::invalid_argument("attention score size overflows usize"))?;
let mut values = vec![0.0_f32; count];
if causal {
for row in 0..score_rows {
let query = row % sequence_length;
for key in query + 1..sequence_length {
values[row * sequence_length + key] = -1.0e9;
}
}
}
let mask = Matrix::allocate(
input.engine_handle(),
vec![score_rows, sequence_length],
count,
DType::F32,
)?;
mask.write_values(&values)?;
self.mask_cache.borrow_mut().push(AttentionMask {
batch,
sequence_length,
causal,
value: mask.clone(),
});
Ok(mask)
}
fn forward_flash(&self, input: &Matrix) -> Result<Matrix> {
let batch = self.geometry(input)?;
let sequence_length = self.sequence_length.get();
let (query, key, value) = self.project(input)?;
let query = split_heads(&query, batch, sequence_length, self.num_heads)?;
let key = split_heads(&key, batch, sequence_length, self.num_heads)?;
let value = split_heads(&value, batch, sequence_length, self.num_heads)?;
let scale = 1.0 / ((self.model_width / self.num_heads) as f32).sqrt();
let context = flash_attention_causal(&query, &key, &value, scale)?;
let context = merge_heads(&context, batch, sequence_length, self.num_heads)?;
let output = self.output_projection.forward(&context)?;
self.last_backend.set(AttentionBackend::Flash);
Ok(output)
}
pub fn forward(&self, input: &Matrix) -> Result<Matrix> {
let mode = self.mode.get();
match self.backend.get() {
AttentionBackend::Flash => {
if mode != AttentionMode::Causal
|| input.dtype() != DType::F32
|| self.dropout_probability != 0.0
|| self.sequence_length.get() > 1024
{
return Err(Error::invalid_argument(
"FlashAttention requires causal F32 input, dropout=0, and sequence length <= 1024",
));
}
self.forward_flash(input)
}
AttentionBackend::Linear => Err(Error::invalid_argument(
"AttentionBackend::Linear is not supported by MultiHeadAttention; use CausalLinearAttention instead",
)),
AttentionBackend::Auto | AttentionBackend::Standard => {
self.forward_standard(input, None, mode == AttentionMode::Causal)
}
}
}
pub fn forward_masked(&self, input: &Matrix, additive_mask: &Matrix) -> Result<Matrix> {
if self.backend.get() == AttentionBackend::Flash {
return Err(Error::invalid_argument(
"FlashAttention does not accept an arbitrary additive mask",
));
}
self.forward_standard(input, Some(additive_mask), false)
}
pub fn set_backend(&self, backend: AttentionBackend) {
self.backend.set(backend);
}
pub fn backend(&self) -> AttentionBackend {
self.backend.get()
}
pub fn last_backend(&self) -> AttentionBackend {
self.last_backend.get()
}
pub fn set_mode(&self, mode: AttentionMode) {
self.mode.set(mode);
}
pub fn mode(&self) -> AttentionMode {
self.mode.get()
}
pub const fn model_width(&self) -> usize {
self.model_width
}
pub const fn num_heads(&self) -> usize {
self.num_heads
}
pub fn sequence_length(&self) -> usize {
self.sequence_length.get()
}
pub fn set_sequence_length(&self, sequence_length: usize) -> Result<()> {
if sequence_length == 0 {
return Err(Error::invalid_argument(
"attention sequence length must be nonzero",
));
}
if self.sequence_length.replace(sequence_length) != sequence_length {
self.mask_cache.borrow_mut().clear();
}
Ok(())
}
pub fn has_bias(&self) -> bool {
self.query_projection.has_bias()
}
pub const fn dropout_probability(&self) -> f32 {
self.dropout_probability
}
}
impl Module for MultiHeadAttention {
fn forward(&self, input: &Matrix) -> Result<Matrix> {
MultiHeadAttention::forward(self, input)
}
fn registry(&self) -> &ModuleRegistry {
&self.registry
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum LinearAttentionMode {
#[default]
Causal,
Bidirectional,
}
pub struct CausalLinearAttention {
model_width: usize,
num_heads: usize,
sequence_length: Cell<usize>,
query_projection: Rc<Linear>,
key_projection: Rc<Linear>,
value_projection: Rc<Linear>,
output_projection: Rc<Linear>,
mode: Cell<LinearAttentionMode>,
registry: ModuleRegistry,
}
impl CausalLinearAttention {
pub fn new_with_seed(
engine: &Engine,
model_width: usize,
num_heads: usize,
sequence_length: usize,
seed: u64,
) -> Result<Self> {
Self::new_with_seed_and_options(engine, model_width, num_heads, sequence_length, true, seed)
}
pub fn new_with_seed_and_bias(
engine: &Engine,
model_width: usize,
num_heads: usize,
sequence_length: usize,
bias: bool,
seed: u64,
) -> Result<Self> {
Self::new_with_seed_and_options(engine, model_width, num_heads, sequence_length, bias, seed)
}
pub fn new_with_seed_and_options(
engine: &Engine,
model_width: usize,
num_heads: usize,
sequence_length: usize,
bias: bool,
seed: u64,
) -> Result<Self> {
if model_width == 0
|| num_heads == 0
|| sequence_length == 0
|| !model_width.is_multiple_of(num_heads)
{
return Err(Error::invalid_argument(
"CausalLinearAttention requires positive D/H/S and D divisible by H",
));
}
let query_projection = Rc::new(Linear::new_with_seed_and_bias(
engine,
model_width,
model_width,
bias,
seed,
)?);
let key_projection = Rc::new(Linear::new_with_seed_and_bias(
engine,
model_width,
model_width,
bias,
seed.wrapping_add(1),
)?);
let value_projection = Rc::new(Linear::new_with_seed_and_bias(
engine,
model_width,
model_width,
bias,
seed.wrapping_add(2),
)?);
let output_projection = Rc::new(Linear::new_with_seed_and_bias(
engine,
model_width,
model_width,
bias,
seed.wrapping_add(3),
)?);
let mut registry = ModuleRegistry::new();
registry.register_module("q_proj", query_projection.clone())?;
registry.register_module("k_proj", key_projection.clone())?;
registry.register_module("v_proj", value_projection.clone())?;
registry.register_module("out_proj", output_projection.clone())?;
Ok(Self {
model_width,
num_heads,
sequence_length: Cell::new(sequence_length),
query_projection,
key_projection,
value_projection,
output_projection,
mode: Cell::new(LinearAttentionMode::Causal),
registry,
})
}
fn geometry(&self, input: &Matrix) -> Result<usize> {
let sequence_length = self.sequence_length.get();
let [rows, width] = input.shape() else {
return Err(Error::invalid_argument(
"CausalLinearAttention expects F32 [B*S, D] input",
));
};
if input.dtype() != DType::F32
|| *width != self.model_width
|| *rows == 0
|| !rows.is_multiple_of(sequence_length)
{
return Err(Error::invalid_argument(format!(
"CausalLinearAttention expects nonempty F32 [B*S, {}] with S={}; found {:?} {}",
self.model_width,
sequence_length,
input.shape(),
input.dtype().token(),
)));
}
Ok(rows / sequence_length)
}
fn feature_map(x: &Matrix) -> Result<Matrix> {
core_matrix::add(
&core_matrix::exp(&core_matrix::clamp_max(x, 0.0)?)?,
&super::super::matrix::relu(x)?,
)
}
fn forward_causal(&self, input: &Matrix) -> Result<Matrix> {
let batch = self.geometry(input)?;
let sequence_length = self.sequence_length.get();
let head_dim = self.model_width / self.num_heads;
if head_dim <= 64 {
let q = split_heads(
&self.query_projection.forward(input)?,
batch,
sequence_length,
self.num_heads,
)?;
let k = split_heads(
&self.key_projection.forward(input)?,
batch,
sequence_length,
self.num_heads,
)?;
let v = split_heads(
&self.value_projection.forward(input)?,
batch,
sequence_length,
self.num_heads,
)?;
let context = super::super::matrix::causal_linear_attention(&q, &k, &v, 1.0e-6)?;
let context = merge_heads(&context, batch, sequence_length, self.num_heads)?;
return self.output_projection.forward(&context);
}
let phi_q = Self::feature_map(&self.query_projection.forward(input)?)?;
let phi_k = Self::feature_map(&self.key_projection.forward(input)?)?;
let phi_v = self.value_projection.forward(input)?;
let phi_q = split_heads(&phi_q, batch, sequence_length, self.num_heads)?;
let phi_k = split_heads(&phi_k, batch, sequence_length, self.num_heads)?;
let phi_v = split_heads(&phi_v, batch, sequence_length, self.num_heads)?;
let batch_heads = batch
.checked_mul(self.num_heads)
.ok_or_else(|| Error::invalid_argument("linear attention batch-head count overflows"))?;
let kv_count = batch_heads
.checked_mul(head_dim)
.and_then(|n| n.checked_mul(head_dim))
.ok_or_else(|| Error::invalid_argument("linear attention kv size overflows"))?;
let ksum_count = batch_heads
.checked_mul(head_dim)
.ok_or_else(|| Error::invalid_argument("linear attention k_sum size overflows"))?;
let kv = Matrix::allocate(
input.engine_handle(),
vec![batch_heads, head_dim, head_dim],
kv_count,
DType::F32,
)?;
kv.write_values(&vec![0.0_f32; kv_count])?;
let k_sum = Matrix::allocate(
input.engine_handle(),
vec![batch_heads, head_dim, 1],
ksum_count,
DType::F32,
)?;
k_sum.write_values(&vec![0.0_f32; ksum_count])?;
let mut kv = kv;
let mut k_sum = k_sum;
let mut steps: Vec<Matrix> = Vec::with_capacity(sequence_length);
for t in 0..sequence_length {
let t_i64 = t as i64;
let phi_q_t = core_matrix::slice(&phi_q, 1, t_i64, t_i64 + 1)?;
let phi_k_t = core_matrix::slice(&phi_k, 1, t_i64, t_i64 + 1)?;
let phi_v_t = core_matrix::slice(&phi_v, 1, t_i64, t_i64 + 1)?;
kv = core_matrix::add(&kv, &bmm_tn(&phi_k_t, &phi_v_t)?)?;
let phi_k_t_col = phi_k_t.reshape([batch_heads, head_dim, 1])?;
k_sum = core_matrix::add(&k_sum, &phi_k_t_col)?;
let ctx_t = bmm(&phi_q_t, &kv)?;
let denom_t = bmm(&phi_q_t, &k_sum)?;
steps.push(core_matrix::div(
&ctx_t,
&core_matrix::clamp_min(&denom_t, 1.0e-6)?,
)?);
}
let context = core_matrix::concat(&steps, 1)?;
let context = merge_heads(&context, batch, sequence_length, self.num_heads)?;
self.output_projection.forward(&context)
}
fn forward_bidirectional(&self, input: &Matrix) -> Result<Matrix> {
let batch = self.geometry(input)?;
let sequence_length = self.sequence_length.get();
let head_dim = self.model_width / self.num_heads;
let phi_q = Self::feature_map(&self.query_projection.forward(input)?)?;
let phi_k = Self::feature_map(&self.key_projection.forward(input)?)?;
let phi_v = self.value_projection.forward(input)?;
let phi_q = split_heads(&phi_q, batch, sequence_length, self.num_heads)?;
let phi_k = split_heads(&phi_k, batch, sequence_length, self.num_heads)?;
let phi_v = split_heads(&phi_v, batch, sequence_length, self.num_heads)?;
let batch_heads = batch
.checked_mul(self.num_heads)
.ok_or_else(|| Error::invalid_argument("linear attention batch-head count overflows"))?;
let kv = bmm_tn(&phi_k, &phi_v)?;
let context_raw = bmm(&phi_q, &kv)?;
let k_sum = core_matrix::sum(&phi_k, 1)?;
let k_sum_col = k_sum.reshape([batch_heads, head_dim, 1])?;
let denom = bmm(&phi_q, &k_sum_col)?;
let context = core_matrix::div(&context_raw, &core_matrix::clamp_min(&denom, 1.0e-6)?)?;
let context = merge_heads(&context, batch, sequence_length, self.num_heads)?;
self.output_projection.forward(&context)
}
pub fn forward(&self, input: &Matrix) -> Result<Matrix> {
match self.mode.get() {
LinearAttentionMode::Causal => self.forward_causal(input),
LinearAttentionMode::Bidirectional => self.forward_bidirectional(input),
}
}
pub fn set_mode(&self, mode: LinearAttentionMode) {
self.mode.set(mode);
}
pub fn mode(&self) -> LinearAttentionMode {
self.mode.get()
}
pub const fn model_width(&self) -> usize {
self.model_width
}
pub const fn num_heads(&self) -> usize {
self.num_heads
}
pub fn sequence_length(&self) -> usize {
self.sequence_length.get()
}
pub fn set_sequence_length(&self, sequence_length: usize) -> Result<()> {
if sequence_length == 0 {
return Err(Error::invalid_argument(
"linear attention sequence length must be nonzero",
));
}
self.sequence_length.set(sequence_length);
Ok(())
}
pub fn has_bias(&self) -> bool {
self.query_projection.has_bias()
}
}
impl Module for CausalLinearAttention {
fn forward(&self, input: &Matrix) -> Result<Matrix> {
CausalLinearAttention::forward(self, input)
}
fn registry(&self) -> &ModuleRegistry {
&self.registry
}
}