#[cfg(feature = "gpu")]
use crate::gpu::{GpuContext, GpuError};
use crate::kv_quant::{pack_kv, KvQuantScheme, PackedKvTensor};
use crate::layers::{Initializer, Linear};
use crate::rope::RopeScaling;
use crate::{IndexTensor, Tensor, TensorError};
pub fn try_causal_softmax(scores: &Tensor) -> Result<Tensor, TensorError> {
let shape = scores.shape();
if shape.len() < 2 {
return Err(TensorError::new(
"causal attention scores must be at least 2D",
));
}
let sequence_length = shape[shape.len() - 1];
if shape[shape.len() - 2] != sequence_length {
return Err(TensorError::new("causal scores must be square"));
}
scores.try_masked_softmax_last_dim(0)
}
pub fn causal_softmax(scores: &Tensor) -> Tensor {
try_causal_softmax(scores).unwrap_or_else(|error| panic!("{error}"))
}
#[derive(Clone)]
pub struct AttentionKvCache {
pub key: Tensor,
pub value: Tensor,
packed: Option<(PackedKvTensor, PackedKvTensor)>,
scheme: Option<KvQuantScheme>,
}
impl AttentionKvCache {
pub fn from_full(key: Tensor, value: Tensor) -> Self {
Self {
key,
value,
packed: None,
scheme: None,
}
}
pub fn from_quantized(
key: Tensor,
value: Tensor,
packed_key: PackedKvTensor,
packed_value: PackedKvTensor,
scheme: KvQuantScheme,
) -> Self {
Self {
key,
value,
packed: Some((packed_key, packed_value)),
scheme: Some(scheme),
}
}
pub fn is_quantized(&self) -> bool {
self.packed.is_some()
}
pub fn scheme(&self) -> Option<KvQuantScheme> {
self.scheme
}
pub fn storage_bytes(&self) -> usize {
if let Some((key, value)) = &self.packed {
key.storage_bytes() + value.storage_bytes()
} else {
(self.key.numel() + self.value.numel()) * std::mem::size_of::<f32>()
}
}
pub fn sequence_length(&self) -> usize {
self.key.shape()[2]
}
pub fn batch_size(&self) -> usize {
self.key.shape()[0]
}
pub fn concat_batch(caches: &[&Self]) -> Result<Self, TensorError> {
if caches.is_empty() {
return Err(TensorError::new("cannot concat an empty KV cache batch"));
}
let key = concat_batch_tensors(
&caches
.iter()
.map(|cache| &cache.key)
.collect::<Vec<_>>(),
)?;
let value = concat_batch_tensors(
&caches
.iter()
.map(|cache| &cache.value)
.collect::<Vec<_>>(),
)?;
let scheme = caches[0].scheme;
if let Some(scheme) = scheme {
if caches.iter().all(|cache| cache.scheme == Some(scheme)) {
return quantize_cache_tensors(key, value, scheme)
.map_err(|error| TensorError::new(error));
}
}
Ok(Self::from_full(key, value))
}
pub fn split_batch(&self) -> Result<Vec<Self>, TensorError> {
let keys = split_batch_tensor(&self.key)?;
let values = split_batch_tensor(&self.value)?;
if keys.len() != values.len() {
return Err(TensorError::new("KV cache batch sizes differ during split"));
}
let scheme = self.scheme;
keys.into_iter()
.zip(values)
.map(|(key, value)| {
if let Some(scheme) = scheme {
quantize_cache_tensors(key, value, scheme)
.map_err(|error| TensorError::new(error))
} else {
Ok(Self::from_full(key, value))
}
})
.collect()
}
pub fn truncate(&mut self, sequence_length: usize) -> Result<(), TensorError> {
let current = self.sequence_length();
if sequence_length > current {
return Err(TensorError::new(
"cannot truncate KV cache past its sequence length",
));
}
if sequence_length == current {
return Ok(());
}
if sequence_length == 0 {
return Err(TensorError::new(
"truncate length must be positive; clear the layer cache instead",
));
}
self.key = truncate_sequence_tensor(&self.key, sequence_length)?;
self.value = truncate_sequence_tensor(&self.value, sequence_length)?;
if let Some(scheme) = self.scheme {
*self = quantize_cache_tensors(self.key.clone(), self.value.clone(), scheme)
.map_err(|error| TensorError::new(error))?;
} else {
self.packed = None;
}
Ok(())
}
}
fn quantize_cache_tensors(
key: Tensor,
value: Tensor,
scheme: KvQuantScheme,
) -> Result<AttentionKvCache, String> {
let key_shape = key.shape();
let value_shape = value.shape();
if key_shape != value_shape || key_shape.len() != 4 {
return Err("KV tensors must share shape [B, H, S, D]".into());
}
let (batch, heads, seq, head_dim) = (key_shape[0], key_shape[1], key_shape[2], key_shape[3]);
let packed_key = pack_kv(&key.shared_data(), batch, heads, seq, head_dim, scheme)?;
let packed_value = pack_kv(&value.shared_data(), batch, heads, seq, head_dim, scheme)?;
let key_dequant = Tensor::new(packed_key.dequantize(), key_shape.clone(), false);
let value_dequant = Tensor::new(packed_value.dequantize(), value_shape, false);
Ok(AttentionKvCache::from_quantized(
key_dequant,
value_dequant,
packed_key,
packed_value,
scheme,
))
}
fn cpu_owned_tensor(tensor: &Tensor) -> Result<Tensor, TensorError> {
let cpu = tensor
.to_cpu()
.map_err(|error| TensorError::new(error.to_string()))?;
Ok(Tensor::new(cpu.data(), cpu.shape(), false))
}
fn truncate_sequence_tensor(tensor: &Tensor, sequence_length: usize) -> Result<Tensor, TensorError> {
let owned = cpu_owned_tensor(tensor)?;
let shape = owned.shape();
if shape.len() != 4 {
return Err(TensorError::new(
"KV cache tensors must have shape [B, H, S, D]",
));
}
if sequence_length > shape[2] {
return Err(TensorError::new(
"cannot truncate KV cache past its sequence length",
));
}
if sequence_length == shape[2] {
return Ok(owned);
}
let narrowed = owned.try_narrow(2, 0, sequence_length)?;
Ok(Tensor::new(narrowed.data(), narrowed.shape(), false))
}
fn concat_batch_tensors(tensors: &[&Tensor]) -> Result<Tensor, TensorError> {
if tensors.is_empty() {
return Err(TensorError::new("cannot concat an empty tensor batch"));
}
let mut owned = Vec::with_capacity(tensors.len());
for tensor in tensors {
owned.push(cpu_owned_tensor(tensor)?);
}
let reference = owned[0].shape();
if reference.len() != 4 {
return Err(TensorError::new(
"KV cache tensors must have shape [B, H, S, D]",
));
}
let mut total_batch = 0usize;
let mut data = Vec::new();
for tensor in &owned {
let shape = tensor.shape();
if shape[1..] != reference[1..] {
return Err(TensorError::new(
"KV cache concat requires matching head, sequence, and width dims",
));
}
total_batch += shape[0];
data.extend_from_slice(&tensor.shared_data());
}
Ok(Tensor::new(
data,
vec![total_batch, reference[1], reference[2], reference[3]],
false,
))
}
fn split_batch_tensor(tensor: &Tensor) -> Result<Vec<Tensor>, TensorError> {
let owned = cpu_owned_tensor(tensor)?;
let shape = owned.shape();
if shape.len() != 4 {
return Err(TensorError::new(
"KV cache tensors must have shape [B, H, S, D]",
));
}
let batch = shape[0];
let stride = shape[1] * shape[2] * shape[3];
let data = owned.shared_data();
let mut parts = Vec::with_capacity(batch);
for batch_index in 0..batch {
let start = batch_index * stride;
parts.push(Tensor::new(
data[start..start + stride].to_vec(),
vec![1, shape[1], shape[2], shape[3]],
false,
));
}
Ok(parts)
}
#[derive(Clone)]
pub struct MultiHeadAttention {
pub q_proj: Linear,
pub k_proj: Linear,
pub v_proj: Linear,
pub o_proj: Linear,
pub hidden_size: usize,
pub num_heads: usize,
pub num_key_value_heads: usize,
pub head_dim: usize,
pub rope_theta: f32,
pub rope_scaling: Option<RopeScaling>,
kv_quant: Option<KvQuantScheme>,
}
impl MultiHeadAttention {
pub fn new(hidden_size: usize, num_heads: usize, rope_theta: f32, seed: u64) -> Self {
Self::new_grouped(hidden_size, num_heads, num_heads, rope_theta, seed)
}
pub fn new_grouped(
hidden_size: usize,
num_heads: usize,
num_key_value_heads: usize,
rope_theta: f32,
seed: u64,
) -> Self {
let mut initializer = Initializer::new(seed);
Self::from_initializer(
hidden_size,
num_heads,
num_key_value_heads,
rope_theta,
None,
&mut initializer,
1.0,
false,
)
}
pub(crate) fn from_initializer(
hidden_size: usize,
num_heads: usize,
num_key_value_heads: usize,
rope_theta: f32,
rope_scaling: Option<RopeScaling>,
initializer: &mut Initializer,
residual_scale: f32,
use_bias: bool,
) -> Self {
assert!(num_heads > 0, "attention needs at least one head");
assert!(
num_key_value_heads > 0 && num_heads % num_key_value_heads == 0,
"query heads must be divisible by key/value heads"
);
assert_eq!(hidden_size % num_heads, 0, "heads must divide hidden size");
let head_dim = hidden_size / num_heads;
let key_value_size = num_key_value_heads * head_dim;
assert_eq!(head_dim % 2, 0, "RoPE head width must be even");
assert!(rope_theta > 0.0, "RoPE theta must be positive");
Self {
q_proj: Linear::from_initializer(hidden_size, hidden_size, initializer, use_bias),
k_proj: Linear::from_initializer(hidden_size, key_value_size, initializer, use_bias),
v_proj: Linear::from_initializer(hidden_size, key_value_size, initializer, use_bias),
o_proj: Linear::from_initializer_scaled(
hidden_size,
hidden_size,
initializer,
residual_scale,
use_bias,
),
hidden_size,
num_heads,
num_key_value_heads,
head_dim,
rope_theta,
rope_scaling,
kv_quant: None,
}
}
pub fn enable_kv_quant(&mut self, scheme: KvQuantScheme) {
self.kv_quant = Some(scheme);
}
pub fn clear_kv_quant(&mut self) {
self.kv_quant = None;
}
pub fn kv_quant(&self) -> Option<KvQuantScheme> {
self.kv_quant
}
pub fn forward(&self, input: &Tensor) -> Tensor {
self.forward_with_optional_attention_mask(input, None)
}
pub fn forward_with_attention_mask(
&self,
input: &Tensor,
attention_mask: &IndexTensor,
) -> Tensor {
self.forward_with_optional_attention_mask(input, Some(attention_mask))
}
pub(crate) fn forward_with_optional_attention_mask(
&self,
input: &Tensor,
attention_mask: Option<&IndexTensor>,
) -> Tensor {
self.forward_training(input, attention_mask, 0.0, 0)
}
pub(crate) fn forward_training(
&self,
input: &Tensor,
attention_mask: Option<&IndexTensor>,
dropout_probability: f32,
dropout_seed: u32,
) -> Tensor {
self.forward_training_chunked(input, attention_mask, dropout_probability, dropout_seed, 0)
}
pub(crate) fn forward_training_chunked(
&self,
input: &Tensor,
attention_mask: Option<&IndexTensor>,
dropout_probability: f32,
dropout_seed: u32,
chunk_size: usize,
) -> Tensor {
let shape = input.shape();
assert_eq!(shape.len(), 3, "attention input must have shape [B, S, D]");
let (batch_size, sequence_length, hidden_size) = (shape[0], shape[1], shape[2]);
assert_eq!(
hidden_size, self.hidden_size,
"attention input width mismatch"
);
let query = self
.split_heads(
&self.q_proj.forward(input),
batch_size,
sequence_length,
self.num_heads,
)
.rope_with_offset_and_scaling(2, self.rope_theta, 0, self.rope_scaling.as_ref());
let key = self
.split_heads(
&self.k_proj.forward(input),
batch_size,
sequence_length,
self.num_key_value_heads,
)
.rope_with_offset_and_scaling(2, self.rope_theta, 0, self.rope_scaling.as_ref());
let value = self.split_heads(
&self.v_proj.forward(input),
batch_size,
sequence_length,
self.num_key_value_heads,
);
let scale = 1.0 / (self.head_dim as f32).sqrt();
let context = if chunk_size == 0 || chunk_size >= sequence_length {
query.tiled_causal_attention_with_optional_mask(
&key,
&value,
scale,
0,
attention_mask,
dropout_probability,
dropout_seed,
)
} else {
let mut chunks = Vec::new();
let mut start = 0;
while start < sequence_length {
let length = chunk_size.min(sequence_length - start);
let query_chunk = query.narrow(2, start, length);
let chunk = query_chunk.tiled_causal_attention_with_optional_mask(
&key,
&value,
scale,
start,
attention_mask,
dropout_probability,
dropout_seed,
);
chunks.push(chunk);
start += length;
}
Tensor::cat(&chunks, 2)
};
let merged = context.permute(vec![0, 2, 1, 3]).reshape(vec![
batch_size,
sequence_length,
hidden_size,
]);
self.o_proj.forward(&merged)
}
#[cfg(feature = "gpu")]
pub fn forward_gpu(&self, input: &Tensor, gpu: &GpuContext) -> Result<Tensor, GpuError> {
self.forward_gpu_with_optional_attention_mask(input, None, gpu)
}
#[cfg(feature = "gpu")]
pub fn forward_gpu_with_attention_mask(
&self,
input: &Tensor,
attention_mask: &IndexTensor,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
self.forward_gpu_with_optional_attention_mask(input, Some(attention_mask), gpu)
}
#[cfg(feature = "gpu")]
pub(crate) fn forward_gpu_with_optional_attention_mask(
&self,
input: &Tensor,
attention_mask: Option<&IndexTensor>,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
self.forward_training_gpu(input, attention_mask, 0.0, 0, gpu)
}
#[cfg(feature = "gpu")]
pub(crate) fn forward_training_gpu(
&self,
input: &Tensor,
attention_mask: Option<&IndexTensor>,
dropout_probability: f32,
dropout_seed: u32,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
let shape = input.shape();
assert_eq!(shape.len(), 3, "attention input must have shape [B, S, D]");
let (batch_size, sequence_length, hidden_size) = (shape[0], shape[1], shape[2]);
assert_eq!(
hidden_size, self.hidden_size,
"attention input width mismatch"
);
let query = self
.split_heads_gpu(
&self.q_proj.forward_gpu(input, gpu)?,
batch_size,
sequence_length,
self.num_heads,
gpu,
)?
.gpu_rope_with_offset_and_scaling(
2,
self.rope_theta,
0,
self.rope_scaling.as_ref(),
gpu,
)?;
let key = self
.split_heads_gpu(
&self.k_proj.forward_gpu(input, gpu)?,
batch_size,
sequence_length,
self.num_key_value_heads,
gpu,
)?
.gpu_rope_with_offset_and_scaling(
2,
self.rope_theta,
0,
self.rope_scaling.as_ref(),
gpu,
)?;
let value = self.split_heads_gpu(
&self.v_proj.forward_gpu(input, gpu)?,
batch_size,
sequence_length,
self.num_key_value_heads,
gpu,
)?;
let context = gpu.tiled_causal_attention_tensor_with_optional_mask(
&query,
&key,
&value,
1.0 / (self.head_dim as f32).sqrt(),
0,
attention_mask,
dropout_probability,
dropout_seed,
)?;
let merged = context.gpu_permute(vec![0, 2, 1, 3], gpu)?.reshape(vec![
batch_size,
sequence_length,
hidden_size,
]);
self.o_proj.forward_gpu(&merged, gpu)
}
pub fn forward_cached(
&self,
input: &Tensor,
cache: &mut Option<AttentionKvCache>,
position_offset: usize,
max_sequence_length: usize,
) -> Tensor {
let shape = input.shape();
assert_eq!(shape.len(), 3, "attention input must have shape [B, S, D]");
let (batch_size, sequence_length, hidden_size) = (shape[0], shape[1], shape[2]);
assert_eq!(
hidden_size, self.hidden_size,
"attention input width mismatch"
);
let query = self
.split_heads(
&self.q_proj.forward(input),
batch_size,
sequence_length,
self.num_heads,
)
.rope_with_offset_and_scaling(
2,
self.rope_theta,
position_offset,
self.rope_scaling.as_ref(),
);
let key = self
.split_heads(
&self.k_proj.forward(input),
batch_size,
sequence_length,
self.num_key_value_heads,
)
.rope_with_offset_and_scaling(
2,
self.rope_theta,
position_offset,
self.rope_scaling.as_ref(),
);
let value = self.split_heads(
&self.v_proj.forward(input),
batch_size,
sequence_length,
self.num_key_value_heads,
);
let past_length = update_cache(
cache,
&key,
&value,
max_sequence_length,
sequence_length,
self.kv_quant,
);
let merged = self.cached_context(&query, cache.as_ref().unwrap(), past_length);
self.o_proj.forward(&merged)
}
#[cfg(feature = "gpu")]
pub fn forward_cached_gpu(
&self,
input: &Tensor,
cache: &mut Option<AttentionKvCache>,
position_offset: usize,
max_sequence_length: usize,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
let shape = input.shape();
assert_eq!(shape.len(), 3, "attention input must have shape [B, S, D]");
let (batch_size, sequence_length, hidden_size) = (shape[0], shape[1], shape[2]);
assert_eq!(
hidden_size, self.hidden_size,
"attention input width mismatch"
);
let query = self
.split_heads_gpu(
&self.q_proj.forward_gpu(input, gpu)?,
batch_size,
sequence_length,
self.num_heads,
gpu,
)?
.gpu_rope_with_offset_and_scaling(
2,
self.rope_theta,
position_offset,
self.rope_scaling.as_ref(),
gpu,
)?;
let key = self
.split_heads_gpu(
&self.k_proj.forward_gpu(input, gpu)?,
batch_size,
sequence_length,
self.num_key_value_heads,
gpu,
)?
.gpu_rope_with_offset_and_scaling(
2,
self.rope_theta,
position_offset,
self.rope_scaling.as_ref(),
gpu,
)?;
let value = self.split_heads_gpu(
&self.v_proj.forward_gpu(input, gpu)?,
batch_size,
sequence_length,
self.num_key_value_heads,
gpu,
)?;
let past_length = update_cache_gpu(
cache,
&key,
&value,
max_sequence_length,
sequence_length,
self.kv_quant,
gpu,
)?;
let merged = self.cached_context_gpu(&query, cache.as_ref().unwrap(), past_length, gpu)?;
self.o_proj.forward_gpu(&merged, gpu)
}
#[cfg(feature = "gpu")]
fn cached_context_gpu(
&self,
query: &Tensor,
cache: &AttentionKvCache,
past_length: usize,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
let query_shape = query.shape();
let batch_size = query_shape[0];
let query_length = query_shape[2];
let context = query.gpu_tiled_causal_attention(
&cache.key,
&cache.value,
1.0 / (self.head_dim as f32).sqrt(),
past_length,
gpu,
)?;
Ok(context.gpu_permute(vec![0, 2, 1, 3], gpu)?.reshape(vec![
batch_size,
query_length,
self.hidden_size,
]))
}
fn cached_context(
&self,
query: &Tensor,
cache: &AttentionKvCache,
past_length: usize,
) -> Tensor {
let query_shape = query.shape();
let batch_size = query_shape[0];
let query_length = query_shape[2];
let context = query.tiled_causal_attention(
&cache.key,
&cache.value,
1.0 / (self.head_dim as f32).sqrt(),
past_length,
);
context
.permute(vec![0, 2, 1, 3])
.reshape(vec![batch_size, query_length, self.hidden_size])
}
fn split_heads(
&self,
input: &Tensor,
batch_size: usize,
sequence_length: usize,
heads: usize,
) -> Tensor {
input
.reshape(vec![batch_size, sequence_length, heads, self.head_dim])
.permute(vec![0, 2, 1, 3])
}
#[cfg(feature = "gpu")]
fn split_heads_gpu(
&self,
input: &Tensor,
batch_size: usize,
sequence_length: usize,
heads: usize,
gpu: &GpuContext,
) -> Result<Tensor, GpuError> {
input
.reshape(vec![batch_size, sequence_length, heads, self.head_dim])
.gpu_permute(vec![0, 2, 1, 3], gpu)
}
pub fn parameters(&self) -> Vec<Tensor> {
let mut parameters = Vec::new();
parameters.extend(self.q_proj.parameters());
parameters.extend(self.k_proj.parameters());
parameters.extend(self.v_proj.parameters());
parameters.extend(self.o_proj.parameters());
parameters
}
}
fn update_cache(
cache: &mut Option<AttentionKvCache>,
key: &Tensor,
value: &Tensor,
max_sequence_length: usize,
new_sequence_length: usize,
kv_quant: Option<KvQuantScheme>,
) -> usize {
assert!(new_sequence_length <= max_sequence_length);
let updated_key = append_sequence(
cache.as_ref().map(|cached| &cached.key),
key,
max_sequence_length,
);
let updated_value = append_sequence(
cache.as_ref().map(|cached| &cached.value),
value,
max_sequence_length,
);
let total_length = updated_key.shape()[2];
*cache = Some(match kv_quant {
Some(scheme) => quantize_cache_tensors(updated_key, updated_value, scheme)
.unwrap_or_else(|error| panic!("KV quant failed: {error}")),
None => AttentionKvCache::from_full(updated_key, updated_value),
});
total_length - new_sequence_length
}
#[cfg(feature = "gpu")]
fn update_cache_gpu(
cache: &mut Option<AttentionKvCache>,
key: &Tensor,
value: &Tensor,
max_sequence_length: usize,
new_sequence_length: usize,
kv_quant: Option<KvQuantScheme>,
gpu: &GpuContext,
) -> Result<usize, GpuError> {
assert!(new_sequence_length <= max_sequence_length);
let updated_key = gpu.append_sequence_tensor(
cache.as_ref().map(|cached| &cached.key),
key,
max_sequence_length,
)?;
let updated_value = gpu.append_sequence_tensor(
cache.as_ref().map(|cached| &cached.value),
value,
max_sequence_length,
)?;
let total_length = updated_key.shape()[2];
*cache = Some(match kv_quant {
Some(scheme) => {
let key_cpu = updated_key
.to_cpu()
.map_err(|error| GpuError::Map(error.to_string()))?;
let value_cpu = updated_value
.to_cpu()
.map_err(|error| GpuError::Map(error.to_string()))?;
let quantized = quantize_cache_tensors(
Tensor::new(key_cpu.data(), key_cpu.shape(), false),
Tensor::new(value_cpu.data(), value_cpu.shape(), false),
scheme,
)
.map_err(GpuError::InvalidShape)?;
AttentionKvCache::from_quantized(
quantized.key.to_gpu(gpu)?,
quantized.value.to_gpu(gpu)?,
quantized.packed.as_ref().unwrap().0.clone(),
quantized.packed.as_ref().unwrap().1.clone(),
scheme,
)
}
None => AttentionKvCache::from_full(updated_key, updated_value),
});
Ok(total_length - new_sequence_length)
}
fn append_sequence(existing: Option<&Tensor>, addition: &Tensor, maximum: usize) -> Tensor {
let addition_shape = addition.shape();
assert_eq!(addition_shape.len(), 4);
let outer = addition_shape[0] * addition_shape[1];
let added_length = addition_shape[2];
let width = addition_shape[3];
let retained_length = existing
.map(|tensor| tensor.shape()[2].min(maximum - added_length))
.unwrap_or(0);
let mut data = Vec::with_capacity(outer * (retained_length + added_length) * width);
let addition_data = addition.shared_data();
if let Some(existing) = existing {
let existing_shape = existing.shape();
assert_eq!(existing_shape[0], addition_shape[0]);
assert_eq!(existing_shape[1], addition_shape[1]);
assert_eq!(existing_shape[3], width);
let existing_length = existing_shape[2];
let existing_data = existing.shared_data();
for outer_index in 0..outer {
let retained_start =
(outer_index * existing_length + existing_length - retained_length) * width;
data.extend_from_slice(
&existing_data[retained_start..retained_start + retained_length * width],
);
let addition_start = outer_index * added_length * width;
data.extend_from_slice(
&addition_data[addition_start..addition_start + added_length * width],
);
}
} else {
data = (*addition_data).clone();
}
Tensor::new(
data,
vec![
addition_shape[0],
addition_shape[1],
retained_length + added_length,
width,
],
false,
)
}
#[cfg(test)]
mod tests {
use super::causal_softmax;
use crate::{IndexTensor, Tensor};
#[test]
fn causal_mask_blocks_future_positions() {
let scores = Tensor::zeros(vec![1, 1, 4, 4], false);
let probabilities = causal_softmax(&scores).data();
for query in 0..4 {
for key in query + 1..4 {
assert_eq!(probabilities[query * 4 + key], 0.0);
}
let allowed = &probabilities[query * 4..=query * 4 + query];
let sum: f32 = allowed.iter().sum();
assert!((sum - 1.0).abs() < 1e-5);
}
}
#[test]
fn causal_softmax_matches_additive_mask_reference() {
let scores = Tensor::new(
vec![0.2, -0.3, 0.7, 1.1, -0.8, 0.4, 0.6, -0.2, 0.9],
vec![1, 3, 3],
false,
);
let mut mask = vec![0.0; 9];
for query in 0..3 {
for key in query + 1..3 {
mask[query * 3 + key] = f32::NEG_INFINITY;
}
}
let reference = scores
.add(&Tensor::new(mask, vec![3, 3], false))
.softmax_last_dim()
.data();
let actual = causal_softmax(&scores).data();
for (left, right) in actual.iter().zip(reference.iter()) {
assert!((left - right).abs() < 1e-6);
}
}
#[test]
fn tiled_attention_ignores_padding_keys_and_queries() {
let query = Tensor::new(
vec![1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0],
vec![2, 1, 2, 2],
false,
);
let key = Tensor::new(
vec![1.0, 0.0, 100.0, 0.0, 100.0, 0.0, 1.0, 0.0],
vec![2, 1, 2, 2],
false,
);
let value = Tensor::new(
vec![3.0, 4.0, 9.0, 9.0, 9.0, 9.0, 5.0, 6.0],
vec![2, 1, 2, 2],
false,
);
let mask = IndexTensor::new(vec![1, 0, 0, 1], vec![2, 2]);
let output = query.tiled_causal_attention_with_mask(&key, &value, 1.0, 0, &mask);
assert_eq!(output.data(), vec![3.0, 4.0, 0.0, 0.0, 0.0, 0.0, 5.0, 6.0]);
}
}