burn-msa 0.2.4

MiniMax Sparse Attention (MSA) - block-sparse attention with per-GQA-group top-k block selection for Burn
Documentation
use burn::module::Module;
use burn::nn::{Initializer, Linear, LinearConfig};
use burn::tensor::backend::Backend;
use burn::tensor::{ops::PadMode, Int, Tensor};

use crate::config::MsaConfig;

#[derive(Module, Debug)]
pub struct IndexBranch<B: Backend> {
    pub q_proj: Linear<B>,
    pub k_proj: Linear<B>,
}

impl<B: Backend> IndexBranch<B> {
    pub fn new(cfg: &MsaConfig, device: &B::Device) -> Self {
        Self {
            q_proj: LinearConfig::new(cfg.d_model, cfg.n_heads_kv * cfg.d_idx)
                .with_bias(false)
                .with_initializer(Initializer::XavierUniform { gain: 1.0 })
                .init(device),
            k_proj: LinearConfig::new(cfg.d_model, cfg.d_idx)
                .with_bias(false)
                .with_initializer(Initializer::XavierUniform { gain: 1.0 })
                .init(device),
        }
    }

    pub fn forward(
        &self,
        hidden_states: Tensor<B, 3>,
        kv_states: Tensor<B, 3>,
        n_heads_kv: usize,
        d_idx: usize,
    ) -> (Tensor<B, 4>, Tensor<B, 4>) {
        let q_3d = self.q_proj.forward(hidden_states);
        let k_3d = self.k_proj.forward(kv_states);

        let [batch, seq_q, _] = q_3d.dims();
        let [_, _seq_kv, _] = k_3d.dims();

        let q_idx = q_3d
            .reshape::<4, _>([batch, seq_q, n_heads_kv, d_idx])
            .swap_dims(1, 2);

        let k_idx = k_3d.unsqueeze_dims(&[1]);

        (q_idx, k_idx)
    }

    pub fn compute_block_scores(
        &self,
        q_idx: Tensor<B, 4>,
        k_idx: Tensor<B, 4>,
        block_size: usize,
        scale: f64,
        causal: bool,
    ) -> Tensor<B, 4> {
        let [batch, h_kv, seq_q, _] = q_idx.dims();
        let seq_kv = k_idx.dims()[2];

        let k_t = k_idx.swap_dims(2, 3);
        let mut scores = q_idx.matmul(k_t).div_scalar(scale);

        if causal && seq_q > 1 && seq_kv > 1 {
            let device = scores.device();
            let q_pos = Tensor::<B, 1, Int>::arange(0..seq_q as i64, &device)
                .reshape::<4, _>([1, 1, seq_q, 1]);
            let kv_pos = Tensor::<B, 1, Int>::arange(0..seq_kv as i64, &device)
                .reshape::<4, _>([1, 1, 1, seq_kv]);
            scores = scores.add(
                kv_pos
                    .greater(q_pos)
                    .float()
                    .mul_scalar(crate::NEG_INF_SAFE),
            );
        }

        let n_blocks = seq_kv.div_ceil(block_size);
        let pad_len = (block_size - seq_kv % block_size) % block_size;
        if pad_len > 0 {
            scores = scores.pad(
                [(0, 0), (0, 0), (0, 0), (0, pad_len)],
                PadMode::Constant(f32::NEG_INFINITY),
            );
        }

        scores
            .reshape::<5, _>([batch, h_kv, seq_q, n_blocks, block_size])
            .max_dim(4)
            .squeeze_dim::<4>(4)
    }
}