burn-msa 0.2.4

MiniMax Sparse Attention (MSA) - block-sparse attention with per-GQA-group top-k block selection for Burn
Documentation
use burn::module::Module;
use burn::tensor::{Int, Tensor, TensorData};
use burn_msa::{
    IndexBranch, KlAlignmentLoss, MsaCache, MsaConfig, MsaModule, SparseAttention, TopKSelector,
};
use burn_ndarray::NdArray;
use burn_ndarray::NdArrayDevice;

type B = NdArray;
fn dvc() -> NdArrayDevice {
    NdArrayDevice::default()
}

fn cfg_small() -> MsaConfig {
    MsaConfig {
        d_model: 32,
        n_heads_q: 4,
        n_heads_kv: 2,
        d_head: 8,
        d_idx: 8,
        block_size: 4,
        topk: 2,
        force_local_block: false,
        causal: false,
        use_kl_loss: false,
        warmup_steps: 0,
        kl_coeff: 0.0,
        gradient_detach: true,
        use_rope: false,
        rope_base: 10000.0,
        rope_max_seq_len: 1024,
    }
}

#[test]
fn config_validates() {
    assert!(cfg_small().validate().is_ok());
}

#[test]
fn config_rejects_bad_dims() {
    let mut c = cfg_small();
    c.d_model = 0;
    assert!(c.validate().is_err());
    let mut c = cfg_small();
    c.n_heads_q = 3;
    assert!(c.validate().is_err());
}

#[test]
fn config_defaults() {
    let d = MsaConfig::default();
    assert_eq!(d.d_model, 1152);
    assert_eq!(d.n_heads_kv, 1);
    assert_eq!(d.topk, 16);
}

#[test]
fn index_branch_shapes() {
    let ib = IndexBranch::<B>::new(&cfg_small(), &dvc());
    let x = Tensor::<B, 3>::random(
        [2, 8, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let (q_idx, k_idx) = ib.forward(x.clone(), x, 2, 8);
    assert_eq!(q_idx.dims(), [2, 2, 8, 8]);
    assert_eq!(k_idx.dims(), [2, 1, 8, 8]);
}

#[test]
fn index_branch_block_scores() {
    let ib = IndexBranch::<B>::new(&cfg_small(), &dvc());
    let q = Tensor::<B, 3>::random(
        [1, 4, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let kv = Tensor::<B, 3>::random(
        [1, 10, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let (q_idx, k_idx) = ib.forward(q, kv, 2, 8);
    let s = ib.compute_block_scores(q_idx, k_idx, 4, 3.0f64.sqrt(), false);
    assert_eq!(s.dims(), [1, 2, 4, 3]);
}

#[test]
fn topk_shapes() {
    let sel = TopKSelector::new(3, 4, false);
    let s = Tensor::<B, 4>::random(
        [1, 2, 4, 8],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    assert_eq!(sel.select(s).dims(), [1, 2, 4, 3]);

    let s = Tensor::<B, 4>::random(
        [1, 2, 2, 5],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    assert_eq!(sel.select(s).dims(), [1, 2, 2, 3]);
}

#[test]
fn topk_force_local() {
    let sel = TopKSelector::new(3, 4, true);
    let s = Tensor::<B, 4>::random(
        [1, 1, 8, 4],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let idx = sel.select(s);
    let td = idx.into_data();
    let vals: Vec<i64> = td
        .bytes
        .chunks_exact(8)
        .map(|b| i64::from_le_bytes(b.try_into().unwrap()))
        .collect();
    for i in 0..8 {
        let local = (i / 4) as i64;
        assert!(
            vals[i * 3..i * 3 + 3].contains(&local),
            "q_pos={} missing local block {}",
            i,
            local
        );
    }
}

#[test]
fn topk_single_row() {
    let sel = TopKSelector::new(2, 4, true);
    let s = Tensor::<B, 4>::random(
        [1, 1, 1, 4],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let _ = sel.select(s);
}

#[test]
fn dense_forward() {
    let attn = SparseAttention::<B>::new(&cfg_small(), &dvc());
    let x = Tensor::<B, 3>::random(
        [1, 4, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let (q, k, v) = {
        let qp = attn.q_proj.forward(x.clone());
        let kp = attn.k_proj.forward(x.clone());
        let vp = attn.v_proj.forward(x);
        (qp, kp, vp)
    };
    assert_eq!(attn.forward_dense(q, k, v).dims(), [1, 4, 32]);
}

#[test]
fn sparse_forward() {
    let attn = SparseAttention::<B>::new(&cfg_small(), &dvc());
    let q = Tensor::<B, 3>::random(
        [1, 4, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let kv = Tensor::<B, 3>::random(
        [1, 8, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let (qp, kp, vp) = {
        let qp = attn.q_proj.forward(q);
        let kp = attn.k_proj.forward(kv.clone());
        let vp = attn.v_proj.forward(kv);
        (qp, kp, vp)
    };
    let idx = Tensor::<B, 4, Int>::from_data(
        TensorData::new(
            vec![0i64, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1],
            [1, 2, 4, 2],
        ),
        &dvc(),
    );
    assert_eq!(attn.forward_sparse(qp, kp, vp, idx).dims(), [1, 4, 32]);
}

#[test]
fn module_self_attn() {
    let m = MsaModule::<B>::new(&cfg_small(), &dvc());
    let x = Tensor::<B, 3>::random(
        [1, 4, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let result = m.forward(x);
    assert_eq!(result.output.dims(), [1, 4, 32]);
    assert!(result.kl_loss.is_none());
}

#[test]
fn module_cross_attn() {
    let m = MsaModule::<B>::new(&cfg_small(), &dvc());
    let q = Tensor::<B, 3>::random(
        [1, 4, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let kv = Tensor::<B, 3>::random(
        [1, 8, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let result = m.forward_cross(q, kv);
    assert_eq!(result.output.dims(), [1, 4, 32]);
}

#[test]
fn module_dense_forward() {
    let m = MsaModule::<B>::new(&cfg_small(), &dvc());
    let x = Tensor::<B, 3>::random(
        [1, 4, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    assert_eq!(m.forward_dense(x.clone(), x).dims(), [1, 4, 32]);
}

#[test]
fn no_nan_sparse() {
    let m = MsaModule::<B>::new(&cfg_small(), &dvc());
    let x = Tensor::<B, 3>::random(
        [1, 6, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let td = m.forward(x).output.into_data();
    for chunk in td.bytes.chunks_exact(4) {
        assert!(f32::from_le_bytes(chunk.try_into().unwrap()).is_finite());
    }
}

#[test]
fn no_nan_dense() {
    let m = MsaModule::<B>::new(&cfg_small(), &dvc());
    let x = Tensor::<B, 3>::random(
        [1, 6, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let td = m.forward_dense(x.clone(), x).into_data();
    for chunk in td.bytes.chunks_exact(4) {
        assert!(f32::from_le_bytes(chunk.try_into().unwrap()).is_finite());
    }
}

#[test]
fn kl_loss() {
    let loss = KlAlignmentLoss;
    let bs = Tensor::<B, 4>::random(
        [1, 2, 4, 4],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let aw = Tensor::<B, 4>::random(
        [1, 2, 4, 2],
        burn::tensor::Distribution::Uniform(0.0, 1.0),
        &dvc(),
    );
    let si = Tensor::<B, 4, Int>::from_data(
        TensorData::new(
            vec![0i64, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1],
            [1, 2, 4, 2],
        ),
        &dvc(),
    );
    assert_eq!(loss.compute(bs, aw, si).dims(), [1]);
}

#[test]
fn cache_ops() {
    let k = Tensor::<B, 4>::ones([1, 2, 4, 8], &dvc());
    let v = Tensor::<B, 4>::ones([1, 2, 4, 8], &dvc());
    let ki = Tensor::<B, 4>::ones([1, 1, 4, 8], &dvc());
    let cache = MsaCache::new(k, v, ki);
    assert_eq!(cache.k_cache.dims(), [1, 2, 4, 8]);

    let new_k = Tensor::<B, 4>::ones([1, 2, 1, 8], &dvc());
    let new_v = Tensor::<B, 4>::ones([1, 2, 1, 8], &dvc());
    let new_ki = Tensor::<B, 4>::ones([1, 1, 1, 8], &dvc());
    let cache = cache.update(new_k, new_v, new_ki);
    assert_eq!(cache.k_cache.dims(), [1, 2, 5, 8]);
}

#[test]
fn edge_single_token() {
    let m = MsaModule::<B>::new(&cfg_small(), &dvc());
    let x = Tensor::<B, 3>::random(
        [1, 1, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let td = m.forward(x).output.into_data();
    assert!(td.bytes.iter().any(|&b| b != 0));
}

#[test]
fn edge_kv_longer() {
    let m = MsaModule::<B>::new(&cfg_small(), &dvc());
    let q = Tensor::<B, 3>::random(
        [1, 2, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let kv = Tensor::<B, 3>::random(
        [1, 16, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    assert_eq!(m.forward_cross(q, kv).output.dims(), [1, 2, 32]);
}

#[test]
fn edge_q_longer() {
    let m = MsaModule::<B>::new(&cfg_small(), &dvc());
    let q = Tensor::<B, 3>::random(
        [1, 16, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let kv = Tensor::<B, 3>::random(
        [1, 2, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    assert_eq!(m.forward_cross(q, kv).output.dims(), [1, 16, 32]);
}

#[test]
fn edge_multi_batch() {
    let c = cfg_small();
    let m = MsaModule::<B>::new(&c, &dvc());
    let q = Tensor::<B, 3>::random(
        [3, 8, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    let kv = Tensor::<B, 3>::random(
        [3, 12, 32],
        burn::tensor::Distribution::Normal(0.0, 1.0),
        &dvc(),
    );
    assert_eq!(m.forward_cross(q, kv).output.dims(), [3, 8, 32]);
}

#[test]
fn serialization_roundtrip() {
    let c = cfg_small();
    let m1 = MsaModule::<B>::new(&c, &dvc());
    let rec = m1.into_record();
    let _ = MsaModule::<B>::new(&c, &dvc()).load_record(rec);
}