burn-msa 0.2.4

MiniMax Sparse Attention (MSA) - block-sparse attention with per-GQA-group top-k block selection for Burn
Documentation
use burn::tensor::backend::Backend;
use burn::tensor::{Int, Tensor};

pub struct KlAlignmentLoss;

impl KlAlignmentLoss {
    pub fn compute<B: Backend>(
        &self,
        block_scores: Tensor<B, 4>,
        attn_weights: Tensor<B, 4>,
        selected_indices: Tensor<B, 4, Int>,
    ) -> Tensor<B, 1> {
        let gathered = block_scores.gather(3, selected_indices);
        let smax = gathered.clone().max_dim(3);
        let shifted = gathered.sub(smax);
        let log_p = shifted
            .clone()
            .sub(shifted.exp().sum_dim(3).log().add_scalar(1e-10));

        let p_teacher = attn_weights.detach();
        let p_target = p_teacher
            .clone()
            .div(p_teacher.sum_dim(3).add_scalar(1e-10));

        p_target.mul(log_p).neg().mean()
    }
}