use burn::module::Module;
use burn::tensor::backend::Backend;
use burn::tensor::Tensor;
use burn_rope::RotaryEmbedding;
use crate::attention::SparseAttention;
use crate::config::MsaConfig;
use crate::index_branch::IndexBranch;
use crate::loss::KlAlignmentLoss;
use crate::topk::TopKSelector;
pub struct MsaOutput<B: Backend> {
pub output: Tensor<B, 3>,
pub kl_loss: Option<Tensor<B, 1>>,
}
#[derive(Module, Debug)]
pub struct MsaModule<B: Backend> {
pub index_branch: IndexBranch<B>,
pub attention: SparseAttention<B>,
#[module(skip)]
pub rope: Option<RotaryEmbedding<B>>,
pub cfg: MsaConfig,
}
impl<B: Backend> MsaModule<B> {
pub fn new(cfg: &MsaConfig, device: &B::Device) -> Self {
let rope = if cfg.use_rope {
Some(RotaryEmbedding::new(
cfg.d_model,
cfg.n_heads_q,
cfg.rope_max_seq_len,
cfg.rope_base,
device,
))
} else {
None
};
Self {
index_branch: IndexBranch::new(cfg, device),
attention: SparseAttention::new(cfg, device),
rope,
cfg: cfg.clone(),
}
}
pub fn forward(&self, x: Tensor<B, 3>) -> MsaOutput<B> {
self.forward_cross(x.clone(), x)
}
pub fn forward_cross(
&self,
hidden_states: Tensor<B, 3>,
kv_states: Tensor<B, 3>,
) -> MsaOutput<B> {
let (output, kl_loss) = self.forward_internal(hidden_states, kv_states);
MsaOutput {
output,
kl_loss: if self.cfg.use_kl_loss {
Some(kl_loss)
} else {
None
},
}
}
fn forward_internal(
&self,
hidden_states: Tensor<B, 3>,
kv_states: Tensor<B, 3>,
) -> (Tensor<B, 3>, Tensor<B, 1>) {
let cfg = &self.cfg;
let idx_input = if cfg.gradient_detach {
hidden_states.clone().detach()
} else {
hidden_states.clone()
};
let kv_idx_input = if cfg.gradient_detach {
kv_states.clone().detach()
} else {
kv_states.clone()
};
let (q_idx, k_idx) =
self.index_branch
.forward(idx_input, kv_idx_input, cfg.n_heads_kv, cfg.d_idx);
let scale = (cfg.d_idx as f64).sqrt();
let block_scores =
self.index_branch
.compute_block_scores(q_idx, k_idx, cfg.block_size, scale, cfg.causal);
let selector = TopKSelector::new(cfg.topk, cfg.block_size, cfg.force_local_block);
let block_indices = selector.select(block_scores.clone());
let q = self.attention.q_proj.forward(hidden_states);
let k = self.attention.k_proj.forward(kv_states.clone());
let v = self.attention.v_proj.forward(kv_states);
let (q, k) = if let Some(rope) = &self.rope {
let cos = rope.cos.clone();
let sin = rope.sin.clone();
let q = burn_rope::apply_rope_3d(q, cos.clone(), sin.clone(), cfg.n_heads_q);
let k = burn_rope::apply_rope_3d(k, cos, sin, cfg.n_heads_kv);
(q, k)
} else {
(q, k)
};
let (output, block_attn) =
self.attention
.forward_sparse_with_weights(q, k, v, block_indices.clone());
let kl_loss = if cfg.use_kl_loss {
KlAlignmentLoss
.compute(block_scores, block_attn, block_indices)
.mul_scalar(cfg.kl_coeff)
} else {
Tensor::zeros([1], &output.device())
};
(output, kl_loss)
}
pub fn forward_dense(
&self,
hidden_states: Tensor<B, 3>,
kv_states: Tensor<B, 3>,
) -> Tensor<B, 3> {
let q = self.attention.q_proj.forward(hidden_states);
let k = self.attention.k_proj.forward(kv_states.clone());
let v = self.attention.v_proj.forward(kv_states);
self.attention.forward_dense(q, k, v)
}
}