1use burn::module::Module;
2use burn::tensor::backend::Backend;
3use burn::tensor::Tensor;
4use burn_rope::RotaryEmbedding;
5
6use crate::attention::SparseAttention;
7use crate::config::MsaConfig;
8use crate::index_branch::IndexBranch;
9use crate::loss::KlAlignmentLoss;
10use crate::topk::TopKSelector;
11
12pub struct MsaOutput<B: Backend> {
13 pub output: Tensor<B, 3>,
14 pub kl_loss: Option<Tensor<B, 1>>,
15}
16
17#[derive(Module, Debug)]
18pub struct MsaModule<B: Backend> {
19 pub index_branch: IndexBranch<B>,
20 pub attention: SparseAttention<B>,
21 #[module(skip)]
22 pub rope: Option<RotaryEmbedding<B>>,
23 pub cfg: MsaConfig,
24}
25
26impl<B: Backend> MsaModule<B> {
27 pub fn new(cfg: &MsaConfig, device: &B::Device) -> Self {
28 let rope = if cfg.use_rope {
29 Some(RotaryEmbedding::new(
30 cfg.d_model,
31 cfg.n_heads_q,
32 cfg.rope_max_seq_len,
33 cfg.rope_base,
34 device,
35 ))
36 } else {
37 None
38 };
39 Self {
40 index_branch: IndexBranch::new(cfg, device),
41 attention: SparseAttention::new(cfg, device),
42 rope,
43 cfg: cfg.clone(),
44 }
45 }
46
47 pub fn forward(&self, x: Tensor<B, 3>) -> MsaOutput<B> {
48 self.forward_cross(x.clone(), x)
49 }
50
51 pub fn forward_cross(
52 &self,
53 hidden_states: Tensor<B, 3>,
54 kv_states: Tensor<B, 3>,
55 ) -> MsaOutput<B> {
56 let (output, kl_loss) = self.forward_internal(hidden_states, kv_states);
57 MsaOutput {
58 output,
59 kl_loss: if self.cfg.use_kl_loss {
60 Some(kl_loss)
61 } else {
62 None
63 },
64 }
65 }
66
67 fn forward_internal(
68 &self,
69 hidden_states: Tensor<B, 3>,
70 kv_states: Tensor<B, 3>,
71 ) -> (Tensor<B, 3>, Tensor<B, 1>) {
72 let cfg = &self.cfg;
73
74 let idx_input = if cfg.gradient_detach {
75 hidden_states.clone().detach()
76 } else {
77 hidden_states.clone()
78 };
79 let kv_idx_input = if cfg.gradient_detach {
80 kv_states.clone().detach()
81 } else {
82 kv_states.clone()
83 };
84
85 let (q_idx, k_idx) =
86 self.index_branch
87 .forward(idx_input, kv_idx_input, cfg.n_heads_kv, cfg.d_idx);
88 let scale = (cfg.d_idx as f64).sqrt();
89 let block_scores =
90 self.index_branch
91 .compute_block_scores(q_idx, k_idx, cfg.block_size, scale, cfg.causal);
92
93 let selector = TopKSelector::new(cfg.topk, cfg.block_size, cfg.force_local_block);
94 let block_indices = selector.select(block_scores.clone());
95
96 let q = self.attention.q_proj.forward(hidden_states);
97 let k = self.attention.k_proj.forward(kv_states.clone());
98 let v = self.attention.v_proj.forward(kv_states);
99 let (q, k) = if let Some(rope) = &self.rope {
100 let cos = rope.cos.clone();
101 let sin = rope.sin.clone();
102 let q = burn_rope::apply_rope_3d(q, cos.clone(), sin.clone(), cfg.n_heads_q);
103 let k = burn_rope::apply_rope_3d(k, cos, sin, cfg.n_heads_kv);
104 (q, k)
105 } else {
106 (q, k)
107 };
108 let (output, block_attn) =
109 self.attention
110 .forward_sparse_with_weights(q, k, v, block_indices.clone());
111
112 let kl_loss = if cfg.use_kl_loss {
113 KlAlignmentLoss
114 .compute(block_scores, block_attn, block_indices)
115 .mul_scalar(cfg.kl_coeff)
116 } else {
117 Tensor::zeros([1], &output.device())
118 };
119 (output, kl_loss)
120 }
121
122 pub fn forward_dense(
123 &self,
124 hidden_states: Tensor<B, 3>,
125 kv_states: Tensor<B, 3>,
126 ) -> Tensor<B, 3> {
127 let q = self.attention.q_proj.forward(hidden_states);
128 let k = self.attention.k_proj.forward(kv_states.clone());
129 let v = self.attention.v_proj.forward(kv_states);
130 self.attention.forward_dense(q, k, v)
131 }
132}