Skip to main content

burn_msa/
module.rs

1use burn::module::Module;
2use burn::tensor::backend::Backend;
3use burn::tensor::Tensor;
4use burn_rope::RotaryEmbedding;
5
6use crate::attention::SparseAttention;
7use crate::config::MsaConfig;
8use crate::index_branch::IndexBranch;
9use crate::loss::KlAlignmentLoss;
10use crate::topk::TopKSelector;
11
12pub struct MsaOutput<B: Backend> {
13    pub output: Tensor<B, 3>,
14    pub kl_loss: Option<Tensor<B, 1>>,
15}
16
17#[derive(Module, Debug)]
18pub struct MsaModule<B: Backend> {
19    pub index_branch: IndexBranch<B>,
20    pub attention: SparseAttention<B>,
21    #[module(skip)]
22    pub rope: Option<RotaryEmbedding<B>>,
23    pub cfg: MsaConfig,
24}
25
26impl<B: Backend> MsaModule<B> {
27    pub fn new(cfg: &MsaConfig, device: &B::Device) -> Self {
28        let rope = if cfg.use_rope {
29            Some(RotaryEmbedding::new(
30                cfg.d_model,
31                cfg.n_heads_q,
32                cfg.rope_max_seq_len,
33                cfg.rope_base,
34                device,
35            ))
36        } else {
37            None
38        };
39        Self {
40            index_branch: IndexBranch::new(cfg, device),
41            attention: SparseAttention::new(cfg, device),
42            rope,
43            cfg: cfg.clone(),
44        }
45    }
46
47    pub fn forward(&self, x: Tensor<B, 3>) -> MsaOutput<B> {
48        self.forward_cross(x.clone(), x)
49    }
50
51    pub fn forward_cross(
52        &self,
53        hidden_states: Tensor<B, 3>,
54        kv_states: Tensor<B, 3>,
55    ) -> MsaOutput<B> {
56        let (output, kl_loss) = self.forward_internal(hidden_states, kv_states);
57        MsaOutput {
58            output,
59            kl_loss: if self.cfg.use_kl_loss {
60                Some(kl_loss)
61            } else {
62                None
63            },
64        }
65    }
66
67    fn forward_internal(
68        &self,
69        hidden_states: Tensor<B, 3>,
70        kv_states: Tensor<B, 3>,
71    ) -> (Tensor<B, 3>, Tensor<B, 1>) {
72        let cfg = &self.cfg;
73
74        let idx_input = if cfg.gradient_detach {
75            hidden_states.clone().detach()
76        } else {
77            hidden_states.clone()
78        };
79        let kv_idx_input = if cfg.gradient_detach {
80            kv_states.clone().detach()
81        } else {
82            kv_states.clone()
83        };
84
85        let (q_idx, k_idx) =
86            self.index_branch
87                .forward(idx_input, kv_idx_input, cfg.n_heads_kv, cfg.d_idx);
88        let scale = (cfg.d_idx as f64).sqrt();
89        let block_scores =
90            self.index_branch
91                .compute_block_scores(q_idx, k_idx, cfg.block_size, scale, cfg.causal);
92
93        let selector = TopKSelector::new(cfg.topk, cfg.block_size, cfg.force_local_block);
94        let block_indices = selector.select(block_scores.clone());
95
96        let q = self.attention.q_proj.forward(hidden_states);
97        let k = self.attention.k_proj.forward(kv_states.clone());
98        let v = self.attention.v_proj.forward(kv_states);
99        let (q, k) = if let Some(rope) = &self.rope {
100            let cos = rope.cos.clone();
101            let sin = rope.sin.clone();
102            let q = burn_rope::apply_rope_3d(q, cos.clone(), sin.clone(), cfg.n_heads_q);
103            let k = burn_rope::apply_rope_3d(k, cos, sin, cfg.n_heads_kv);
104            (q, k)
105        } else {
106            (q, k)
107        };
108        let (output, block_attn) =
109            self.attention
110                .forward_sparse_with_weights(q, k, v, block_indices.clone());
111
112        let kl_loss = if cfg.use_kl_loss {
113            KlAlignmentLoss
114                .compute(block_scores, block_attn, block_indices)
115                .mul_scalar(cfg.kl_coeff)
116        } else {
117            Tensor::zeros([1], &output.device())
118        };
119        (output, kl_loss)
120    }
121
122    pub fn forward_dense(
123        &self,
124        hidden_states: Tensor<B, 3>,
125        kv_states: Tensor<B, 3>,
126    ) -> Tensor<B, 3> {
127        let q = self.attention.q_proj.forward(hidden_states);
128        let k = self.attention.k_proj.forward(kv_states.clone());
129        let v = self.attention.v_proj.forward(kv_states);
130        self.attention.forward_dense(q, k, v)
131    }
132}