use burn::module::Module;
use burn::nn::{Initializer, Linear, LinearConfig};
use burn::tensor::backend::Backend;
use burn::tensor::{ops::PadMode, Int, Tensor};
use crate::config::MsaConfig;
#[derive(Module, Debug)]
pub struct IndexBranch<B: Backend> {
pub q_proj: Linear<B>,
pub k_proj: Linear<B>,
}
impl<B: Backend> IndexBranch<B> {
pub fn new(cfg: &MsaConfig, device: &B::Device) -> Self {
Self {
q_proj: LinearConfig::new(cfg.d_model, cfg.n_heads_kv * cfg.d_idx)
.with_bias(false)
.with_initializer(Initializer::XavierUniform { gain: 1.0 })
.init(device),
k_proj: LinearConfig::new(cfg.d_model, cfg.d_idx)
.with_bias(false)
.with_initializer(Initializer::XavierUniform { gain: 1.0 })
.init(device),
}
}
pub fn forward(
&self,
hidden_states: Tensor<B, 3>,
kv_states: Tensor<B, 3>,
n_heads_kv: usize,
d_idx: usize,
) -> (Tensor<B, 4>, Tensor<B, 4>) {
let q_3d = self.q_proj.forward(hidden_states);
let k_3d = self.k_proj.forward(kv_states);
let [batch, seq_q, _] = q_3d.dims();
let [_, _seq_kv, _] = k_3d.dims();
let q_idx = q_3d
.reshape::<4, _>([batch, seq_q, n_heads_kv, d_idx])
.swap_dims(1, 2);
let k_idx = k_3d.unsqueeze_dims(&[1]);
(q_idx, k_idx)
}
pub fn compute_block_scores(
&self,
q_idx: Tensor<B, 4>,
k_idx: Tensor<B, 4>,
block_size: usize,
scale: f64,
causal: bool,
) -> Tensor<B, 4> {
let [batch, h_kv, seq_q, _] = q_idx.dims();
let seq_kv = k_idx.dims()[2];
let k_t = k_idx.swap_dims(2, 3);
let mut scores = q_idx.matmul(k_t).div_scalar(scale);
if causal && seq_q > 1 && seq_kv > 1 {
let device = scores.device();
let q_pos = Tensor::<B, 1, Int>::arange(0..seq_q as i64, &device)
.reshape::<4, _>([1, 1, seq_q, 1]);
let kv_pos = Tensor::<B, 1, Int>::arange(0..seq_kv as i64, &device)
.reshape::<4, _>([1, 1, 1, seq_kv]);
scores = scores.add(
kv_pos
.greater(q_pos)
.float()
.mul_scalar(crate::NEG_INF_SAFE),
);
}
let n_blocks = seq_kv.div_ceil(block_size);
let pad_len = (block_size - seq_kv % block_size) % block_size;
if pad_len > 0 {
scores = scores.pad(
[(0, 0), (0, 0), (0, 0), (0, pad_len)],
PadMode::Constant(f32::NEG_INFINITY),
);
}
scores
.reshape::<5, _>([batch, h_kv, seq_q, n_blocks, block_size])
.max_dim(4)
.squeeze_dim::<4>(4)
}
}