use candle_core::{Result, Tensor, D};
pub const ARCH: &str = "kimi-k3";
pub const DEFAULT_ATTN_RES_BLOCK: usize = 12;
pub const SITU_BETA: f32 = 4.0;
pub const SITU_LINEAR_BETA: f32 = 25.0;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LayerKind {
Kda,
Mla,
}
pub fn layer_kind(il: usize, n_layer: usize) -> LayerKind {
if il + 1 == n_layer || il % 4 == 3 {
LayerKind::Mla
} else {
LayerKind::Kda
}
}
pub fn layer_kinds_from_head_count_kv(head_count_kv: &[u32]) -> Vec<LayerKind> {
head_count_kv
.iter()
.map(|&n| {
if n == 0 {
LayerKind::Kda
} else {
LayerKind::Mla
}
})
.collect()
}
pub fn situ(gate: &Tensor, up: &Tensor, beta: f32, linear_beta: f32) -> Result<Tensor> {
let b = beta as f64;
let soft_gate = ((gate / b)?.tanh()? * b)?;
let gated = (soft_gate * candle_nn::ops::sigmoid(gate)?)?;
if linear_beta <= 0.0 {
return gated * up;
}
let lb = linear_beta as f64;
let soft_up = ((up / lb)?.tanh()? * lb)?;
gated * soft_up
}
pub fn kda_gate(a: &Tensor, a_log_exp: &Tensor, lower_bound: f32) -> Result<Tensor> {
let dims = a.dims();
let n_head = dims[dims.len() - 2];
let scaled = a.broadcast_mul(&a_log_exp.reshape((n_head, 1))?)?;
candle_nn::ops::sigmoid(&scaled)? * lower_bound as f64
}
#[allow(clippy::needless_range_loop, clippy::too_many_arguments)]
pub fn kda_recurrent_head(
q: &[f32],
k: &[f32],
v: &[f32],
g: &[f32],
beta: &[f32],
state: &mut [f32],
seq_len: usize,
key_dim: usize,
value_dim: usize,
) -> Vec<f32> {
let mut out = vec![0f32; seq_len * value_dim];
let mut delta = vec![0f32; value_dim];
for t in 0..seq_len {
let (qt, kt) = (&q[t * key_dim..], &k[t * key_dim..]);
let vt = &v[t * value_dim..];
let gt = &g[t * key_dim..];
for i in 0..key_dim {
let decay = gt[i].exp();
let row = &mut state[i * value_dim..(i + 1) * value_dim];
for s in row.iter_mut() {
*s *= decay;
}
}
for j in 0..value_dim {
let mut acc = 0f32;
for i in 0..key_dim {
acc += kt[i] * state[i * value_dim + j];
}
delta[j] = vt[j] - acc;
}
let b = beta[t];
for i in 0..key_dim {
let bk = b * kt[i];
if bk == 0.0 {
continue;
}
let row = &mut state[i * value_dim..(i + 1) * value_dim];
for (j, s) in row.iter_mut().enumerate() {
*s += bk * delta[j];
}
}
for j in 0..value_dim {
let mut acc = 0f32;
for i in 0..key_dim {
acc += qt[i] * state[i * value_dim + j];
}
out[t * value_dim + j] = acc;
}
}
out
}
pub fn attn_res_mix(candidates: &Tensor, score: &Tensor, eps: f64) -> Result<Tensor> {
let raw = candidates.to_dtype(candle_core::DType::F32)?;
let ms = raw.sqr()?.mean_keepdim(D::Minus1)?;
let normed = raw.broadcast_div(&(ms + eps)?.sqrt()?)?;
let scores = normed
.broadcast_mul(&score.to_dtype(candle_core::DType::F32)?)?
.sum(D::Minus1)?;
let probs = candle_nn::ops::softmax_last_dim(&scores)?;
raw.broadcast_mul(&probs.unsqueeze(D::Minus1)?)?.sum(0)
}
pub fn checkpoint_layers(n_layer: usize, block: usize) -> Vec<usize> {
if block == 0 {
return Vec::new();
}
(0..n_layer).filter(|il| il % block == 0).collect()
}
pub fn route_experts(
logits: &Tensor,
bias: Option<&Tensor>,
n_used: usize,
renormalize: bool,
scale: f64,
) -> Result<(Tensor, Tensor)> {
let scores = candle_nn::ops::sigmoid(&logits.to_dtype(candle_core::DType::F32)?)?;
let selection = match bias {
Some(b) => scores.broadcast_add(&b.reshape((1, ()))?)?,
None => scores.clone(),
};
let idx = selection
.arg_sort_last_dim(false)?
.narrow(D::Minus1, 0, n_used)?
.contiguous()?;
let mut w = scores.gather(&idx, D::Minus1)?;
if renormalize {
let denom = (w.sum_keepdim(D::Minus1)? + 1e-20)?;
w = w.broadcast_div(&denom)?;
}
if scale != 1.0 {
w = (w * scale)?;
}
Ok((idx, w))
}
#[cfg(test)]
mod tests {
use super::*;
use candle_core::Device;
#[test]
fn layer_topology_matches_the_reference_pattern() {
let n = 93;
let mla: Vec<usize> = (0..n)
.filter(|&il| layer_kind(il, n) == LayerKind::Mla)
.collect();
assert_eq!(mla.len(), 24, "24 full-attention layers");
assert_eq!(&mla[..3], &[3, 7, 11]);
assert_eq!(&mla[mla.len() - 2..], &[91, 92]);
assert_eq!(
(0..n).filter(|&il| layer_kind(il, n) == LayerKind::Kda).count(),
69,
"69 KDA layers"
);
assert_eq!(layer_kind(0, n), LayerKind::Kda);
}
#[test]
fn head_count_kv_array_drives_the_topology() {
let kinds = layer_kinds_from_head_count_kv(&[0, 0, 0, 1, 0, 0, 0, 1]);
assert_eq!(kinds[0], LayerKind::Kda);
assert_eq!(kinds[3], LayerKind::Mla);
assert_eq!(kinds[7], LayerKind::Mla);
for (il, kind) in kinds.iter().enumerate() {
assert_eq!(*kind, layer_kind(il, 93), "layer {il}");
}
}
#[test]
fn situ_matches_its_definition() {
let dev = Device::Cpu;
let gate = Tensor::new(&[0.0f32, 1.0, -2.0, 8.0], &dev).unwrap();
let up = Tensor::new(&[1.0f32, -1.0, 2.0, 50.0], &dev).unwrap();
let got: Vec<f32> = situ(&gate, &up, SITU_BETA, SITU_LINEAR_BETA)
.unwrap()
.to_vec1()
.unwrap();
let (b, lb) = (SITU_BETA, SITU_LINEAR_BETA);
for (i, (g, u)) in [(0.0f32, 1.0f32), (1.0, -1.0), (-2.0, 2.0), (8.0, 50.0)]
.iter()
.enumerate()
{
let expect = b * (g / b).tanh() * (1.0 / (1.0 + (-g).exp())) * (lb * (u / lb).tanh());
assert!(
(got[i] - expect).abs() < 1e-5,
"situ[{i}] = {}, expected {expect}",
got[i]
);
}
assert!(got[3].abs() < (SITU_BETA * SITU_LINEAR_BETA) + 1.0);
}
#[test]
fn kda_gate_is_negative_and_respects_its_lower_bound() {
let dev = Device::Cpu;
let a = Tensor::new(&[[-50.0f32, 0.0, 50.0], [-50.0, 0.0, 50.0]], &dev).unwrap();
let a_log_exp = Tensor::new(&[1.0f32, 1.0], &dev).unwrap();
let g = kda_gate(&a, &a_log_exp, -5.0).unwrap();
let v: Vec<f32> = g.flatten_all().unwrap().to_vec1().unwrap();
for x in &v {
assert!(*x <= 0.0 && *x >= -5.0, "gate {x} outside (lower_bound, 0]");
}
assert!((v[2] - -5.0).abs() < 1e-3);
assert!(v[0].abs() < 1e-3);
assert!((v[1] - -2.5).abs() < 1e-5);
}
#[test]
fn kda_gate_uses_a_per_head_scalar_across_channels() {
let dev = Device::Cpu;
let a = Tensor::new(&[[1.0f32, 1.0], [1.0, 1.0]], &dev).unwrap();
let a_log_exp = Tensor::new(&[0.1f32, 10.0], &dev).unwrap();
let v: Vec<f32> = kda_gate(&a, &a_log_exp, -5.0)
.unwrap()
.flatten_all()
.unwrap()
.to_vec1()
.unwrap();
assert_eq!(v[0], v[1], "channels within a head share the scalar");
assert_eq!(v[2], v[3]);
assert!(v[3] < v[1], "a larger exp(A_log) decays harder");
}
#[test]
fn delta_rule_writes_a_value_then_recalls_it() {
let (kd, vd) = (4usize, 3usize);
let k = vec![1.0f32, 0.0, 0.0, 0.0];
let q = k.clone();
let v = vec![0.5f32, -1.0, 2.0];
let g = vec![0.0f32; kd]; let beta = vec![1.0f32];
let mut state = vec![0f32; kd * vd];
let out = kda_recurrent_head(&q, &k, &v, &g, &beta, &mut state, 1, kd, vd);
for j in 0..vd {
assert!((out[j] - v[j]).abs() < 1e-6, "recall {j}: {} vs {}", out[j], v[j]);
}
}
#[test]
fn delta_rule_decay_forgets_previous_writes() {
let (kd, vd) = (2usize, 1usize);
let k = vec![1.0f32, 0.0, 1.0, 0.0];
let q = k.clone();
let v = vec![1.0f32, 0.0];
let g = vec![0.0f32, 0.0, -2.0, -2.0];
let beta = vec![1.0f32, 0.0];
let mut state = vec![0f32; kd * vd];
let out = kda_recurrent_head(&q, &k, &v, &g, &beta, &mut state, 2, kd, vd);
assert!((out[0] - 1.0).abs() < 1e-6);
let expect = (-2.0f32).exp();
assert!(
(out[1] - expect).abs() < 1e-6,
"decayed recall {} vs {expect}",
out[1]
);
}
#[test]
fn attn_res_mix_scores_normalised_but_averages_raw() {
let dev = Device::Cpu;
let candidates = Tensor::new(&[[1.0f32, 0.0], [100.0, 0.0]], &dev).unwrap();
let score = Tensor::new(&[1.0f32, 0.0], &dev).unwrap();
let out: Vec<f32> = attn_res_mix(&candidates, &score, 1e-6)
.unwrap()
.to_vec1()
.unwrap();
assert!(
(out[0] - 50.5).abs() < 1e-3,
"expected the raw mean 50.5, got {}",
out[0]
);
}
#[test]
fn routing_biases_selection_but_not_weights() {
let dev = Device::Cpu;
let logits = Tensor::new(&[[1.0f32, 0.5, -5.0, -6.0]], &dev).unwrap();
let bias = Tensor::new(&[0.0f32, 10.0, 0.0, 0.0], &dev).unwrap();
let (idx, w) = route_experts(&logits, Some(&bias), 1, false, 1.0).unwrap();
let idx: Vec<u32> = idx.flatten_all().unwrap().to_vec1().unwrap();
let w: Vec<f32> = w.flatten_all().unwrap().to_vec1().unwrap();
assert_eq!(idx[0], 1, "bias must steer selection");
let unbiased = 1.0f32 / (1.0 + (-0.5f32).exp());
assert!(
(w[0] - unbiased).abs() < 1e-6,
"weight {} must be the unbiased score {unbiased}",
w[0]
);
}
#[test]
fn routing_renormalises_selected_weights() {
let dev = Device::Cpu;
let logits = Tensor::new(&[[2.0f32, 1.0, 0.0, -1.0]], &dev).unwrap();
let (_, w) = route_experts(&logits, None, 2, true, 1.0).unwrap();
let w: Vec<f32> = w.flatten_all().unwrap().to_vec1().unwrap();
let sum: f32 = w.iter().sum();
assert!((sum - 1.0).abs() < 1e-6, "renormalised weights sum to {sum}");
assert!(w[0] > w[1], "the stronger expert keeps the larger share");
}
#[test]
fn checkpoints_land_every_block_layers() {
let cps = checkpoint_layers(93, DEFAULT_ATTN_RES_BLOCK);
assert_eq!(cps, vec![0, 12, 24, 36, 48, 60, 72, 84]);
assert_eq!(cps.len(), 8);
assert!(checkpoint_layers(93, 0).is_empty());
}
}