use std::ffi::c_int;
use std::sync::atomic::{AtomicBool, Ordering};
use anyhow::{anyhow, bail, Result};
use super::graph_nodes;
use super::model::{CanaryModel, EncBlock};
use crate::mediagen::backend::{Backend, Graph};
use crate::mediagen::ffi::{ty, Tensor};
const BN_EPS: f32 = 1e-5;
const LN_EPS: f32 = 1e-5;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Probe {
PreEncode,
PosEmb,
Block(usize),
}
pub struct Run {
pub out: Vec<f32>,
pub frames: usize,
pub probes: Vec<Vec<f32>>,
}
pub struct Encoder {
d_model: usize,
n_heads: usize,
n_mels: usize,
bn: Vec<(Vec<f32>, Vec<f32>)>,
logged: AtomicBool,
}
impl Encoder {
pub fn new(model: &CanaryModel, be: &Backend) -> Result<Encoder> {
let hp = model.hp();
if hp.enc_subsampling_factor != 8 {
bail!(
"encoder: subsampling factor {} (the dw_striding graph is 8x)",
hp.enc_subsampling_factor
);
}
if hp.frontend.num_mels != 128 {
bail!("encoder: {} mel bins (expected 128)", hp.frontend.num_mels);
}
if !hp.enc_d_model.is_multiple_of(hp.enc_n_heads) {
bail!(
"encoder: d_model {} is not a multiple of {} heads",
hp.enc_d_model,
hp.enc_n_heads
);
}
check_conv_kernel(hp.enc_conv_kernel)?;
if !hp.enc_d_model.is_multiple_of(2) {
bail!(
"encoder: d_model {} is odd; the interleaved position table needs an even width",
hp.enc_d_model
);
}
let bn = model
.tensors()?
.enc_blocks
.iter()
.map(|b| {
fold_batch_norm(
&be.read_f32(b.bn),
&be.read_f32(b.bn_b),
&be.read_f32(b.bn_mean),
&be.read_f32(b.bn_var),
BN_EPS,
)
})
.collect();
Ok(Encoder {
d_model: hp.enc_d_model as usize,
n_heads: hp.enc_n_heads as usize,
n_mels: hp.frontend.num_mels as usize,
bn,
logged: AtomicBool::new(false),
})
}
pub fn run(
&self,
model: &CanaryModel,
be: &Backend,
mel: &[f32],
probes: &[Probe],
) -> Result<Run> {
if mel.is_empty() || !mel.len().is_multiple_of(self.n_mels) {
bail!(
"encoder: {} mel values are not a whole number of {}-bin frames",
mel.len(),
self.n_mels
);
}
let frames = mel.len() / self.n_mels;
let t_enc = subsampled_len(frames);
let tensors = model.tensors()?;
let pre = &tensors.pre_encode;
let gr = Graph::new(be.api, graph_nodes(tensors.enc_blocks.len()))?;
let x = gr.input_f32(&[self.n_mels as i64, frames as i64, 1, 1], mel, "mel");
let pos = gr.input_f32(
&[self.d_model as i64, 2 * t_enc as i64 - 1],
&pos_table(t_enc, self.d_model),
"pos_emb",
);
let bn: Vec<(Tensor, Tensor)> = self
.bn
.iter()
.enumerate()
.map(|(i, (scale, shift))| {
let n = &[self.d_model as i64];
(
gr.input_f32(n, scale, &format!("bn_scale.{i}")),
gr.input_f32(n, shift, &format!("bn_shift.{i}")),
)
})
.collect();
let g = &gr;
let bias = |b| g.reshape_4d(b, 1, 1, b.shape()[0], 1);
let mut cur = g.conv_2d(pre.conv0, x, 2, 2, 1, 1, 1, 1);
cur = g.relu(g.add(cur, bias(pre.conv0_b)));
cur = g.conv_2d_dw_direct(pre.conv2, cur, 2, 2, 1, 1, 1, 1);
cur = g.add(cur, bias(pre.conv2_b));
cur = g.conv_2d(pre.conv3, cur, 1, 1, 0, 0, 1, 1);
cur = g.relu(g.add(cur, bias(pre.conv3_b)));
cur = g.conv_2d_dw_direct(pre.conv5, cur, 2, 2, 1, 1, 1, 1);
cur = g.add(cur, bias(pre.conv5_b));
cur = g.conv_2d(pre.conv6, cur, 1, 1, 0, 0, 1, 1);
cur = g.relu(g.add(cur, bias(pre.conv6_b)));
cur = g.cont(g.permute(cur, 0, 2, 1, 3));
let [n_freq, n_chan, n_time, _] = cur.shape();
let in_dim = pre.out.shape()[0];
if n_time != t_enc as i64 || n_freq * n_chan != in_dim {
bail!(
"encoder: subsampler produced [{n_freq}, {n_chan}, {n_time}] for {frames} frames, \
expected {in_dim} values per frame over {t_enc} frames"
);
}
cur = g.reshape_2d(cur, in_dim, n_time);
let pre_out = g.linear(cur, pre.out, Some(pre.out_b));
let mut blocks = Vec::with_capacity(tensors.enc_blocks.len());
cur = pre_out;
for (b, &(bn_scale, bn_shift)) in tensors.enc_blocks.iter().zip(&bn) {
cur = feed_forward(
g,
cur,
[b.norm_ff1, b.norm_ff1_b],
[b.ff1_linear1, b.ff1_linear1_b],
[b.ff1_linear2, b.ff1_linear2_b],
);
cur = attention(g, b, cur, pos, self.n_heads as i64);
cur = conv_module(g, b, cur, bn_scale, bn_shift);
cur = feed_forward(
g,
cur,
[b.norm_ff2, b.norm_ff2_b],
[b.ff2_linear1, b.ff2_linear1_b],
[b.ff2_linear2, b.ff2_linear2_b],
);
cur = layer_norm(g, cur, b.norm_out, b.norm_out_b);
blocks.push(cur);
}
let out = cur;
gr.mark_output(out);
let probe_tensors = probes
.iter()
.map(|p| match p {
Probe::PreEncode => Ok(pre_out),
Probe::PosEmb => Ok(pos),
Probe::Block(i) => blocks.get(*i).copied().ok_or_else(|| {
anyhow!(
"encoder: block {i} probe requested of {} blocks",
blocks.len()
)
}),
})
.collect::<Result<Vec<Tensor>>>()?;
for &t in &probe_tensors {
gr.mark_output(t);
}
if !self.logged.swap(true, Ordering::Relaxed) {
let bytes = gr.measure(be)?;
eprintln!("[llmman] speech: encoder compute buffer {bytes} bytes for {frames} frames");
}
gr.compute(be)?;
Ok(Run {
out: gr.output_f32(out),
frames: t_enc,
probes: probe_tensors.iter().map(|&t| gr.output_f32(t)).collect(),
})
}
}
pub(super) fn layer_norm<'g>(
g: &'g Graph<'g>,
x: Tensor<'g>,
w: Tensor<'g>,
b: Tensor<'g>,
) -> Tensor<'g> {
g.add(g.mul(g.norm(x, LN_EPS), w), b)
}
fn feed_forward<'g>(
g: &'g Graph<'g>,
x: Tensor<'g>,
norm: [Tensor<'g>; 2],
linear1: [Tensor<'g>; 2],
linear2: [Tensor<'g>; 2],
) -> Tensor<'g> {
let h = layer_norm(g, x, norm[0], norm[1]);
let h = g.silu(g.linear(h, linear1[0], Some(linear1[1])));
let h = g.linear(h, linear2[0], Some(linear2[1]));
g.add(x, g.scale(h, 0.5))
}
fn attention<'g>(
g: &'g Graph<'g>,
b: &EncBlock<'g>,
x: Tensor<'g>,
pos: Tensor<'g>,
n_heads: i64,
) -> Tensor<'g> {
let [d_model, t, _, _] = x.shape();
let d_head = d_model / n_heads;
let n_pos = pos.shape()[1];
let h = layer_norm(g, x, b.norm_attn, b.norm_attn_b);
let heads = |y| g.reshape_3d(y, d_head, n_heads, t);
let q = heads(g.linear(h, b.attn_q, Some(b.attn_q_b)));
let k = heads(g.linear(h, b.attn_k, Some(b.attn_k_b)));
let v = heads(g.linear(h, b.attn_v, Some(b.attn_v_b)));
let k = g.permute(k, 0, 2, 1, 3);
let q_u = g.permute(g.add(q, b.pos_bias_u), 0, 2, 1, 3);
let q_v = g.cont(g.permute(g.add(q, b.pos_bias_v), 0, 2, 1, 3));
let p = g.reshape_3d(g.mul_mat(b.attn_pos, pos), d_head, n_heads, n_pos);
let p = g.cont(g.permute(p, 0, 2, 1, 3));
let ac = g.prec_f32(g.mul_mat(k, q_u));
let bd = g.prec_f32(g.mul_mat(p, q_v));
let nb = bd.strides();
let shifted = g.view_3d(
bd,
t,
t,
n_heads,
(n_pos - 1) as usize * nb[0],
nb[2],
(t - 1) as usize * nb[0],
);
let scores = g.add(ac, g.cont(shifted));
let probs = g.soft_max_ext(scores, None, 1.0 / (d_head as f32).sqrt(), 0.0);
let v = g.cont(g.permute(v, 1, 2, 0, 3));
let o = g.prec_f32(g.mul_mat(probs, v));
let o = g.cont_2d(g.permute(o, 2, 0, 1, 3), d_model, t);
g.add(x, g.linear(o, b.attn_out, Some(b.attn_out_b)))
}
fn conv_module<'g>(
g: &'g Graph<'g>,
b: &EncBlock<'g>,
x: Tensor<'g>,
bn_scale: Tensor<'g>,
bn_shift: Tensor<'g>,
) -> Tensor<'g> {
let [d_model, t, _, _] = x.shape();
let h = layer_norm(g, x, b.norm_conv, b.norm_conv_b);
let [_, pw_in, pw_out, _] = b.pointwise1.shape();
let pw1 = g.reshape_2d(b.pointwise1, pw_in, pw_out);
let h = g.linear(h, pw1, Some(b.pointwise1_b));
let nb = h.strides();
let signal = g.view_2d(h, d_model, t, nb[1], 0);
let gate = g.view_2d(h, d_model, t, nb[1], d_model as usize * nb[0]);
let h = g.mul(signal, g.sigmoid(gate));
let h = g.cont(g.transpose(h));
let [k, _, _, _] = b.depthwise.shape();
let pad = ((k - 1) / 2) as c_int;
let h = g.pad_ext(h, pad, pad, 0, 0, 0, 0, 0, 0);
let kernel = if b.depthwise.dtype() == ty::F32 {
b.depthwise
} else {
g.cast(b.depthwise, ty::F32)
};
let kernel = g.reshape_2d(kernel, k, d_model);
let h = g.ssm_conv(g.reshape_3d(h, t + k - 1, d_model, 1), kernel);
let h = g.add(g.reshape_2d(h, d_model, t), b.depthwise_b);
let h = g.add(g.mul(h, bn_scale), bn_shift);
let h = g.silu(h);
let [_, pw_in, pw_out, _] = b.pointwise2.shape();
let pw2 = g.reshape_2d(b.pointwise2, pw_in, pw_out);
g.add(x, g.linear(h, pw2, Some(b.pointwise2_b)))
}
fn check_conv_kernel(k: u64) -> Result<()> {
if k.is_multiple_of(2) {
bail!("encoder: conv kernel {k} (the depthwise conv needs an odd kernel)");
}
Ok(())
}
pub fn subsampled_len(frames: usize) -> usize {
(0..3).fold(frames, |len, _| {
(len + 2).checked_sub(3).map_or(0, |n| n / 2 + 1)
})
}
pub fn pos_table(t_enc: usize, d_model: usize) -> Vec<f32> {
assert!(
d_model.is_multiple_of(2),
"pos_table: d_model {d_model} is odd"
);
let rows = 2 * t_enc - 1;
let div: Vec<f64> = (0..d_model / 2)
.map(|i| (-((2 * i) as f64) * 10_000f64.ln() / d_model as f64).exp())
.collect();
let mut table = Vec::with_capacity(rows * d_model);
for r in 0..rows {
let p = (t_enc as f64 - 1.0) - r as f64;
for d in &div {
let (sin, cos) = (p * d).sin_cos();
table.push(sin as f32);
table.push(cos as f32);
}
}
table
}
pub fn fold_batch_norm(
weight: &[f32],
bias: &[f32],
mean: &[f32],
var: &[f32],
eps: f32,
) -> (Vec<f32>, Vec<f32>) {
let n = weight.len();
assert!(
bias.len() == n && mean.len() == n && var.len() == n,
"BatchNorm fold: channel counts differ ({n}, {}, {}, {})",
bias.len(),
mean.len(),
var.len()
);
let eps = eps as f64;
let mut scale = Vec::with_capacity(n);
let mut shift = Vec::with_capacity(n);
for (((w, b), m), v) in weight.iter().zip(bias).zip(mean).zip(var) {
let s = *w as f64 / (*v as f64 + eps).sqrt();
scale.push(s as f32);
shift.push((*b as f64 - *m as f64 * s) as f32);
}
(scale, shift)
}
#[cfg(test)]
mod tests {
use super::*;
fn close(a: f32, b: f32) -> bool {
(a - b).abs() <= 1e-6
}
#[test]
fn subsampled_len_matches_the_fixture_and_the_dump() {
assert_eq!(subsampled_len(1056), 132);
assert_eq!(subsampled_len(1057), 133);
}
#[test]
fn subsampled_len_of_nothing_is_nothing() {
assert_eq!(subsampled_len(0), 0);
assert_eq!(subsampled_len(1), 1);
}
#[test]
fn subsampled_len_halves_three_times() {
let step = |len: usize| (len + 2 - 3) / 2 + 1;
assert_eq!(step(1056), 528);
assert_eq!(step(528), 264);
assert_eq!(step(264), 132);
assert_eq!(subsampled_len(1056), step(step(step(1056))));
}
#[test]
fn check_conv_kernel_accepts_the_odd_nemo_kernel() {
check_conv_kernel(9).unwrap();
check_conv_kernel(31).unwrap();
}
#[test]
fn check_conv_kernel_rejects_even_and_zero_kernels() {
for k in [0, 8] {
let err = check_conv_kernel(k).unwrap_err().to_string();
assert_eq!(
err,
format!("encoder: conv kernel {k} (the depthwise conv needs an odd kernel)")
);
}
}
#[test]
fn pos_table_middle_row_is_position_zero() {
let (t, d) = (5, 8);
let table = pos_table(t, d);
assert_eq!(table.len(), (2 * t - 1) * d);
let mid = &table[(t - 1) * d..t * d];
assert_eq!(mid, &[0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0]);
}
#[test]
fn pos_table_first_row_is_position_t_minus_one() {
let (t, d) = (5, 8);
let table = pos_table(t, d);
let p = (t - 1) as f64;
assert!(close(table[0], p.sin() as f32));
assert!(close(table[1], p.cos() as f32));
let div_1 = (-2.0 * 10_000f64.ln() / d as f64).exp();
assert!(close(table[2], (p * div_1).sin() as f32));
assert!(close(table[3], (p * div_1).cos() as f32));
}
#[test]
fn pos_table_mirrored_rows_negate_sines_and_share_cosines() {
let (t, d) = (6, 8);
let table = pos_table(t, d);
let rows = 2 * t - 1;
for r in 0..rows {
let a = &table[r * d..(r + 1) * d];
let b = &table[(rows - 1 - r) * d..(rows - r) * d];
for i in 0..d / 2 {
assert!(close(a[2 * i], -b[2 * i]), "row {r} col {}", 2 * i);
assert!(
close(a[2 * i + 1], b[2 * i + 1]),
"row {r} col {}",
2 * i + 1
);
}
}
}
#[test]
fn pos_table_rms_matches_the_dump_sidecar() {
let table = pos_table(133, 1024);
let sum: f64 = table.iter().map(|v| (*v as f64).powi(2)).sum();
let rms = (sum / table.len() as f64).sqrt();
assert!(
(rms - std::f64::consts::FRAC_1_SQRT_2).abs() < 1e-6,
"rms {rms}"
);
}
#[test]
fn fold_batch_norm_reproduces_the_normalisation() {
let (w, b, m, v) = (
[2.0f32, 0.5],
[0.25f32, -1.0],
[1.0f32, -3.0],
[4.0f32, 0.25],
);
let eps = 1e-5;
let (scale, shift) = fold_batch_norm(&w, &b, &m, &v, eps);
for c in 0..2 {
for x in [-2.0f32, 0.0, 3.5] {
let want = (x as f64 - m[c] as f64) / (v[c] as f64 + eps as f64).sqrt()
* w[c] as f64
+ b[c] as f64;
let got = x * scale[c] + shift[c];
assert!(
(got as f64 - want).abs() < 1e-5,
"channel {c} x {x}: {got} vs {want}"
);
}
}
}
#[test]
fn fold_batch_norm_identity_parameters() {
let eps = 1e-5f32;
let (scale, shift) =
fold_batch_norm(&[1.0, 1.0], &[0.0, 0.0], &[0.0, 0.0], &[1.0, 1.0], eps);
let want = 1.0 / (1.0 + eps as f64).sqrt();
assert!(scale.iter().all(|s| close(*s, want as f32)));
assert_eq!(shift, vec![0.0, 0.0]);
}
}