use anyhow::{bail, Result};
use super::Hparams;
use crate::mediagen::backend::{Backend, Graph};
use crate::mediagen::ffi::{self, Tensor};
use crate::mediagen::ltx::rms;
use crate::mediagen::weights::Weights;
pub const MAX_PROMPT_TOKENS: i64 = 4096;
pub struct UndCache {
pub n_tokens: i64,
pub k: Vec<Vec<f32>>,
pub v: Vec<Vec<f32>>,
}
pub fn mrope_tables(hp: &Hparams, positions: &[[f64; 3]]) -> (Vec<f32>, Vec<f32>) {
let half = (hp.head_dim / 2) as usize;
let sect = hp.mrope_section;
let mut cos = Vec::with_capacity(positions.len() * half);
let mut sin = Vec::with_capacity(positions.len() * half);
for p in positions {
for j in 0..half {
let inv = 1.0 / hp.rope_theta.powf(2.0 * j as f64 / hp.head_dim as f64);
let axis = if j < 3 * sect[1] as usize && j % 3 == 1 {
1
} else if j < 3 * sect[2] as usize && j % 3 == 2 {
2
} else {
0
};
let angle = p[axis] * inv;
cos.push(angle.cos() as f32);
sin.push(angle.sin() as f32);
}
}
(cos, sin)
}
pub fn text_positions(n: i64) -> Vec<[f64; 3]> {
(0..n).map(|i| [i as f64; 3]).collect()
}
pub fn vision_positions(hp: &Hparams, n_text: i64, grid: [i64; 3], fps: f32) -> Vec<[f64; 3]> {
let offset = (n_text + hp.temporal_margin) as f64;
let [gt, gh, gw] = grid;
let modulated = hp.enable_fps_modulation && gt > 1;
let mut out = Vec::with_capacity((gt * gh * gw) as usize);
for t in 0..gt {
let tp = if modulated {
let tps = fps as f64 / hp.vae_scale_t as f64;
let base_tps = hp.base_fps as f64 / hp.vae_scale_t as f64;
t as f64 / tps * base_tps + offset
} else {
t as f64 + offset
};
for h in 0..gh {
for w in 0..gw {
out.push([tp, h as f64, w as f64]);
}
}
}
out
}
pub fn sound_positions(hp: &Hparams, n_text: i64, n: i64, sound_fps: f32) -> Vec<[f64; 3]> {
let offset = (n_text + hp.temporal_margin) as f64;
(0..n)
.map(|f| {
let t = if hp.enable_fps_modulation && n > 1 {
f as f64 / sound_fps as f64 * hp.base_fps as f64 + offset
} else {
f as f64 + offset
};
[t, 0.0, 0.0]
})
.collect()
}
pub fn action_positions(hp: &Hparams, n_text: i64, n: i64, fps: f32) -> Vec<[f64; 3]> {
let offset = (n_text + hp.temporal_margin) as f64;
let base_tps = hp.base_fps as f64 / hp.vae_scale_t as f64;
(0..n)
.map(|f| {
let t = if hp.enable_fps_modulation && n > 1 {
(f + 1) as f64 / fps as f64 * base_tps + offset
} else {
(f + 1) as f64 + offset
};
[t, 0.0, 0.0]
})
.collect()
}
pub type Tables = (Vec<f32>, Vec<f32>);
pub fn timestep_embedding(t: f32) -> Vec<f32> {
let half = 128;
let mut e = vec![0f32; 256];
for j in 0..half {
let freq = (-(10000f32).ln() * j as f32 / half as f32).exp();
let arg = t * freq;
e[j] = arg.cos();
e[half + j] = arg.sin();
}
e
}
pub fn rope<'g>(
g: &'g Graph<'g>,
x: Tensor<'g>,
cos: Tensor<'g>,
sin: Tensor<'g>,
n_heads: i64,
) -> Tensor<'g> {
let [dim, t, ..] = x.shape();
let hd = dim / n_heads;
let x4 = g.reshape_4d(g.cont(x), hd / 2, 2, n_heads, t);
let nb = x4.strides();
let x1 = g.cont(g.view_4d(x4, hd / 2, 1, n_heads, t, nb[1], nb[2], nb[3], 0));
let x2 = g.cont(g.view_4d(x4, hd / 2, 1, n_heads, t, nb[1], nb[2], nb[3], nb[1]));
let c = g.reshape_4d(cos, hd / 2, 1, 1, t);
let s = g.reshape_4d(sin, hd / 2, 1, 1, t);
let o1 = g.sub(g.mul(x1, c), g.mul(x2, s));
let o2 = g.add(g.mul(x1, s), g.mul(x2, c));
g.reshape_2d(g.concat(o1, o2, 1), dim, t)
}
pub fn head_rms<'g>(
g: &'g Graph<'g>,
x: Tensor<'g>,
hd: i64,
eps: f32,
w: Option<Tensor<'g>>,
) -> Tensor<'g> {
let [dim, t, ..] = x.shape();
let x3 = g.reshape_3d(g.cont(x), hd, dim / hd, t);
let y = rms(g, x3, eps, w);
g.reshape_2d(y, dim, t)
}
pub fn sdpa_gqa<'g>(
g: &'g Graph<'g>,
q: Tensor<'g>,
k: Tensor<'g>,
v: Tensor<'g>,
n_heads: i64,
n_kv: i64,
mask: Option<Tensor<'g>>,
flash: bool,
) -> Tensor<'g> {
let tq = q.shape()[1];
let tk = k.shape()[1];
let hd = q.shape()[0] / n_heads;
let scale = 1.0 / (hd as f32).sqrt();
let q3 = g.permute(g.reshape_3d(g.cont(q), hd, n_heads, tq), 0, 2, 1, 3);
let k3 = g.permute(g.reshape_3d(g.cont(k), hd, n_kv, tk), 0, 2, 1, 3);
if flash && mask.is_none() {
let v3 = g.permute(g.reshape_3d(g.cont(v), hd, n_kv, tk), 0, 2, 1, 3);
let k3 = g.cast(k3, ffi::ty::F16);
let v3 = g.cast(v3, ffi::ty::F16);
let cur = g.prec_f32(g.flash_attn_ext(q3, k3, v3, None, scale, 0.0, 0.0));
g.reshape_2d(cur, hd * n_heads, tq)
} else {
let v3 = g.cont(g.permute(g.reshape_3d(g.cont(v), hd, n_kv, tk), 1, 2, 0, 3));
let kq = g.mul_mat(k3, q3);
let kq = g.soft_max_ext(kq, mask, scale, 0.0);
let kqv = g.mul_mat(v3, kq);
g.cont_2d(g.permute(kqv, 0, 2, 1, 3), hd * n_heads, tq)
}
}
fn mlp<'g>(
g: &'g Graph<'g>,
w: &'g Weights,
prefix: &str,
x: Tensor<'g>,
relu2: bool,
) -> Result<Tensor<'g>> {
let up = g.linear(x, w.get(&format!("{prefix}.up_proj.weight"))?, None);
let h = if relu2 {
g.sqr(g.clamp(up, 0.0, f32::INFINITY))
} else {
let gate = g.linear(x, w.get(&format!("{prefix}.gate_proj.weight"))?, None);
g.mul(g.silu(gate), up)
};
Ok(g.linear(h, w.get(&format!("{prefix}.down_proj.weight"))?, None))
}
fn text_decoder<'g>(
hp: &Hparams,
w: &'g Weights,
gr: &'g Graph<'g>,
ids: &[u32],
mut on_layer: impl FnMut(&'g Graph<'g>, Tensor<'g>, Tensor<'g>),
) -> Result<Tensor<'g>> {
let n = ids.len() as i64;
if n == 0 || n > MAX_PROMPT_TOKENS {
bail!("the prompt must be 1 to {MAX_PROMPT_TOKENS} tokens, got {n}");
}
let ids_i32: Vec<i32> = ids.iter().map(|&i| i as i32).collect();
let ids_t = gr.input_i32(&[n], &ids_i32, "ids");
let (cos, sin) = mrope_tables(hp, &text_positions(n));
let half = hp.head_dim / 2;
let cos_t = gr.input_f32(&[half, n], &cos, "cos");
let sin_t = gr.input_f32(&[half, n], &sin, "sin");
let mask_t = gr.input_f32(&[n, n], &causal_mask(n), "mask");
let g = &gr;
let eps = hp.rms_eps;
let hd = hp.head_dim;
let mut x = g.get_rows(w.get("embed_tokens.weight")?, ids_t);
for l in 0..hp.n_layers {
let p = format!("layers.{l}");
let h = rms(
g,
x,
eps,
Some(w.get(&format!("{p}.input_layernorm.weight"))?),
);
let mut q = g.linear(h, w.get(&format!("{p}.self_attn.to_q.weight"))?, None);
let mut k = g.linear(h, w.get(&format!("{p}.self_attn.to_k.weight"))?, None);
let v = g.linear(h, w.get(&format!("{p}.self_attn.to_v.weight"))?, None);
if hp.qk_norm_for_text {
q = head_rms(
g,
q,
hd,
eps,
Some(w.get(&format!("{p}.self_attn.norm_q.weight"))?),
);
k = head_rms(
g,
k,
hd,
eps,
Some(w.get(&format!("{p}.self_attn.norm_k.weight"))?),
);
}
let mut k_gen = match w.opt(&format!("{p}.self_attn.k_norm_und_for_gen.weight")) {
Some(nw) => head_rms(g, k, hd, eps, Some(nw)),
None => k,
};
q = rope(g, q, cos_t, sin_t, hp.n_heads);
k = rope(g, k, cos_t, sin_t, hp.n_kv_heads);
k_gen = rope(g, k_gen, cos_t, sin_t, hp.n_kv_heads);
on_layer(gr, k_gen, v);
let attn = sdpa_gqa(g, q, k, v, hp.n_heads, hp.n_kv_heads, Some(mask_t), false);
x = g.add(
x,
g.linear(attn, w.get(&format!("{p}.self_attn.to_out.weight"))?, None),
);
let h2 = rms(
g,
x,
eps,
Some(w.get(&format!("{p}.post_attention_layernorm.weight"))?),
);
x = g.add(x, mlp(g, w, &format!("{p}.mlp"), h2, hp.relu2)?);
}
Ok(x)
}
pub fn causal_mask(n: i64) -> Vec<f32> {
let n = n as usize;
let mut mask = vec![0f32; n * n];
for q in 0..n {
mask[q * n + q + 1..(q + 1) * n].fill(f32::NEG_INFINITY);
}
mask
}
fn text_graph<'o>(be: &Backend, hp: &Hparams) -> Result<Graph<'o>> {
Graph::new(be.api, 4 * 1024 + hp.n_layers as usize * 96)
}
pub fn und_forward(hp: &Hparams, w: &Weights, be: &Backend, ids: &[u32]) -> Result<UndCache> {
let gr = text_graph(be, hp)?;
let (mut ks, mut vs) = (Vec::new(), Vec::new());
text_decoder(hp, w, &gr, ids, |gr, k, v| {
let (k, v) = (gr.cont(k), gr.cont(v));
gr.mark_output(k);
gr.mark_output(v);
ks.push(k);
vs.push(v);
})?;
gr.compute(be)?;
Ok(UndCache {
n_tokens: ids.len() as i64,
k: ks.iter().map(|&t| gr.output_f32(t)).collect(),
v: vs.iter().map(|&t| gr.output_f32(t)).collect(),
})
}
pub fn text_hidden(hp: &Hparams, w: &Weights, be: &Backend, ids: &[u32]) -> Result<Vec<f32>> {
let gr = text_graph(be, hp)?;
let x = text_decoder(hp, w, &gr, ids, |_, _, _| {})?;
let x = gr.cont(x);
gr.mark_output(x);
gr.compute(be)?;
Ok(gr.output_f32(x))
}
pub struct Segment<'a> {
pub tokens: &'a [f32],
pub n: i64,
pub cos: &'a [f32],
pub sin: &'a [f32],
pub noisy: Option<&'a [f32]>,
}
pub struct GenInputs<'a> {
pub vision: Segment<'a>,
pub sound: Option<Segment<'a>>,
pub action: Option<(Segment<'a>, i64)>,
pub timestep: f32,
pub und: &'a UndCache,
pub flash: bool,
}
pub struct GenOutputs {
pub vision: Vec<f32>,
pub sound: Vec<f32>,
pub action: Vec<f32>,
}
fn domain_linear<'g>(
g: &'g Graph<'g>,
w: &'g Weights,
prefix: &str,
domain: Tensor<'g>,
n_in: i64,
n_out: i64,
) -> Result<(Tensor<'g>, Tensor<'g>)> {
let row = g.get_rows(w.get(&format!("{prefix}.fc.weight"))?, domain); let wt = g.cont(g.transpose(g.reshape_2d(row, n_out, n_in)));
let b = g.get_rows(w.get(&format!("{prefix}.bias.weight"))?, domain); Ok((wt, g.reshape_2d(b, n_out, 1)))
}
pub fn gen_forward(hp: &Hparams, w: &Weights, be: &Backend, inp: &GenInputs) -> Result<GenOutputs> {
let nt = inp.und.n_tokens;
let nv = inp.vision.n;
let ns = inp.sound.as_ref().map_or(0, |s| s.n);
let na = inp.action.as_ref().map_or(0, |(s, _)| s.n);
let n = nv + ns + na;
let gr = Graph::new(be.api, 4 * 1024 + hp.n_layers as usize * 128)?;
let half = hp.head_dim / 2;
let v_tok = gr.input_f32(&[hp.patch_dim, nv], inp.vision.tokens, "vision");
let s_tok = inp
.sound
.as_ref()
.map(|s| gr.input_f32(&[hp.sound_dim, s.n], s.tokens, "sound"));
let a_tok = inp
.action
.as_ref()
.map(|(s, _)| gr.input_f32(&[hp.action_dim, s.n], s.tokens, "action"));
let domain = inp
.action
.as_ref()
.map(|(_, d)| gr.input_i32(&[1], &[*d as i32], "domain"));
let mut cos = inp.vision.cos.to_vec();
let mut sin = inp.vision.sin.to_vec();
let mut noisy: Vec<f32> = match inp.vision.noisy {
Some(m) => m.to_vec(),
None => vec![1.0; nv as usize],
};
let mut any_cond = inp.vision.noisy.is_some();
for seg in inp.sound.iter().chain(inp.action.iter().map(|(s, _)| s)) {
cos.extend_from_slice(seg.cos);
sin.extend_from_slice(seg.sin);
match seg.noisy {
Some(m) => {
noisy.extend_from_slice(m);
any_cond = true;
}
None => noisy.extend(std::iter::repeat_n(1.0, seg.n as usize)),
}
}
let cos_t = gr.input_f32(&[half, n], &cos, "cos");
let sin_t = gr.input_f32(&[half, n], &sin, "sin");
let noisy_t = any_cond.then(|| gr.input_f32(&[1, n], &noisy, "noisy"));
let temb_in = gr.input_f32(&[256], ×tep_embedding(inp.timestep), "temb");
let kv_dim = hp.head_dim * hp.n_kv_heads;
let und_k: Vec<Tensor> = (0..hp.n_layers as usize)
.map(|l| gr.input_f32(&[kv_dim, nt], &inp.und.k[l], &format!("und_k{l}")))
.collect();
let und_v: Vec<Tensor> = (0..hp.n_layers as usize)
.map(|l| gr.input_f32(&[kv_dim, nt], &inp.und.v[l], &format!("und_v{l}")))
.collect();
let g = &gr;
let eps = hp.rms_eps;
let hd = hp.head_dim;
let mut x = g.linear(
v_tok,
w.get("proj_in.weight")?,
Some(w.get("proj_in.bias")?),
);
if let Some(st) = s_tok {
let e = g.linear(
st,
w.get("audio_proj_in.weight")?,
Some(w.get("audio_proj_in.bias")?),
);
let e = g.add(
e,
g.reshape_2d(w.get("audio_modality_embed")?, hp.hidden, 1),
);
x = g.concat(x, e, 1);
}
let mut action_out_w = None;
if let (Some(at), Some(d)) = (a_tok, domain) {
let (wi, bi) = domain_linear(g, w, "action_proj_in", d, hp.action_dim, hp.hidden)?;
let e = g.add(g.mul_mat(wi, at), bi);
let e = g.add(
e,
g.reshape_2d(w.get("action_modality_embed")?, hp.hidden, 1),
);
x = g.concat(x, e, 1);
action_out_w = Some(domain_linear(
g,
w,
"action_proj_out",
d,
hp.hidden,
hp.action_dim,
)?);
}
let te = g.linear(
g.reshape_2d(temb_in, 256, 1),
w.get("time_embedder.linear_1.weight")?,
Some(w.get("time_embedder.linear_1.bias")?),
);
let te = g.linear(
g.silu(te),
w.get("time_embedder.linear_2.weight")?,
Some(w.get("time_embedder.linear_2.bias")?),
);
x = match noisy_t {
None => g.add(x, te),
Some(m) => g.add(x, g.mul(g.repeat(te, x), m)),
};
for l in 0..hp.n_layers {
let p = format!("layers.{l}");
let h = rms(
g,
x,
eps,
Some(w.get(&format!("{p}.input_layernorm_moe_gen.weight"))?),
);
let q = g.linear(h, w.get(&format!("{p}.self_attn.add_q_proj.weight"))?, None);
let k = g.linear(h, w.get(&format!("{p}.self_attn.add_k_proj.weight"))?, None);
let v = g.linear(h, w.get(&format!("{p}.self_attn.add_v_proj.weight"))?, None);
let q = head_rms(
g,
q,
hd,
eps,
Some(w.get(&format!("{p}.self_attn.norm_added_q.weight"))?),
);
let k = head_rms(
g,
k,
hd,
eps,
Some(w.get(&format!("{p}.self_attn.norm_added_k.weight"))?),
);
let q = rope(g, q, cos_t, sin_t, hp.n_heads);
let k = rope(g, k, cos_t, sin_t, hp.n_kv_heads);
let kk = g.concat(und_k[l as usize], g.cont(k), 1);
let vv = g.concat(und_v[l as usize], g.cont(v), 1);
let attn = sdpa_gqa(g, q, kk, vv, hp.n_heads, hp.n_kv_heads, None, inp.flash);
x = g.add(
x,
g.linear(
attn,
w.get(&format!("{p}.self_attn.to_add_out.weight"))?,
None,
),
);
let h2 = rms(
g,
x,
eps,
Some(w.get(&format!("{p}.post_attention_layernorm_moe_gen.weight"))?),
);
x = g.add(x, mlp(g, w, &format!("{p}.mlp_moe_gen"), h2, hp.relu2)?);
}
let out = g.cont(rms(g, x, eps, Some(w.get("norm_moe_gen.weight")?)));
let nb = out.strides();
let part = |off: i64, len: i64| g.view_2d(out, hp.hidden, len, nb[1], (off as usize) * nb[1]);
let v_out = g.cont(g.linear(
part(0, nv),
w.get("proj_out.weight")?,
Some(w.get("proj_out.bias")?),
));
gr.mark_output(v_out);
let s_out = (ns > 0)
.then(|| -> Result<Tensor> {
let t = g.cont(g.linear(
part(nv, ns),
w.get("audio_proj_out.weight")?,
Some(w.get("audio_proj_out.bias")?),
));
gr.mark_output(t);
Ok(t)
})
.transpose()?;
let a_out = match action_out_w {
Some((wo, bo)) if na > 0 => {
let t = g.cont(g.add(g.mul_mat(wo, part(nv + ns, na)), bo));
gr.mark_output(t);
Some(t)
}
_ => None,
};
gr.compute(be)?;
Ok(GenOutputs {
vision: gr.output_f32(v_out),
sound: s_out.map(|t| gr.output_f32(t)).unwrap_or_default(),
action: a_out.map(|t| gr.output_f32(t)).unwrap_or_default(),
})
}
pub fn patchify(hp: &Hparams, lat: &[f32], t: i64, h: i64, w: i64) -> Vec<f32> {
let (c, p) = (hp.latent_channels, hp.patch);
let (hp_, wp) = ((h + p - 1) / p, (w + p - 1) / p);
let mut out = vec![0f32; (t * hp_ * wp * hp.patch_dim) as usize];
let idx = |ci: i64, ti: i64, hi: i64, wi: i64| (((ci * t + ti) * h + hi) * w + wi) as usize;
let mut n = 0usize;
for ti in 0..t {
for hh in 0..hp_ {
for ww in 0..wp {
let base = n * hp.patch_dim as usize;
for pp in 0..p {
for q in 0..p {
let (hi, wi) = (hh * p + pp, ww * p + q);
if hi >= h || wi >= w {
continue;
}
for ci in 0..c {
out[base + ((pp * p + q) * c + ci) as usize] = lat[idx(ci, ti, hi, wi)];
}
}
}
n += 1;
}
}
}
out
}
pub fn unpatchify(hp: &Hparams, tokens: &[f32], t: i64, h: i64, w: i64) -> Vec<f32> {
let (c, p) = (hp.latent_channels, hp.patch);
let (hp_, wp) = ((h + p - 1) / p, (w + p - 1) / p);
let mut out = vec![0f32; (c * t * h * w) as usize];
let idx = |ci: i64, ti: i64, hi: i64, wi: i64| (((ci * t + ti) * h + hi) * w + wi) as usize;
let mut n = 0usize;
for ti in 0..t {
for hh in 0..hp_ {
for ww in 0..wp {
let base = n * hp.patch_dim as usize;
for pp in 0..p {
for q in 0..p {
let (hi, wi) = (hh * p + pp, ww * p + q);
if hi >= h || wi >= w {
continue;
}
for ci in 0..c {
out[idx(ci, ti, hi, wi)] =
tokens[base + ((pp * p + q) * c + ci) as usize];
}
}
}
n += 1;
}
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn patchify_roundtrip() {
let hp = Hparams::test_default();
let (t, h, w) = (2, 4, 6);
let lat: Vec<f32> = (0..(hp.latent_channels * t * h * w))
.map(|i| i as f32)
.collect();
let tok = patchify(&hp, &lat, t, h, w);
assert_eq!(tok.len(), (t * (h / 2) * (w / 2) * hp.patch_dim) as usize);
assert_eq!(tok[0], lat[0]); assert_eq!(tok[hp.latent_channels as usize], lat[1]); assert_eq!(tok[2 * hp.latent_channels as usize], lat[w as usize]); assert_eq!(unpatchify(&hp, &tok, t, h, w), lat);
let (h, w) = (3, 5);
let lat: Vec<f32> = (0..(hp.latent_channels * t * h * w))
.map(|i| i as f32)
.collect();
let tok = patchify(&hp, &lat, t, h, w);
assert_eq!(tok.len(), (t * 2 * 3 * hp.patch_dim) as usize);
assert_eq!(unpatchify(&hp, &tok, t, h, w), lat);
}
#[test]
fn mrope_axes() {
let hp = Hparams::test_default();
let (_, s0) = mrope_tables(&hp, &[[3.0, 0.0, 0.0]]);
let (_, s1) = mrope_tables(&hp, &[[3.0, 5.0, 0.0]]);
for j in 0..64 {
let h_axis = j < 60 && j % 3 == 1;
assert_eq!(s0[j] != s1[j], h_axis, "j={j}");
}
let (_, s2) = mrope_tables(&hp, &[[3.0, 0.0, 5.0]]);
for j in 0..64 {
let w_axis = j < 60 && j % 3 == 2;
assert_eq!(s0[j] != s2[j], w_axis, "j={j}");
}
}
#[test]
fn timestep_embedding_layout() {
let e = timestep_embedding(0.995);
assert_eq!(e.len(), 256);
assert!((e[0] - 0.995f32.cos()).abs() < 1e-6);
assert!((e[128] - 0.995f32.sin()).abs() < 1e-6);
let f = (-(10000f32).ln() * 127.0 / 128.0).exp();
assert!((e[127] - (0.995 * f).cos()).abs() < 1e-6);
}
#[test]
fn vision_positions_layout() {
let hp = Hparams::test_default();
let pos = vision_positions(&hp, 10, [3, 2, 2], 24.0);
assert_eq!(pos.len(), 12);
assert_eq!(pos[0], [15010.0, 0.0, 0.0]);
assert_eq!(pos[3], [15010.0, 1.0, 1.0]);
assert_eq!(pos[4], [15011.0, 0.0, 0.0]);
let pos12 = vision_positions(&hp, 10, [3, 2, 2], 12.0);
assert_eq!(pos12[4][0], 15012.0);
assert_eq!(vision_positions(&hp, 10, [1, 2, 2], 12.0)[0][0], 15010.0);
}
}