use std::cell::RefCell;
use std::path::Path;
use anyhow::{bail, Context, Result};
use crate::mediagen::backend::{Backend, Graph, Resident};
use crate::mediagen::ffi::{self, Tensor};
use crate::mediagen::weights::{read_safetensors_raw, write_safetensors_raw, Weights};
const IM2COL_BUDGET: i64 = 256 << 20;
const NORM_EPS: f32 = 1e-12;
pub struct Chunk {
pub frames: i64,
pub height: i64,
pub width: i64,
pub channels: i64,
pub rgb: Vec<f32>,
}
pub struct Layout {
pub temporal_up: Vec<bool>,
pub patch: bool,
}
impl Layout {
pub fn wan22() -> Layout {
Layout {
temporal_up: vec![true, true, false],
patch: true,
}
}
}
struct Caches<'b> {
be: &'b Backend,
slots: RefCell<Vec<Option<CacheSlot>>>,
read: usize,
filled: Vec<bool>,
}
struct CacheSlot {
pair: Resident,
ne: [i64; 4],
}
impl<'b> Caches<'b> {
fn new(be: &'b Backend) -> Caches<'b> {
Caches {
be,
slots: RefCell::new(Vec::new()),
read: 0,
filled: Vec::new(),
}
}
fn get(&self, slot: usize) -> Option<Tensor<'_>> {
let slots = self.slots.borrow();
match (slots.get(slot), self.filled.get(slot)) {
(Some(Some(s)), Some(true)) => {
let ptr = s.pair.tensor(self.read).as_ptr();
Some(unsafe { Tensor::from_raw(ptr, self) })
}
_ => None,
}
}
fn target(&self, slot: usize, ne: [i64; 4]) -> Result<Tensor<'_>> {
let mut slots = self.slots.borrow_mut();
if slots.len() <= slot {
slots.resize_with(slot + 1, || None);
}
if let Some(s) = &slots[slot] {
if s.ne != ne {
bail!("cache slot {slot}: shape {ne:?} does not match {:?}", s.ne);
}
} else {
let pair = Resident::new(self.be, ffi::ty::F32, &[&ne, &ne])
.with_context(|| format!("cache slot {slot} {ne:?}"))?;
slots[slot] = Some(CacheSlot { pair, ne });
}
let ptr = slots[slot]
.as_ref()
.unwrap()
.pair
.tensor(1 - self.read)
.as_ptr();
Ok(unsafe { Tensor::from_raw(ptr, self) })
}
fn advance(&mut self, filled: Vec<bool>) {
self.filled = filled;
self.read = 1 - self.read;
}
}
struct Build<'a, 'b> {
gr: &'a Graph<'a>,
w: &'a Weights,
direct: bool,
caches: &'a Caches<'b>,
slot: usize,
filled: Vec<bool>,
first_chunk: bool,
}
impl<'a> Build<'a, '_> {
fn g(&self) -> &'a Graph<'a> {
self.gr
}
fn zeros(&self, ne: [i64; 4], name: &str) -> Tensor<'a> {
let n = ne.iter().product::<i64>() as usize;
self.gr.input_f32(&ne, &vec![0f32; n], name)
}
fn vec_f32(&self, t: Tensor<'a>) -> Tensor<'a> {
let g = self.g();
let n: i64 = t.shape().iter().product();
let t = if t.dtype() == ffi::ty::F32 {
t
} else {
g.cast(t, ffi::ty::F32)
};
g.reshape_4d(g.cont(t), n, 1, 1, 1)
}
fn norm(&self, x: Tensor<'a>, gamma: &str) -> Result<Tensor<'a>> {
let gm = self.vec_f32(self.w.get(gamma)?);
let g = self.g();
let p = g.cont(g.permute(x, 1, 2, 0, 3)); let p = g.mul(g.rms_norm(p, NORM_EPS), gm);
Ok(g.cont(g.permute(p, 2, 0, 1, 3)))
}
fn bias(&self, y: Tensor<'a>, name: &str) -> Result<Tensor<'a>> {
let g = self.g();
let b = self.w.get(name)?;
let oc = y.shape()[2];
Ok(g.add(y, g.reshape_4d(b, 1, 1, oc, 1)))
}
fn conv1(&self, prefix: &str, x: Tensor<'a>) -> Result<Tensor<'a>> {
let k = self.w.get(&format!("{prefix}.weight"))?;
let y = self.conv2(k, x, 1, 0);
self.bias(y, &format!("{prefix}.bias"))
}
fn conv2d(&self, prefix: &str, x: Tensor<'a>) -> Result<Tensor<'a>> {
let k = self.w.get(&format!("{prefix}.weight"))?;
let y = self.conv_frames(k, x, 1, x.shape()[3])?;
self.bias(y, &format!("{prefix}.bias"))
}
fn conv2(&self, k: Tensor<'a>, x: Tensor<'a>, s: i32, pad: i32) -> Tensor<'a> {
let direct = if self.direct {
self.g().conv_2d_direct(k, x, s, pad)
} else {
None
};
direct.unwrap_or_else(|| self.g().conv_2d(k, x, s, s, pad, pad, 1, 1))
}
fn conv_frames(
&self,
k: Tensor<'a>,
x: Tensor<'a>,
pad: i32,
nframes: i64,
) -> Result<Tensor<'a>> {
let g = self.g();
let [w, h, c, _] = x.shape();
let kw = k.shape()[0];
let per_frame = kw * kw * c * w * h * 2;
if !self.direct && pad > 0 && per_frame > IM2COL_BUDGET {
return Ok(self.conv_bands(k, x, pad as i64, nframes));
}
let group = if self.direct {
nframes
} else {
(IM2COL_BUDGET / per_frame.max(1)).clamp(1, nframes)
};
let nb = x.strides();
let mut out: Option<Tensor<'a>> = None;
let mut f0 = 0;
while f0 < nframes {
let nf = group.min(nframes - f0);
let xv = g.view_4d(x, w, h, c, nf, nb[1], nb[2], nb[3], (f0 as usize) * nb[3]);
let y = self.conv2(k, xv, 1, pad);
out = Some(match out {
None => y,
Some(o) => g.concat(o, y, 3),
});
f0 += nf;
}
Ok(out.expect("at least one frame"))
}
fn conv_bands(&self, k: Tensor<'a>, x: Tensor<'a>, pad: i64, nframes: i64) -> Tensor<'a> {
let g = self.g();
let [w, h, c, _] = x.shape();
let kw = k.shape()[0];
let rows = (IM2COL_BUDGET / (kw * kw * c * w * 2)).clamp(1, h);
let nb = x.strides();
let mut out: Option<Tensor<'a>> = None;
for f in 0..nframes {
let mut frame: Option<Tensor<'a>> = None;
let mut y0 = 0;
while y0 < h {
let y1 = (y0 + rows).min(h);
let (a, b) = ((y0 - pad).max(0), (y1 + pad).min(h));
let off = a as usize * nb[1] + f as usize * nb[3];
let band = g.cont(g.view_4d(x, w, b - a, c, 1, nb[1], nb[2], nb[3], off));
let (top, bottom) = (a - (y0 - pad), y1 + pad - b);
let band = if top > 0 || bottom > 0 {
g.pad_ext(band, 0, 0, top as i32, bottom as i32, 0, 0, 0, 0)
} else {
band
};
let y = g.conv_2d(k, band, 1, 1, pad as i32, 0, 1, 1);
frame = Some(frame.map_or(y, |o| g.concat(o, y, 1)));
y0 = y1;
}
let frame = frame.expect("at least one row");
out = Some(out.map_or(frame, |o| g.concat(o, frame, 3)));
}
out.expect("at least one frame")
}
fn take_cache(&mut self, prefix: &str, ne: [i64; 4]) -> Result<(usize, Tensor<'a>)> {
let slot = self.slot;
self.slot += 1;
let Some(t) = self.caches.get(slot) else {
let zeros = self.zeros(ne, &format!("zcache{slot}"));
return Ok((slot, zeros));
};
if t.shape() != ne {
bail!(
"{prefix}: cache shape {:?} does not match {ne:?}",
t.shape()
);
}
Ok((slot, t))
}
fn keep_cache(&mut self, slot: usize, t: Option<Tensor<'a>>) -> Result<()> {
if self.filled.len() <= slot {
self.filled.resize(slot + 1, false);
}
if let Some(t) = t {
let dst = self.caches.target(slot, t.shape())?;
self.gr.store(t, dst);
}
self.filled[slot] = t.is_some();
Ok(())
}
fn stack_taps(&self, xx: Tensor<'a>, n: i64) -> Tensor<'a> {
let g = self.g();
let [w, h, c, _] = xx.shape();
let nb = xx.strides();
let tap = |k: usize| g.view_4d(xx, w, h, c, n, nb[1], nb[2], nb[3], k * nb[3]);
g.concat(g.concat(tap(0), tap(1), 2), tap(2), 2)
}
fn cconv(&mut self, prefix: &str, x: Tensor<'a>, spatial_pad: i32) -> Result<Tensor<'a>> {
let [w, h, c, n] = x.shape();
let k = self.w.get(&format!("{prefix}.weight"))?;
if k.shape()[2] == c {
let y = self.conv_frames(k, x, spatial_pad, n)?;
return self.bias(y, &format!("{prefix}.bias"));
}
let g = self.g();
let (slot, cache) = self.take_cache(prefix, [w, h, c, 2])?;
let xx = g.concat(cache, x, 3); let nb = xx.strides();
let keep = g.view_4d(xx, w, h, c, 2, nb[1], nb[2], nb[3], (n as usize) * nb[3]);
self.keep_cache(slot, Some(keep))?;
let stacked = self.stack_taps(xx, n);
let y = self.conv_frames(k, stacked, spatial_pad, n)?; self.bias(y, &format!("{prefix}.bias"))
}
fn resnet(&mut self, prefix: &str, x: Tensor<'a>) -> Result<Tensor<'a>> {
let short = if self.w.has(&format!("{prefix}.conv_shortcut.weight")) {
self.conv1(&format!("{prefix}.conv_shortcut"), x)?
} else {
x
};
let h = self.norm(x, &format!("{prefix}.norm1.gamma"))?;
let h = self.cconv(&format!("{prefix}.conv1"), self.g().silu(h), 1)?;
let h = self.norm(h, &format!("{prefix}.norm2.gamma"))?;
let h = self.cconv(&format!("{prefix}.conv2"), self.g().silu(h), 1)?;
Ok(self.g().add(h, short))
}
fn attention(&mut self, prefix: &str, x: Tensor<'a>) -> Result<Tensor<'a>> {
let [w, h, c, n] = x.shape();
let hn = self.norm(x, &format!("{prefix}.norm.gamma"))?;
let qkv = self.conv1(&format!("{prefix}.to_qkv"), hn)?; let g = self.g();
let hw = w * h;
let qkv = g.reshape_3d(qkv, hw, 3 * c, n);
let nb = qkv.strides();
let q = g.view_3d(qkv, hw, c, n, nb[1], nb[2], 0);
let k = g.view_3d(qkv, hw, c, n, nb[1], nb[2], (c as usize) * nb[1]);
let v = g.view_3d(qkv, hw, c, n, nb[1], nb[2], (2 * c as usize) * nb[1]);
let qc = g.cont(g.permute(q, 1, 0, 2, 3)); let kc = g.cont(g.permute(k, 1, 0, 2, 3));
let vc = g.cont(v); let scores = g.mul_mat(kc, qc); let scores = g.soft_max_ext(scores, None, 1.0 / (c as f32).sqrt(), 0.0);
let o = g.mul_mat(vc, scores); let o = g.cont(g.permute(o, 1, 0, 2, 3)); let o = g.reshape_4d(o, w, h, c, n);
let o = self.conv1(&format!("{prefix}.proj"), o)?;
Ok(self.g().add(o, x))
}
fn upsample(&mut self, prefix: &str, x: Tensor<'a>, time: bool) -> Result<Tensor<'a>> {
let mut x = x;
if time {
let [w, h, c, n] = x.shape();
if self.first_chunk {
let slot = self.slot;
self.slot += 1;
self.keep_cache(slot, None)?;
} else {
let y = self.cconv(&format!("{prefix}.time_conv"), x, 0)?; x = self.g().reshape_4d(y, w, h, c, 2 * n);
}
}
let [w, h, c, n] = x.shape();
let g = self.g();
let up = g.interpolate(x, 2 * w, 2 * h, c, n, ffi::GGML_SCALE_MODE_NEAREST);
self.conv2d(&format!("{prefix}.resample.1"), up)
}
fn dup_up_3d(&self, x: Tensor<'a>) -> Tensor<'a> {
let g = self.g();
let [w, h, c, n] = x.shape();
let up = g.interpolate(x, 2 * w, 2 * h, c, 2 * n, ffi::GGML_SCALE_MODE_NEAREST);
if self.first_chunk {
let nb = up.strides();
g.cont(g.view_4d(up, 2 * w, 2 * h, c, 2 * n - 1, nb[1], nb[2], nb[3], nb[3]))
} else {
up
}
}
fn dup_up_2d(&self, x: Tensor<'a>) -> Tensor<'a> {
let g = self.g();
let [w, h, c, n] = x.shape();
let up = g.interpolate(x, 2 * w, h, c, n, ffi::GGML_SCALE_MODE_NEAREST);
let s = g.reshape_4d(up, 2 * w, h, 2, (c / 2) * n);
let s = g.cont(g.permute(s, 0, 2, 1, 3)); g.reshape_4d(s, 2 * w, 2 * h, c / 2, n)
}
fn dup_up(&self, x: Tensor<'a>, time: bool, out_c: i64) -> Result<Tensor<'a>> {
let c = x.shape()[2];
let factor = if time { 8 } else { 4 };
match (time, out_c * factor / c) {
(true, 8) => Ok(self.dup_up_3d(x)),
(true, 4) if self.first_chunk => Ok(self.dup_up_3d_halve(x)),
(false, 2) => Ok(self.dup_up_2d(x)),
(_, r) => bail!("unsupported DupUp3D: {c} -> {out_c} channels, repeats {r}"),
}
}
fn dup_up_3d_halve(&self, x: Tensor<'a>) -> Tensor<'a> {
let g = self.g();
let [w, h, c, n] = x.shape();
let s = g.reshape_4d(x, w * h, 2, c / 2, n);
let nb = s.strides();
let odd = g.cont(g.view_4d(s, w * h, 1, c / 2, n, nb[1], nb[2], nb[3], nb[1]));
let odd = g.reshape_4d(odd, w, h, c / 2, n);
g.interpolate(odd, 2 * w, 2 * h, c / 2, n, ffi::GGML_SCALE_MODE_NEAREST)
}
fn unpatchify(&self, x: Tensor<'a>) -> Result<Tensor<'a>> {
let g = self.g();
let [w, h, c12, f] = x.shape();
if c12 != 12 {
bail!("decoder.conv_out has {c12} channels, expected 12");
}
let s = g.reshape_4d(x, w, h, 2, 6 * f);
let s = g.cont(g.permute(s, 0, 2, 1, 3)); let s = g.reshape_4d(s, w, 2 * h, 2, 3 * f);
let s = g.cont(g.permute(s, 1, 2, 0, 3)); Ok(g.reshape_4d(s, 2 * w, 2 * h, 3, f))
}
fn patchify(&self, img: Tensor<'a>) -> Tensor<'a> {
let g = self.g();
let [w2, h2, c, f] = img.shape();
debug_assert_eq!(c, 3);
let (w, h) = (w2 / 2, h2 / 2);
let s = g.reshape_4d(img, 2, w, h2, 3 * f); let s = g.cont(g.permute(s, 2, 0, 1, 3)); let s = g.reshape_4d(s, w, 2, h, 6 * f); let s = g.cont(g.permute(s, 0, 2, 1, 3)); g.reshape_4d(s, w, h, 12, f)
}
fn pool2(&self, x: Tensor<'a>) -> Tensor<'a> {
let g = self.g();
let [w, h, c, n] = x.shape();
let s = g.reshape_4d(x, 2, w / 2, h, c * n);
let nb = s.strides();
let a = g.view_4d(s, 1, w / 2, h, c * n, nb[1], nb[2], nb[3], 0);
let b = g.view_4d(s, 1, w / 2, h, c * n, nb[1], nb[2], nb[3], nb[0]);
let s = g.reshape_4d(g.add(a, b), w / 2, 2, h / 2, c * n);
let nb = s.strides();
let a = g.view_4d(s, w / 2, 1, h / 2, c * n, nb[1], nb[2], nb[3], 0);
let b = g.view_4d(s, w / 2, 1, h / 2, c * n, nb[1], nb[2], nb[3], nb[1]);
g.scale(g.reshape_4d(g.add(a, b), w / 2, h / 2, c, n), 0.25)
}
fn avg_down(&mut self, x: Tensor<'a>, time: bool) -> Tensor<'a> {
let p = self.pool2(x);
if !time {
return p;
}
let [w, h, c, n] = p.shape();
let (p, n) = if n % 2 == 1 {
let z = self.zeros([w, h, c, 1], "avgpad");
(self.g().concat(z, p, 3), n + 1)
} else {
(p, n)
};
let g = self.g();
let s = g.reshape_4d(p, w * h, c, 2, n / 2); let s = g.cont(g.permute(s, 0, 2, 1, 3)); g.reshape_4d(s, w, h, 2 * c, n / 2)
}
fn downsample(&mut self, prefix: &str, x: Tensor<'a>, time: bool) -> Result<Tensor<'a>> {
let g = self.g();
let padded = g.pad(x, 1, 1, 0, 0);
let k = self.w.get(&format!("{prefix}.resample.1.weight"))?;
let y = self.conv2(k, padded, 2, 0);
let mut x = self.bias(y, &format!("{prefix}.resample.1.bias"))?;
if time {
let [w, h, c, n] = x.shape();
if self.first_chunk {
let slot = self.slot;
self.slot += 1;
self.keep_cache(slot, Some(x))?;
} else {
let g = self.g();
let (slot, cache) = self.take_cache(prefix, [w, h, c, 1])?;
let xx = g.concat(cache, x, 3); let nb = xx.strides();
let last = g.view_4d(xx, w, h, c, 1, nb[1], nb[2], nb[3], (n as usize) * nb[3]);
self.keep_cache(slot, Some(last))?;
let g = self.g();
let tap =
|k: usize| g.view_4d(xx, w, h, c, n / 2, nb[1], nb[2], 2 * nb[3], k * nb[3]);
let stacked = g.concat(g.concat(tap(0), tap(1), 2), tap(2), 2);
let kt = self.w.get(&format!("{prefix}.time_conv.weight"))?; let y = self.conv2(kt, stacked, 1, 0);
x = self.bias(y, &format!("{prefix}.time_conv.bias"))?;
}
}
Ok(x)
}
fn encoder(&mut self, img: Tensor<'a>) -> Result<Tensor<'a>> {
let mut x = self.patchify(img);
x = self.cconv("encoder.conv_in", x, 1)?;
for b in 0..4 {
let p = format!("encoder.down_blocks.{b}");
let x_copy = x;
for r in 0..2 {
x = self.resnet(&format!("{p}.resnets.{r}"), x)?;
}
match b {
0 => {
x = self.downsample(&format!("{p}.downsampler"), x, false)?;
let short = self.avg_down(x_copy, false);
x = self.g().add(x, short);
}
1 | 2 => {
x = self.downsample(&format!("{p}.downsampler"), x, true)?;
let short = self.avg_down(x_copy, true);
x = self.g().add(x, short);
}
_ => x = self.g().add(x, x_copy),
}
}
x = self.resnet("encoder.mid_block.resnets.0", x)?;
x = self.attention("encoder.mid_block.attentions.0", x)?;
x = self.resnet("encoder.mid_block.resnets.1", x)?;
let x = self.norm(x, "encoder.norm_out.gamma")?;
let x = self.cconv("encoder.conv_out", self.g().silu(x), 1)?;
let x = self.conv1("quant_conv", x)?; let g = self.g();
let [w, h, _, t] = x.shape();
let nb = x.strides();
Ok(g.cont(g.view_4d(x, w, h, 48, t, nb[1], nb[2], nb[3], 0)))
}
fn decoder(&mut self, z: Tensor<'a>, layout: &Layout) -> Result<Tensor<'a>> {
let mut x = self.conv1("post_quant_conv", z)?;
x = self.cconv("decoder.conv_in", x, 1)?;
x = self.resnet("decoder.mid_block.resnets.0", x)?;
x = self.attention("decoder.mid_block.attentions.0", x)?;
x = self.resnet("decoder.mid_block.resnets.1", x)?;
for b in 0.. {
let p = format!("decoder.up_blocks.{b}");
if !self.w.has(&format!("{p}.resnets.0.norm1.gamma")) {
break;
}
let x_copy = x;
for r in 0..3 {
x = self.resnet(&format!("{p}.resnets.{r}"), x)?;
}
if self.w.has(&format!("{p}.upsampler.resample.1.weight")) {
let time = layout.temporal_up.get(b).copied().unwrap_or(false);
x = self.upsample(&format!("{p}.upsampler"), x, time)?;
let short = self.dup_up(x_copy, time, x.shape()[2])?;
x = self.g().add(x, short);
}
}
let x = self.norm(x, "decoder.norm_out.gamma")?;
let x = self.cconv("decoder.conv_out", self.g().silu(x), 1)?;
if layout.patch {
self.unpatchify(x)
} else {
Ok(x)
}
}
}
fn direct_conv(be: &Backend) -> bool {
let wanted = match std::env::var("MEDIAGEN_DIRECT_CONV").as_deref() {
Ok("1") => true,
Ok("0") => false,
_ => be.gpu_name().starts_with("Vulkan"),
};
wanted && be.supports_direct_conv()
}
pub fn decode(
w: &Weights,
be: &Backend,
z: &[Vec<f32>],
c: i64,
h: i64,
wd: i64,
layout: &Layout,
) -> Result<Chunk> {
let want = w.get("post_quant_conv.weight")?.shape()[2];
if c != want {
bail!("expected {want} latent channels, got {c}");
}
let mut caches = Caches::new(be);
let mut out = Chunk {
frames: 0,
height: 0,
width: 0,
channels: 0,
rgb: Vec::new(),
};
let timing = std::env::var_os("MEDIAGEN_TIMING").is_some();
let direct = direct_conv(be);
for (i, frame) in z.iter().enumerate() {
let t_start = std::time::Instant::now();
let gr = Graph::new(be.api, 16 * 1024)?;
let mut x = vec![0f32; (wd * h * c) as usize];
for ci in 0..c as usize {
for hi in 0..h as usize {
for wi in 0..wd as usize {
x[wi + (wd as usize) * (hi + (h as usize) * ci)] =
frame[(ci * h as usize + hi) * wd as usize + wi];
}
}
}
let zt = gr.input_f32(&[wd, h, c, 1], &x, "z");
let mut b = Build {
gr: &gr,
w,
direct,
caches: &caches,
slot: 0,
filled: Vec::new(),
first_chunk: i == 0,
};
let rgb = b.decoder(zt, layout)?;
let rgb = b.g().cont(rgb);
b.gr.mark_output(rgb);
let filled = std::mem::take(&mut b.filled);
drop(b);
let t_built = t_start.elapsed();
gr.compute(be)?;
let t_computed = t_start.elapsed();
let [ow, oh, nc, of] = rgb.shape();
let data = gr.output_f32(rgb);
if out.frames == 0 {
out.width = ow;
out.height = oh;
out.channels = nc;
}
let (ow, oh, nc) = (ow as usize, oh as usize, nc as usize);
for f in 0..of as usize {
for y in 0..oh {
for xx in 0..ow {
for ch in 0..nc {
out.rgb.push(data[xx + ow * (y + oh * (ch + nc * f))]);
}
}
}
}
out.frames += of;
caches.advance(filled);
if timing {
eprintln!(
"[llmman] mediagen: vae chunk {i}: build {:.2} s, compute {:.2} s, readback {:.2} s",
t_built.as_secs_f64(),
(t_computed - t_built).as_secs_f64(),
(t_start.elapsed() - t_computed).as_secs_f64()
);
}
}
Ok(out)
}
pub fn encode(
w: &Weights,
be: &Backend,
rgb: &[f32],
frames: i64,
h: i64,
wd: i64,
) -> Result<Vec<Vec<f32>>> {
if frames < 1 || (frames - 1) % 4 != 0 {
bail!("the encoder takes 4k + 1 frames, got {frames}");
}
let mut caches = Caches::new(be);
let mut out = Vec::new();
let direct = direct_conv(be);
let (hu, wu) = (h as usize, wd as usize);
let mut f0 = 0usize;
let mut first = true;
while (f0 as i64) < frames {
let n = if first { 1 } else { 4 };
let mut x = vec![0f32; wu * hu * 3 * n];
for fi in 0..n {
for y in 0..hu {
for xx in 0..wu {
for ch in 0..3 {
x[xx + wu * (y + hu * (ch + 3 * fi))] =
rgb[(((f0 + fi) * hu + y) * wu + xx) * 3 + ch];
}
}
}
}
let gr = Graph::new(be.api, 16 * 1024)?;
let img = gr.input_f32(&[wd, h, 3, n as i64], &x, "img");
let mut b = Build {
gr: &gr,
w,
direct,
caches: &caches,
slot: 0,
filled: Vec::new(),
first_chunk: first,
};
let z = b.encoder(img)?;
b.gr.mark_output(z);
let filled = std::mem::take(&mut b.filled);
drop(b);
gr.compute(be)?;
let [lw, lh, c, lt] = z.shape();
let data = gr.output_f32(z);
let (lw, lh, c) = (lw as usize, lh as usize, c as usize);
for t in 0..lt as usize {
let mut frame = vec![0f32; c * lh * lw];
for ci in 0..c {
for y in 0..lh {
for xx in 0..lw {
frame[(ci * lh + y) * lw + xx] = data[xx + lw * (y + lh * (ci + c * t))];
}
}
}
out.push(frame);
}
caches.advance(filled);
f0 += n;
first = false;
}
Ok(out)
}
pub fn merge_time_taps(src: &Path, dst: &Path) -> Result<()> {
let mut tensors = read_safetensors_raw(src)?;
for t in &mut tensors {
if t.shape.len() != 5 {
continue;
}
let esz = match t.dtype.as_str() {
"BF16" | "F16" => 2usize,
"F32" => 4,
other => bail!("{}: unsupported dtype {other}", t.name),
};
let [oc, ic, kt, kh, kw] = [0, 1, 2, 3, 4].map(|i| t.shape[i] as usize);
let spatial = kh * kw * esz;
let mut merged = vec![0u8; t.bytes.len()];
for o in 0..oc {
for i in 0..ic {
for k in 0..kt {
let from = ((o * ic + i) * kt + k) * spatial;
let to = ((o * kt + k) * ic + i) * spatial;
merged[to..to + spatial].copy_from_slice(&t.bytes[from..from + spatial]);
}
}
}
t.bytes = merged;
t.shape = vec![oc as i64, (kt * ic) as i64, kh as i64, kw as i64];
}
write_safetensors_raw(dst, &tensors, "time taps merged")
}
pub fn merged_path(src: &Path, cache_dir: &Path) -> Result<std::path::PathBuf> {
let name = src.file_stem().and_then(|s| s.to_str()).unwrap_or("vae");
let dst = cache_dir.join(format!("{name}.merged.safetensors"));
if !dst.exists() {
std::fs::create_dir_all(cache_dir)?;
let t0 = std::time::Instant::now();
merge_time_taps(src, &dst)
.with_context(|| format!("merging the time taps of {}", src.display()))?;
eprintln!(
"[llmman] mediagen: merged the VAE's conv time taps into {} in {:.1} s",
dst.display(),
t0.elapsed().as_secs_f64()
);
}
Ok(dst)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn merged_taps_are_tap_major() {
let dir = std::env::temp_dir().join(format!("llmman-merge-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let vals: Vec<f32> = (0..2)
.flat_map(|i| (0..3).map(move |t| (i * 10 + t) as f32))
.collect();
let src = dir.join("in.safetensors");
write_safetensors_raw(
&src,
&[crate::mediagen::weights::RawTensor {
name: "k.weight".into(),
dtype: "F32".into(),
shape: vec![1, 2, 3, 1, 1],
bytes: vals.iter().flat_map(|v| v.to_le_bytes()).collect(),
}],
"test",
)
.unwrap();
let dst = dir.join("out.safetensors");
merge_time_taps(&src, &dst).unwrap();
let out = read_safetensors_raw(&dst).unwrap();
assert_eq!(out[0].shape, [1, 6, 1, 1]);
assert_eq!(out[0].f32s().unwrap(), [0.0, 10.0, 1.0, 11.0, 2.0, 12.0]);
let _ = std::fs::remove_dir_all(&dir);
}
}