use crate::conv::FeatureMap;
use crate::matrix::Matrix;
use crate::network::NetworkError;
use crate::t5::{T5Config, T5Encoder};
use crate::tokenizer::Unigram;
use crate::transformer::Precision;
use crate::wan_dit::{WanDit, WanDitConfig};
use crate::wan_vae::{WanVaeConfig, WanVaeDecoder, WanVaeEncoder};
use rand::SeedableRng;
use rand::rngs::StdRng;
use std::path::{Path, PathBuf};
const SPATIAL: usize = 16;
const TEMPORAL: usize = 4;
#[derive(Clone, Debug, PartialEq)]
pub struct VideoConfig {
pub steps: usize,
pub guidance: f32,
pub prompt_length: usize,
pub dmd: Option<Dmd>,
pub skip: Option<StepSkip>,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct StepSkip {
pub full: usize,
pub every: usize,
}
impl StepSkip {
pub fn skips(&self, step: usize, steps: usize) -> bool {
self.every > 0
&& step >= self.full
&& step + 1 < steps
&& (step - self.full) % self.every == self.every - 1
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct Dmd {
pub timesteps: Vec<u32>,
pub seed: u64,
}
pub fn dmd_sigmas(timesteps: &[u32], shift: f64, train_steps: usize) -> Vec<f64> {
let train = train_steps as f64;
let schedule: Vec<f64> = (0..train_steps)
.map(|i| {
let s = 1.0 - i as f64 * (1.0 - 1.0 / train) / (train - 1.0).max(1.0);
shift * s / (1.0 + (shift - 1.0) * s)
})
.collect();
timesteps
.iter()
.map(|&t| {
*schedule
.iter()
.min_by(|a, b| {
(*a * train - t as f64)
.abs()
.total_cmp(&(*b * train - t as f64).abs())
})
.expect("train_steps > 0")
})
.collect()
}
impl Default for VideoConfig {
fn default() -> Self {
Self {
steps: 50,
guidance: 5.0,
prompt_length: 512,
dmd: None,
skip: None,
}
}
}
#[derive(Clone, Copy, Debug)]
pub struct VideoRequest<'a> {
pub width: usize,
pub height: usize,
pub frames: usize,
pub seed: Option<u64>,
pub image: Option<&'a FeatureMap>,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Video {
pub width: usize,
pub height: usize,
pub frames: Vec<Vec<u8>>,
}
pub fn latent_grid(width: usize, height: usize, frames: usize) -> Result<[usize; 3], NetworkError> {
let side = SPATIAL * 2;
if width == 0 || height == 0 || !width.is_multiple_of(side) || !height.is_multiple_of(side) {
return Err(NetworkError::InvalidConfig(format!(
"a Wan clip is a multiple of {side} on each side, not {width}x{height}"
)));
}
if frames == 0 || !(frames - 1).is_multiple_of(TEMPORAL) {
return Err(NetworkError::InvalidConfig(format!(
"a Wan clip has 1 + 4k frames, not {frames}"
)));
}
Ok([
(frames - 1) / TEMPORAL + 1,
height / SPATIAL,
width / SPATIAL,
])
}
fn check_image(image: &FeatureMap, width: usize, height: usize) -> Result<(), NetworkError> {
if image.channels != 3 || image.width != width || image.height != height {
return Err(NetworkError::InvalidConfig(format!(
"the first frame is {}x{}x{} and the clip is 3x{height}x{width}",
image.channels, image.height, image.width
)));
}
Ok(())
}
#[derive(Clone, Debug)]
pub struct UniPc {
timesteps: Vec<f32>,
sigmas: Vec<f32>,
outputs: [Option<Vec<f32>>; 2],
last_sample: Option<Vec<f32>>,
this_order: usize,
lower_order_nums: usize,
index: usize,
}
impl UniPc {
pub fn new(steps: usize, shift: f64, train_steps: usize) -> Result<Self, NetworkError> {
if steps == 0 || train_steps == 0 || shift.is_nan() || shift <= 0.0 {
return Err(NetworkError::InvalidConfig(format!(
"UniPC needs steps, training steps and a positive shift, not {steps}, \
{train_steps}, {shift}"
)));
}
let train = train_steps as f64;
let delta = (1.0 / train - 1.0) / steps as f64;
let mut sigmas: Vec<f64> = (0..steps)
.map(|i| i as f64 * delta + 1.0)
.map(|s| shift * s / (1.0 + (shift - 1.0) * s))
.collect();
if (sigmas[0] - 1.0).abs() < 1e-6 {
sigmas[0] -= 1e-6;
}
let timesteps = sigmas.iter().map(|s| (s * train).trunc() as f32).collect();
let sigmas = sigmas
.iter()
.map(|&s| s as f32)
.chain(std::iter::once(0.0))
.collect();
Ok(Self {
timesteps,
sigmas,
outputs: [None, None],
last_sample: None,
this_order: 0,
lower_order_nums: 0,
index: 0,
})
}
pub fn timesteps(&self) -> &[f32] {
&self.timesteps
}
pub fn sigmas(&self) -> &[f32] {
&self.sigmas
}
pub fn step(&mut self, velocity: &[f32], sample: &[f32]) -> Result<Vec<f32>, NetworkError> {
if velocity.len() != sample.len() {
return Err(NetworkError::InvalidInput {
expected: sample.len(),
actual: velocity.len(),
});
}
let sigma = self.sigma(self.index)?;
let x0 = sample
.iter()
.zip(velocity)
.map(|(x, v)| x - sigma * v)
.collect();
self.step_x0(x0, sample)
}
fn sigma(&self, index: usize) -> Result<f32, NetworkError> {
match index < self.timesteps.len() {
true => Ok(self.sigmas[index]),
false => Err(NetworkError::InvalidConfig(format!(
"UniPC has {} steps and was asked for another",
self.timesteps.len()
))),
}
}
pub(crate) fn step_x0(
&mut self,
x0: Vec<f32>,
sample: &[f32],
) -> Result<Vec<f32>, NetworkError> {
let i = self.index;
self.sigma(i)?;
let mut sample = sample.to_vec();
if i > 0
&& let Some(last) = &self.last_sample
{
sample = self.corrector(&x0, last, &sample, self.this_order);
}
self.outputs[0] = self.outputs[1].take();
self.outputs[1] = Some(x0);
let this_order = 2.min(self.timesteps.len() - i);
self.this_order = this_order.min(self.lower_order_nums + 1);
let next = self.predictor(&sample, self.this_order);
self.last_sample = Some(sample);
self.lower_order_nums = (self.lower_order_nums + 1).min(2);
self.index += 1;
Ok(next)
}
fn predictor(&self, x: &[f32], order: usize) -> Vec<f32> {
let i = self.index;
let m0 = self.outputs[1]
.as_deref()
.expect("step stored the prediction");
let (sigma_t, sigma_s0) = (self.sigmas[i + 1], self.sigmas[i]);
let h = lambda(sigma_t) - lambda(sigma_s0);
let (h_phi_1, b_h, _) = coefficients(h, order);
let alpha_t = 1.0 - sigma_t;
let ratio = sigma_t / sigma_s0;
let (c0, cb) = (alpha_t * h_phi_1, alpha_t * b_h);
match order {
1 => x
.iter()
.zip(m0)
.map(|(x, m0)| ratio * x - c0 * m0)
.collect(),
_ => {
let m1 = self.outputs[0].as_deref().expect("order 2 has a history");
let rk = (lambda(self.sigmas[i - 1]) - lambda(sigma_s0)) / h;
x.iter()
.zip(m0)
.zip(m1)
.map(|((x, m0), m1)| {
let d1 = (m1 - m0) / rk;
(ratio * x - c0 * m0) - cb * (0.5 * d1)
})
.collect()
}
}
}
fn corrector(&self, model_t: &[f32], x: &[f32], this_sample: &[f32], order: usize) -> Vec<f32> {
let i = self.index;
let m0 = self.outputs[1]
.as_deref()
.expect("a corrector follows a step");
let (sigma_t, sigma_s0) = (self.sigmas[i], self.sigmas[i - 1]);
let h = lambda(sigma_t) - lambda(sigma_s0);
let (h_phi_1, b_h, b) = coefficients(h, order);
let alpha_t = 1.0 - sigma_t;
let ratio = sigma_t / sigma_s0;
let (c0, cb) = (alpha_t * h_phi_1, alpha_t * b_h);
debug_assert_eq!(this_sample.len(), x.len());
match order {
1 => x
.iter()
.zip(m0)
.zip(model_t)
.map(|((x, m0), mt)| (ratio * x - c0 * m0) - cb * (0.5 * (mt - m0)))
.collect(),
_ => {
let m1 = self.outputs[0].as_deref().expect("order 2 has a history");
let rk = (lambda(self.sigmas[i - 2]) - lambda(sigma_s0)) / h;
let rhos = solve2([[1.0, 1.0], [rk, 1.0]], [b[0], b[1]]);
x.iter()
.zip(m0)
.zip(m1)
.zip(model_t)
.map(|(((x, m0), m1), mt)| {
let d1 = (m1 - m0) / rk;
(ratio * x - c0 * m0) - cb * (rhos[0] * d1 + rhos[1] * (mt - m0))
})
.collect()
}
}
}
}
fn lambda(sigma: f32) -> f32 {
(1.0 - sigma).ln() - sigma.ln()
}
fn coefficients(h: f32, order: usize) -> (f32, f32, Vec<f32>) {
let hh = -h;
let h_phi_1 = hh.exp_m1();
let mut h_phi_k = h_phi_1 / hh - 1.0;
let b_h = hh.exp_m1();
let mut factorial = 1.0f32;
let mut b = Vec::with_capacity(order);
for i in 1..=order {
b.push(h_phi_k * factorial / b_h);
factorial *= (i + 1) as f32;
h_phi_k = h_phi_k / hh - 1.0 / factorial;
}
(h_phi_1, b_h, b)
}
fn solve2(mut a: [[f32; 2]; 2], mut b: [f32; 2]) -> [f32; 2] {
if a[1][0].abs() > a[0][0].abs() {
a.swap(0, 1);
b.swap(0, 1);
}
let l = a[1][0] / a[0][0];
let x1 = (b[1] - l * b[0]) / (a[1][1] - l * a[0][1]);
[(b[0] - a[0][1] * x1) / a[0][0], x1]
}
fn flatten(maps: &[FeatureMap]) -> Vec<f32> {
maps.iter()
.flat_map(|map| map.data.iter().copied())
.collect()
}
fn refill(maps: &mut [FeatureMap], data: &[f32]) {
let mut at = 0;
for map in maps {
let size = map.data.len();
map.data.copy_from_slice(&data[at..at + size]);
at += size;
}
}
pub struct VideoPipeline {
pub config: VideoConfig,
pub encoder: Option<T5Encoder>,
tokenizer: Unigram,
denoiser: Option<WanDit>,
dit_path: PathBuf,
dit_config: WanDitConfig,
precision: Precision,
decoder: WanVaeDecoder,
image_encoder: WanVaeEncoder,
flow_shift: f64,
train_steps: usize,
#[cfg(feature = "cuda")]
gpu: Option<(usize, std::sync::Arc<crate::gpu_transformer::GpuContext>)>,
}
impl VideoPipeline {
pub fn load(directory: impl AsRef<Path>) -> Result<Self, NetworkError> {
Self::load_at(directory, Precision::Q8)
}
pub fn load_at(
directory: impl AsRef<Path>,
precision: Precision,
) -> Result<Self, NetworkError> {
let dir = directory.as_ref();
let index = read_json(&dir.join("model_index.json"))?;
if index.get("expand_timesteps").and_then(|v| v.as_bool()) != Some(true)
|| index.get("transformer_2").is_some_and(|v| {
!v.is_null() && v.as_array().is_none_or(|a| a.iter().any(|x| !x.is_null()))
})
{
return Err(NetworkError::InvalidConfig(
"only Wan 2.2 TI2V-5B (one transformer, expand_timesteps) is implemented".into(),
));
}
let (flow_shift, train_steps) =
read_scheduler(&dir.join("scheduler/scheduler_config.json"))?;
let dit_config = WanDitConfig::from_file(dir.join("transformer/config.json"))?;
let vae = WanVaeConfig::from_file(dir.join("vae/config.json"))?;
if vae.z_dim != dit_config.in_channels
|| dit_config.out_channels != dit_config.in_channels
|| dit_config.patch != [1, 2, 2]
{
return Err(NetworkError::InvalidConfig(format!(
"a {}-channel VAE does not feed a {}->{}-channel DiT with patch {:?}",
vae.z_dim, dit_config.in_channels, dit_config.out_channels, dit_config.patch
)));
}
let t5 = T5Config::from_file(dir.join("text_encoder/config.json"))?;
if t5.d_model != dit_config.text_dim {
return Err(NetworkError::InvalidConfig(format!(
"umT5 makes {} features and the DiT reads {}",
t5.d_model, dit_config.text_dim
)));
}
let encoder = T5Encoder::load_at(
dir.join("text_encoder/model.safetensors.index.json"),
t5,
precision,
)?;
let weights = dir.join("vae/diffusion_pytorch_model.safetensors");
Ok(Self {
config: VideoConfig::default(),
encoder: Some(encoder),
tokenizer: Unigram::from_file(dir.join("tokenizer/tokenizer.json"))?,
denoiser: None,
dit_path: dir.join("transformer/diffusion_pytorch_model.safetensors.index.json"),
dit_config,
precision,
decoder: WanVaeDecoder::load(&weights, vae.clone())?,
image_encoder: WanVaeEncoder::load(&weights, vae)?,
flow_shift,
train_steps,
#[cfg(feature = "cuda")]
gpu: None,
})
}
pub fn set_dit_weights(&mut self, path: impl Into<PathBuf>) {
self.dit_path = path.into();
self.denoiser = None;
}
#[cfg(feature = "cuda")]
pub fn to_cuda(&mut self, device: usize) -> Result<(), NetworkError> {
if self.gpu.is_some() {
return Ok(());
}
let image = crate::cuda_image::ImageGpu::new(device)?;
let context = crate::gpu_transformer::GpuContext::new(device)?;
let mut dit = match self.denoiser.take() {
Some(dit) => dit,
None => WanDit::load_at(&self.dit_path, self.dit_config.clone(), self.precision)?,
};
let uploaded = dit.attach_device_keeping_host(&context);
self.denoiser = Some(dit);
uploaded?;
self.decoder.attach_device(&image)?;
self.gpu = Some((device, context));
Ok(())
}
#[cfg(feature = "cuda")]
pub fn try_cuda(&mut self, device: usize) -> bool {
self.to_cuda(device).is_ok()
}
#[cfg(not(feature = "cuda"))]
pub fn try_cuda(&mut self, _device: usize) -> bool {
false
}
pub fn condition(&self, prompt: &str) -> Result<Matrix, NetworkError> {
let encoder = self
.encoder
.as_ref()
.ok_or_else(|| NetworkError::InvalidConfig("the prompt encoder was dropped".into()))?;
encoder.encode_real(&self.tokenizer, prompt, self.config.prompt_length)
}
pub fn encode_image(&self, image: &FeatureMap) -> Result<FeatureMap, NetworkError> {
latent_grid(image.width, image.height, 1)?;
check_image(image, image.width, image.height)?;
let mut latent = self.image_encoder.encode(std::slice::from_ref(image))?;
Ok(latent.remove(0))
}
pub fn noise(&self, request: &VideoRequest) -> Result<Vec<FeatureMap>, NetworkError> {
let [frames, height, width] = latent_grid(request.width, request.height, request.frames)?;
let mut rng = match request.seed {
Some(seed) => StdRng::seed_from_u64(seed),
None => StdRng::from_entropy(),
};
let channels = self.dit_config.in_channels;
(0..frames)
.map(|_| {
let data = (0..channels * height * width)
.map(|_| crate::diffusion::gaussian(&mut rng))
.collect();
FeatureMap::from_vec(channels, height, width, data)
})
.collect()
}
fn denoiser(&mut self) -> Result<&WanDit, NetworkError> {
if self.denoiser.is_none() {
let dit = WanDit::load_at(&self.dit_path, self.dit_config.clone(), self.precision)?;
self.denoiser = Some(dit);
}
#[cfg(feature = "cuda")]
if let (Some((_, context)), Some(dit)) = (&self.gpu, &mut self.denoiser)
&& !dit.on_device()
{
dit.attach_device_keeping_host(context)?;
}
Ok(self.denoiser.as_ref().expect("loaded above"))
}
pub fn sample_from_noise(
&mut self,
positive: &Matrix,
negative: Option<&Matrix>,
first: Option<&FeatureMap>,
mut noise: Vec<FeatureMap>,
mut on_step: impl FnMut(usize, &[FeatureMap]) -> bool,
) -> Result<Vec<FeatureMap>, NetworkError> {
let channels = self.dit_config.in_channels;
let Some(frame) = noise.first() else {
return Err(NetworkError::InvalidConfig(
"the noise has no frames".into(),
));
};
let (height, width) = (frame.height, frame.width);
if noise
.iter()
.any(|map| map.channels != channels || map.height != height || map.width != width)
{
return Err(NetworkError::InvalidConfig(format!(
"the noise is not {channels}x{height}x{width} in every frame"
)));
}
if let Some(first) = first
&& (first.channels != channels || first.height != height || first.width != width)
{
return Err(NetworkError::InvalidConfig(format!(
"the first frame's latent is {}x{}x{} and the noise {channels}x{height}x{width}",
first.channels, first.height, first.width
)));
}
if let Some(dmd) = self.config.dmd.clone() {
return self.sample_dmd(&dmd, positive, first, noise, on_step);
}
let guidance = self.config.guidance;
let negative = negative.filter(|_| guidance > 1.0);
let mut scheduler = UniPc::new(self.config.steps, self.flow_shift, self.train_steps)?;
let skip = self.config.skip;
let mut guided = Vec::new();
let steps = self.config.steps;
for step in 0..steps {
let t = scheduler.timesteps()[step];
if !guided.is_empty() && skip.is_some_and(|skip| skip.skips(step, steps)) {
let next = scheduler.step(&guided, &flatten(&noise))?;
refill(&mut noise, &next);
if !on_step(step, &noise) {
break;
}
continue;
}
let mut input = noise.clone();
let mut times = vec![t; noise.len()];
if let Some(first) = first {
input[0] = first.clone();
times[0] = 0.0;
}
let dit = self.denoiser()?;
let mut velocity = flatten(&dit.forward(&input, ×, positive)?);
if let Some(negative) = negative {
let unconditional = flatten(&dit.forward(&input, ×, negative)?);
for (v, u) in velocity.iter_mut().zip(&unconditional) {
*v = u + guidance * (*v - u);
}
}
guided = velocity;
let next = scheduler.step(&guided, &flatten(&noise))?;
refill(&mut noise, &next);
if !on_step(step, &noise) {
break;
}
}
if let Some(first) = first {
noise[0] = first.clone();
}
Ok(noise)
}
fn sample_dmd(
&mut self,
dmd: &Dmd,
positive: &Matrix,
first: Option<&FeatureMap>,
mut noise: Vec<FeatureMap>,
mut on_step: impl FnMut(usize, &[FeatureMap]) -> bool,
) -> Result<Vec<FeatureMap>, NetworkError> {
let sigmas = dmd_sigmas(
&dmd.timesteps,
8.0,
self.train_steps,
);
eprintln!("dmd sigmas {sigmas:?}");
let mut rng = StdRng::seed_from_u64(dmd.seed);
for (step, (&t, &sigma)) in dmd.timesteps.iter().zip(&sigmas).enumerate() {
let mut input = noise.clone();
let mut times = vec![t as f32; noise.len()];
if let Some(first) = first {
input[0] = first.clone();
times[0] = 0.0;
}
let velocity = flatten(&self.denoiser()?.forward(&input, ×, positive)?);
let mut x0 = flatten(&noise);
for (x, v) in x0.iter_mut().zip(&velocity) {
*x = (*x as f64 - sigma * *v as f64) as f32;
}
if let Some(&next) = sigmas.get(step + 1) {
for x in &mut x0 {
*x = ((1.0 - next) * *x as f64
+ next * crate::diffusion::gaussian(&mut rng) as f64)
as f32;
}
}
refill(&mut noise, &x0);
if let Some(first) = first {
noise[0] = first.clone();
}
if !on_step(step, &noise) {
break;
}
}
Ok(noise)
}
pub fn decode(&mut self, latent: &[FeatureMap]) -> Result<Vec<FeatureMap>, NetworkError> {
let mut latent = latent.to_vec();
self.decoder.config.denormalize(&mut latent);
#[cfg(feature = "cuda")]
if let Some(frame) = latent.first() {
self.make_room(frame.width * SPATIAL, frame.height * SPATIAL)?;
}
self.decoder.decode(&latent)
}
#[cfg(feature = "cuda")]
fn make_room(&mut self, width: usize, height: usize) -> Result<(), NetworkError> {
let Some((device, _)) = &self.gpu else {
return Ok(());
};
let device = *device;
if self.decoder.device.is_none() || self.denoiser.is_none() {
return Ok(());
}
let need = (650.0 + 5.0e-3 * (width * height) as f64 + 512.0) * 1048576.0;
crate::cuda_training::release_freed_memory(device)?;
let free = crate::cuda_image::memory_info().map_or(0, |(free, _)| free);
if (free as f64) < need {
if let Some(dit) = &mut self.denoiser {
dit.detach_device();
}
crate::cuda_training::release_freed_memory(device)?;
}
Ok(())
}
pub fn generate_conditioned(
&mut self,
positive: &Matrix,
negative: Option<&Matrix>,
request: &VideoRequest,
on_step: impl FnMut(usize, &[FeatureMap]) -> bool,
) -> Result<Video, NetworkError> {
let noise = self.noise(request)?;
let first = match request.image {
Some(image) => {
check_image(image, request.width, request.height)?;
Some(self.encode_image(image)?)
}
None => None,
};
let latent = self.sample_from_noise(positive, negative, first.as_ref(), noise, on_step)?;
let frames = self.decode(&latent)?;
Ok(Video {
width: request.width,
height: request.height,
frames: frames.iter().map(crate::vae::to_rgb8).collect(),
})
}
pub fn generate(
&mut self,
prompt: &str,
negative: &str,
request: &VideoRequest,
on_step: impl FnMut(usize, &[FeatureMap]) -> bool,
) -> Result<Video, NetworkError> {
latent_grid(request.width, request.height, request.frames)?;
let positive = self.condition(prompt)?;
let negative = match self.config.guidance > 1.0 {
true => Some(self.condition(negative)?),
false => None,
};
self.generate_conditioned(&positive, negative.as_ref(), request, on_step)
}
}
fn read_json(path: &Path) -> Result<serde_json::Value, NetworkError> {
Ok(serde_json::from_str(&std::fs::read_to_string(path)?)?)
}
fn read_scheduler(path: &Path) -> Result<(f64, usize), NetworkError> {
let json = read_json(path)?;
let text = |key: &str| json.get(key).and_then(|v| v.as_str());
let flag = |key: &str| json.get(key).and_then(|v| v.as_bool());
let supported = text("_class_name") == Some("UniPCMultistepScheduler")
&& json.get("solver_order").and_then(|v| v.as_u64()) == Some(2)
&& text("prediction_type") == Some("flow_prediction")
&& text("solver_type") == Some("bh2")
&& text("final_sigmas_type") == Some("zero")
&& flag("predict_x0") == Some(true)
&& flag("use_flow_sigmas") == Some(true)
&& flag("lower_order_final") == Some(true)
&& flag("use_dynamic_shifting") != Some(true)
&& flag("thresholding") != Some(true)
&& json.get("solver_p").is_none_or(|v| v.is_null())
&& json.get("shift_terminal").is_none_or(|v| v.is_null())
&& json
.get("disable_corrector")
.and_then(|v| v.as_array())
.is_none_or(|v| v.is_empty());
let shift = json.get("flow_shift").and_then(|v| v.as_f64());
let train = json.get("num_train_timesteps").and_then(|v| v.as_u64());
match (supported, shift, train) {
(true, Some(shift), Some(train)) => Ok((shift, train as usize)),
_ => Err(NetworkError::InvalidConfig(format!(
"{} is not the UniPC variant Wan 2.2 ships",
path.display()
))),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn unipc_matches_diffusers_schedule() {
let unipc = UniPc::new(6, 5.0, 1000).unwrap();
assert_eq!(
unipc.timesteps(),
[999.0, 961.0, 909.0, 833.0, 714.0, 501.0]
);
let want = [
0.9999989867210388f64,
0.9615828394889832,
0.9092148542404175,
0.8336109519004822,
0.7148972749710083,
0.5014970302581787,
0.0,
]
.map(|sigma| sigma as f32);
assert_eq!(unipc.sigmas(), want);
}
#[test]
fn unipc_follows_an_exact_flow() {
let (x0, noise) = ([0.5f32, -1.25, 2.0], [1.0f32, 0.3, -0.7]);
let mut unipc = UniPc::new(6, 5.0, 1000).unwrap();
let sigmas = unipc.sigmas().to_vec();
let mut sample: Vec<f32> = (0..3)
.map(|i| (1.0 - sigmas[0]) * x0[i] + sigmas[0] * noise[i])
.collect();
for step in 0..6 {
let velocity: Vec<f32> = (0..3).map(|i| noise[i] - x0[i]).collect();
sample = unipc.step(&velocity, &sample).unwrap();
let s = sigmas[step + 1];
for i in 0..3 {
let want = (1.0 - s) * x0[i] + s * noise[i];
assert!((sample[i] - want).abs() < 1e-5, "step {step}: {sample:?}");
}
}
assert!(unipc.step(&[0.0; 3], &sample).is_err());
}
#[test]
fn solve2_pivots() {
let [x, y] = solve2([[1.0, 1.0], [-3.0, 1.0]], [2.0, -2.0]);
assert!((x - 1.0).abs() < 1e-6 && (y - 1.0).abs() < 1e-6);
}
#[test]
fn shapes_are_checked() {
assert_eq!(latent_grid(256, 256, 5).unwrap(), [2, 16, 16]);
assert_eq!(latent_grid(1280, 704, 121).unwrap(), [31, 44, 80]);
for (w, h, f) in [
(0, 256, 5),
(256, 48, 5),
(240, 256, 5),
(256, 256, 4),
(256, 256, 0),
] {
assert!(
matches!(latent_grid(w, h, f), Err(NetworkError::InvalidConfig(_))),
"{w}x{h}x{f}"
);
}
let image = FeatureMap::new(3, 256, 224);
assert!(check_image(&image, 224, 256).is_ok());
assert!(check_image(&image, 256, 256).is_err());
assert!(check_image(&FeatureMap::new(4, 256, 224), 224, 256).is_err());
}
#[test]
fn scheduler_config_is_checked() {
let dir = std::env::temp_dir().join(format!("rb-unipc-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("scheduler_config.json");
let good = r#"{"_class_name": "UniPCMultistepScheduler", "solver_order": 2,
"prediction_type": "flow_prediction", "solver_type": "bh2",
"final_sigmas_type": "zero", "predict_x0": true, "use_flow_sigmas": true,
"lower_order_final": true, "use_dynamic_shifting": false, "thresholding": false,
"solver_p": null, "disable_corrector": [], "flow_shift": 5.0,
"num_train_timesteps": 1000}"#;
std::fs::write(&path, good).unwrap();
assert_eq!(read_scheduler(&path).unwrap(), (5.0, 1000));
std::fs::write(&path, good.replace("bh2", "bh1")).unwrap();
assert!(read_scheduler(&path).is_err());
std::fs::write(&path, good.replace("[]", "[0]")).unwrap();
assert!(read_scheduler(&path).is_err());
let index = r#"{"expand_timesteps": true, "transformer_2": [null, null]}"#;
let model_error = |index: &str| {
std::fs::write(dir.join("model_index.json"), index).unwrap();
match VideoPipeline::load(&dir) {
Err(NetworkError::InvalidConfig(message)) => message,
Err(other) => format!("{other:?}"),
Ok(_) => panic!("an empty directory loaded"),
}
};
assert!(!model_error(index).contains("only Wan 2.2 TI2V-5B"));
for bad in [
index.replace("[null, null]", r#"["diffusers", "WanTransformer3DModel"]"#),
index.replace("true", "false"),
] {
assert!(model_error(&bad).contains("only Wan 2.2 TI2V-5B"));
}
std::fs::remove_dir_all(&dir).ok();
}
pub(crate) fn bf16(x: f32) -> f32 {
let bits = x.to_bits();
let rounded = bits.wrapping_add(0x7fff + ((bits >> 16) & 1)) & 0xffff_0000;
f32::from_bits(rounded)
}
pub(crate) fn bf16_x0(
cond: &[f32],
uncond: &[f32],
sample: &[f32],
sigma: f32,
g: f32,
) -> Vec<f32> {
cond.iter()
.zip(uncond)
.zip(sample)
.map(|((c, u), x)| {
let v = bf16(u + bf16(g * bf16(c - u)));
x - bf16(sigma * v)
})
.collect()
}
#[test]
#[ignore = "needs the output of RustingFrame's tools/wan_diffusers_bench.py w4"]
fn wan22_unipc_replays_diffusers() {
use crate::wan_vae::tests::relative;
let dir = PathBuf::from(std::env::var("RB_WAN_REF").expect("RB_WAN_REF"));
let mut reference =
crate::safetensors::SafeTensors::open(dir.join("w4.safetensors")).unwrap();
let mut read = |name: &str| reference.tensor(name).unwrap().0;
let mut worst: f32 = 0.0;
for mode in ["t2v", "ti2v"] {
let mut unipc = UniPc::new(6, 5.0, 1000).unwrap();
assert_eq!(unipc.timesteps(), read(&format!("{mode}.timesteps")));
assert_eq!(unipc.sigmas(), read(&format!("{mode}.sigmas")));
let (mut emulated, mut plain) = (
read(&format!("{mode}.noise")),
read(&format!("{mode}.noise")),
);
let mut float = UniPc::new(6, 5.0, 1000).unwrap();
for step in 0..6 {
let cond = read(&format!("{mode}.step{step}.cond"));
let uncond = read(&format!("{mode}.step{step}.uncond"));
let want = read(&format!("{mode}.step{step}.latent"));
let sigma = unipc.sigmas()[step];
let x0 = bf16_x0(&cond, &uncond, &emulated, sigma, 5.0);
emulated = unipc.step_x0(x0, &emulated).unwrap();
let velocity: Vec<f32> = cond
.iter()
.zip(&uncond)
.map(|(c, u)| u + 5.0 * (c - u))
.collect();
plain = float.step(&velocity, &plain).unwrap();
let (error, f32_error) = (relative(&emulated, &want), relative(&plain, &want));
eprintln!(
"unipc {mode} step {step}: BF16-emulated {error:.3e}, plain f32 {f32_error:.3e}"
);
worst = worst.max(error);
}
}
assert!(worst < 1e-5, "UniPC replay {worst:.3e} over the 1e-5 gate");
}
fn jittered_run(
pipeline: &mut VideoPipeline,
positive: &Matrix,
negative: &Matrix,
first: Option<&FeatureMap>,
mut noise: Vec<FeatureMap>,
jitter: f32,
seed: u64,
) -> Vec<FeatureMap> {
let mut rng = StdRng::seed_from_u64(seed);
let mut shake = |values: &mut Vec<f32>| {
let rms = (values.iter().map(|v| v * v).sum::<f32>() / values.len() as f32).sqrt();
for v in values.iter_mut() {
*v += jitter * rms * crate::diffusion::gaussian(&mut rng);
}
};
let guidance = pipeline.config.guidance;
let mut scheduler = UniPc::new(6, pipeline.flow_shift, pipeline.train_steps).unwrap();
for step in 0..6 {
let mut input = noise.clone();
let mut times = vec![scheduler.timesteps()[step]; noise.len()];
if let Some(first) = first {
input[0] = first.clone();
times[0] = 0.0;
}
let dit = pipeline.denoiser().unwrap();
let mut cond = flatten(&dit.forward(&input, ×, positive).unwrap());
let mut uncond = flatten(&dit.forward(&input, ×, negative).unwrap());
shake(&mut cond);
shake(&mut uncond);
let velocity: Vec<f32> = cond
.iter()
.zip(&uncond)
.map(|(c, u)| u + guidance * (c - u))
.collect();
let next = scheduler.step(&velocity, &flatten(&noise)).unwrap();
refill(&mut noise, &next);
}
if let Some(first) = first {
noise[0] = first.clone();
}
noise
}
fn shaken(states: &Matrix, jitter: f32, seed: u64) -> Matrix {
let mut rng = StdRng::seed_from_u64(seed);
let mut out = states.clone();
let rows: Vec<_> = out
.data
.chunks_mut(states.cols)
.filter(|row| row.iter().any(|v| *v != 0.0))
.collect();
let count: usize = rows.iter().map(|row| row.len()).sum();
let rms = (rows
.iter()
.flat_map(|row| row.iter())
.map(|v| v * v)
.sum::<f32>()
/ count as f32)
.sqrt();
for row in rows {
for v in row.iter_mut() {
*v += jitter * rms * crate::diffusion::gaussian(&mut rng);
}
}
out
}
fn peak_rss_gib() -> f64 {
std::fs::read_to_string("/proc/self/status")
.ok()
.and_then(|status| {
let line = status.lines().find(|line| line.starts_with("VmHWM:"))?;
line.split_whitespace().nth(1)?.parse::<f64>().ok()
})
.map_or(0.0, |kib| kib / 1048576.0)
}
fn device_used_mib() -> f64 {
#[cfg(feature = "cuda")]
if let Some((free, total)) = crate::cuda_image::memory_info() {
return (total - free) as f64 / 1048576.0;
}
0.0
}
fn save_clip(video: &[FeatureMap], dir: &Path, name: &str) {
use std::io::Write;
let (width, height) = (video[0].width, video[0].height);
let bytes: Vec<u8> = video.iter().flat_map(crate::vae::to_rgb8).collect();
let outputs = [
vec![
"-c:v".to_string(),
"libx264".into(),
"-pix_fmt".into(),
"yuv420p".into(),
"-crf".into(),
"16".into(),
format!("{name}.mp4"),
],
vec![format!("{name}_frame%02d.png")],
];
for output in outputs {
let mut child = std::process::Command::new("ffmpeg")
.args([
"-y",
"-loglevel",
"error",
"-f",
"rawvideo",
"-pix_fmt",
"rgb24",
])
.args(["-s", &format!("{width}x{height}"), "-r", "24", "-i", "-"])
.args(output)
.current_dir(dir)
.stdin(std::process::Stdio::piped())
.spawn()
.expect("ffmpeg");
child.stdin.take().unwrap().write_all(&bytes).unwrap();
assert!(child.wait().unwrap().success(), "ffmpeg failed on {name}");
}
}
#[cfg(feature = "cuda")]
#[test]
#[ignore = "needs Wan2.2-TI2V-5B and RB_WAN_CUDA=1"]
fn wan22_second_large_clip_reuses_the_host_dit() {
use std::time::Instant;
if std::env::var("RB_WAN_CUDA").as_deref() != Ok("1") {
return;
}
let model = PathBuf::from(
std::env::var("RB_WAN_MODEL")
.unwrap_or_else(|_| "/mnt/ollama/RustingFrameModels/Wan2.2-TI2V-5B".into()),
);
let mut pipeline = VideoPipeline::load(&model).unwrap();
pipeline.encoder = None;
pipeline.config.steps = 2;
assert!(pipeline.try_cuda(0));
let request = VideoRequest {
width: 1280,
height: 704,
frames: 17,
seed: Some(0),
image: None,
};
let text = Matrix::new(16, pipeline.dit_config.text_dim);
for run in 0..2 {
let started = Instant::now();
let latent = pipeline
.sample_from_noise(
&text,
None,
None,
pipeline.noise(&request).unwrap(),
|_, _| true,
)
.unwrap();
let sampled = started.elapsed();
pipeline.decode(&latent).unwrap();
eprintln!(
"wan clip {run}: sample {sampled:?}, total {:?}",
started.elapsed()
);
let on_device = pipeline.denoiser.as_ref().is_some_and(WanDit::on_device);
eprintln!("wan clip {run}: DiT on device after decode: {on_device}");
if run == 0 {
assert!(
!on_device,
"make_room did not fire; the test proves nothing"
);
}
let started = Instant::now();
pipeline.denoiser().unwrap();
eprintln!("wan clip {run}: reattach {:?}", started.elapsed());
if let Some((free, _)) = crate::cuda_image::memory_info() {
eprintln!(
"wan clip {run}: free with the DiT on device {} MiB",
free >> 20
);
}
}
}
#[cfg(feature = "cuda")]
#[test]
#[ignore = "needs FastWan2.2-TI2V-5B, Wan2.2-TI2V-5B and RB_WAN_CUDA=1"]
fn fastwan_dmd_three_steps() {
use std::time::Instant;
if std::env::var("RB_WAN_CUDA").as_deref() != Ok("1") {
return;
}
let model = PathBuf::from(
std::env::var("RB_WAN_MODEL")
.unwrap_or_else(|_| "/mnt/ollama/RustingFrameModels/Wan2.2-TI2V-5B".into()),
);
let fast = PathBuf::from(std::env::var("RB_FASTWAN").unwrap_or_else(|_| {
"/mnt/ollama/RustingFrameModels/FastWan2.2-TI2V-5B-FullAttn-Diffusers".into()
}));
let dir = PathBuf::from("/tmp/ws5");
std::fs::create_dir_all(&dir).unwrap();
let mut pipeline = VideoPipeline::load(&model).unwrap();
pipeline.set_dit_weights(fast.join("transformer/diffusion_pytorch_model.safetensors"));
pipeline.config.guidance = 1.0;
pipeline.config.dmd = Some(Dmd {
timesteps: vec![1000, 757, 522],
seed: 42,
});
let text = pipeline
.condition(
"A red fox trotting through fresh snow in a pine forest at golden hour, \
cinematic, shallow depth of field",
)
.unwrap();
pipeline.encoder = None;
assert!(pipeline.try_cuda(0));
for frames in [49, 17] {
let request = VideoRequest {
width: 1280,
height: 704,
frames,
seed: Some(42),
image: None,
};
let started = Instant::now();
let latent = pipeline
.sample_from_noise(
&text,
None,
None,
pipeline.noise(&request).unwrap(),
|_, _| true,
)
.unwrap();
let sampled = started.elapsed();
let decoding = Instant::now();
let clip = pipeline.decode(&latent).unwrap();
eprintln!(
"fastwan 1280x704x{frames}: sample {sampled:?}, decode {:?}, total {:?}",
decoding.elapsed(),
started.elapsed()
);
let name = format!("fastwan_{frames}");
save_clip(&clip, &dir, &name);
let picks: Vec<usize> = (0..5).map(|k| k * (frames - 1) / 4).collect();
let mut command = std::process::Command::new("ffmpeg");
command.args(["-y", "-loglevel", "error"]).current_dir(&dir);
for pick in &picks {
command
.arg("-i")
.arg(format!("{name}_frame{:02}.png", pick + 1));
}
let scaled: String = (0..5)
.map(|k| format!("[{k}]scale=640:-1[s{k}];"))
.collect();
let stacked: String = (0..5).map(|k| format!("[s{k}]")).collect();
command
.args([
"-filter_complex",
&format!("{scaled}{stacked}hstack=inputs=5"),
])
.arg(if frames == 49 {
"tiles.png".to_string()
} else {
format!("tiles{frames}.png")
});
assert!(command.status().unwrap().success(), "tile failed");
}
}
#[test]
fn step_skip_keeps_the_head_and_the_last_step() {
let skip = StepSkip { full: 4, every: 2 };
let skipped: Vec<usize> = (0..20).filter(|&s| skip.skips(s, 20)).collect();
assert_eq!(skipped, [5, 7, 9, 11, 13, 15, 17]);
assert!(!StepSkip { full: 0, every: 0 }.skips(3, 20));
}
#[cfg(all(feature = "cuda", feature = "images"))]
#[test]
#[ignore = "needs Wan2.2-TI2V-5B, a 1280x704 image and RB_WAN_CUDA=1"]
fn wan22_image_to_video_step_skip_sweep() {
use std::time::Instant;
let env = |name: &str| std::env::var(name).ok();
let Some(image_path) =
env("RB_WS3_IMAGE").filter(|_| env("RB_WAN_CUDA").as_deref() == Some("1"))
else {
eprintln!("skipped: needs RB_WAN_CUDA=1 and RB_WS3_IMAGE");
return;
};
let model = PathBuf::from(
env("RB_WAN_MODEL")
.unwrap_or_else(|| "/mnt/ollama/RustingFrameModels/Wan2.2-TI2V-5B".into()),
);
let frames: usize = env("RB_WS3_FRAMES").map_or(17, |v| v.parse().unwrap());
let runs: usize = env("RB_WS3_RUNS").map_or(1, |v| v.parse().unwrap());
let dir = PathBuf::from("/tmp/ws3");
std::fs::create_dir_all(&dir).unwrap();
let picture = image::open(image_path).unwrap().to_rgb8();
let (width, height) = (picture.width() as usize, picture.height() as usize);
let mut planes = vec![0f32; 3 * width * height];
for (i, pixel) in picture.pixels().enumerate() {
for c in 0..3 {
planes[c * width * height + i] = pixel[c] as f32 / 127.5 - 1.0;
}
}
let image = FeatureMap::from_vec(3, height, width, planes).unwrap();
let mut pipeline = VideoPipeline::load(&model).unwrap();
let text = pipeline
.condition("A cinematic shot of the scene coming to life, gentle camera push in, natural motion")
.unwrap();
let negative = pipeline.condition("").unwrap();
pipeline.encoder = None;
assert!(pipeline.try_cuda(0));
let request = VideoRequest {
width,
height,
frames,
seed: Some(7),
image: Some(&image),
};
let first = pipeline.encode_image(&image).unwrap();
let spec = env("RB_WS3_VARIANTS").unwrap_or_else(|| "base:20".into());
let mut variants: Vec<&str> = spec.split_whitespace().collect();
let base = variants
.iter()
.position(|v| v.starts_with("base:20"))
.map_or("base:20", |i| variants.remove(i));
variants.insert(0, base);
pipeline.config.steps = 2;
pipeline
.sample_from_noise(
&text,
Some(&negative),
Some(&first),
pipeline.noise(&request).unwrap(),
|_, _| true,
)
.unwrap();
let mut reference: Vec<f32> = Vec::new();
let mut pending = Vec::new();
for variant in variants {
let parts: Vec<&str> = variant.split(':').collect();
let number = |i: usize| parts.get(i).map_or(0, |v| v.parse::<usize>().unwrap());
pipeline.config.steps = number(1);
pipeline.config.skip = (number(3) > 0).then(|| StepSkip {
full: number(2),
every: number(3),
});
let steps = pipeline.config.steps;
let calls: usize = (0..steps)
.map(|step| match pipeline.config.skip {
Some(skip) if skip.skips(step, steps) => 0,
_ => 2,
})
.sum();
let mut times = Vec::new();
let mut latent = Vec::new();
let runs = parts.get(4).map_or(runs, |v| v.parse().unwrap());
for _ in 0..runs {
let started = Instant::now();
latent = pipeline
.sample_from_noise(
&text,
Some(&negative),
Some(&first),
pipeline.noise(&request).unwrap(),
|_, _| true,
)
.unwrap();
times.push(started.elapsed().as_secs_f64());
}
let flat: Vec<f32> = latent[1..].iter().flat_map(|m| m.data.clone()).collect();
if reference.is_empty() {
reference = flat.clone();
}
let (mut diff, mut norm) = (0f64, 0f64);
for (a, b) in flat.iter().zip(&reference) {
diff += ((a - b) as f64).powi(2);
norm += (*b as f64).powi(2);
}
let mean = times.iter().sum::<f64>() / runs as f64;
let spread = times.iter().cloned().fold(f64::MIN, f64::max)
- times.iter().cloned().fold(f64::MAX, f64::min);
let load = std::fs::read_to_string("/proc/loadavg").unwrap_or_default();
eprintln!(
"WS3 {variant} dit_calls {calls} sample_mean {mean:.1}s spread {spread:.1}s rel_l2 {:.4} load {}",
(diff / norm).sqrt(),
load.split_whitespace()
.take(3)
.collect::<Vec<_>>()
.join(" ")
);
if env("RB_WS3_DECODE").as_deref() == Some("1") {
pending.push((variant.replace(':', "_"), latent));
}
}
for (name, latent) in pending {
let clip = pipeline.decode(&latent).unwrap();
save_tiles(&clip, &dir.join(format!("{name}_tiles.png")));
}
}
#[cfg(all(feature = "cuda", feature = "images"))]
fn save_tiles(clip: &[FeatureMap], path: &Path) {
use std::io::Write;
let (width, height, frames) = (clip[0].width, clip[0].height, clip.len());
let mut child = std::process::Command::new("ffmpeg")
.args([
"-y",
"-loglevel",
"error",
"-f",
"rawvideo",
"-pix_fmt",
"rgb24",
])
.args(["-s", &format!("{width}x{height}"), "-i", "-"])
.args(["-vf", "scale=640:-1,tile=5x1", "-frames:v", "1"])
.arg(path)
.stdin(std::process::Stdio::piped())
.spawn()
.expect("ffmpeg");
let mut stdin = child.stdin.take().unwrap();
for k in 0..5 {
stdin
.write_all(&crate::vae::to_rgb8(&clip[k * (frames - 1) / 4]))
.unwrap();
}
drop(stdin);
assert!(child.wait().unwrap().success(), "tile failed");
}
#[cfg(all(feature = "cuda", feature = "images"))]
#[test]
#[ignore = "needs Wan2.2-TI2V-5B, RB_WAN_CUDA=1 and RB_WS9D_REF"]
fn wan22_ti2v_long_clip_replays_diffusers() {
use crate::wan_vae::tests::{frames, relative};
let env = |name: &str| std::env::var(name).ok();
let Some(dir) = env("RB_WS9D_REF").filter(|_| env("RB_WAN_CUDA").as_deref() == Some("1"))
else {
eprintln!("skipped: needs RB_WAN_CUDA=1 and RB_WS9D_REF");
return;
};
let dir = PathBuf::from(dir);
let model = PathBuf::from(
env("RB_WAN_MODEL")
.unwrap_or_else(|| "/mnt/ollama/RustingFrameModels/Wan2.2-TI2V-5B".into()),
);
let size = env("RB_WS9D_SIZE").unwrap_or_else(|| "352x640x121".into());
let mut reference =
crate::safetensors::SafeTensors::open(dir.join(format!("ti2v_{size}.safetensors")))
.unwrap();
let mut read = |name: &str| reference.tensor(name).unwrap();
let matrix =
|(data, shape): (Vec<f32>, Vec<usize>)| Matrix::from_vec(shape[0], shape[1], data);
let positive = matrix(read("prompt_embeds"));
let negative = matrix(read("negative_prompt_embeds"));
let (noise, noise_shape) = read("noise");
let steps = read("timesteps").0.len();
let (input, shape) = read("step0.dit_in");
let first = frames(&input, &shape).remove(0);
let per_frame = |got: &[FeatureMap], want: &[FeatureMap]| -> Vec<f32> {
got.iter()
.zip(want)
.map(|(g, w)| relative(&g.data, &w.data))
.collect()
};
let show = |errors: &[f32]| {
errors
.iter()
.map(|e| format!("{e:.2e}"))
.collect::<Vec<_>>()
.join(" ")
};
let mut pipeline = VideoPipeline::load(&model).unwrap();
pipeline.encoder = None;
pipeline.config.steps = steps;
pipeline.config.guidance = 5.0;
pipeline.config.skip = None;
assert!(pipeline.try_cuda(0));
let sigmas = UniPc::new(steps, pipeline.flow_shift, pipeline.train_steps).unwrap();
for (a, b) in sigmas.sigmas().iter().zip(&read("sigmas").0) {
assert!((a - b).abs() < 1e-6, "sigma {a} vs {b}");
}
let mut unipc = UniPc::new(steps, pipeline.flow_shift, pipeline.train_steps).unwrap();
let mut sample = flatten(&frames(&noise, &noise_shape));
let forced: usize = env("RB_WS9D_FORCED").map_or(steps, |v| v.parse().unwrap());
for step in 0..forced.min(steps) {
let (input, shape) = read(&format!("step{step}.dit_in"));
let input = frames(&input, &shape);
let per_token = read(&format!("step{step}.dit_t")).0;
let times: Vec<f32> = per_token
.iter()
.step_by(per_token.len() / input.len())
.copied()
.collect();
let dit = pipeline.denoiser().unwrap();
let cond = dit.forward(&input, ×, &positive).unwrap();
let uncond = dit.forward(&input, ×, &negative).unwrap();
let (data, shape) = read(&format!("step{step}.cond"));
let want_cond = frames(&data, &shape);
let (data, shape) = read(&format!("step{step}.uncond"));
let want_uncond = frames(&data, &shape);
let guided: Vec<f32> = flatten(&cond)
.iter()
.zip(flatten(&uncond))
.map(|(c, u)| u + 5.0 * (c - u))
.collect();
let next = unipc.step(&guided, &sample).unwrap();
let mut ours = frames(&noise, &noise_shape);
refill(&mut ours, &next);
let (data, shape) = read(&format!("step{step}.latent"));
let want = frames(&data, &shape);
eprintln!(
"ws9d step {step} t={}: cond {:.3e} uncond {:.3e} latent {:.3e}\n cond per frame {}\n latent per frame {}",
times[times.len() - 1],
relative(&flatten(&cond), &flatten(&want_cond)),
relative(&flatten(&uncond), &flatten(&want_uncond)),
relative(&next, &flatten(&want)),
show(&per_frame(&cond, &want_cond)),
show(&per_frame(&ours, &want)),
);
sample = flatten(&want);
}
let latent = pipeline
.sample_from_noise(
&positive,
Some(&negative),
Some(&first),
frames(&noise, &noise_shape),
|_, _| true,
)
.unwrap();
let (data, shape) = read("final_latent");
let want = frames(&data, &shape);
eprintln!(
"ws9d free-running final latent {:.3e}; per frame {}",
relative(&flatten(&latent), &flatten(&want)),
show(&per_frame(&latent, &want))
);
let clip = pipeline.decode(&latent).unwrap();
save_tiles(&clip, &dir.join(format!("rb_ti2v_{size}_tiles.png")));
let clip = pipeline.decode(&want).unwrap();
save_tiles(
&clip,
&dir.join(format!("rb_decode_of_diffusers_{size}_tiles.png")),
);
}
#[test]
#[ignore = "needs Wan2.2-TI2V-5B and the output of RustingFrame's tools/wan_diffusers_bench.py w4"]
fn wan22_video_pipeline_matches_diffusers() {
use crate::wan_vae::tests::{frames, relative, stack};
use std::time::Instant;
let env = |name: &str| std::env::var(name).ok();
let dir = PathBuf::from(env("RB_WAN_REF").expect("RB_WAN_REF"));
let model = PathBuf::from(
env("RB_WAN_MODEL")
.unwrap_or_else(|| "/mnt/ollama/RustingFrameModels/Wan2.2-TI2V-5B".into()),
);
let cuda = env("RB_WAN_CUDA").as_deref() == Some("1");
let meta = read_json(&dir.join("reference.json")).unwrap();
let prompts = &meta["w4"]["prompts"];
let mut reference =
crate::safetensors::SafeTensors::open(dir.join("w4.safetensors")).unwrap();
let mut read = |name: &str| reference.tensor(name).unwrap();
let mut report = Vec::new();
let mut gate = |stage: &str, error: f32, limit: f32| {
eprintln!("wan video {stage}: relative L2 {error:.3e} (gate {limit:.2e})");
report.push((stage.to_string(), error, limit));
};
let base_mib = device_used_mib();
let mut peak_mib = base_mib;
let started = Instant::now();
let mut pipeline = VideoPipeline::load(&model).unwrap();
eprintln!(
"wan video load (umT5 Q8, tokenizer, VAE): {:?}",
started.elapsed()
);
pipeline.config.steps = 6;
let embeds = |data: Vec<f32>, shape: Vec<usize>| Matrix::from_vec(shape[0], shape[1], data);
let mut ours = Vec::new();
let mut state_errors = Vec::new();
for (key, name) in [
("t2v", "t2v.prompt_embeds"),
("ti2v", "ti2v.prompt_embeds"),
("negative", "t2v.negative_prompt_embeds"),
] {
let started = Instant::now();
let states = pipeline.condition(prompts[key].as_str().unwrap()).unwrap();
eprintln!("wan video umT5 {key}: {:?}", started.elapsed());
let want = read(name).0;
let real = |data: &[f32]| {
data.chunks(states.cols)
.filter(|row| row.iter().any(|v| *v != 0.0))
.count()
};
assert_eq!(real(&states.data), real(&want), "umT5 {key} token count");
let error = relative(&states.data, &want);
gate(&format!("umT5 {key}"), error, 1.2e-1);
state_errors.push(error);
ours.push(states);
}
let [our_negative, our_ti2v, our_t2v] =
[ours.pop(), ours.pop(), ours.pop()].map(Option::unwrap);
pipeline.encoder = None;
let (image, shape) = read("ti2v.image");
let image = FeatureMap::from_vec(shape[0], shape[1], shape[2], image).unwrap();
let started = Instant::now();
let our_first = pipeline.encode_image(&image).unwrap();
eprintln!(
"wan video first-frame encode (host): {:?}",
started.elapsed()
);
let (condition, shape) = read("ti2v.condition");
let ref_first = frames(&condition, &shape).remove(0);
gate(
"ti2v first-frame latent",
relative(&our_first.data, &ref_first.data),
5e-2,
);
if cuda {
let started = Instant::now();
assert!(
pipeline.try_cuda(0),
"RB_WAN_CUDA=1 needs the cuda feature and a device"
);
eprintln!(
"wan video DiT load + upload, decoder upload: {:?}",
started.elapsed()
);
peak_mib = peak_mib.max(device_used_mib());
} else {
pipeline.denoiser().unwrap();
}
eprintln!("wan video peak RSS after load: {:.2} GiB", peak_rss_gib());
for mode in ["t2v", "ti2v"] {
let (data, shape) = read(&format!("{mode}.prompt_embeds"));
let positive = embeds(data, shape);
let (data, shape) = read(&format!("{mode}.negative_prompt_embeds"));
let negative = embeds(data, shape);
let first = (mode == "ti2v").then_some(&ref_first);
let mut unipc = UniPc::new(6, 5.0, 1000).unwrap();
let mut call_errors = Vec::new();
let mut sample = read(&format!("{mode}.noise")).0;
for step in 0..6 {
let (input, shape) = read(&format!("{mode}.step{step}.dit_in"));
let input = frames(&input, &shape);
let per_token = read(&format!("{mode}.step{step}.dit_t")).0;
let per_frame = per_token.len() / input.len();
let times: Vec<f32> = per_token.iter().step_by(per_frame).copied().collect();
let dit = pipeline.denoiser().unwrap();
let started = Instant::now();
let cond = stack(&dit.forward(&input, ×, &positive).unwrap());
let uncond = stack(&dit.forward(&input, ×, &negative).unwrap());
eprintln!("wan video {mode} DiT call: {:?}", started.elapsed() / 2);
peak_mib = peak_mib.max(device_used_mib());
let want_cond = read(&format!("{mode}.step{step}.cond")).0;
let want_uncond = read(&format!("{mode}.step{step}.uncond")).0;
call_errors.push(relative(&cond, &want_cond));
call_errors.push(relative(&uncond, &want_uncond));
gate(
&format!("{mode} step {step} cond"),
relative(&cond, &want_cond),
8e-2,
);
gate(
&format!("{mode} step {step} uncond"),
relative(&uncond, &want_uncond),
8e-2,
);
let guided: Vec<f32> = cond
.iter()
.zip(&uncond)
.map(|(c, u)| u + 5.0 * (c - u))
.collect();
let want_guided: Vec<f32> = want_cond
.iter()
.zip(&want_uncond)
.map(|(c, u)| u + 5.0 * (c - u))
.collect();
eprintln!(
"wan video {mode} step {step} guided velocity: relative L2 {:.3e}",
relative(&guided, &want_guided)
);
let next = unipc.step(&guided, &sample).unwrap();
let want = read(&format!("{mode}.step{step}.latent")).0;
gate(
&format!("{mode} step {step} latent"),
relative(&next, &want),
1e-1,
);
sample = want;
}
let (noise, noise_shape) = read(&format!("{mode}.noise"));
let started = Instant::now();
let latent = pipeline
.sample_from_noise(
&positive,
Some(&negative),
first,
frames(&noise, &noise_shape),
|_, _| true,
)
.unwrap();
eprintln!("wan video {mode} 6 steps: {:?}", started.elapsed());
let (final_latent, shape) = read(&format!("{mode}.final_latent"));
let want_video = read(&format!("{mode}.video")).0;
let started = Instant::now();
let video = stack(&pipeline.decode(&latent).unwrap());
eprintln!("wan video {mode} decode: {:?}", started.elapsed());
peak_mib = peak_mib.max(device_used_mib());
let decoded = stack(&pipeline.decode(&frames(&final_latent, &shape)).unwrap());
gate(
&format!("{mode} decode of the reference latent"),
relative(&decoded, &want_video),
3e-2,
);
let jitter = call_errors.iter().sum::<f32>() / call_errors.len() as f32;
let replay = jittered_run(
&mut pipeline,
&positive,
&negative,
first,
frames(&noise, &noise_shape),
0.0,
0,
);
gate(
&format!("{mode} jitter-free replay of our sampler"),
relative(&flatten(&replay), &flatten(&latent)),
1e-6,
);
let state_jitter = [state_errors[usize::from(mode == "ti2v")], state_errors[2]];
let mut floor = [[0.0f32; 2]; 2];
for seed in 1..=2 {
for (row, [p, n]) in [[0.0, 0.0], state_jitter].into_iter().enumerate() {
let moved = jittered_run(
&mut pipeline,
&shaken(&positive, p, seed + 10),
&shaken(&negative, n, seed + 20),
first,
frames(&noise, &noise_shape),
jitter,
seed,
);
let moved_video = stack(&pipeline.decode(&moved).unwrap());
let errors = [
relative(&stack(&moved), &stack(&latent)),
relative(&moved_video, &video),
];
eprintln!(
"wan video {mode} noise floor, per-call {jitter:.2e}, states \
{p:.2e}/{n:.2e}, seed {seed}: latent {:.3e}, video {:.3e}",
errors[0], errors[1]
);
for (most, error) in floor[row].iter_mut().zip(errors) {
*most = most.max(error);
}
}
}
gate(
&format!("{mode} final latent"),
relative(&stack(&latent), &final_latent),
1.5 * floor[0][0],
);
gate(
&format!("{mode} video"),
relative(&video, &want_video),
1.5 * floor[0][1],
);
let (ours_positive, ours_first) = match mode {
"t2v" => (&our_t2v, None),
_ => (&our_ti2v, Some(&our_first)),
};
let latent = pipeline
.sample_from_noise(
ours_positive,
Some(&our_negative),
ours_first,
frames(&noise, &noise_shape),
|_, _| true,
)
.unwrap();
gate(
&format!("{mode} final latent, our states"),
relative(&stack(&latent), &final_latent),
1.5 * floor[1][0],
);
let clip = pipeline.decode(&latent).unwrap();
gate(
&format!("{mode} video, our states"),
relative(&stack(&clip), &want_video),
1.5 * floor[1][1],
);
save_clip(&clip, &dir, &format!("rb_w4_{mode}"));
let request = VideoRequest {
width: 256,
height: 256,
frames: 5,
seed: Some(0),
image: (mode == "ti2v").then_some(&image),
};
let started = Instant::now();
let mut marks = Vec::new();
let video = pipeline
.generate_conditioned(ours_positive, Some(&our_negative), &request, |_, _| {
marks.push(started.elapsed());
true
})
.unwrap();
eprintln!(
"wan video {mode} whole 256x256x5 clip: {:?} (steps end at {marks:?})",
started.elapsed()
);
assert_eq!(video.frames.len(), 5);
assert_eq!(video.frames[0].len(), 256 * 256 * 3);
}
if let Some(size) = env("RB_WAN_BENCH") {
let [width, height, count] = size
.split('x')
.map(|part| part.parse().unwrap())
.collect::<Vec<usize>>()[..]
else {
panic!("RB_WAN_BENCH is WxHxF")
};
let request = VideoRequest {
width,
height,
frames: count,
seed: Some(0),
image: None,
};
let started = Instant::now();
let mut marks = Vec::new();
let latent = pipeline
.sample_from_noise(
&our_t2v,
Some(&our_negative),
None,
pipeline.noise(&request).unwrap(),
|_, _| {
marks.push(started.elapsed());
true
},
)
.unwrap();
peak_mib = peak_mib.max(device_used_mib());
eprintln!("wan video {size}: 6 steps end at {marks:?}");
let started = Instant::now();
let clip = pipeline.decode(&latent).unwrap();
peak_mib = peak_mib.max(device_used_mib());
eprintln!(
"wan video {size} decode: {:?}, DiT on device: {}",
started.elapsed(),
pipeline.denoiser.as_ref().is_some_and(WanDit::on_device)
);
save_clip(&clip, &dir, &format!("rb_w4_t2v_{size}"));
}
eprintln!(
"wan video peak RSS {:.2} GiB, device memory in use at most {peak_mib:.0} MiB \
({base_mib:.0} MiB before the run)",
peak_rss_gib()
);
let failures: Vec<_> = report
.iter()
.filter(|(_, error, limit)| error.is_nan() || error > limit)
.collect();
assert!(failures.is_empty(), "over the gate: {failures:?}");
}
}