use crate::conv::{Dense, FeatureMap};
use crate::cuda_flash::compile_ptx;
use crate::cuda_training::{cfg, cuda_err};
use crate::gpu_model::{Act, FlashShape, Gpu};
use crate::gpu_transformer::GpuContext;
use crate::matrix::Matrix;
use crate::network::NetworkError;
use crate::wan_dit::{WanBlock, WanDit, rope};
use cudarc::driver::{CudaFunction, CudaSlice, CudaView, CudaViewMut, LaunchConfig, PushKernelArg};
use cudarc::nvrtc::Ptx;
use std::sync::{Arc, OnceLock};
const KERNELS: &str = r#"
#define ROW_THREADS 256
// The RMS norm over the `cols` values of each row at `x + r * stride + off`,
// times `weight`, in place, then with `rope` the rotation of each adjacent
// pair by the row's angle for the pair's index within its head. Tables are
// `[rows, head_dim / 2]`. One block per row.
extern "C" __global__ void rms_rope(float*x,const float*weight,const float*cs,const float*sn,int rows,int cols,int stride,int off,int head_dim,int rope,float eps){
__shared__ float red[ROW_THREADS];int tid=threadIdx.x,half=head_dim/2;
for(int r=blockIdx.x;r<rows;r+=gridDim.x){
float2*v=(float2*)(x+(size_t)r*stride+off);const float2*w=(const float2*)weight;
float a=0.f;for(int p=tid;p<cols/2;p+=ROW_THREADS){float2 e=v[p];a+=e.x*e.x+e.y*e.y;}
red[tid]=a;__syncthreads();
for(int s=ROW_THREADS/2;s>0;s>>=1){if(tid<s)red[tid]+=red[tid+s];__syncthreads();}
float inv=rsqrtf(red[0]/cols+eps);__syncthreads();
const float*c=cs+(size_t)r*half;const float*n=sn+(size_t)r*half;
for(int p=tid;p<cols/2;p+=ROW_THREADS){
float2 e=v[p],g=w[p];float a=e.x*inv*g.x,b=e.y*inv*g.y;
if(rope){int q=p%half;v[p]=make_float2(a*c[q]-b*n[q],a*n[q]+b*c[q]);}
else v[p]=make_float2(a,b);
}
}
}
"#;
fn ptx() -> Result<&'static Ptx, NetworkError> {
static PTX: OnceLock<Result<Ptx, String>> = OnceLock::new();
PTX.get_or_init(|| compile_ptx(KERNELS).map_err(|error| error.to_string()))
.as_ref()
.map_err(|error| NetworkError::Cuda(format!("Wan kernel compilation failed: {error}")))
}
const CHUNK_ABOVE: usize = 12288;
const CHUNK_ROWS: usize = 4096;
struct Linear {
weight: Act,
bias: CudaSlice<f32>,
units: usize,
inner: usize,
}
struct DeviceBlock {
qkv: Linear,
query_norm: CudaSlice<f32>,
key_norm: CudaSlice<f32>,
output: Linear,
cross_query: Linear,
cross_kv: Linear,
cross_query_norm: CudaSlice<f32>,
cross_key_norm: CudaSlice<f32>,
cross_output: Linear,
norm2: CudaSlice<f32>,
ffn_in: Linear,
ffn_out: Linear,
}
pub struct DeviceWanDit {
context: Arc<GpuContext>,
layer_norm_modulate: CudaFunction,
bias_gelu: CudaFunction,
rms_rope: CudaFunction,
patch: Linear,
text_in: Linear,
text_out: Linear,
proj_out: Linear,
blocks: Vec<DeviceBlock>,
tables: Vec<f32>,
chunk_above: usize,
chunk_rows: usize,
}
impl std::fmt::Debug for DeviceWanDit {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DeviceWanDit").finish_non_exhaustive()
}
}
fn invalid(what: String) -> NetworkError {
NetworkError::InvalidConfig(format!("Wan transformer upload: {what}"))
}
fn check_dense(name: &str, layer: &Dense, units: usize, inner: usize) -> Result<(), NetworkError> {
let bias = layer.bias.as_ref().map(Vec::len);
if layer.out_dim() != units || layer.in_dim() != inner || bias != Some(units) {
return Err(invalid(format!(
"{name} is [{}, {}] with a bias of {bias:?}, not [{units}, {inner}] with {units}",
layer.out_dim(),
layer.in_dim()
)));
}
Ok(())
}
fn check_len(name: &str, values: &[f32], len: usize) -> Result<(), NetworkError> {
if values.len() != len {
return Err(invalid(format!("{name} holds {}, not {len}", values.len())));
}
Ok(())
}
fn validate(model: &WanDit) -> Result<(), NetworkError> {
let config = &model.config;
config.validate()?;
let (dim, ffn) = (config.dim(), config.ffn_dim);
let [pt, ph, pw] = config.patch;
check_dense(
"patch_embedding",
&model.patch,
dim,
config.in_channels * pt * ph * pw,
)?;
check_dense(
"text_embedder.linear_1",
&model.text_in,
dim,
config.text_dim,
)?;
check_dense("text_embedder.linear_2", &model.text_out, dim, dim)?;
check_dense(
"proj_out",
&model.proj_out,
config.out_channels * pt * ph * pw,
dim,
)?;
check_len("scale_shift_table", &model.out_table, 2 * dim)?;
if model.blocks.len() != config.layers {
return Err(invalid(format!(
"{} blocks, not {}",
model.blocks.len(),
config.layers
)));
}
for (index, block) in model.blocks.iter().enumerate() {
let name = |part: &str| format!("blocks.{index}.{part}");
let square = [
("attn1.to_q", &block.query),
("attn1.to_k", &block.key),
("attn1.to_v", &block.value),
("attn1.to_out.0", &block.output),
("attn2.to_q", &block.cross_query),
("attn2.to_k", &block.cross_key),
("attn2.to_v", &block.cross_value),
("attn2.to_out.0", &block.cross_output),
];
for (part, layer) in square {
check_dense(&name(part), layer, dim, dim)?;
}
check_dense(&name("ffn.net.0.proj"), &block.ffn_in, ffn, dim)?;
check_dense(&name("ffn.net.2"), &block.ffn_out, dim, ffn)?;
let vectors = [
("attn1.norm_q", &block.query_norm),
("attn1.norm_k", &block.key_norm),
("attn2.norm_q", &block.cross_query_norm),
("attn2.norm_k", &block.cross_key_norm),
("norm2.weight", &block.norm_weight),
("norm2.bias", &block.norm_bias),
];
for (part, values) in vectors {
check_len(&name(part), values, dim)?;
}
check_len(&name("scale_shift_table"), &block.table, 6 * dim)?;
}
Ok(())
}
impl DeviceWanDit {
pub fn upload(context: &Arc<GpuContext>, model: &WanDit) -> Result<Self, NetworkError> {
validate(model)?;
if context.half_accumulate {
return Err(NetworkError::InvalidConfig(
"the Wan transformer needs a BF16 context, not an FP16 one".into(),
));
}
let head_dim = model.config.head_dim;
if context.flash.is_none() || !crate::cuda_flash::eligible(true, head_dim) {
return Err(NetworkError::InvalidConfig(format!(
"the device Wan transformer needs the fused attention kernel, which takes \
heads of 64 or 128 on Ampere or later; this one has {head_dim}"
)));
}
let gpu = Gpu { context };
let load = |ptx: &Ptx, names: &[&str]| {
let module = context
.stream
.context()
.load_module(ptx.clone())
.map_err(cuda_err("Wan module loading"))?;
names
.iter()
.map(|name| {
module
.load_function(name)
.map_err(cuda_err("Wan kernel lookup"))
})
.collect::<Result<Vec<_>, _>>()
};
let mut mmdit = load(
crate::cuda_mmdit::ptx()?,
&["layer_norm_modulate", "bias_gelu"],
)?;
let rms_rope = load(ptx()?, &["rms_rope"])?.remove(0);
let bias_gelu = mmdit.remove(1);
let layer_norm_modulate = mmdit.remove(0);
let stacked = |layers: &[&Dense]| {
let (mut weights, mut biases) = (Vec::new(), Vec::new());
let (mut q, mut scales) = (Vec::new(), Vec::new());
let int8 = layers.iter().all(|layer| layer.int8_parts().is_some());
for layer in layers {
match layer.int8_parts().filter(|_| int8) {
Some((bytes, row_scales)) => {
q.extend_from_slice(bytes);
scales.extend_from_slice(row_scales);
}
None => weights.extend_from_slice(&layer.weight_f32().data),
}
biases.extend_from_slice(layer.bias.as_deref().expect("checked by validate"));
}
let (units, inner) = (biases.len(), layers[0].in_dim());
let wide = match int8 {
true => gpu.dequantized(&q, &scales, units, inner)?,
false => gpu.upload(&weights)?,
};
Ok::<_, NetworkError>(Linear {
weight: match layers[0].is_quantized() {
true => gpu.quantized(&wide.slice(..), units, inner)?,
false => gpu.narrowed(&wide.slice(..), units * inner, true)?,
},
bias: gpu.upload(&biases)?,
units,
inner,
})
};
let dense = |layer: &Dense| stacked(&[layer]);
let blocks = model
.blocks
.iter()
.map(|block: &WanBlock| {
let norm2: Vec<f32> = block
.norm_bias
.iter()
.copied()
.chain(block.norm_weight.iter().map(|weight| weight - 1.0))
.collect();
Ok(DeviceBlock {
qkv: stacked(&[&block.query, &block.key, &block.value])?,
query_norm: gpu.upload(&block.query_norm)?,
key_norm: gpu.upload(&block.key_norm)?,
output: dense(&block.output)?,
cross_query: dense(&block.cross_query)?,
cross_kv: stacked(&[&block.cross_key, &block.cross_value])?,
cross_query_norm: gpu.upload(&block.cross_query_norm)?,
cross_key_norm: gpu.upload(&block.cross_key_norm)?,
cross_output: dense(&block.cross_output)?,
norm2: gpu.upload(&norm2)?,
ffn_in: dense(&block.ffn_in)?,
ffn_out: dense(&block.ffn_out)?,
})
})
.collect::<Result<_, NetworkError>>()?;
Ok(Self {
layer_norm_modulate,
bias_gelu,
rms_rope,
patch: dense(&model.patch)?,
text_in: dense(&model.text_in)?,
text_out: dense(&model.text_out)?,
proj_out: dense(&model.proj_out)?,
blocks,
tables: model
.blocks
.iter()
.flat_map(|block| block.table.iter().copied())
.collect(),
context: context.clone(),
chunk_above: CHUNK_ABOVE,
chunk_rows: CHUNK_ROWS,
})
}
pub fn forward(
&self,
model: &WanDit,
latent: &[FeatureMap],
timesteps: &[f32],
context: &Matrix,
) -> Result<Vec<FeatureMap>, NetworkError> {
let config = &model.config;
if self.blocks.len() != config.layers || self.patch.units != config.dim() {
return Err(invalid(
"forward with a model other than the one uploaded".into(),
));
}
let grid = model.grid(latent, timesteps, context)?;
let gpu = Gpu {
context: &self.context,
};
let (dim, heads, eps) = (config.dim(), config.num_heads, config.eps);
let frames = grid[0];
let per_frame = grid[1] * grid[2];
let rows = frames * per_frame;
let (temb, projection) = model.time(timesteps, frames)?;
let mut modulation = Vec::with_capacity((self.blocks.len() * 6 + 2) * frames * dim);
for table in self.tables.chunks(6 * dim) {
for half in [0, 3 * dim] {
for frame in projection.data.chunks(6 * dim) {
let range = half..half + 3 * dim;
modulation.extend(
frame[range.clone()]
.iter()
.zip(&table[range])
.map(|(a, b)| a + b),
);
}
}
}
let output_at = modulation.len();
for frame in temb.data.chunks(dim) {
for chunk in model.out_table.chunks(dim) {
modulation.extend(chunk.iter().zip(frame).map(|(a, b)| a + b));
}
}
let modulation = gpu.upload(&modulation)?;
let positions = rope(config, grid);
let (cos, sin) = (gpu.upload(&positions.cos)?, gpu.upload(&positions.sin)?);
let patches = model.patches(latent, grid);
let mut hidden = self.input(&gpu, &self.patch, &patches)?;
let text = self.input(&gpu, &self.text_in, context)?;
let text = self.gelu(&gpu, &text, None, context.rows, dim)?;
let text = self.apply(&gpu, &self.text_out, &text.all(), context.rows)?;
let text = gpu.narrowed(&text.slice(..), context.rows * dim, true)?;
let scale = (config.head_dim as f32).sqrt().recip();
let flash = self.context.flash.as_ref().expect("checked at upload");
let mut merged = gpu.act(rows * dim, true)?;
let mut log_sum_exp = gpu.uninit(heads * rows)?;
let shape = FlashShape {
rows,
seq_len: rows,
kv_len: rows,
sequences: 1,
heads,
group: 1,
qkv_width: 3 * dim,
kv_width: 3 * dim,
query_width: dim,
key_base: dim,
value_base: 2 * dim,
causal: false,
};
let cross = FlashShape {
kv_len: context.rows,
qkv_width: dim,
kv_width: 2 * dim,
key_base: 0,
value_base: dim,
..shape
};
for (index, block) in self.blocks.iter().enumerate() {
let at = index * 6 * frames * dim;
let attention = modulation.slice(at..at + 3 * frames * dim);
let feed_forward = modulation.slice(at + 3 * frames * dim..at + 6 * frames * dim);
let qkv = {
let normed =
self.frame_norm(&gpu, &hidden, &attention, frames, per_frame, 3, dim, eps)?;
let step = if rows > self.chunk_above {
self.chunk_rows
} else {
rows
};
let mut narrow = None;
for start in (0..rows).step_by(step) {
let part = step.min(rows - start);
let mut qkv = self.apply(&gpu, &block.qkv, &normed.at(start * dim), part)?;
let half = config.head_dim / 2;
let (cos, sin) = (cos.slice(start * half..), sin.slice(start * half..));
for (weight, offset) in [(&block.query_norm, 0), (&block.key_norm, dim)] {
self.rms_rope(
&mut qkv,
weight,
Some((&cos, &sin)),
part,
3 * dim,
offset,
config.head_dim,
eps,
)?;
}
if narrow.is_none() {
narrow = Some(gpu.act(rows * 3 * dim, true)?);
}
let narrow = narrow.as_mut().expect("just set");
gpu.cast_into(
&mut narrow.destination().slice_mut(start * 3 * dim * 2..),
&qkv.slice(..),
part * 3 * dim,
true,
)?;
}
narrow.expect("rows > 0")
};
gpu.flash_attention(
flash,
&qkv,
&qkv,
&mut merged,
&mut log_sum_exp,
shape,
scale,
)?;
drop(qkv);
let out = self.apply(&gpu, &block.output, &merged.all(), rows)?;
hidden = gpu.gate_residual(
&hidden.slice(..),
&out.slice(..),
&attention,
rows,
dim,
per_frame,
)?;
drop(out);
let query = {
let normed = self.norm(
&gpu,
&hidden.slice(..),
&block.norm2.slice(..),
rows,
dim,
eps,
)?;
let mut query = self.apply(&gpu, &block.cross_query, &normed.all(), rows)?;
drop(normed);
self.rms_rope(
&mut query,
&block.cross_query_norm,
None,
rows,
dim,
0,
config.head_dim,
eps,
)?;
gpu.narrowed(&query.slice(..), rows * dim, true)?
};
let kv = {
let mut kv = self.apply(&gpu, &block.cross_kv, &text.all(), context.rows)?;
self.rms_rope(
&mut kv,
&block.cross_key_norm,
None,
context.rows,
2 * dim,
0,
config.head_dim,
eps,
)?;
gpu.narrowed(&kv.slice(..), context.rows * 2 * dim, true)?
};
gpu.flash_attention(
flash,
&query,
&kv,
&mut merged,
&mut log_sum_exp,
cross,
scale,
)?;
drop((query, kv));
let out = self.apply(&gpu, &block.cross_output, &merged.all(), rows)?;
gpu.add(&mut hidden, &out, 0, rows * dim, true)?;
drop(out);
let normed =
self.frame_norm(&gpu, &hidden, &feed_forward, frames, per_frame, 3, dim, eps)?;
let units = block.ffn_in.units;
let out = if rows > self.chunk_above {
let mut out = gpu.uninit(rows * dim)?;
for start in (0..rows).step_by(self.chunk_rows) {
let part = self.chunk_rows.min(rows - start);
let pre = self.gemm(&gpu, &block.ffn_in, &normed.at(start * dim), part)?;
let activated = self.gelu(&gpu, &pre, Some(&block.ffn_in.bias), part, units)?;
drop(pre);
let raw = self.gemm(&gpu, &block.ffn_out, &activated.all(), part)?;
let mut into = out.slice_mut(start * dim..);
self.bias(&mut into, &raw, &block.ffn_out, part)?;
}
drop(normed);
out
} else {
let pre = self.gemm(&gpu, &block.ffn_in, &normed.all(), rows)?;
drop(normed);
let activated = self.gelu(&gpu, &pre, Some(&block.ffn_in.bias), rows, units)?;
drop(pre);
self.apply(&gpu, &block.ffn_out, &activated.all(), rows)?
};
hidden = gpu.gate_residual(
&hidden.slice(..),
&out.slice(..),
&feed_forward,
rows,
dim,
per_frame,
)?;
}
let output = modulation.slice(output_at..);
let normed = self.frame_norm(&gpu, &hidden, &output, frames, per_frame, 2, dim, eps)?;
let out = self.apply(&gpu, &self.proj_out, &normed.all(), rows)?;
let out = Matrix::from_vec(rows, self.proj_out.units, gpu.download(&out)?);
Ok(model.unpatchify(&out, grid, config.out_channels))
}
fn input(
&self,
gpu: &Gpu<'_>,
linear: &Linear,
tokens: &Matrix,
) -> Result<CudaSlice<f32>, NetworkError> {
let wide = gpu.upload(&tokens.data)?;
let narrow = gpu.narrowed(&wide.slice(..), tokens.data.len(), true)?;
self.apply(gpu, linear, &narrow.all(), tokens.rows)
}
fn gemm(
&self,
gpu: &Gpu<'_>,
linear: &Linear,
x: &CudaView<'_, u8>,
rows: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let mut raw = gpu.uninit(rows * linear.units)?;
gpu.project(
&linear.weight,
linear.units,
linear.inner,
x,
true,
&mut raw.slice_mut(..),
rows,
0.0,
)?;
Ok(raw)
}
fn apply(
&self,
gpu: &Gpu<'_>,
linear: &Linear,
x: &CudaView<'_, u8>,
rows: usize,
) -> Result<CudaSlice<f32>, NetworkError> {
let raw = self.gemm(gpu, linear, x, rows)?;
let mut out = gpu.uninit(rows * linear.units)?;
self.bias(&mut out.slice_mut(..), &raw, linear, rows)?;
Ok(out)
}
fn bias(
&self,
out: &mut CudaViewMut<'_, f32>,
raw: &CudaSlice<f32>,
linear: &Linear,
rows: usize,
) -> Result<(), NetworkError> {
let units = linear.units;
unsafe {
self.context
.stream
.launch_builder(&self.bias_gelu)
.arg(out)
.arg(raw)
.arg(&linear.bias)
.arg(&(rows as i32))
.arg(&(units as i32))
.arg(&(units as i32))
.arg(&0i32)
.arg(&0i32)
.arg(&1i32)
.arg(&0i32)
.launch(cfg(rows * units))
.map_err(cuda_err("Wan bias kernel"))?;
}
Ok(())
}
fn gelu(
&self,
gpu: &Gpu<'_>,
source: &CudaSlice<f32>,
bias: Option<&CudaSlice<f32>>,
rows: usize,
cols: usize,
) -> Result<Act, NetworkError> {
let mut out = gpu.act(rows * cols, true)?;
unsafe {
self.context
.stream
.launch_builder(&self.bias_gelu)
.arg(out.destination())
.arg(source)
.arg(bias.unwrap_or(source))
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(cols as i32))
.arg(&0i32)
.arg(&1i32)
.arg(&i32::from(bias.is_some()))
.arg(&1i32)
.launch(cfg(rows * cols))
.map_err(cuda_err("Wan GELU kernel"))?;
}
Ok(out)
}
#[allow(clippy::too_many_arguments)]
fn frame_norm(
&self,
gpu: &Gpu<'_>,
hidden: &CudaSlice<f32>,
modulation: &CudaView<'_, f32>,
frames: usize,
per_frame: usize,
chunks: usize,
dim: usize,
eps: f32,
) -> Result<Act, NetworkError> {
let mut normed = gpu.uninit(frames * per_frame * dim)?;
for frame in 0..frames {
let rows = frame * per_frame * dim..(frame + 1) * per_frame * dim;
let mut out = normed.slice_mut(rows.clone());
let shift_scale = modulation.slice(frame * chunks * dim..);
self.launch_norm(
&mut out,
&hidden.slice(rows),
&shift_scale,
per_frame,
dim,
eps,
)?;
}
gpu.narrowed(&normed.slice(..), frames * per_frame * dim, true)
}
fn norm(
&self,
gpu: &Gpu<'_>,
hidden: &CudaView<'_, f32>,
shift_scale: &CudaView<'_, f32>,
rows: usize,
dim: usize,
eps: f32,
) -> Result<Act, NetworkError> {
let mut normed = gpu.uninit(rows * dim)?;
self.launch_norm(
&mut normed.slice_mut(..),
hidden,
shift_scale,
rows,
dim,
eps,
)?;
gpu.narrowed(&normed.slice(..), rows * dim, true)
}
fn launch_norm(
&self,
out: &mut cudarc::driver::CudaViewMut<'_, f32>,
hidden: &CudaView<'_, f32>,
shift_scale: &CudaView<'_, f32>,
rows: usize,
dim: usize,
eps: f32,
) -> Result<(), NetworkError> {
unsafe {
self.context
.stream
.launch_builder(&self.layer_norm_modulate)
.arg(out)
.arg(hidden)
.arg(shift_scale)
.arg(&(rows as i32))
.arg(&(dim as i32))
.arg(&eps)
.launch(row_blocks(rows))
.map_err(cuda_err("Wan layer norm kernel"))?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn rms_rope(
&self,
x: &mut CudaSlice<f32>,
weight: &CudaSlice<f32>,
tables: Option<(&CudaView<'_, f32>, &CudaView<'_, f32>)>,
rows: usize,
stride: usize,
offset: usize,
head_dim: usize,
eps: f32,
) -> Result<(), NetworkError> {
let unused = weight.slice(..);
let (cos, sin) = tables.unwrap_or((&unused, &unused));
let cols = weight.len();
unsafe {
self.context
.stream
.launch_builder(&self.rms_rope)
.arg(x)
.arg(weight)
.arg(cos)
.arg(sin)
.arg(&(rows as i32))
.arg(&(cols as i32))
.arg(&(stride as i32))
.arg(&(offset as i32))
.arg(&(head_dim as i32))
.arg(&i32::from(tables.is_some()))
.arg(&eps)
.launch(row_blocks(rows))
.map_err(cuda_err("Wan RMS norm kernel"))?;
}
Ok(())
}
}
fn row_blocks(rows: usize) -> LaunchConfig {
LaunchConfig {
grid_dim: (rows.clamp(1, 1024) as u32, 1, 1),
block_dim: (256, 1, 1),
shared_mem_bytes: 0,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cuda_training::cuda_doctor;
use crate::transformer::Precision;
use crate::wan_dit::tests::{checkpoint, context, latent, tiny};
fn cuda_or_skip() -> Option<Arc<GpuContext>> {
match cuda_doctor(0, 8192) {
Ok(_) => Some(GpuContext::new(0).expect("the CUDA doctor passed")),
Err(NetworkError::Cuda(message))
if message.contains("NO_DEVICE") || message.contains("no CUDA-capable device") =>
{
crate::cuda_training::skip_without_cuda();
None
}
Err(error) => panic!("CUDA is present but the CUDA doctor failed: {error}"),
}
}
#[test]
fn a_device_wan_transformer_matches_the_host_or_skips_without_device() {
let Some(gpu) = cuda_or_skip() else { return };
let config = tiny();
let path = checkpoint(&config, "rusting_brain_cuda_wan_dit.safetensors");
let (latent, context) = (latent(&config, 3, 6, 8), context(&config, 7));
let timesteps = [0.0, 700.0, 700.0];
let flat = |frames: &[FeatureMap]| {
frames
.iter()
.flat_map(|f| f.data.clone())
.collect::<Vec<_>>()
};
for precision in [Precision::F32, Precision::Q8] {
let host = WanDit::load_at(&path, config.clone(), precision).unwrap();
let want = host.forward(&latent, ×teps, &context).unwrap();
let mut device = WanDit::load_at(&path, config.clone(), precision).unwrap();
device.attach_device(&gpu).unwrap();
let got = device.forward(&latent, ×teps, &context).unwrap();
let error = crate::wan_vae::tests::relative(&flat(&got), &flat(&want));
eprintln!("device wan transformer {precision:?}: relative L2 {error:.3e}");
assert!(error < 2e-2, "{precision:?}: {error}");
let mut kept = WanDit::load_at(&path, config.clone(), precision).unwrap();
kept.attach_device_keeping_host(&gpu).unwrap();
kept.detach_device();
assert!(!kept.on_device());
let on_host = kept.forward(&latent, ×teps, &context).unwrap();
assert_eq!(
flat(&on_host),
flat(&want),
"{precision:?}: host after detach"
);
kept.attach_device_keeping_host(&gpu).unwrap();
assert!(kept.on_device());
let again = kept.forward(&latent, ×teps, &context).unwrap();
assert_eq!(flat(&again), flat(&got), "{precision:?}: reattached");
}
}
#[test]
fn chunked_tokens_match_the_whole_pass_or_skip_without_device() {
let Some(gpu) = cuda_or_skip() else { return };
let config = tiny();
let path = checkpoint(&config, "rusting_brain_cuda_wan_dit_chunks.safetensors");
let (latent, context) = (latent(&config, 3, 6, 8), context(&config, 7));
let timesteps = [0.0, 700.0, 700.0];
let flat = |frames: &[FeatureMap]| {
frames
.iter()
.flat_map(|f| f.data.clone())
.collect::<Vec<_>>()
};
for precision in [Precision::F32, Precision::Q8] {
let model = WanDit::load_at(&path, config.clone(), precision).unwrap();
let mut device = DeviceWanDit::upload(&gpu, &model).unwrap();
let whole = device
.forward(&model, &latent, ×teps, &context)
.unwrap();
(device.chunk_above, device.chunk_rows) = (0, 5);
let parts = device
.forward(&model, &latent, ×teps, &context)
.unwrap();
let error = crate::wan_vae::tests::relative(&flat(&parts), &flat(&whole));
eprintln!(
"device wan transformer {precision:?} in 5-token parts: relative L2 {error:.3e}"
);
assert!(error < 5e-3, "{precision:?}: {error}");
}
}
#[test]
#[ignore = "needs Wan2.2-TI2V-5B and RB_WAN_CUDA=1"]
fn real_chunked_tokens_match_the_whole_pass() {
if std::env::var("RB_WAN_CUDA").as_deref() != Ok("1") {
return;
}
let dir = std::path::PathBuf::from(
std::env::var("RB_WAN_MODEL")
.unwrap_or_else(|_| "/mnt/ollama/RustingFrameModels/Wan2.2-TI2V-5B".into()),
);
let config =
crate::wan_dit::WanDitConfig::from_file(dir.join("transformer/config.json")).unwrap();
let weights = dir.join("transformer/diffusion_pytorch_model.safetensors.index.json");
let model = WanDit::load_at(&weights, config.clone(), Precision::Q8).unwrap();
let gpu = cuda_or_skip().expect("RB_WAN_CUDA=1 needs a device");
let mut device = DeviceWanDit::upload(&gpu, &model).unwrap();
let context = context(&config, 512);
let flat = |frames: &[FeatureMap]| {
frames
.iter()
.flat_map(|f| f.data.clone())
.collect::<Vec<_>>()
};
let list = std::env::var("RB_WAN_CHUNK_FRAMES").unwrap_or_else(|_| "49".into());
for video in list.split(',').map(|f| f.trim().parse::<usize>().unwrap()) {
let frames = (video - 1) / 4 + 1;
let latent = latent(&config, frames, 44, 80);
let mut timesteps = vec![700.0; frames];
timesteps[0] = 0.0;
let mut run = |above, rows| {
(device.chunk_above, device.chunk_rows) = (above, rows);
flat(
&device
.forward(&model, &latent, ×teps, &context)
.unwrap(),
)
};
let tokens = frames * 880;
let (reference, name) = match tokens <= CHUNK_ABOVE {
true => (run(usize::MAX, CHUNK_ROWS), "whole"),
false => (run(0, 1000), "1000-token parts"),
};
let parts = run(0, CHUNK_ROWS);
if tokens <= CHUNK_ABOVE {
let floor = crate::wan_vae::tests::relative(&run(0, 1000), &reference);
eprintln!("real wan 1280x704x{video}: 1000-token parts vs whole {floor:.3e}");
}
let error = crate::wan_vae::tests::relative(&parts, &reference);
let per_frame = parts.len() / frames;
let frame = |f: usize| f * per_frame..(f + 1) * per_frame;
let worst = (0..frames)
.map(|f| crate::wan_vae::tests::relative(&parts[frame(f)], &reference[frame(f)]))
.fold(0.0f32, f32::max);
let rms = |f: usize| {
let values = &parts[frame(f)];
(values.iter().map(|v| v * v).sum::<f32>() / values.len() as f32).sqrt()
};
eprintln!(
"real wan 1280x704x{video} ({tokens} tokens): 4096-token parts vs {name} \
{error:.3e}, worst latent frame {worst:.3e}; output rms frame 1 {:.3}, \
middle {:.3}, last {:.3}",
rms(1),
rms(frames / 2),
rms(frames - 1)
);
assert!(error < 1e-2, "{video} frames: {error}");
}
}
#[test]
#[ignore = "needs Wan2.2-TI2V-5B, RB_WAN_CUDA=1 and RB_WS9D_REF"]
fn real_forward_matches_diffusers_per_latent_frame() {
use crate::wan_vae::tests::{frames, relative};
let env = |name: &str| std::env::var(name).ok();
let Some(dir) = env("RB_WS9D_REF").filter(|_| env("RB_WAN_CUDA").as_deref() == Some("1"))
else {
eprintln!("skipped: needs RB_WAN_CUDA=1 and RB_WS9D_REF");
return;
};
let model_dir = std::path::PathBuf::from(
env("RB_WAN_MODEL")
.unwrap_or_else(|| "/mnt/ollama/RustingFrameModels/Wan2.2-TI2V-5B".into()),
);
let embeds = env("RB_WAN_EMBEDS").unwrap_or_else(|| {
"/home/vasylt/Code/rusting/RustingFrame/tools/ref/wan/embeds.safetensors".into()
});
let (data, shape) = crate::safetensors::SafeTensors::open(embeds)
.unwrap()
.tensor("ti2v")
.unwrap();
let context = Matrix::from_vec(shape[0], shape[1], data);
let config =
crate::wan_dit::WanDitConfig::from_file(model_dir.join("transformer/config.json"))
.unwrap();
let weights = model_dir.join("transformer/diffusion_pytorch_model.safetensors.index.json");
let model = WanDit::load_at(&weights, config.clone(), Precision::Q8).unwrap();
let gpu = cuda_or_skip().expect("RB_WAN_CUDA=1 needs a device");
let device = DeviceWanDit::upload(&gpu, &model).unwrap();
let sizes = env("RB_WS9D_SIZES").unwrap_or_else(|| "352x640x17,352x640x121".into());
let mut worst = 0.0f32;
for size in sizes.split(',').map(str::trim) {
let path = std::path::Path::new(&dir).join(format!("dit1_{size}.safetensors"));
let mut reference = crate::safetensors::SafeTensors::open(path).unwrap();
let (data, shape) = reference.tensor("latent").unwrap();
let latent = frames(&data, &shape);
let (per_token, _) = reference.tensor("timestep").unwrap();
let per_frame = per_token.len() / latent.len();
let timesteps: Vec<f32> = per_token.iter().step_by(per_frame).copied().collect();
for (frame, t) in per_token.chunks(per_frame).zip(×teps) {
assert!(frame.iter().all(|value| value == t));
}
assert_eq!(timesteps[0], 0.0);
let (data, shape) = reference.tensor("output").unwrap();
let want = frames(&data, &shape);
let started = std::time::Instant::now();
let got = device
.forward(&model, &latent, ×teps, &context)
.unwrap();
let seconds = started.elapsed().as_secs_f32();
let per: Vec<f32> = got
.iter()
.zip(&want)
.map(|(got, want)| relative(&got.data, &want.data))
.collect();
let flat =
|maps: &[FeatureMap]| maps.iter().flat_map(|m| m.data.clone()).collect::<Vec<_>>();
let all = relative(&flat(&got), &flat(&want));
worst = per.iter().copied().fold(worst, f32::max);
eprintln!(
"ws9d {size} ({} latent frames, t={}, {seconds:.1} s): rel_l2 {all:.3e}; per frame {}",
latent.len(),
timesteps[timesteps.len() - 1],
per.iter()
.map(|e| format!("{e:.3e}"))
.collect::<Vec<_>>()
.join(" ")
);
}
assert!(worst < 5e-2, "worst latent frame {worst}");
}
#[test]
fn malformed_shapes_are_refused_before_upload_or_skip_without_device() {
let Some(gpu) = cuda_or_skip() else { return };
let config = tiny();
let path = checkpoint(&config, "rusting_brain_cuda_wan_dit_bad.safetensors");
let refused = |model: &WanDit| {
assert!(matches!(
DeviceWanDit::upload(&gpu, model),
Err(NetworkError::InvalidConfig(_))
))
};
let load = || WanDit::load_at(&path, config.clone(), Precision::F32).unwrap();
let mut model = load();
model.blocks[1].key_norm.pop();
refused(&model);
let mut model = load();
model.blocks.pop();
refused(&model);
let mut model = load();
model.blocks[0].ffn_out = Dense::new(Matrix::new(128, 255), Some(vec![0.0; 128])).unwrap();
refused(&model);
let mut model = load();
model.proj_out.bias = None;
refused(&model);
let mut model = load();
model.config.head_dim = 32;
model.config.num_heads = 4;
refused(&model);
let mut model = load();
model.attach_device(&gpu).unwrap();
let context = context(&config, 3);
assert!(matches!(
model.forward(&latent(&config, 2, 5, 6), &[1.0], &context),
Err(NetworkError::InvalidConfig(_))
));
}
}