use crate::matrix::Matrix;
use crate::network::NetworkError;
use rand::{Rng, SeedableRng, rngs::StdRng};
pub trait Denoiser {
fn denoise(&mut self, latents: &Matrix, sigma: f32) -> Result<Matrix, NetworkError>;
fn denoise_unconditional(
&mut self,
_latents: &Matrix,
_sigma: f32,
) -> Result<Option<Matrix>, NetworkError> {
Ok(None)
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Scheduler {
FlowMatch { shift: f32 },
Ddim {
beta_start: f32,
beta_end: f32,
train_steps: usize,
},
}
impl Scheduler {
pub fn flow_match(shift: f32) -> Self {
Self::FlowMatch { shift }
}
pub fn ddim() -> Self {
Self::Ddim {
beta_start: 0.00085,
beta_end: 0.012,
train_steps: 1000,
}
}
pub fn sigmas(&self, steps: usize) -> Result<Vec<f32>, NetworkError> {
if steps == 0 {
return Err(NetworkError::InvalidConfig(
"a sampling run of no steps produces noise".into(),
));
}
Ok(match *self {
Self::FlowMatch { shift } => (0..=steps)
.map(|step| {
let level = 1.0 - step as f32 / steps as f32;
shift * level / (1.0 + (shift - 1.0) * level)
})
.collect(),
Self::Ddim {
beta_start,
beta_end,
train_steps,
} => {
let alphas = alphas_cumprod(beta_start, beta_end, train_steps);
let mut sigmas: Vec<f32> = (0..steps)
.map(|step| {
let index = train_steps - 1 - step * train_steps / steps;
let alpha = alphas[index];
((1.0 - alpha) / alpha).sqrt()
})
.collect();
sigmas.push(0.0);
sigmas
}
})
}
pub fn step(
&self,
latents: &mut Matrix,
prediction: &Matrix,
sigma: f32,
next_sigma: f32,
) -> Result<(), NetworkError> {
if latents.data.len() != prediction.data.len() {
return Err(NetworkError::InvalidTarget {
expected: latents.data.len(),
actual: prediction.data.len(),
});
}
match self {
Self::FlowMatch { .. } => {
let delta = next_sigma - sigma;
for (latent, velocity) in latents.data.iter_mut().zip(&prediction.data) {
*latent += delta * velocity;
}
}
Self::Ddim { .. } => {
for (latent, noise) in latents.data.iter_mut().zip(&prediction.data) {
let clean = *latent - sigma * noise;
*latent = clean + next_sigma * noise;
}
}
}
Ok(())
}
pub fn add_noise(
&self,
latents: &mut Matrix,
noise: &Matrix,
sigma: f32,
) -> Result<(), NetworkError> {
if latents.data.len() != noise.data.len() {
return Err(NetworkError::InvalidTarget {
expected: latents.data.len(),
actual: noise.data.len(),
});
}
for (latent, noise) in latents.data.iter_mut().zip(&noise.data) {
*latent = match self {
Self::FlowMatch { .. } => (1.0 - sigma) * *latent + sigma * noise,
Self::Ddim { .. } => *latent + sigma * noise,
};
}
Ok(())
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Solver {
#[default]
Euler,
EulerAncestral,
DpmPlusPlus2m,
}
pub(crate) fn alphas_cumprod(beta_start: f32, beta_end: f32, steps: usize) -> Vec<f32> {
let (start, end) = (beta_start.sqrt(), beta_end.sqrt());
let mut product = 1.0;
(0..steps)
.map(|step| {
let fraction = step as f32 / (steps - 1).max(1) as f32;
let beta = (start + fraction * (end - start)).powi(2);
product *= 1.0 - beta;
product
})
.collect()
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct SamplingConfig {
pub steps: usize,
pub solver: Solver,
pub guidance: f32,
pub seed: Option<u64>,
}
impl Default for SamplingConfig {
fn default() -> Self {
Self {
steps: 28,
solver: Solver::default(),
guidance: 1.0,
seed: None,
}
}
}
pub fn noise(rows: usize, cols: usize, seed: Option<u64>) -> Matrix {
let mut rng = match seed {
Some(seed) => StdRng::seed_from_u64(seed),
None => StdRng::from_entropy(),
};
let data = (0..rows * cols).map(|_| gaussian(&mut rng)).collect();
Matrix::from_vec(rows, cols, data)
}
pub fn gaussian<R: Rng>(rng: &mut R) -> f32 {
let first: f32 = rng.gen_range(f32::MIN_POSITIVE..1.0);
let second: f32 = rng.gen_range(0.0..1.0);
(-2.0 * first.ln()).sqrt() * (std::f32::consts::TAU * second).cos()
}
pub fn sample<D: Denoiser>(
denoiser: &mut D,
scheduler: Scheduler,
latents: Matrix,
config: &SamplingConfig,
on_step: impl FnMut(usize, &Matrix) -> bool,
) -> Result<Matrix, NetworkError> {
sample_from(denoiser, scheduler, latents, config, 0, on_step)
}
pub fn sample_from<D: Denoiser>(
denoiser: &mut D,
scheduler: Scheduler,
mut latents: Matrix,
config: &SamplingConfig,
first: usize,
mut on_step: impl FnMut(usize, &Matrix) -> bool,
) -> Result<Matrix, NetworkError> {
let sigmas = scheduler.sigmas(config.steps)?;
let mut previous: Option<(Vec<f32>, f32)> = None;
for step in first.min(config.steps)..config.steps {
let (sigma, next) = (sigmas[step], sigmas[step + 1]);
let mut prediction = denoiser.denoise(&latents, sigma)?;
if config.guidance != 1.0
&& let Some(unconditional) = denoiser.denoise_unconditional(&latents, sigma)?
{
if unconditional.data.len() != prediction.data.len() {
return Err(NetworkError::InvalidTarget {
expected: prediction.data.len(),
actual: unconditional.data.len(),
});
}
for (conditional, unconditional) in prediction.data.iter_mut().zip(&unconditional.data)
{
*conditional = unconditional + config.guidance * (*conditional - unconditional);
}
}
match config.solver {
Solver::Euler => scheduler.step(&mut latents, &prediction, sigma, next)?,
Solver::EulerAncestral => {
let up = (next * next * (sigma * sigma - next * next) / (sigma * sigma))
.max(0.0)
.sqrt()
.min(next);
let down = (next * next - up * up).max(0.0).sqrt();
scheduler.step(&mut latents, &prediction, sigma, down)?;
if up > 0.0 {
let seed = config
.seed
.map(|seed| seed ^ (step as u64 + 1).wrapping_mul(0x9E37_79B9_7F4A_7C15));
let extra = noise(latents.rows, latents.cols, seed);
for (latent, value) in latents.data.iter_mut().zip(&extra.data) {
*latent += up * value;
}
}
}
Solver::DpmPlusPlus2m => {
let denoised: Vec<f32> = latents
.data
.iter()
.zip(&prediction.data)
.map(|(latent, prediction)| latent - sigma * prediction)
.collect();
let height = (sigma / next).ln();
let ratio = next / sigma;
let mut target = denoised.clone();
if let Some((before, last)) = &previous
&& next > 0.0
{
let slope = 0.5 * height / last;
for (target, before) in target.iter_mut().zip(before) {
*target += slope * (*target - before);
}
}
for (latent, target) in latents.data.iter_mut().zip(&target) {
*latent = ratio * *latent + (1.0 - ratio) * target;
}
previous = Some((denoised, height));
}
}
if !on_step(step, &latents) {
break;
}
}
Ok(latents)
}
#[cfg(test)]
mod tests {
use super::*;
struct Straight {
target: Vec<f32>,
calls: usize,
}
impl Denoiser for Straight {
fn denoise(&mut self, latents: &Matrix, sigma: f32) -> Result<Matrix, NetworkError> {
self.calls += 1;
let data = latents
.data
.iter()
.zip(&self.target)
.map(|(value, target)| (value - target) / sigma.max(1e-6))
.collect();
Ok(Matrix::from_vec(latents.rows, latents.cols, data))
}
}
#[test]
fn flow_matching_walks_a_straight_path_to_the_image() {
let target = vec![0.5, -0.25, 1.0, 0.0];
let mut model = Straight {
target: target.clone(),
calls: 0,
};
let start = Matrix::from_vec(1, 4, vec![2.0, -3.0, 0.5, 1.5]);
let config = SamplingConfig {
steps: 30,
..SamplingConfig::default()
};
let image = sample(
&mut model,
Scheduler::flow_match(1.0),
start,
&config,
|_, _| true,
)
.unwrap();
assert_eq!(model.calls, 30);
for (value, target) in image.data.iter().zip(&target) {
assert!((value - target).abs() < 0.05, "{value} vs {target}");
}
}
struct Curved {
target: Vec<f32>,
calls: usize,
}
impl Denoiser for Curved {
fn denoise(&mut self, latents: &Matrix, sigma: f32) -> Result<Matrix, NetworkError> {
self.calls += 1;
let data = latents
.data
.iter()
.zip(&self.target)
.map(|(value, target)| {
let clean = target + 0.5 * sigma * (value - target);
(value - clean) / sigma.max(1e-6)
})
.collect();
Ok(Matrix::from_vec(latents.rows, latents.cols, data))
}
}
#[test]
fn the_higher_order_solver_lands_on_the_image_in_fewer_steps() {
let target = vec![0.5, -0.25, 1.0, 0.0];
let start = Matrix::from_vec(1, 4, vec![2.0, -3.0, 0.5, 1.5]);
let run = |solver: Solver, steps: usize| {
let mut model = Curved {
target: target.clone(),
calls: 0,
};
let config = SamplingConfig {
steps,
solver,
..SamplingConfig::default()
};
let image = sample(
&mut model,
Scheduler::flow_match(1.0),
start.clone(),
&config,
|_, _| true,
)
.unwrap();
assert_eq!(model.calls, steps);
image
};
let settled = run(Solver::Euler, 500);
let error = |solver: Solver, steps: usize| {
run(solver, steps)
.data
.iter()
.zip(&settled.data)
.map(|(value, settled)| (value - settled).abs())
.fold(0.0f32, f32::max)
};
assert!(error(Solver::DpmPlusPlus2m, 4) < error(Solver::Euler, 4));
assert!(error(Solver::DpmPlusPlus2m, 8) < error(Solver::Euler, 8));
assert!(error(Solver::DpmPlusPlus2m, 30) < 0.01);
}
#[test]
fn the_ancestral_solver_repeats_on_a_seed_and_wanders_without_one() {
let target = vec![0.5, -0.25, 1.0, 0.0];
let start = Matrix::from_vec(1, 4, vec![2.0, -3.0, 0.5, 1.5]);
let run = |seed: Option<u64>| {
let mut model = Straight {
target: target.clone(),
calls: 0,
};
let config = SamplingConfig {
steps: 12,
solver: Solver::EulerAncestral,
seed,
..SamplingConfig::default()
};
let mut route = Vec::new();
let landed = sample(
&mut model,
Scheduler::ddim(),
start.clone(),
&config,
|_, latents| {
route.extend_from_slice(&latents.data);
true
},
)
.unwrap();
(route, landed)
};
assert_eq!(run(Some(7)).0, run(Some(7)).0);
assert_ne!(run(Some(7)).0, run(Some(8)).0);
for (value, target) in run(Some(7)).1.data.iter().zip(&target) {
assert!((value - target).abs() < 0.05, "{value} vs {target}");
}
}
#[test]
fn the_levels_fall_to_zero_and_the_shift_slows_the_noisy_end() {
for scheduler in [Scheduler::flow_match(1.0), Scheduler::flow_match(3.0)] {
let sigmas = scheduler.sigmas(20).unwrap();
assert_eq!(sigmas.len(), 21);
assert!((sigmas[0] - 1.0).abs() < 1e-6);
assert_eq!(*sigmas.last().unwrap(), 0.0);
for pair in sigmas.windows(2) {
assert!(pair[0] > pair[1], "{pair:?}");
}
}
let even = Scheduler::flow_match(1.0).sigmas(20).unwrap();
let shifted = Scheduler::flow_match(3.0).sigmas(20).unwrap();
assert!(
even.iter()
.zip(&shifted)
.skip(1)
.take(19)
.all(|(even, shifted)| shifted > even)
);
let ddim = Scheduler::ddim().sigmas(20).unwrap();
assert_eq!(ddim.len(), 21);
assert_eq!(*ddim.last().unwrap(), 0.0);
for pair in ddim.windows(2) {
assert!(pair[0] > pair[1], "{pair:?}");
}
assert!(Scheduler::ddim().sigmas(0).is_err());
}
#[test]
fn a_noise_predicting_model_lands_on_the_same_image() {
struct Noisy {
target: Vec<f32>,
}
impl Denoiser for Noisy {
fn denoise(&mut self, latents: &Matrix, sigma: f32) -> Result<Matrix, NetworkError> {
let data = latents
.data
.iter()
.zip(&self.target)
.map(|(value, target)| (value - target) / sigma.max(1e-6))
.collect();
Ok(Matrix::from_vec(latents.rows, latents.cols, data))
}
}
let target = vec![0.25, -0.75, 0.0];
let image = sample(
&mut Noisy {
target: target.clone(),
},
Scheduler::ddim(),
Matrix::from_vec(1, 3, vec![3.0, 1.0, -2.0]),
&SamplingConfig {
steps: 12,
..SamplingConfig::default()
},
|_, _| true,
)
.unwrap();
for (value, target) in image.data.iter().zip(&target) {
assert!((value - target).abs() < 1e-3, "{value} vs {target}");
}
}
#[test]
fn guidance_pushes_away_from_the_unconditional_prediction() {
struct Guided;
impl Denoiser for Guided {
fn denoise(&mut self, latents: &Matrix, _: f32) -> Result<Matrix, NetworkError> {
Ok(Matrix::from_vec(
latents.rows,
latents.cols,
vec![2.0; latents.data.len()],
))
}
fn denoise_unconditional(
&mut self,
latents: &Matrix,
_: f32,
) -> Result<Option<Matrix>, NetworkError> {
Ok(Some(Matrix::from_vec(
latents.rows,
latents.cols,
vec![1.0; latents.data.len()],
)))
}
}
let start = Matrix::from_vec(1, 2, vec![0.0, 0.0]);
let config = SamplingConfig {
steps: 1,
guidance: 3.0,
..SamplingConfig::default()
};
let guided = sample(
&mut Guided,
Scheduler::flow_match(1.0),
start.clone(),
&config,
|_, _| true,
)
.unwrap();
assert_eq!(guided.data, vec![-4.0, -4.0]);
struct Distilled;
impl Denoiser for Distilled {
fn denoise(&mut self, latents: &Matrix, _: f32) -> Result<Matrix, NetworkError> {
Ok(Matrix::from_vec(
latents.rows,
latents.cols,
vec![2.0; latents.data.len()],
))
}
}
let plain = sample(
&mut Distilled,
Scheduler::flow_match(1.0),
start,
&config,
|_, _| true,
)
.unwrap();
assert_eq!(plain.data, vec![-2.0, -2.0]);
}
#[test]
fn the_callback_can_stop_a_run_early() {
let mut model = Straight {
target: vec![0.0; 4],
calls: 0,
};
sample(
&mut model,
Scheduler::flow_match(1.0),
Matrix::new(1, 4),
&SamplingConfig {
steps: 20,
..SamplingConfig::default()
},
|step, _| step < 4,
)
.unwrap();
assert_eq!(model.calls, 5);
}
#[test]
fn seeded_noise_repeats_and_looks_normal() {
let first = noise(8, 16, Some(9));
assert_eq!(first.data, noise(8, 16, Some(9)).data);
assert_ne!(first.data, noise(8, 16, Some(10)).data);
let mean = first.data.iter().sum::<f32>() / first.data.len() as f32;
let variance = first
.data
.iter()
.map(|value| (value - mean).powi(2))
.sum::<f32>()
/ first.data.len() as f32;
assert!(mean.abs() < 0.2, "{mean}");
assert!((variance - 1.0).abs() < 0.3, "{variance}");
}
}