use crate::conv::{Conv2d, Dense, FeatureMap, GroupNorm, silu, upsample_nearest};
use crate::matrix::Matrix;
use crate::network::NetworkError;
use crate::safetensors::ShardedSafeTensors;
#[derive(Clone, Debug, PartialEq)]
pub struct VaeConfig {
pub latent_channels: usize,
pub out_channels: usize,
pub block_out_channels: Vec<usize>,
pub layers_per_block: usize,
pub norm_groups: usize,
pub scaling_factor: f32,
pub shift_factor: f32,
pub batch_norm_eps: f32,
}
impl Default for VaeConfig {
fn default() -> Self {
Self {
latent_channels: 4,
out_channels: 3,
block_out_channels: vec![128, 256, 512, 512],
layers_per_block: 2,
norm_groups: 32,
scaling_factor: 0.18215,
shift_factor: 0.0,
batch_norm_eps: 1e-4,
}
}
}
impl VaeConfig {
pub fn flux() -> Self {
Self {
latent_channels: 16,
scaling_factor: 0.3611,
shift_factor: 0.1159,
..Self::default()
}
}
pub fn from_file(path: impl AsRef<std::path::Path>) -> Result<Self, NetworkError> {
let text = std::fs::read_to_string(path)?;
let json: serde_json::Value = serde_json::from_str(&text)
.map_err(|error| NetworkError::InvalidDataset(format!("vae config.json: {error}")))?;
let default = Self::default();
let flux2 =
json.get("_class_name").and_then(|name| name.as_str()) == Some("AutoencoderKLFlux2");
let number = |name: &str, fallback: usize| -> usize {
json.get(name)
.and_then(|value| value.as_u64())
.map_or(fallback, |value| value as usize)
};
let float = |name: &str, fallback: f32| -> f32 {
json.get(name)
.and_then(|value| value.as_f64())
.map_or(fallback, |value| value as f32)
};
Ok(Self {
latent_channels: number("latent_channels", default.latent_channels),
out_channels: number("out_channels", default.out_channels),
block_out_channels: match json
.get("block_out_channels")
.and_then(|value| value.as_array())
{
Some(channels) => channels
.iter()
.map(|count| count.as_u64().map(|count| count as usize))
.collect::<Option<Vec<_>>>()
.ok_or_else(|| {
NetworkError::InvalidDataset(
"vae config.json: block_out_channels is not a list of sizes".into(),
)
})?,
None => default.block_out_channels,
},
layers_per_block: number("layers_per_block", default.layers_per_block),
norm_groups: number("norm_num_groups", default.norm_groups),
scaling_factor: float(
"scaling_factor",
match flux2 {
true => 1.0,
false => default.scaling_factor,
},
),
shift_factor: float("shift_factor", 0.0),
batch_norm_eps: float("batch_norm_eps", default.batch_norm_eps),
})
}
pub fn upscale(&self) -> usize {
1 << (self.block_out_channels.len() - 1)
}
}
#[derive(Clone, Debug)]
pub(crate) struct ResnetBlock {
pub(crate) norm1: GroupNorm,
pub(crate) conv1: Conv2d,
pub(crate) norm2: GroupNorm,
pub(crate) conv2: Conv2d,
pub(crate) shortcut: Option<Conv2d>,
}
impl ResnetBlock {
fn forward(&self, input: &FeatureMap) -> Result<FeatureMap, NetworkError> {
let mut hidden = input.clone();
self.norm1.forward(&mut hidden)?;
silu(&mut hidden);
let mut hidden = self.conv1.forward(&hidden)?;
self.norm2.forward(&mut hidden)?;
silu(&mut hidden);
let mut hidden = self.conv2.forward(&hidden)?;
let skip = match &self.shortcut {
Some(shortcut) => shortcut.forward(input)?,
None => input.clone(),
};
if skip.data.len() != hidden.data.len() {
return Err(NetworkError::InvalidTarget {
expected: hidden.data.len(),
actual: skip.data.len(),
});
}
for (value, skip) in hidden.data.iter_mut().zip(&skip.data) {
*value += skip;
}
Ok(hidden)
}
}
#[derive(Clone, Debug)]
pub(crate) struct AttentionBlock {
pub(crate) norm: GroupNorm,
pub(crate) query: Dense,
pub(crate) key: Dense,
pub(crate) value: Dense,
pub(crate) output: Dense,
}
impl AttentionBlock {
fn forward(&self, input: &FeatureMap) -> Result<FeatureMap, NetworkError> {
let mut normed = input.clone();
self.norm.forward(&mut normed)?;
let tokens = normed.to_tokens();
let queries = self.query.forward(&tokens)?;
let keys = self.key.forward(&tokens)?;
let values = self.value.forward(&tokens)?;
let attended = attend(&queries, &keys, &values);
let projected = self.output.forward(&attended)?;
let mut output = FeatureMap::from_tokens(&projected, input.height, input.width)?;
for (value, skip) in output.data.iter_mut().zip(&input.data) {
*value += skip;
}
Ok(output)
}
}
pub(crate) fn attend(queries: &Matrix, keys: &Matrix, values: &Matrix) -> Matrix {
let scale = (queries.cols as f32).sqrt().recip();
let mut attended = Matrix::new(queries.rows, values.cols);
let mut weights = vec![0.0; queries.rows];
for pixel in 0..queries.rows {
let query = queries.row(pixel);
let mut largest = f32::NEG_INFINITY;
for (weight, other) in weights.iter_mut().zip(0..queries.rows) {
let score = query
.iter()
.zip(keys.row(other))
.map(|(query, key)| query * key)
.sum::<f32>()
* scale;
*weight = score;
largest = largest.max(score);
}
let mut total = 0.0;
for weight in &mut weights {
*weight = (*weight - largest).exp();
total += *weight;
}
let row = attended.row_mut(pixel);
for (other, weight) in weights.iter().enumerate() {
let weight = weight / total;
for (value, source) in row.iter_mut().zip(values.row(other)) {
*value += weight * source;
}
}
}
attended
}
#[derive(Clone, Debug)]
pub(crate) struct UpBlock {
pub(crate) resnets: Vec<ResnetBlock>,
pub(crate) upsampler: Option<Conv2d>,
}
#[derive(Clone, Debug)]
pub struct VaeDecoder {
config: VaeConfig,
pub(crate) post_quant: Option<Conv2d>,
patch_norm: Option<(Vec<f32>, Vec<f32>)>,
pub(crate) conv_in: Conv2d,
pub(crate) mid_first: ResnetBlock,
pub(crate) mid_attention: AttentionBlock,
pub(crate) mid_last: ResnetBlock,
pub(crate) up_blocks: Vec<UpBlock>,
pub(crate) norm_out: GroupNorm,
pub(crate) conv_out: Conv2d,
#[cfg(feature = "cuda")]
device: Option<std::sync::Arc<crate::cuda_image::DeviceVae>>,
}
impl VaeDecoder {
pub fn config(&self) -> &VaeConfig {
&self.config
}
pub fn patch_norm(&self) -> Option<(&[f32], &[f32])> {
self.patch_norm
.as_ref()
.map(|(mean, std)| (mean.as_slice(), std.as_slice()))
}
pub fn load(
path: impl AsRef<std::path::Path>,
prefix: &str,
config: VaeConfig,
) -> Result<Self, NetworkError> {
let mut file = ShardedSafeTensors::open(path)?;
Self::read(&mut file, prefix, config)
}
pub fn read(
file: &mut ShardedSafeTensors,
prefix: &str,
config: VaeConfig,
) -> Result<Self, NetworkError> {
if config.block_out_channels.is_empty() {
return Err(NetworkError::InvalidConfig(
"a decoder needs at least one resolution".into(),
));
}
let ladder: Vec<usize> = config.block_out_channels.iter().rev().copied().collect();
let deepest = ladder[0];
let root = prefix.strip_suffix("decoder.").unwrap_or("");
let post_quant = read_conv(file, &format!("{root}post_quant_conv"), 1, 0).ok();
let patch_norm = match (
file.tensor(&format!("{root}bn.running_mean")),
file.tensor(&format!("{root}bn.running_var")),
) {
(Ok((mean, _)), Ok((variance, _))) => {
let eps = config.batch_norm_eps;
let std = variance.iter().map(|value| (value + eps).sqrt()).collect();
Some((mean, std))
}
_ => None,
};
let conv_in = read_conv(file, &format!("{prefix}conv_in"), 1, 1)?;
let mid = format!("{prefix}mid_block");
let mid_first = read_resnet(file, &format!("{mid}.resnets.0"), config.norm_groups)?;
let mid_attention =
read_attention(file, &format!("{mid}.attentions.0"), config.norm_groups)?;
let mid_last = read_resnet(file, &format!("{mid}.resnets.1"), config.norm_groups)?;
let mut up_blocks = Vec::with_capacity(ladder.len());
for (rung, _) in ladder.iter().enumerate() {
let base = format!("{prefix}up_blocks.{rung}");
let resnets = (0..config.layers_per_block + 1)
.map(|index| {
read_resnet(file, &format!("{base}.resnets.{index}"), config.norm_groups)
})
.collect::<Result<Vec<_>, _>>()?;
let upsampler = if rung + 1 < ladder.len() {
Some(read_conv(file, &format!("{base}.upsamplers.0.conv"), 1, 1)?)
} else {
None
};
up_blocks.push(UpBlock { resnets, upsampler });
}
let norm_out =
read_group_norm(file, &format!("{prefix}conv_norm_out"), config.norm_groups)?;
let conv_out = read_conv(file, &format!("{prefix}conv_out"), 1, 1)?;
let takes = post_quant
.as_ref()
.map_or(conv_in.in_channels, |quant| quant.in_channels);
if takes != config.latent_channels || conv_in.out_channels() != deepest {
return Err(NetworkError::InvalidConfig(format!(
"the checkpoint's stem maps {} channels to {}, and the configuration says {} to \
{deepest}",
takes,
conv_in.out_channels(),
config.latent_channels
)));
}
Ok(Self {
config,
post_quant,
patch_norm,
conv_in,
mid_first,
mid_attention,
mid_last,
up_blocks,
norm_out,
conv_out,
#[cfg(feature = "cuda")]
device: None,
})
}
#[cfg(feature = "cuda")]
pub fn attach_device(
&mut self,
gpu: std::sync::Arc<crate::cuda_image::ImageGpu>,
) -> Result<(), NetworkError> {
self.device = Some(std::sync::Arc::new(crate::cuda_image::DeviceVae::upload(
&gpu, self,
)?));
Ok(())
}
pub fn decode(&self, latent: &FeatureMap) -> Result<FeatureMap, NetworkError> {
if latent.channels != self.config.latent_channels {
return Err(NetworkError::InvalidTarget {
expected: self.config.latent_channels,
actual: latent.channels,
});
}
let mut scaled = latent.clone();
for value in &mut scaled.data {
*value = *value / self.config.scaling_factor + self.config.shift_factor;
}
#[cfg(feature = "cuda")]
if let Some(device) = &self.device {
return device.decode(&scaled);
}
let scaled = match &self.post_quant {
Some(quant) => quant.forward(&scaled)?,
None => scaled,
};
let mut hidden = self.conv_in.forward(&scaled)?;
hidden = self.mid_first.forward(&hidden)?;
hidden = self.mid_attention.forward(&hidden)?;
hidden = self.mid_last.forward(&hidden)?;
for block in &self.up_blocks {
for resnet in &block.resnets {
hidden = resnet.forward(&hidden)?;
}
if let Some(upsampler) = &block.upsampler {
hidden = upsampler.forward(&upsample_nearest(&hidden, 2))?;
}
}
self.norm_out.forward(&mut hidden)?;
silu(&mut hidden);
self.conv_out.forward(&hidden)
}
pub fn decode_tiled(
&self,
latent: &FeatureMap,
tile: usize,
overlap: usize,
) -> Result<FeatureMap, NetworkError> {
if overlap >= tile {
return Err(NetworkError::InvalidTarget {
expected: tile.saturating_sub(1),
actual: overlap,
});
}
stitch_tiles(latent, tile, overlap, self.config.upscale(), |part| {
self.decode(part)
})
}
}
fn tile_starts(len: usize, tile: usize, overlap: usize) -> Vec<usize> {
if len <= tile {
return vec![0];
}
let mut starts: Vec<usize> = (0..len - tile).step_by(tile - overlap).collect();
starts.push(len - tile);
starts
}
fn ramp_weight(at: usize, len: usize, start: usize, axis: usize, ramp: usize) -> f32 {
let mut weight = 1.0f32;
if start > 0 {
weight = weight.min((at + 1) as f32 / (ramp + 1) as f32);
}
if start + len < axis {
weight = weight.min((len - at) as f32 / (ramp + 1) as f32);
}
weight
}
fn stitch_tiles(
latent: &FeatureMap,
tile: usize,
overlap: usize,
scale: usize,
decode: impl Fn(&FeatureMap) -> Result<FeatureMap, NetworkError>,
) -> Result<FeatureMap, NetworkError> {
let (height, width) = (latent.height * scale, latent.width * scale);
let ramp = overlap * scale;
let mut sum: Vec<f32> = Vec::new();
let mut weights = vec![0.0f32; height * width];
let mut channels = 0;
for top in tile_starts(latent.height, tile, overlap) {
for left in tile_starts(latent.width, tile, overlap) {
let (rows, cols) = (tile.min(latent.height), tile.min(latent.width));
let mut part = FeatureMap::new(latent.channels, rows, cols);
for c in 0..latent.channels {
for y in 0..rows {
let from = (c * latent.height + top + y) * latent.width + left;
let to = (c * rows + y) * cols;
part.data[to..to + cols].copy_from_slice(&latent.data[from..from + cols]);
}
}
let out = decode(&part)?;
if sum.is_empty() {
channels = out.channels;
sum = vec![0.0; channels * height * width];
}
for y in 0..out.height {
let wy = ramp_weight(y, out.height, top * scale, height, ramp);
for x in 0..out.width {
let w = wy * ramp_weight(x, out.width, left * scale, width, ramp);
let at = (top * scale + y) * width + left * scale + x;
weights[at] += w;
for c in 0..channels {
sum[c * height * width + at] +=
w * out.data[(c * out.height + y) * out.width + x];
}
}
}
}
}
for (index, value) in sum.iter_mut().enumerate() {
*value /= weights[index % (height * width)];
}
FeatureMap::from_vec(channels, height, width, sum)
}
#[derive(Clone, Debug)]
pub(crate) struct DownBlock {
pub(crate) resnets: Vec<ResnetBlock>,
pub(crate) downsampler: Option<Conv2d>,
}
#[derive(Clone, Debug)]
pub struct VaeEncoder {
config: VaeConfig,
pub(crate) conv_in: Conv2d,
pub(crate) down_blocks: Vec<DownBlock>,
pub(crate) mid_first: ResnetBlock,
pub(crate) mid_attention: AttentionBlock,
pub(crate) mid_last: ResnetBlock,
pub(crate) norm_out: GroupNorm,
pub(crate) conv_out: Conv2d,
pub(crate) quant: Option<Conv2d>,
#[cfg(feature = "cuda")]
device: Option<std::sync::Arc<crate::cuda_image::DeviceVaeEncoder>>,
}
impl VaeEncoder {
pub fn config(&self) -> &VaeConfig {
&self.config
}
pub fn load(
path: impl AsRef<std::path::Path>,
prefix: &str,
config: VaeConfig,
) -> Result<Self, NetworkError> {
let mut file = ShardedSafeTensors::open(path)?;
Self::read(&mut file, prefix, config)
}
pub fn read(
file: &mut ShardedSafeTensors,
prefix: &str,
config: VaeConfig,
) -> Result<Self, NetworkError> {
if config.block_out_channels.is_empty() {
return Err(NetworkError::InvalidConfig(
"an encoder needs at least one resolution".into(),
));
}
let ladder = &config.block_out_channels;
let conv_in = read_conv(file, &format!("{prefix}conv_in"), 1, 1)?;
let mut down_blocks = Vec::with_capacity(ladder.len());
for rung in 0..ladder.len() {
let base = format!("{prefix}down_blocks.{rung}");
let resnets = (0..config.layers_per_block)
.map(|index| {
read_resnet(file, &format!("{base}.resnets.{index}"), config.norm_groups)
})
.collect::<Result<Vec<_>, _>>()?;
let downsampler = if rung + 1 < ladder.len() {
Some(read_conv(
file,
&format!("{base}.downsamplers.0.conv"),
2,
0,
)?)
} else {
None
};
down_blocks.push(DownBlock {
resnets,
downsampler,
});
}
let mid = format!("{prefix}mid_block");
let mid_first = read_resnet(file, &format!("{mid}.resnets.0"), config.norm_groups)?;
let mid_attention =
read_attention(file, &format!("{mid}.attentions.0"), config.norm_groups)?;
let mid_last = read_resnet(file, &format!("{mid}.resnets.1"), config.norm_groups)?;
let norm_out =
read_group_norm(file, &format!("{prefix}conv_norm_out"), config.norm_groups)?;
let conv_out = read_conv(file, &format!("{prefix}conv_out"), 1, 1)?;
let root = prefix.strip_suffix("encoder.").unwrap_or("");
let quant = read_conv(file, &format!("{root}quant_conv"), 1, 0).ok();
if conv_in.in_channels != config.out_channels || conv_in.out_channels() != ladder[0] {
return Err(NetworkError::InvalidConfig(format!(
"the checkpoint's stem maps {} channels to {}, and the configuration says {} to {}",
conv_in.in_channels,
conv_in.out_channels(),
config.out_channels,
ladder[0]
)));
}
let produced = quant
.as_ref()
.map_or(conv_out.out_channels(), |quant| quant.out_channels());
if produced != config.latent_channels && produced != 2 * config.latent_channels {
return Err(NetworkError::InvalidConfig(format!(
"the encoder ends in {produced} channels, which is neither {} nor twice it",
config.latent_channels
)));
}
Ok(Self {
config,
conv_in,
down_blocks,
mid_first,
mid_attention,
mid_last,
norm_out,
conv_out,
quant,
#[cfg(feature = "cuda")]
device: None,
})
}
#[cfg(feature = "cuda")]
pub fn attach_device(
&mut self,
gpu: std::sync::Arc<crate::cuda_image::ImageGpu>,
) -> Result<(), NetworkError> {
self.device = Some(std::sync::Arc::new(
crate::cuda_image::DeviceVaeEncoder::upload(&gpu, self)?,
));
Ok(())
}
pub fn encode(&self, image: &FeatureMap) -> Result<FeatureMap, NetworkError> {
if image.channels != self.config.out_channels {
return Err(NetworkError::InvalidTarget {
expected: self.config.out_channels,
actual: image.channels,
});
}
let moments = self.moments(image)?;
let channels = self.config.latent_channels;
let pixels = moments.pixels();
let mut latent = FeatureMap::from_vec(
channels,
moments.height,
moments.width,
moments.data[..channels * pixels].to_vec(),
)?;
for value in &mut latent.data {
*value = (*value - self.config.shift_factor) * self.config.scaling_factor;
}
Ok(latent)
}
fn moments(&self, image: &FeatureMap) -> Result<FeatureMap, NetworkError> {
#[cfg(feature = "cuda")]
if let Some(device) = &self.device {
return device.moments(image);
}
let mut hidden = self.conv_in.forward(image)?;
for block in &self.down_blocks {
for resnet in &block.resnets {
hidden = resnet.forward(&hidden)?;
}
if let Some(downsampler) = &block.downsampler {
hidden = downsampler.forward(&pad_right_bottom(&hidden))?;
}
}
hidden = self.mid_first.forward(&hidden)?;
hidden = self.mid_attention.forward(&hidden)?;
hidden = self.mid_last.forward(&hidden)?;
self.norm_out.forward(&mut hidden)?;
silu(&mut hidden);
let mut moments = self.conv_out.forward(&hidden)?;
if let Some(quant) = &self.quant {
moments = quant.forward(&moments)?;
}
Ok(moments)
}
}
pub(crate) fn pad_right_bottom(map: &FeatureMap) -> FeatureMap {
let (height, width) = (map.height + 1, map.width + 1);
let mut padded = FeatureMap::new(map.channels, height, width);
for channel in 0..map.channels {
for row in 0..map.height {
let from = (channel * map.height + row) * map.width;
let to = (channel * height + row) * width;
padded.data[to..to + map.width].copy_from_slice(&map.data[from..from + map.width]);
}
}
padded
}
pub fn to_rgb8(image: &FeatureMap) -> Vec<u8> {
let pixels = image.pixels();
let mut bytes = Vec::with_capacity(pixels * image.channels);
for pixel in 0..pixels {
for channel in 0..image.channels {
let value = image.data[channel * pixels + pixel];
bytes.push(((value * 0.5 + 0.5).clamp(0.0, 1.0) * 255.0).round() as u8);
}
}
bytes
}
#[cfg(feature = "images")]
pub fn save_png(image: &FeatureMap, path: impl AsRef<std::path::Path>) -> Result<(), NetworkError> {
let path = path.as_ref();
dynamic_image(image)?
.save(path)
.map_err(|error| NetworkError::InvalidDataset(format!("{}: {error}", path.display())))
}
#[cfg(feature = "images")]
pub fn png_bytes(image: &FeatureMap) -> Result<Vec<u8>, NetworkError> {
let mut bytes = std::io::Cursor::new(Vec::new());
dynamic_image(image)?
.write_to(&mut bytes, image::ImageFormat::Png)
.map_err(|error| NetworkError::InvalidDataset(format!("PNG encoding: {error}")))?;
Ok(bytes.into_inner())
}
#[cfg(feature = "images")]
pub fn from_image_bytes(
bytes: &[u8],
width: usize,
height: usize,
) -> Result<FeatureMap, NetworkError> {
if width == 0 || height == 0 {
return Err(NetworkError::InvalidDataset(format!(
"cannot resize an image to {width}x{height}"
)));
}
let decoded = image::load_from_memory(bytes)
.map_err(|error| NetworkError::InvalidDataset(format!("image decoding: {error}")))?;
let (w, h) = (width as u32, height as u32);
let rgb = if decoded.width() == w && decoded.height() == h {
decoded.to_rgb8()
} else {
decoded
.resize_exact(w, h, image::imageops::FilterType::Lanczos3)
.to_rgb8()
};
let pixels = width * height;
let mut map = FeatureMap::new(3, height, width);
for (pixel, rgb) in rgb.as_raw().chunks_exact(3).enumerate() {
for (channel, &byte) in rgb.iter().enumerate() {
map.data[channel * pixels + pixel] = byte as f32 / 255.0 * 2.0 - 1.0;
}
}
Ok(map)
}
#[cfg(feature = "images")]
fn dynamic_image(image: &FeatureMap) -> Result<image::DynamicImage, NetworkError> {
let bytes = to_rgb8(image);
let (width, height) = (image.width as u32, image.height as u32);
let shape = || NetworkError::InvalidConfig("the image's shape and its bytes disagree".into());
match image.channels {
3 => image::RgbImage::from_raw(width, height, bytes)
.map(image::DynamicImage::ImageRgb8)
.ok_or_else(shape),
1 => image::GrayImage::from_raw(width, height, bytes)
.map(image::DynamicImage::ImageLuma8)
.ok_or_else(shape),
channels => Err(NetworkError::InvalidConfig(format!(
"a {channels}-channel image is neither grey nor RGB"
))),
}
}
pub(crate) fn read_conv(
file: &mut ShardedSafeTensors,
name: &str,
stride: usize,
padding: usize,
) -> Result<Conv2d, NetworkError> {
let (values, shape) = file.tensor(&format!("{name}.weight"))?;
if shape.len() != 4 || shape[2] != shape[3] {
return Err(NetworkError::InvalidConfig(format!(
"{name}.weight is {shape:?}, which is not a square convolution kernel"
)));
}
let (out_channels, in_channels, kernel) = (shape[0], shape[1], shape[2]);
let bias = file
.tensor(&format!("{name}.bias"))
.ok()
.map(|(bias, _)| bias);
let padding = if kernel == 1 { 0 } else { padding };
Conv2d::new(
Matrix::from_vec(out_channels, in_channels * kernel * kernel, values),
bias,
in_channels,
kernel,
stride,
padding,
)
}
fn read_group_norm(
file: &mut ShardedSafeTensors,
name: &str,
groups: usize,
) -> Result<GroupNorm, NetworkError> {
let (weight, _) = file.tensor(&format!("{name}.weight"))?;
let (bias, _) = file.tensor(&format!("{name}.bias"))?;
GroupNorm::new(groups, weight, bias, 1e-6)
}
fn read_resnet(
file: &mut ShardedSafeTensors,
name: &str,
groups: usize,
) -> Result<ResnetBlock, NetworkError> {
let conv1 = read_conv(file, &format!("{name}.conv1"), 1, 1)?;
let conv2 = read_conv(file, &format!("{name}.conv2"), 1, 1)?;
let shortcut = match read_conv(file, &format!("{name}.conv_shortcut"), 1, 0) {
Ok(conv) => Some(conv),
Err(_) if conv1.in_channels == conv2.out_channels() => None,
Err(error) => return Err(error),
};
Ok(ResnetBlock {
norm1: read_group_norm(file, &format!("{name}.norm1"), groups)?,
conv1,
norm2: read_group_norm(file, &format!("{name}.norm2"), groups)?,
conv2,
shortcut,
})
}
fn read_attention(
file: &mut ShardedSafeTensors,
name: &str,
groups: usize,
) -> Result<AttentionBlock, NetworkError> {
let projection = |file: &mut ShardedSafeTensors, suffix: &str| -> Result<Dense, NetworkError> {
let (values, shape) = file.tensor(&format!("{name}.{suffix}.weight"))?;
let channels = shape[0];
let (bias, _) = file.tensor(&format!("{name}.{suffix}.bias"))?;
Dense::new(
Matrix::from_vec(channels, values.len() / channels, values),
Some(bias),
)
};
Ok(AttentionBlock {
norm: read_group_norm(file, &format!("{name}.group_norm"), groups)?,
query: projection(file, "to_q")?,
key: projection(file, "to_k")?,
value: projection(file, "to_v")?,
output: projection(file, "to_out.0")?,
})
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
use std::collections::BTreeMap;
pub(crate) fn checkpoint(config: &VaeConfig, path: &std::path::Path) {
checkpoint_at(config, path, true)
}
pub(crate) fn checkpoint_at(config: &VaeConfig, path: &std::path::Path, post_quant: bool) {
let ladder: Vec<usize> = config.block_out_channels.iter().rev().copied().collect();
let mut tensors: BTreeMap<String, (Vec<usize>, Vec<f32>)> = BTreeMap::new();
let mut add = |name: String, shape: Vec<usize>| {
let count = shape.iter().product::<usize>();
let values = (0..count)
.map(|index| ((index % 7) as f32 - 3.0) * 0.02)
.collect();
tensors.insert(name, (shape, values));
};
let deepest = ladder[0];
add(
"decoder.conv_in.weight".into(),
vec![deepest, config.latent_channels, 3, 3],
);
add("decoder.conv_in.bias".into(), vec![deepest]);
let resnet =
|name: &str, input: usize, output: usize, add: &mut dyn FnMut(String, Vec<usize>)| {
add(format!("{name}.norm1.weight"), vec![input]);
add(format!("{name}.norm1.bias"), vec![input]);
add(format!("{name}.conv1.weight"), vec![output, input, 3, 3]);
add(format!("{name}.conv1.bias"), vec![output]);
add(format!("{name}.norm2.weight"), vec![output]);
add(format!("{name}.norm2.bias"), vec![output]);
add(format!("{name}.conv2.weight"), vec![output, output, 3, 3]);
add(format!("{name}.conv2.bias"), vec![output]);
if input != output {
add(
format!("{name}.conv_shortcut.weight"),
vec![output, input, 1, 1],
);
add(format!("{name}.conv_shortcut.bias"), vec![output]);
}
};
resnet("decoder.mid_block.resnets.0", deepest, deepest, &mut add);
resnet("decoder.mid_block.resnets.1", deepest, deepest, &mut add);
for suffix in ["group_norm", "to_q", "to_k", "to_v", "to_out.0"] {
let shape = if suffix == "group_norm" {
vec![deepest]
} else {
vec![deepest, deepest]
};
add(
format!("decoder.mid_block.attentions.0.{suffix}.weight"),
shape,
);
add(
format!("decoder.mid_block.attentions.0.{suffix}.bias"),
vec![deepest],
);
}
let mut input = deepest;
for (rung, channels) in ladder.iter().enumerate() {
for index in 0..config.layers_per_block + 1 {
resnet(
&format!("decoder.up_blocks.{rung}.resnets.{index}"),
input,
*channels,
&mut add,
);
input = *channels;
}
if rung + 1 < ladder.len() {
add(
format!("decoder.up_blocks.{rung}.upsamplers.0.conv.weight"),
vec![*channels, *channels, 3, 3],
);
add(
format!("decoder.up_blocks.{rung}.upsamplers.0.conv.bias"),
vec![*channels],
);
}
}
add("decoder.conv_norm_out.weight".into(), vec![input]);
add("decoder.conv_norm_out.bias".into(), vec![input]);
add(
"decoder.conv_out.weight".into(),
vec![config.out_channels, input, 3, 3],
);
add("decoder.conv_out.bias".into(), vec![config.out_channels]);
let forward = &config.block_out_channels;
add(
"encoder.conv_in.weight".into(),
vec![forward[0], config.out_channels, 3, 3],
);
add("encoder.conv_in.bias".into(), vec![forward[0]]);
let mut input = forward[0];
for (rung, channels) in forward.iter().enumerate() {
for index in 0..config.layers_per_block {
resnet(
&format!("encoder.down_blocks.{rung}.resnets.{index}"),
input,
*channels,
&mut add,
);
input = *channels;
}
if rung + 1 < forward.len() {
add(
format!("encoder.down_blocks.{rung}.downsamplers.0.conv.weight"),
vec![*channels, *channels, 3, 3],
);
add(
format!("encoder.down_blocks.{rung}.downsamplers.0.conv.bias"),
vec![*channels],
);
}
}
resnet("encoder.mid_block.resnets.0", input, input, &mut add);
resnet("encoder.mid_block.resnets.1", input, input, &mut add);
for suffix in ["group_norm", "to_q", "to_k", "to_v", "to_out.0"] {
let shape = if suffix == "group_norm" {
vec![input]
} else {
vec![input, input]
};
add(
format!("encoder.mid_block.attentions.0.{suffix}.weight"),
shape,
);
add(
format!("encoder.mid_block.attentions.0.{suffix}.bias"),
vec![input],
);
}
add("encoder.conv_norm_out.weight".into(), vec![input]);
add("encoder.conv_norm_out.bias".into(), vec![input]);
let moments = 2 * config.latent_channels;
add("encoder.conv_out.weight".into(), vec![moments, input, 3, 3]);
add("encoder.conv_out.bias".into(), vec![moments]);
add("quant_conv.weight".into(), vec![moments, moments, 1, 1]);
add("quant_conv.bias".into(), vec![moments]);
if post_quant {
let latent = config.latent_channels;
add("post_quant_conv.weight".into(), vec![latent, latent, 1, 1]);
add("post_quant_conv.bias".into(), vec![latent]);
}
crate::safetensors::write_checkpoint(path, &tensors);
}
pub(crate) fn tiny() -> VaeConfig {
VaeConfig {
latent_channels: 4,
out_channels: 3,
block_out_channels: vec![8, 16],
layers_per_block: 1,
norm_groups: 4,
scaling_factor: 0.5,
shift_factor: 0.25,
batch_norm_eps: 1e-4,
}
}
#[test]
fn a_flux2_autoencoder_reads_no_scale_and_its_patch_norm() {
let directory = std::env::temp_dir().join("rusting_brain_vae_flux2");
std::fs::create_dir_all(&directory).unwrap();
let config_path = directory.join("config.json");
std::fs::write(
&config_path,
r#"{"_class_name": "AutoencoderKLFlux2", "latent_channels": 4,
"block_out_channels": [8, 16], "layers_per_block": 1,
"norm_num_groups": 4, "batch_norm_eps": 0.0001}"#,
)
.unwrap();
let config = VaeConfig::from_file(&config_path).unwrap();
assert_eq!(config.scaling_factor, 1.0);
assert_eq!(config.shift_factor, 0.0);
assert_eq!(config.batch_norm_eps, 1e-4);
let path = directory.join("vae.safetensors");
checkpoint_at(&config, &path, true);
let decoder = VaeDecoder::load(&path, "decoder.", config.clone()).unwrap();
assert!(decoder.patch_norm().is_none());
let mut tensors = crate::safetensors::read_checkpoint(&path);
let packed = config.latent_channels * 4;
tensors.insert(
"bn.running_mean".into(),
(
vec![packed],
(0..packed).map(|index| index as f32).collect(),
),
);
tensors.insert(
"bn.running_var".into(),
(vec![packed], vec![3.9999; packed]),
);
crate::safetensors::write_checkpoint(&path, &tensors);
let decoder = VaeDecoder::load(&path, "decoder.", config).unwrap();
let (mean, std) = decoder.patch_norm().unwrap();
assert_eq!(mean[5], 5.0);
assert!((std[5] - 2.0).abs() < 1e-6);
}
#[test]
fn a_decoder_runs_the_convolution_that_sits_in_front_of_its_stem() {
let config = tiny();
let latent = FeatureMap::from_vec(
4,
3,
3,
(0..36).map(|value| (value as f32 * 0.1).sin()).collect(),
)
.unwrap();
let mut images = Vec::new();
for (carries, name) in [(true, "with"), (false, "without")] {
let path = std::env::temp_dir()
.join(format!("rusting_brain_vae_post_quant_{name}.safetensors"));
checkpoint_at(&config, &path, carries);
let decoder = VaeDecoder::load(&path, "decoder.", config.clone()).unwrap();
assert_eq!(decoder.post_quant.is_some(), carries);
images.push(decoder.decode(&latent).unwrap());
}
let (with, without) = (&images[0], &images[1]);
let apart = with
.data
.iter()
.zip(&without.data)
.map(|(with, without)| (with - without).abs())
.fold(0.0f32, f32::max);
assert!(
apart > 1e-4,
"the convolution in front of the stem changed nothing, so it was not applied"
);
}
#[test]
fn a_decoder_reads_a_checkpoint_and_turns_a_latent_into_an_image() {
let config = tiny();
let path = std::env::temp_dir().join("rusting_brain_vae_decode.safetensors");
checkpoint(&config, &path);
let decoder = VaeDecoder::load(&path, "decoder.", config.clone()).unwrap();
assert_eq!(decoder.config().upscale(), 2);
let latent = FeatureMap::from_vec(
4,
3,
3,
(0..36).map(|value| (value as f32 * 0.1).sin()).collect(),
)
.unwrap();
let image = decoder.decode(&latent).unwrap();
assert_eq!(image.channels, 3);
assert_eq!((image.height, image.width), (6, 6));
assert!(image.data.iter().all(|value| value.is_finite()));
assert!(decoder.decode(&FeatureMap::new(3, 3, 3)).is_err());
std::fs::remove_file(path).ok();
}
#[test]
fn an_encoder_turns_an_image_into_a_latent_the_decoder_accepts() {
let config = tiny();
let path = std::env::temp_dir().join("rusting_brain_vae_encode.safetensors");
checkpoint(&config, &path);
let encoder = VaeEncoder::load(&path, "encoder.", config.clone()).unwrap();
let decoder = VaeDecoder::load(&path, "decoder.", config.clone()).unwrap();
let image = FeatureMap::from_vec(
3,
6,
6,
(0..108).map(|value| (value as f32 * 0.07).cos()).collect(),
)
.unwrap();
let latent = encoder.encode(&image).unwrap();
assert_eq!(latent.channels, config.latent_channels);
assert_eq!((latent.height, latent.width), (3, 3));
assert!(latent.data.iter().all(|value| value.is_finite()));
let round_trip = decoder.decode(&latent).unwrap();
assert_eq!((round_trip.height, round_trip.width), (6, 6));
assert!(encoder.encode(&FeatureMap::new(1, 6, 6)).is_err());
std::fs::remove_file(path).ok();
}
#[test]
fn a_missing_layer_is_an_error_rather_than_a_silent_hole() {
let config = tiny();
let path = std::env::temp_dir().join("rusting_brain_vae_missing.safetensors");
checkpoint(&config, &path);
assert!(VaeDecoder::load(&path, "vae.decoder.", config.clone()).is_err());
let mut deeper = config;
deeper.block_out_channels.push(32);
assert!(VaeDecoder::load(&path, "decoder.", deeper).is_err());
std::fs::remove_file(path).ok();
}
#[test]
fn tiles_blend_back_into_the_whole_image() {
let latent = FeatureMap::from_vec(
2,
11,
7,
(0..154).map(|value| (value as f32 * 0.37).sin()).collect(),
)
.unwrap();
let decode = |map: &FeatureMap| Ok(crate::conv::upsample_nearest(map, 2));
let whole = decode(&latent).unwrap();
for (tile, overlap) in [(4, 1), (5, 2), (3, 0), (20, 4)] {
let tiled = stitch_tiles(&latent, tile, overlap, 2, decode).unwrap();
assert_eq!((tiled.height, tiled.width), (22, 14));
for (a, b) in tiled.data.iter().zip(&whole.data) {
assert!((a - b).abs() < 1e-5, "tile {tile} overlap {overlap}");
}
}
}
#[test]
fn a_tiled_decode_matches_the_whole_decode_when_one_tile_covers_it() {
let config = tiny();
let path = std::env::temp_dir().join("rusting_brain_vae_tiled.safetensors");
checkpoint(&config, &path);
let decoder = VaeDecoder::load(&path, "decoder.", config).unwrap();
let latent = FeatureMap::from_vec(
4,
5,
4,
(0..80).map(|value| (value as f32 * 0.1).cos()).collect(),
)
.unwrap();
assert_eq!(
decoder.decode_tiled(&latent, 5, 1).unwrap(),
decoder.decode(&latent).unwrap()
);
let tiled = decoder.decode_tiled(&latent, 3, 1).unwrap();
assert_eq!((tiled.channels, tiled.height, tiled.width), (3, 10, 8));
assert!(tiled.data.iter().all(|value| value.is_finite()));
assert!(decoder.decode_tiled(&latent, 3, 3).is_err());
std::fs::remove_file(path).ok();
}
#[test]
fn eight_bit_output_clamps_the_range_the_decoder_produces() {
let image = FeatureMap::from_vec(3, 1, 2, vec![-1.0, 0.0, 1.0, 2.0, -5.0, 0.5]).unwrap();
assert_eq!(to_rgb8(&image), vec![0, 255, 0, 128, 255, 191]);
}
#[test]
fn the_flux_configuration_carries_its_own_latent_shape() {
let flux = VaeConfig::flux();
assert_eq!(flux.latent_channels, 16);
assert_eq!(flux.upscale(), 8);
assert!(flux.shift_factor > 0.0);
}
#[cfg(feature = "images")]
#[test]
fn png_bytes_are_the_file_save_png_writes() {
for channels in [3, 1] {
let mut image = FeatureMap::new(channels, 5, 7);
for (index, value) in image.data.iter_mut().enumerate() {
*value = (index % 11) as f32 / 5.0 - 1.0;
}
let path = std::env::temp_dir().join(format!(
"rusting_brain_png_bytes_{}_{channels}.png",
std::process::id()
));
save_png(&image, &path).unwrap();
let file = std::fs::read(&path).unwrap();
std::fs::remove_file(&path).unwrap();
assert_eq!(png_bytes(&image).unwrap(), file, "{channels} channels");
}
assert!(png_bytes(&FeatureMap::new(2, 1, 1)).is_err());
}
}
#[cfg(feature = "images")]
#[test]
fn from_image_bytes_round_trips_and_resizes() {
let mut image = FeatureMap::new(3, 5, 7);
for (index, value) in image.data.iter_mut().enumerate() {
*value = (index % 13) as f32 / 6.0 - 1.0;
}
let png = png_bytes(&image).unwrap();
let same = from_image_bytes(&png, 7, 5).unwrap();
assert_eq!(to_rgb8(&same), to_rgb8(&image));
let resized = from_image_bytes(&png, 16, 9).unwrap();
assert_eq!(
(resized.channels, resized.width, resized.height),
(3, 16, 9)
);
assert!(from_image_bytes(&png, 0, 9).is_err());
assert!(from_image_bytes(b"nope", 4, 4).is_err());
}