use crate::clip::{Layer, Norm, dense, norm, run_layer};
use crate::conv::{Dense, FeatureMap};
use crate::matrix::Matrix;
use crate::network::NetworkError;
use crate::safetensors::ShardedSafeTensors;
use crate::transformer::Precision;
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct VitEncoderConfig {
pub num_heads: usize,
pub eps: f32,
pub quick_gelu: bool,
}
impl Default for VitEncoderConfig {
fn default() -> Self {
Self {
num_heads: 12,
eps: 1e-5,
quick_gelu: true,
}
}
}
pub struct VitEncoder {
config: VitEncoderConfig,
pub(crate) patch: Dense,
class_token: Vec<f32>,
positions: Matrix,
pub(crate) pre_norm: Option<Norm>,
pub(crate) layers: Vec<Layer>,
pub(crate) post_norm: Norm,
projection: Option<Dense>,
channels: usize,
patch_size: usize,
pub(crate) exact_gelu: bool,
#[cfg(feature = "cuda")]
device: Option<crate::cuda_image::DeviceVitEncoder>,
}
impl VitEncoder {
pub fn load<P: AsRef<std::path::Path>>(
path: P,
prefix: &str,
config: VitEncoderConfig,
precision: Precision,
) -> Result<Self, NetworkError> {
let mut file = ShardedSafeTensors::open(path)?;
Self::read_at(&mut file, prefix, config, precision)
}
pub fn read_at(
file: &mut ShardedSafeTensors,
prefix: &str,
config: VitEncoderConfig,
precision: Precision,
) -> Result<Self, NetworkError> {
if config.num_heads == 0 {
return Err(NetworkError::InvalidConfig(
"an image tower needs at least one attention head".into(),
));
}
let mut base = format!("{prefix}vision_model.");
let patches = |base: &str| format!("{base}embeddings.patch_embedding.weight");
if !file.names().any(|name| name == patches(&base)) {
base = prefix.to_string();
}
let (values, shape) = file.tensor(&patches(&base))?;
let [d_model, channels, patch_rows, patch_cols] = shape[..] else {
return Err(NetworkError::InvalidConfig(format!(
"the patch embedding is {shape:?}, which is not a convolution weight"
)));
};
if patch_rows != patch_cols {
return Err(NetworkError::InvalidConfig(format!(
"the patch embedding reads a {patch_rows}x{patch_cols} patch, which is not square"
)));
}
let patch = Dense::new(
Matrix::from_vec(d_model, channels * patch_rows * patch_cols, values),
None,
)?;
let class_token = file.tensor(&format!("{base}embeddings.class_embedding"))?.0;
if class_token.len() != d_model {
return Err(NetworkError::InvalidTarget {
expected: d_model,
actual: class_token.len(),
});
}
let positions = file.matrix(&format!("{base}embeddings.position_embedding.weight"))?;
if positions.cols != d_model {
return Err(NetworkError::InvalidConfig(format!(
"the position table is {} wide and the patch embedding produces {d_model}",
positions.cols
)));
}
let patches = positions.rows - 1;
if patches.isqrt().pow(2) != patches || patches == 0 {
return Err(NetworkError::InvalidConfig(format!(
"the position table holds {} rows, which is not a square grid plus a class token",
positions.rows
)));
}
let pre_norm = norm(file, &format!("{base}pre_layrnorm"))
.or_else(|_| norm(file, &format!("{base}pre_layernorm")))?;
let mut layers = Vec::new();
while let Ok(layer) = read_layer(file, &base, layers.len(), precision) {
layers.push(layer);
}
if layers.is_empty() {
return Err(NetworkError::InvalidConfig(format!(
"{base}encoder.layers.0 is missing, so this is not an image tower"
)));
}
if d_model % config.num_heads != 0 {
return Err(NetworkError::InvalidConfig(format!(
"{} heads do not divide a width of {d_model}",
config.num_heads
)));
}
Ok(Self {
post_norm: norm(file, &format!("{base}post_layernorm"))?,
projection: file
.matrix(&format!("{prefix}visual_projection.weight"))
.ok()
.map(|weight| Dense::new(weight, None))
.transpose()?,
config,
patch,
class_token,
positions,
pre_norm: Some(pre_norm),
layers,
channels,
patch_size: patch_rows,
exact_gelu: false,
#[cfg(feature = "cuda")]
device: None,
})
}
pub fn load_dinov2<P: AsRef<std::path::Path>>(
path: P,
image_size: usize,
precision: Precision,
) -> Result<Self, NetworkError> {
let file = &mut ShardedSafeTensors::open(path)?;
let (values, shape) = file.tensor("embeddings.patch_embeddings.projection.weight")?;
let [d_model, channels, patch, patch_cols] = shape[..] else {
return Err(NetworkError::InvalidConfig(format!(
"the patch embedding is {shape:?}, which is not a convolution weight"
)));
};
if patch != patch_cols || !image_size.is_multiple_of(patch) || image_size == 0 {
return Err(NetworkError::InvalidConfig(format!(
"{image_size}-pixel images do not split into {patch}x{patch_cols} patches"
)));
}
let patch_embedding = Dense::new(
Matrix::from_vec(d_model, channels * patch * patch, values),
Some(
file.tensor("embeddings.patch_embeddings.projection.bias")?
.0,
),
)?;
let class_token = file.tensor("embeddings.cls_token")?.0;
let table = file.tensor("embeddings.position_embeddings")?.0;
let table = Matrix::from_vec(table.len() / d_model.max(1), d_model, table);
let mut layers = Vec::new();
while let Ok(layer) = read_dinov2_layer(file, layers.len(), precision) {
layers.push(layer);
}
let grid = (table.rows.max(1) - 1).isqrt();
if layers.is_empty() || class_token.len() != d_model || grid * grid + 1 != table.rows {
return Err(NetworkError::InvalidConfig(
"this is not a DINOv2 checkpoint: no layers, a class token of another width \
or a position table that is not a square grid plus a class row"
.into(),
));
}
Ok(Self {
config: VitEncoderConfig {
num_heads: d_model / 64,
eps: 1e-6,
quick_gelu: false,
},
patch: patch_embedding,
class_token,
positions: resample_positions(&table, image_size / patch),
pre_norm: None,
layers,
post_norm: norm(file, "layernorm")?,
projection: None,
channels,
patch_size: patch,
exact_gelu: true,
#[cfg(feature = "cuda")]
device: None,
})
}
pub fn config(&self) -> &VitEncoderConfig {
&self.config
}
#[cfg(feature = "cuda")]
pub fn attach_device(
&mut self,
gpu: std::sync::Arc<crate::cuda_image::ImageGpu>,
) -> Result<(), NetworkError> {
self.device = Some(crate::cuda_image::DeviceVitEncoder::upload(&gpu, self)?);
Ok(())
}
pub fn image_size(&self) -> usize {
self.patch_size * (self.positions.rows - 1).isqrt()
}
pub fn channels(&self) -> usize {
self.channels
}
pub fn d_model(&self) -> usize {
self.positions.cols
}
pub fn tokens(&self) -> usize {
self.positions.rows
}
pub fn encode(&self, image: &FeatureMap) -> Result<Matrix, NetworkError> {
#[cfg(feature = "cuda")]
if self.device.is_some() {
return Ok(self.encode_batch(std::slice::from_ref(image))?.remove(0));
}
let flat = self.flatten(image)?;
let projected = self.patch.forward(&flat)?;
let mut hidden = Matrix::new(self.positions.rows, self.d_model());
hidden.row_mut(0).copy_from_slice(&self.class_token);
for index in 0..projected.rows {
hidden
.row_mut(index + 1)
.copy_from_slice(projected.row(index));
}
for index in 0..hidden.rows {
for (value, learned) in hidden
.row_mut(index)
.iter_mut()
.zip(self.positions.row(index))
{
*value += learned;
}
}
if let Some(norm) = &self.pre_norm {
hidden = norm.forward(&hidden, self.config.eps);
}
let activation = match (self.exact_gelu, self.config.quick_gelu) {
(true, _) => crate::ffn::gelu_erf,
(false, true) => crate::clip::quick_gelu_value,
(false, false) => crate::ffn::gelu,
};
for layer in &self.layers {
run_layer(
layer,
&mut hidden,
self.config.num_heads,
self.config.eps,
activation,
false,
)?;
}
Ok(self.post_norm.forward(&hidden, self.config.eps))
}
pub fn encode_batch(&self, images: &[FeatureMap]) -> Result<Vec<Matrix>, NetworkError> {
#[cfg(feature = "cuda")]
if let Some(device) = self.device.as_ref().filter(|_| !images.is_empty()) {
let tokens = self.tokens();
let width = self.d_model();
let mut patches = Matrix::new(images.len() * tokens, self.patch.in_dim());
let mut embedding = Matrix::new(images.len() * tokens, width);
for (index, image) in images.iter().enumerate() {
let flat = self.flatten(image)?;
let first = index * tokens;
let start = (first + 1) * flat.cols;
patches.data[start..start + flat.data.len()].copy_from_slice(&flat.data);
let rows = first * width..(first + tokens) * width;
embedding.data[rows].copy_from_slice(&self.positions.data);
for (value, class) in embedding.row_mut(first).iter_mut().zip(&self.class_token) {
*value += class;
}
}
let stacked = device.forward(&patches, &embedding, tokens)?;
return Ok(stacked
.data
.chunks_exact(tokens * width)
.map(|image| Matrix::from_vec(tokens, width, image.to_vec()))
.collect());
}
images.iter().map(|image| self.encode(image)).collect()
}
pub fn embed(&self, image: &FeatureMap) -> Result<Vec<f32>, NetworkError> {
let tokens = self.encode(image)?;
match &self.projection {
Some(projection) => projection.apply(tokens.row(0)),
None => Ok(tokens.row(0).to_vec()),
}
}
fn flatten(&self, image: &FeatureMap) -> Result<Matrix, NetworkError> {
let size = self.image_size();
if image.channels != self.channels || image.height != size || image.width != size {
return Err(NetworkError::InvalidConfig(format!(
"the tower reads a {}x{size}x{size} image and this one is {}x{}x{}",
self.channels, image.channels, image.height, image.width
)));
}
let patch = self.patch_size;
let grid = size / patch;
let mut flat = Matrix::new(grid * grid, self.patch.in_dim());
for row in 0..grid {
for column in 0..grid {
let target = flat.row_mut(row * grid + column);
for channel in 0..self.channels {
for y in 0..patch {
let source = (channel * size + row * patch + y) * size + column * patch;
let start = (channel * patch + y) * patch;
target[start..start + patch]
.copy_from_slice(&image.data[source..source + patch]);
}
}
}
}
Ok(flat)
}
}
pub const CLIP_MEAN: [f32; 3] = [0.481_454_82, 0.457_827_5, 0.408_210_74];
pub const CLIP_STD: [f32; 3] = [0.268_629_54, 0.261_302_6, 0.275_777_1];
pub const IMAGENET_MEAN: [f32; 3] = [0.485, 0.456, 0.406];
pub const IMAGENET_STD: [f32; 3] = [0.229, 0.224, 0.225];
pub fn composite_rgba(
pixels: &[f32],
size: usize,
background: [f32; 3],
) -> Result<FeatureMap, NetworkError> {
if pixels.len() != size * size * 4 {
return Err(NetworkError::InvalidTarget {
expected: size * size * 4,
actual: pixels.len(),
});
}
let mut map = FeatureMap::new(3, size, size);
for (index, pixel) in pixels.as_chunks::<4>().0.iter().enumerate() {
let alpha = pixel[3];
for channel in 0..3 {
let composited = pixel[channel] * alpha + background[channel] * (1.0 - alpha);
map.data[channel * size * size + index] =
(composited - CLIP_MEAN[channel]) / CLIP_STD[channel];
}
}
Ok(map)
}
#[cfg(feature = "images")]
pub fn read_image<P: AsRef<std::path::Path>>(
path: P,
size: usize,
background: f32,
) -> Result<FeatureMap, NetworkError> {
let path = path.as_ref();
let decoded = image::open(path)
.map_err(|error| NetworkError::InvalidDataset(format!("{}: {error}", path.display())))?;
let side = size as u32;
let decoded = decoded.resize_exact(side, side, image::imageops::FilterType::CatmullRom);
if decoded.color().has_alpha() {
let rgba: Vec<f32> = decoded
.to_rgba8()
.iter()
.map(|&value| f32::from(value) / 255.0)
.collect();
return composite_rgba(&rgba, size, [background; 3]);
}
let mut map = FeatureMap::new(3, size, size);
for (index, pixel) in decoded.to_rgb8().pixels().enumerate() {
for channel in 0..3 {
map.data[channel * size * size + index] =
(f32::from(pixel[channel]) / 255.0 - CLIP_MEAN[channel]) / CLIP_STD[channel];
}
}
Ok(map)
}
fn resample_positions(table: &Matrix, grid: usize) -> Matrix {
let (from, width) = ((table.rows - 1).isqrt(), table.cols);
if from == grid {
return table.clone();
}
let cubic = |x: f32| {
const A: f32 = -0.75;
match x.abs() {
x if x <= 1.0 => ((A + 2.0) * x - (A + 3.0)) * x * x + 1.0,
x => ((A * x - 5.0 * A) * x + 8.0 * A) * x - 4.0 * A,
}
};
let scale = from as f32 / grid as f32;
let taps: Vec<[(usize, f32); 4]> = (0..grid)
.map(|out| {
let real = scale * (out as f32 + 0.5) - 0.5;
let floor = real.floor();
let t = real - floor;
std::array::from_fn(|k| {
let index = (floor as isize + k as isize - 1).clamp(0, from as isize - 1);
(index as usize, cubic(t + 1.0 - k as f32))
})
})
.collect();
let mut out = Matrix::new(grid * grid + 1, width);
out.row_mut(0).copy_from_slice(table.row(0));
for y in 0..grid {
for x in 0..grid {
let target = out.row_mut(1 + y * grid + x);
for &(sy, wy) in &taps[y] {
let mut row = vec![0.0f32; width];
for &(sx, wx) in &taps[x] {
for (r, v) in row.iter_mut().zip(table.row(1 + sy * from + sx)) {
*r += wx * v;
}
}
for (t, r) in target.iter_mut().zip(row) {
*t += wy * r;
}
}
}
}
out
}
fn read_dinov2_layer(
file: &mut ShardedSafeTensors,
index: usize,
precision: Precision,
) -> Result<Layer, NetworkError> {
let layer = format!("encoder.layer.{index}");
let scaled = |file: &mut ShardedSafeTensors, name: &str, scale: &str| {
let mut dense = dense(file, name, Precision::F32)?;
let lambda = file.tensor(scale)?.0;
for (row, &l) in lambda.iter().enumerate() {
dense.weight.row_mut(row).iter_mut().for_each(|w| *w *= l);
}
if let Some(bias) = &mut dense.bias {
bias.iter_mut().zip(&lambda).for_each(|(b, l)| *b *= l);
}
if precision == Precision::Q8 {
dense.quantize();
}
Ok::<_, NetworkError>(dense)
};
Ok(Layer {
attention_norm: norm(file, &format!("{layer}.norm1"))?,
query: dense(
file,
&format!("{layer}.attention.attention.query"),
precision,
)?,
key: dense(file, &format!("{layer}.attention.attention.key"), precision)?,
value: dense(
file,
&format!("{layer}.attention.attention.value"),
precision,
)?,
output: scaled(
file,
&format!("{layer}.attention.output.dense"),
&format!("{layer}.layer_scale1.lambda1"),
)?,
mlp_norm: norm(file, &format!("{layer}.norm2"))?,
mlp_in: dense(file, &format!("{layer}.mlp.fc1"), precision)?,
mlp_out: scaled(
file,
&format!("{layer}.mlp.fc2"),
&format!("{layer}.layer_scale2.lambda1"),
)?,
})
}
fn read_layer(
file: &mut ShardedSafeTensors,
base: &str,
index: usize,
precision: Precision,
) -> Result<Layer, NetworkError> {
let layer = format!("{base}encoder.layers.{index}");
Ok(Layer {
attention_norm: norm(file, &format!("{layer}.layer_norm1"))?,
query: dense(file, &format!("{layer}.self_attn.q_proj"), precision)?,
key: dense(file, &format!("{layer}.self_attn.k_proj"), precision)?,
value: dense(file, &format!("{layer}.self_attn.v_proj"), precision)?,
output: dense(file, &format!("{layer}.self_attn.out_proj"), precision)?,
mlp_norm: norm(file, &format!("{layer}.layer_norm2"))?,
mlp_in: dense(file, &format!("{layer}.mlp.fc1"), precision)?,
mlp_out: dense(file, &format!("{layer}.mlp.fc2"), precision)?,
})
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
use std::collections::BTreeMap;
const D_MODEL: usize = 8;
const PATCH: usize = 2;
const GRID: usize = 3;
const LAYERS: usize = 2;
pub(crate) fn checkpoint(path: &std::path::Path, identity_layers: bool) {
let mut tensors: BTreeMap<String, (Vec<usize>, Vec<f32>)> = BTreeMap::new();
let mut add = |name: String, shape: Vec<usize>, zero: bool| {
let count = shape.iter().product::<usize>();
let values = (0..count)
.map(|index| match zero {
true => 0.0,
false => ((index % 11) as f32 - 5.0) * 0.05,
})
.collect();
tensors.insert(name, (shape, values));
};
add(
"vision_model.embeddings.patch_embedding.weight".into(),
vec![D_MODEL, 3, PATCH, PATCH],
false,
);
add(
"vision_model.embeddings.class_embedding".into(),
vec![D_MODEL],
false,
);
add(
"vision_model.embeddings.position_embedding.weight".into(),
vec![GRID * GRID + 1, D_MODEL],
false,
);
for name in ["pre_layrnorm", "post_layernorm"] {
add(format!("vision_model.{name}.weight"), vec![D_MODEL], false);
add(format!("vision_model.{name}.bias"), vec![D_MODEL], false);
}
for index in 0..LAYERS {
let base = format!("vision_model.encoder.layers.{index}");
for norm in ["layer_norm1", "layer_norm2"] {
add(format!("{base}.{norm}.weight"), vec![D_MODEL], false);
add(format!("{base}.{norm}.bias"), vec![D_MODEL], false);
}
for name in ["q_proj", "k_proj", "v_proj", "out_proj"] {
let zero = identity_layers && name == "out_proj";
add(
format!("{base}.self_attn.{name}.weight"),
vec![D_MODEL, D_MODEL],
zero,
);
add(format!("{base}.self_attn.{name}.bias"), vec![D_MODEL], zero);
}
add(
format!("{base}.mlp.fc1.weight"),
vec![D_MODEL * 2, D_MODEL],
false,
);
add(format!("{base}.mlp.fc1.bias"), vec![D_MODEL * 2], false);
add(
format!("{base}.mlp.fc2.weight"),
vec![D_MODEL, D_MODEL * 2],
identity_layers,
);
add(
format!("{base}.mlp.fc2.bias"),
vec![D_MODEL],
identity_layers,
);
}
add("visual_projection.weight".into(), vec![4, D_MODEL], false);
crate::safetensors::write_checkpoint(path, &tensors);
}
fn scratch(name: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!(
"rusting-brain-vit-{name}-{}.safetensors",
std::process::id()
))
}
pub(crate) fn tiny() -> VitEncoderConfig {
VitEncoderConfig {
num_heads: 2,
..VitEncoderConfig::default()
}
}
pub(crate) fn ramp(size: usize) -> FeatureMap {
FeatureMap::from_vec(
3,
size,
size,
(0..3 * size * size)
.map(|index| ((index % 7) as f32 - 3.0) * 0.1)
.collect(),
)
.unwrap()
}
#[test]
fn the_shape_is_read_out_of_the_checkpoint() {
let path = scratch("shape");
checkpoint(&path, false);
let tower = VitEncoder::load(&path, "", tiny(), Precision::F32).unwrap();
assert_eq!(tower.d_model(), D_MODEL);
assert_eq!(tower.channels(), 3);
assert_eq!(tower.image_size(), GRID * PATCH);
assert_eq!(tower.tokens(), GRID * GRID + 1);
let tokens = tower.encode(&ramp(tower.image_size())).unwrap();
assert_eq!((tokens.rows, tokens.cols), (GRID * GRID + 1, D_MODEL));
assert!(tokens.data.iter().all(|value| value.is_finite()));
std::fs::remove_file(&path).ok();
}
#[test]
fn a_tower_saved_without_its_vision_model_prefix_loads_alike() {
let path = scratch("unprefixed");
checkpoint(&path, false);
let prefixed = VitEncoder::load(&path, "", tiny(), Precision::F32).unwrap();
let mut file = crate::safetensors::SafeTensors::open(&path).unwrap();
let names: Vec<String> = file.names().map(str::to_string).collect();
let tensors: BTreeMap<String, (Vec<usize>, Vec<f32>)> = names
.into_iter()
.map(|name| {
let (data, shape) = file.tensor(&name).unwrap();
(
name.trim_start_matches("vision_model.").to_string(),
(shape, data),
)
})
.collect();
crate::safetensors::write_checkpoint(&path, &tensors);
let unprefixed = VitEncoder::load(&path, "", tiny(), Precision::F32).unwrap();
std::fs::remove_file(&path).ok();
let image = ramp(prefixed.image_size());
assert_eq!(
unprefixed.encode(&image).unwrap().data,
prefixed.encode(&image).unwrap().data
);
}
#[test]
fn the_embedding_is_the_projected_class_token() {
let path = scratch("embed");
checkpoint(&path, false);
let tower = VitEncoder::load(&path, "", tiny(), Precision::F32).unwrap();
let image = ramp(tower.image_size());
let class = tower.encode(&image).unwrap().row(0).to_vec();
let projection = tower.projection.as_ref().unwrap();
let embedding = tower.embed(&image).unwrap();
assert_eq!(embedding.len(), 4);
for (unit, got) in embedding.iter().enumerate() {
let want: f32 = projection
.weight
.row(unit)
.iter()
.zip(&class)
.map(|(a, b)| a * b)
.sum();
assert!((got - want).abs() < 1e-5, "{got} against {want}");
}
std::fs::remove_file(&path).ok();
}
#[test]
fn zeroed_residual_branches_leave_the_embedding_alone() {
let path = scratch("identity");
checkpoint(&path, true);
let tower = VitEncoder::load(&path, "", tiny(), Precision::F32).unwrap();
let size = tower.image_size();
let image = ramp(size);
let tokens = tower.encode(&image).unwrap();
let mut expected = Matrix::new(tower.tokens(), D_MODEL);
expected.row_mut(0).copy_from_slice(&tower.class_token);
for row in 0..GRID {
for column in 0..GRID {
let mut patch = Vec::new();
for channel in 0..3 {
for y in 0..PATCH {
for x in 0..PATCH {
let index =
(channel * size + row * PATCH + y) * size + column * PATCH + x;
patch.push(image.data[index]);
}
}
}
let target = expected.row_mut(row * GRID + column + 1);
for (unit, weight) in target.iter_mut().enumerate() {
*weight = tower
.patch
.weight
.row(unit)
.iter()
.zip(&patch)
.map(|(a, b)| a * b)
.sum();
}
}
}
for index in 0..expected.rows {
for (value, learned) in expected
.row_mut(index)
.iter_mut()
.zip(tower.positions.row(index))
{
*value += learned;
}
}
let expected = tower
.pre_norm
.as_ref()
.unwrap()
.forward(&expected, tiny().eps);
let expected = tower.post_norm.forward(&expected, tiny().eps);
for (got, want) in tokens.data.iter().zip(&expected.data) {
assert!((got - want).abs() < 1e-5, "{got} against {want}");
}
std::fs::remove_file(&path).ok();
}
#[test]
fn an_image_of_the_wrong_size_is_refused() {
let path = scratch("wrong-size");
checkpoint(&path, false);
let tower = VitEncoder::load(&path, "", tiny(), Precision::F32).unwrap();
assert!(tower.encode(&ramp(tower.image_size() + PATCH)).is_err());
std::fs::remove_file(&path).ok();
}
#[test]
fn heads_that_do_not_divide_the_width_are_refused() {
let path = scratch("bad-heads");
checkpoint(&path, false);
let config = VitEncoderConfig {
num_heads: 3,
..tiny()
};
assert!(VitEncoder::load(&path, "", config, Precision::F32).is_err());
std::fs::remove_file(&path).ok();
}
#[test]
fn alpha_chooses_between_the_pixel_and_the_background() {
let opaque = composite_rgba(&[0.5, 0.5, 0.5, 1.0], 1, [0.0, 0.0, 0.0]).unwrap();
let clear = composite_rgba(&[0.5, 0.5, 0.5, 0.0], 1, [1.0, 1.0, 1.0]).unwrap();
for channel in 0..3 {
let expected = (0.5 - CLIP_MEAN[channel]) / CLIP_STD[channel];
assert!((opaque.data[channel] - expected).abs() < 1e-6);
let expected = (1.0 - CLIP_MEAN[channel]) / CLIP_STD[channel];
assert!((clear.data[channel] - expected).abs() < 1e-6);
}
assert!(composite_rgba(&[0.0; 3], 1, [0.0; 3]).is_err());
}
#[test]
#[ignore]
fn dinov2_matches_transformers() {
let dir = std::path::PathBuf::from(std::env::var("RB_DINOV2_REF").unwrap());
let paths: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(dir.join("reference.json")).unwrap())
.unwrap();
for name in ["small", "base"] {
let tower = VitEncoder::load_dinov2(paths[name].as_str().unwrap(), 224, Precision::F32)
.unwrap();
let mut reference =
crate::safetensors::SafeTensors::open(dir.join(format!("{name}.safetensors")))
.unwrap();
let (pixels, _) = reference.tensor("pixels").unwrap();
let (tokens, _) = reference.tensor("tokens").unwrap();
let per_image = 3 * 224 * 224;
let mut images = Vec::new();
for image in 0..2 {
let mut map = FeatureMap::new(3, 224, 224);
map.data
.copy_from_slice(&pixels[image * per_image..(image + 1) * per_image]);
images.push(map);
}
for (image, got) in images.iter().map(|m| tower.encode(m).unwrap()).enumerate() {
let expected = &tokens[image * got.data.len()..(image + 1) * got.data.len()];
let (got, expected) = (&got.data[got.cols..], &expected[got.cols..]);
let scale = expected.iter().fold(0.0f32, |m, v| m.max(v.abs()));
let gap = got
.iter()
.zip(expected)
.fold(0.0f32, |m, (a, b)| m.max((a - b).abs()));
eprintln!("dinov2-{name} image {image}: gap {gap:e} at scale {scale}");
assert!(
gap <= 1e-3 * scale,
"dinov2-{name}: gap {gap} at scale {scale}"
);
}
}
}
}