use crate::conv::Dense;
use crate::matrix::Matrix;
use crate::mmdit::{attention, layer_norm};
use crate::network::NetworkError;
use crate::safetensors::ShardedSafeTensors;
use crate::tokenizer::Bpe;
use crate::transformer::Precision;
use rayon::prelude::*;
#[derive(Clone, Debug, PartialEq)]
pub struct ClipTextConfig {
pub vocab_size: usize,
pub d_model: usize,
pub layers: usize,
pub num_heads: usize,
pub mlp_dim: usize,
pub max_positions: usize,
pub eps: f32,
pub quick_gelu: bool,
pub projection_dim: Option<usize>,
pub bos_token: u32,
pub eos_token: u32,
pub argmax_pool: bool,
}
impl Default for ClipTextConfig {
fn default() -> Self {
Self {
vocab_size: 49408,
d_model: 768,
layers: 12,
num_heads: 12,
mlp_dim: 3072,
max_positions: 77,
eps: 1e-5,
quick_gelu: true,
projection_dim: None,
bos_token: 49406,
eos_token: 49407,
argmax_pool: true,
}
}
}
impl ClipTextConfig {
pub fn from_file(path: impl AsRef<std::path::Path>) -> Result<Self, NetworkError> {
let text = std::fs::read_to_string(path)?;
let json: serde_json::Value = serde_json::from_str(&text).map_err(|error| {
NetworkError::InvalidDataset(format!("text encoder config.json: {error}"))
})?;
let json = json.get("text_config").unwrap_or(&json).clone();
let default = Self::default();
let number = |name: &str, fallback: usize| -> usize {
json.get(name)
.and_then(|value| value.as_u64())
.map_or(fallback, |value| value as usize)
};
let token = |name: &str, fallback: u32| -> u32 {
json.get(name)
.and_then(|value| value.as_u64())
.map_or(fallback, |value| value as u32)
};
Ok(Self {
vocab_size: number("vocab_size", default.vocab_size),
d_model: number("hidden_size", default.d_model),
layers: number("num_hidden_layers", default.layers),
num_heads: number("num_attention_heads", default.num_heads),
mlp_dim: number("intermediate_size", default.mlp_dim),
max_positions: number("max_position_embeddings", default.max_positions),
eps: json
.get("layer_norm_eps")
.and_then(|value| value.as_f64())
.unwrap_or(default.eps as f64) as f32,
quick_gelu: json
.get("hidden_act")
.and_then(|value| value.as_str())
.is_none_or(|name| name == "quick_gelu"),
projection_dim: json
.get("projection_dim")
.and_then(|value| value.as_u64())
.map(|value| value as usize),
bos_token: token("bos_token_id", default.bos_token),
eos_token: token("eos_token_id", default.eos_token),
argmax_pool: false,
}
.legacy_special_tokens())
}
fn legacy_special_tokens(mut self) -> Self {
if self.eos_token == 2 && self.vocab_size >= 2 {
self.bos_token = self.vocab_size as u32 - 2;
self.eos_token = self.vocab_size as u32 - 1;
self.argmax_pool = true;
}
self
}
pub fn head_dim(&self) -> usize {
self.d_model / self.num_heads.max(1)
}
}
pub(crate) struct Layer {
pub(crate) attention_norm: Norm,
pub(crate) query: Dense,
pub(crate) key: Dense,
pub(crate) value: Dense,
pub(crate) output: Dense,
pub(crate) mlp_norm: Norm,
pub(crate) mlp_in: Dense,
pub(crate) mlp_out: Dense,
}
pub(crate) struct Norm {
pub(crate) weight: Vec<f32>,
pub(crate) bias: Vec<f32>,
}
impl Norm {
pub(crate) fn forward(&self, tokens: &Matrix, eps: f32) -> Matrix {
let mut output = layer_norm(tokens, eps);
output.data.par_chunks_mut(tokens.cols).for_each(|row| {
for ((value, weight), bias) in row.iter_mut().zip(&self.weight).zip(&self.bias) {
*value = *value * weight + bias;
}
});
output
}
}
pub struct ClipTextEncoder {
config: ClipTextConfig,
tokens: Matrix,
positions: Matrix,
pub(crate) layers: Vec<Layer>,
pub(crate) final_norm: Norm,
projection: Option<Dense>,
pad: u32,
#[cfg(feature = "cuda")]
device: Option<crate::cuda_image::DeviceClip>,
}
impl ClipTextEncoder {
pub fn config(&self) -> &ClipTextConfig {
&self.config
}
pub fn load(
path: impl AsRef<std::path::Path>,
prefix: &str,
config: ClipTextConfig,
) -> Result<Self, NetworkError> {
Self::load_at(path, prefix, config, Precision::F32)
}
pub fn load_at(
path: impl AsRef<std::path::Path>,
prefix: &str,
config: ClipTextConfig,
precision: Precision,
) -> Result<Self, NetworkError> {
let mut file = ShardedSafeTensors::open(path)?;
Self::read_at(&mut file, prefix, config, precision)
}
pub fn read_at(
file: &mut ShardedSafeTensors,
prefix: &str,
config: ClipTextConfig,
precision: Precision,
) -> Result<Self, NetworkError> {
let mut base = format!("{prefix}text_model.");
let token_key = |base: &str| format!("{base}embeddings.token_embedding.weight");
if !file.names().any(|name| name == token_key(&base)) {
base = prefix.to_string();
}
let tokens = file.matrix(&token_key(&base))?;
let positions = file.matrix(&format!("{base}embeddings.position_embedding.weight"))?;
if tokens.cols != config.d_model {
return Err(NetworkError::InvalidConfig(format!(
"the checkpoint's embeddings are {} wide and the configuration says {}",
tokens.cols, config.d_model
)));
}
let eos = config.eos_token;
let layers = (0..config.layers)
.map(|index| {
let layer = format!("{base}encoder.layers.{index}");
Ok(Layer {
attention_norm: norm(file, &format!("{layer}.layer_norm1"))?,
query: dense(file, &format!("{layer}.self_attn.q_proj"), precision)?,
key: dense(file, &format!("{layer}.self_attn.k_proj"), precision)?,
value: dense(file, &format!("{layer}.self_attn.v_proj"), precision)?,
output: dense(file, &format!("{layer}.self_attn.out_proj"), precision)?,
mlp_norm: norm(file, &format!("{layer}.layer_norm2"))?,
mlp_in: dense(file, &format!("{layer}.mlp.fc1"), precision)?,
mlp_out: dense(file, &format!("{layer}.mlp.fc2"), precision)?,
})
})
.collect::<Result<Vec<_>, NetworkError>>()?;
Ok(Self {
final_norm: norm(file, &format!("{base}final_layer_norm"))?,
projection: file
.matrix(&format!("{prefix}text_projection.weight"))
.ok()
.map(|weight| Dense::new(weight, None))
.transpose()?,
config,
tokens,
pad: eos,
positions,
layers,
#[cfg(feature = "cuda")]
device: None,
})
}
pub fn tokenize(&self, tokenizer: &Bpe, prompt: &str) -> Result<Vec<u32>, NetworkError> {
let mut ids = vec![self.config.bos_token];
let room = self.config.max_positions.saturating_sub(2);
ids.extend(tokenizer.encode(prompt)?.into_iter().take(room));
ids.push(self.config.eos_token);
ids.resize(self.config.max_positions, self.pad);
Ok(ids)
}
pub fn add_token(
&mut self,
tokenizer: &mut Bpe,
token: &str,
vectors: &Matrix,
) -> Result<u32, NetworkError> {
if vectors.cols != self.config.d_model || tokenizer.vocab_size() != self.tokens.rows {
return Err(NetworkError::InvalidConfig(format!(
"a {}-wide embedding for a tokenizer of {} ids cannot join a {}-wide table of {} rows",
vectors.cols,
tokenizer.vocab_size(),
self.config.d_model,
self.tokens.rows
)));
}
let first = tokenizer.add_token(token, vectors.rows)?;
self.tokens.data.extend_from_slice(&vectors.data);
self.tokens.rows += vectors.rows;
self.config.vocab_size = self.tokens.rows;
Ok(first)
}
pub fn set_pad_token(&mut self, id: u32) {
self.pad = id;
}
pub fn forward(&self, ids: &[u32]) -> Result<Matrix, NetworkError> {
Ok(self.forward_skipping(ids, 0)?.1)
}
pub fn forward_skipping(
&self,
ids: &[u32],
skip: usize,
) -> Result<(Matrix, Matrix), NetworkError> {
if ids.is_empty() {
return Err(NetworkError::InvalidConfig(
"an empty prompt has no hidden states".into(),
));
}
if ids.len() > self.positions.rows {
return Err(NetworkError::InvalidTarget {
expected: self.positions.rows,
actual: ids.len(),
});
}
let mut hidden = Matrix::new(ids.len(), self.config.d_model);
for (position, id) in ids.iter().enumerate() {
if *id as usize >= self.tokens.rows {
return Err(NetworkError::InvalidTarget {
expected: self.tokens.rows,
actual: *id as usize,
});
}
let row = hidden.row_mut(position);
row.copy_from_slice(self.tokens.row(*id as usize));
for (value, learned) in row.iter_mut().zip(self.positions.row(position)) {
*value += learned;
}
}
#[cfg(feature = "cuda")]
if let Some(device) = &self.device {
return device.forward(&hidden, skip);
}
let heads = self.config.num_heads;
let stop = self.layers.len().saturating_sub(skip);
let mut skipped = None;
for (index, layer) in self.layers.iter().enumerate() {
if index == stop {
skipped = Some(hidden.clone());
}
run_layer(
layer,
&mut hidden,
heads,
self.config.eps,
match self.config.quick_gelu {
true => quick_gelu_value,
false => crate::ffn::gelu,
},
true,
)?;
}
let normalized = self.final_norm.forward(&hidden, self.config.eps);
Ok((skipped.unwrap_or_else(|| normalized.clone()), normalized))
}
#[cfg(feature = "cuda")]
pub fn attach_device(
&mut self,
gpu: std::sync::Arc<crate::cuda_image::ImageGpu>,
) -> Result<(), NetworkError> {
self.device = Some(crate::cuda_image::DeviceClip::upload(&gpu, self)?);
Ok(())
}
pub fn pool(&self, ids: &[u32], hidden: &Matrix) -> Result<Vec<f32>, NetworkError> {
let position = match self.config.argmax_pool {
true => ids
.iter()
.enumerate()
.rev()
.max_by_key(|(_, id)| **id)
.map(|(at, _)| at),
false => ids.iter().position(|id| *id == self.config.eos_token),
}
.unwrap_or(hidden.rows - 1)
.min(hidden.rows - 1);
let pooled = hidden.row(position).to_vec();
match &self.projection {
Some(projection) => projection.apply(&pooled),
None => Ok(pooled),
}
}
pub fn encode(
&self,
tokenizer: &Bpe,
prompt: &str,
) -> Result<(Matrix, Vec<f32>), NetworkError> {
let ids = self.tokenize(tokenizer, prompt)?;
let hidden = self.forward(&ids)?;
let pooled = self.pool(&ids, &hidden)?;
Ok((hidden, pooled))
}
}
pub(crate) fn run_layer(
layer: &Layer,
hidden: &mut Matrix,
heads: usize,
eps: f32,
activation: fn(f32) -> f32,
causal: bool,
) -> Result<(), NetworkError> {
let normed = layer.attention_norm.forward(hidden, eps);
let attended = attention(
&layer.query.forward(&normed)?,
&layer.key.forward(&normed)?,
&layer.value.forward(&normed)?,
heads,
heads,
causal,
);
add(hidden, &layer.output.forward(&attended)?);
let normed = layer.mlp_norm.forward(hidden, eps);
let mut wide = layer.mlp_in.forward(&normed)?;
wide.data
.par_iter_mut()
.for_each(|value| *value = activation(*value));
add(hidden, &layer.mlp_out.forward(&wide)?);
Ok(())
}
pub(crate) fn quick_gelu_value(value: f32) -> f32 {
value / (1.0 + (-1.702 * value).exp())
}
pub(crate) fn add(target: &mut Matrix, source: &Matrix) {
target
.data
.par_iter_mut()
.zip(source.data.par_iter())
.for_each(|(target, source)| *target += source);
}
pub fn read_embedding(
path: impl AsRef<std::path::Path>,
width: usize,
) -> Result<Matrix, NetworkError> {
let mut file = crate::safetensors::SafeTensors::open(path)?;
let names: Vec<String> = file
.names()
.filter(|name| {
let shape = &file.info(name).expect("listed").shape;
matches!(shape.len(), 1 | 2) && shape.last() == Some(&width)
})
.map(str::to_string)
.collect();
let [name] = names.as_slice() else {
return Err(NetworkError::InvalidConfig(format!(
"a textual inversion needs one tensor {width} wide, and this file has {names:?}"
)));
};
let (data, _) = file.tensor(name)?;
Ok(Matrix::from_vec(data.len() / width, width, data))
}
pub fn logit_scale(file: &mut ShardedSafeTensors, prefix: &str) -> Result<f32, NetworkError> {
let (values, _) = file.tensor(&format!("{prefix}logit_scale"))?;
match values[..] {
[value] => Ok(value.exp()),
_ => Err(NetworkError::InvalidTarget {
expected: 1,
actual: values.len(),
}),
}
}
pub fn zero_shot(image: &[f32], captions: &[Vec<f32>], scale: f32) -> Vec<f32> {
let unit = |vector: &[f32]| {
let length = vector.iter().map(|value| value * value).sum::<f32>().sqrt();
vector
.iter()
.map(|value| value / length)
.collect::<Vec<_>>()
};
let image = unit(image);
let logits: Vec<f32> = captions
.iter()
.map(|caption| {
scale
* unit(caption)
.iter()
.zip(&image)
.map(|(a, b)| a * b)
.sum::<f32>()
})
.collect();
let top = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let weights: Vec<f32> = logits.iter().map(|logit| (logit - top).exp()).collect();
let total: f32 = weights.iter().sum();
weights.iter().map(|weight| weight / total).collect()
}
pub(crate) fn norm(file: &mut ShardedSafeTensors, name: &str) -> Result<Norm, NetworkError> {
Ok(Norm {
weight: file.tensor(&format!("{name}.weight"))?.0,
bias: file.tensor(&format!("{name}.bias"))?.0,
})
}
pub(crate) fn dense(
file: &mut ShardedSafeTensors,
name: &str,
precision: Precision,
) -> Result<Dense, NetworkError> {
let (values, shape) = file.tensor(&format!("{name}.weight"))?;
if shape.len() != 2 {
return Err(NetworkError::InvalidConfig(format!(
"{name}.weight is {shape:?}, which is not a linear layer"
)));
}
let bias = file
.tensor(&format!("{name}.bias"))
.ok()
.map(|(bias, _)| bias);
let mut layer = Dense::new(Matrix::from_vec(shape[0], shape[1], values), bias)?;
if precision == Precision::Q8 {
layer.quantize();
}
Ok(layer)
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
use std::collections::BTreeMap;
pub(crate) fn tiny() -> ClipTextConfig {
ClipTextConfig {
vocab_size: 24,
d_model: 16,
layers: 2,
num_heads: 4,
mlp_dim: 32,
max_positions: 8,
eps: 1e-5,
quick_gelu: true,
projection_dim: None,
bos_token: 20,
eos_token: 21,
argmax_pool: false,
}
}
pub(crate) fn checkpoint(config: &ClipTextConfig, path: &std::path::Path, projection: bool) {
let mut tensors: BTreeMap<String, (Vec<usize>, Vec<f32>)> = BTreeMap::new();
let mut add = |name: String, shape: Vec<usize>| {
let count = shape.iter().product::<usize>();
let values = (0..count)
.map(|index| ((index % 11) as f32 - 5.0) * 0.05)
.collect();
tensors.insert(name, (shape, values));
};
add(
"text_model.embeddings.token_embedding.weight".into(),
vec![config.vocab_size, config.d_model],
);
add(
"text_model.embeddings.position_embedding.weight".into(),
vec![config.max_positions, config.d_model],
);
for index in 0..config.layers {
let base = format!("text_model.encoder.layers.{index}");
for norm in ["layer_norm1", "layer_norm2"] {
add(format!("{base}.{norm}.weight"), vec![config.d_model]);
add(format!("{base}.{norm}.bias"), vec![config.d_model]);
}
for name in ["q_proj", "k_proj", "v_proj", "out_proj"] {
add(
format!("{base}.self_attn.{name}.weight"),
vec![config.d_model, config.d_model],
);
add(
format!("{base}.self_attn.{name}.bias"),
vec![config.d_model],
);
}
add(
format!("{base}.mlp.fc1.weight"),
vec![config.mlp_dim, config.d_model],
);
add(format!("{base}.mlp.fc1.bias"), vec![config.mlp_dim]);
add(
format!("{base}.mlp.fc2.weight"),
vec![config.d_model, config.mlp_dim],
);
add(format!("{base}.mlp.fc2.bias"), vec![config.d_model]);
}
add(
"text_model.final_layer_norm.weight".into(),
vec![config.d_model],
);
add(
"text_model.final_layer_norm.bias".into(),
vec![config.d_model],
);
if let Some(width) = config.projection_dim.filter(|_| projection) {
add("text_projection.weight".into(), vec![width, config.d_model]);
}
crate::safetensors::write_checkpoint(path, &tensors);
}
fn encoder(config: &ClipTextConfig, name: &str, precision: Precision) -> ClipTextEncoder {
let path = std::env::temp_dir().join(name);
checkpoint(config, &path, true);
let encoder = ClipTextEncoder::load_at(&path, "", config.clone(), precision).unwrap();
std::fs::remove_file(path).ok();
encoder
}
#[test]
fn a_tower_saved_without_its_text_model_prefix_loads_alike() {
let config = tiny();
let path = std::env::temp_dir().join("rusting_brain_clip_unprefixed.safetensors");
checkpoint(&config, &path, true);
let mut file = crate::safetensors::SafeTensors::open(&path).unwrap();
let names: Vec<String> = file.names().map(str::to_string).collect();
let tensors: BTreeMap<String, (Vec<usize>, Vec<f32>)> = names
.into_iter()
.map(|name| {
let (data, shape) = file.tensor(&name).unwrap();
(
name.trim_start_matches("text_model.").to_string(),
(shape, data),
)
})
.collect();
crate::safetensors::write_checkpoint(&path, &tensors);
let unprefixed = ClipTextEncoder::load(&path, "", config.clone()).unwrap();
std::fs::remove_file(path).ok();
let prefixed = encoder(
&config,
"rusting_brain_clip_prefixed.safetensors",
Precision::F32,
);
let ids = [20, 3, 7, 21];
assert_eq!(
unprefixed.forward(&ids).unwrap().data,
prefixed.forward(&ids).unwrap().data
);
}
#[test]
fn a_prompt_becomes_one_hidden_state_per_position() {
let config = tiny();
let encoder = encoder(
&config,
"rusting_brain_clip_forward.safetensors",
Precision::F32,
);
let hidden = encoder.forward(&[20, 3, 7, 21]).unwrap();
assert_eq!((hidden.rows, hidden.cols), (4, config.d_model));
assert!(hidden.data.iter().all(|value| value.is_finite()));
}
#[test]
fn the_pooled_vector_is_read_at_the_end_token_and_not_the_last_position() {
let config = tiny();
let encoder = encoder(
&config,
"rusting_brain_clip_pool.safetensors",
Precision::F32,
);
let ids = [20, 3, 7, 21, 21, 21, 21, 21];
let hidden = encoder.forward(&ids).unwrap();
let pooled = encoder.pool(&ids, &hidden).unwrap();
assert_eq!(pooled, hidden.row(3).to_vec());
assert_ne!(pooled, hidden.row(hidden.rows - 1).to_vec());
}
#[test]
fn a_prompt_is_wrapped_and_padded_to_the_position_table() {
let config = tiny();
let encoder = encoder(
&config,
"rusting_brain_clip_tokens.safetensors",
Precision::F32,
);
let tokenizer = crate::tokenizer::tests::tokenizer(&["h i"], &["hi"], None);
let ids = encoder.tokenize(&tokenizer, "hi").unwrap();
assert_eq!(ids.len(), config.max_positions);
assert_eq!(ids[0], config.bos_token);
assert!(ids[1..].contains(&config.eos_token));
let long = encoder.tokenize(&tokenizer, "hihihihihihi").unwrap();
assert_eq!(long.len(), config.max_positions);
assert_eq!(long[config.max_positions - 1], config.eos_token);
}
#[test]
fn a_named_pad_token_fills_after_the_end_token() {
let config = tiny();
let mut encoder = encoder(
&config,
"rusting_brain_clip_pad.safetensors",
Precision::F32,
);
let tokenizer = crate::tokenizer::tests::tokenizer(&["h i"], &["hi"], None);
encoder.set_pad_token(0);
let ids = encoder.tokenize(&tokenizer, "hi").unwrap();
let end = ids.iter().position(|id| *id == config.eos_token).unwrap();
assert!(ids[end + 1..].iter().all(|id| *id == 0));
}
#[test]
fn zero_shot_is_a_softmax_over_scaled_cosines() {
let image = [3.0, 0.0];
let captions = [vec![2.0, 0.0], vec![0.0, 5.0], vec![-0.5, 0.0]];
let probabilities = zero_shot(&image, &captions, 1.0);
let total: f32 = [1.0f32, 0.0, -1.0].iter().map(|x| x.exp()).sum();
for (got, logit) in probabilities.iter().zip([1.0f32, 0.0, -1.0]) {
assert!((got - logit.exp() / total).abs() < 1e-6, "{got}");
}
assert!(zero_shot(&image, &captions, 100.0)[0] > 0.999);
}
#[test]
fn a_projection_widens_the_pooled_vector() {
let mut config = tiny();
config.projection_dim = Some(6);
let encoder = encoder(
&config,
"rusting_brain_clip_projection.safetensors",
Precision::F32,
);
let ids = [20, 3, 21];
let hidden = encoder.forward(&ids).unwrap();
assert_eq!(encoder.pool(&ids, &hidden).unwrap().len(), 6);
}
#[test]
fn a_quantized_tower_says_what_the_float_one_does() {
let config = tiny();
let ids = [20, 3, 7, 12, 21];
let float = encoder(
&config,
"rusting_brain_clip_f32.safetensors",
Precision::F32,
)
.forward(&ids)
.unwrap();
let quantized = encoder(&config, "rusting_brain_clip_q8.safetensors", Precision::Q8)
.forward(&ids)
.unwrap();
for (float, quantized) in float.data.iter().zip(&quantized.data) {
assert!(
(float - quantized).abs() < 0.05,
"{float} against {quantized}"
);
}
}
#[test]
fn a_published_config_states_the_shape() {
let path = std::env::temp_dir().join("rusting_brain_clip_config.json");
std::fs::write(
&path,
r#"{"hidden_size": 1280, "num_hidden_layers": 32, "num_attention_heads": 20,
"intermediate_size": 5120, "max_position_embeddings": 77, "vocab_size": 49408,
"layer_norm_eps": 1e-5, "hidden_act": "gelu", "projection_dim": 1280,
"bos_token_id": 0, "eos_token_id": 2}"#,
)
.unwrap();
let config = ClipTextConfig::from_file(&path).unwrap();
std::fs::remove_file(path).ok();
assert_eq!(config.d_model, 1280);
assert_eq!(config.layers, 32);
assert_eq!(config.head_dim(), 64);
assert_eq!(config.projection_dim, Some(1280));
assert_eq!((config.bos_token, config.eos_token), (49406, 49407));
assert!(!config.quick_gelu);
}
#[test]
fn a_token_past_the_vocabulary_is_refused() {
let config = tiny();
let encoder = encoder(
&config,
"rusting_brain_clip_range.safetensors",
Precision::F32,
);
assert!(encoder.forward(&[20, 900, 21]).is_err());
assert!(encoder.forward(&[]).is_err());
assert!(encoder.forward(&[20; 9]).is_err());
}
#[test]
fn skipping_the_last_layer_reads_a_state_the_end_of_the_tower_does_not() {
let config = ClipTextConfig {
vocab_size: 320,
bos_token: 300,
eos_token: 301,
..tiny()
};
let encoder = encoder(
&config,
"rusting_brain_clip_skip.safetensors",
Precision::F32,
);
let ids = encoder
.tokenize(
&crate::tokenizer::tests::tokenizer(&["h i"], &["hi"], None),
"a lighthouse",
)
.unwrap();
let (skipped, normalized) = encoder.forward_skipping(&ids, 1).unwrap();
assert_eq!(skipped.rows, normalized.rows);
assert_ne!(skipped.data, normalized.data);
let (states, end) = encoder.forward_skipping(&ids, 0).unwrap();
assert_eq!(states.data, end.data);
assert_eq!(end.data, normalized.data);
}
#[test]
#[ignore]
fn textual_inversion_matches_diffusers() {
let dir = std::path::PathBuf::from(std::env::var("RB_TI_REF").unwrap());
let reference: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(dir.join("reference.json")).unwrap())
.unwrap();
let floats = |value: &serde_json::Value| -> Vec<f32> {
let values = value.as_array().unwrap().iter();
values.map(|x| x.as_f64().unwrap() as f32).collect()
};
let prompt = reference["prompt"].as_str().unwrap();
let towers = [
(
"tokenizer",
&["diffusers", "a1111", "sdxl"][..],
&["<cat-toy>", "<style>", "<sdxl>"][..],
),
("tokenizer_2", &["sdxl"][..], &["<sdxl>"][..]),
];
for (name, files, tokens) in towers {
let tower = dir.join(name);
let config = ClipTextConfig::from_file(tower.join("config.json")).unwrap();
let mut encoder =
ClipTextEncoder::load(tower.join("model.safetensors"), "", config).unwrap();
let mut tokenizer = Bpe::from_file(tower.join("tokenizer.json")).unwrap();
if name == "tokenizer_2" {
encoder.set_pad_token(tokenizer.token_id("!").unwrap());
}
for (file, token) in files.iter().zip(tokens) {
let path = dir.join(format!("{file}.safetensors"));
let vectors = read_embedding(&path, encoder.config().d_model).unwrap();
encoder.add_token(&mut tokenizer, token, &vectors).unwrap();
}
let expected = &reference[name];
let ids = encoder.tokenize(&tokenizer, prompt).unwrap();
let want: Vec<u32> = expected["ids"]
.as_array()
.unwrap()
.iter()
.map(|id| id.as_u64().unwrap() as u32)
.collect();
assert_eq!(ids, want, "{name}");
let (hidden, pooled) = encoder.encode(&tokenizer, prompt).unwrap();
for (got, want) in [
(hidden.data, floats(&expected["hidden"])),
(pooled, floats(&expected["pooled"])),
] {
assert_eq!(got.len(), want.len());
let worst = got
.iter()
.zip(&want)
.map(|(a, b)| (a - b).abs())
.fold(0.0, f32::max);
assert!(worst < 1e-5, "{name}: off by {worst}");
}
}
}
}