use std::fs;
use std::path::Path;
#[cfg(not(target_arch = "wasm32"))]
use std::path::PathBuf;
#[cfg(not(target_arch = "wasm32"))]
use std::sync::atomic::{AtomicU64, Ordering};
#[cfg(not(target_arch = "wasm32"))]
use std::time::{Duration, SystemTime};
use safetensors::SafeTensors;
use safetensors::tensor::Dtype;
use sha2::{Digest, Sha256};
use tokenizers::Tokenizer;
#[cfg(not(target_arch = "wasm32"))]
use crate::config::Config;
use crate::digest::hex;
use crate::domain::{self, Embedding, Error};
use crate::embed::Embedder;
const DEFAULT_MAX_TOKENS: usize = 512;
const ARTIFACT_FILES: [&str; 3] = ["model.safetensors", "tokenizer.json", "config.json"];
const POTION_BASE_8M_SHA256: &str =
"81c3592150873b1c5a8c4262850f795bff4fd568fbde80ac69889d087f16a0b4";
#[cfg(not(target_arch = "wasm32"))]
const POTION_BASE_8M_URLS: [&str; 3] = [
"https://huggingface.co/minishlab/potion-base-8M/resolve/bf8b056651a2c21b8d2565580b8569da283cab23/model.safetensors",
"https://huggingface.co/minishlab/potion-base-8M/resolve/bf8b056651a2c21b8d2565580b8569da283cab23/tokenizer.json",
"https://huggingface.co/minishlab/potion-base-8M/resolve/bf8b056651a2c21b8d2565580b8569da283cab23/config.json",
];
const POTION_BASE_8M_NAME: &str = "potion-base-8M";
#[cfg(not(target_arch = "wasm32"))]
const POTION_BASE_8M_BYTES: u64 = 30_920_628;
const POTION_BASE_8M_LICENSE: &str = "MIT";
#[cfg(not(target_arch = "wasm32"))]
const POTION_BASE_8M_LICENSE_URL: &str = "https://opensource.org/license/mit";
#[cfg(not(target_arch = "wasm32"))]
const POTION_BASE_8M: Pin<'static> = Pin {
name: POTION_BASE_8M_NAME,
bytes: POTION_BASE_8M_BYTES,
digest: POTION_BASE_8M_SHA256,
urls: POTION_BASE_8M_URLS,
license: POTION_BASE_8M_LICENSE,
license_url: POTION_BASE_8M_LICENSE_URL,
};
#[cfg(not(target_arch = "wasm32"))]
fn potion_dir(models_dir: &Path) -> PathBuf {
models_dir.join(POTION_BASE_8M_NAME)
}
#[cfg(not(target_arch = "wasm32"))]
#[derive(Debug, Clone, Copy)]
struct Pin<'a> {
name: &'a str,
bytes: u64,
digest: &'a str,
urls: [&'a str; 3],
license: &'a str,
license_url: &'a str,
}
#[cfg(not(target_arch = "wasm32"))]
use crate::domain::limits::MAX_DOWNLOAD_BYTES as MAX_ARTIFACT_BYTES;
#[cfg(not(target_arch = "wasm32"))]
const FETCH_TIMEOUT: Duration = Duration::from_secs(300);
#[cfg(not(target_arch = "wasm32"))]
const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
#[cfg(not(target_arch = "wasm32"))]
const TEMP_FILE_PREFIX: &str = ".tmp.";
#[cfg(not(target_arch = "wasm32"))]
const STALE_TEMP_AGE: Duration = Duration::from_secs(600);
pub struct Model2Vec {
matrix: Vec<f32>,
dim: usize,
weights: Option<Vec<f32>>,
mapping: Option<Vec<usize>>,
tokenizer: Tokenizer,
unk_id: Option<u32>,
median_token_length: usize,
normalize: bool,
model_hash: String,
}
impl Model2Vec {
pub fn from_dir(dir: impl AsRef<Path>) -> domain::Result<Self> {
let dir = dir.as_ref();
let [model_path, tokenizer_path, config_path] = ARTIFACT_FILES.map(|f| dir.join(f));
for p in [&model_path, &tokenizer_path, &config_path] {
if !p.exists() {
return Err(Error::ModelNotFound(p.clone()));
}
}
let model_bytes = fs::read(&model_path).map_err(|e| io_at("read", &model_path, &e))?;
let tokenizer_bytes =
fs::read(&tokenizer_path).map_err(|e| io_at("read", &tokenizer_path, &e))?;
let config_bytes = fs::read(&config_path).map_err(|e| io_at("read", &config_path, &e))?;
catch_embed_panic(|| Self::from_bytes(&model_bytes, &tokenizer_bytes, &config_bytes))
}
#[cfg(not(target_arch = "wasm32"))]
pub fn potion_base_8m(models_dir: impl AsRef<Path>) -> domain::Result<Self> {
Self::potion_base_8m_with(&potion_dir(models_dir.as_ref()), fetch_capped, &mut |_| {})
}
#[cfg(not(target_arch = "wasm32"))]
pub fn from_config(cfg: &Config) -> domain::Result<Self> {
Self::potion_base_8m_with(
&cfg.model_dir().join(cfg.embedding_model()),
fetch_capped,
&mut |_| {},
)
}
#[cfg(not(target_arch = "wasm32"))]
pub fn from_config_with_notice(
cfg: &Config,
mut notice: impl FnMut(&domain::ProvisionNotice),
) -> domain::Result<Self> {
Self::potion_base_8m_with(
&cfg.model_dir().join(cfg.embedding_model()),
fetch_capped,
&mut notice,
)
}
#[cfg(not(target_arch = "wasm32"))]
fn potion_base_8m_with<F>(
dir: &Path,
fetch: F,
notice: &mut dyn FnMut(&domain::ProvisionNotice),
) -> domain::Result<Self>
where
F: Fn(&str) -> domain::Result<Vec<u8>>,
{
Self::provision(dir, &POTION_BASE_8M, notice, &fetch)
}
#[cfg(not(target_arch = "wasm32"))]
fn provision<F>(
dir: &Path,
pin: &Pin<'_>,
notice: &mut dyn FnMut(&domain::ProvisionNotice),
fetch: &F,
) -> domain::Result<Self>
where
F: Fn(&str) -> domain::Result<Vec<u8>>,
{
let expected_digest = pin.digest;
fs::create_dir_all(dir).map_err(|e| io_at("create the model directory", dir, &e))?;
let paths: [PathBuf; 3] = ARTIFACT_FILES.map(|f| dir.join(f));
let present = paths.iter().filter(|p| p.exists()).count();
if present == ARTIFACT_FILES.len() {
return match read_verify_load(&paths, expected_digest)? {
Some(model) => Ok(model),
None => Err(not_the_pinned_model(
dir,
"the three artifacts already there are not the pinned reference model",
expected_digest,
)),
};
}
if present > 0 {
return Err(not_the_pinned_model(
dir,
"only part of an artifact set is there, and a partial set carries no provenance",
expected_digest,
));
}
let artifacts = fetch_verified(dir, pin, notice, fetch)?;
install_artifacts(dir, &paths, &artifacts)?;
catch_embed_panic(|| Model2Vec::from_bytes(&artifacts[0], &artifacts[1], &artifacts[2]))
}
fn from_bytes(
model_bytes: &[u8],
tokenizer_bytes: &[u8],
config_bytes: &[u8],
) -> domain::Result<Self> {
let mut hasher = Sha256::new();
hasher.update(model_bytes);
hasher.update(tokenizer_bytes);
hasher.update(config_bytes);
let model_hash = hex(&hasher.finalize());
let tensors = SafeTensors::deserialize(model_bytes)
.map_err(|e| Error::ModelInvalid(format!("safetensors: {e}")))?;
let names = tensors.names();
let emb_name = ["embeddings", "0", "embedding.weight"]
.into_iter()
.find(|n| names.iter().any(|t| t == n))
.ok_or_else(|| {
Error::ModelInvalid(
"no embeddings tensor (tried: embeddings, 0, embedding.weight)".into(),
)
})?;
let emb = tensors
.tensor(emb_name)
.map_err(|e| Error::ModelInvalid(format!("embeddings tensor: {e}")))?;
if emb.shape().len() != 2 {
return Err(Error::ModelInvalid(format!(
"embeddings tensor must be 2-dimensional, got shape {:?}",
emb.shape()
)));
}
let (vocab, dim) = (emb.shape()[0], emb.shape()[1]);
if vocab == 0 || dim == 0 {
return Err(Error::ModelInvalid(format!(
"embeddings tensor has a zero dimension: shape {vocab}x{dim}"
)));
}
let matrix = f32_tensor(emb.dtype(), emb.data(), "embeddings")?;
if matrix.len() != vocab * dim {
return Err(Error::ModelInvalid(format!(
"embeddings tensor data length {} does not match shape {}x{}",
matrix.len(),
vocab,
dim
)));
}
let weights = match tensors.tensor("weights") {
Ok(t) => Some(f32_tensor(t.dtype(), t.data(), "weights")?),
Err(_) => None,
};
let mapping = match tensors.tensor("mapping") {
Ok(t) => Some(index_tensor(t.dtype(), t.data())?),
Err(_) => None,
};
if let Some(map) = &mapping
&& let Some(&bad) = map.iter().find(|&&row| row >= vocab)
{
return Err(Error::ModelInvalid(format!(
"mapping entry {bad} outside matrix vocab {vocab}"
)));
}
if let Some(w) = &weights {
let id_space = mapping.as_ref().map_or(vocab, Vec::len);
if w.len() < id_space {
return Err(Error::ModelInvalid(format!(
"weights tensor length {} does not cover the token id space {id_space}",
w.len()
)));
}
}
let tokenizer = Tokenizer::from_bytes(tokenizer_bytes)
.map_err(|e| Error::ModelInvalid(format!("tokenizer: {e}")))?;
let tokenizer_json: serde_json::Value = serde_json::from_slice(tokenizer_bytes)
.map_err(|e| Error::ModelInvalid(format!("tokenizer.json: {e}")))?;
let model_obj = tokenizer_json.get("model");
let unk_id = model_obj
.and_then(|m| m.get("unk_token"))
.and_then(|u| u.as_str())
.and_then(|s| tokenizer.token_to_id(s))
.or_else(|| {
model_obj
.and_then(|m| m.get("unk_id"))
.and_then(serde_json::Value::as_u64)
.and_then(|v| u32::try_from(v).ok())
});
let mut lengths: Vec<usize> = tokenizer
.get_vocab(false)
.keys()
.map(|token| token.len())
.collect();
lengths.sort_unstable();
let median_token_length = lengths.get(lengths.len() / 2).copied().unwrap_or(1).max(1);
let config: serde_json::Value = serde_json::from_slice(config_bytes)
.map_err(|e| Error::ModelInvalid(format!("config.json: {e}")))?;
let normalize = config
.get("normalize")
.and_then(serde_json::Value::as_bool)
.unwrap_or(true);
Ok(Self {
matrix,
dim,
weights,
mapping,
tokenizer,
unk_id,
median_token_length,
normalize,
model_hash,
})
}
#[must_use]
pub fn model_hash(&self) -> &str {
&self.model_hash
}
#[must_use]
pub fn dimensions(&self) -> usize {
self.dim
}
fn truncate_str(text: &str, max_bytes: usize) -> &str {
if text.len() <= max_bytes {
return text;
}
let mut end = max_bytes;
while end > 0 && !text.is_char_boundary(end) {
end -= 1;
}
&text[..end]
}
fn embed_one(&self, text: &str) -> domain::Result<Embedding> {
let truncated = Self::truncate_str(
text,
DEFAULT_MAX_TOKENS.saturating_mul(self.median_token_length),
);
let encoding = self
.tokenizer
.encode_fast(truncated, false)
.map_err(|e| Error::ModelInvalid(format!("tokenize: {e}")))?;
let mut ids: Vec<u32> = encoding.get_ids().to_vec();
if let Some(unk) = self.unk_id {
ids.retain(|&id| id != unk);
}
ids.truncate(DEFAULT_MAX_TOKENS);
let vocab = self.matrix.len() / self.dim;
let mut sum = vec![0.0f32; self.dim];
let mut count = 0usize;
for &id in &ids {
let row = match &self.mapping {
Some(map) => match map.get(id as usize) {
Some(&mapped) => mapped,
None => {
return Err(Error::ModelInvalid(format!(
"token id {id} outside mapping (len {})",
map.len()
)));
}
},
None => id as usize,
};
if row >= vocab {
return Err(Error::ModelInvalid(format!(
"token id {id} maps to row {row} outside vocab {vocab}"
)));
}
let scale = self
.weights
.as_ref()
.and_then(|w| w.get(id as usize))
.copied()
.unwrap_or(1.0);
let base = row * self.dim;
for (s, v) in sum.iter_mut().zip(&self.matrix[base..base + self.dim]) {
*s += v * scale;
}
count += 1;
}
let denom = count.max(1) as f32;
for s in &mut sum {
*s /= denom;
}
if self.normalize {
let norm = sum.iter().map(|v| v * v).sum::<f32>().sqrt().max(1e-12);
for s in &mut sum {
*s /= norm;
}
}
Ok(Embedding(sum))
}
}
const EMBEDDER_NAME: &str = "model2vec";
impl Embedder for Model2Vec {
fn embed(&self, texts: &[&str]) -> domain::Result<Vec<Embedding>> {
catch_embed_panic(|| texts.iter().map(|t| self.embed_one(t)).collect())
}
fn identity(&self) -> domain::Identity {
let pinned = self.model_hash.eq_ignore_ascii_case(POTION_BASE_8M_SHA256);
domain::Identity::new(
EMBEDDER_NAME,
pinned.then(|| POTION_BASE_8M_NAME.to_string()),
Some(self.model_hash.clone()),
pinned.then(|| POTION_BASE_8M_LICENSE.to_string()),
)
}
}
#[cfg(not(target_arch = "wasm32"))]
fn read_verify_load(
paths: &[PathBuf; 3],
expected_digest: &str,
) -> domain::Result<Option<Model2Vec>> {
let mut artifacts: [Vec<u8>; 3] = Default::default();
for (slot, path) in artifacts.iter_mut().zip(paths) {
*slot = fs::read(path).map_err(|e| io_at("read", path, &e))?;
}
if !digest_of(&artifacts).eq_ignore_ascii_case(expected_digest) {
return Ok(None);
}
catch_embed_panic(|| Model2Vec::from_bytes(&artifacts[0], &artifacts[1], &artifacts[2]))
.map(Some)
}
#[cfg(not(target_arch = "wasm32"))]
fn digest_of(artifacts: &[Vec<u8>; 3]) -> String {
let mut hasher = Sha256::new();
for bytes in artifacts {
hasher.update(bytes);
}
hex(&hasher.finalize())
}
#[cfg(not(target_arch = "wasm32"))]
fn not_the_pinned_model(dir: &Path, reason: &str, expected_digest: &str) -> Error {
Error::ModelInvalid(format!(
"{}: {reason} (expected the three-file SHA-256 {expected_digest}). \
Nothing was downloaded and nothing was removed. Load a model of your own with \
Model2Vec::from_dir, remove the files there to provision the pinned model again, \
or point MATRA_MODEL_DIR or the configured embedding model name at another \
directory.",
dir.display()
))
}
fn io_at(operation: &str, path: &Path, error: &std::io::Error) -> Error {
Error::Io(crate::domain::IoContext::cannot(operation, path, error))
}
#[cfg(not(target_arch = "wasm32"))]
fn remove_artifacts(paths: &[PathBuf]) {
for path in paths {
let _ = fs::remove_file(path);
}
}
#[cfg(not(target_arch = "wasm32"))]
struct Transaction {
temps: Vec<PathBuf>,
landed: Vec<PathBuf>,
committed: bool,
}
#[cfg(not(target_arch = "wasm32"))]
impl Transaction {
fn new() -> Transaction {
Transaction {
temps: Vec::new(),
landed: Vec::new(),
committed: false,
}
}
fn write_temp(&mut self, tmp: &Path, bytes: &[u8]) -> domain::Result<()> {
use std::io::Write;
let mut file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(tmp)
.map_err(|e| {
domain::IoContext::at_path(
e.kind(),
tmp,
format!(
"cannot create the temporary file {}: {e} (a leftover from a killed \
process, or one planted there; remove it and try again)",
tmp.display()
),
)
})?;
self.temps.push(tmp.to_path_buf());
file.write_all(bytes)
.map_err(|e| io_at("write the artifact to", tmp, &e))?;
Ok(())
}
fn rename_into_place(&mut self, tmp: &Path, final_path: &Path) -> domain::Result<()> {
fs::rename(tmp, final_path)
.map_err(|e| io_at("move the artifact into place at", final_path, &e))?;
self.landed.push(final_path.to_path_buf());
Ok(())
}
fn commit(&mut self) {
self.committed = true;
}
}
#[cfg(not(target_arch = "wasm32"))]
impl Drop for Transaction {
fn drop(&mut self) {
if self.committed {
return;
}
remove_artifacts(&self.temps);
remove_artifacts(&self.landed);
}
}
#[cfg(not(target_arch = "wasm32"))]
fn fetch_verified<F>(
dir: &Path,
pin: &Pin<'_>,
notice: &mut dyn FnMut(&domain::ProvisionNotice),
fetch: &F,
) -> domain::Result<[Vec<u8>; 3]>
where
F: Fn(&str) -> domain::Result<Vec<u8>>,
{
let expected_digest = pin.digest;
for _ in 0..2 {
notice(&domain::ProvisionNotice {
artifact: pin.name.to_string(),
bytes: pin.bytes,
destination: dir.to_path_buf(),
license: pin.license.to_string(),
license_url: pin.license_url.to_string(),
});
let artifacts = fetch_all(&pin.urls, fetch)?;
if digest_of(&artifacts).eq_ignore_ascii_case(expected_digest) {
return Ok(artifacts);
}
}
Err(Error::ModelInvalid(format!(
"artifact digest mismatch after re-download in {}: expected {expected_digest}",
dir.display()
)))
}
#[cfg(not(target_arch = "wasm32"))]
fn fetch_all<F>(urls: &[&str; 3], fetch: &F) -> domain::Result<[Vec<u8>; 3]>
where
F: Fn(&str) -> domain::Result<Vec<u8>>,
{
let mut artifacts: [Vec<u8>; 3] = Default::default();
for (slot, url) in artifacts.iter_mut().zip(urls) {
let bytes = fetch(url)?;
if bytes.len() > MAX_ARTIFACT_BYTES {
return Err(Error::InputTooLarge {
limit: MAX_ARTIFACT_BYTES,
actual: bytes.len(),
what: "embedding_model_download",
});
}
*slot = bytes;
}
Ok(artifacts)
}
#[cfg(not(target_arch = "wasm32"))]
fn install_artifacts(
dir: &Path,
paths: &[PathBuf; 3],
artifacts: &[Vec<u8>; 3],
) -> domain::Result<()> {
reclaim_stale_temps(dir, SystemTime::now());
let stamp = temp_stamp();
let temps: [PathBuf; 3] =
ARTIFACT_FILES.map(|name| dir.join(format!("{TEMP_FILE_PREFIX}{name}.{stamp}")));
let mut transaction = Transaction::new();
for (tmp, bytes) in temps.iter().zip(artifacts) {
transaction.write_temp(tmp, bytes)?;
}
for (tmp, final_path) in temps.iter().zip(paths) {
transaction.rename_into_place(tmp, final_path)?;
}
transaction.commit();
Ok(())
}
#[cfg(not(target_arch = "wasm32"))]
fn temp_stamp() -> String {
static COUNTER: AtomicU64 = AtomicU64::new(0);
let nanos = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map_or(0, |d| d.as_nanos() as u64);
format!(
"{}.{nanos:x}.{:x}",
std::process::id(),
COUNTER.fetch_add(1, Ordering::Relaxed)
)
}
#[cfg(not(target_arch = "wasm32"))]
fn reclaim_stale_temps(dir: &Path, now: SystemTime) {
let Ok(entries) = fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let name = entry.file_name();
let Some(name) = name.to_str() else { continue };
if !name.starts_with(TEMP_FILE_PREFIX) {
continue;
}
let stale = entry
.metadata()
.and_then(|m| m.modified())
.ok()
.and_then(|modified| now.duration_since(modified).ok())
.is_some_and(|age| age >= STALE_TEMP_AGE);
if stale {
let _ = fs::remove_file(entry.path());
}
}
}
#[cfg(not(target_arch = "wasm32"))]
fn fetch_capped(url: &str) -> domain::Result<Vec<u8>> {
use std::io::Read;
let response = download_agent()
.get(url)
.call()
.map_err(|e| transport_failure(url, &e))?;
let mut bytes = Vec::new();
response
.into_body()
.into_reader()
.take(MAX_ARTIFACT_BYTES as u64 + 1)
.read_to_end(&mut bytes)
.map_err(|e| body_failure(url, e))?;
Ok(bytes)
}
#[cfg(not(target_arch = "wasm32"))]
fn download_agent() -> ureq::Agent {
ureq::Agent::config_builder()
.timeout_global(Some(FETCH_TIMEOUT))
.timeout_connect(Some(CONNECT_TIMEOUT))
.https_only(true)
.build()
.into()
}
#[cfg(not(target_arch = "wasm32"))]
fn body_failure(url: &str, error: std::io::Error) -> Error {
transport_failure(url, &ureq::Error::from(error))
}
#[cfg(not(target_arch = "wasm32"))]
fn transport_failure(url: &str, error: &ureq::Error) -> Error {
use std::io::ErrorKind;
let kind = match error {
ureq::Error::Io(e) => e.kind(),
ureq::Error::Timeout(_) => ErrorKind::TimedOut,
ureq::Error::ConnectionFailed | ureq::Error::HostNotFound => ErrorKind::NotConnected,
_ => ErrorKind::Other,
};
let retryable = !is_certificate_rejection(&error.to_string(), error);
Error::Io(crate::domain::IoContext::download(
kind,
url,
download_message(url, error),
retryable,
))
}
#[cfg(not(target_arch = "wasm32"))]
fn download_message(url: &str, error: &ureq::Error) -> String {
let detail = error.to_string();
if !is_certificate_rejection(&detail, error) {
return format!("download {url}: {detail}");
}
format!(
"download {url}: the TLS certificate offered for {} was rejected. matra verifies \
TLS against root certificates compiled into it and never reads the system trust \
store, so a proxy that re-signs TLS cannot be trusted by installing its CA. \
Fetch the artifacts by hand into a directory of your own and load it with \
Model2Vec::from_dir instead. Underlying failure: {detail}",
host_of(url),
)
}
#[cfg(not(target_arch = "wasm32"))]
fn is_certificate_rejection(detail: &str, error: &ureq::Error) -> bool {
if matches!(error, ureq::Error::Tls(_)) {
return true;
}
let lowered = detail.to_ascii_lowercase();
lowered.contains("certificate") || lowered.contains("rustls")
}
#[cfg(not(target_arch = "wasm32"))]
fn host_of(url: &str) -> &str {
let after_scheme = url.split_once("://").map_or(url, |(_, rest)| rest);
let authority = after_scheme
.split(['/', '?', '#'])
.next()
.unwrap_or(after_scheme);
let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h);
host.split(':').next().unwrap_or(host)
}
fn f32_tensor(dtype: Dtype, data: &[u8], name: &str) -> domain::Result<Vec<f32>> {
match dtype {
Dtype::F32 => Ok(data
.as_chunks::<4>()
.0
.iter()
.map(|b| f32::from_le_bytes(*b))
.collect()),
other => Err(Error::ModelInvalid(format!(
"{name} tensor dtype {other:?} not supported (f32 only in this build)"
))),
}
}
fn index_tensor(dtype: Dtype, data: &[u8]) -> domain::Result<Vec<usize>> {
let to_usize = |v: i64| -> domain::Result<usize> {
usize::try_from(v).map_err(|_| Error::ModelInvalid(format!("negative mapping index {v}")))
};
match dtype {
Dtype::I32 => data
.as_chunks::<4>()
.0
.iter()
.map(|b| to_usize(i64::from(i32::from_le_bytes(*b))))
.collect(),
Dtype::I64 => data
.as_chunks::<8>()
.0
.iter()
.map(|b| to_usize(i64::from_le_bytes(*b)))
.collect(),
other => Err(Error::ModelInvalid(format!(
"mapping tensor dtype {other:?} not supported (i32/i64 only)"
))),
}
}
fn catch_embed_panic<F, T>(f: F) -> domain::Result<T>
where
F: FnOnce() -> domain::Result<T>,
{
use std::panic::{AssertUnwindSafe, catch_unwind};
match catch_unwind(AssertUnwindSafe(f)) {
Ok(result) => result,
Err(payload) => {
let message = payload
.downcast_ref::<&'static str>()
.map(|s| (*s).to_string())
.or_else(|| payload.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "model2vec panic (no message captured)".to_string());
Err(Error::ModelInvalid(format!(
"model2vec panicked: {message}"
)))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
pub(super) fn write_model_normalized(dir: &Path) {
write_model(dir, None, r#"{"normalize": true}"#);
}
use safetensors::tensor::TensorView;
use std::collections::HashMap;
const MATRIX: [[f32; 4]; 3] = [
[9.0, 9.0, 9.0, 9.0],
[1.0, 0.0, 2.0, 0.0],
[0.0, 2.0, 0.0, 4.0],
];
const TOKENIZER_JSON: &str = r###"{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [],
"normalizer": {"type": "BertNormalizer", "clean_text": true,
"handle_chinese_chars": true, "strip_accents": null,
"lowercase": true},
"pre_tokenizer": {"type": "BertPreTokenizer"},
"post_processor": null,
"decoder": null,
"model": {"type": "WordPiece", "unk_token": "[UNK]",
"continuing_subword_prefix": "##",
"max_input_chars_per_word": 100,
"vocab": {"[UNK]": 0, "hello": 1, "world": 2}}
}"###;
fn tensor_bytes(rows: &[[f32; 4]]) -> Vec<u8> {
rows.iter()
.flatten()
.flat_map(|v| v.to_le_bytes())
.collect()
}
fn write_model_full(
dir: &Path,
weights: Option<&[f32]>,
mapping: Option<&[i64]>,
config: &str,
) {
let emb = tensor_bytes(&MATRIX);
let mut tensors = HashMap::new();
tensors.insert(
"embeddings".to_string(),
TensorView::new(Dtype::F32, vec![3, 4], &emb).unwrap(),
);
let wbytes: Vec<u8>;
if let Some(w) = weights {
wbytes = w.iter().flat_map(|v| v.to_le_bytes()).collect();
tensors.insert(
"weights".to_string(),
TensorView::new(Dtype::F32, vec![w.len()], &wbytes).unwrap(),
);
}
let mbytes: Vec<u8>;
if let Some(m) = mapping {
mbytes = m.iter().flat_map(|v| v.to_le_bytes()).collect();
tensors.insert(
"mapping".to_string(),
TensorView::new(Dtype::I64, vec![m.len()], &mbytes).unwrap(),
);
}
let bytes = safetensors::serialize(&tensors, None).unwrap();
fs::write(dir.join("model.safetensors"), bytes).unwrap();
fs::write(dir.join("tokenizer.json"), TOKENIZER_JSON).unwrap();
fs::write(dir.join("config.json"), config).unwrap();
}
fn write_model(dir: &Path, weights: Option<&[f32]>, config: &str) {
write_model_full(dir, weights, None, config);
}
fn load(weights: Option<&[f32]>, config: &str) -> Model2Vec {
let dir = tempfile::tempdir().unwrap();
write_model(dir.path(), weights, config);
Model2Vec::from_dir(dir.path()).unwrap()
}
fn load_full(weights: Option<&[f32]>, mapping: Option<&[i64]>, config: &str) -> Model2Vec {
let dir = tempfile::tempdir().unwrap();
write_model_full(dir.path(), weights, mapping, config);
Model2Vec::from_dir(dir.path()).unwrap()
}
#[test]
fn mean_pool_is_exact() {
let m = load(None, r#"{"normalize": false}"#);
let out = m.embed(&["hello world"]).unwrap();
assert_eq!(out.len(), 1);
assert_eq!(out[0].0, vec![0.5, 1.0, 1.0, 2.0]);
}
#[test]
fn normalization_is_exact() {
let m = load(None, r#"{"normalize": true}"#);
let out = m.embed(&["hello world"]).unwrap();
assert_eq!(out[0].0, vec![0.2, 0.4, 0.4, 0.8]);
}
#[test]
fn normalize_defaults_to_true_when_absent() {
let m = load(None, "{}");
let out = m.embed(&["hello world"]).unwrap();
assert_eq!(out[0].0, vec![0.2, 0.4, 0.4, 0.8]);
}
#[test]
fn unknown_tokens_are_dropped_not_pooled() {
let m = load(None, r#"{"normalize": false}"#);
let out = m.embed(&["hello xyzzy"]).unwrap();
assert_eq!(out[0].0, vec![1.0, 0.0, 2.0, 0.0]);
}
#[test]
fn per_token_weights_scale_before_pooling() {
let m = load(Some(&[1.0, 2.0, 0.5]), r#"{"normalize": false}"#);
let out = m.embed(&["hello world"]).unwrap();
assert_eq!(out[0].0, vec![1.0, 0.5, 2.0, 1.0]);
}
#[test]
fn empty_and_all_unknown_input_yield_zero_vectors() {
let m = load(None, r#"{"normalize": true}"#);
let out = m.embed(&["", "xyzzy qwerty"]).unwrap();
assert_eq!(out.len(), 2);
assert_eq!(out[0].0, vec![0.0; 4]);
assert_eq!(out[1].0, vec![0.0; 4]);
}
#[test]
fn contract_length_and_dimension_hold() {
let m = load(None, "{}");
let texts = ["hello", "world", "hello world hello", ""];
let out = m.embed(&texts).unwrap();
assert_eq!(out.len(), texts.len());
assert!(out.iter().all(|e| e.0.len() == m.dimensions()));
}
#[test]
fn model_hash_covers_all_three_artifact_files() {
let dir = tempfile::tempdir().unwrap();
write_model(dir.path(), None, "{}");
let mut hasher = Sha256::new();
for f in ["model.safetensors", "tokenizer.json", "config.json"] {
hasher.update(fs::read(dir.path().join(f)).unwrap());
}
let expected = hex(&hasher.finalize());
let m = Model2Vec::from_dir(dir.path()).unwrap();
assert_eq!(m.model_hash(), expected);
let dir2 = tempfile::tempdir().unwrap();
write_model(dir2.path(), None, r#"{"normalize": false}"#);
let m2 = Model2Vec::from_dir(dir2.path()).unwrap();
assert_ne!(m.model_hash(), m2.model_hash());
}
#[test]
fn an_unpinned_model_reports_its_digest_and_nothing_it_cannot_know() {
let dir = tempfile::tempdir().unwrap();
write_model(dir.path(), None, "{}");
let m = Model2Vec::from_dir(dir.path()).unwrap();
assert_eq!(
m.identity(),
domain::Identity::new("model2vec", None, Some(m.model_hash().to_string()), None)
);
}
#[test]
fn mapping_remaps_token_ids_to_rows() {
let m = load_full(None, Some(&[0, 2, 1]), r#"{"normalize": false}"#);
let out = m.embed(&["hello"]).unwrap();
assert_eq!(out[0].0, vec![0.0, 2.0, 0.0, 4.0]);
}
#[test]
fn weights_index_by_original_token_id_not_mapped_row() {
let m = load_full(
Some(&[1.0, 2.0, 1.0]),
Some(&[0, 2, 1]),
r#"{"normalize": false}"#,
);
let out = m.embed(&["hello"]).unwrap();
assert_eq!(out[0].0, vec![0.0, 4.0, 0.0, 8.0]);
}
#[test]
fn mapping_entry_outside_vocab_is_rejected_at_load() {
let dir = tempfile::tempdir().unwrap();
write_model_full(dir.path(), None, Some(&[0, 3, 1]), "{}");
match Model2Vec::from_dir(dir.path()) {
Err(Error::ModelInvalid(msg)) => assert!(msg.contains("mapping entry")),
other => panic!("expected ModelInvalid, got {:?}", other.map(|_| ())),
}
}
#[test]
fn short_weights_tensor_is_rejected_at_load() {
let dir = tempfile::tempdir().unwrap();
write_model(dir.path(), Some(&[1.0, 2.0]), "{}");
match Model2Vec::from_dir(dir.path()) {
Err(Error::ModelInvalid(msg)) => assert!(msg.contains("weights tensor length")),
other => panic!("expected ModelInvalid, got {:?}", other.map(|_| ())),
}
}
#[test]
fn zero_dimension_matrix_is_rejected_at_load() {
let dir = tempfile::tempdir().unwrap();
let mut tensors = HashMap::new();
let empty: Vec<u8> = Vec::new();
tensors.insert(
"embeddings".to_string(),
TensorView::new(Dtype::F32, vec![3, 0], &empty).unwrap(),
);
let bytes = safetensors::serialize(&tensors, None).unwrap();
fs::write(dir.path().join("model.safetensors"), bytes).unwrap();
fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
fs::write(dir.path().join("config.json"), "{}").unwrap();
match Model2Vec::from_dir(dir.path()) {
Err(Error::ModelInvalid(msg)) => assert!(msg.contains("zero dimension")),
other => panic!("expected ModelInvalid, got {:?}", other.map(|_| ())),
}
}
#[test]
fn missing_file_is_model_not_found() {
let dir = tempfile::tempdir().unwrap();
match Model2Vec::from_dir(dir.path()) {
Err(Error::ModelNotFound(p)) => {
assert!(p.ends_with("model.safetensors"));
}
Err(other) => panic!("expected ModelNotFound, got {other:?}"),
Ok(_) => panic!("expected ModelNotFound, got a loaded model"),
}
}
#[test]
fn corrupt_safetensors_is_model_invalid_not_panic() {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("model.safetensors"), b"garbage").unwrap();
fs::write(dir.path().join("tokenizer.json"), TOKENIZER_JSON).unwrap();
fs::write(dir.path().join("config.json"), "{}").unwrap();
match Model2Vec::from_dir(dir.path()) {
Err(Error::ModelInvalid(_)) => {}
Err(other) => panic!("expected ModelInvalid, got {other:?}"),
Ok(_) => panic!("expected ModelInvalid, got a loaded model"),
}
}
#[test]
fn truncation_respects_char_boundaries() {
let text = "hé";
let t = Model2Vec::truncate_str(text, 2);
assert_eq!(t, "h");
assert_eq!(Model2Vec::truncate_str(text, 3), "hé");
}
}
#[cfg(test)]
mod parity {
use super::tests::write_model_normalized;
use super::*;
const TOLERANCE: f32 = 1e-6;
#[test]
fn matches_python_reference_on_pinned_inputs() {
let dir = tempfile::tempdir().unwrap();
write_model_normalized(dir.path());
let m = Model2Vec::from_dir(dir.path()).unwrap();
let cases: [(&str, [f64; 4]); 6] = [
("hello world", [0.2, 0.4, 0.4, 0.8]),
("hello", [0.4472135954999579, 0.0, 0.8944271909999159, 0.0]),
(
"hello xyzzy",
[0.4472135954999579, 0.0, 0.8944271909999159, 0.0],
),
("", [0.0, 0.0, 0.0, 0.0]),
("xyzzy qwerty", [0.0, 0.0, 0.0, 0.0]),
(
"HELLO World hello",
[
0.31622776601683794,
0.31622776601683794,
0.6324555320336759,
0.6324555320336759,
],
),
];
let texts: Vec<&str> = cases.iter().map(|(t, _)| *t).collect();
let out = m.embed(&texts).unwrap();
for ((text, expected), got) in cases.iter().zip(&out) {
for (i, (e, g)) in expected.iter().zip(&got.0).enumerate() {
let diff = (*e as f32 - g).abs();
assert!(
diff <= TOLERANCE,
"{text:?} dim {i}: python {e} vs rust {g} (diff {diff})"
);
}
}
}
}
#[cfg(test)]
mod bit_parity {
use super::tests::write_model_normalized;
use super::*;
#[test]
fn vectors_are_bit_identical_across_targets() {
let dir = tempfile::tempdir().unwrap();
write_model_normalized(dir.path());
let m = Model2Vec::from_dir(dir.path()).unwrap();
let cases: [(&str, [u32; 4]); 3] = [
(
"hello world",
[0x3e4c_cccd, 0x3ecc_cccd, 0x3ecc_cccd, 0x3f4c_cccd],
),
(
"hello",
[0x3ee4_f92e, 0x0000_0000, 0x3f64_f92e, 0x0000_0000],
),
(
"HELLO World hello",
[0x3ea1_e89c, 0x3ea1_e89c, 0x3f21_e89c, 0x3f21_e89c],
),
];
let texts: Vec<&str> = cases.iter().map(|(t, _)| *t).collect();
let out = m.embed(&texts).unwrap();
for ((text, expected), got) in cases.iter().zip(&out) {
let bits: Vec<u32> = got.0.iter().map(|v| v.to_bits()).collect();
assert_eq!(&bits[..], expected, "bit drift on {text:?}");
}
}
}
#[cfg(test)]
#[cfg(not(target_arch = "wasm32"))]
mod provisioning {
use std::cell::RefCell;
use super::tests::write_model_normalized;
use super::*;
const URLS: [&str; 3] = ["test://model", "test://tokenizer", "test://config"];
struct Fixture {
bytes: [Vec<u8>; 3],
digest: String,
}
impl Fixture {
fn new() -> Fixture {
let dir = tempfile::tempdir().unwrap();
write_model_normalized(dir.path());
let bytes = ARTIFACT_FILES.map(|name| fs::read(dir.path().join(name)).unwrap());
let mut hasher = Sha256::new();
for buf in &bytes {
hasher.update(buf);
}
Fixture {
bytes,
digest: hex(&hasher.finalize()),
}
}
fn write_into(&self, dir: &Path) {
for (name, buf) in ARTIFACT_FILES.iter().zip(&self.bytes) {
fs::write(dir.join(name), buf).unwrap();
}
}
}
enum Serves {
TheModel,
Garbage,
GarbageThenTheModel,
TooMuch,
TheFirstThenAFailure,
TooMuchOnTheLast,
Nothing,
}
struct Fetcher<'a> {
fixture: &'a Fixture,
serves: Serves,
log: RefCell<Vec<String>>,
}
impl<'a> Fetcher<'a> {
fn new(fixture: &'a Fixture, serves: Serves) -> Fetcher<'a> {
Fetcher {
fixture,
serves,
log: RefCell::new(Vec::new()),
}
}
fn fetch(&self, url: &str) -> domain::Result<Vec<u8>> {
self.log.borrow_mut().push(url.to_string());
let index = URLS.iter().position(|u| *u == url).expect("known url");
let round = self.log.borrow().len();
match self.serves {
Serves::TheModel => Ok(self.fixture.bytes[index].clone()),
Serves::Garbage => Ok(b"not a model".to_vec()),
Serves::GarbageThenTheModel => {
if round <= ARTIFACT_FILES.len() {
Ok(b"not a model".to_vec())
} else {
Ok(self.fixture.bytes[index].clone())
}
}
Serves::TooMuch => Ok(vec![0u8; MAX_ARTIFACT_BYTES + 1]),
Serves::TheFirstThenAFailure => {
if round == 1 {
Ok(self.fixture.bytes[index].clone())
} else {
Err(Error::Io(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("download {url}: the transfer stalled"),
)))
}
}
Serves::TooMuchOnTheLast => {
if round < ARTIFACT_FILES.len() {
Ok(self.fixture.bytes[index].clone())
} else {
Ok(vec![0u8; MAX_ARTIFACT_BYTES + 1])
}
}
Serves::Nothing => {
panic!("the fetcher was called for {url}, and should not have been")
}
}
}
fn calls(&self) -> Vec<String> {
self.log.borrow().clone()
}
}
const FIXTURE_NAME: &str = "fixture-model";
const FIXTURE_LICENSE: &str = "Fixture License 1.0";
const FIXTURE_LICENSE_URL: &str = "https://licenses.example/fixture";
fn pin_of(fixture: &Fixture) -> Pin<'_> {
Pin {
name: FIXTURE_NAME,
bytes: fixture.bytes.iter().map(|b| b.len() as u64).sum(),
digest: &fixture.digest,
urls: URLS,
license: FIXTURE_LICENSE,
license_url: FIXTURE_LICENSE_URL,
}
}
fn provision(
dir: &Path,
fixture: &Fixture,
fetcher: &Fetcher<'_>,
) -> domain::Result<Model2Vec> {
provision_noting(dir, fixture, fetcher, &mut |_| {})
}
fn provision_noting(
dir: &Path,
fixture: &Fixture,
fetcher: &Fetcher<'_>,
notice: &mut dyn FnMut(&domain::ProvisionNotice),
) -> domain::Result<Model2Vec> {
Model2Vec::provision(dir, &pin_of(fixture), notice, &|url| fetcher.fetch(url))
}
fn expect_err(result: domain::Result<Model2Vec>) -> Error {
match result {
Ok(_) => panic!("expected an error, got a loaded model"),
Err(e) => e,
}
}
fn present(dir: &Path) -> Vec<&'static str> {
ARTIFACT_FILES
.into_iter()
.filter(|name| dir.join(name).exists())
.collect()
}
fn temporaries(dir: &Path) -> Vec<String> {
let Ok(entries) = fs::read_dir(dir) else {
return Vec::new();
};
entries
.map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned())
.filter(|name| name.starts_with(".tmp."))
.collect()
}
#[test]
fn a_matching_digest_loads_and_nothing_is_fetched() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
fixture.write_into(dir.path());
let fetcher = Fetcher::new(&fixture, Serves::Nothing);
let model = provision(dir.path(), &fixture, &fetcher).unwrap();
assert_eq!(model.model_hash(), fixture.digest);
assert!(fetcher.calls().is_empty());
}
#[test]
fn a_corrupted_download_is_refetched_and_the_second_set_lands() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
let fetcher = Fetcher::new(&fixture, Serves::GarbageThenTheModel);
let model = provision(dir.path(), &fixture, &fetcher).unwrap();
assert_eq!(fetcher.calls(), [URLS, URLS].concat());
assert_eq!(model.model_hash(), fixture.digest);
assert_eq!(temporaries(dir.path()), Vec::<String>::new());
}
#[test]
fn a_second_mismatch_is_model_invalid_and_leaves_no_files() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
let fetcher = Fetcher::new(&fixture, Serves::Garbage);
let err = expect_err(provision(dir.path(), &fixture, &fetcher));
assert!(
matches!(err, Error::ModelInvalid(ref m) if m.contains(&fixture.digest)),
"expected a digest mismatch, got {err:?}"
);
assert_eq!(fetcher.calls().len(), 2 * ARTIFACT_FILES.len());
assert_eq!(present(dir.path()), Vec::<&str>::new());
}
#[test]
fn an_oversized_download_is_input_too_large() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
let fetcher = Fetcher::new(&fixture, Serves::TooMuch);
let err = expect_err(provision(dir.path(), &fixture, &fetcher));
match err {
Error::InputTooLarge {
limit,
actual,
what,
} => {
assert_eq!(limit, MAX_ARTIFACT_BYTES);
assert_eq!(actual, MAX_ARTIFACT_BYTES + 1);
assert_eq!(what, "embedding_model_download");
}
other => panic!("expected InputTooLarge, got {other:?}"),
}
assert_eq!(present(dir.path()), Vec::<&str>::new());
}
#[test]
fn a_rejected_certificate_reads_as_a_sentence() {
let error = ureq::Error::Io(std::io::Error::other(
"invalid peer certificate: Other(OtherError(CaUsedAsEndEntity))",
));
let message = download_message(POTION_BASE_8M_URLS[0], &error);
assert!(message.contains("huggingface.co"), "{message}");
assert!(message.contains("system trust store"), "{message}");
assert!(message.contains("from_dir"), "{message}");
assert!(message.contains("CaUsedAsEndEntity"), "{message}");
let mapped = transport_failure(POTION_BASE_8M_URLS[0], &error);
assert_eq!(mapped.kind(), "io");
assert_eq!(
mapped.details().get("retryable"),
Some(&domain::DetailValue::Bool(false)),
"the message says to load by hand, so a retry is not offered"
);
}
#[test]
fn a_transport_failure_is_retryable_and_names_its_url() {
for error in [
ureq::Error::HostNotFound,
ureq::Error::Timeout(ureq::Timeout::Connect),
ureq::Error::StatusCode(502),
] {
let details = transport_failure(POTION_BASE_8M_URLS[0], &error).details();
assert_eq!(
details.get("url"),
Some(&domain::DetailValue::Text(
POTION_BASE_8M_URLS[0].to_string()
)),
"{error}"
);
assert_eq!(
details.get("retryable"),
Some(&domain::DetailValue::Bool(true)),
"{error}"
);
}
}
#[test]
fn a_url_yields_the_host_a_message_should_name() {
assert_eq!(host_of(POTION_BASE_8M_URLS[0]), "huggingface.co");
assert_eq!(host_of("https://example.org:8443/a?b=c"), "example.org");
}
#[test]
fn a_fetch_that_fails_partway_leaves_no_artifacts() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
let fetcher = Fetcher::new(&fixture, Serves::TheFirstThenAFailure);
let err = expect_err(provision(dir.path(), &fixture, &fetcher));
match err {
Error::Io(e) => assert_eq!(e.kind(), std::io::ErrorKind::TimedOut),
other => panic!("expected the fetcher's Io error, got {other:?}"),
}
assert_eq!(fetcher.calls(), URLS[..2].to_vec());
assert_eq!(present(dir.path()), Vec::<&str>::new());
assert_eq!(temporaries(dir.path()), Vec::<String>::new());
}
#[test]
fn an_oversized_last_artifact_leaves_no_artifacts() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
let fetcher = Fetcher::new(&fixture, Serves::TooMuchOnTheLast);
let err = expect_err(provision(dir.path(), &fixture, &fetcher));
match err {
Error::InputTooLarge {
limit,
actual,
what,
} => {
assert_eq!(limit, MAX_ARTIFACT_BYTES);
assert_eq!(actual, MAX_ARTIFACT_BYTES + 1);
assert_eq!(what, "embedding_model_download");
}
other => panic!("expected InputTooLarge, got {other:?}"),
}
assert_eq!(fetcher.calls(), URLS.to_vec());
assert_eq!(present(dir.path()), Vec::<&str>::new());
assert_eq!(temporaries(dir.path()), Vec::<String>::new());
}
#[test]
fn pre_existing_mismatching_files_are_refused_and_untouched() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
fixture.write_into(dir.path());
let mine: &[u8] = br#"{"normalize": false}"#;
fs::write(dir.path().join("config.json"), mine).unwrap();
let fetcher = Fetcher::new(&fixture, Serves::Nothing);
let err = expect_err(provision(dir.path(), &fixture, &fetcher));
let dir_name = dir.path().display().to_string();
assert!(
matches!(err, Error::ModelInvalid(ref m) if m.contains(&dir_name)),
"expected the directory to be named, got {err:?}"
);
assert_eq!(present(dir.path()).len(), ARTIFACT_FILES.len());
assert_eq!(fs::read(dir.path().join("config.json")).unwrap(), mine);
assert_eq!(
fs::read(dir.path().join("model.safetensors")).unwrap(),
fixture.bytes[0]
);
assert_eq!(
fs::read(dir.path().join("tokenizer.json")).unwrap(),
fixture.bytes[1]
);
}
#[test]
fn a_partial_directory_is_refused_and_untouched() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
fixture.write_into(dir.path());
fs::remove_file(dir.path().join("tokenizer.json")).unwrap();
let fetcher = Fetcher::new(&fixture, Serves::Nothing);
let err = expect_err(provision(dir.path(), &fixture, &fetcher));
let dir_name = dir.path().display().to_string();
assert!(
matches!(err, Error::ModelInvalid(ref m) if m.contains(&dir_name)),
"expected the directory to be named, got {err:?}"
);
assert_eq!(
present(dir.path()),
vec!["model.safetensors", "config.json"]
);
assert_eq!(
fs::read(dir.path().join("model.safetensors")).unwrap(),
fixture.bytes[0]
);
assert_eq!(
fs::read(dir.path().join("config.json")).unwrap(),
fixture.bytes[2]
);
}
#[cfg(unix)]
#[test]
fn a_planted_temporary_is_not_written_through() {
let victim_bytes: &[u8] = b"the caller's bytes";
let dir = tempfile::tempdir().unwrap();
let victim = dir.path().join("victim");
fs::write(&victim, victim_bytes).unwrap();
let planted = dir.path().join(".tmp.model.safetensors.planted");
std::os::unix::fs::symlink(&victim, &planted).unwrap();
let mut transaction = Transaction::new();
let err = transaction
.write_temp(&planted, b"thirty megabytes")
.expect_err("a path already there fails the open");
match err {
Error::Io(e) => {
assert_eq!(e.kind(), std::io::ErrorKind::AlreadyExists);
assert!(e.to_string().contains(".tmp.model.safetensors"), "{e}");
}
other => panic!("expected Io(AlreadyExists), got {other:?}"),
}
assert_eq!(fs::read(&victim).unwrap(), victim_bytes);
drop(transaction);
assert!(fs::symlink_metadata(&planted).is_ok());
assert_eq!(fs::read(&victim).unwrap(), victim_bytes);
}
#[test]
fn nothing_is_written_until_the_whole_set_has_satisfied_the_pin() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
let seen: RefCell<Vec<(Vec<&'static str>, Vec<String>)>> = RefCell::new(Vec::new());
let model = Model2Vec::provision(dir.path(), &pin_of(&fixture), &mut |_| {}, &|url| {
seen.borrow_mut()
.push((present(dir.path()), temporaries(dir.path())));
let index = URLS.iter().position(|u| *u == url).expect("known url");
Ok(fixture.bytes[index].clone())
})
.unwrap();
assert_eq!(model.model_hash(), fixture.digest);
assert_eq!(seen.borrow().len(), ARTIFACT_FILES.len());
for (artifacts, temps) in seen.borrow().iter() {
assert!(
artifacts.is_empty(),
"an artifact name existed mid-download: {artifacts:?}"
);
assert!(
temps.is_empty(),
"a temporary existed mid-download: {temps:?}"
);
}
assert_eq!(present(dir.path()).len(), ARTIFACT_FILES.len());
assert_eq!(temporaries(dir.path()), Vec::<String>::new());
}
#[test]
fn a_model_directory_that_cannot_be_created_fails_before_the_download() {
let fixture = Fixture::new();
let parent = tempfile::tempdir().unwrap();
let blocker = parent.path().join("not-a-directory");
fs::write(&blocker, b"x").unwrap();
let dir = blocker.join("models");
let fetcher = Fetcher::new(&fixture, Serves::Nothing);
let err = expect_err(provision(&dir, &fixture, &fetcher));
assert!(
matches!(err, Error::Io(_)),
"expected an Io failure, got {err:?}"
);
assert!(fetcher.calls().is_empty());
let message = err.to_string();
assert!(
message.contains("create the model directory"),
"names the operation: {message}"
);
assert!(
message.contains(&dir.display().to_string()),
"names the path: {message}"
);
assert_eq!(
err.details().get("path"),
Some(&domain::DetailValue::Text(dir.display().to_string())),
"the path is data, not only words in the message"
);
}
#[test]
fn an_artifact_that_cannot_be_read_names_the_operation_and_the_path() {
let dir = tempfile::tempdir().unwrap();
for name in ARTIFACT_FILES {
fs::write(dir.path().join(name), b"x").unwrap();
}
let blocked = dir.path().join(ARTIFACT_FILES[0]);
fs::remove_file(&blocked).unwrap();
fs::create_dir(&blocked).unwrap();
let err = expect_err(Model2Vec::from_dir(dir.path()));
assert_eq!(err.kind(), "io");
let message = err.to_string();
assert!(
message.contains("cannot read"),
"names the operation: {message}"
);
assert!(
message.contains(&blocked.display().to_string()),
"names the path: {message}"
);
}
#[test]
fn an_aged_orphan_temporary_is_reclaimed() {
let dir = tempfile::tempdir().unwrap();
let orphan = dir.path().join(".tmp.model.safetensors.424242.0");
fs::write(&orphan, b"half an artifact").unwrap();
reclaim_stale_temps(dir.path(), SystemTime::now());
assert!(orphan.exists(), "nothing is old yet");
reclaim_stale_temps(
dir.path(),
SystemTime::now() + STALE_TEMP_AGE + Duration::from_secs(1),
);
assert!(!orphan.exists(), "an aged leftover is reclaimed");
}
#[test]
fn a_fresh_peer_temporary_survives_a_provision() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
let peer = dir
.path()
.join(format!(".tmp.model.safetensors.{}", std::process::id()));
fs::write(&peer, b"in flight").unwrap();
let fetcher = Fetcher::new(&fixture, Serves::TheModel);
let model = provision(dir.path(), &fixture, &fetcher).unwrap();
assert_eq!(model.model_hash(), fixture.digest);
assert!(peer.exists(), "a fresh peer is not this call's to remove");
assert_eq!(fs::read(&peer).unwrap(), b"in flight");
assert_eq!(present(dir.path()).len(), ARTIFACT_FILES.len());
}
#[test]
fn a_temporary_from_the_future_is_left_alone() {
let dir = tempfile::tempdir().unwrap();
let odd = dir.path().join(".tmp.config.json.7.0");
fs::write(&odd, b"scratch").unwrap();
reclaim_stale_temps(dir.path(), SystemTime::now() - Duration::from_secs(3600));
assert!(odd.exists());
}
#[test]
fn the_sweep_only_matches_its_own_temporary_names() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
fixture.write_into(dir.path());
let temp = dir.path().join(".tmp.config.json.7.0");
fs::write(&temp, b"scratch").unwrap();
reclaim_stale_temps(
dir.path(),
SystemTime::now() + STALE_TEMP_AGE + Duration::from_secs(1),
);
assert_eq!(present(dir.path()).len(), ARTIFACT_FILES.len());
assert!(!temp.exists());
}
#[test]
fn two_temporary_names_in_one_process_differ() {
assert_ne!(temp_stamp(), temp_stamp());
}
#[test]
fn a_timeout_while_reading_the_body_is_reported_as_a_timeout() {
let from_ureq = std::io::Error::other(ureq::Error::Timeout(ureq::Timeout::Global));
assert_eq!(
from_ureq.kind(),
std::io::ErrorKind::Other,
"the wrapping this test exists to undo"
);
match body_failure(POTION_BASE_8M_URLS[0], from_ureq) {
Error::Io(e) => {
assert_eq!(e.kind(), std::io::ErrorKind::TimedOut);
assert!(e.to_string().contains("huggingface.co"), "{e}");
}
other => panic!("expected Io(TimedOut), got {other:?}"),
}
}
#[test]
fn a_plain_body_read_failure_keeps_its_own_kind() {
let raw = std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "peer disconnected");
match body_failure(POTION_BASE_8M_URLS[0], raw) {
Error::Io(e) => assert_eq!(e.kind(), std::io::ErrorKind::UnexpectedEof),
other => panic!("expected Io(UnexpectedEof), got {other:?}"),
}
}
#[test]
fn the_download_agent_refuses_to_leave_https() {
let agent = download_agent();
assert!(agent.config().https_only());
assert!(
agent.config().max_redirects() > 0,
"the redirect following that makes https_only load-bearing"
);
}
#[test]
fn the_pinned_digest_matches_the_conformance_fixture() {
let path =
Path::new(env!("CARGO_MANIFEST_DIR")).join("spec/tests/semantic/reference-model.json");
let raw = fs::read_to_string(&path)
.unwrap_or_else(|e| panic!("cannot read {}: {e}", path.display()));
let fixture: serde_json::Value = serde_json::from_str(&raw)
.unwrap_or_else(|e| panic!("malformed fixture {}: {e}", path.display()));
let pinned = fixture["model"]["artifact_hash"]
.as_str()
.expect("model.artifact_hash is a string");
assert_eq!(
pinned,
POTION_BASE_8M_SHA256,
"{} pins a different artifact digest than src/embed/model2vec.rs; \
refresh both with scripts/fetch-embedding-hash.sh",
path.display()
);
}
#[test]
fn a_fetch_is_announced_once_for_the_set() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
let fetcher = Fetcher::new(&fixture, Serves::TheModel);
let mut notices = Vec::new();
provision_noting(dir.path(), &fixture, &fetcher, &mut |n| {
notices.push(n.clone())
})
.unwrap();
assert_eq!(notices.len(), 1, "one notice for the set of three files");
let notice = ¬ices[0];
assert_eq!(notice.artifact, FIXTURE_NAME);
assert_eq!(
notice.bytes,
fixture.bytes.iter().map(|b| b.len() as u64).sum::<u64>()
);
assert_eq!(notice.destination, dir.path());
assert_eq!(notice.license, FIXTURE_LICENSE);
assert_eq!(notice.license_url, FIXTURE_LICENSE_URL);
}
#[test]
fn the_notice_follows_the_fetches() {
let fixture = Fixture::new();
let dir = tempfile::tempdir().unwrap();
let fetcher = Fetcher::new(&fixture, Serves::TheModel);
provision(dir.path(), &fixture, &fetcher).unwrap();
let mut notices = 0;
let nothing = Fetcher::new(&fixture, Serves::Nothing);
provision_noting(dir.path(), &fixture, ¬hing, &mut |_| notices += 1).unwrap();
assert_eq!(notices, 0, "a warm start announces nothing");
let fresh = tempfile::tempdir().unwrap();
let flaky = Fetcher::new(&fixture, Serves::GarbageThenTheModel);
provision_noting(fresh.path(), &fixture, &flaky, &mut |_| notices += 1).unwrap();
assert_eq!(
notices, 2,
"one notice per round, and the first round failed"
);
}
#[test]
fn the_pin_carries_its_name_size_and_license() {
assert_eq!(POTION_BASE_8M.name, "potion-base-8M");
assert_eq!(POTION_BASE_8M.bytes, 30_236_760 + 683_666 + 202);
assert_eq!(POTION_BASE_8M.license, "MIT");
assert_eq!(POTION_BASE_8M.digest, POTION_BASE_8M_SHA256);
assert_eq!(POTION_BASE_8M.urls, POTION_BASE_8M_URLS);
}
#[test]
fn potion_base_8m_takes_the_models_directory_from_config_reads() {
let models = Path::new("/srv/matra/models");
let cfg = Config::from_sources(|key| (key == "HOME").then(|| "/home/u".to_string()), None)
.unwrap()
.with_model_dir(models);
assert_eq!(
potion_dir(models),
cfg.model_dir().join(cfg.embedding_model())
);
assert_eq!(potion_dir(models), models.join("potion-base-8M"));
}
#[test]
fn a_missing_directory_is_created_and_filled() {
let fixture = Fixture::new();
let parent = tempfile::tempdir().unwrap();
let dir = parent.path().join("models").join("potion-base-8M");
let fetcher = Fetcher::new(&fixture, Serves::TheModel);
let model = provision(&dir, &fixture, &fetcher).unwrap();
assert_eq!(model.model_hash(), fixture.digest);
assert_eq!(present(&dir).len(), 3);
assert_eq!(temporaries(&dir), Vec::<String>::new());
}
#[test]
fn from_dir_never_downloads() {
let fixture = Fixture::new();
let fetcher = Fetcher::new(&fixture, Serves::TheModel);
let provisioned = tempfile::tempdir().unwrap();
provision(provisioned.path(), &fixture, &fetcher).unwrap();
assert_eq!(fetcher.calls().len(), 3);
let empty = tempfile::tempdir().unwrap();
let err = expect_err(Model2Vec::from_dir(empty.path()));
assert!(
matches!(err, Error::ModelNotFound(ref p) if p.ends_with("model.safetensors")),
"expected ModelNotFound, got {err:?}"
);
assert_eq!(fetcher.calls().len(), 3);
assert_eq!(present(empty.path()), Vec::<&str>::new());
}
}