use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::path::{Path, PathBuf};
use serde::Serialize;
use crate::schema::{ModelSchema, ModelSource};
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct HfReference {
pub repo: String,
pub files: Option<BTreeSet<String>>,
}
pub fn hf_references(schema: &ModelSchema) -> Vec<HfReference> {
let file = |repo: &str, path: &str| HfReference {
repo: repo.to_string(),
files: Some(BTreeSet::from([path.to_string()])),
};
match &schema.source {
ModelSource::Local {
hf_repo,
hf_filename,
tokenizer_repo,
} => vec![
file(hf_repo, hf_filename),
file(tokenizer_repo, "tokenizer.json"),
],
ModelSource::Mlx { hf_repo, .. } | ModelSource::ManagedVllmMlx { hf_repo, .. } => {
let mut refs = vec![HfReference {
repo: hf_repo.clone(),
files: None,
}];
if crate::registry::uses_flux_auxiliary(&schema.name, hf_repo) {
let (_, _, base, path) = crate::registry::FLUX_AUXILIARY;
refs.push(file(base, path));
}
if crate::registry::uses_ltx_text_encoder(schema) {
refs.push(HfReference {
repo: crate::registry::LTX_TEXT_ENCODER.to_string(),
files: None,
});
}
refs
}
_ => Vec::new(),
}
}
pub fn primary_whole_repo(schema: &ModelSchema) -> Option<&str> {
match &schema.source {
ModelSource::Mlx { hf_repo, .. } | ModelSource::ManagedVllmMlx { hf_repo, .. } => {
Some(hf_repo.as_str())
}
_ => None,
}
}
pub const PROVENANCE_FILE: &str = "hf-provenance.jsonl";
#[derive(Debug, Serialize, serde::Deserialize)]
struct ProvenanceRecord {
at: u64,
model_id: String,
repo: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
revision: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
files: Option<BTreeSet<String>>,
hub: PathBuf,
}
fn provenance_path(state_root: &Path) -> PathBuf {
state_root.join("model-management").join(PROVENANCE_FILE)
}
pub type Fetched = (String, String, Option<String>);
tokio::task_local! {
static FETCHED: std::cell::RefCell<Vec<Fetched>>;
}
pub(crate) fn snapshot_revision(path: &Path) -> Option<String> {
let mut parts = path.components().map(|c| c.as_os_str().to_string_lossy());
parts.by_ref().find(|c| c.starts_with("models--"))?;
(parts.next()? == "snapshots").then_some(())?;
parts.next().map(|c| c.into_owned())
}
pub(crate) fn note_fetched(repo: &str, file: &str, revision: Option<String>) {
if revision.is_none() {
return;
}
let _ = FETCHED.try_with(|f| {
f.borrow_mut()
.push((repo.to_string(), file.to_string(), revision))
});
}
pub(crate) async fn collecting_fetches<F: std::future::Future>(
fut: F,
) -> (F::Output, Vec<Fetched>) {
FETCHED
.scope(std::cell::RefCell::new(Vec::new()), async move {
let out = fut.await;
let fetched = FETCHED.with(|f| std::mem::take(&mut *f.borrow_mut()));
(out, fetched)
})
.await
}
pub fn record_download(state_root: &Path, hub: &Path, model_id: &str, fetched: &[Fetched]) {
use std::io::Write;
if fetched.is_empty() {
return;
}
let hub = hub.canonicalize().unwrap_or_else(|_| hub.to_path_buf());
let at = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
let mut by_repo: BTreeMap<(&str, Option<&str>), BTreeSet<String>> = BTreeMap::new();
for (repo, file, revision) in fetched {
by_repo
.entry((repo, revision.as_deref()))
.or_default()
.insert(file.clone());
}
let path = provenance_path(state_root);
let mut body = String::new();
for ((repo, revision), files) in by_repo {
let record = ProvenanceRecord {
at,
model_id: model_id.to_string(),
repo: repo.to_string(),
revision: revision.map(str::to_string),
files: Some(files),
hub: hub.clone(),
};
if let Ok(line) = serde_json::to_string(&record) {
body.push_str(&line);
body.push('\n');
}
}
let write = || -> std::io::Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let mut f = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&path)?;
f.write_all(body.as_bytes())
};
if let Err(error) = write() {
tracing::warn!(%error, path = %path.display(), "could not record download provenance");
}
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct OwnedFile {
pub path: String,
pub revision: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Owned {
Whole,
Files(BTreeSet<OwnedFile>),
}
pub type Ownership = BTreeMap<String, Owned>;
pub fn recorded_files(state_root: &Path, hub: &Path) -> Ownership {
let mut out: Ownership = BTreeMap::new();
let Ok(text) = std::fs::read_to_string(provenance_path(state_root)) else {
return out;
};
let hub = hub.canonicalize().unwrap_or_else(|_| hub.to_path_buf());
for record in text
.lines()
.filter_map(|line| serde_json::from_str::<ProvenanceRecord>(line).ok())
.filter(|r| r.hub == hub)
{
let files = record
.files
.unwrap_or_default()
.into_iter()
.map(|path| OwnedFile {
path,
revision: record.revision.clone(),
});
match out
.entry(record.repo)
.or_insert(Owned::Files(BTreeSet::new()))
{
Owned::Files(set) => set.extend(files),
Owned::Whole => {}
}
}
out
}
pub fn catalog_ownership() -> Ownership {
let mut out: Ownership = BTreeMap::new();
for r in crate::registry::builtin_catalog()
.iter()
.flat_map(hf_references)
{
let entry = out.entry(r.repo).or_insert(Owned::Files(BTreeSet::new()));
match (entry, r.files) {
(slot, None) => *slot = Owned::Whole,
(Owned::Files(set), Some(files)) => {
set.extend(files.into_iter().map(|path| OwnedFile {
path,
revision: None,
}))
}
(Owned::Whole, Some(_)) => {}
}
}
out
}
pub fn ownership(catalog: Ownership, recorded: Ownership) -> Ownership {
let mut out = recorded;
for (repo, owned) in catalog {
match (
out.entry(repo).or_insert(Owned::Files(BTreeSet::new())),
owned,
) {
(slot, Owned::Whole) => *slot = Owned::Whole,
(Owned::Files(set), Owned::Files(more)) => set.extend(more),
(Owned::Whole, Owned::Files(_)) => {}
}
}
out
}
#[derive(Debug, Clone, Default, Serialize)]
pub struct RetirePlan {
pub model_id: String,
pub also_retires: Vec<String>,
pub projections: Vec<Projection>,
pub user_content: Vec<PathBuf>,
pub deletions: Vec<HubDeletion>,
pub kept: Vec<KeptReference>,
pub unowned: Vec<String>,
pub refusals: Vec<RetireRefusal>,
pub freed_bytes: u64,
pub digest: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Projection {
pub path: PathBuf,
#[serde(skip_serializing_if = "Option::is_none")]
pub receipt_for: Option<String>,
}
impl RetirePlan {
pub fn seal(&mut self) {
use sha2::{Digest, Sha256};
self.also_retires.sort();
self.projections.sort_by(|a, b| a.path.cmp(&b.path));
for deletion in &mut self.deletions {
if let HubDeletion::Files { entries, blobs, .. } = deletion {
entries.sort();
blobs.sort();
}
}
self.deletions.sort_by(|a, b| a.repo().cmp(b.repo()));
let material = serde_json::json!({
"model_id": self.model_id,
"also_retires": self.also_retires,
"projections": self.projections,
"deletions": self.deletions,
});
let mut hash = Sha256::new();
hash.update(material.to_string().as_bytes());
self.digest = format!("{:x}", hash.finalize());
}
}
impl HubDeletion {
fn repo(&self) -> &str {
match self {
HubDeletion::Repo { repo, .. } | HubDeletion::Files { repo, .. } => repo,
}
}
}
impl RetirePlan {
pub fn is_refused(&self) -> bool {
!self.refusals.is_empty()
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum HubDeletion {
Repo {
repo: String,
path: PathBuf,
bytes: u64,
},
Files {
repo: String,
entries: Vec<PathBuf>,
blobs: Vec<PathBuf>,
bytes: u64,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct KeptReference {
pub repo: String,
pub kept_for: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(tag = "reason", rename_all = "snake_case")]
pub enum RetireRefusal {
InUse,
LaneDefault { use_case: String },
Protected { why: String },
Downloading,
StateChanged,
UnsafePath { path: PathBuf, why: String },
NothingToRetire,
Unsupported { why: String },
}
#[derive(Debug, Clone, Serialize)]
pub struct RetireOutcome {
pub plan: RetirePlan,
pub executed: bool,
pub removed: Vec<PathBuf>,
pub errors: Vec<String>,
}
pub fn same_entry(a: &Path, b: &Path) -> bool {
let parent = |p: &Path| p.parent().and_then(|d| d.canonicalize().ok());
a.file_name().is_some()
&& a.file_name() == b.file_name()
&& matches!((parent(a), parent(b)), (Some(x), Some(y)) if x == y)
}
pub(crate) fn verify_repo_dir(dir: &Path, hub: &Path) -> Result<(), String> {
let hub = hub.canonicalize().map_err(|e| e.to_string())?;
check_repo_dir(dir, &hub)
}
pub(crate) fn verify_entry(entry: &Path, repo_snapshots: &Path) -> Result<(), String> {
let meta = std::fs::symlink_metadata(entry).map_err(|e| e.to_string())?;
if !(meta.file_type().is_symlink() || meta.is_file()) {
return Err("no longer a link or a file".into());
}
let parent = entry
.parent()
.ok_or("no parent")?
.canonicalize()
.map_err(|e| e.to_string())?;
let snapshots = repo_snapshots.canonicalize().map_err(|e| e.to_string())?;
if !parent.starts_with(&snapshots) {
return Err("no longer inside the repo's snapshots".into());
}
Ok(())
}
pub(crate) fn verify_blob(blob: &Path, repo_blobs: &Path) -> Result<(), String> {
let meta = std::fs::symlink_metadata(blob).map_err(|e| e.to_string())?;
if !meta.is_file() {
return Err("no longer a regular file".into());
}
let parent = blob
.parent()
.ok_or("no parent")?
.canonicalize()
.map_err(|e| e.to_string())?;
if parent != repo_blobs.canonicalize().map_err(|e| e.to_string())? {
return Err("no longer in the repo's blobs".into());
}
Ok(())
}
pub(crate) fn remove_links_only_tree(
root: &Path,
models_dir: &Path,
) -> Result<Vec<PathBuf>, String> {
use crate::model_management::{unlink_checked, UnlinkKind};
let models_dir = models_dir.canonicalize().map_err(|e| e.to_string())?;
let parent = root
.parent()
.ok_or("projection has no parent")?
.canonicalize()
.map_err(|e| e.to_string())?;
if parent != models_dir {
return Err("the projection is no longer directly in the models directory".into());
}
let meta = std::fs::symlink_metadata(root).map_err(|e| e.to_string())?;
if meta.file_type().is_symlink() {
unlink_checked(root, UnlinkKind::Link).map_err(|e| e.to_string())?;
return Ok(vec![root.to_path_buf()]);
}
if !links_only(root) {
return Err("the projection now holds real files".into());
}
let mut dirs = vec![root.to_path_buf()];
let mut links = Vec::new();
let mut i = 0;
while i < dirs.len() {
for child in std::fs::read_dir(&dirs[i]).map_err(|e| e.to_string())? {
let path = child.map_err(|e| e.to_string())?.path();
let meta = std::fs::symlink_metadata(&path).map_err(|e| e.to_string())?;
if meta.file_type().is_symlink() {
links.push(path);
} else if meta.is_dir() {
dirs.push(path);
} else {
return Err(format!("{} is a real file", path.display()));
}
}
i += 1;
}
let mut removed = Vec::new();
for link in links {
unlink_checked(&link, UnlinkKind::Link).map_err(|e| e.to_string())?;
removed.push(link);
}
for dir in dirs.into_iter().rev() {
std::fs::remove_dir(&dir).map_err(|e| e.to_string())?;
}
removed.push(root.to_path_buf());
Ok(removed)
}
pub(crate) fn prune_empty_repo(dir: &Path) {
let mut files = Vec::new();
let mut dirs = vec![dir.to_path_buf()];
let mut i = 0;
while i < dirs.len() {
let Ok(read) = std::fs::read_dir(&dirs[i]) else {
return;
};
for child in read {
let Ok(child) = child else { return };
let path = child.path();
let Ok(meta) = std::fs::symlink_metadata(&path) else {
return;
};
if meta.is_dir() {
dirs.push(path);
} else if meta.is_file() && path.starts_with(dir.join("refs")) {
files.push(path);
} else {
return;
}
}
i += 1;
}
for file in files {
if std::fs::remove_file(&file).is_err() {
return;
}
}
for d in dirs.into_iter().rev() {
if std::fs::remove_dir(&d).is_err() {
return;
}
}
}
pub struct RetireInputs<'a> {
pub targets: &'a [&'a ModelSchema],
pub registry: &'a [&'a ModelSchema],
pub installed: &'a dyn Fn(&ModelSchema) -> bool,
pub owned: &'a Ownership,
pub hub: &'a Path,
}
pub fn plan_hub_retirement(inputs: &RetireInputs<'_>) -> RetirePlan {
let Some(target) = inputs.targets.first().copied() else {
return RetirePlan::default();
};
let mut plan = RetirePlan {
model_id: target.id.clone(),
also_retires: inputs.targets[1..].iter().map(|t| t.id.clone()).collect(),
..Default::default()
};
let target_ids: HashSet<&str> = inputs.targets.iter().map(|t| t.id.as_str()).collect();
let refs: Vec<HfReference> = inputs
.targets
.iter()
.flat_map(|t| hf_references(t))
.collect();
if !target.downloads_weights() {
plan.refusals.push(RetireRefusal::NothingToRetire);
return plan;
}
if refs.is_empty() {
plan.refusals.push(RetireRefusal::Unsupported {
why: format!(
"{} keeps its weights outside the Hugging Face hub",
target.id
),
});
return plan;
}
let hub = match inputs.hub.is_absolute().then(|| inputs.hub.canonicalize()) {
Some(Ok(hub)) => hub,
Some(Err(error)) if error.kind() == std::io::ErrorKind::NotFound => return plan,
_ => {
plan.refusals.push(RetireRefusal::UnsafePath {
path: inputs.hub.to_path_buf(),
why: "the hub cache path is not an absolute, readable directory".into(),
});
return plan;
}
};
let mut held: BTreeMap<String, (Option<BTreeSet<String>>, String)> = BTreeMap::new();
for row in inputs.registry.iter().copied() {
if target_ids.contains(row.id.as_str()) || !(inputs.installed)(row) {
continue;
}
for r in hf_references(row) {
let entry = held
.entry(r.repo)
.or_insert((Some(BTreeSet::new()), row.id.clone()));
merge_files(&mut entry.0, r.files);
}
}
let mut mine: BTreeMap<String, Option<BTreeSet<String>>> = BTreeMap::new();
for r in refs {
merge_files(mine.entry(r.repo).or_insert(Some(BTreeSet::new())), r.files);
}
for (repo, claimed) in mine {
let Some(owned) = inputs.owned.get(&repo) else {
plan.unowned.push(repo);
continue;
};
let files = claimed;
let Some(dir) = crate::hf_cache::checked_repo_dir_in(&hub, &repo) else {
plan.refusals.push(RetireRefusal::UnsafePath {
path: hub.join(&repo),
why: "the repo id is not a Hub repo id".into(),
});
continue;
};
match std::fs::symlink_metadata(&dir) {
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => {
plan.refusals.push(RetireRefusal::UnsafePath {
path: dir,
why: error.to_string(),
});
continue;
}
Ok(_) => {}
}
if let Err(why) = check_repo_dir(&dir, &hub) {
plan.refusals
.push(RetireRefusal::UnsafePath { path: dir, why });
continue;
}
let kept_files = match held.get(&repo) {
Some((None, holder)) => {
plan.kept.push(KeptReference {
repo: repo.clone(),
kept_for: holder.clone(),
});
continue;
}
Some((Some(held_files), holder)) => {
let overlaps = files
.as_ref()
.is_none_or(|mine| !mine.is_disjoint(held_files));
if overlaps {
plan.kept.push(KeptReference {
repo: repo.clone(),
kept_for: holder.clone(),
});
}
held_files.clone()
}
None => BTreeSet::new(),
};
match plan_repo(&dir, &repo, files.as_ref(), owned, &kept_files) {
Ok(Some(deletion)) => {
plan.freed_bytes += match &deletion {
HubDeletion::Repo { bytes, .. } | HubDeletion::Files { bytes, .. } => *bytes,
};
plan.deletions.push(deletion);
}
Ok(None) => {}
Err(why) => plan
.refusals
.push(RetireRefusal::UnsafePath { path: dir, why }),
}
}
plan
}
fn merge_files(slot: &mut Option<BTreeSet<String>>, more: Option<BTreeSet<String>>) {
match (slot.as_mut(), more) {
(Some(files), Some(more)) => files.extend(more),
(_, None) => *slot = None,
(None, Some(_)) => {}
}
}
fn check_repo_dir(dir: &Path, hub: &Path) -> Result<(), String> {
let meta = std::fs::symlink_metadata(dir).map_err(|e| e.to_string())?;
if meta.file_type().is_symlink() {
return Err("the repo directory is a symlink".into());
}
if !meta.is_dir() {
return Err("the repo path is not a directory".into());
}
let name = dir.file_name().ok_or("the repo path has no name")?;
if dir.canonicalize().map_err(|e| e.to_string())? != hub.join(name) {
return Err("the repo directory is not directly inside the hub".into());
}
for sub in ["blobs", "snapshots"] {
match std::fs::symlink_metadata(dir.join(sub)) {
Ok(m) if m.file_type().is_symlink() => {
return Err(format!("the repo's {sub}/ is a symlink"));
}
Ok(m) if !m.is_dir() => return Err(format!("the repo's {sub}/ is not a directory")),
Ok(_) => {}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e.to_string()),
}
}
Ok(())
}
struct Entry {
snapshot_path: PathBuf,
snapshot: String,
relative: String,
bytes_at: PathBuf,
}
fn plan_repo(
dir: &Path,
repo: &str,
files: Option<&BTreeSet<String>>,
owned: &Owned,
kept: &BTreeSet<String>,
) -> Result<Option<HubDeletion>, String> {
let entries = snapshot_entries(dir)?;
if *owned == Owned::Whole && files.is_none() && kept.is_empty() {
if transfer_in_progress(dir)? {
return Err("the repo has a download in progress".into());
}
let bytes = tree_bytes(dir);
return Ok(
(bytes > 0 || !entries.is_empty()).then(|| HubDeletion::Repo {
repo: repo.to_string(),
path: dir.to_path_buf(),
bytes,
}),
);
}
let claimed_and_free =
|e: &Entry| !kept.contains(&e.relative) && files.is_none_or(|f| f.contains(&e.relative));
let doomed: Box<dyn Fn(&Entry) -> bool> = match owned {
Owned::Whole => Box::new(claimed_and_free),
Owned::Files(of) => {
let owns = |e: &Entry| {
of.contains(&OwnedFile {
path: e.relative.clone(),
revision: None,
}) || of.contains(&OwnedFile {
path: e.relative.clone(),
revision: Some(e.snapshot.clone()),
})
};
let clean: HashSet<&str> = {
let mut dirty: HashSet<&str> = HashSet::new();
for e in &entries {
if !(owns(e) && claimed_and_free(e)) {
dirty.insert(e.snapshot.as_str());
}
}
entries
.iter()
.map(|e| e.snapshot.as_str())
.filter(|s| !dirty.contains(s))
.collect()
};
let clean: HashSet<String> = clean.into_iter().map(str::to_string).collect();
Box::new(move |e: &Entry| clean.contains(&e.snapshot) && owns(e) && claimed_and_free(e))
}
};
let surviving: HashSet<&Path> = entries
.iter()
.filter(|e| !doomed(e))
.map(|e| e.bytes_at.as_path())
.collect();
let mut gone_entries = Vec::new();
let mut blobs = BTreeSet::new();
for e in entries.iter().filter(|e| doomed(e)) {
gone_entries.push(e.snapshot_path.clone());
if !surviving.contains(e.bytes_at.as_path()) && e.bytes_at != e.snapshot_path {
blobs.insert(e.bytes_at.clone());
}
}
if gone_entries.is_empty() {
return Ok(None);
}
let bytes = unique_file_bytes(
blobs
.iter()
.chain(gone_entries.iter())
.map(PathBuf::as_path),
);
Ok(Some(HubDeletion::Files {
repo: repo.to_string(),
entries: gone_entries,
blobs: blobs.into_iter().collect(),
bytes,
}))
}
pub const ABANDONED_PARTIAL_SECS: u64 = 24 * 60 * 60;
pub fn discard_abandoned_partials(
hub: &Path,
owned: &Ownership,
) -> (Vec<(PathBuf, u64)>, Vec<String>) {
use crate::model_management::{unlink_checked, UnlinkKind};
let now = std::time::SystemTime::now();
let mut discarded = Vec::new();
let mut errors = Vec::new();
for (repo, owns) in owned {
let Some(dir) = crate::hf_cache::checked_repo_dir_in(hub, repo) else {
continue;
};
if *owns != Owned::Whole {
continue;
}
let Ok(read) = std::fs::read_dir(dir.join("blobs")) else {
continue;
};
let dir_name = dir
.file_name()
.map(|n| n.to_os_string())
.unwrap_or_default();
for entry in read.filter_map(Result::ok) {
let name = entry.file_name().to_string_lossy().into_owned();
let Some(blob) = name
.strip_suffix(".incomplete")
.or_else(|| name.strip_suffix(".sync.part"))
else {
continue;
};
let path = entry.path();
let abandoned = || {
let held = [
dir.join("blobs").join(format!("{blob}.lock")),
hub.join(".locks")
.join(&dir_name)
.join(format!("{blob}.lock")),
]
.iter()
.any(|lock| lock.exists() && lock_is_held(lock));
let meta = std::fs::symlink_metadata(&path).ok()?;
let age = meta
.modified()
.ok()
.and_then(|modified| now.duration_since(modified).ok())?;
(meta.is_file() && !held && age.as_secs() >= ABANDONED_PARTIAL_SECS)
.then_some(meta.len())
};
let Some(bytes) = abandoned() else {
continue;
};
match unlink_checked(&path, UnlinkKind::File) {
Ok(()) => discarded.push((path, bytes)),
Err(error) => errors.push(format!("{}: {error}", path.display())),
}
}
}
(discarded, errors)
}
pub(crate) fn transfer_in_progress(dir: &Path) -> Result<bool, String> {
let mut lock_dirs = vec![dir.join("blobs")];
if let (Some(hub), Some(name)) = (dir.parent(), dir.file_name()) {
lock_dirs.push(hub.join(".locks").join(name));
}
for lock_dir in &lock_dirs {
let read = match std::fs::read_dir(lock_dir) {
Ok(read) => read,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
Err(e) => return Err(e.to_string()),
};
for entry in read {
let path = entry.map_err(|e| e.to_string())?.path();
if path.extension().is_some_and(|e| e == "lock") && lock_is_held(&path) {
return Ok(true);
}
}
}
let now = std::time::SystemTime::now();
match std::fs::read_dir(dir.join("blobs")) {
Ok(read) => {
for entry in read {
let entry = entry.map_err(|e| e.to_string())?;
let name = entry.file_name().to_string_lossy().into_owned();
if !(name.ends_with(".incomplete") || name.ends_with(".sync.part")) {
continue;
}
let age = entry
.metadata()
.and_then(|m| m.modified())
.ok()
.and_then(|modified| now.duration_since(modified).ok());
if age.is_none_or(|age| age.as_secs() < ABANDONED_PARTIAL_SECS) {
return Ok(true);
}
}
Ok(false)
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(e) => Err(e.to_string()),
}
}
#[cfg(unix)]
fn lock_is_held(path: &Path) -> bool {
use std::os::fd::AsRawFd;
let Ok(file) = std::fs::File::open(path) else {
return true;
};
let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
if rc == 0 {
unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_UN) };
false
} else {
true
}
}
#[cfg(not(unix))]
fn lock_is_held(_path: &Path) -> bool {
false
}
fn unique_file_bytes<'a>(paths: impl Iterator<Item = &'a Path>) -> u64 {
let mut seen = HashSet::new();
let mut total = 0;
for path in paths {
let Ok(meta) = std::fs::symlink_metadata(path) else {
continue;
};
if !meta.is_file() {
continue;
}
#[cfg(unix)]
let key = {
use std::os::unix::fs::MetadataExt;
(meta.dev(), meta.ino())
};
#[cfg(not(unix))]
let key = path.to_path_buf();
if seen.insert(key) {
total += meta.len();
}
}
total
}
fn snapshot_entries(dir: &Path) -> Result<Vec<Entry>, String> {
let io = |e: std::io::Error| e.to_string();
let blobs = match dir.join("blobs").canonicalize() {
Ok(blobs) if blobs == dir.join("blobs") => Some(blobs),
Ok(_) => return Err("the repo's blobs/ resolves elsewhere".into()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => None,
Err(e) => return Err(io(e)),
};
let snapshots = dir.join("snapshots");
let read = match std::fs::read_dir(&snapshots) {
Ok(read) => read,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(io(e)),
};
let mut out = Vec::new();
for snapshot in read {
let root = snapshot.map_err(io)?.path();
if !std::fs::symlink_metadata(&root).map_err(io)?.is_dir() {
continue;
}
let mut stack = vec![root.clone()];
while let Some(at) = stack.pop() {
for child in std::fs::read_dir(&at).map_err(io)? {
let path = child.map_err(io)?.path();
let meta = std::fs::symlink_metadata(&path).map_err(io)?;
if meta.is_dir() {
stack.push(path);
continue;
}
let relative = path
.strip_prefix(&root)
.map_err(|e| e.to_string())?
.to_string_lossy()
.replace('\\', "/");
let bytes_at = if meta.file_type().is_symlink() {
match (path.canonicalize(), &blobs) {
(Ok(target), Some(blobs))
if target.parent() == Some(blobs.as_path())
&& std::fs::symlink_metadata(&target)
.is_ok_and(|m| m.is_file()) =>
{
target
}
_ => path.clone(),
}
} else {
path.clone()
};
out.push(Entry {
snapshot_path: path,
snapshot: root
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default(),
relative,
bytes_at,
});
}
}
}
Ok(out)
}
pub fn links_only(path: &Path) -> bool {
let mut stack = vec![path.to_path_buf()];
while let Some(at) = stack.pop() {
let Ok(meta) = std::fs::symlink_metadata(&at) else {
return false;
};
if meta.file_type().is_symlink() {
continue;
}
if !meta.is_dir() {
return false;
}
let Ok(read) = std::fs::read_dir(&at) else {
return false;
};
for child in read {
let Ok(child) = child else {
return false;
};
stack.push(child.path());
}
}
true
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum OrphanOwner {
Car,
User,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct HubOrphan {
pub repo: String,
#[serde(skip)]
pub path: PathBuf,
pub bytes: u64,
pub owner: OrphanOwner,
}
pub(crate) fn single_component(name: &str) -> bool {
let mut components = Path::new(name).components();
matches!(
(components.next(), components.next()),
(Some(std::path::Component::Normal(_)), None)
)
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, serde::Deserialize)]
pub struct RevokedCopy {
pub model_id: String,
pub path: PathBuf,
pub bytes: u64,
}
pub fn revoked_copies(state_root: &Path, models_dir: &Path) -> Vec<RevokedCopy> {
let names = crate::catalog::load_revoked_names(&crate::catalog::revoked_names_path(state_root));
if names.is_empty() {
return Vec::new();
}
let mut seen = HashSet::new();
let mut out = Vec::new();
for (name, id) in names {
if !single_component(&name) {
continue;
}
let receipt =
crate::model_management::read_receipt(state_root, &id).ok().flatten().filter(|r| {
r.artifact_kind != crate::model_management::ManagedArtifactKind::Symlink
});
for path in std::iter::once(models_dir.join(&name)).chain(receipt.map(|r| r.managed_path)) {
if std::fs::symlink_metadata(&path).is_err()
|| links_only(&path)
|| !seen.insert(path.clone())
{
continue;
}
out.push(RevokedCopy {
model_id: id.clone(),
bytes: tree_bytes(&path),
path,
});
}
}
out
}
pub(crate) fn path_resolving_into(models_dir: &Path, repo_dir: &Path) -> Option<PathBuf> {
let mut stack = vec![(models_dir.to_path_buf(), 0usize)];
while let Some((dir, depth)) = stack.pop() {
let Ok(read) = std::fs::read_dir(&dir) else {
continue;
};
for entry in read.filter_map(Result::ok) {
let path = entry.path();
let Ok(meta) = std::fs::symlink_metadata(&path) else {
continue;
};
if meta.file_type().is_symlink() {
if path
.canonicalize()
.is_ok_and(|target| target.starts_with(repo_dir))
{
return Some(path);
}
} else if meta.is_dir() && depth < 4 {
stack.push((path, depth + 1));
}
}
}
None
}
fn wholly_cars(dir: &Path, owned: Option<&Owned>) -> bool {
match owned {
Some(Owned::Whole) => true,
Some(Owned::Files(of)) => snapshot_entries(dir).is_ok_and(|entries| {
!entries.is_empty()
&& entries.iter().all(|e| {
of.contains(&OwnedFile {
path: e.relative.clone(),
revision: None,
}) || of.contains(&OwnedFile {
path: e.relative.clone(),
revision: Some(e.snapshot.clone()),
})
})
}),
None => false,
}
}
pub fn unreferenced_repos(
hub: &Path,
referenced: &std::collections::HashSet<String>,
owned: &Ownership,
) -> Vec<HubOrphan> {
let Ok(read) = std::fs::read_dir(hub) else {
return Vec::new();
};
let mut out = Vec::new();
for entry in read.filter_map(Result::ok) {
let name = entry.file_name().to_string_lossy().into_owned();
let Some(encoded) = name.strip_prefix("models--") else {
continue;
};
let repo = match encoded.split_once("--") {
Some((owner, rest)) if !rest.contains("--") => format!("{owner}/{rest}"),
Some(_) => continue,
None => encoded.to_string(),
};
if referenced.contains(&repo)
|| crate::hf_cache::checked_repo_dir_in(hub, &repo).as_deref()
!= Some(entry.path().as_path())
|| !entry.file_type().is_ok_and(|t| t.is_dir())
{
continue;
}
out.push(HubOrphan {
owner: if wholly_cars(&entry.path(), owned.get(&repo)) {
OrphanOwner::Car
} else {
OrphanOwner::User
},
bytes: tree_bytes(&entry.path()),
path: entry.path(),
repo,
});
}
out.sort_by(|a, b| a.repo.cmp(&b.repo));
out
}
pub(crate) fn tree_bytes(dir: &Path) -> u64 {
match std::fs::symlink_metadata(dir) {
Ok(meta) if meta.file_type().is_symlink() => return 0,
Ok(meta) if meta.is_file() => return meta.len(),
Ok(_) => {}
Err(_) => return 0,
}
let mut files = Vec::new();
let mut stack = vec![dir.to_path_buf()];
while let Some(at) = stack.pop() {
let Ok(read) = std::fs::read_dir(&at) else {
continue;
};
for child in read.filter_map(Result::ok) {
let Ok(meta) = std::fs::symlink_metadata(child.path()) else {
continue;
};
if meta.is_dir() {
stack.push(child.path());
} else if meta.is_file() {
files.push(child.path());
}
}
}
unique_file_bytes(files.iter().map(PathBuf::as_path))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_video_row_references_its_text_encoder() {
let video = crate::registry::builtin_catalog()
.into_iter()
.find(crate::registry::uses_ltx_text_encoder)
.expect("a catalog LTX row");
assert!(hf_references(&video).contains(&HfReference {
repo: crate::registry::LTX_TEXT_ENCODER.to_string(),
files: None,
}));
}
#[test]
fn unreferenced_repos_are_split_by_whose_they_are() {
let hub = tempfile::tempdir().unwrap();
for dir in [
"models--org--wanted",
"models--org--dropped",
"models--someone--theirs",
"models--bad--na--me",
"datasets--org--data",
] {
std::fs::create_dir_all(hub.path().join(dir).join("blobs")).unwrap();
}
std::fs::write(
hub.path().join("models--org--dropped/blobs/b"),
vec![0u8; 9],
)
.unwrap();
let referenced = std::collections::HashSet::from(["org/wanted".to_string()]);
let snap = hub.path().join("models--org--fetched/snapshots/r1");
std::fs::create_dir_all(&snap).unwrap();
std::fs::write(snap.join("model.safetensors"), b"w").unwrap();
std::fs::write(snap.join("config.json"), b"c").unwrap();
let mixed = hub.path().join("models--org--mixed/snapshots/r1");
std::fs::create_dir_all(&mixed).unwrap();
std::fs::write(mixed.join("tokenizer.json"), b"t").unwrap();
std::fs::write(mixed.join("weights.bin"), b"user").unwrap();
let file = |path: &str, rev: Option<&str>| OwnedFile {
path: path.into(),
revision: rev.map(str::to_owned),
};
let owned = Ownership::from([
("org/dropped".to_string(), Owned::Whole),
("someone/theirs".to_string(), Owned::Files(BTreeSet::new())),
(
"org/fetched".to_string(),
Owned::Files(BTreeSet::from([
file("model.safetensors", Some("r1")),
file("config.json", None),
])),
),
(
"org/mixed".to_string(),
Owned::Files(BTreeSet::from([file("tokenizer.json", None)])),
),
]);
let orphans = unreferenced_repos(hub.path(), &referenced, &owned);
let summary: Vec<(&str, OrphanOwner, u64)> = orphans
.iter()
.map(|o| (o.repo.as_str(), o.owner, o.bytes))
.collect();
assert_eq!(
summary,
vec![
("org/dropped", OrphanOwner::Car, 9),
("org/fetched", OrphanOwner::Car, 2),
("org/mixed", OrphanOwner::User, 5),
("someone/theirs", OrphanOwner::User, 0)
]
);
}
#[test]
fn the_digest_ignores_discovery_order() {
let plan = |twins: [&str; 2], entries: [&str; 2]| {
let mut plan = RetirePlan {
model_id: "a".into(),
also_retires: twins.iter().map(|t| t.to_string()).collect(),
deletions: vec![HubDeletion::Files {
repo: "o/r".into(),
entries: entries.iter().map(PathBuf::from).collect(),
blobs: Vec::new(),
bytes: 1,
}],
..RetirePlan::default()
};
plan.seal();
plan.digest
};
assert_eq!(plan(["b", "c"], ["x", "y"]), plan(["c", "b"], ["y", "x"]));
assert_ne!(plan(["b", "c"], ["x", "y"]), plan(["b", "d"], ["x", "y"]));
}
use crate::schema::{CostModel, ModelCapability, PerformanceEnvelope, TrustTier};
fn row(id: &str, name: &str, source: ModelSource) -> ModelSchema {
ModelSchema {
id: id.into(),
name: name.into(),
provider: "test".into(),
family: "qwen3".into(),
version: String::new(),
capabilities: vec![ModelCapability::Generate],
context_length: 4096,
max_output_tokens: None,
param_count: "4B".into(),
quantization: None,
performance: PerformanceEnvelope::default(),
cost: CostModel::default(),
source,
tags: vec![],
supported_params: vec![],
public_benchmarks: vec![],
trust_tier: TrustTier::Curated,
deprecated: false,
available: true,
weights_ready: true,
}
}
fn mlx(id: &str, repo: &str) -> ModelSchema {
row(
id,
id,
ModelSource::Mlx {
hf_repo: repo.into(),
hf_weight_file: None,
},
)
}
#[cfg(unix)]
fn gguf(id: &str, repo: &str, file: &str, tokenizer: &str) -> ModelSchema {
row(
id,
id,
ModelSource::Local {
hf_repo: repo.into(),
hf_filename: file.into(),
tokenizer_repo: tokenizer.into(),
},
)
}
#[cfg(unix)]
fn repo(hub: &Path, id: &str, files: &[(&str, &str, usize)]) -> PathBuf {
let dir = crate::hf_cache::repo_dir_in(hub, id);
let snap = dir.join("snapshots/abc");
std::fs::create_dir_all(dir.join("blobs")).unwrap();
std::fs::create_dir_all(dir.join("refs")).unwrap();
std::fs::write(dir.join("refs/main"), "abc").unwrap();
for (path, blob, size) in files {
let blob_path = dir.join("blobs").join(blob);
if !blob_path.exists() {
std::fs::write(&blob_path, vec![0u8; *size]).unwrap();
}
let entry = snap.join(path);
std::fs::create_dir_all(entry.parent().unwrap()).unwrap();
#[cfg(unix)]
std::os::unix::fs::symlink(&blob_path, &entry).unwrap();
}
dir
}
fn as_catalog(rows: &[&ModelSchema]) -> Ownership {
let mut out = Ownership::new();
for r in rows.iter().flat_map(|row| hf_references(row)) {
let entry = out.entry(r.repo).or_insert(Owned::Files(BTreeSet::new()));
match (entry, r.files) {
(slot, None) => *slot = Owned::Whole,
(Owned::Files(set), Some(files)) => {
set.extend(files.into_iter().map(|path| OwnedFile {
path,
revision: None,
}))
}
(Owned::Whole, Some(_)) => {}
}
}
out
}
fn plan(
target: &ModelSchema,
registry: &[&ModelSchema],
installed: &[&str],
hub: &Path,
) -> RetirePlan {
plan_owned(target, registry, installed, hub, &as_catalog(&[target]))
}
fn plan_owned(
target: &ModelSchema,
registry: &[&ModelSchema],
installed: &[&str],
hub: &Path,
owned: &Ownership,
) -> RetirePlan {
let installed: HashSet<String> = installed.iter().map(|s| s.to_string()).collect();
let is_installed = |m: &ModelSchema| installed.contains(&m.id);
plan_hub_retirement(&RetireInputs {
targets: &[target],
registry,
installed: &is_installed,
owned,
hub,
})
}
#[cfg(unix)]
fn snapshot(dir: &Path, rev: &str, files: &[(&str, &str, usize)]) {
for (path, blob, size) in files {
let blob_path = dir.join("blobs").join(blob);
if !blob_path.exists() {
std::fs::write(&blob_path, vec![0u8; *size]).unwrap();
}
let entry = dir.join("snapshots").join(rev).join(path);
std::fs::create_dir_all(entry.parent().unwrap()).unwrap();
std::os::unix::fs::symlink(&blob_path, &entry).unwrap();
}
}
fn hub() -> (tempfile::TempDir, PathBuf) {
let tmp = tempfile::tempdir().unwrap();
let hub = tmp.path().canonicalize().unwrap().join("hub");
std::fs::create_dir_all(&hub).unwrap();
(tmp, hub)
}
#[cfg(unix)]
#[test]
fn an_mlx_row_owns_its_whole_repo() {
let (_tmp, hub) = hub();
let dir = repo(
&hub,
"mlx-community/Qwen3-30B-A3B-4bit",
&[("config.json", "c", 10), ("model.safetensors", "w", 1000)],
);
let target = mlx("mlx/qwen3-30b-a3b:4bit", "mlx-community/Qwen3-30B-A3B-4bit");
let p = plan(&target, &[&target], &[], &hub);
assert!(!p.is_refused(), "{p:?}");
assert_eq!(
p.deletions,
vec![HubDeletion::Repo {
repo: "mlx-community/Qwen3-30B-A3B-4bit".into(),
path: dir,
bytes: 1013,
}]
);
assert_eq!(p.freed_bytes, 1013);
}
#[cfg(unix)]
#[test]
fn a_gguf_row_owns_only_the_files_it_fetched() {
let (_tmp, hub) = hub();
let quant = repo(
&hub,
"Qwen/Qwen3-8B-GGUF",
&[("Qwen3-8B-Q4_K_M.gguf", "q4", 500)],
);
repo(
&hub,
"Qwen/Qwen3-8B",
&[
("tokenizer.json", "tok", 7),
("model-00001.safetensors", "big", 9000),
("config.json", "cfg", 3),
],
);
let target = gguf(
"qwen/qwen3-8b:q4_k_m",
"Qwen/Qwen3-8B-GGUF",
"Qwen3-8B-Q4_K_M.gguf",
"Qwen/Qwen3-8B",
);
let p = plan(&target, &[&target], &[], &hub);
assert!(!p.is_refused(), "{p:?}");
assert_eq!(
p.deletions,
vec![HubDeletion::Files {
repo: "Qwen/Qwen3-8B-GGUF".into(),
entries: vec![quant.join("snapshots/abc/Qwen3-8B-Q4_K_M.gguf")],
blobs: vec![quant.join("blobs/q4")],
bytes: 500,
}],
"the user's base checkout keeps the shared tokenizer"
);
let (_tmp2, hub2) = super::tests::hub();
repo(&hub2, "Qwen/Qwen3-8B", &[("tokenizer.json", "tok", 7)]);
let p = plan(&target, &[&target], &[], &hub2);
assert!(
matches!(&p.deletions[..],
[HubDeletion::Files { repo, bytes: 7, .. }] if repo == "Qwen/Qwen3-8B"),
"{p:?}"
);
}
#[cfg(unix)]
#[test]
fn a_repo_another_installed_row_uses_is_kept() {
let (_tmp, hub) = hub();
repo(
&hub,
"mlx-community/gemma-4-12B-it-4bit",
&[("w", "w", 100)],
);
let a = mlx(
"mlx/gemma-4-12b-it:4bit",
"mlx-community/gemma-4-12B-it-4bit",
);
let b = mlx(
"vllm-mlx/gemma-4-12b-it",
"mlx-community/gemma-4-12B-it-4bit",
);
let p = plan(&a, &[&a, &b], &["vllm-mlx/gemma-4-12b-it"], &hub);
assert!(p.deletions.is_empty(), "{p:?}");
assert_eq!(p.kept[0].kept_for, "vllm-mlx/gemma-4-12b-it");
let p = plan(&a, &[&a, &b], &[], &hub);
assert_eq!(p.deletions.len(), 1);
}
#[cfg(unix)]
#[test]
fn a_shared_blob_survives_while_any_entry_keeps_it() {
let (_tmp, hub) = hub();
let dir = repo(&hub, "Qwen/Qwen3-8B", &[("tokenizer.json", "same", 7)]);
snapshot(&dir, "def", &[("other.json", "same", 7)]);
let target = gguf("g", "Qwen/Qwen3-8B-GGUF", "q.gguf", "Qwen/Qwen3-8B");
let p = plan(&target, &[&target], &[], &hub);
let [HubDeletion::Files {
entries,
blobs,
bytes,
..
}] = &p.deletions[..]
else {
panic!("{p:?}");
};
assert_eq!(entries, &vec![dir.join("snapshots/abc/tokenizer.json")]);
assert!(blobs.is_empty(), "another snapshot still points at it");
assert_eq!(*bytes, 0);
}
#[cfg(unix)]
#[test]
fn an_unowned_repo_is_never_touched_even_by_its_own_row() {
let (_tmp, hub) = hub();
repo(&hub, "google/flan-t5-large", &[("w", "w", 100)]);
let target = mlx("mlx/x", "google/flan-t5-large");
let p = plan_owned(&target, &[&target], &[], &hub, &Ownership::new());
assert!(p.deletions.is_empty());
assert_eq!(p.unowned, vec!["google/flan-t5-large".to_string()]);
}
#[cfg(unix)]
#[test]
fn twins_retired_together_free_their_shared_repo() {
let (_tmp, hub) = hub();
repo(
&hub,
"mlx-community/gemma-4-12B-it-4bit",
&[("w", "w", 100)],
);
let a = mlx(
"mlx/gemma-4-12b-it:4bit",
"mlx-community/gemma-4-12B-it-4bit",
);
let b = mlx(
"vllm-mlx/gemma-4-12b-it",
"mlx-community/gemma-4-12B-it-4bit",
);
let owned: Ownership = [(
"mlx-community/gemma-4-12B-it-4bit".to_string(),
Owned::Whole,
)]
.into();
let installed = |_: &ModelSchema| true;
let p = plan_hub_retirement(&RetireInputs {
targets: &[&a, &b],
registry: &[&a, &b],
installed: &installed,
owned: &owned,
hub: &hub,
});
assert!(p.kept.is_empty(), "{p:?}");
assert_eq!(p.also_retires, vec!["vllm-mlx/gemma-4-12b-it".to_string()]);
assert!(matches!(
&p.deletions[..],
[HubDeletion::Repo { bytes: 103, .. }]
));
}
#[cfg(unix)]
#[test]
fn a_recorded_only_repo_is_owned_file_by_file_and_revision() {
let (_tmp, hub) = hub();
let dir = repo(&hub, "someone/model", &[("model.safetensors", "w", 100)]);
snapshot(&dir, "def", &[("model.safetensors", "w2", 400)]);
let target = mlx("user/model", "someone/model");
let recorded: Ownership = [(
"someone/model".to_string(),
Owned::Files(BTreeSet::from([OwnedFile {
path: "model.safetensors".into(),
revision: Some("abc".into()),
}])),
)]
.into();
let p = plan_owned(
&target,
&[&target],
&[],
&hub,
&ownership(Ownership::new(), recorded),
);
let [HubDeletion::Files { entries, bytes, .. }] = &p.deletions[..] else {
panic!("{p:?}");
};
assert_eq!(entries, &vec![dir.join("snapshots/abc/model.safetensors")]);
assert_eq!(*bytes, 100, "revision def is not CAR's");
let catalog: Ownership = [("someone/model".to_string(), Owned::Whole)].into();
let p = plan_owned(
&target,
&[&target],
&[],
&hub,
&ownership(catalog, Ownership::new()),
);
assert!(matches!(&p.deletions[..], [HubDeletion::Repo { .. }]));
}
#[cfg(unix)]
#[test]
fn a_whole_claim_on_a_file_level_catalog_repo_is_never_a_repo_delete() {
let (_tmp, hub) = hub();
repo(
&hub,
"Qwen/Qwen3-8B",
&[
("tokenizer.json", "tok", 7),
("model-00001.safetensors", "big", 9000),
],
);
let catalog_gguf = gguf(
"qwen/qwen3-8b:q4_k_m",
"Qwen/Qwen3-8B-GGUF",
"q.gguf",
"Qwen/Qwen3-8B",
);
let user = mlx("user/qwen3-8b", "Qwen/Qwen3-8B");
let p = plan_owned(
&user,
&[&user, &catalog_gguf],
&[],
&hub,
&as_catalog(&[&catalog_gguf]),
);
assert!(p.deletions.is_empty(), "{p:?}");
}
#[cfg(unix)]
#[test]
fn a_whole_repo_with_an_hf_hub_partial_blob_is_refused() {
let (_tmp, hub) = hub();
let dir = repo(&hub, "mlx-community/Qwen3-4B-4bit", &[("w", "w", 10)]);
let partial = dir.join("blobs/z.sync.part");
std::fs::write(&partial, b"partial").unwrap();
let target = mlx("mlx/q", "mlx-community/Qwen3-4B-4bit");
let p = plan(&target, &[&target], &[], &hub);
assert!(p.deletions.is_empty());
assert!(
matches!(&p.refusals[..], [RetireRefusal::UnsafePath { .. }]),
"{p:?}"
);
let stale = std::time::SystemTime::now()
- std::time::Duration::from_secs(ABANDONED_PARTIAL_SECS + 60);
std::fs::File::options()
.write(true)
.open(&partial)
.unwrap()
.set_modified(stale)
.unwrap();
let p = plan(&target, &[&target], &[], &hub);
assert!(
matches!(&p.deletions[..], [HubDeletion::Repo { .. }]),
"{p:?}"
);
}
#[cfg(unix)]
#[test]
fn links_only_tells_a_projection_from_user_content() {
let tmp = tempfile::tempdir().unwrap();
let proj = tmp.path().join("proj");
std::fs::create_dir_all(proj.join("sub")).unwrap();
std::os::unix::fs::symlink("/nowhere", proj.join("a")).unwrap();
std::os::unix::fs::symlink("/nowhere", proj.join("sub/b")).unwrap();
assert!(links_only(&proj));
std::fs::write(proj.join("sub/real.bin"), b"user bytes").unwrap();
assert!(!links_only(&proj));
let link = tmp.path().join("link");
std::os::unix::fs::symlink(&proj, &link).unwrap();
assert!(
links_only(&link),
"a link is CAR's to unlink, whatever it points at"
);
}
#[cfg(unix)]
#[test]
fn symlinked_blobs_or_snapshots_refuse_the_plan() {
for sub in ["blobs", "snapshots"] {
let (tmp, hub) = hub();
let dir = repo(&hub, "Qwen/Qwen3-8B", &[("tokenizer.json", "tok", 7)]);
let elsewhere = tmp.path().join("elsewhere");
std::fs::rename(dir.join(sub), &elsewhere).unwrap();
std::os::unix::fs::symlink(&elsewhere, dir.join(sub)).unwrap();
let target = gguf("g", "Qwen/Qwen3-8B-GGUF", "q.gguf", "Qwen/Qwen3-8B");
let p = plan(&target, &[&target], &[], &hub);
assert!(
p.refusals
.iter()
.any(|r| matches!(r, RetireRefusal::UnsafePath { .. })),
"{sub}: {p:?}"
);
assert!(p.deletions.is_empty(), "{sub}");
}
}
#[cfg(unix)]
#[test]
fn an_empty_or_partial_repo_is_not_widened_to_a_whole_repo_delete() {
let (_tmp, hub) = hub();
let dir = crate::hf_cache::repo_dir_in(&hub, "Qwen/Qwen3-8B");
std::fs::create_dir_all(dir.join("blobs")).unwrap();
std::fs::write(dir.join("blobs/x.incomplete"), vec![0u8; 900]).unwrap();
let target = gguf("g", "Qwen/Qwen3-8B-GGUF", "q.gguf", "Qwen/Qwen3-8B");
let p = plan(&target, &[&target], &[], &hub);
assert!(p.deletions.is_empty(), "{p:?}");
assert_eq!(p.freed_bytes, 0);
}
#[cfg(unix)]
#[test]
fn a_whole_repo_with_an_incomplete_blob_is_refused() {
let (_tmp, hub) = hub();
let dir = repo(&hub, "mlx-community/Qwen3-4B-4bit", &[("w", "w", 10)]);
std::fs::write(dir.join("blobs/y.incomplete"), b"partial").unwrap();
let target = mlx("mlx/q", "mlx-community/Qwen3-4B-4bit");
let p = plan(&target, &[&target], &[], &hub);
assert!(p.deletions.is_empty());
assert!(
matches!(&p.refusals[..], [RetireRefusal::UnsafePath { .. }]),
"{p:?}"
);
}
#[cfg(unix)]
#[test]
fn an_unreadable_snapshot_refuses_the_plan() {
use std::os::unix::fs::PermissionsExt;
let (_tmp, hub) = hub();
let dir = repo(
&hub,
"Qwen/Qwen3-8B",
&[("tokenizer.json", "tok", 7), ("sub/x", "x", 3)],
);
let locked = dir.join("snapshots/abc/sub");
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap();
let target = gguf("g", "Qwen/Qwen3-8B-GGUF", "q.gguf", "Qwen/Qwen3-8B");
let p = plan(&target, &[&target], &[], &hub);
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).unwrap();
assert!(p.deletions.is_empty(), "{p:?}");
assert!(p
.refusals
.iter()
.any(|r| matches!(r, RetireRefusal::UnsafePath { .. })));
}
#[cfg(unix)]
#[test]
fn hardlinked_bytes_are_counted_once() {
let (_tmp, hub) = hub();
let dir = crate::hf_cache::repo_dir_in(&hub, "mlx-community/H");
std::fs::create_dir_all(dir.join("snapshots/s")).unwrap();
std::fs::create_dir_all(dir.join("blobs")).unwrap();
std::fs::write(dir.join("blobs/b"), vec![0u8; 100]).unwrap();
std::fs::hard_link(dir.join("blobs/b"), dir.join("snapshots/s/w")).unwrap();
let target = mlx("mlx/h", "mlx-community/H");
let p = plan(&target, &[&target], &[], &hub);
assert_eq!(p.freed_bytes, 100, "{p:?}");
}
#[cfg(unix)]
#[test]
fn unsafe_paths_refuse_the_plan() {
let (tmp, hub) = hub();
let elsewhere = tmp.path().join("elsewhere");
std::fs::create_dir_all(&elsewhere).unwrap();
std::os::unix::fs::symlink(&elsewhere, hub.join("models--a--b")).unwrap();
let target = mlx("mlx/a", "a/b");
let p = plan(&target, &[&target], &[], &hub);
assert!(
matches!(&p.refusals[..], [RetireRefusal::UnsafePath { .. }]),
"{p:?}"
);
assert!(p.deletions.is_empty());
let p = plan(&target, &[&target], &[], Path::new("rel/hub"));
assert!(matches!(
&p.refusals[..],
[RetireRefusal::UnsafePath { .. }]
));
let bad = mlx("mlx/bad", "../../etc");
let p = plan(&bad, &[&bad], &[], &hub);
assert!(
matches!(&p.refusals[..], [RetireRefusal::UnsafePath { .. }]),
"{p:?}"
);
}
#[cfg(unix)]
#[test]
fn a_link_out_of_the_repo_never_frees_its_target() {
let (tmp, hub) = hub();
let foreign = tmp.path().join("foreign.bin");
std::fs::write(&foreign, vec![0u8; 50]).unwrap();
let dir = crate::hf_cache::repo_dir_in(&hub, "Qwen/Qwen3-8B");
std::fs::create_dir_all(dir.join("snapshots/abc")).unwrap();
std::fs::create_dir_all(dir.join("blobs")).unwrap();
std::os::unix::fs::symlink(&foreign, dir.join("snapshots/abc/tokenizer.json")).unwrap();
let target = gguf("g", "Qwen/Qwen3-8B-GGUF", "q.gguf", "Qwen/Qwen3-8B");
let p = plan(&target, &[&target], &[], &hub);
let HubDeletion::Files { entries, blobs, .. } = &p.deletions[0] else {
panic!("{p:?}");
};
assert_eq!(entries, &vec![dir.join("snapshots/abc/tokenizer.json")]);
assert!(blobs.is_empty());
}
#[test]
fn a_remote_row_has_nothing_to_retire() {
let (_tmp, hub) = hub();
let target = crate::openrouter::curated_schemas()
.into_iter()
.next()
.expect("a remote row");
let p = plan(&target, &[&target], &[], &hub);
assert_eq!(p.refusals, vec![RetireRefusal::NothingToRetire]);
}
#[test]
fn a_cache_path_names_its_snapshot_revision() {
assert_eq!(
snapshot_revision(Path::new("/h/models--a--b/snapshots/0f1e/sub/tok.json")),
Some("0f1e".into())
);
assert_eq!(
snapshot_revision(Path::new("/h/models--a--b/blobs/x")),
None
);
assert_eq!(
snapshot_revision(Path::new(
"/Volumes/snapshots/hub/models--a--b/snapshots/r9/f"
)),
Some("r9".into())
);
}
#[tokio::test]
async fn only_noted_fetches_are_recorded() {
note_fetched("outside/scope", "x", Some("r0".into()));
let ((), unknown) = collecting_fetches(async { note_fetched("a/b", "x", None) }).await;
assert!(unknown.is_empty(), "an unknown revision is not recorded");
let ((), fetched) = collecting_fetches(async {
note_fetched("a/b", "model.safetensors", Some("r1".into()));
note_fetched("a/b", "config.json", Some("r1".into()));
})
.await;
assert_eq!(fetched.len(), 2);
let root = tempfile::tempdir().unwrap();
record_download(root.path(), Path::new("/hub"), "m", &fetched);
let text = std::fs::read_to_string(provenance_path(root.path())).unwrap();
assert_eq!(text.lines().count(), 1, "{text}");
assert!(text.contains("model.safetensors") && text.contains("config.json"));
assert!(!text.contains("outside/scope"));
record_download(root.path(), Path::new("/hub"), "m", &[]);
assert_eq!(
std::fs::read_to_string(provenance_path(root.path())).unwrap(),
text
);
}
#[test]
fn the_flux_row_takes_one_file_from_its_base_checkpoint() {
let flux = mlx("mlx/flux", crate::registry::FLUX_AUXILIARY.0);
let refs = hf_references(&flux);
assert_eq!(refs.len(), 2);
assert_eq!(refs[1].repo, crate::registry::FLUX_AUXILIARY.2);
assert_eq!(
refs[1].files,
Some(BTreeSet::from([crate::registry::FLUX_AUXILIARY
.3
.to_string()]))
);
}
}