use serde::Serialize;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum ImageKind {
Container,
RunnerEnv,
ModelCache,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct Image {
pub id: String,
pub kind: ImageKind,
pub size_bytes: u64,
pub last_used: Option<SystemTime>,
pub created: SystemTime,
pub earnings: f64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum RemoveError {
InUse,
Timeout,
NotFound,
Other(String),
}
pub trait Store {
fn list(&self) -> Vec<Image>;
fn in_use(&self, id: &str) -> bool;
fn remove(&self, id: &str, timeout: Duration) -> Result<(), RemoveError>;
}
pub const REMOVE_TIMEOUT: Duration = Duration::from_secs(60);
pub const GC_EVERY: Duration = Duration::from_secs(3600);
#[derive(Debug, Clone, Default, PartialEq, Serialize)]
pub struct GcReport {
pub removed: Vec<(String, u64)>,
pub skipped: Vec<(String, RemoveError)>,
}
pub fn gc(store: &dyn Store, now: SystemTime, ttl: Duration) -> GcReport {
let mut report = GcReport::default();
for img in store.list() {
if img.kind == ImageKind::RunnerEnv {
continue;
}
let since = img.last_used.unwrap_or(img.created);
if now.duration_since(since).unwrap_or_default() < ttl {
continue;
}
if store.in_use(&img.id) {
report.skipped.push((img.id, RemoveError::InUse));
continue;
}
match store.remove(&img.id, REMOVE_TIMEOUT) {
Ok(()) => report.removed.push((img.id, img.size_bytes)),
Err(e) => report.skipped.push((img.id, e)),
}
}
report
}
pub fn remove_now(store: &dyn Store, id: &str) -> Result<u64, RemoveError> {
let img = store
.list()
.into_iter()
.find(|i| i.id == id)
.ok_or(RemoveError::NotFound)?;
if store.in_use(id) {
return Err(RemoveError::InUse);
}
store.remove(id, REMOVE_TIMEOUT)?;
Ok(img.size_bytes)
}
pub fn run_with_timeout(
timeout: Duration,
f: impl FnOnce() -> Result<(), RemoveError> + Send + 'static,
) -> Result<(), RemoveError> {
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let _ = tx.send(f());
});
rx.recv_timeout(timeout)
.unwrap_or(Err(RemoveError::Timeout))
}
pub struct HostStore {
pub root: PathBuf,
pub enabled_backends: Vec<String>,
pub jobs: Box<dyn Fn() -> Option<Vec<String>> + Send + Sync>,
pub last_job_seen: Option<SystemTime>,
}
fn dir_size(p: &Path) -> u64 {
let Ok(rd) = std::fs::read_dir(p) else {
return 0;
};
rd.flatten()
.map(|e| match e.file_type() {
Ok(t) if t.is_dir() => dir_size(&e.path()),
Ok(t) if t.is_file() => e.metadata().map(|m| m.len()).unwrap_or(0),
_ => 0,
})
.sum()
}
fn mtime(p: &Path) -> SystemTime {
std::fs::metadata(p)
.and_then(|m| m.modified())
.unwrap_or(SystemTime::UNIX_EPOCH)
}
impl HostStore {
fn entries(&self) -> Vec<(String, PathBuf, ImageKind)> {
let mut out = Vec::new();
let mut add =
|dir: PathBuf, prefix: &str, kind: ImageKind, filter: &dyn Fn(&str) -> bool| {
if let Ok(rd) = std::fs::read_dir(&dir) {
for e in rd.flatten() {
let name = e.file_name().to_string_lossy().into_owned();
if e.path().is_dir() && filter(&name) {
out.push((format!("{prefix}/{name}"), e.path(), kind));
}
}
}
};
add(self.root.join("envs"), "env", ImageKind::RunnerEnv, &|_| {
true
});
add(
self.root.join("work"),
"work",
ImageKind::ModelCache,
&|_| true,
);
add(
self.root.join("hf/hub"),
"hf",
ImageKind::ModelCache,
&|n| n.starts_with("models--"),
);
out
}
fn path_of(&self, id: &str) -> Option<PathBuf> {
self.entries()
.into_iter()
.find(|(i, _, _)| i == id)
.map(|(_, p, _)| p)
}
}
impl Store for HostStore {
fn list(&self) -> Vec<Image> {
self.entries()
.into_iter()
.map(|(id, path, kind)| {
let t = mtime(&path);
let last_used = match kind {
ImageKind::RunnerEnv => None,
_ if id.starts_with("hf/") => Some(self.last_job_seen.map_or(t, |j| j.max(t))),
_ => Some(t),
};
Image {
id,
kind,
size_bytes: dir_size(&path),
last_used,
created: t,
earnings: 0.0,
}
})
.collect()
}
fn in_use(&self, id: &str) -> bool {
match id.split_once('/') {
Some(("env", b)) => self.enabled_backends.iter().any(|e| e == b),
Some(("work", j)) => (self.jobs)().is_none_or(|r| r.iter().any(|x| x == j)),
Some(("hf", _)) => (self.jobs)().is_none_or(|r| !r.is_empty()),
_ => true,
}
}
fn remove(&self, id: &str, timeout: Duration) -> Result<(), RemoveError> {
let path = self.path_of(id).ok_or(RemoveError::NotFound)?;
run_with_timeout(timeout, move || {
std::fs::remove_dir_all(&path).map_err(|e| RemoveError::Other(e.to_string()))
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::cell::RefCell;
use std::time::{Duration, SystemTime};
const DAY: Duration = Duration::from_secs(86_400);
struct Fake {
images: Vec<Image>,
in_use: RefCell<Vec<String>>,
flips: Vec<String>,
hangs: Vec<String>,
removed: RefCell<Vec<String>>,
}
impl Fake {
fn new(images: Vec<Image>) -> Self {
Self {
images,
in_use: RefCell::new(vec![]),
flips: vec![],
hangs: vec![],
removed: RefCell::new(vec![]),
}
}
}
impl Store for Fake {
fn list(&self) -> Vec<Image> {
self.in_use.borrow_mut().extend(self.flips.iter().cloned());
self.images.clone()
}
fn in_use(&self, id: &str) -> bool {
self.in_use.borrow().iter().any(|i| i == id)
}
fn remove(&self, id: &str, _timeout: Duration) -> Result<(), RemoveError> {
if self.hangs.iter().any(|h| h == id) {
return Err(RemoveError::Timeout);
}
self.removed.borrow_mut().push(id.to_string());
Ok(())
}
}
fn now() -> SystemTime {
SystemTime::UNIX_EPOCH + 100 * DAY
}
fn img(id: &str, days_ago: Option<u64>, created_days_ago: u64) -> Image {
Image {
id: id.into(),
kind: ImageKind::ModelCache,
size_bytes: 1 << 30,
last_used: days_ago.map(|d| now() - DAY * d as u32),
created: now() - DAY * created_days_ago as u32,
earnings: 0.0,
}
}
#[test]
fn gc_removes_only_past_ttl() {
let f = Fake::new(vec![img("old", Some(8), 30), img("fresh", Some(6), 30)]);
let r = gc(&f, now(), 7 * DAY);
assert_eq!(*f.removed.borrow(), vec!["old".to_string()]);
assert_eq!(r.removed, vec![("old".to_string(), 1 << 30)]);
}
#[test]
fn gc_never_removes_in_use() {
let f = Fake::new(vec![img("old", Some(30), 60)]);
f.in_use.borrow_mut().push("old".into());
let r = gc(&f, now(), 7 * DAY);
assert!(f.removed.borrow().is_empty());
assert_eq!(r.skipped, vec![("old".to_string(), RemoveError::InUse)]);
}
#[test]
fn gc_rechecks_in_use_before_each_removal() {
let mut f = Fake::new(vec![img("old", Some(30), 60)]);
f.flips = vec!["old".into()];
let r = gc(&f, now(), 7 * DAY);
assert!(f.removed.borrow().is_empty());
assert_eq!(r.skipped, vec![("old".to_string(), RemoveError::InUse)]);
}
#[test]
fn gc_removal_timeout_is_per_item() {
let mut f = Fake::new(vec![img("stuck", Some(30), 60), img("next", Some(30), 60)]);
f.hangs = vec!["stuck".into()];
let r = gc(&f, now(), 7 * DAY);
assert_eq!(*f.removed.borrow(), vec!["next".to_string()]);
assert_eq!(r.skipped, vec![("stuck".to_string(), RemoveError::Timeout)]);
}
#[test]
fn gc_never_auto_removes_runner_envs() {
let mut f = Fake::new(vec![img("env/peft", None, 60)]);
f.images[0].kind = ImageKind::RunnerEnv;
gc(&f, now(), 7 * DAY);
assert!(f.removed.borrow().is_empty());
assert_eq!(remove_now(&f, "env/peft"), Ok(1 << 30));
}
fn host_store(root: &std::path::Path, jobs: Option<Vec<String>>) -> HostStore {
HostStore {
root: root.to_path_buf(),
enabled_backends: vec![],
jobs: Box::new(move || jobs.clone()),
last_job_seen: None,
}
}
fn tmp_root() -> std::path::PathBuf {
let d = std::env::temp_dir().join(format!(
"zc-images-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(d.join("hf/hub/models--a--b")).unwrap();
std::fs::create_dir_all(d.join("work/j1")).unwrap();
d
}
#[test]
fn in_use_asks_the_runner_at_each_check() {
let root = tmp_root();
let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let c = calls.clone();
let store = HostStore {
root: root.clone(),
enabled_backends: vec![],
jobs: Box::new(move || {
let n = c.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Some(if n == 0 {
vec![]
} else {
vec!["j1".to_string()]
})
}),
last_job_seen: None,
};
assert!(!store.in_use("work/j1"));
assert!(store.in_use("work/j1"));
assert!(store.in_use("hf/models--a--b"));
}
#[test]
fn unknown_jobs_mean_in_use() {
let root = tmp_root();
let store = host_store(&root, None);
assert!(store.in_use("work/j1"));
assert!(store.in_use("hf/models--a--b"));
let store = host_store(&root, Some(vec![]));
assert!(!store.in_use("work/j1"));
}
#[test]
fn a_model_ages_from_the_last_job_not_its_download() {
let root = tmp_root();
let mut store = host_store(&root, Some(vec![]));
let recent = SystemTime::now() - Duration::from_secs(3600);
store.last_job_seen = Some(recent);
let hf = store
.list()
.into_iter()
.find(|i| i.id == "hf/models--a--b")
.unwrap();
assert!(hf.last_used.unwrap() >= recent);
}
#[test]
fn never_used_counts_from_creation() {
let f = Fake::new(vec![img("unused-old", None, 8), img("unused-new", None, 2)]);
gc(&f, now(), 7 * DAY);
assert_eq!(*f.removed.borrow(), vec!["unused-old".to_string()]);
}
#[test]
fn remove_now_refuses_in_use() {
let f = Fake::new(vec![img("a", Some(0), 1)]);
f.in_use.borrow_mut().push("a".into());
assert_eq!(remove_now(&f, "a"), Err(RemoveError::InUse));
let f = Fake::new(vec![img("a", Some(0), 1)]);
assert_eq!(remove_now(&f, "a"), Ok(1 << 30));
assert_eq!(remove_now(&f, "missing"), Err(RemoveError::NotFound));
}
#[test]
fn remove_with_timeout_times_out() {
let r = run_with_timeout(Duration::from_millis(50), || {
std::thread::sleep(Duration::from_secs(2));
Ok(())
});
assert_eq!(r, Err(RemoveError::Timeout));
assert_eq!(run_with_timeout(Duration::from_secs(1), || Ok(())), Ok(()));
}
}