use std::collections::HashSet;
use std::num::NonZeroUsize;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime};
use lru::LruCache;
use wasmtime::component::{Component, Linker};
use wasmtime::Engine;
use crate::cache;
use crate::sandbox::HandlerPre;
use crate::types::{valid_name, HostCtx};
use crate::Error;
pub(crate) fn cwasm_dir(modules_dir: &Path) -> PathBuf {
modules_dir.join("cwasm")
}
pub(crate) fn wasm_dir(modules_dir: &Path) -> PathBuf {
modules_dir.join("wasm")
}
fn wasm_path(modules_dir: &Path, digest: &str) -> PathBuf {
wasm_dir(modules_dir).join(format!("{digest}.wasm"))
}
pub(crate) fn wasm_exists(modules_dir: &Path, digest: &str) -> bool {
cache::is_valid_digest(digest) && wasm_path(modules_dir, digest).is_file()
}
fn pointer_path(modules_dir: &Path, tenant: &str, func: &str) -> Option<PathBuf> {
if !valid_name(tenant) || !valid_name(func) {
return None;
}
Some(modules_dir.join("tenants").join(tenant).join(func))
}
pub(crate) fn write_pointer(
modules_dir: &Path,
tenant: &str,
func: &str,
digest: &str,
) -> Result<(), Error> {
if !cache::is_valid_digest(digest) {
return Err(Error::Corrupt(
"invalid digest: expected 64 lowercase hex characters".into(),
));
}
let path = pointer_path(modules_dir, tenant, func)
.ok_or_else(|| Error::Corrupt("invalid tenant or function name".into()))?;
let parent = path.parent().expect("pointer_path always has a parent");
cache::atomic_write(parent, &path, digest.as_bytes())
}
pub(crate) fn remove_pointer(modules_dir: &Path, tenant: &str, func: &str) -> Result<(), Error> {
let Some(path) = pointer_path(modules_dir, tenant, func) else {
return Ok(());
};
match std::fs::remove_file(&path) {
Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
_ => Ok(()),
}
}
pub(crate) fn touch(path: &Path) -> std::io::Result<()> {
std::fs::OpenOptions::new()
.write(true)
.open(path)?
.set_modified(SystemTime::now())
}
pub(crate) fn write_wasm_source(
modules_dir: &Path,
digest: &str,
wasm_bytes: &[u8],
) -> Result<(), Error> {
if !cache::is_valid_digest(digest) {
return Err(Error::Corrupt(
"invalid digest: expected 64 lowercase hex characters".into(),
));
}
let dest = wasm_path(modules_dir, digest);
match touch(&dest) {
Ok(()) => return Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e.into()),
}
cache::atomic_write(&wasm_dir(modules_dir), &dest, wasm_bytes)
}
pub(crate) fn read_pointer(
modules_dir: &Path,
tenant: &str,
func: &str,
) -> Result<Option<String>, Error> {
let Some(path) = pointer_path(modules_dir, tenant, func) else {
return Ok(None);
};
let digest = match std::fs::read_to_string(&path) {
Ok(s) => s,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(e.into()),
};
let digest = digest.trim();
if !cache::is_valid_digest(digest) {
return Err(Error::Corrupt(format!(
"corrupt pointer file {}: not a valid digest",
path.display()
)));
}
Ok(Some(digest.to_string()))
}
struct CacheEntry {
pre: Arc<HandlerPre<HostCtx>>,
weight: usize,
}
pub(crate) enum Lookup {
NotFound,
Hit(Arc<HandlerPre<HostCtx>>),
Miss(String),
}
pub(crate) enum Loaded {
Ready(Arc<HandlerPre<HostCtx>>),
NeedsCompile,
}
struct ComponentCacheInner {
lru: LruCache<String, CacheEntry>,
total_bytes: usize,
}
pub(crate) struct ComponentCache {
engine: Engine,
linker: Arc<Linker<HostCtx>>,
modules_dir: PathBuf,
inner: Mutex<ComponentCacheInner>,
byte_budget: usize,
}
impl ComponentCache {
pub(crate) fn new(
engine: Engine,
linker: Arc<Linker<HostCtx>>,
modules_dir: PathBuf,
cap: usize,
byte_budget: usize,
) -> Self {
Self {
engine,
linker,
modules_dir,
inner: Mutex::new(ComponentCacheInner {
lru: LruCache::new(NonZeroUsize::new(cap.max(1)).unwrap()),
total_bytes: 0,
}),
byte_budget,
}
}
pub(crate) fn lookup(&self, tenant: &str, func: &str) -> Result<Lookup, Error> {
let Some(digest) = read_pointer(&self.modules_dir, tenant, func)? else {
return Ok(Lookup::NotFound);
};
let hit = self
.inner
.lock()
.unwrap_or_else(|p| p.into_inner())
.lru
.get(&digest)
.map(|e| e.pre.clone());
Ok(match hit {
Some(pre) => Lookup::Hit(pre),
None => Lookup::Miss(digest),
})
}
pub(crate) fn get_or_load(&self, digest: &str) -> Result<Arc<HandlerPre<HostCtx>>, Error> {
match self.load_existing(digest)? {
Loaded::Ready(pre) => Ok(pre),
Loaded::NeedsCompile => self.recompile(digest),
}
}
pub(crate) fn load_existing(&self, digest: &str) -> Result<Loaded, Error> {
if let Some(hit) = self
.inner
.lock()
.unwrap_or_else(|p| p.into_inner())
.lru
.get(digest)
.map(|e| e.pre.clone())
{
return Ok(Loaded::Ready(hit));
}
let cwasm_dir = cwasm_dir(&self.modules_dir);
let component = match cache::load_cwasm(&self.engine, &cwasm_dir, digest) {
Ok(c) => c,
Err(e) => {
tracing::warn!(
digest,
error = %e,
"cwasm cache miss/corrupt, recompiling from stored source"
);
return Ok(Loaded::NeedsCompile);
}
};
let weight =
std::fs::metadata(cwasm_dir.join(cache::cache_file_name(&self.engine, digest)))
.map(|m| m.len() as usize)
.unwrap_or(0);
let pre = Arc::new(crate::sandbox::instantiate_pre(&self.linker, &component)?);
self.insert(digest, pre.clone(), weight);
Ok(Loaded::Ready(pre))
}
pub(crate) fn recompile(&self, digest: &str) -> Result<Arc<HandlerPre<HostCtx>>, Error> {
let cwasm_dir = cwasm_dir(&self.modules_dir);
let wasm_bytes = std::fs::read(wasm_path(&self.modules_dir, digest)).map_err(|_| {
Error::Corrupt(format!(
"no cwasm and no source wasm on disk for digest {digest}"
))
})?;
let component = Component::new(&self.engine, &wasm_bytes)?;
let weight = match cache::persist_cwasm(&component, &self.engine, &cwasm_dir, digest) {
Ok(len) => len,
Err(e) => {
tracing::warn!(
digest,
error = %e,
"could not re-publish recompiled .cwasm; serving from memory"
);
wasm_bytes.len()
}
};
let pre = Arc::new(crate::sandbox::instantiate_pre(&self.linker, &component)?);
self.insert(digest, pre.clone(), weight);
Ok(pre)
}
fn insert(&self, digest: &str, pre: Arc<HandlerPre<HostCtx>>, weight: usize) {
let mut inner = self.inner.lock().unwrap_or_else(|p| p.into_inner());
if let Some(evicted) = inner
.lru
.put(digest.to_string(), CacheEntry { pre, weight })
{
inner.total_bytes = inner.total_bytes.saturating_sub(evicted.weight);
}
inner.total_bytes += weight;
while inner.total_bytes > self.byte_budget {
match inner.lru.pop_lru() {
Some((_, evicted)) => {
inner.total_bytes = inner.total_bytes.saturating_sub(evicted.weight)
}
None => break,
}
}
}
}
pub const GC_GRACE_PERIOD: Duration = Duration::from_secs(60 * 60);
fn older_than(path: &Path, grace_period: Duration) -> bool {
std::fs::metadata(path)
.and_then(|m| m.modified())
.map(|mtime| mtime.elapsed().unwrap_or(Duration::ZERO) >= grace_period)
.unwrap_or(false)
}
fn sweep_stale_temp_files(dir: &Path, grace_period: Duration) -> usize {
let Ok(entries) = std::fs::read_dir(dir) else {
return 0;
};
entries
.flatten()
.filter(|entry| {
entry.file_name().to_string_lossy().starts_with(".tmp")
&& older_than(&entry.path(), grace_period)
})
.filter(|entry| std::fs::remove_file(entry.path()).is_ok())
.count()
}
pub(crate) fn gc_unreferenced_blobs(
modules_dir: &Path,
engine: &Engine,
grace_period: Duration,
) -> Result<usize, Error> {
let mut referenced: HashSet<String> = HashSet::new();
let mut tenant_dirs: Vec<PathBuf> = Vec::new();
let tenants_dir = modules_dir.join("tenants");
match std::fs::read_dir(&tenants_dir) {
Ok(tenants) => {
for tenant_entry in tenants {
let tenant_entry = tenant_entry?;
if !tenant_entry.file_type()?.is_dir() {
continue;
}
for func_entry in std::fs::read_dir(tenant_entry.path())? {
let func_entry = func_entry?;
if func_entry.file_name().to_string_lossy().starts_with(".tmp") {
continue;
}
let contents = match std::fs::read_to_string(func_entry.path()) {
Ok(c) => c,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
Err(e) => return Err(e.into()),
};
let digest = contents.trim();
if cache::is_valid_digest(digest) {
referenced.insert(digest.to_string());
}
}
tenant_dirs.push(tenant_entry.path());
}
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e.into()),
}
let mut removed = 0usize;
for dir in &tenant_dirs {
removed += sweep_stale_temp_files(dir, grace_period);
}
let wdir = wasm_dir(modules_dir);
if wdir.exists() {
removed += sweep_stale_temp_files(&wdir, grace_period);
for entry in std::fs::read_dir(&wdir)? {
let entry = entry?;
let name = entry.file_name().to_string_lossy().into_owned();
if let Some(digest) = name.strip_suffix(".wasm") {
if !referenced.contains(digest)
&& older_than(&entry.path(), grace_period)
&& std::fs::remove_file(entry.path()).is_ok()
{
removed += 1;
}
}
}
}
let current_compat = cache::compat_hash(engine);
let cdir = cwasm_dir(modules_dir);
if cdir.exists() {
removed += sweep_stale_temp_files(&cdir, grace_period);
for entry in std::fs::read_dir(&cdir)? {
let entry = entry?;
let name = entry.file_name().to_string_lossy().into_owned();
if let Some(rest) = name.strip_suffix(".cwasm") {
if let Some((digest, compat)) = rest.split_once('-') {
let collectible = !referenced.contains(digest) || compat != current_compat;
if collectible
&& older_than(&entry.path(), grace_period)
&& std::fs::remove_file(entry.path()).is_ok()
{
removed += 1;
}
}
}
}
}
Ok(removed)
}
impl std::fmt::Debug for ComponentCache {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ComponentCache")
.field("byte_budget", &self.byte_budget)
.finish_non_exhaustive()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::sandbox::{build_engine, build_linker};
const TEST_GUEST_WASM: &[u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/fixtures/test_guest.wasm"
));
fn test_cache(engine: &Engine, dir: &Path) -> ComponentCache {
let linker = Arc::new(build_linker(engine).unwrap());
ComponentCache::new(engine.clone(), linker, dir.to_path_buf(), 256, usize::MAX)
}
#[test]
fn write_then_read_pointer_roundtrips() {
let dir = tempfile::tempdir().unwrap();
let digest = "a".repeat(64);
write_pointer(dir.path(), "tenant-a", "fn-a", &digest).unwrap();
assert_eq!(
read_pointer(dir.path(), "tenant-a", "fn-a").unwrap(),
Some(digest)
);
}
#[test]
fn read_pointer_missing_is_none() {
let dir = tempfile::tempdir().unwrap();
assert_eq!(read_pointer(dir.path(), "tenant-a", "fn-a").unwrap(), None);
}
#[test]
fn read_pointer_invalid_names_are_none() {
let dir = tempfile::tempdir().unwrap();
assert_eq!(read_pointer(dir.path(), "../etc", "fn-a").unwrap(), None);
}
#[test]
fn write_pointer_rejects_bad_digest() {
let dir = tempfile::tempdir().unwrap();
assert!(write_pointer(dir.path(), "tenant-a", "fn-a", "not-a-digest").is_err());
}
#[test]
fn re_upload_overwrites_pointer() {
let dir = tempfile::tempdir().unwrap();
let d1 = "a".repeat(64);
let d2 = "b".repeat(64);
write_pointer(dir.path(), "tenant-a", "fn-a", &d1).unwrap();
write_pointer(dir.path(), "tenant-a", "fn-a", &d2).unwrap();
assert_eq!(
read_pointer(dir.path(), "tenant-a", "fn-a").unwrap(),
Some(d2)
);
}
#[test]
fn get_or_load_recompiles_from_source_after_cwasm_deleted() {
let engine = build_engine().unwrap();
let dir = tempfile::tempdir().unwrap();
let cache_obj = test_cache(&engine, dir.path());
let digest = cache::digest(TEST_GUEST_WASM);
write_wasm_source(dir.path(), &digest, TEST_GUEST_WASM).unwrap();
let component = wasmtime::component::Component::new(&engine, TEST_GUEST_WASM).unwrap();
cache::persist_cwasm(&component, &engine, &cwasm_dir(dir.path()), &digest).unwrap();
std::fs::remove_dir_all(cwasm_dir(dir.path())).unwrap();
assert!(!cwasm_dir(dir.path()).exists());
let _loaded = cache_obj
.get_or_load(&digest)
.expect("should recompile from the persisted source wasm");
assert!(cwasm_dir(dir.path())
.join(cache::cache_file_name(&engine, &digest))
.exists());
}
#[test]
fn get_or_load_fails_cleanly_with_no_cwasm_and_no_source() {
let engine = build_engine().unwrap();
let dir = tempfile::tempdir().unwrap();
let cache_obj = test_cache(&engine, dir.path());
assert!(cache_obj.get_or_load(&"c".repeat(64)).is_err());
}
#[test]
fn gc_removes_unreferenced_blobs_but_keeps_referenced() {
let engine = build_engine().unwrap();
let dir = tempfile::tempdir().unwrap();
let kept = "a".repeat(64);
let orphan = "b".repeat(64);
write_pointer(dir.path(), "tenant-a", "fn-a", &kept).unwrap();
std::fs::create_dir_all(wasm_dir(dir.path())).unwrap();
std::fs::write(wasm_dir(dir.path()).join(format!("{kept}.wasm")), b"x").unwrap();
std::fs::write(wasm_dir(dir.path()).join(format!("{orphan}.wasm")), b"y").unwrap();
let removed =
gc_unreferenced_blobs(dir.path(), &engine, std::time::Duration::ZERO).unwrap();
assert_eq!(removed, 1);
assert!(wasm_dir(dir.path()).join(format!("{kept}.wasm")).exists());
assert!(!wasm_dir(dir.path()).join(format!("{orphan}.wasm")).exists());
}
#[test]
fn gc_leaves_young_unreferenced_blobs_alone() {
let engine = build_engine().unwrap();
let dir = tempfile::tempdir().unwrap();
let orphan = "b".repeat(64);
std::fs::create_dir_all(wasm_dir(dir.path())).unwrap();
std::fs::write(wasm_dir(dir.path()).join(format!("{orphan}.wasm")), b"y").unwrap();
let removed =
gc_unreferenced_blobs(dir.path(), &engine, Duration::from_secs(3600)).unwrap();
assert_eq!(removed, 0);
assert!(wasm_dir(dir.path()).join(format!("{orphan}.wasm")).exists());
}
#[test]
fn gc_removes_cwasm_with_stale_compat_hash_even_if_referenced() {
let engine = build_engine().unwrap();
let dir = tempfile::tempdir().unwrap();
let digest = "c".repeat(64);
write_pointer(dir.path(), "tenant-a", "fn-a", &digest).unwrap();
std::fs::create_dir_all(cwasm_dir(dir.path())).unwrap();
let stale_path = cwasm_dir(dir.path()).join(format!("{digest}-deadbeef.cwasm"));
std::fs::write(&stale_path, b"stale").unwrap();
let current_path = cwasm_dir(dir.path()).join(cache::cache_file_name(&engine, &digest));
std::fs::write(¤t_path, b"current").unwrap();
let removed =
gc_unreferenced_blobs(dir.path(), &engine, std::time::Duration::ZERO).unwrap();
assert_eq!(removed, 1);
assert!(!stale_path.exists());
assert!(current_path.exists());
}
#[test]
fn gc_sweeps_stale_temp_files() {
let engine = build_engine().unwrap();
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir_all(wasm_dir(dir.path())).unwrap();
let tmp = wasm_dir(dir.path()).join(".tmpABCDEF");
std::fs::write(&tmp, b"partial").unwrap();
let removed =
gc_unreferenced_blobs(dir.path(), &engine, std::time::Duration::ZERO).unwrap();
assert_eq!(removed, 1);
assert!(!tmp.exists());
}
#[test]
fn component_cache_evicts_by_byte_budget() {
let engine = build_engine().unwrap();
let linker = Arc::new(build_linker(&engine).unwrap());
let cache = ComponentCache::new(engine.clone(), linker.clone(), PathBuf::new(), 256, 1);
let component = wasmtime::component::Component::new(&engine, TEST_GUEST_WASM).unwrap();
let pre = Arc::new(crate::sandbox::instantiate_pre(&linker, &component).unwrap());
cache.insert("d1", pre, 100);
assert_eq!(cache.inner.lock().unwrap().total_bytes, 0);
assert!(cache.inner.lock().unwrap().lru.is_empty());
}
#[test]
fn gc_aborts_on_unreadable_pointer_and_deletes_nothing() {
let engine = build_engine().unwrap();
let dir = tempfile::tempdir().unwrap();
let orphan = "b".repeat(64);
std::fs::create_dir_all(wasm_dir(dir.path())).unwrap();
let blob = wasm_dir(dir.path()).join(format!("{orphan}.wasm"));
std::fs::write(&blob, b"y").unwrap();
std::fs::create_dir_all(dir.path().join("tenants/tenant-a/fn-a")).unwrap();
let err = gc_unreferenced_blobs(dir.path(), &engine, Duration::ZERO);
assert!(err.is_err(), "expected GC to abort, got {err:?}");
assert!(
blob.exists(),
"nothing may be deleted when a pointer is unreadable"
);
}
#[test]
fn re_upload_of_existing_source_bumps_mtime() {
let dir = tempfile::tempdir().unwrap();
let digest = cache::digest(b"x");
write_wasm_source(dir.path(), &digest, b"x").unwrap();
let path = wasm_path(dir.path(), &digest);
let old = SystemTime::now() - Duration::from_secs(7200);
std::fs::OpenOptions::new()
.write(true)
.open(&path)
.unwrap()
.set_modified(old)
.unwrap();
write_wasm_source(dir.path(), &digest, b"x").unwrap();
assert!(!older_than(&path, Duration::from_secs(3600)));
}
}