use crate::host::SerObj;
use fusevm::Chunk;
use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
use std::collections::HashMap;
use std::fs::File;
use std::io::Write as IoWrite;
use std::os::unix::io::AsRawFd;
use std::path::{Path, PathBuf};
use std::sync::OnceLock;
use std::time::{SystemTime, UNIX_EPOCH};
pub const SHARD_MAGIC: u32 = 0x454C_5350;
pub const SHARD_FORMAT_VERSION: u32 = 17;
pub fn schema_key(builtin_fingerprint: u64) -> String {
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
builtin_fingerprint.hash(&mut hasher);
crate::prelude::PRELUDE.hash(&mut hasher);
crate::prelude::NADVICE.hash(&mut hasher);
for segment in crate::prelude_31::SEGMENTS {
segment.hash(&mut hasher);
}
format!("{}-{:016x}", env!("CARGO_PKG_VERSION"), hasher.finish())
}
#[derive(Archive, RkyvSerialize, RkyvDeserialize)]
#[archive(check_bytes)]
struct ShardHeader {
magic: u32,
format_version: u32,
pointer_width: u32,
built_at_secs: u64,
schema_key: String,
}
#[derive(Archive, RkyvSerialize, RkyvDeserialize)]
#[archive(check_bytes)]
struct Entry {
mtime_ns: i64,
binary_mtime_at_cache: i64,
cached_at_secs: i64,
forms: Vec<Vec<u8>>,
heap_tail: Vec<u8>,
base_fingerprint: u64,
oclosure_meta: Vec<u8>,
introspection_cells: Vec<u8>,
}
#[derive(Archive, RkyvSerialize, RkyvDeserialize)]
#[archive(check_bytes)]
struct BaseImage {
heap: Vec<u8>,
builtin_cells: Vec<u8>,
buffer_state: Vec<u8>,
fingerprint: u64,
}
impl BaseImage {
fn fingerprint_of(heap: &[u8], builtin_cells: &[u8], buffer_state: &[u8]) -> u64 {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
heap.hash(&mut h);
builtin_cells.hash(&mut h);
buffer_state.hash(&mut h);
h.finish()
}
fn new(heap: Vec<u8>, builtin_cells: Vec<u8>, buffer_state: Vec<u8>) -> Self {
let fingerprint = Self::fingerprint_of(&heap, &builtin_cells, &buffer_state);
Self {
heap,
builtin_cells,
buffer_state,
fingerprint,
}
}
fn bytes(&self) -> u64 {
(self.heap.len() + self.builtin_cells.len() + self.buffer_state.len()) as u64
}
}
#[derive(Archive, RkyvSerialize, RkyvDeserialize)]
#[archive(check_bytes)]
struct Shard {
header: ShardHeader,
base: BaseImage,
entries: HashMap<String, Entry>,
}
fn elisprs_dir() -> Option<PathBuf> {
let home = std::env::var_os("HOME")?;
Some(PathBuf::from(home).join(".elisprs"))
}
fn shard_path() -> Option<PathBuf> {
Some(elisprs_dir()?.join("scripts.rkyv"))
}
fn lock_path() -> Option<PathBuf> {
Some(elisprs_dir()?.join("scripts.rkyv.lock"))
}
pub fn default_cache_path() -> PathBuf {
shard_path().unwrap_or_else(|| PathBuf::from("/tmp/.elisprs/scripts.rkyv"))
}
pub fn cache_enabled() -> bool {
!matches!(
std::env::var("ELISPRS_CACHE").as_deref(),
Ok("0") | Ok("false") | Ok("no")
)
}
pub const DEFAULT_MAX_SHARD_BYTES: u64 = 64 * 1024 * 1024;
pub fn max_shard_bytes() -> u64 {
match std::env::var("ELISPRS_CACHE_MAX_BYTES") {
Ok(v) => v.trim().parse().unwrap_or(DEFAULT_MAX_SHARD_BYTES),
Err(_) => DEFAULT_MAX_SHARD_BYTES,
}
}
fn entry_bytes(e: &Entry) -> u64 {
(e.forms.iter().map(Vec::len).sum::<usize>()
+ e.heap_tail.len()
+ e.oclosure_meta.len()
+ e.introspection_cells.len()) as u64
}
fn enforce_budget(shard: &mut Shard, keep: &str, budget: u64) {
shard.entries.retain(|p, e| {
p == keep
|| match file_mtime_ns(Path::new(p)) {
Some(ns) => ns == e.mtime_ns,
None => false,
}
});
if budget == 0 {
return;
}
let mut total: u64 = shard.base.bytes() + shard.entries.values().map(entry_bytes).sum::<u64>();
if total <= budget {
return;
}
let mut by_age: Vec<(i64, String)> = shard
.entries
.iter()
.filter(|(p, _)| p.as_str() != keep)
.map(|(p, e)| (e.cached_at_secs, p.clone()))
.collect();
by_age.sort_unstable();
for (_, path) in by_age {
if total <= budget {
break;
}
if let Some(e) = shard.entries.remove(&path) {
total -= entry_bytes(&e);
}
}
}
struct FlockGuard {
_file: File,
}
fn acquire_lock() -> Option<FlockGuard> {
let path = lock_path()?;
if let Some(dir) = path.parent() {
let _ = std::fs::create_dir_all(dir);
}
let file = File::options()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&path)
.ok()?;
let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) };
if rc != 0 {
return None;
}
Some(FlockGuard { _file: file })
}
fn header_ok(h: &ArchivedShardHeader, schema_key: &str) -> bool {
let magic: u32 = h.magic.into();
let fv: u32 = h.format_version.into();
let pw: u32 = h.pointer_width.into();
magic == SHARD_MAGIC
&& fv == SHARD_FORMAT_VERSION
&& pw as usize == std::mem::size_of::<usize>()
&& h.schema_key.as_str() == schema_key
}
fn now_secs() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
fn current_binary_mtime_secs() -> Option<i64> {
static BIN_MTIME: OnceLock<Option<i64>> = OnceLock::new();
*BIN_MTIME.get_or_init(|| {
use std::os::unix::fs::MetadataExt;
let exe = std::env::current_exe().ok()?;
Some(std::fs::metadata(exe).ok()?.mtime())
})
}
fn file_mtime_ns(path: &Path) -> Option<i64> {
let m = std::fs::metadata(path).ok()?;
let t = m.modified().ok()?;
Some(t.duration_since(UNIX_EPOCH).ok()?.as_nanos() as i64)
}
fn read_shard() -> Option<Shard> {
let bytes = std::fs::read(shard_path()?).ok()?;
let archived = rkyv::check_archived_root::<Shard>(&bytes).ok()?;
archived.deserialize(&mut rkyv::Infallible).ok()
}
fn fresh_shard(schema_key: &str, base: BaseImage) -> Shard {
Shard {
base,
header: ShardHeader {
magic: SHARD_MAGIC,
format_version: SHARD_FORMAT_VERSION,
pointer_width: std::mem::size_of::<usize>() as u32,
built_at_secs: now_secs() as u64,
schema_key: schema_key.to_string(),
},
entries: HashMap::new(),
}
}
fn owned_header_ok(h: &ShardHeader, schema_key: &str) -> bool {
h.magic == SHARD_MAGIC
&& h.format_version == SHARD_FORMAT_VERSION
&& h.pointer_width as usize == std::mem::size_of::<usize>()
&& h.schema_key == schema_key
}
fn write_shard(shard: &Shard) -> std::io::Result<()> {
let dir = elisprs_dir().ok_or_else(|| std::io::Error::other("no HOME"))?;
std::fs::create_dir_all(&dir)?;
let bytes = rkyv::to_bytes::<_, 4096>(shard)
.map_err(|e| std::io::Error::other(format!("rkyv: {e:?}")))?;
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let tmp = dir.join(format!("scripts.rkyv.tmp.{}.{}", std::process::id(), nanos));
{
let mut f = File::create(&tmp)?;
f.write_all(&bytes)?;
f.sync_all()?;
}
std::fs::rename(&tmp, dir.join("scripts.rkyv"))
}
pub struct CachedScript {
pub chunks: Vec<Chunk>,
pub heap: Vec<SerObj>,
pub oclosure_meta: Vec<(u32, u32, Vec<u32>)>,
pub introspection_cells: Vec<(u32, fusevm::Value)>,
pub builtin_cells: Vec<Option<crate::host::SymbolBaseline>>,
pub buffer_state: crate::host::BufferState,
pub base_len: usize,
}
#[allow(clippy::type_complexity)]
pub fn get(path: &str, mtime_ns: i64, schema_key: &str) -> Option<CachedScript> {
if !cache_enabled() {
return None;
}
let bytes = std::fs::read(shard_path()?).ok()?;
let shard = rkyv::check_archived_root::<Shard>(&bytes).ok()?;
if !header_ok(&shard.header, schema_key) {
return None;
}
let entry = shard.entries.get(path)?;
let entry_mtime: i64 = entry.mtime_ns.into();
if entry_mtime != mtime_ns {
return None;
}
if let Some(bin_mtime) = current_binary_mtime_secs() {
let cached: i64 = entry.binary_mtime_at_cache.into();
if cached < bin_mtime {
return None;
}
}
let chunks: Vec<Chunk> = entry
.forms
.iter()
.map(|b| bincode::deserialize(b))
.collect::<Result<_, _>>()
.ok()?;
let base_fingerprint: u64 = entry.base_fingerprint.into();
if base_fingerprint
!= BaseImage::fingerprint_of(
&shard.base.heap,
&shard.base.builtin_cells,
&shard.base.buffer_state,
)
{
return None;
}
let mut heap: Vec<SerObj> = bincode::deserialize(&shard.base.heap).ok()?;
let base_len = heap.len();
let tail: Vec<SerObj> = bincode::deserialize(&entry.heap_tail).ok()?;
heap.extend(tail);
let oclosure_meta: Vec<(u32, u32, Vec<u32>)> =
bincode::deserialize(&entry.oclosure_meta).ok()?;
let introspection_cells: Vec<(u32, fusevm::Value)> =
bincode::deserialize(&entry.introspection_cells).ok()?;
let builtin_cells: Vec<Option<crate::host::SymbolBaseline>> =
bincode::deserialize(&shard.base.builtin_cells).ok()?;
let buffer_state: crate::host::BufferState =
bincode::deserialize(&shard.base.buffer_state).ok()?;
Some(CachedScript {
chunks,
heap,
oclosure_meta,
introspection_cells,
builtin_cells,
buffer_state,
base_len,
})
}
pub struct ScriptParts<'a> {
pub chunks: &'a [Chunk],
pub prelude_heap: &'a [SerObj],
pub heap_tail: &'a [SerObj],
pub oclosure_meta: &'a [(u32, u32, Vec<u32>)],
pub introspection_cells: &'a [(u32, fusevm::Value)],
pub builtin_cells: &'a [Option<crate::host::SymbolBaseline>],
pub buffer_state: &'a crate::host::BufferState,
}
pub fn put(path: &str, mtime_ns: i64, schema_key: &str, parts: ScriptParts<'_>) {
let ScriptParts {
chunks,
prelude_heap,
heap_tail,
oclosure_meta,
introspection_cells,
builtin_cells,
buffer_state,
} = parts;
if !cache_enabled() {
return;
}
let Ok(forms) = chunks
.iter()
.map(bincode::serialize)
.collect::<Result<Vec<_>, _>>()
else {
return;
};
let Ok(tail_blob) = bincode::serialize(heap_tail) else {
return;
};
let Ok(oclosure_blob) = bincode::serialize(oclosure_meta) else {
return;
};
let Ok(introspection_blob) = bincode::serialize(introspection_cells) else {
return;
};
let Ok(builtin_cells_blob) = bincode::serialize(builtin_cells) else {
return;
};
let Ok(buffer_state_blob) = bincode::serialize(buffer_state) else {
return;
};
let Ok(prelude_blob) = bincode::serialize(prelude_heap) else {
return;
};
let _lock = acquire_lock();
let base = BaseImage::new(prelude_blob, builtin_cells_blob, buffer_state_blob);
let mut shard = read_shard()
.filter(|s| owned_header_ok(&s.header, schema_key))
.unwrap_or_else(|| {
fresh_shard(
schema_key,
BaseImage::new(Vec::new(), Vec::new(), Vec::new()),
)
});
if shard.base.fingerprint != base.fingerprint {
shard.base = base;
shard.entries.clear();
}
let bin_mtime = current_binary_mtime_secs().unwrap_or(0);
shard.entries.insert(
path.to_string(),
Entry {
mtime_ns,
binary_mtime_at_cache: bin_mtime,
cached_at_secs: now_secs(),
forms,
heap_tail: tail_blob,
base_fingerprint: shard.base.fingerprint,
oclosure_meta: oclosure_blob,
introspection_cells: introspection_blob,
},
);
if std::env::var_os("ELISPRS_CACHE_DEBUG").is_some() {
let e = &shard.entries[path];
eprintln!(
"elisprs: entry bytes forms={} heap_tail={} oclosure={} intro={}; shard base={} entries={}",
e.forms.iter().map(Vec::len).sum::<usize>(),
e.heap_tail.len(),
e.oclosure_meta.len(),
e.introspection_cells.len(),
shard.base.bytes(),
shard.entries.len()
);
}
enforce_budget(&mut shard, path, max_shard_bytes());
shard.header.built_at_secs = now_secs() as u64;
let _ = write_shard(&shard);
}
pub fn stats() -> (i64, i64) {
let Some(shard) = read_shard() else {
return (0, 0);
};
let count = shard.entries.len() as i64;
let bytes: i64 = shard.base.bytes() as i64
+ shard
.entries
.values()
.map(|e| entry_bytes(e) as i64)
.sum::<i64>();
(count, bytes)
}
pub fn clear() -> std::io::Result<()> {
let _lock = acquire_lock();
let Some(p) = shard_path() else {
return Ok(());
};
match std::fs::remove_file(&p) {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e),
}
}
pub fn evict_stale() -> usize {
let _lock = acquire_lock();
let Some(mut shard) = read_shard() else {
return 0;
};
let before = shard.entries.len();
shard
.entries
.retain(|p, e| match file_mtime_ns(Path::new(p)) {
Some(ns) => ns == e.mtime_ns,
None => false,
});
let evicted = before - shard.entries.len();
if evicted > 0 {
let _ = write_shard(&shard);
}
evicted
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn shard_roundtrip_via_rkyv() {
let mut shard = fresh_shard("v-test", BaseImage::new(vec![11, 12], vec![13], vec![14]));
shard.entries.insert(
"/tmp/x.el".to_string(),
Entry {
mtime_ns: 1,
binary_mtime_at_cache: 3,
cached_at_secs: 4,
forms: vec![vec![9, 9, 9]],
heap_tail: vec![1, 2],
base_fingerprint: 0,
oclosure_meta: vec![3, 4],
introspection_cells: vec![5, 6],
},
);
let bytes = rkyv::to_bytes::<_, 4096>(&shard).unwrap();
let archived = rkyv::check_archived_root::<Shard>(&bytes[..]).unwrap();
assert!(header_ok(&archived.header, "v-test"));
assert!(!header_ok(&archived.header, "v-other"));
let back: Shard = archived.deserialize(&mut rkyv::Infallible).unwrap();
assert_eq!(back.entries["/tmp/x.el"].forms, vec![vec![9, 9, 9]]);
assert_eq!(back.entries["/tmp/x.el"].introspection_cells, vec![5, 6]);
assert_eq!(back.base.heap, vec![11, 12]);
assert_eq!(back.base.builtin_cells, vec![13]);
assert_eq!(back.entries["/tmp/x.el"].heap_tail, vec![1, 2]);
assert_eq!(back.header.magic, SHARD_MAGIC);
}
#[test]
fn cache_enabled_env() {
std::env::set_var("ELISPRS_CACHE", "0");
assert!(!cache_enabled());
std::env::set_var("ELISPRS_CACHE", "1");
assert!(cache_enabled());
std::env::remove_var("ELISPRS_CACHE");
assert!(cache_enabled());
}
fn sized_entry(dir: &Path, name: &str, bytes: usize, age: i64) -> (String, Entry) {
let p = dir.join(name);
std::fs::write(&p, b"x").unwrap();
let mtime_ns = file_mtime_ns(&p).unwrap();
(
p.to_string_lossy().into_owned(),
Entry {
mtime_ns,
binary_mtime_at_cache: 0,
cached_at_secs: age,
forms: vec![vec![0u8; bytes]],
heap_tail: vec![],
base_fingerprint: 0,
oclosure_meta: vec![],
introspection_cells: vec![],
},
)
}
#[test]
fn budget_evicts_oldest_and_never_the_entry_just_written() {
let dir = std::env::temp_dir().join(format!("elisprs-budget-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let mut shard = fresh_shard("v-test", BaseImage::new(vec![11, 12], vec![13], vec![14]));
let mut paths = Vec::new();
for (i, age) in [10, 20, 30, 40].iter().enumerate() {
let (p, e) = sized_entry(&dir, &format!("e{i}.el"), 1000, *age);
shard.entries.insert(p.clone(), e);
paths.push(p);
}
assert_eq!(shard.entries.len(), 4);
enforce_budget(&mut shard, &paths[0], 2500);
assert!(
shard.entries.contains_key(&paths[0]),
"the entry just written was evicted; a shard at its budget would then never serve a hit"
);
let total: u64 = shard.entries.values().map(entry_bytes).sum();
assert!(total <= 2500, "budget not enforced: {total} bytes remain");
assert!(shard.entries.contains_key(&paths[3]));
assert!(!shard.entries.contains_key(&paths[1]));
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn a_single_oversized_entry_survives_its_own_budget() {
let dir = std::env::temp_dir().join(format!("elisprs-budget1-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let mut shard = fresh_shard("v-test", BaseImage::new(vec![11, 12], vec![13], vec![14]));
let (p, e) = sized_entry(&dir, "big.el", 10_000, 1);
shard.entries.insert(p.clone(), e);
enforce_budget(&mut shard, &p, 100);
assert_eq!(shard.entries.len(), 1);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn budget_sweep_drops_unservable_entries() {
let dir = std::env::temp_dir().join(format!("elisprs-budget2-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let mut shard = fresh_shard("v-test", BaseImage::new(vec![11, 12], vec![13], vec![14]));
let (live, e1) = sized_entry(&dir, "live.el", 10, 5);
let (gone, e2) = sized_entry(&dir, "gone.el", 10, 5);
let (edited, mut e3) = sized_entry(&dir, "edited.el", 10, 5);
e3.mtime_ns += 1; shard.entries.insert(live.clone(), e1);
shard.entries.insert(gone.clone(), e2);
shard.entries.insert(edited.clone(), e3);
std::fs::remove_file(&gone).unwrap();
enforce_budget(&mut shard, &live, 0);
assert!(shard.entries.contains_key(&live));
assert!(
!shard.entries.contains_key(&gone),
"deleted source retained"
);
assert!(
!shard.entries.contains_key(&edited),
"entry whose source changed since caching was retained"
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn max_shard_bytes_env_override() {
std::env::set_var("ELISPRS_CACHE_MAX_BYTES", "4096");
assert_eq!(max_shard_bytes(), 4096);
std::env::set_var("ELISPRS_CACHE_MAX_BYTES", "0");
assert_eq!(max_shard_bytes(), 0);
std::env::set_var("ELISPRS_CACHE_MAX_BYTES", "not-a-number");
assert_eq!(max_shard_bytes(), DEFAULT_MAX_SHARD_BYTES);
std::env::remove_var("ELISPRS_CACHE_MAX_BYTES");
assert_eq!(max_shard_bytes(), DEFAULT_MAX_SHARD_BYTES);
}
}