use std::collections::HashMap;
use std::fs::File;
use std::io::Write as IoWrite;
use std::path::{Path, PathBuf};
use std::sync::{Mutex, OnceLock};
use std::time::{SystemTime, UNIX_EPOCH};
use memmap2::Mmap;
use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
pub const SHARD_MAGIC: u32 = 0x54_45_58_52;
pub const SHARD_FORMAT_VERSION: u32 = 1;
#[derive(Archive, RkyvDeserialize, RkyvSerialize, Debug, Clone)]
#[archive(check_bytes)]
pub struct ShardHeader {
pub magic: u32,
pub format_version: u32,
pub texrs_version: String,
pub pointer_width: u32,
pub built_at_secs: u64,
}
#[derive(Archive, RkyvDeserialize, RkyvSerialize, Debug, Clone)]
#[archive(check_bytes)]
pub struct ScriptEntry {
pub mtime_secs: i64,
pub mtime_nsecs: i64,
pub binary_mtime_at_cache: i64,
pub cached_at_secs: i64,
pub chunk_blob: Vec<u8>,
}
#[derive(Archive, RkyvDeserialize, RkyvSerialize, Debug, Clone)]
#[archive(check_bytes)]
pub struct ScriptShard {
pub header: ShardHeader,
pub entries: HashMap<String, ScriptEntry>,
}
pub struct MmappedShard {
_mmap: Mmap,
archived: *const ArchivedScriptShard,
}
unsafe impl Send for MmappedShard {}
unsafe impl Sync for MmappedShard {}
impl MmappedShard {
pub fn open(path: &Path) -> Option<Self> {
let file = File::open(path).ok()?;
let mmap = unsafe { Mmap::map(&file).ok()? };
let archived = rkyv::check_archived_root::<ScriptShard>(&mmap[..]).ok()?;
let archived = archived as *const ArchivedScriptShard;
Some(Self {
_mmap: mmap,
archived,
})
}
fn shard(&self) -> &ArchivedScriptShard {
unsafe { &*self.archived }
}
fn header_ok(&self) -> bool {
let h = &self.shard().header;
let magic: u32 = h.magic.into();
let format_version: u32 = h.format_version.into();
let pointer_width: u32 = h.pointer_width.into();
magic == SHARD_MAGIC
&& format_version == SHARD_FORMAT_VERSION
&& pointer_width as usize == std::mem::size_of::<usize>()
&& h.texrs_version.as_str() == env!("CARGO_PKG_VERSION")
}
fn lookup(&self, path: &str) -> Option<&ArchivedScriptEntry> {
self.shard().entries.get(path)
}
}
pub fn document_key(input_digests: &[String], profile: &str) -> String {
let mut hasher = <sha2::Sha256 as sha2::Digest>::new();
sha2::Digest::update(&mut hasher, profile.as_bytes());
for digest in input_digests {
sha2::Digest::update(&mut hasher, b"\0");
sha2::Digest::update(&mut hasher, digest.as_bytes());
}
let hex: String = sha2::Digest::finalize(hasher)
.iter()
.map(|b| format!("{b:02x}"))
.collect();
format!("doc:{hex}")
}
pub struct ScriptCache {
path: PathBuf,
lock_path: PathBuf,
mmap: Mutex<Option<MmappedShard>>,
}
impl ScriptCache {
pub fn open(path: &Path) -> std::io::Result<Self> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let parent = path.parent().unwrap_or_else(|| Path::new("/tmp"));
let lock_path = parent.join(format!(
"{}.lock",
path.file_name()
.and_then(|s| s.to_str())
.unwrap_or("scripts.rkyv")
));
Ok(Self {
path: path.to_path_buf(),
lock_path,
mmap: Mutex::new(None),
})
}
fn ensure_mmap(&self) {
let mut guard = self.mmap.lock().unwrap_or_else(|e| e.into_inner());
if guard.is_none() {
*guard = MmappedShard::open(&self.path);
}
}
fn invalidate_mmap(&self) {
*self.mmap.lock().unwrap_or_else(|e| e.into_inner()) = None;
}
pub fn get(&self, path: &str, mtime_secs: i64, mtime_nsecs: i64) -> Option<Vec<u8>> {
self.ensure_mmap();
let guard = self.mmap.lock().unwrap_or_else(|e| e.into_inner());
let shard = guard.as_ref()?;
if !shard.header_ok() {
return None;
}
let entry = shard.lookup(path)?;
let entry_secs: i64 = entry.mtime_secs.into();
let entry_nsecs: i64 = entry.mtime_nsecs.into();
if entry_secs != mtime_secs || entry_nsecs != mtime_nsecs {
return None;
}
if let Some(binary_mtime) = current_binary_mtime_secs() {
let cached: i64 = entry.binary_mtime_at_cache.into();
if cached < binary_mtime {
return None;
}
}
Some(entry.chunk_blob.as_slice().to_vec())
}
pub fn put(
&self,
path: &str,
mtime_secs: i64,
mtime_nsecs: i64,
chunk_blob: Vec<u8>,
) -> std::io::Result<()> {
let _lock = match acquire_lock(&self.lock_path) {
Some(lock) => lock,
None => return Ok(()),
};
let mut shard = match read_owned_shard(&self.path) {
Some(s)
if s.header.texrs_version == env!("CARGO_PKG_VERSION")
&& s.header.pointer_width as usize == std::mem::size_of::<usize>()
&& s.header.format_version == SHARD_FORMAT_VERSION =>
{
s
}
_ => fresh_shard(),
};
shard.entries.insert(
path.to_string(),
ScriptEntry {
mtime_secs,
mtime_nsecs,
binary_mtime_at_cache: current_binary_mtime_secs().unwrap_or(0),
cached_at_secs: now_secs(),
chunk_blob,
},
);
shard.header.built_at_secs = now_secs() as u64;
write_shard_atomic(&self.path, &shard)?;
self.invalidate_mmap();
Ok(())
}
pub fn evict_stale(&self) -> usize {
let _lock = match acquire_lock(&self.lock_path) {
Some(lock) => lock,
None => return 0,
};
let mut shard = match read_owned_shard(&self.path) {
Some(s) => s,
None => return 0,
};
let before = shard.entries.len();
shard.entries.retain(|p, e| match file_mtime(Path::new(p)) {
Some((secs, nsecs)) => secs == e.mtime_secs && nsecs == e.mtime_nsecs,
None => false,
});
let evicted = before - shard.entries.len();
if evicted > 0 {
let _ = write_shard_atomic(&self.path, &shard);
self.invalidate_mmap();
}
evicted
}
pub fn len(&self) -> usize {
self.ensure_mmap();
let guard = self.mmap.lock().unwrap_or_else(|e| e.into_inner());
guard.as_ref().map(|s| s.shard().entries.len()).unwrap_or(0)
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn clear(&self) -> std::io::Result<()> {
let _lock = acquire_lock(&self.lock_path);
let result = match std::fs::remove_file(&self.path) {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e),
};
self.invalidate_mmap();
result
}
}
fn acquire_lock(path: &Path) -> Option<nix::fcntl::Flock<File>> {
let file = File::options()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(path)
.ok()?;
nix::fcntl::Flock::lock(file, nix::fcntl::FlockArg::LockExclusive).ok()
}
fn fresh_shard() -> ScriptShard {
ScriptShard {
header: ShardHeader {
magic: SHARD_MAGIC,
format_version: SHARD_FORMAT_VERSION,
texrs_version: env!("CARGO_PKG_VERSION").to_string(),
pointer_width: std::mem::size_of::<usize>() as u32,
built_at_secs: now_secs() as u64,
},
entries: HashMap::new(),
}
}
fn read_owned_shard(path: &Path) -> Option<ScriptShard> {
let bytes = std::fs::read(path).ok()?;
let archived = rkyv::check_archived_root::<ScriptShard>(&bytes[..]).ok()?;
archived.deserialize(&mut rkyv::Infallible).ok()
}
fn write_shard_atomic(path: &Path, shard: &ScriptShard) -> std::io::Result<()> {
let bytes = rkyv::to_bytes::<_, 4096>(shard)
.map_err(|e| std::io::Error::other(format!("rkyv serialize: {e}")))?;
let parent = path.parent().unwrap_or_else(|| Path::new("/tmp"));
let _ = std::fs::create_dir_all(parent);
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let tmp_path = parent.join(format!(
"{}.tmp.{}.{nanos}",
path.file_name()
.and_then(|s| s.to_str())
.unwrap_or("scripts.rkyv"),
std::process::id()
));
{
let mut f = File::create(&tmp_path)?;
f.write_all(&bytes)?;
f.sync_all()?;
}
std::fs::rename(&tmp_path, path)?;
Ok(())
}
fn now_secs() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
pub fn file_mtime(path: &Path) -> Option<(i64, i64)> {
use std::os::unix::fs::MetadataExt;
let meta = std::fs::metadata(path).ok()?;
Some((meta.mtime(), meta.mtime_nsec()))
}
fn current_binary_mtime_secs() -> Option<i64> {
static BINARY_MTIME: OnceLock<Option<i64>> = OnceLock::new();
*BINARY_MTIME.get_or_init(|| {
let exe = std::env::current_exe().ok()?;
let (secs, _) = file_mtime(&exe)?;
Some(secs)
})
}
pub fn default_cache_path() -> PathBuf {
dirs::cache_dir()
.or_else(dirs::home_dir)
.unwrap_or_else(|| PathBuf::from("/tmp"))
.join("texrs/scripts.rkyv")
}
pub fn cache_enabled() -> bool {
!matches!(
std::env::var("TEXRS_CACHE").as_deref(),
Ok("0") | Ok("false") | Ok("no")
)
}
pub static CACHE: once_cell::sync::Lazy<Option<ScriptCache>> = once_cell::sync::Lazy::new(|| {
if !cache_enabled() {
return None;
}
ScriptCache::open(&default_cache_path()).ok()
});
pub fn try_load_keyed(key: &str) -> Option<fusevm::Chunk> {
let cache = CACHE.as_ref()?;
let blob = cache.get(key, DIGEST_KEYED, DIGEST_KEYED)?;
bincode::deserialize::<fusevm::Chunk>(&blob).ok()
}
pub fn store_keyed(key: &str, chunk: &fusevm::Chunk) {
let Some(cache) = CACHE.as_ref() else {
return;
};
let Ok(blob) = bincode::serialize(chunk) else {
return;
};
let _ = cache.put(key, DIGEST_KEYED, DIGEST_KEYED, blob);
}
const DIGEST_KEYED: i64 = 0;
pub fn try_load_mode(path: &Path, mode: &str) -> Option<fusevm::Chunk> {
let cache = CACHE.as_ref()?;
let canonical = path.canonicalize().ok()?;
let key = format!("{}#{mode}", canonical.to_str()?);
let (mtime_secs, mtime_nsecs) = file_mtime(&canonical)?;
let blob = cache.get(&key, mtime_secs, mtime_nsecs)?;
bincode::deserialize::<fusevm::Chunk>(&blob).ok()
}
pub fn store_mode(path: &Path, mode: &str, chunk: &fusevm::Chunk) {
let Some(cache) = CACHE.as_ref() else {
return;
};
let Ok(canonical) = path.canonicalize() else {
return;
};
let Some(base) = canonical.to_str() else {
return;
};
let Some((mtime_secs, mtime_nsecs)) = file_mtime(&canonical) else {
return;
};
let Ok(blob) = bincode::serialize(chunk) else {
return;
};
let _ = cache.put(&format!("{base}#{mode}"), mtime_secs, mtime_nsecs, blob);
}
pub fn try_load(path: &Path) -> Option<fusevm::Chunk> {
let cache = CACHE.as_ref()?;
let canonical = path.canonicalize().ok()?;
let key = canonical.to_str()?;
let (mtime_secs, mtime_nsecs) = file_mtime(&canonical)?;
let blob = cache.get(key, mtime_secs, mtime_nsecs)?;
bincode::deserialize::<fusevm::Chunk>(&blob).ok()
}
pub fn store(path: &Path, chunk: &fusevm::Chunk) {
let Some(cache) = CACHE.as_ref() else {
return;
};
let Ok(canonical) = path.canonicalize() else {
return;
};
let Some(key) = canonical.to_str() else {
return;
};
let Some((mtime_secs, mtime_nsecs)) = file_mtime(&canonical) else {
return;
};
let Ok(blob) = bincode::serialize(chunk) else {
return;
};
let _ = cache.put(key, mtime_secs, mtime_nsecs, blob);
}
#[cfg(test)]
mod tests {
use super::*;
fn scratch(name: &str) -> PathBuf {
static N: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let seq = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let dir =
std::env::temp_dir().join(format!("texrs_cache_{}_{seq}_{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir.join("scripts.rkyv")
}
fn source(dir: &Path, name: &str, body: &str) -> PathBuf {
let path = dir.join(name);
std::fs::write(&path, body).unwrap();
path
}
#[test]
fn a_stored_chunk_comes_back_byte_for_byte() {
let shard = scratch("roundtrip");
let cache = ScriptCache::open(&shard).unwrap();
assert!(cache.is_empty(), "a cache with no file holds nothing");
cache.put("/tmp/doc.tex", 7, 11, vec![1, 2, 3]).unwrap();
assert_eq!(cache.get("/tmp/doc.tex", 7, 11), Some(vec![1, 2, 3]));
assert_eq!(cache.len(), 1);
cache.put("/tmp/other.tex", 1, 2, vec![9]).unwrap();
assert_eq!(cache.len(), 2);
assert_eq!(cache.get("/tmp/doc.tex", 7, 11), Some(vec![1, 2, 3]));
cache.put("/tmp/doc.tex", 7, 11, vec![4, 5]).unwrap();
assert_eq!(cache.len(), 2);
assert_eq!(cache.get("/tmp/doc.tex", 7, 11), Some(vec![4, 5]));
}
#[test]
fn an_edited_source_is_a_miss_rather_than_a_stale_hit() {
let shard = scratch("mtime");
let cache = ScriptCache::open(&shard).unwrap();
cache.put("/tmp/doc.tex", 100, 0, vec![1]).unwrap();
assert_eq!(cache.get("/tmp/doc.tex", 100, 0), Some(vec![1]));
assert_eq!(cache.get("/tmp/doc.tex", 101, 0), None, "a later second");
assert_eq!(
cache.get("/tmp/doc.tex", 100, 1),
None,
"an edit inside the same second, which seconds alone would miss"
);
assert_eq!(cache.get("/tmp/elsewhere.tex", 100, 0), None);
}
#[test]
fn a_shard_this_build_cannot_read_is_rebuilt_not_misread() {
let shard = scratch("foreign");
std::fs::write(&shard, b"not an rkyv archive").unwrap();
let cache = ScriptCache::open(&shard).unwrap();
assert_eq!(cache.get("/tmp/doc.tex", 1, 2), None);
assert!(cache.is_empty());
cache.put("/tmp/doc.tex", 1, 2, vec![7]).unwrap();
assert_eq!(cache.get("/tmp/doc.tex", 1, 2), Some(vec![7]));
let mut owned = read_owned_shard(&shard).expect("readable");
owned.header.format_version = SHARD_FORMAT_VERSION + 1;
write_shard_atomic(&shard, &owned).unwrap();
let cache = ScriptCache::open(&shard).unwrap();
assert_eq!(
cache.get("/tmp/doc.tex", 1, 2),
None,
"a newer format is a miss"
);
cache.put("/tmp/doc.tex", 1, 2, vec![8]).unwrap();
let back = read_owned_shard(&shard).expect("readable");
assert_eq!(back.header.format_version, SHARD_FORMAT_VERSION);
assert_eq!(
back.entries.len(),
1,
"the old entries went with the format"
);
}
#[test]
fn the_header_says_what_wrote_the_shard() {
let shard = scratch("header");
let cache = ScriptCache::open(&shard).unwrap();
cache.put("/tmp/doc.tex", 1, 2, vec![1]).unwrap();
let owned = read_owned_shard(&shard).expect("readable");
assert_eq!(owned.header.magic, SHARD_MAGIC);
assert_eq!(
&SHARD_MAGIC.to_be_bytes(),
b"TEXR",
"the tag reads as itself"
);
assert_eq!(owned.header.format_version, SHARD_FORMAT_VERSION);
assert_eq!(owned.header.texrs_version, env!("CARGO_PKG_VERSION"));
assert_eq!(
owned.header.pointer_width as usize,
std::mem::size_of::<usize>()
);
assert!(owned.header.built_at_secs > 0, "and when it was written");
}
#[test]
fn eviction_drops_what_the_disk_no_longer_backs() {
let shard = scratch("evict");
let dir = shard.parent().unwrap().to_path_buf();
let kept = source(&dir, "kept.tex", "\\message{a}");
let edited = source(&dir, "edited.tex", "\\message{b}");
let removed = source(&dir, "removed.tex", "\\message{c}");
let cache = ScriptCache::open(&shard).unwrap();
for path in [&kept, &edited, &removed] {
let (secs, nsecs) = file_mtime(path).unwrap();
cache
.put(path.to_str().unwrap(), secs, nsecs, vec![1])
.unwrap();
}
assert_eq!(cache.len(), 3);
std::fs::write(&edited, "\\message{different}").unwrap();
std::fs::remove_file(&removed).unwrap();
assert_eq!(cache.evict_stale(), 2);
assert_eq!(cache.len(), 1);
let (secs, nsecs) = file_mtime(&kept).unwrap();
assert!(cache.get(kept.to_str().unwrap(), secs, nsecs).is_some());
assert_eq!(cache.evict_stale(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn clearing_is_idempotent_and_leaves_nothing_behind() {
let shard = scratch("clear");
let cache = ScriptCache::open(&shard).unwrap();
cache.put("/tmp/doc.tex", 1, 2, vec![1]).unwrap();
assert!(shard.exists());
cache.clear().unwrap();
assert!(!shard.exists(), "the file is gone");
assert!(cache.is_empty(), "and so is what it holds");
cache.clear().unwrap();
cache.put("/tmp/doc.tex", 1, 2, vec![2]).unwrap();
assert_eq!(cache.get("/tmp/doc.tex", 1, 2), Some(vec![2]));
let dir = shard.parent().unwrap();
let mut names: Vec<String> = std::fs::read_dir(dir)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
names.sort();
assert_eq!(
names,
vec!["scripts.rkyv", "scripts.rkyv.lock"],
"no temp files"
);
let _ = std::fs::remove_dir_all(dir);
}
#[test]
fn a_real_document_compiles_once_and_is_read_back_from_the_shard() {
let shard = scratch("chunk");
let dir = shard.parent().unwrap().to_path_buf();
let src = "\\catcode`\\{=1 \\catcode`\\}=2 \\message{hello}";
let doc = source(&dir, "doc.tex", src);
let chunk = crate::compile(src).expect("compiles");
let blob = bincode::serialize(&chunk).unwrap();
let cache = ScriptCache::open(&shard).unwrap();
let (secs, nsecs) = file_mtime(&doc).unwrap();
cache
.put(doc.to_str().unwrap(), secs, nsecs, blob.clone())
.unwrap();
let back = cache
.get(doc.to_str().unwrap(), secs, nsecs)
.expect("a hit");
assert_eq!(back, blob);
let decoded: fusevm::Chunk = bincode::deserialize(&back).unwrap();
assert_eq!(decoded.ops.len(), chunk.ops.len());
assert_eq!(
crate::runtime::run(decoded).unwrap(),
vec!["hello".to_string()]
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_cache_can_be_turned_off_by_the_environment() {
for value in ["0", "false", "no"] {
std::env::set_var("TEXRS_CACHE", value);
assert!(!cache_enabled(), "TEXRS_CACHE={value} turns it off");
}
for value in ["1", "true", "yes", ""] {
std::env::set_var("TEXRS_CACHE", value);
assert!(cache_enabled(), "TEXRS_CACHE={value:?} leaves it on");
}
std::env::remove_var("TEXRS_CACHE");
assert!(cache_enabled(), "and unset is on");
}
#[test]
fn a_document_key_is_its_content_and_its_profile() {
let a = "a".repeat(64);
let b = "b".repeat(64);
assert_eq!(
document_key(&[a.clone(), b.clone()], "default"),
document_key(&[a.clone(), b.clone()], "default")
);
assert_ne!(
document_key(&[a.clone(), b.clone()], "default"),
document_key(&[b.clone(), a.clone()], "default")
);
assert_ne!(
document_key(std::slice::from_ref(&a), "default"),
document_key(std::slice::from_ref(&a), "bytes")
);
assert!(document_key(&[a], "default").starts_with("doc:"));
}
#[test]
fn the_default_path_is_under_the_cache_directory() {
let path = default_cache_path();
assert!(path.ends_with("texrs/scripts.rkyv"), "{path:?}");
assert!(path.is_absolute(), "{path:?}");
}
}