arblang 0.1.24

Visualize and modify Unix pipelines: pipe a stream in, arb spawns a dynamic TUI
Documentation
//! rkyv-backed script cache for parsed `.arb` specs (mirrors the sibling langs
//! zshrs/rubylang/elisprs/vimlrs/awkrs). Versioned from day one so a spec that
//! parsed once never breaks on a later run.
//!
//! Layout: a single shard at `~/.arb/scripts.rkyv` (or `$ARB_CACHE`). The *outer*
//! container is a zero-copy rkyv archive (`Shard`), validated on load; each
//! *inner* entry blob is a bincode-encoded parsed AST (`Vec<ast::Command>`). arb
//! caches the AST rather than the built `Spec` because the `Spec` carries
//! `regex::Regex` (in `expect`/`match`/`replace`), which is neither serde- nor
//! rkyv-serializable — so a cache hit skips lex+parse and hands the AST straight
//! to `spec::build`, which recompiles the (cheap) regex/pipeline. The key is a
//! 64-bit hash of the source plus a schema version, so a source or format change
//! misses cleanly instead of loading a stale tree.

use std::hash::{Hash, Hasher};
use std::path::{Path, PathBuf};

use rkyv::{Archive, Deserialize as RkyvDe, Serialize as RkyvSer};

use crate::ast::Command;

/// Bump on any incompatible change to `ast::Command`/`ast::Arg` or the parser
/// output shape, so old blobs miss instead of decoding wrong.
///
/// 2: the lexer now reads a jq literal at command position as one verbatim atom,
/// so the same source parses to a different `Command` split than schema 1 wrote.
/// (The build stamp in [`build_stamp`] is the AUTOMATIC guard — a rebuild
/// invalidates the shard on its own. This constant remains as the record of the
/// format changes that made it necessary.)
///
/// 3: that atom widened to the whole jq surface — `reduce`/`if`/`try`, the
/// `name(` call spelling, `[`/`{`/`(`/`$`/`"` openers and `@format` — and the
/// scan now balances braces and keeps a `;` inside `(`/`[`/`{`. Every one of
/// those changes the `Command` split for source schema 2 already cached.
const SCHEMA: u64 = 3;

/// The outer, rkyv-archived shard: a flat list of (key, bincode-blob) entries,
/// stamped with the build that wrote them.
#[derive(Archive, RkyvSer, RkyvDe, Default)]
#[archive(check_bytes)]
struct Shard {
    /// [`build_stamp`] of the binary that wrote this shard. A shard written by a
    /// DIFFERENT build is dropped whole rather than read — see that function for
    /// why hand-bumped salts are not enough.
    stamp: u64,
    entries: Vec<Entry>,
}

#[derive(Archive, RkyvSer, RkyvDe)]
#[archive(check_bytes)]
struct Entry {
    key: u64,
    blob: Vec<u8>,
}

/// A stable content key for a source string, salted with the schema AND the arb
/// version. The version salt matters because this caches PARSER OUTPUT: a build
/// that changes how a source lexes produces a different AST for the same text, and
/// a key over the source alone would keep serving the pre-change tree from a warm
/// `~/.arb` — the new behavior would silently not apply on any machine that had
/// run the spec before. Re-lexing a spec is cheap; serving a stale parse is not.
pub fn key_for(src: &str) -> u64 {
    let mut h = rustc_hash::FxHasher::default();
    SCHEMA.hash(&mut h);
    env!("CARGO_PKG_VERSION").hash(&mut h);
    src.hash(&mut h);
    h.finish()
}

/// A fingerprint of the RUNNING BINARY: its size and modification time.
///
/// The salts above are not sufficient on their own, and that is measured, not
/// argued. `SCHEMA` is hand-bumped and `CARGO_PKG_VERSION` only moves at
/// release, so a development build that changes how a source LEXES leaves both
/// unchanged — and the shard then replays the previous build's AST for the same
/// text. Reproduced on this tree: with a warm shard, `out { in.json; {k: .a} }`
/// kept printing `{"k":1}` after the lexer rule that makes that parse work was
/// deliberately removed and the binary rebuilt; the same binary against a cold
/// shard correctly reported `command cannot start with a block`.
///
/// Both fields are needed. Size alone misses a same-size rebuild (the common
/// case for a one-line change); mtime alone misses a restored copy. Hashing them
/// costs one `stat` per process.
fn build_stamp() -> u64 {
    static STAMP: std::sync::OnceLock<u64> = std::sync::OnceLock::new();
    *STAMP.get_or_init(|| {
        let mut h = rustc_hash::FxHasher::default();
        // No readable executable (a stripped-down container, a deleted binary)
        // leaves the stamp at the hash of nothing — the cache then behaves as it
        // did before, which is a stale-read risk only where `stat` is impossible.
        if let Ok(md) = std::env::current_exe().and_then(std::fs::metadata) {
            md.len().hash(&mut h);
            if let Ok(nanos) = md
                .modified()
                .and_then(|t| {
                    t.duration_since(std::time::UNIX_EPOCH)
                        .map_err(|e| std::io::Error::other(e.to_string()))
                })
                .map(|d| d.as_nanos())
            {
                nanos.hash(&mut h);
            }
        }
        h.finish()
    })
}

/// `~/.arb/scripts.rkyv` (or `$ARB_CACHE`), creating the parent dir. `None` when
/// there is no `$HOME`/`$ARB_CACHE` (the cache is then silently disabled).
fn shard_path() -> Option<PathBuf> {
    if let Some(p) = std::env::var_os("ARB_CACHE") {
        let p = PathBuf::from(p);
        if let Some(parent) = p.parent() {
            let _ = std::fs::create_dir_all(parent);
        }
        return Some(p);
    }
    let dir = std::env::var_os("HOME").map(|h| Path::new(&h).join(".arb"))?;
    let _ = std::fs::create_dir_all(&dir);
    Some(dir.join("scripts.rkyv"))
}

fn load_shard() -> Shard {
    let Some(path) = shard_path() else {
        return Shard::default();
    };
    let Ok(bytes) = std::fs::read(&path) else {
        return Shard::default();
    };
    // A corrupt/old-format shard validates-fails and resets cleanly.
    let shard = rkyv::from_bytes::<Shard>(&bytes).unwrap_or_default();
    // A shard written by a different BUILD is stale by construction: it holds
    // ASTs produced by the previous lexer/parser. Drop it whole — that also
    // prunes it, since every entry in it is unreachable from this build.
    if shard.stamp == build_stamp() {
        shard
    } else {
        Shard::default()
    }
}

fn write_shard(shard: &mut Shard) -> Result<(), String> {
    shard.stamp = build_stamp();
    let path = shard_path().ok_or("no home dir for cache")?;
    let bytes = rkyv::to_bytes::<_, 4096>(shard).map_err(|e| format!("cache serialize: {e}"))?;
    std::fs::write(&path, &bytes).map_err(|e| format!("cache write: {e}"))
}

/// Encode a parsed AST to the inner bincode blob.
fn encode(ast: &[Command]) -> Result<Vec<u8>, String> {
    bincode::serialize(ast).map_err(|e| format!("cache encode: {e}"))
}

/// Decode an inner bincode blob back to a parsed AST.
fn decode(blob: &[u8]) -> Option<Vec<Command>> {
    bincode::deserialize(blob).ok()
}

/// The blob for `key` in `shard`, if present (pure; testable without the FS).
fn shard_get(shard: &Shard, key: u64) -> Option<&[u8]> {
    shard
        .entries
        .iter()
        .find(|e| e.key == key)
        .map(|e| e.blob.as_slice())
}

/// Largest number of cached specs kept in the shard. Every run reads and
/// deserializes the whole shard, so an unbounded cache would slow every startup
/// as it accreted distinct specs over time; cap it and evict oldest-first.
const MAX_ENTRIES: usize = 1024;

/// Insert/replace `key`'s blob in `shard` (pure; testable without the FS).
fn shard_put(shard: &mut Shard, key: u64, blob: Vec<u8>) {
    shard.entries.retain(|e| e.key != key);
    // Bound the shard: drop the oldest entries (FIFO) so the on-disk file — and
    // the per-run read+deserialize cost — stay bounded no matter how many
    // distinct specs are ever cached.
    if shard.entries.len() >= MAX_ENTRIES {
        let overflow = shard.entries.len() + 1 - MAX_ENTRIES;
        shard.entries.drain(..overflow);
    }
    shard.entries.push(Entry { key, blob });
}

/// Look up the parsed AST for `src`, if present and current.
pub fn load(src: &str) -> Option<Vec<Command>> {
    let shard = load_shard();
    decode(shard_get(&shard, key_for(src))?)
}

/// Store the parsed `ast` for `src`, replacing any prior entry. Best-effort: any
/// error is returned but callers treat a cache miss/failure as non-fatal.
pub fn store(src: &str, ast: &[Command]) -> Result<(), String> {
    let blob = encode(ast)?;
    let mut shard = load_shard();
    shard_put(&mut shard, key_for(src), blob);
    write_shard(&mut shard)
}

/// Parse `src`, using the script cache: a hit skips lex+parse, a miss parses and
/// populates the cache (best-effort — a cache write failure never fails a run).
pub fn parse_cached(src: &str) -> Result<Vec<Command>, crate::err::SpecError> {
    if let Some(ast) = load(src) {
        return Ok(ast);
    }
    let ast = crate::parser::parse(src)?;
    let _ = store(src, &ast);
    Ok(ast)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::ast::Arg;

    /// `ARB_CACHE` is process-global, so the two tests that redirect the shard to
    /// a scratch file must not run at the same time — one clearing the variable
    /// sends the other's `load` at the real `~/.arb` shard.
    static FS_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());

    #[test]
    fn key_is_stable_and_schema_salted() {
        assert_eq!(key_for("text .t <- in"), key_for("text .t <- in"));
        assert_ne!(key_for("text .t <- in"), key_for("tail .t <- in"));
    }

    #[test]
    fn encode_decode_round_trips_the_ast() {
        let ast = vec![Command {
            name: "gauge".into(),
            args: vec![
                Arg::Word(".cpu".into()),
                Arg::Block(vec![Command {
                    name: "in".into(),
                    args: vec![],
                    pos: 12,
                }]),
            ],
            pos: 0,
        }];
        let blob = encode(&ast).unwrap();
        let back = decode(&blob).unwrap();
        assert_eq!(back.len(), 1);
        assert_eq!(back[0].name, "gauge");
        // Nested block survives the round trip.
        match &back[0].args[1] {
            Arg::Block(inner) => assert_eq!(inner[0].name, "in"),
            _ => panic!("nested block lost"),
        }
    }

    #[test]
    fn shard_get_put_replaces_by_key() {
        let mut s = Shard::default();
        shard_put(&mut s, 7, vec![1, 2, 3]);
        shard_put(&mut s, 9, vec![4]);
        assert_eq!(shard_get(&s, 7), Some(&[1, 2, 3][..]));
        // Re-put the same key replaces, not appends.
        shard_put(&mut s, 7, vec![8, 8]);
        assert_eq!(s.entries.len(), 2);
        assert_eq!(shard_get(&s, 7), Some(&[8, 8][..]));
        assert_eq!(shard_get(&s, 1), None);
    }

    #[test]
    fn shard_put_evicts_oldest_at_capacity() {
        let mut s = Shard::default();
        // Fill past capacity with distinct keys.
        for k in 0..(MAX_ENTRIES as u64 + 50) {
            shard_put(&mut s, k, vec![k as u8]);
        }
        assert_eq!(s.entries.len(), MAX_ENTRIES, "shard is capped");
        // The 50 oldest keys were evicted; the newest survive.
        assert_eq!(shard_get(&s, 0), None);
        assert_eq!(shard_get(&s, 49), None);
        assert!(shard_get(&s, 50).is_some());
        assert!(shard_get(&s, MAX_ENTRIES as u64 + 49).is_some());
    }

    /// The regression this file's `stamp` exists for: a shard written by a
    /// different BUILD must not be read. Reproduced by hand here — writing a
    /// shard, then rewriting it with a stamp that is not this build's — because
    /// the real trigger (rebuilding the binary mid-test) is not available to a
    /// unit test.
    #[test]
    fn shard_from_another_build_is_not_read() {
        let _guard = FS_LOCK
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        let tmp = std::env::temp_dir().join(format!("arb_cache_stamp_{}.rkyv", std::process::id()));
        std::env::set_var("ARB_CACHE", &tmp);
        let _ = std::fs::remove_file(&tmp);
        let src = "text .t <- in";
        store(src, &crate::parser::parse(src).unwrap()).unwrap();
        assert!(load(src).is_some(), "same build reads its own shard");

        // Re-stamp the shard as if another build had written it.
        let mut shard = rkyv::from_bytes::<Shard>(&std::fs::read(&tmp).unwrap()).unwrap();
        shard.stamp = build_stamp().wrapping_add(1);
        let bytes = rkyv::to_bytes::<_, 4096>(&shard).unwrap();
        std::fs::write(&tmp, &bytes).unwrap();

        assert!(
            load(src).is_none(),
            "a shard from another build must be dropped, not replayed"
        );
        let _ = std::fs::remove_file(&tmp);
        std::env::remove_var("ARB_CACHE");
    }

    #[test]
    fn parse_cached_matches_direct_parse_via_fs_round_trip() {
        let _guard = FS_LOCK
            .lock()
            .unwrap_or_else(std::sync::PoisonError::into_inner);
        // Isolate the shard to a scratch file so the test never touches ~/.arb.
        let tmp = std::env::temp_dir().join(format!("arb_cache_test_{}.rkyv", std::process::id()));
        std::env::set_var("ARB_CACHE", &tmp);
        let _ = std::fs::remove_file(&tmp);
        let src = "text .t <- in";
        // First call: cache miss -> parses + stores.
        let a = parse_cached(src).unwrap();
        // Second call: cache hit -> same AST as a direct parse.
        let b = parse_cached(src).unwrap();
        let direct = crate::parser::parse(src).unwrap();
        assert_eq!(a.len(), direct.len());
        assert_eq!(b.len(), direct.len());
        assert_eq!(b[0].name, direct[0].name);
        assert!(tmp.exists(), "shard should have been written");
        let _ = std::fs::remove_file(&tmp);
        std::env::remove_var("ARB_CACHE");
    }
}