sherd 0.6.0

two-axis federation of SPEC.md and source over a dir DAG, for small-context local models
Documentation
//! One cached, idempotent state file for every command that needs to remember
//! something: `sherd-state` in the git common dir (`src/state:V4`).
//!
//! Line-oriented and greppable, like `.context-limits` and `.spec-records` in
//! the sibling repos -- not JSON, so the core stays free of `serde` (§C: the
//! deterministic core must build with `--no-default-features`).
//!
//! ```text
//! kind key value
//! pace prefill 1738
//! tok  src/fed/SPEC.md 1023
//! hash src/fed/SPEC.md 8f2a1c...
//! ```
//!
//! IDEMPOTENT: writing unchanged state twice yields a byte-identical file, and
//! reading it back yields the same map. Asserted, because a cache that drifts
//! on rewrite is worse than no cache -- the same guarantee microlith asserts
//! for `fmt`.

use std::collections::BTreeMap;
use std::hash::{Hash, Hasher};
use std::path::{Path, PathBuf};

/// `(kind, key) -> value`. `BTreeMap` so serialisation is ordered, which is
/// what makes the file byte-stable across runs.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct State {
    map: BTreeMap<(String, String), String>,
    /// Where this state was read from, and where [`Self::save`] writes.
    ///
    /// Carried rather than re-resolved, so a `State` is a value and not a
    /// handle on one ambient file. Reading `SHERD_STATE` inside `save` meant
    /// every test in the process shared one file: they wrote `obs` rows,
    /// `derived_prefill` branched on how many existed, and which lines
    /// executed depended on what a previous run left behind (`src/ollama:B8`).
    ///
    /// `std::env::set_var` is unsafe under edition 2024 and this crate
    /// FORBIDS unsafe, so a per-test env var was never available -- the path
    /// had to become a parameter (`src/ollama:V19`).
    path: PathBuf,
}

/// The default state file: `SHERD_STATE`, else `sherd-state` in the git
/// common dir, else `.sherd-state` in the cwd (`src/state:V4`).
///
/// Read at the EDGE only. Everything below takes the path as a value, the
/// same split `tdd::cargo_bin`/`gate_with` uses for the toolchain.
#[must_use]
pub fn default_path() -> PathBuf {
    resolve(
        std::env::var("SHERD_STATE").ok(),
        git_common_dir(Path::new(".")),
    )
}

/// Which file the default store is, given what the edge read.
///
/// The cwd-relative `.sherd-state` wrote into whatever repo sherd ran in, and
/// a consumer's `.crate` shipped it (`src/state:B3`). The git common dir is
/// never tracked or packaged, and every worktree shares it.
#[must_use]
pub fn resolve(env: Option<String>, git_common: Option<PathBuf>) -> PathBuf {
    match (env, git_common) {
        (Some(p), _) => PathBuf::from(p),
        (None, Some(dir)) => dir.join("sherd-state"),
        (None, None) => PathBuf::from(".sherd-state"),
    }
}

/// The git common dir of the repository holding `dir`, absolute, or `None`
/// outside one.
#[must_use]
pub fn git_common_dir(dir: &Path) -> Option<PathBuf> {
    let out = crate::git::at(
        dir,
        &["rev-parse", "--path-format=absolute", "--git-common-dir"],
    )
    .output()
    .ok()?;
    let s = String::from_utf8(out.stdout).ok()?;
    let s = s.trim();
    (out.status.success() && !s.is_empty()).then(|| PathBuf::from(s))
}

impl State {
    /// Read state, or an empty one. A missing file is a cold start, not an
    /// error; a malformed LINE is skipped, but never silently -- an
    /// unparsable file would otherwise read as "nothing cached" forever.
    #[must_use]
    pub fn load() -> Self {
        Self::at(default_path())
    }

    /// Read state from an EXPLICIT path.
    ///
    /// What makes the suite hermetic: a test hands its own temp file and
    /// shares nothing with the tests running beside it.
    #[must_use]
    pub fn at(path: impl Into<PathBuf>) -> Self {
        let path = path.into();
        let mut s = Self {
            path: path.clone(),
            ..Self::default()
        };
        let Ok(text) = std::fs::read_to_string(&path) else {
            return s;
        };
        for (n, line) in text.lines().enumerate() {
            if line.starts_with('#') || line.trim().is_empty() {
                continue;
            }
            let mut it = line.splitn(3, ' ');
            match (it.next(), it.next(), it.next()) {
                (Some(k), Some(key), Some(v)) => {
                    s.map.insert((k.into(), unescape_key(key)), v.into());
                }
                _ => {
                    eprintln!("sherd-state:{}: unparsable line, ignored", n + 1)
                }
            }
        }
        s
    }

    /// Write. Best-effort: a read-only tree must not fail a run over a cache.
    pub fn save(&self) {
        // Through `serialise`, not a second loop of its own. The two had
        // drifted -- escaping the key in `serialise` changed nothing because
        // the WRITE path never called it (B1), which is `.:src/fed:B13`'s two
        // readings of one rule inside a nine-line function.
        let out = format!(
            "# generated by sherd. Idempotent -- rewriting unchanged state is a no-op.\n{}",
            self.serialise()
        );
        let _ = std::fs::write(&self.path, out);
    }

    #[must_use]
    pub fn get(&self, kind: &str, key: &str) -> Option<&str> {
        self.map.get(&(kind.into(), key.into())).map(String::as_str)
    }

    pub fn set(&mut self, kind: &str, key: &str, value: impl Into<String>) {
        self.map.insert((kind.into(), key.into()), value.into());
    }

    #[must_use]
    pub fn get_u64(&self, kind: &str, key: &str) -> Option<u64> {
        self.get(kind, key).and_then(|v| v.parse().ok())
    }

    /// Drop every entry of one kind -- used when a plan is superseded, so a
    /// stale step never outlives the plan that produced it.
    pub fn clear_kind(&mut self, kind: &str) {
        self.map.retain(|(k, _), _| k != kind);
    }

    /// Every value of one kind.
    #[must_use]
    pub fn all(&self, kind: &str) -> Vec<String> {
        self.map
            .iter()
            .filter(|((k, _), _)| k == kind)
            .map(|(_, v)| v.clone())
            .collect()
    }

    /// Append one entry to an ORDERED, capped log, deduplicated by `dedup`.
    ///
    /// The key carries a zero-padded sequence, so lexical order IS insertion
    /// order and the cap drops the genuinely oldest. A plain key-order trim
    /// cannot do that: its only ordering is the key's own, and the one real
    /// caller keys by CONTENT HASH, which carries no time at all (B2).
    ///
    /// Idempotent, and that is what lets this avoid a timestamp: an entry
    /// whose `dedup` is already present is not re-appended, so recording the
    /// same observation twice leaves the file byte-identical. A clock would
    /// change the file when nothing changed, which is not a cache.
    pub fn push(&mut self, log: Log, dedup: &str, value: impl Into<String>) {
        let kind = log.kind;
        if self.find_key(kind, dedup).is_some() {
            return;
        }
        self.set(kind, &format!("{:08}-{dedup}", self.next_seq(kind)), value);
        let ordered = self.oldest_first(kind);
        let over = ordered.len().saturating_sub(log.cap);
        for k in ordered.into_iter().take(over) {
            self.map.remove(&(kind.to_string(), k));
        }
    }

    /// The sequence the next entry of a kind gets.
    ///
    /// MAX, not the last key's: a file written before ordered keys existed
    /// holds bare content hashes, and `ff78..` sorts after `00000000-..`, so
    /// reading the last key's sequence found none and every append restarted
    /// at zero. An unsequenced key is older than all of them, which is true --
    /// it was written first.
    fn next_seq(&self, kind: &str) -> u64 {
        self.keys_of(kind)
            .iter()
            .filter_map(|k| seq_of(k))
            .max()
            .map_or(0, |n| n.saturating_add(1))
    }

    /// The full key an ordered entry was stored under, found by its `dedup`.
    #[must_use]
    pub fn find_key(&self, kind: &str, dedup: &str) -> Option<String> {
        let suffix = format!("-{dedup}");
        self.keys_of(kind)
            .into_iter()
            .find(|k| k.ends_with(&suffix))
    }

    /// Every key of one kind, oldest first.
    ///
    /// By SEQUENCE, not by the key's own bytes: a legacy bare hash like
    /// `ff7833..` sorts after `00000000-..` lexically, so plain map order put
    /// the oldest entries LAST and the cap evicted the newest. `None` orders
    /// before `Some` in Rust, which is exactly right here -- an unsequenced
    /// entry predates every sequenced one.
    fn oldest_first(&self, kind: &str) -> Vec<String> {
        let mut keys = self.keys_of(kind);
        keys.sort_by_key(|k| (seq_of(k), k.clone()));
        keys
    }

    /// Every key of one kind, in the map's order, which is lexical.
    fn keys_of(&self, kind: &str) -> Vec<String> {
        self.map
            .keys()
            .filter(|(k, _)| k == kind)
            .map(|(_, key)| key.clone())
            .collect()
    }

    #[must_use]
    pub fn serialise(&self) -> String {
        self.map
            .iter()
            .map(|((k, key), v)| format!("{k} {} {v}\n", escape_key(key)))
            .collect()
    }
}

/// The log an ordered entry is appended to: which kind, and how many to keep.
///
/// A struct because `kind` and `cap` describe the LOG and travel together,
/// while `dedup` and `value` describe the ENTRY. Passing all four positionally
/// is the five-argument helper this repo has already written once and
/// reverted -- one lint traded for a worse one.
#[derive(Debug, Clone, Copy)]
pub struct Log<'a> {
    /// The kind these entries share.
    pub kind: &'a str,
    /// How many to keep; the oldest beyond this are evicted.
    pub cap: usize,
}

impl<'a> Log<'a> {
    /// One log: which kind, and how many to keep.
    #[must_use]
    pub const fn new(kind: &'a str, cap: usize) -> Self {
        Self { kind, cap }
    }
}

/// The sequence an ordered key opens with, if it has one.
fn seq_of(key: &str) -> Option<u64> {
    key.split_once('-').and_then(|(n, _)| n.parse().ok())
}
/// A key, safe to sit in the middle field of `kind key value`.
///
/// V1: the VALUE may contain spaces -- an `obs` row carries a whole telemetry
/// line -- so the parser splits three and the key is the middle field. A key
/// with a space silently became its own first word, and everything after it
/// joined the value (B1). `%` is escaped first, or unescaping would be
/// ambiguous.
fn escape_key(key: &str) -> String {
    key.replace('%', "%25").replace(' ', "%20")
}

/// The inverse of [`escape_key`]. `%20` before `%25`, so a literal `%20`
/// written as `%2520` comes back as `%20` and not as a space.
fn unescape_key(key: &str) -> String {
    key.replace("%20", " ").replace("%25", "%")
}

/// Content hash, for cache keys.
///
/// `DefaultHasher` is not stable across Rust releases, which is fine here and
/// nowhere else: a changed hash function means a cache MISS and a recompute,
/// never a wrong answer. It would not be acceptable for anything a gate reads.
#[must_use]
pub fn content_hash(bytes: &[u8]) -> String {
    let mut h = std::collections::hash_map::DefaultHasher::new();
    bytes.hash(&mut h);
    format!("{:016x}", h.finish())
}

/// Cached token count for a file, recomputed only when its content changed.
///
/// # Errors
/// Propagates the read failure -- unreadable is not zero (`.:tokens` V5).
pub fn cached_tokens(
    st: &mut State,
    path: &std::path::Path,
) -> std::io::Result<u64> {
    let bytes = std::fs::read(path)?;
    let key = path.to_string_lossy().to_string();
    let h = content_hash(&bytes);
    if st.get("hash", &key) == Some(h.as_str())
        && let Some(n) = st.get_u64("tok", &key)
    {
        return Ok(n);
    }
    let n = crate::tokens::count(&String::from_utf8_lossy(&bytes)).tokens;
    st.set("hash", &key, h);
    st.set("tok", &key, n.to_string());
    Ok(n)
}

#[cfg(test)]
#[path = "tests/state.rs"]
mod tests;

#[cfg(test)]
#[path = "tests/path.rs"]
mod path_tests;