tokenfold_core/

retrieval_store.rs

//! Reversible evidence store and retrieval: content-addressed storage of pre-transform
//! originals, plus the `[retrieval]` `tokenfold.toml` schema block that configures it.
//!
//! Granularity in this pass is whole-payload, not per-span: `pipeline.rs` stores the entire
//! pre-transform input under its SHA-256 content hash when `CompressionPolicy.store_originals`
//! is set and the payload contains no secret-shaped content. Per-span inline
//! `[tokenfold:retrieve ...]` markers are an explicitly out-of-scope future enhancement (see
//! the marker grammar's own fallback rule: "If a format cannot carry markers safely, markers
//! live only in `CompressionReport.retrieval`").
//!
//! Hash algorithm is SHA-256 only in this pass; `blake3` is a documented, rejected scope cut
//! (see [`RetrievalStore::open`]). Backends are `memory` (in-process, used in tests) and
//! `filesystem` (the default persistent backend); `sqlite` is likewise a documented, rejected
//! scope cut.

use std::collections::HashMap;
use std::fs::{File, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};

use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};

use crate::errors::TokenFoldError;
use crate::transforms::redaction;

/// `tokenfold.toml`'s documented `[retrieval].ttl_seconds` default (7 days).
pub const DEFAULT_TTL_SECONDS: u64 = 604_800;

/// One retrieval marker's worth of metadata, in the documented marker grammar:
/// `[tokenfold:retrieve hash=<hex> alg=sha256 namespace=<ns> bytes=<n> ttl=<seconds>]`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RetrievalMarker {
    pub hash: String,
    pub alg: &'static str,
    pub namespace: String,
    pub bytes: usize,
    pub ttl_seconds: Option<u64>,
}

/// A validated reference to one stored original. Accepted inputs are a raw SHA-256 hash,
/// the legacy `[tokenfold:retrieve ...]` marker, or the JSON `{"$tf_ref": ...}` marker
/// emitted by lossy JSON pruning.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RetrievalReference {
    pub hash: String,
    pub namespace: Option<String>,
}

pub fn parse_retrieval_reference(reference: &str) -> Result<RetrievalReference, TokenFoldError> {
    let reference = reference.trim();
    if reference.starts_with('{') {
        let value: serde_json::Value = serde_json::from_str(reference).map_err(|error| {
            TokenFoldError::InvalidInput(format!("invalid retrieval JSON marker: {error}"))
        })?;
        let marker = value.get("$tf_ref").unwrap_or(&value);
        let hash = marker
            .get("hash")
            .and_then(serde_json::Value::as_str)
            .ok_or_else(|| {
                TokenFoldError::InvalidInput(
                    "retrieval JSON marker has no string hash field".into(),
                )
            })?;
        let alg = marker
            .get("alg")
            .and_then(serde_json::Value::as_str)
            .unwrap_or("sha256");
        let namespace = marker
            .get("namespace")
            .and_then(serde_json::Value::as_str)
            .map(str::to_string);
        return validate_reference(hash, alg, namespace);
    }
    if reference.contains("tokenfold:retrieve") {
        let hash = extract_marker_field(reference, "hash").ok_or_else(|| {
            TokenFoldError::InvalidInput("retrieval marker has no hash=<hex> field".into())
        })?;
        let alg = extract_marker_field(reference, "alg").unwrap_or_else(|| "sha256".into());
        return validate_reference(&hash, &alg, extract_marker_field(reference, "namespace"));
    }
    validate_reference(reference, "sha256", None)
}

fn validate_reference(
    hash: &str,
    alg: &str,
    namespace: Option<String>,
) -> Result<RetrievalReference, TokenFoldError> {
    if alg != "sha256" {
        return Err(TokenFoldError::InvalidInput(format!(
            "unsupported retrieval hash algorithm {alg:?}; expected \"sha256\""
        )));
    }
    if hash.len() != 64 || !hash.chars().all(|c| c.is_ascii_hexdigit()) {
        return Err(TokenFoldError::InvalidInput(format!(
            "{hash:?} is not a valid SHA-256 hex hash"
        )));
    }
    if namespace
        .as_deref()
        .is_some_and(|value| !is_safe_path_component(value))
    {
        return Err(TokenFoldError::InvalidInput(format!(
            "invalid retrieval namespace: {:?}",
            namespace.as_deref().unwrap_or_default()
        )));
    }
    Ok(RetrievalReference {
        hash: hash.to_ascii_lowercase(),
        namespace,
    })
}

fn extract_marker_field(marker: &str, field: &str) -> Option<String> {
    let needle = format!("{field}=");
    let start = marker.find(&needle)? + needle.len();
    let rest = &marker[start..];
    let end = rest
        .find(|c: char| c.is_whitespace() || c == ']')
        .unwrap_or(rest.len());
    Some(rest[..end].to_string())
}

/// Result of a retrieval lookup. Deliberately has no "partial" variant: a caller either gets
/// the exact original bytes back, or an explicit reason it did not.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RetrievalOutcome {
    Found(Vec<u8>),
    Missing,
    Expired,
}

#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct GcOutcome {
    pub expired_removed: usize,
    pub evicted_removed: usize,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
struct EntryMeta {
    stored_at_unix: u64,
    ttl_seconds: Option<u64>,
    bytes: usize,
}

// `pub` only because it's the type of a field inside the public `RetrievalStore::Memory`
// variant (tuple-variant fields of a `pub enum` are implicitly public); its own fields stay
// private; nothing outside this module ever constructs or reads one directly.
pub struct MemoryEntry {
    bytes: Vec<u8>,
    meta: EntryMeta,
}

/// A content-addressed, namespaced store for reversible originals. Backend-dispatching enum
/// rather than a trait object: only two live variants exist in this pass, so a trait would add
/// indirection without buying any real polymorphism.
pub enum RetrievalStore {
    Memory(Mutex<HashMap<(String, String), MemoryEntry>>),
    Filesystem { root: PathBuf },
}

impl RetrievalStore {
    pub fn memory() -> Self {
        RetrievalStore::Memory(Mutex::new(HashMap::new()))
    }

    pub fn filesystem(root: impl Into<PathBuf>) -> Self {
        RetrievalStore::Filesystem { root: root.into() }
    }

    /// The default persistent store used when nothing overrides it: a `filesystem` backend
    /// rooted at [`default_store_path`].
    pub fn default_filesystem() -> Self {
        Self::filesystem(default_store_path())
    }

    /// Builds a store from `tokenfold.toml`'s `[retrieval]` schema values. `hash_algorithm` and
    /// `backend` are validated here so that selecting an unimplemented option (`blake3`,
    /// `sqlite`) fails clearly instead of silently behaving like `sha256`/`filesystem`.
    pub fn open(
        backend: &str,
        hash_algorithm: &str,
        store_path_override: Option<PathBuf>,
    ) -> Result<Self, TokenFoldError> {
        if hash_algorithm != "sha256" {
            return Err(TokenFoldError::ConfigError(format!(
                "retrieval hash_algorithm {hash_algorithm:?} is not implemented yet in v0.2; only \"sha256\" is supported"
            )));
        }
        match backend {
            "memory" => Ok(Self::memory()),
            "filesystem" => Ok(Self::filesystem(
                store_path_override.unwrap_or_else(default_store_path),
            )),
            "sqlite" => Err(TokenFoldError::ConfigError(
                "retrieval backend \"sqlite\" is not implemented yet in v0.2; use \"memory\" or \"filesystem\"".to_string(),
            )),
            other => Err(TokenFoldError::ConfigError(format!(
                "unknown retrieval backend {other:?}; expected \"memory\" or \"filesystem\" (\"sqlite\" is a documented v0.2 scope cut)"
            ))),
        }
    }

    /// Persists `bytes` under their SHA-256 hex hash, namespaced by `namespace`. Refuses to
    /// store (and never partially stores) anything [`redaction::contains_secret`] flags — this
    /// check runs unconditionally inside `store`, so no caller anywhere (pipeline, CLI, tests)
    /// can reach a code path that stores secret-shaped bytes.
    pub fn store(
        &self,
        bytes: &[u8],
        namespace: &str,
        ttl_seconds: Option<u64>,
    ) -> Result<RetrievalMarker, TokenFoldError> {
        self.store_batch(&[(bytes, namespace, ttl_seconds)])
            .map(|mut markers| markers.remove(0))
    }

    /// Stores a set as one publication unit. On failure, no newly-created entry remains.
    pub fn store_batch(
        &self,
        entries: &[(&[u8], &str, Option<u64>)],
    ) -> Result<Vec<RetrievalMarker>, TokenFoldError> {
        let prepared: Vec<_> = entries
            .iter()
            .map(|(bytes, namespace, ttl_seconds)| {
                validate_store_input(bytes, namespace)?;
                let hash = hex_sha256(bytes);
                let meta = EntryMeta {
                    stored_at_unix: now_unix(),
                    ttl_seconds: *ttl_seconds,
                    bytes: bytes.len(),
                };
                let marker = RetrievalMarker {
                    hash,
                    alg: "sha256",
                    namespace: (*namespace).to_string(),
                    bytes: bytes.len(),
                    ttl_seconds: *ttl_seconds,
                };
                Ok((bytes.to_vec(), meta, marker))
            })
            .collect::<Result<_, TokenFoldError>>()?;

        match self {
            RetrievalStore::Memory(map) => {
                let mut guard = map.lock().unwrap_or_else(|e| e.into_inner());
                for (bytes, meta, marker) in &prepared {
                    guard.insert(
                        (marker.namespace.clone(), marker.hash.clone()),
                        MemoryEntry {
                            bytes: bytes.clone(),
                            meta: meta.clone(),
                        },
                    );
                }
            }
            RetrievalStore::Filesystem { root } => {
                std::fs::create_dir_all(root)?;
                let _lock = lock_store(root)?;
                let mut created = Vec::new();
                for (bytes, meta, marker) in &prepared {
                    match store_filesystem_entry(root, bytes, meta, marker) {
                        Ok(true) => created.push((marker.namespace.clone(), marker.hash.clone())),
                        Ok(false) => {}
                        Err(error) => {
                            for (namespace, hash) in created {
                                let dir = root.join(namespace);
                                std::fs::remove_file(dir.join(format!("{hash}.meta.json"))).ok();
                                std::fs::remove_file(dir.join(format!("{hash}.bin"))).ok();
                            }
                            return Err(error);
                        }
                    }
                }
            }
        }

        Ok(prepared.into_iter().map(|(_, _, marker)| marker).collect())
    }

    /// Looks up `hash` in `namespace`. Never returns a partial result: exactly one of
    /// `Found`/`Missing`/`Expired`.
    pub fn retrieve(&self, hash: &str, namespace: &str) -> RetrievalOutcome {
        if !is_safe_path_component(namespace) || !is_safe_path_component(hash) {
            return RetrievalOutcome::Missing;
        }

        match self {
            RetrievalStore::Memory(map) => {
                let guard = map.lock().unwrap_or_else(|e| e.into_inner());
                match guard.get(&(namespace.to_string(), hash.to_string())) {
                    None => RetrievalOutcome::Missing,
                    Some(entry) if is_expired(&entry.meta) => RetrievalOutcome::Expired,
                    Some(entry) => RetrievalOutcome::Found(entry.bytes.clone()),
                }
            }
            RetrievalStore::Filesystem { root } => {
                if !root.is_dir() {
                    return RetrievalOutcome::Missing;
                }
                let Ok(_lock) = lock_store(root) else {
                    return RetrievalOutcome::Missing;
                };
                let dir = root.join(namespace);
                let meta_path = dir.join(format!("{hash}.meta.json"));
                let data_path = dir.join(format!("{hash}.bin"));
                let Ok(meta_bytes) = std::fs::read(&meta_path) else {
                    return RetrievalOutcome::Missing;
                };
                let Ok(meta) = serde_json::from_slice::<EntryMeta>(&meta_bytes) else {
                    return RetrievalOutcome::Missing;
                };
                if is_expired(&meta) {
                    return RetrievalOutcome::Expired;
                }
                match std::fs::read(&data_path) {
                    Ok(bytes) => RetrievalOutcome::Found(bytes),
                    Err(_) => RetrievalOutcome::Missing,
                }
            }
        }
    }

    /// Deletes entries whose `ttl_seconds` has elapsed (entries stored with `ttl_seconds:
    /// None` never expire), then — if `max_store_bytes` is given and total remaining stored
    /// bytes still exceed it — evicts the oldest-`stored_at` entries first until under the cap.
    pub fn gc(&self, max_store_bytes: Option<u64>) -> Result<GcOutcome, TokenFoldError> {
        match self {
            RetrievalStore::Memory(map) => {
                let mut guard = map.lock().unwrap_or_else(|e| e.into_inner());
                let mut outcome = GcOutcome::default();

                let expired: Vec<_> = guard
                    .iter()
                    .filter(|(_, entry)| is_expired(&entry.meta))
                    .map(|(key, _)| key.clone())
                    .collect();
                for key in expired {
                    guard.remove(&key);
                    outcome.expired_removed += 1;
                }

                if let Some(cap) = max_store_bytes {
                    let mut total: u64 = guard.values().map(|e| e.meta.bytes as u64).sum();
                    if total > cap {
                        let mut remaining: Vec<_> = guard
                            .iter()
                            .map(|(key, e)| {
                                (key.clone(), e.meta.stored_at_unix, e.meta.bytes as u64)
                            })
                            .collect();
                        remaining.sort_by_key(|(_, stored_at, _)| *stored_at);
                        for (key, _, bytes) in remaining {
                            if total <= cap {
                                break;
                            }
                            guard.remove(&key);
                            total = total.saturating_sub(bytes);
                            outcome.evicted_removed += 1;
                        }
                    }
                }
                Ok(outcome)
            }
            RetrievalStore::Filesystem { root } => {
                let mut outcome = GcOutcome::default();
                if !root.is_dir() {
                    return Ok(outcome);
                }
                let _lock = lock_store(root)?;

                let mut live: Vec<(PathBuf, PathBuf, EntryMeta)> = Vec::new();
                for ns_entry in std::fs::read_dir(root)? {
                    let ns_entry = ns_entry?;
                    if !ns_entry.file_type()?.is_dir() {
                        continue;
                    }
                    let ns_dir = ns_entry.path();
                    for file_entry in std::fs::read_dir(&ns_dir)? {
                        let file_entry = file_entry?;
                        let meta_path = file_entry.path();
                        let Some(name) = meta_path.file_name().and_then(|n| n.to_str()) else {
                            continue;
                        };
                        let Some(hash) = name.strip_suffix(".meta.json") else {
                            continue;
                        };
                        let Ok(meta_bytes) = std::fs::read(&meta_path) else {
                            continue;
                        };
                        let Ok(meta) = serde_json::from_slice::<EntryMeta>(&meta_bytes) else {
                            continue;
                        };
                        let data_path = ns_dir.join(format!("{hash}.bin"));
                        if is_expired(&meta) {
                            std::fs::remove_file(&meta_path).ok();
                            std::fs::remove_file(&data_path).ok();
                            outcome.expired_removed += 1;
                            continue;
                        }
                        live.push((meta_path, data_path, meta));
                    }
                }

                if let Some(cap) = max_store_bytes {
                    let mut total: u64 = live.iter().map(|(_, _, m)| m.bytes as u64).sum();
                    if total > cap {
                        live.sort_by_key(|(_, _, m)| m.stored_at_unix);
                        for (meta_path, data_path, meta) in live {
                            if total <= cap {
                                break;
                            }
                            std::fs::remove_file(&meta_path).ok();
                            std::fs::remove_file(&data_path).ok();
                            total = total.saturating_sub(meta.bytes as u64);
                            outcome.evicted_removed += 1;
                        }
                    }
                }
                Ok(outcome)
            }
        }
    }
}

fn validate_store_input(bytes: &[u8], namespace: &str) -> Result<(), TokenFoldError> {
    if redaction::contains_secret(bytes) {
        return Err(TokenFoldError::SafetyViolation(
            "refusing to persist bytes that match a secret-redaction pattern".to_string(),
        ));
    }
    if !is_safe_path_component(namespace) {
        return Err(TokenFoldError::InvalidInput(format!(
            "invalid retrieval namespace: {namespace:?}"
        )));
    }
    Ok(())
}

fn lock_store(root: &Path) -> Result<File, TokenFoldError> {
    std::fs::create_dir_all(root)?;
    let file = OpenOptions::new()
        .read(true)
        .write(true)
        .create(true)
        .truncate(false)
        .open(root.join(".tokenfold.lock"))?;
    file.lock()?;
    Ok(file)
}

static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);

fn unique_sidecar(path: &Path, label: &str) -> PathBuf {
    let id = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed);
    let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("entry");
    path.with_file_name(format!(".{name}.{}.{}.{label}", std::process::id(), id))
}

fn stage_file(path: &Path, bytes: &[u8]) -> Result<PathBuf, TokenFoldError> {
    let temp = unique_sidecar(path, "tmp");
    let mut file = OpenOptions::new()
        .write(true)
        .create_new(true)
        .open(&temp)?;
    if let Err(error) = file.write_all(bytes).and_then(|_| file.sync_all()) {
        std::fs::remove_file(&temp).ok();
        return Err(error.into());
    }
    Ok(temp)
}

fn publish_file(temp: &Path, destination: &Path) -> Result<(), TokenFoldError> {
    let backup = unique_sidecar(destination, "bak");
    let had_destination = destination.exists();
    if had_destination {
        std::fs::rename(destination, &backup)?;
    }
    if let Err(error) = std::fs::rename(temp, destination) {
        if had_destination {
            std::fs::rename(&backup, destination).ok();
        }
        std::fs::remove_file(temp).ok();
        return Err(error.into());
    }
    if had_destination {
        std::fs::remove_file(backup).ok();
    }
    Ok(())
}

fn store_filesystem_entry(
    root: &Path,
    bytes: &[u8],
    meta: &EntryMeta,
    marker: &RetrievalMarker,
) -> Result<bool, TokenFoldError> {
    let dir = root.join(&marker.namespace);
    std::fs::create_dir_all(&dir)?;
    let data_path = dir.join(format!("{}.bin", marker.hash));
    let meta_path = dir.join(format!("{}.meta.json", marker.hash));
    let existed = data_path.is_file() && meta_path.is_file();
    if !existed {
        std::fs::remove_file(&data_path).ok();
        std::fs::remove_file(&meta_path).ok();
    }
    let meta_json = serde_json::to_vec_pretty(meta).map_err(|e| {
        TokenFoldError::InternalError(format!("failed to encode retrieval metadata: {e}"))
    })?;
    let data_temp = stage_file(&data_path, bytes)?;
    let meta_temp = match stage_file(&meta_path, &meta_json) {
        Ok(path) => path,
        Err(error) => {
            std::fs::remove_file(data_temp).ok();
            return Err(error);
        }
    };
    if let Err(error) = publish_file(&data_temp, &data_path) {
        std::fs::remove_file(meta_temp).ok();
        return Err(error);
    }
    if let Err(error) = publish_file(&meta_temp, &meta_path) {
        if !existed {
            std::fs::remove_file(data_path).ok();
        }
        return Err(error);
    }
    Ok(!existed)
}

fn is_expired(meta: &EntryMeta) -> bool {
    match meta.ttl_seconds {
        None => false,
        Some(ttl) => now_unix().saturating_sub(meta.stored_at_unix) >= ttl,
    }
}

/// Rejects values that would let a namespace or hash escape the store root via path
/// traversal (`..`, embedded separators) when used as a directory/file name component.
fn is_safe_path_component(value: &str) -> bool {
    !value.is_empty()
        && !value.contains('/')
        && !value.contains('\\')
        && value != "."
        && value != ".."
}

fn now_unix() -> u64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|d| d.as_secs())
        .unwrap_or(0)
}

/// Lowercase hex SHA-256 of `bytes`.
pub fn hex_sha256(bytes: &[u8]) -> String {
    let digest = Sha256::digest(bytes);
    let mut hex = String::with_capacity(digest.len() * 2);
    for byte in digest {
        use std::fmt::Write;
        let _ = write!(hex, "{byte:02x}");
    }
    hex
}

fn home_dir() -> Option<PathBuf> {
    std::env::var_os("HOME")
        .or_else(|| std::env::var_os("USERPROFILE"))
        .map(PathBuf::from)
}

/// `$XDG_DATA_HOME/tokenfold/retrieve`, falling back to `<home>/.local/share/tokenfold/retrieve`
/// when `XDG_DATA_HOME` is unset — mirrors `tokenfold-cli::config`'s HOME/USERPROFILE fallback
/// for `home_dir()`. Deliberately not a Windows-native path (e.g. `%LOCALAPPDATA%`): the rest
/// of the codebase is XDG-everywhere by convention.
pub fn default_store_path() -> PathBuf {
    if let Some(dir) = std::env::var_os("XDG_DATA_HOME") {
        return PathBuf::from(dir).join("tokenfold").join("retrieve");
    }
    let home = home_dir().unwrap_or_else(|| PathBuf::from("."));
    home.join(".local")
        .join("share")
        .join("tokenfold")
        .join("retrieve")
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::sync::atomic::{AtomicU64, Ordering};

    fn temp_root(tag: &str) -> PathBuf {
        static COUNTER: AtomicU64 = AtomicU64::new(0);
        let n = COUNTER.fetch_add(1, Ordering::Relaxed);
        std::env::temp_dir().join(format!(
            "tokenfold_retrieval_store_test_{tag}_{}_{n}",
            std::process::id()
        ))
    }

    #[test]
    fn hex_sha256_matches_known_test_vector() {
        // sha256("") is a widely published test vector.
        assert_eq!(
            hex_sha256(b""),
            "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
        );
    }

    #[test]
    fn stores_by_content_hash_and_namespace_independently() {
        let store = RetrievalStore::memory();
        let marker_a = store.store(b"hello world", "project-a", None).unwrap();
        let marker_b = store.store(b"hello world", "project-b", None).unwrap();
        assert_eq!(marker_a.hash, marker_b.hash, "same bytes hash identically");

        assert_eq!(
            store.retrieve(&marker_a.hash, "project-a"),
            RetrievalOutcome::Found(b"hello world".to_vec())
        );
        // Same hash, wrong namespace: not found (namespaces are independent).
        assert_eq!(
            store.retrieve(&marker_a.hash, "project-nope"),
            RetrievalOutcome::Missing
        );
        assert_eq!(
            store.retrieve(&marker_b.hash, "project-b"),
            RetrievalOutcome::Found(b"hello world".to_vec())
        );
    }

    #[test]
    fn memory_retrieve_restores_exact_bytes_including_non_utf8() {
        let store = RetrievalStore::memory();
        let original: Vec<u8> = vec![0, 159, 146, 150, 1, 2, 3, 255, 0, 254];
        let marker = store.store(&original, "default", None).unwrap();
        match store.retrieve(&marker.hash, "default") {
            RetrievalOutcome::Found(bytes) => assert_eq!(bytes, original),
            other => panic!("expected Found, got {other:?}"),
        }
    }

    #[test]
    fn filesystem_retrieve_restores_exact_bytes() {
        let root = temp_root("roundtrip");
        let store = RetrievalStore::filesystem(&root);
        let original = b"the quick brown fox jumps over the lazy dog";
        let marker = store.store(original, "default", None).unwrap();

        match store.retrieve(&marker.hash, "default") {
            RetrievalOutcome::Found(bytes) => assert_eq!(bytes, original),
            other => panic!("expected Found, got {other:?}"),
        }

        std::fs::remove_dir_all(&root).ok();
    }

    #[test]
    fn missing_hash_returns_missing_with_no_partial_output() {
        let store = RetrievalStore::memory();
        store.store(b"stored content", "default", None).unwrap();
        assert_eq!(
            store.retrieve(
                "0000000000000000000000000000000000000000000000000000000000000000",
                "default"
            ),
            RetrievalOutcome::Missing
        );
    }

    #[test]
    fn expired_entry_returns_expired_with_no_partial_output() {
        let store = RetrievalStore::memory();
        // ttl_seconds: Some(0) means "already elapsed" the instant it's stored.
        let marker = store
            .store(b"will expire immediately", "default", Some(0))
            .unwrap();
        assert_eq!(
            store.retrieve(&marker.hash, "default"),
            RetrievalOutcome::Expired
        );
    }

    #[test]
    fn none_ttl_never_expires() {
        let store = RetrievalStore::memory();
        let marker = store.store(b"never expires", "default", None).unwrap();
        assert_eq!(
            store.retrieve(&marker.hash, "default"),
            RetrievalOutcome::Found(b"never expires".to_vec())
        );
    }

    #[test]
    fn gc_removes_only_expired_entries() {
        let store = RetrievalStore::memory();
        let expired = store.store(b"expired entry", "default", Some(0)).unwrap();
        let alive = store.store(b"alive entry", "default", None).unwrap();

        let outcome = store.gc(None).unwrap();
        assert_eq!(outcome.expired_removed, 1);
        assert_eq!(outcome.evicted_removed, 0);
        assert_eq!(
            store.retrieve(&expired.hash, "default"),
            RetrievalOutcome::Missing
        );
        assert_eq!(
            store.retrieve(&alive.hash, "default"),
            RetrievalOutcome::Found(b"alive entry".to_vec())
        );
    }

    #[test]
    fn gc_evicts_oldest_entries_first_when_over_size_cap() {
        let store = RetrievalStore::memory();
        // Each entry is stored with a slightly later `stored_at_unix` via a manual meta
        // override isn't available on the public API, so rely on filesystem gc's stable
        // ordering test below for eviction-order coverage, and just prove the cap is enforced
        // here (all entries share the same instant, so ties are broken by iteration order).
        store.store(b"aaaaaaaaaa", "default", None).unwrap();
        store.store(b"bbbbbbbbbb", "default", None).unwrap();
        store.store(b"cccccccccc", "default", None).unwrap();

        let outcome = store.gc(Some(15)).unwrap();
        assert!(
            outcome.evicted_removed >= 1,
            "at least one entry must be evicted over cap"
        );
        assert_eq!(outcome.expired_removed, 0);
    }

    #[test]
    fn filesystem_gc_evicts_oldest_stored_at_first() {
        let root = temp_root("gc_order");
        let store = RetrievalStore::filesystem(&root);
        let old = store.store(b"oldest-entry-here", "default", None).unwrap();
        std::thread::sleep(std::time::Duration::from_millis(1100));
        let newer = store.store(b"newest-entry", "default", None).unwrap();

        // Force the older entry's stored_at further into the past so ordering is unambiguous
        // regardless of clock resolution, then cap tight enough to evict exactly one entry.
        let meta_path = root.join("default").join(format!("{}.meta.json", old.hash));
        let mut meta: serde_json::Value =
            serde_json::from_slice(&std::fs::read(&meta_path).unwrap()).unwrap();
        meta["stored_at_unix"] = serde_json::json!(1);
        std::fs::write(&meta_path, serde_json::to_vec(&meta).unwrap()).unwrap();

        let outcome = store.gc(Some(newer.bytes as u64)).unwrap();
        assert_eq!(outcome.evicted_removed, 1);
        assert_eq!(
            store.retrieve(&old.hash, "default"),
            RetrievalOutcome::Missing,
            "the older entry must be the one evicted"
        );
        assert!(matches!(
            store.retrieve(&newer.hash, "default"),
            RetrievalOutcome::Found(_)
        ));

        std::fs::remove_dir_all(&root).ok();
    }

    #[test]
    fn store_refuses_bytes_containing_a_known_secret_pattern() {
        let store = RetrievalStore::memory();
        let err = store
            .store(b"AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE", "default", None)
            .unwrap_err();
        assert!(matches!(err, TokenFoldError::SafetyViolation(_)));
    }

    #[test]
    fn filesystem_backend_also_refuses_secret_bearing_bytes() {
        let root = temp_root("secret_gate");
        let store = RetrievalStore::filesystem(&root);
        let err = store
            .store(
                b"Authorization: Bearer abcDEF123.token-value",
                "default",
                None,
            )
            .unwrap_err();
        assert!(matches!(err, TokenFoldError::SafetyViolation(_)));
        // Nothing should have been written to disk.
        assert!(!root.join("default").exists());
        std::fs::remove_dir_all(&root).ok();
    }

    #[test]
    fn filesystem_batch_failure_commits_no_new_entries() {
        let root = temp_root("batch_rollback");
        let store = RetrievalStore::filesystem(&root);
        let entries = [
            (b"safe first".as_slice(), "default", None),
            (
                b"Authorization: Bearer abcDEF123.token-value".as_slice(),
                "default",
                None,
            ),
        ];
        assert!(store.store_batch(&entries).is_err());
        assert_eq!(
            store.retrieve(&hex_sha256(b"safe first"), "default"),
            RetrievalOutcome::Missing
        );
        std::fs::remove_dir_all(&root).ok();
    }

    #[test]
    fn filesystem_store_and_gc_do_not_expose_partial_entries() {
        let root = temp_root("store_gc_race");
        let store = std::sync::Arc::new(RetrievalStore::filesystem(&root));
        let writer = {
            let store = std::sync::Arc::clone(&store);
            std::thread::spawn(move || {
                for i in 0..50 {
                    let bytes = format!("entry-{i}-{}", "x".repeat(256));
                    let marker = store.store(bytes.as_bytes(), "default", None).unwrap();
                    assert_eq!(
                        store.retrieve(&marker.hash, "default"),
                        RetrievalOutcome::Found(bytes.into_bytes())
                    );
                }
            })
        };
        let collector = {
            let store = std::sync::Arc::clone(&store);
            std::thread::spawn(move || {
                for _ in 0..50 {
                    store.gc(None).unwrap();
                }
            })
        };
        writer.join().unwrap();
        collector.join().unwrap();
        std::fs::remove_dir_all(&root).ok();
    }

    #[test]
    fn open_rejects_sqlite_backend_as_a_clear_config_error() {
        // `RetrievalStore` isn't `Debug` (it holds a `Mutex`), so assert via `Result::err`
        // rather than `unwrap_err`.
        let err = RetrievalStore::open("sqlite", "sha256", None)
            .err()
            .unwrap();
        assert!(matches!(err, TokenFoldError::ConfigError(_)));
    }

    #[test]
    fn open_rejects_blake3_hash_algorithm_as_a_clear_config_error() {
        let err = RetrievalStore::open("filesystem", "blake3", None)
            .err()
            .unwrap();
        assert!(matches!(err, TokenFoldError::ConfigError(_)));
    }

    #[test]
    fn open_accepts_memory_and_filesystem_with_sha256() {
        assert!(RetrievalStore::open("memory", "sha256", None).is_ok());
        assert!(RetrievalStore::open("filesystem", "sha256", Some(temp_root("open_ok"))).is_ok());
    }

    #[test]
    fn unsafe_namespace_is_rejected_by_store_and_missing_from_retrieve() {
        let store = RetrievalStore::memory();
        assert!(store.store(b"data", "../escape", None).is_err());
        assert_eq!(
            store.retrieve("deadbeef", "../escape"),
            RetrievalOutcome::Missing
        );
    }

    #[test]
    fn parses_all_supported_retrieval_reference_forms() {
        let hash = "A".repeat(64);
        let raw = parse_retrieval_reference(&hash).unwrap();
        assert_eq!(raw.hash, "a".repeat(64));
        assert_eq!(raw.namespace, None);

        let legacy = parse_retrieval_reference(&format!(
            "[tokenfold:retrieve hash={hash} alg=sha256 namespace=project bytes=1]"
        ))
        .unwrap();
        assert_eq!(legacy.namespace.as_deref(), Some("project"));

        let json = parse_retrieval_reference(&format!(
            r#"{{"$tf_ref":{{"hash":"{hash}","alg":"sha256","namespace":"project"}}}}"#
        ))
        .unwrap();
        assert_eq!(json, legacy);
    }

    #[test]
    fn rejects_malformed_retrieval_references() {
        assert!(parse_retrieval_reference("deadbeef").is_err());
        assert!(
            parse_retrieval_reference(&format!(
                "[tokenfold:retrieve hash={} alg=blake3]",
                "a".repeat(64)
            ))
            .is_err()
        );
        assert!(
            parse_retrieval_reference(&format!(
                r#"{{"$tf_ref":{{"hash":"{}","namespace":"../escape"}}}}"#,
                "a".repeat(64)
            ))
            .is_err()
        );
    }
}