haz_cache/

clean.rs

//! Cache invalidation per `CACHE-021` and `AUX-022..AUX-027`.
//!
//! Two entry points:
//!
//! - [`CacheWriter::clear`] is the unconditional reset behind
//!   `haz cache clear`: it removes the entire cache root in a
//!   single recursive call. Subsequent lookups against the same
//!   workspace are misses until new entries are stored.
//! - [`CacheWriter::clean`] is the composable selective reclamation
//!   behind `haz cache clean`. It accepts a [`CleanOptions`] mode
//!   set and applies the modes in spec-mandated priority order:
//!     1. `soft` (`CACHE-022`): reclaim every *objectively
//!        stale* artefact, entry directories with a missing,
//!        unparseable, or schema-mismatched manifest plus
//!        `.tmp-<key>-<random>` store-time directories and
//!        `.restore-<key>-<random>` restore-time staging
//!        directories.
//!     2. `max_age` (`AUX-023` step 4): evict every remaining
//!        well-formed entry whose `created_at` is strictly older
//!        than `now_unix - max_age`.
//!     3. `max_size` (`AUX-023` step 5): if the well-formed
//!        survivors' footprint exceeds `max_size`, evict
//!        oldest-`created_at`-first until the residual footprint
//!        is at or below `max_size`.
//!     4. `dry_run` (`AUX-024`): compute the eviction set but
//!        make no on-disk changes.
//!
//! Per `AUX-024`, when more than one mode would name the same
//! entry, the entry counts in the highest-priority mode
//! (`soft` > `max_age` > `max_size`). [`CleanReport::evicted_entries`]
//! carries one [`EvictedEntry`] per evicted entry, labelled with
//! its priority mode.
//!
//! Both methods are idempotent on an absent cache root: calling
//! them when `<workspace>/.haz/cache` does not exist is a no-op,
//! not an error.

use std::path::{Path, PathBuf};

use haz_domain::settings::cache_clean::max_age::MaxAge;
use haz_domain::settings::cache_clean::max_size::MaxSize;
use haz_vfs::{EntryKind, FsError, WritableFilesystem};
use snafu::Snafu;

use crate::layout;
use crate::manifest::{HashFunctionLabel, Manifest};
use crate::writer::CacheWriter;

/// Failure modes shared by [`CacheWriter::clear`] and [`CacheWriter::clean`].
#[derive(Debug, Snafu)]
pub enum CleanError {
    /// Underlying filesystem error during the walk or removal.
    /// The wrapped [`FsError`] carries the specific path.
    #[snafu(display("filesystem error during cache invalidation: {source}"))]
    Io {
        /// The originating filesystem error.
        source: FsError,
    },
}

/// One best-effort failure collected during [`CacheWriter::clean`]
/// per `AUX-028`.
///
/// A `clean` run records these and keeps going rather than aborting,
/// so a single un-removable entry or unreadable shard does not
/// strand the rest of the plan. Each variant carries the path it
/// concerns and the originating [`FsError`].
#[derive(Debug, Snafu)]
pub enum CleanFailure {
    /// A cache shard directory could not be read during planning, so
    /// its entries were skipped.
    #[snafu(display("failed to read cache shard at: {}: {source}", path.display()))]
    Shard {
        /// Shard directory that could not be read.
        path: PathBuf,
        /// Originating filesystem error.
        source: FsError,
    },
    /// An entry's manifest could not be read (a real I/O error, not
    /// a simply-absent manifest), so the entry was skipped.
    #[snafu(display("failed to read entry manifest at: {}: {source}", path.display()))]
    Manifest {
        /// Manifest path that could not be read.
        path: PathBuf,
        /// Originating filesystem error.
        source: FsError,
    },
    /// A planned entry directory could not be removed.
    #[snafu(display("failed to remove cache entry at: {}: {source}", path.display()))]
    Entry {
        /// Entry directory that could not be removed.
        path: PathBuf,
        /// Originating filesystem error.
        source: FsError,
    },
    /// An orphan `.tmp-` directory could not be removed.
    #[snafu(display("failed to remove orphan tmp directory at: {}: {source}", path.display()))]
    Tmp {
        /// Tmp directory that could not be removed.
        path: PathBuf,
        /// Originating filesystem error.
        source: FsError,
    },
    /// An orphan `.restore-` directory could not be removed.
    #[snafu(display(
        "failed to remove orphan restore directory at: {}: {source}",
        path.display()
    ))]
    Restore {
        /// Restore directory that could not be removed.
        path: PathBuf,
        /// Originating filesystem error.
        source: FsError,
    },
}

/// Outcome of [`CacheWriter::clean`]: the `AUX-024` report plus the
/// best-effort [`CleanFailure`]s collected per `AUX-028`.
///
/// `failures` is empty on a fully-successful run. `report` is always
/// present and describes what was actually reclaimed (under
/// `--dry-run`, what would be reclaimed); a non-empty `failures`
/// means some planned work could not be completed, and the caller
/// surfaces those while still reporting the rest.
#[derive(Debug, Default)]
pub struct CleanOutcome {
    /// The `AUX-024` report for the work that succeeded.
    pub report: CleanReport,
    /// Per-item failures collected best-effort per `AUX-028`.
    pub failures: Vec<CleanFailure>,
}

/// Composable mode flags for [`CacheWriter::clean`] per `AUX-022`.
///
/// At least one of `soft`, `max_age`, `max_size` MUST be supplied
/// for the call to remove anything; the CLI layer enforces the
/// "must supply a mode" rule per `AUX-022`, so a [`CacheWriter::clean`]
/// call with all-false / all-`None` mode fields is a well-defined
/// no-op (the report shows zero counts).
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct CleanOptions {
    /// `CACHE-022` / `--soft`: reclaim objectively-stale state
    /// (manifest missing, unparseable, or schema-mismatched, plus
    /// orphan `.tmp-` and `.restore-` staging directories).
    pub soft: bool,
    /// `AUX-022` / `--max-age <DURATION>`: evict well-formed
    /// entries whose `created_at` is strictly older than
    /// `now_unix - max_age`. The cutoff is computed via
    /// [`u64::saturating_sub`], so a workspace clock that has not
    /// advanced past the threshold yields an empty eviction set
    /// rather than an overflow.
    pub max_age: Option<MaxAge>,
    /// `AUX-022` / `--max-size <BYTES>`: after `soft` and
    /// `max_age` have run, evict oldest-`created_at`-first
    /// well-formed survivors until the residual footprint is at
    /// or below `max_size`.
    pub max_size: Option<MaxSize>,
    /// `AUX-024` / `--dry-run`: compute the eviction set but DO
    /// NOT remove anything from disk. The returned [`CleanReport`]
    /// mirrors the non-dry-run report exactly, modulo the absence
    /// of on-disk changes.
    pub dry_run: bool,
    /// Reference "now" in Unix seconds since the epoch. Used as
    /// the right-hand side of the `AUX-023` step 4 cutoff
    /// (`now_unix - max_age`). Required to be non-zero when
    /// `max_age` is set; ignored otherwise. The injected value
    /// makes the operation deterministic under test.
    pub now_unix: u64,
}

/// Mode that accounted for evicting a given entry per `AUX-024`.
///
/// When more than one mode would match the same entry, the
/// highest-priority mode wins (in this declaration order).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EvictionMode {
    /// `CACHE-022` `--soft` pass: schema-stale or incomplete
    /// entry.
    Soft,
    /// `AUX-023` `--max-age` pass: well-formed but too old.
    MaxAge,
    /// `AUX-023` `--max-size` pass: well-formed but oldest among
    /// survivors of the previous passes when the residual
    /// footprint exceeded the bound.
    MaxSize,
}

/// Per-entry eviction detail surfaced for `AUX-024` dry-run
/// rendering.
///
/// One instance per evicted entry; the CLI displays the list
/// only under `--dry-run` per `AUX-024`, but the cache layer
/// always populates it (the cost is one `Vec` allocation of
/// modest size, and the audit-friendliness is worth it).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct EvictedEntry {
    /// First eight hex characters of the entry's cache key, per
    /// `AUX-024` ("first 8 hex characters suffice for
    /// human-readable output"). Sourced from the entry
    /// directory's basename when the manifest is unparseable.
    pub key_hex_prefix: String,
    /// Manifest's `created_at_unix` field when available; zero
    /// when the manifest is missing or unparseable.
    pub created_at_unix: u64,
    /// Manifest-declared footprint of the entry (`stdout_len +
    /// stderr_len + sum(outputs[].size)`). Zero when the manifest
    /// is unavailable.
    pub footprint: u64,
    /// Priority mode that accounted for this entry's eviction.
    pub matched_mode: EvictionMode,
}

/// Outcome of [`CacheWriter::clean`] per `AUX-024`.
///
/// All counts are zero on an absent cache root. Per-mode counts
/// follow the `AUX-024` priority rule (`soft` > `max_age` >
/// `max_size`): an entry never contributes to more than one
/// per-mode counter.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct CleanReport {
    /// `AUX-024` step 1: total number of entry directories the
    /// walk looked at (well-formed + corrupt). Excludes
    /// `.tmp-`/`.restore-` reclaimables.
    pub inspected: u64,
    /// `AUX-024` step 2: entries evicted under the `--soft` mode.
    pub evicted_by_soft: u64,
    /// `AUX-024` step 2: entries evicted under the `--max-age`
    /// mode.
    pub evicted_by_max_age: u64,
    /// `AUX-024` step 2: entries evicted under the `--max-size`
    /// mode.
    pub evicted_by_max_size: u64,
    /// `CACHE-022`: `.tmp-<key>-<random>` directories reclaimed.
    /// Always zero when `soft` is false.
    pub removed_tmp_dirs: u64,
    /// `CACHE-022`: `.restore-<key>-<random>` directories
    /// reclaimed. Always zero when `soft` is false.
    pub removed_restore_dirs: u64,
    /// `AUX-024` step 3: total bytes reclaimed (or projected
    /// under `--dry-run`). Sums each evicted entry's
    /// manifest-declared footprint; entries with no parseable
    /// manifest contribute zero. `.tmp-`/`.restore-` reclamations
    /// contribute zero.
    pub bytes_reclaimed: u64,
    /// One detail per evicted entry per `AUX-024`'s dry-run
    /// requirement (always populated regardless of `dry_run`).
    /// Sorted by `(matched_mode priority, created_at, key)` for
    /// determinism.
    pub evicted_entries: Vec<EvictedEntry>,
}

impl<Fs: WritableFilesystem> CacheWriter<Fs> {
    /// Remove every cache entry under
    /// `<workspace_root>/.haz/cache`, per `CACHE-021`.
    ///
    /// Idempotent on an absent cache root: calling `clear` when
    /// the cache tree does not exist returns `Ok(())`, not an
    /// error. The implementation only touches paths under the
    /// cache root; the rest of the workspace is left alone.
    ///
    /// # Errors
    ///
    /// Returns [`CleanError::Io`] wrapping the underlying
    /// [`FsError`] if the recursive removal fails for any reason
    /// other than "the cache root did not exist".
    pub fn clear(&self) -> Result<(), CleanError> {
        match self.fs().remove_dir_all(self.cache_root()) {
            Ok(()) | Err(FsError::NotFound { .. }) => Ok(()),
            Err(e) => Err(CleanError::Io { source: e }),
        }
    }

    /// Walk the cache root and reclaim every artefact named by the
    /// `CleanOptions` mode flags per `AUX-022..AUX-028`.
    ///
    /// Eviction priority follows `AUX-024` (`soft` > `max_age` >
    /// `max_size`); each entry contributes to exactly one per-mode
    /// count. Under `dry_run`, the eviction set is computed and
    /// reported but no file or directory is removed.
    ///
    /// Best-effort per `AUX-028`: a shard or manifest that cannot be
    /// read during planning, and an entry that cannot be removed
    /// during eviction, are recorded in [`CleanOutcome::failures`]
    /// and skipped rather than aborting the run. The returned
    /// [`CleanReport`] counts and `bytes_reclaimed` reflect what
    /// actually succeeded. Idempotent on an absent cache root.
    ///
    /// # Errors
    ///
    /// Returns [`CleanError::Io`] only when the cache root itself
    /// cannot be read, which prevents computing any eviction set.
    /// Per-item read/removal failures are NOT errors: they are
    /// collected in [`CleanOutcome::failures`]. Unparseable or
    /// missing manifests are not failures either; those entries
    /// surface as the `soft`-eligible "objectively stale" case.
    pub fn clean(&self, opts: &CleanOptions) -> Result<CleanOutcome, CleanError> {
        let Some(enumerated) = self.enumerate_for_clean()? else {
            return Ok(CleanOutcome::default());
        };
        let CleanEnumeration {
            well_formed,
            corrupt,
            tmp_paths,
            restore_paths,
            mut failures,
        } = enumerated;

        let mut report = CleanReport {
            inspected: (well_formed.len() + corrupt.len()) as u64,
            ..CleanReport::default()
        };

        let mut plan: Vec<PlannedEviction> = Vec::new();
        apply_soft_pass(opts, corrupt, &mut plan);
        let survivors = apply_max_age_pass(opts, well_formed, &mut plan);
        apply_max_size_pass(opts, survivors, &mut plan);

        // `AUX-028`: apply the plan best-effort. Under `--dry-run`,
        // `evict_path` is a no-op success, so every planned entry
        // counts as reclaimed and nothing is removed; on a real run
        // each removal either counts on success or is recorded as a
        // failure, never aborting the rest of the plan.
        let mut removed: Vec<EvictedEntry> = Vec::new();
        for planned in plan {
            match self.evict_path(opts.dry_run, &planned.path) {
                Ok(()) => {
                    bump_mode_count(&mut report, planned.detail.matched_mode);
                    report.bytes_reclaimed = report
                        .bytes_reclaimed
                        .saturating_add(planned.detail.footprint);
                    removed.push(planned.detail);
                }
                Err(source) => failures.push(CleanFailure::Entry {
                    path: planned.path,
                    source,
                }),
            }
        }

        if opts.soft {
            for path in tmp_paths {
                match self.evict_path(opts.dry_run, &path) {
                    Ok(()) => {
                        report.removed_tmp_dirs = report.removed_tmp_dirs.saturating_add(1);
                    }
                    Err(source) => failures.push(CleanFailure::Tmp { path, source }),
                }
            }
            for path in restore_paths {
                match self.evict_path(opts.dry_run, &path) {
                    Ok(()) => {
                        report.removed_restore_dirs = report.removed_restore_dirs.saturating_add(1);
                    }
                    Err(source) => failures.push(CleanFailure::Restore { path, source }),
                }
            }
        }

        report.evicted_entries = finalize_evicted_entries(removed);
        Ok(CleanOutcome { report, failures })
    }

    /// Remove `path`, unless `dry_run` is set, in which case report
    /// success without touching disk. Lets [`CacheWriter::clean`]
    /// drive the dry-run projection and the real eviction through a
    /// single best-effort loop.
    fn evict_path(&self, dry_run: bool, path: &Path) -> Result<(), FsError> {
        if dry_run {
            Ok(())
        } else {
            self.fs().remove_dir_all(path)
        }
    }

    fn enumerate_for_clean(&self) -> Result<Option<CleanEnumeration>, CleanError> {
        let cache_entries = match self.fs().read_dir(self.cache_root()) {
            Ok(es) => es,
            Err(FsError::NotFound { .. }) => return Ok(None),
            Err(e) => return Err(CleanError::Io { source: e }),
        };
        let mut e = CleanEnumeration::default();
        for cache_entry in cache_entries {
            let name = cache_entry
                .path
                .file_name()
                .map(|n| n.to_string_lossy().into_owned())
                .unwrap_or_default();

            if name.starts_with(".restore-") {
                e.restore_paths.push(cache_entry.path);
                continue;
            }
            if cache_entry.metadata.kind != EntryKind::Dir {
                continue;
            }
            self.clean_classify_shard(
                &cache_entry.path,
                &mut e.well_formed,
                &mut e.corrupt,
                &mut e.tmp_paths,
                &mut e.failures,
            );
        }
        Ok(Some(e))
    }

    fn clean_classify_shard(
        &self,
        shard_dir: &Path,
        well_formed: &mut Vec<EntryRecord>,
        corrupt: &mut Vec<EntryRecord>,
        tmp_paths: &mut Vec<PathBuf>,
        failures: &mut Vec<CleanFailure>,
    ) {
        // `AUX-028`: a shard we cannot read is recorded and skipped,
        // not fatal; the rest of the cache is still enumerated.
        let shard_entries = match self.fs().read_dir(shard_dir) {
            Ok(entries) => entries,
            Err(source) => {
                failures.push(CleanFailure::Shard {
                    path: shard_dir.to_path_buf(),
                    source,
                });
                return;
            }
        };
        for shard_entry in shard_entries {
            let sname = shard_entry
                .path
                .file_name()
                .map(|n| n.to_string_lossy().into_owned())
                .unwrap_or_default();

            if sname.starts_with(".tmp-") {
                tmp_paths.push(shard_entry.path);
                continue;
            }

            if shard_entry.metadata.kind != EntryKind::Dir {
                continue;
            }

            self.clean_classify_entry(&shard_entry.path, &sname, well_formed, corrupt, failures);
        }
    }

    fn clean_classify_entry(
        &self,
        entry_dir: &Path,
        basename: &str,
        well_formed: &mut Vec<EntryRecord>,
        corrupt: &mut Vec<EntryRecord>,
        failures: &mut Vec<CleanFailure>,
    ) {
        let key_hex_prefix: String = basename.chars().take(8).collect();
        let manifest_path = entry_dir.join(layout::MANIFEST_FILE_NAME);
        let bytes = match self.fs().read(&manifest_path) {
            Ok(b) => b,
            Err(FsError::NotFound { .. } | FsError::NotAFile { .. }) => {
                corrupt.push(EntryRecord {
                    path: entry_dir.to_path_buf(),
                    key_hex_prefix,
                    created_at_unix: 0,
                    footprint: 0,
                });
                return;
            }
            // `AUX-028`: an unreadable manifest (a real I/O error,
            // not a simply-absent one) is recorded and the entry
            // skipped, not fatal.
            Err(source) => {
                failures.push(CleanFailure::Manifest {
                    path: manifest_path,
                    source,
                });
                return;
            }
        };
        let Ok(manifest) = Manifest::from_json(&bytes) else {
            corrupt.push(EntryRecord {
                path: entry_dir.to_path_buf(),
                key_hex_prefix,
                created_at_unix: 0,
                footprint: 0,
            });
            return;
        };
        let chapter_ok = manifest.current_chapter_revision_matches();
        let hash_ok = HashFunctionLabel::from(self.hash_algo()) == manifest.hash_function;
        let footprint = manifest_footprint(&manifest);
        let record = EntryRecord {
            path: entry_dir.to_path_buf(),
            key_hex_prefix,
            created_at_unix: manifest.created_at_unix,
            footprint,
        };
        if chapter_ok && hash_ok {
            well_formed.push(record);
        } else {
            corrupt.push(record);
        }
    }
}

struct EntryRecord {
    path: PathBuf,
    key_hex_prefix: String,
    created_at_unix: u64,
    footprint: u64,
}

struct PlannedEviction {
    path: PathBuf,
    detail: EvictedEntry,
}

#[derive(Default)]
struct CleanEnumeration {
    well_formed: Vec<EntryRecord>,
    corrupt: Vec<EntryRecord>,
    tmp_paths: Vec<PathBuf>,
    restore_paths: Vec<PathBuf>,
    failures: Vec<CleanFailure>,
}

fn apply_soft_pass(
    opts: &CleanOptions,
    corrupt: Vec<EntryRecord>,
    plan: &mut Vec<PlannedEviction>,
) {
    if !opts.soft {
        return;
    }
    for c in corrupt {
        plan.push(PlannedEviction {
            path: c.path,
            detail: EvictedEntry {
                key_hex_prefix: c.key_hex_prefix,
                created_at_unix: c.created_at_unix,
                footprint: c.footprint,
                matched_mode: EvictionMode::Soft,
            },
        });
    }
}

fn apply_max_age_pass(
    opts: &CleanOptions,
    well_formed: Vec<EntryRecord>,
    plan: &mut Vec<PlannedEviction>,
) -> Vec<EntryRecord> {
    let Some(max_age) = opts.max_age else {
        return well_formed;
    };
    let cutoff = opts
        .now_unix
        .saturating_sub(max_age.as_duration().as_secs());
    let mut survivors: Vec<EntryRecord> = Vec::with_capacity(well_formed.len());
    for wf in well_formed {
        if wf.created_at_unix < cutoff {
            plan.push(PlannedEviction {
                path: wf.path.clone(),
                detail: EvictedEntry {
                    key_hex_prefix: wf.key_hex_prefix.clone(),
                    created_at_unix: wf.created_at_unix,
                    footprint: wf.footprint,
                    matched_mode: EvictionMode::MaxAge,
                },
            });
        } else {
            survivors.push(wf);
        }
    }
    survivors
}

fn apply_max_size_pass(
    opts: &CleanOptions,
    mut survivors: Vec<EntryRecord>,
    plan: &mut Vec<PlannedEviction>,
) {
    let Some(max_size) = opts.max_size else {
        return;
    };
    let limit = max_size.as_bytes();
    let total: u64 = survivors.iter().map(|e| e.footprint).sum();
    if total <= limit {
        return;
    }
    survivors.sort_by(|a, b| {
        a.created_at_unix
            .cmp(&b.created_at_unix)
            .then_with(|| a.key_hex_prefix.cmp(&b.key_hex_prefix))
    });
    let mut remaining = total;
    for wf in &survivors {
        if remaining <= limit {
            break;
        }
        plan.push(PlannedEviction {
            path: wf.path.clone(),
            detail: EvictedEntry {
                key_hex_prefix: wf.key_hex_prefix.clone(),
                created_at_unix: wf.created_at_unix,
                footprint: wf.footprint,
                matched_mode: EvictionMode::MaxSize,
            },
        });
        remaining = remaining.saturating_sub(wf.footprint);
    }
}

/// Increment the per-mode evicted counter for `mode`. Called once
/// per successful removal (and once per planned entry under
/// `--dry-run`), so the counts track what was actually reclaimed
/// per `AUX-028`.
fn bump_mode_count(report: &mut CleanReport, mode: EvictionMode) {
    match mode {
        EvictionMode::Soft => {
            report.evicted_by_soft = report.evicted_by_soft.saturating_add(1);
        }
        EvictionMode::MaxAge => {
            report.evicted_by_max_age = report.evicted_by_max_age.saturating_add(1);
        }
        EvictionMode::MaxSize => {
            report.evicted_by_max_size = report.evicted_by_max_size.saturating_add(1);
        }
    }
}

fn finalize_evicted_entries(mut details: Vec<EvictedEntry>) -> Vec<EvictedEntry> {
    details.sort_by(|a, b| {
        mode_rank(a.matched_mode)
            .cmp(&mode_rank(b.matched_mode))
            .then(a.created_at_unix.cmp(&b.created_at_unix))
            .then_with(|| a.key_hex_prefix.cmp(&b.key_hex_prefix))
    });
    details
}

fn manifest_footprint(m: &Manifest) -> u64 {
    let mut total = m.stdout_len.saturating_add(m.stderr_len);
    for o in &m.outputs {
        total = total.saturating_add(o.size);
    }
    total
}

const fn mode_rank(m: EvictionMode) -> u8 {
    match m {
        EvictionMode::Soft => 0,
        EvictionMode::MaxAge => 1,
        EvictionMode::MaxSize => 2,
    }
}

#[cfg(test)]
mod tests {
    use std::io::ErrorKind;
    use std::path::Path;

    use haz_domain::path::CanonicalPath;
    use haz_domain::settings::cache::HashAlgo;
    use haz_domain::settings::cache_clean::max_age::MaxAge;
    use haz_domain::settings::cache_clean::max_size::MaxSize;
    use haz_vfs::{Filesystem, WritableFilesystem};
    use haz_vfs_testing::{MemFaultOp, MemFilesystem};

    use crate::clean::{CleanError, CleanFailure, CleanOptions, CleanReport, EvictionMode};
    use crate::key::CacheKey;
    use crate::key::prefix::CHAPTER_REVISION;
    use crate::layout;
    use crate::manifest::{HashFunctionLabel, Manifest, OutputBlob};
    use crate::store::{StoreInputs, StoredOutput};
    use crate::writer::CacheWriter;

    fn cp(s: &str) -> CanonicalPath {
        CanonicalPath::parse_workspace_absolute(s)
            .expect("test helper expects a valid workspace-absolute path")
    }

    const WORKSPACE_ROOT: &str = "/ws";

    fn make_cache(fs: MemFilesystem, algo: HashAlgo) -> CacheWriter<MemFilesystem> {
        CacheWriter::new(fs, Path::new(WORKSPACE_ROOT), algo)
    }

    fn key_with_first_byte(first: u8) -> CacheKey {
        let mut bytes = [0u8; 32];
        bytes[0] = first;
        CacheKey::from_bytes(bytes)
    }

    fn store_entry_at(
        cache: &CacheWriter<MemFilesystem>,
        key: &CacheKey,
        rel: &str,
        bytes: &[u8],
        created_at_unix: u64,
    ) {
        let target = Path::new(WORKSPACE_ROOT).join(rel);
        let anchored = format!("/{rel}");
        cache.fs().create_dir_all(target.parent().unwrap()).unwrap();
        cache.fs().write_file(&target, bytes).unwrap();
        let outs = [StoredOutput {
            workspace_absolute_path: &anchored,
            on_disk_path: &target,
            mode: 0o644,
        }];
        cache
            .store(
                key,
                &StoreInputs {
                    outputs: &outs,
                    stdout: b"",
                    stderr: b"",
                    created_at_unix,
                },
            )
            .unwrap();
    }

    fn store_a_valid_entry(
        cache: &CacheWriter<MemFilesystem>,
        key: &CacheKey,
        rel: &str,
        bytes: &[u8],
    ) {
        store_entry_at(cache, key, rel, bytes, 0);
    }

    fn write_manifest_to_entry(
        cache: &CacheWriter<MemFilesystem>,
        key: &CacheKey,
        manifest: &Manifest,
    ) {
        cache
            .fs()
            .create_dir_all(&layout::entry_dir(cache.cache_root(), key))
            .unwrap();
        cache
            .fs()
            .write_file(
                &layout::manifest_path(cache.cache_root(), key),
                &manifest.to_json_bytes(),
            )
            .unwrap();
    }

    fn soft_only() -> CleanOptions {
        CleanOptions {
            soft: true,
            ..Default::default()
        }
    }

    // ---- clear ----

    #[test]
    fn cache_021_clear_empties_a_populated_cache() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);
        store_a_valid_entry(&cache, &key, "proj/out", b"x");

        assert!(
            cache.reader().lookup(&key).is_some(),
            "precondition: entry present"
        );
        cache.clear().unwrap();
        assert!(
            cache.reader().lookup(&key).is_none(),
            "lookup must be a miss after clear"
        );
    }

    #[test]
    fn cache_021_clear_on_fresh_cache_is_a_noop_not_an_error() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        cache.clear().unwrap();
    }

    #[test]
    fn cache_021_clear_does_not_touch_files_outside_cache_root() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        fs.add_file("/ws/unrelated.txt", b"keep me".to_vec())
            .unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);
        store_a_valid_entry(&cache, &key, "proj/out", b"x");

        cache.clear().unwrap();
        assert_eq!(
            cache.fs().read(Path::new("/ws/unrelated.txt")).unwrap(),
            b"keep me"
        );
    }

    // ---- clean: no-op cases ----

    #[test]
    fn cache_022_clean_soft_on_fresh_cache_is_a_noop_with_zero_counts() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let report = cache.clean(&soft_only()).unwrap().report;
        assert_eq!(report, CleanReport::default());
    }

    #[test]
    fn aux_022_clean_with_no_modes_is_a_noop_on_a_populated_cache() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);
        store_a_valid_entry(&cache, &key, "proj/out", b"x");

        let report = cache.clean(&CleanOptions::default()).unwrap().report;
        assert_eq!(report.evicted_by_soft, 0);
        assert_eq!(report.evicted_by_max_age, 0);
        assert_eq!(report.evicted_by_max_size, 0);
        assert_eq!(report.removed_tmp_dirs, 0);
        assert_eq!(report.removed_restore_dirs, 0);
        assert_eq!(report.inspected, 1);
        assert!(cache.reader().lookup(&key).is_some());
    }

    #[test]
    fn cache_022_clean_soft_keeps_a_valid_entry_intact() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);
        store_a_valid_entry(&cache, &key, "proj/out", b"x");

        let report = cache.clean(&soft_only()).unwrap().report;
        assert_eq!(report.evicted_by_soft, 0);
        assert!(cache.reader().lookup(&key).is_some());
    }

    // ---- clean --soft: schema mismatch ----

    #[test]
    fn cache_022_clean_soft_removes_entry_with_chapter_revision_mismatch() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);
        let manifest = Manifest {
            chapter_revision: CHAPTER_REVISION.saturating_add(1),
            hash_function: HashFunctionLabel::Blake3,
            key,
            outputs: vec![],
            stdout_len: 0,
            stderr_len: 0,
            stdout_hash: [0u8; 32],
            stderr_hash: [0u8; 32],
            exit_status: 0,
            created_at_unix: 0,
        };
        write_manifest_to_entry(&cache, &key, &manifest);
        assert!(
            cache
                .fs()
                .metadata(&layout::entry_dir(cache.cache_root(), &key))
                .is_ok()
        );

        let report = cache.clean(&soft_only()).unwrap().report;
        assert_eq!(report.evicted_by_soft, 1);
        assert!(
            cache
                .fs()
                .metadata(&layout::entry_dir(cache.cache_root(), &key))
                .is_err()
        );
    }

    #[test]
    fn cache_022_clean_soft_removes_entry_with_hash_function_mismatch() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);
        let manifest = Manifest {
            chapter_revision: CHAPTER_REVISION,
            hash_function: HashFunctionLabel::Sha256,
            key,
            outputs: vec![],
            stdout_len: 0,
            stderr_len: 0,
            stdout_hash: [0u8; 32],
            stderr_hash: [0u8; 32],
            exit_status: 0,
            created_at_unix: 0,
        };
        write_manifest_to_entry(&cache, &key, &manifest);

        let report = cache.clean(&soft_only()).unwrap().report;
        assert_eq!(report.evicted_by_soft, 1);
    }

    // ---- clean --soft: incomplete entry ----

    #[test]
    fn cache_022_clean_soft_removes_entry_without_a_manifest() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);
        cache
            .fs()
            .create_dir_all(&layout::entry_dir(cache.cache_root(), &key))
            .unwrap();

        let report = cache.clean(&soft_only()).unwrap().report;
        assert_eq!(report.evicted_by_soft, 1);
    }

    #[test]
    fn cache_022_clean_soft_removes_entry_with_unparseable_manifest() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);
        cache
            .fs()
            .create_dir_all(&layout::entry_dir(cache.cache_root(), &key))
            .unwrap();
        cache
            .fs()
            .write_file(
                &layout::manifest_path(cache.cache_root(), &key),
                b"this is not json",
            )
            .unwrap();

        let report = cache.clean(&soft_only()).unwrap().report;
        assert_eq!(report.evicted_by_soft, 1);
    }

    // ---- clean --soft: tmp / restore dirs ----

    #[test]
    fn cache_022_clean_soft_removes_store_tmp_directory() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);
        let tmp = layout::tmp_entry_dir(cache.cache_root(), &key, "abcdef");
        cache.fs().create_dir_all(&tmp).unwrap();
        cache
            .fs()
            .write_file(&tmp.join("manifest.json"), b"partial")
            .unwrap();

        let report = cache.clean(&soft_only()).unwrap().report;
        assert_eq!(report.removed_tmp_dirs, 1);
        assert!(cache.fs().metadata(&tmp).is_err());
    }

    #[test]
    fn cache_022_clean_soft_removes_restore_staging_directory() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);
        let staging = layout::restore_staging_dir(cache.cache_root(), &key, "feedface");
        cache.fs().create_dir_all(&staging).unwrap();
        cache
            .fs()
            .write_file(&staging.join("00000000"), b"leftover")
            .unwrap();

        let report = cache.clean(&soft_only()).unwrap().report;
        assert_eq!(report.removed_restore_dirs, 1);
        assert!(cache.fs().metadata(&staging).is_err());
    }

    // ---- clean --soft: mixed state ----

    #[test]
    fn cache_022_clean_soft_is_selective_when_mixed_state_is_present() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        let key_good = key_with_first_byte(0xAB);
        store_a_valid_entry(&cache, &key_good, "proj/out", b"x");

        let key_stale = key_with_first_byte(0xCD);
        let stale_manifest = Manifest {
            chapter_revision: CHAPTER_REVISION,
            hash_function: HashFunctionLabel::Sha256,
            key: key_stale,
            outputs: vec![],
            stdout_len: 0,
            stderr_len: 0,
            stdout_hash: [0u8; 32],
            stderr_hash: [0u8; 32],
            exit_status: 0,
            created_at_unix: 0,
        };
        write_manifest_to_entry(&cache, &key_stale, &stale_manifest);

        let key_tmp = key_with_first_byte(0xEF);
        let tmp = layout::tmp_entry_dir(cache.cache_root(), &key_tmp, "rnd1");
        cache.fs().create_dir_all(&tmp).unwrap();

        let key_restore = key_with_first_byte(0x12);
        let staging = layout::restore_staging_dir(cache.cache_root(), &key_restore, "rnd2");
        cache.fs().create_dir_all(&staging).unwrap();

        let report = cache.clean(&soft_only()).unwrap().report;
        assert_eq!(report.evicted_by_soft, 1);
        assert_eq!(report.removed_tmp_dirs, 1);
        assert_eq!(report.removed_restore_dirs, 1);
        assert!(cache.reader().lookup(&key_good).is_some());
    }

    #[test]
    fn cache_022_clean_soft_does_not_touch_files_outside_cache_root() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        fs.add_file("/ws/sibling.txt", b"don't touch".to_vec())
            .unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);
        cache
            .fs()
            .create_dir_all(&layout::entry_dir(cache.cache_root(), &key))
            .unwrap();
        cache.clean(&soft_only()).unwrap();
        assert_eq!(
            cache.fs().read(Path::new("/ws/sibling.txt")).unwrap(),
            b"don't touch"
        );
    }

    #[test]
    fn cache_022_clean_soft_does_not_inspect_blob_contents() {
        // `CACHE-022` only checks `chapter_revision`/
        // `hash_function` and manifest presence/parseability. A
        // blob-content mismatch is a lookup-time concern, not a
        // clean-soft concern.
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);
        let key = key_with_first_byte(0xAB);

        let manifest = Manifest {
            chapter_revision: CHAPTER_REVISION,
            hash_function: HashFunctionLabel::Blake3,
            key,
            outputs: vec![OutputBlob {
                workspace_absolute_path: cp("/proj/out"),
                content_hash: [0xAAu8; 32],
                size: 42,
                mode: 0o644,
            }],
            stdout_len: 0,
            stderr_len: 0,
            stdout_hash: [0u8; 32],
            stderr_hash: [0u8; 32],
            exit_status: 0,
            created_at_unix: 0,
        };
        write_manifest_to_entry(&cache, &key, &manifest);

        let report = cache.clean(&soft_only()).unwrap().report;
        assert_eq!(report.evicted_by_soft, 0);
        assert!(
            cache
                .fs()
                .metadata(&layout::entry_dir(cache.cache_root(), &key))
                .is_ok()
        );
    }

    // ---- clean --max-age ----

    #[test]
    fn aux_023_clean_max_age_evicts_entries_strictly_older_than_cutoff() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        let key_old = key_with_first_byte(0xAA);
        store_entry_at(&cache, &key_old, "proj/old", b"x", 100);
        let key_new = key_with_first_byte(0xBB);
        store_entry_at(&cache, &key_new, "proj/new", b"y", 260);

        let opts = CleanOptions {
            max_age: Some(MaxAge::parse("50s").unwrap()),
            now_unix: 300,
            ..Default::default()
        };
        let report = cache.clean(&opts).unwrap().report;
        assert_eq!(report.evicted_by_max_age, 1);
        assert_eq!(report.evicted_by_soft, 0);
        assert_eq!(report.evicted_by_max_size, 0);
        assert!(cache.reader().lookup(&key_old).is_none());
        assert!(cache.reader().lookup(&key_new).is_some());
    }

    #[test]
    fn aux_023_clean_max_age_keeps_entry_at_exactly_cutoff() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        let key = key_with_first_byte(0xAA);
        store_entry_at(&cache, &key, "proj/x", b"x", 100);

        let opts = CleanOptions {
            max_age: Some(MaxAge::parse("100s").unwrap()),
            now_unix: 200,
            ..Default::default()
        };
        let report = cache.clean(&opts).unwrap().report;
        assert_eq!(report.evicted_by_max_age, 0);
        assert!(cache.reader().lookup(&key).is_some());
    }

    #[test]
    fn aux_023_clean_max_age_ignores_corrupt_entries() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        let key_corrupt = key_with_first_byte(0xCC);
        let m = Manifest {
            chapter_revision: CHAPTER_REVISION,
            hash_function: HashFunctionLabel::Sha256,
            key: key_corrupt,
            outputs: vec![],
            stdout_len: 0,
            stderr_len: 0,
            stdout_hash: [0u8; 32],
            stderr_hash: [0u8; 32],
            exit_status: 0,
            created_at_unix: 0,
        };
        write_manifest_to_entry(&cache, &key_corrupt, &m);

        let key_stale = key_with_first_byte(0xAA);
        store_entry_at(&cache, &key_stale, "proj/x", b"x", 100);

        let opts = CleanOptions {
            max_age: Some(MaxAge::parse("50s").unwrap()),
            now_unix: 300,
            ..Default::default()
        };
        let report = cache.clean(&opts).unwrap().report;
        assert_eq!(report.evicted_by_max_age, 1);
        assert_eq!(report.evicted_by_soft, 0);
        assert!(
            cache
                .fs()
                .metadata(&layout::entry_dir(cache.cache_root(), &key_corrupt))
                .is_ok()
        );
        assert!(cache.reader().lookup(&key_stale).is_none());
    }

    // ---- clean --max-size ----

    #[test]
    fn aux_023_clean_max_size_is_noop_when_under_limit() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        let key = key_with_first_byte(0xAA);
        store_entry_at(&cache, &key, "proj/x", b"hello", 100);

        let opts = CleanOptions {
            max_size: Some(MaxSize::parse("1KB").unwrap()),
            ..Default::default()
        };
        let report = cache.clean(&opts).unwrap().report;
        assert_eq!(report.evicted_by_max_size, 0);
        assert!(cache.reader().lookup(&key).is_some());
    }

    #[test]
    fn aux_023_clean_max_size_evicts_oldest_first_until_at_or_below_limit() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        let bytes = b"0123456789"; // 10 bytes per entry footprint
        let key_old = key_with_first_byte(0x11);
        let key_mid = key_with_first_byte(0x22);
        let key_new = key_with_first_byte(0x33);
        store_entry_at(&cache, &key_old, "proj/a", bytes, 100);
        store_entry_at(&cache, &key_mid, "proj/b", bytes, 200);
        store_entry_at(&cache, &key_new, "proj/c", bytes, 300);

        // Total 30 bytes; limit 15. Evict oldest until <= 15.
        let opts = CleanOptions {
            max_size: Some(MaxSize::parse("15").unwrap()),
            ..Default::default()
        };
        let report = cache.clean(&opts).unwrap().report;
        assert_eq!(report.evicted_by_max_size, 2);
        assert!(cache.reader().lookup(&key_old).is_none());
        assert!(cache.reader().lookup(&key_mid).is_none());
        assert!(cache.reader().lookup(&key_new).is_some());
        assert_eq!(report.bytes_reclaimed, 20);
    }

    #[test]
    fn aux_023_clean_max_size_zero_evicts_every_well_formed_entry() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        let key = key_with_first_byte(0xAA);
        store_entry_at(&cache, &key, "proj/x", b"x", 100);

        let opts = CleanOptions {
            max_size: Some(MaxSize::parse("0").unwrap()),
            ..Default::default()
        };
        let report = cache.clean(&opts).unwrap().report;
        assert_eq!(report.evicted_by_max_size, 1);
        assert!(cache.reader().lookup(&key).is_none());
    }

    // ---- clean: mode composition ----

    #[test]
    fn aux_023_clean_soft_and_max_age_count_separately_per_priority() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        let key_corrupt = key_with_first_byte(0xCC);
        let m = Manifest {
            chapter_revision: CHAPTER_REVISION.saturating_add(1),
            hash_function: HashFunctionLabel::Blake3,
            key: key_corrupt,
            outputs: vec![],
            stdout_len: 0,
            stderr_len: 0,
            stdout_hash: [0u8; 32],
            stderr_hash: [0u8; 32],
            exit_status: 0,
            created_at_unix: 0,
        };
        write_manifest_to_entry(&cache, &key_corrupt, &m);

        let key_stale = key_with_first_byte(0xAA);
        store_entry_at(&cache, &key_stale, "proj/x", b"x", 100);

        let opts = CleanOptions {
            soft: true,
            max_age: Some(MaxAge::parse("50s").unwrap()),
            now_unix: 300,
            ..Default::default()
        };
        let report = cache.clean(&opts).unwrap().report;
        assert_eq!(report.evicted_by_soft, 1);
        assert_eq!(report.evicted_by_max_age, 1);
        assert_eq!(report.inspected, 2);
    }

    #[test]
    fn aux_023_clean_evicted_entries_sorted_by_mode_then_created_at() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        // Two well-formed entries that both fall under max-age.
        let key_a = key_with_first_byte(0x11);
        let key_b = key_with_first_byte(0x22);
        store_entry_at(&cache, &key_a, "proj/a", b"x", 100);
        store_entry_at(&cache, &key_b, "proj/b", b"y", 200);

        // One corrupt entry under --soft.
        let key_corrupt = key_with_first_byte(0xCC);
        let stale = Manifest {
            chapter_revision: CHAPTER_REVISION.saturating_add(1),
            hash_function: HashFunctionLabel::Blake3,
            key: key_corrupt,
            outputs: vec![],
            stdout_len: 0,
            stderr_len: 0,
            stdout_hash: [0u8; 32],
            stderr_hash: [0u8; 32],
            exit_status: 0,
            created_at_unix: 0,
        };
        write_manifest_to_entry(&cache, &key_corrupt, &stale);

        let opts = CleanOptions {
            soft: true,
            max_age: Some(MaxAge::parse("50s").unwrap()),
            now_unix: 300,
            ..Default::default()
        };
        let report = cache.clean(&opts).unwrap().report;
        assert_eq!(report.evicted_entries.len(), 3);
        assert_eq!(report.evicted_entries[0].matched_mode, EvictionMode::Soft);
        assert_eq!(report.evicted_entries[1].matched_mode, EvictionMode::MaxAge);
        assert_eq!(
            report.evicted_entries[1].created_at_unix, 100,
            "older max-age entry sorts before newer one"
        );
        assert_eq!(report.evicted_entries[2].matched_mode, EvictionMode::MaxAge);
        assert_eq!(report.evicted_entries[2].created_at_unix, 200);
    }

    // ---- clean: dry-run ----

    #[test]
    fn aux_023_clean_dry_run_does_not_modify_disk() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        let key = key_with_first_byte(0xAA);
        store_entry_at(&cache, &key, "proj/x", b"x", 100);

        let opts = CleanOptions {
            max_age: Some(MaxAge::parse("50s").unwrap()),
            now_unix: 300,
            dry_run: true,
            ..Default::default()
        };
        let report = cache.clean(&opts).unwrap().report;
        assert_eq!(report.evicted_by_max_age, 1);
        assert_eq!(report.evicted_entries.len(), 1);
        assert_eq!(report.evicted_entries[0].matched_mode, EvictionMode::MaxAge);
        assert!(
            cache.reader().lookup(&key).is_some(),
            "dry-run must leave the entry on disk"
        );
    }

    #[test]
    fn aux_023_clean_dry_run_under_soft_keeps_tmp_and_restore_dirs() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        let key_tmp = key_with_first_byte(0xEF);
        let tmp = layout::tmp_entry_dir(cache.cache_root(), &key_tmp, "rnd1");
        cache.fs().create_dir_all(&tmp).unwrap();

        let key_restore = key_with_first_byte(0x12);
        let staging = layout::restore_staging_dir(cache.cache_root(), &key_restore, "rnd2");
        cache.fs().create_dir_all(&staging).unwrap();

        let opts = CleanOptions {
            soft: true,
            dry_run: true,
            ..Default::default()
        };
        let report = cache.clean(&opts).unwrap().report;
        assert_eq!(report.removed_tmp_dirs, 1);
        assert_eq!(report.removed_restore_dirs, 1);
        assert!(cache.fs().metadata(&tmp).is_ok());
        assert!(cache.fs().metadata(&staging).is_ok());
    }

    #[test]
    fn aux_023_clean_bytes_reclaimed_sums_evicted_footprints() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache = make_cache(fs, HashAlgo::Blake3);

        let key_a = key_with_first_byte(0x11);
        store_entry_at(&cache, &key_a, "proj/a", b"hello", 100); // 5 bytes
        let key_b = key_with_first_byte(0x22);
        store_entry_at(&cache, &key_b, "proj/b", b"world!", 200); // 6 bytes

        let opts = CleanOptions {
            max_age: Some(MaxAge::parse("50s").unwrap()),
            now_unix: 300,
            ..Default::default()
        };
        let report = cache.clean(&opts).unwrap().report;
        assert_eq!(report.evicted_by_max_age, 2);
        assert_eq!(report.bytes_reclaimed, 11);
    }

    // ---- clean: AUX-028 best-effort ----

    /// A schema-stale (hash-function-mismatch) manifest, so the
    /// entry is objectively stale per `CACHE-022` (soft-eligible)
    /// without depending on the current `CHAPTER_REVISION`.
    fn corrupt_manifest(key: CacheKey) -> Manifest {
        Manifest {
            chapter_revision: CHAPTER_REVISION,
            hash_function: HashFunctionLabel::Sha256,
            key,
            outputs: vec![],
            stdout_len: 0,
            stderr_len: 0,
            stdout_hash: [0u8; 32],
            stderr_hash: [0u8; 32],
            exit_status: 0,
            created_at_unix: 0,
        }
    }

    #[test]
    fn aux_028_clean_soft_is_best_effort_when_one_removal_fails() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache_root = layout::cache_root(Path::new(WORKSPACE_ROOT));

        let key_stuck = key_with_first_byte(0xAA);
        let key_ok = key_with_first_byte(0xBB);
        let stuck_dir = layout::entry_dir(&cache_root, &key_stuck);
        let ok_dir = layout::entry_dir(&cache_root, &key_ok);
        fs.fail_on(
            MemFaultOp::RemoveDirAll,
            &stuck_dir,
            ErrorKind::PermissionDenied,
        );

        let cache = make_cache(fs, HashAlgo::Blake3);
        write_manifest_to_entry(&cache, &key_stuck, &corrupt_manifest(key_stuck));
        write_manifest_to_entry(&cache, &key_ok, &corrupt_manifest(key_ok));

        let outcome = cache.clean(&soft_only()).unwrap();

        // The removable entry is evicted despite the other failing.
        assert!(
            cache.fs().metadata(&stuck_dir).is_ok(),
            "stuck entry remains"
        );
        assert!(
            cache.fs().metadata(&ok_dir).is_err(),
            "removable entry evicted despite the other failing"
        );
        // Counts reflect what actually succeeded, not the plan.
        assert_eq!(outcome.report.evicted_by_soft, 1);
        assert_eq!(outcome.report.evicted_entries.len(), 1);
        assert_eq!(outcome.failures.len(), 1);
        assert!(matches!(
            &outcome.failures[0],
            CleanFailure::Entry { path, .. } if path == &stuck_dir
        ));
    }

    #[test]
    fn aux_028_clean_collects_every_removal_failure_not_just_the_first() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache_root = layout::cache_root(Path::new(WORKSPACE_ROOT));
        let key_a = key_with_first_byte(0xA1);
        let key_b = key_with_first_byte(0xB2);
        fs.fail_on(
            MemFaultOp::RemoveDirAll,
            layout::entry_dir(&cache_root, &key_a),
            ErrorKind::PermissionDenied,
        );
        fs.fail_on(
            MemFaultOp::RemoveDirAll,
            layout::entry_dir(&cache_root, &key_b),
            ErrorKind::PermissionDenied,
        );

        let cache = make_cache(fs, HashAlgo::Blake3);
        write_manifest_to_entry(&cache, &key_a, &corrupt_manifest(key_a));
        write_manifest_to_entry(&cache, &key_b, &corrupt_manifest(key_b));

        let outcome = cache.clean(&soft_only()).unwrap();
        assert_eq!(
            outcome.report.evicted_by_soft, 0,
            "neither removal succeeded"
        );
        assert_eq!(
            outcome.failures.len(),
            2,
            "both failures surfaced, not just the first"
        );
    }

    #[test]
    fn aux_028_clean_soft_best_effort_across_tmp_and_restore() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache_root = layout::cache_root(Path::new(WORKSPACE_ROOT));
        let tmp = layout::tmp_entry_dir(&cache_root, &key_with_first_byte(0xEF), "rnd1");
        let staging = layout::restore_staging_dir(&cache_root, &key_with_first_byte(0x12), "rnd2");
        fs.fail_on(MemFaultOp::RemoveDirAll, &tmp, ErrorKind::PermissionDenied);

        let cache = make_cache(fs, HashAlgo::Blake3);
        cache.fs().create_dir_all(&tmp).unwrap();
        cache.fs().create_dir_all(&staging).unwrap();

        let outcome = cache.clean(&soft_only()).unwrap();
        // The restore staging dir is still removed despite tmp failing.
        assert!(
            cache.fs().metadata(&staging).is_err(),
            "restore dir removed"
        );
        assert!(cache.fs().metadata(&tmp).is_ok(), "stuck tmp dir remains");
        assert_eq!(outcome.report.removed_restore_dirs, 1);
        assert_eq!(
            outcome.report.removed_tmp_dirs, 0,
            "tmp removal failed, not counted"
        );
        assert_eq!(outcome.failures.len(), 1);
        assert!(matches!(
            &outcome.failures[0],
            CleanFailure::Tmp { path, .. } if path == &tmp
        ));
    }

    #[test]
    fn aux_028_clean_skips_unreadable_shard_and_cleans_the_rest() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache_root = layout::cache_root(Path::new(WORKSPACE_ROOT));
        let key_blocked = key_with_first_byte(0xAA);
        let key_ok = key_with_first_byte(0xBB);
        let blocked_shard = layout::shard_dir(&cache_root, &key_blocked);
        fs.fail_on(
            MemFaultOp::ReadDir,
            &blocked_shard,
            ErrorKind::PermissionDenied,
        );

        let cache = make_cache(fs, HashAlgo::Blake3);
        write_manifest_to_entry(&cache, &key_blocked, &corrupt_manifest(key_blocked));
        write_manifest_to_entry(&cache, &key_ok, &corrupt_manifest(key_ok));

        let outcome = cache.clean(&soft_only()).unwrap();
        assert!(
            cache
                .fs()
                .metadata(&layout::entry_dir(&cache_root, &key_ok))
                .is_err(),
            "entry in the readable shard is evicted"
        );
        assert!(
            cache
                .fs()
                .metadata(&layout::entry_dir(&cache_root, &key_blocked))
                .is_ok(),
            "entry in the unreadable shard is left untouched"
        );
        assert_eq!(outcome.report.evicted_by_soft, 1);
        assert_eq!(outcome.failures.len(), 1);
        assert!(matches!(
            &outcome.failures[0],
            CleanFailure::Shard { path, .. } if path == &blocked_shard
        ));
    }

    #[test]
    fn aux_028_clean_skips_unreadable_manifest_and_cleans_the_rest() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache_root = layout::cache_root(Path::new(WORKSPACE_ROOT));
        let key_blocked = key_with_first_byte(0xAA);
        let key_ok = key_with_first_byte(0xCC);
        let blocked_manifest = layout::manifest_path(&cache_root, &key_blocked);
        fs.fail_on(
            MemFaultOp::Read,
            &blocked_manifest,
            ErrorKind::PermissionDenied,
        );

        let cache = make_cache(fs, HashAlgo::Blake3);
        write_manifest_to_entry(&cache, &key_blocked, &corrupt_manifest(key_blocked));
        write_manifest_to_entry(&cache, &key_ok, &corrupt_manifest(key_ok));

        let outcome = cache.clean(&soft_only()).unwrap();
        assert!(
            cache
                .fs()
                .metadata(&layout::entry_dir(&cache_root, &key_ok))
                .is_err(),
            "entry with a readable manifest is evicted"
        );
        assert!(
            cache
                .fs()
                .metadata(&layout::entry_dir(&cache_root, &key_blocked))
                .is_ok(),
            "entry with an unreadable manifest is left untouched"
        );
        assert_eq!(outcome.report.evicted_by_soft, 1);
        assert_eq!(outcome.failures.len(), 1);
        assert!(matches!(
            &outcome.failures[0],
            CleanFailure::Manifest { path, .. } if path == &blocked_manifest
        ));
    }

    #[test]
    fn aux_028_clean_unreadable_cache_root_is_fatal_with_no_report() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache_root = layout::cache_root(Path::new(WORKSPACE_ROOT));
        // The cache root cannot be enumerated, so no eviction set can
        // be computed: this is the one fatal case per AUX-028.
        fs.fail_on(
            MemFaultOp::ReadDir,
            &cache_root,
            ErrorKind::PermissionDenied,
        );
        let cache = make_cache(fs, HashAlgo::Blake3);

        let result = cache.clean(&soft_only());
        assert!(
            matches!(result, Err(CleanError::Io { .. })),
            "an unreadable cache root is fatal"
        );
    }

    #[test]
    fn aux_028_dry_run_records_no_failures_even_with_a_remove_fault() {
        let mut fs = MemFilesystem::new();
        fs.add_dir("/ws").unwrap();
        let cache_root = layout::cache_root(Path::new(WORKSPACE_ROOT));
        let key = key_with_first_byte(0xAA);
        let entry_dir = layout::entry_dir(&cache_root, &key);
        fs.fail_on(
            MemFaultOp::RemoveDirAll,
            &entry_dir,
            ErrorKind::PermissionDenied,
        );

        let cache = make_cache(fs, HashAlgo::Blake3);
        write_manifest_to_entry(&cache, &key, &corrupt_manifest(key));

        let opts = CleanOptions {
            soft: true,
            dry_run: true,
            ..Default::default()
        };
        let outcome = cache.clean(&opts).unwrap();
        assert!(
            outcome.failures.is_empty(),
            "dry-run never touches disk, so no removal failure is recorded"
        );
        assert_eq!(
            outcome.report.evicted_by_soft, 1,
            "dry-run still projects the plan"
        );
        assert!(
            cache.fs().metadata(&entry_dir).is_ok(),
            "dry-run leaves the entry on disk"
        );
    }
}