Skip to main content

leviath_cli/
runstate.rs

1//! On-disk run state for background agent executions.
2//!
3//! Each run lives under `~/.leviath/runs/<run-id>/` with:
4//! - `meta.json`    - run metadata, updated atomically (tmp + rename)
5//! - `output.log`  - append-only combined worker stdout (legacy/fallback)
6//! - `stages.json` - index of per-stage records
7//! - `stages/<idx>/output.log` - readable agent output for that stage
8//! - `stages/<idx>/logs.log`   - operational events + tool activity
9//! - `stages/<idx>/context.json` - context snapshot for that stage
10//!
11//! The dashboard's activity log is persisted separately at:
12//! - `~/.leviath/dashboard.log` - never cleared, appended across sessions
13//!
14//! # Who writes, and which copy is authoritative
15//!
16//! There are two answers to "what runs exist", and that is deliberate. The ECS
17//! world is the live one: it knows wait reasons and tick-fresh progress for the
18//! runs the daemon is holding right now, and `host.rs`'s `list()` reads it.
19//! The runs directory is the durable one: it survives a crash or a daemon that
20//! is not running, and `list_runs` below reads it. Disk lags the world by at
21//! most one persistence tick, so the two disagreeing is expected rather than a
22//! bug, and every reconciliation of that gap goes through `looks_abandoned`.
23//!
24//! The runtime's `persistence_bridge` is the only thing that writes a live
25//! run's state. The writers in this module are `#[cfg(test)]` so that stays
26//! true by compilation rather than by convention: a test can lay down a run
27//! directory to read back, and production has no second path to the same files.
28
29use std::path::{Path, PathBuf};
30use std::sync::Arc;
31use std::time::{SystemTime, UNIX_EPOCH};
32
33// The plain run-state data types (RunMeta, RunStatus, the snapshot structs, and
34// the per-stage records) live in `leviath_core::run_meta`. Re-exported here so
35// `crate::runstate::RunMeta` / `runstate::RunMeta` call sites across the cli
36// resolve. All on-disk IO for these types remains in this module.
37pub use leviath_core::run_meta::{
38    ContextSnapshot, RegionEntrySnapshot, RegionSnapshot, RunMeta, RunStatus, StageRecord,
39    StageRunStatus,
40};
41
42/// Atomically write a context snapshot for the run.
43///
44/// Test-only. Production writes go through the runtime's `persistence_bridge`,
45/// which is the sole writer of a live run's on-disk state; this exists so a
46/// test can lay down a run directory to read back. See the module doc.
47#[cfg(test)]
48pub fn write_context_snapshot(run_id: &str, snap: &ContextSnapshot) -> anyhow::Result<()> {
49    write_context_snapshot_to(&run_dir(run_id), snap)
50}
51
52/// Atomically write pre-serialized `json` to `path` (via a `.json.tmp`
53/// sibling + rename).
54///
55/// Non-generic (takes an already-serialized string) so it has a single
56/// monomorphization and every region - including the `std::fs` error `?`
57/// arms - is exercised by real tests. Serialization is performed by the
58/// callers, whose concrete production types
59/// (`ContextSnapshot`/`RunMeta`/`&[StageRecord]`) are provably infallible to
60/// serialize (see the `.expect` sites).
61/// Write `body` to `path` atomically, readable only by this user.
62///
63/// Not JSON-specific despite where it started: the final-output sidecar is raw
64/// content, and wants the same private-then-rename treatment for the same
65/// reason.
66fn write_private_atomic(path: &std::path::Path, body: &str) -> anyhow::Result<()> {
67    let tmp = path.with_extension("tmp");
68    // `write_private`: these files carry the run's full task prompt,
69    // conversation and tool output - and `meta.json` carries the webhook
70    // signing secret. They were written with a plain `fs::write` at the umask
71    // default (typically 0644), protected only by the 0700 on the enclosing run
72    // directory. That is one `chmod` away from being readable, and defence in
73    // depth is the whole point of a mode on the file itself.
74    leviath_sys::write_private(&tmp, body.as_bytes())?;
75    std::fs::rename(&tmp, path)?;
76    Ok(())
77}
78
79#[cfg(test)]
80fn write_context_snapshot_to(dir: &std::path::Path, snap: &ContextSnapshot) -> anyhow::Result<()> {
81    let json = serde_json::to_string_pretty(snap)
82        .expect("infallible: ContextSnapshot always serializes to JSON");
83    write_private_atomic(&dir.join("context.json"), &json)
84}
85
86/// Read the context snapshot for a run, if present.
87pub fn read_context_snapshot(run_id: &str) -> Option<ContextSnapshot> {
88    let path = run_dir(run_id).join("context.json");
89    let json = std::fs::read_to_string(&path).ok()?;
90    serde_json::from_str(&json).ok()
91}
92
93/// A parse cache keyed by a file's `(mtime, len)`: the file is re-read and
94/// re-parsed only when its stat changes.
95///
96/// For pollers reading run state on a tick. The dashboard synced at 10Hz by
97/// re-parsing every run's `meta.json`, `stages.json`, and whole
98/// `context.json`; with 50 runs on disk that was on the order of 100 MB/s of
99/// allocate-and-parse-and-free for files that change at most once per persist
100/// tick. A `stat` costs microseconds; this turns the steady-state tick into
101/// stats plus clones of shared `Arc`s.
102///
103/// `(mtime, len)` rather than mtime alone: the persistence lane's atomic
104/// rename gives every update a fresh temp inode and mtime, but coarse mtime
105/// granularity on some filesystems can miss two updates in the same instant -
106/// the length check catches most of those, and a same-length same-instant
107/// rewrite is indistinguishable anyway one tick later.
108pub struct StatCache<T> {
109    entries: std::collections::HashMap<PathBuf, (std::time::SystemTime, u64, Option<Arc<T>>)>,
110}
111
112impl<T> Default for StatCache<T> {
113    fn default() -> Self {
114        Self {
115            entries: std::collections::HashMap::new(),
116        }
117    }
118}
119
120impl<T> StatCache<T> {
121    /// The value parsed from `path`, re-reading only when the file's stat
122    /// changed since the last call. `None` when the file is missing,
123    /// unreadable, or `parse` rejects it - negative results are cached too, so
124    /// a persistently-bad file costs one stat per tick, not one parse.
125    pub fn get_with(
126        &mut self,
127        path: &Path,
128        parse: impl FnOnce(&str) -> Option<T>,
129    ) -> Option<Arc<T>> {
130        let Ok(meta) = std::fs::metadata(path) else {
131            self.entries.remove(path);
132            return None;
133        };
134        // A filesystem with no mtimes degrades to epoch (so length changes
135        // still refresh) rather than growing an unreachable error arm.
136        let stamp = (meta.modified().unwrap_or(std::time::UNIX_EPOCH), meta.len());
137        if let Some((mtime, len, value)) = self.entries.get(path)
138            && (*mtime, *len) == stamp
139        {
140            return value.clone();
141        }
142        let value = std::fs::read_to_string(path)
143            .ok()
144            .and_then(|text| parse(&text))
145            .map(Arc::new);
146        self.entries
147            .insert(path.to_path_buf(), (stamp.0, stamp.1, value.clone()));
148        value
149    }
150
151    /// Drop entries for files under runs that no longer exist, so a
152    /// long-lived poller's cache stays bounded by the live run set.
153    pub fn retain_under(&mut self, keep: &std::collections::HashSet<PathBuf>) {
154        self.entries.retain(|path, _| {
155            path.parent()
156                .is_some_and(|dir| keep.contains(&dir.to_path_buf()))
157        });
158    }
159}
160
161/// Read + parse a run's portable archive (`<run_dir>/run.lvr`), returning its
162/// records, or `None` if the archive is missing or unreadable.
163///
164/// Materializes the whole journal. For anything that only walks the timeline
165/// (the history API, journal search highlights), prefer [`visit_run_archive`]:
166/// a mature run's journal is tens of MB, and parsing it whole per request was
167/// the API's single largest transient allocation.
168pub fn read_run_archive(run_id: &str) -> Option<Vec<leviath_core::run_archive::RunRecord>> {
169    let path = run_dir(run_id).join("run.lvr");
170    let bytes = std::fs::read(&path).ok()?;
171    leviath_core::run_archive::read_archive(&mut bytes.as_slice())
172        .ok()
173        .map(|(_version, records)| records)
174}
175
176/// Stream a run's raw journal records through `visit`, one at a time, without
177/// materializing the archive. Same lenient tail handling as
178/// [`visit_run_archive`]. For consumers that inspect records rather than
179/// replayed points (journal search).
180pub fn visit_run_records(
181    run_id: &str,
182    visit: &mut dyn FnMut(&leviath_core::run_archive::RunRecord) -> std::ops::ControlFlow<()>,
183) -> Option<()> {
184    let path = run_dir(run_id).join("run.lvr");
185    let file = std::fs::File::open(&path).ok()?;
186    let mut reader = std::io::BufReader::with_capacity(64 * 1024, file);
187    leviath_core::run_archive::read_archive_start(&mut reader).ok()?;
188    while let Ok(Some(record)) = leviath_core::run_archive::read_record(&mut reader) {
189        if visit(&record).is_break() {
190            break;
191        }
192    }
193    Some(())
194}
195
196/// Stream a run's archive through a [`visit_points`] visitor without ever
197/// materializing the journal: one buffered pass over `run.lvr`, one record and
198/// one running window in memory. Returns `None` if the archive is missing or
199/// its preamble is invalid; a torn tail (a live run mid-append) just ends the
200/// walk with the points already visited.
201///
202/// [`visit_points`]: leviath_core::run_archive::visit_points
203pub fn visit_run_archive(
204    run_id: &str,
205    visit: &mut dyn FnMut(leviath_core::run_archive::PointRef<'_>) -> std::ops::ControlFlow<()>,
206) -> Option<()> {
207    let path = run_dir(run_id).join("run.lvr");
208    let file = std::fs::File::open(&path).ok()?;
209    let mut reader = std::io::BufReader::with_capacity(64 * 1024, file);
210    leviath_core::run_archive::visit_archive_points(&mut reader, visit).ok()
211}
212
213/// A run's context-window history: the full window (+ metadata) at each recorded
214/// point over time, oldest first. Empty when there's no readable archive.
215///
216/// Every point's `meta` is [`RunMeta::redacted`]. The journal stores `RunMeta`
217/// whole - including `callback_secret`, which the daemon needs to keep signing
218/// webhooks for a run it reloads - so a replayed point carries the secret unless
219/// it is stripped here. `GET /api/agents/{id}/context/history` serialized these
220/// points directly, which handed the webhook signing key to any holder of the
221/// API token: the same disclosure `redacted()` was introduced for on
222/// `/api/agents`, re-opened through the archive.
223///
224/// Redacted in this shared reader rather than in that one handler so the next
225/// consumer of a run's history inherits the fix instead of having to remember
226/// it. No caller needs the secret: the CLI printer, the dashboard, and the API
227/// all only display these points.
228pub fn context_history(run_id: &str) -> Vec<leviath_core::run_archive::RunPoint> {
229    read_run_archive(run_id)
230        .map(|records| leviath_core::run_archive::replay_points(&records))
231        .unwrap_or_default()
232        .into_iter()
233        .map(|point| leviath_core::run_archive::RunPoint {
234            meta: point.meta.redacted(),
235            ..point
236        })
237        .collect()
238}
239
240fn now_secs() -> i64 {
241    SystemTime::now()
242        .duration_since(UNIX_EPOCH)
243        .map(|d| d.as_secs() as i64)
244        .unwrap_or(0)
245}
246
247/// Inner implementation of `runs_dir`, parameterised so it can be tested
248/// without touching the process-global env. All callers go through `runs_dir`.
249///
250/// The fallback resolves through [`crate::config::leviath_home_dir`], not
251/// `dirs::home_dir` directly, so `LEVIATH_HOME` redirects the runs dir like it
252/// redirects the config, the control socket and the agents dir. With the raw
253/// OS home instead, a test that sets `LEVIATH_HOME` would be isolated
254/// everywhere *except* here and still write runs into the developer's real
255/// `~/.leviath/runs`. `LEVIATH_RUNS_DIR` wins over both.
256fn runs_dir_from(env_override: Option<&str>) -> PathBuf {
257    if let Some(dir) = env_override {
258        return PathBuf::from(dir);
259    }
260    leviath_core::paths::data_dir()
261        .unwrap_or_default()
262        .join("runs")
263}
264
265/// Directory where all run state is stored.
266pub fn runs_dir() -> PathBuf {
267    runs_dir_from(std::env::var("LEVIATH_RUNS_DIR").ok().as_deref())
268}
269
270/// Directory for a specific run.
271///
272/// A `run_id` that is not a single safe path component resolves to
273/// `<runs_dir>/<invalid>`, a name that cannot exist - so a caller that passes an
274/// attacker-supplied id gets a miss rather than a traversal. `run_id` reaches
275/// this from URL segments on `GET /api/agents/{id}/logs` and friends, where
276/// `Path::join` would otherwise happily accept `../../` or an absolute path.
277///
278/// Returning a definitely-missing path rather than an `Option` keeps every
279/// caller's "no such run" branch as the single failure path, instead of adding a
280/// second one that all of them would have to handle identically.
281pub fn run_dir(run_id: &str) -> PathBuf {
282    if !leviath_core::is_safe_path_component(run_id) {
283        tracing::warn!(run_id = %run_id, "rejected an unsafe run id");
284        return runs_dir().join("<invalid>");
285    }
286    runs_dir().join(run_id)
287}
288
289/// Inner implementation of `dashboard_log_path`, parameterised so it can be
290/// tested without touching the process-global env. All callers go through
291/// `dashboard_log_path`.
292fn dashboard_log_path_from(env_override: Option<&str>) -> PathBuf {
293    if let Some(path) = env_override {
294        return PathBuf::from(path);
295    }
296    leviath_core::paths::data_dir()
297        .unwrap_or_default()
298        .join("dashboard.log")
299}
300
301/// Path to the persistent dashboard activity log (~/.leviath/dashboard.log).
302///
303/// Honours the `LEVIATH_DASHBOARD_LOG_PATH` override when set (tests use it via
304/// `isolate_runs_dir_for_test`); otherwise resolves the real home-relative
305/// path. This function only *computes* a `PathBuf` - it never writes - so both
306/// arms are safe to exercise directly in tests. The write side
307/// ([`append_dashboard_log`] and `Dashboard::add_log`) is what must stay off
308/// the user's real log in tests: `append_dashboard_log`'s own tests set the
309/// override, and `Dashboard` carries an injected log path (a temp dir under
310/// `make_test_dashboard`) so no dashboard-input test ever appends to the real
311/// `~/.leviath/dashboard.log`.
312pub fn dashboard_log_path() -> PathBuf {
313    match std::env::var("LEVIATH_DASHBOARD_LOG_PATH") {
314        Ok(path) => dashboard_log_path_from(Some(&path)),
315        Err(_) => dashboard_log_path_from(None),
316    }
317}
318
319/// Append a timestamped line to the persistent dashboard activity log at the
320/// default [`dashboard_log_path`]. Silently ignores I/O errors - best-effort.
321pub fn append_dashboard_log(msg: &str) {
322    append_dashboard_log_to(&dashboard_log_path(), msg);
323}
324
325/// Append a timestamped line to the dashboard activity log at an explicit
326/// `path`. Silently ignores I/O errors - the dashboard log is best-effort.
327///
328/// The path is a parameter so `Dashboard` can inject a test-isolated log
329/// location, guaranteeing no dashboard-input test appends to the user's real
330/// `~/.leviath/dashboard.log` (see [`dashboard_log_path`]).
331pub fn append_dashboard_log_to(path: &Path, msg: &str) {
332    append_dashboard_log_capped(path, msg, DASHBOARD_LOG_MAX_BYTES);
333}
334
335/// The dashboard log is capped at this size; once the live file reaches it, the
336/// file is rolled (see [`roll_log_if_over_cap`]) so it can't grow without bound
337/// across a long-lived daemon's lifetime.
338const DASHBOARD_LOG_MAX_BYTES: u64 = 5 * 1024 * 1024;
339
340/// Append with an explicit cap (the public entry points use
341/// [`DASHBOARD_LOG_MAX_BYTES`]; tests pass a small cap to exercise rolling).
342fn append_dashboard_log_capped(path: &Path, msg: &str, max_bytes: u64) {
343    use std::io::Write;
344    // Ensure the parent directory exists (first-run case).
345    if let Some(parent) = path.parent() {
346        let _ = std::fs::create_dir_all(parent);
347    }
348    roll_log_if_over_cap(path, max_bytes);
349    if let Ok(mut file) = leviath_sys::open_private_append(path) {
350        let timestamp = chrono::Local::now().format("%Y-%m-%d %H:%M:%S");
351        let _ = writeln!(file, "{} {}", timestamp, msg);
352    }
353}
354
355/// The path the rolled (previous-generation) log is moved to: `<name>.1`.
356fn rolled_log_path(path: &Path) -> PathBuf {
357    let mut name = path.as_os_str().to_owned();
358    name.push(".1");
359    PathBuf::from(name)
360}
361
362/// Roll the live log to `<name>.1` once it reaches `max_bytes`, replacing any
363/// existing rolled file, so the live file restarts empty and at most one
364/// previous generation is retained (bounded ~2×cap on disk). Best-effort - a
365/// failed rename just leaves the log to keep growing rather than erroring.
366fn roll_log_if_over_cap(path: &Path, max_bytes: u64) {
367    let over = std::fs::metadata(path)
368        .map(|m| m.len() >= max_bytes)
369        .unwrap_or(false);
370    if over {
371        let _ = std::fs::rename(path, rolled_log_path(path));
372    }
373}
374
375/// How many random bits go in a run ID's suffix, rendered as 12 hex digits.
376/// Collisions only matter within one wall-clock second for one agent name, so 48
377/// bits is many orders of magnitude more than needed while staying short enough
378/// to read in `lev ps` and the dashboard.
379const RUN_ID_ENTROPY_BITS: u32 = 48;
380
381/// Generate a unique run ID: `<agent_name>-<timestamp>-<random>`.
382///
383/// The suffix is **random**, not derived. A derived suffix like
384/// `(now ^ (now >> 16) ^ counter)` over a process-local counter defends a
385/// `lev run --count N` batch inside one process but degenerates to a pure
386/// function of the current second across separate processes: three concurrent
387/// `lev run` invocations all mint `fetcher-1785127214-8b48` and silently share
388/// one run directory. Nothing downstream detects that - `create_dir_all` is a
389/// no-op on an existing directory and the persistence worker then
390/// last-writer-wins over `meta.json` / `context.json` / `run.lvr`, interleaving
391/// two runs' state irrecoverably.
392///
393/// The `<name>-<secs>-<hex>` shape is preserved: the timestamp keeps IDs sorting
394/// and reading chronologically, and the dashboard's short-ID display
395/// (`split('-').next_back()`) still lands on the unique component.
396///
397/// The name is folded to **ASCII** alphanumerics, which is stricter than it
398/// looks necessary. The id becomes a directory name, and [`run_dir`] resolves an
399/// id that is not a safe path component to `<invalid>`. A Unicode fold let an
400/// agent named `café` mint `café-...`: the daemon created that directory
401/// happily, and then every CLI read of the run looked in `<invalid>` and found
402/// nothing. The minter has to satisfy the rule the readers enforce.
403pub fn new_run_id(agent_name: &str) -> String {
404    use rand::RngExt as _;
405    let entropy: u64 = rand::rng().random::<u64>() >> (u64::BITS - RUN_ID_ENTROPY_BITS);
406    let safe_name = agent_name.replace(|c: char| !c.is_ascii_alphanumeric() && c != '-', "-");
407    format!("{}-{}-{:012x}", safe_name, now_secs(), entropy)
408}
409
410/// Create the run directory and write initial metadata.
411pub fn create_run(meta: &RunMeta) -> anyhow::Result<()> {
412    create_run_in(&run_dir(&meta.run_id), meta)
413}
414
415/// Create an explicit run directory and write initial metadata into it.
416///
417/// Callers that already know the directory should prefer this over
418/// [`create_run`], which resolves it from the home directory - the daemon's
419/// spawner stakes out the run dir under its own configured `runs_dir`.
420pub(crate) fn create_run_in(dir: &std::path::Path, meta: &RunMeta) -> anyhow::Result<()> {
421    std::fs::create_dir_all(dir)?;
422
423    // Restrict the run directory to owner-only (no-op on non-Unix).
424    let _ = leviath_sys::secure_dir_perms(dir);
425
426    write_meta_to(dir, meta)
427}
428
429/// Atomically write run metadata (write to tmp, then rename).
430pub fn write_meta(meta: &RunMeta) -> anyhow::Result<()> {
431    write_meta_to(&run_dir(&meta.run_id), meta)
432}
433
434/// Atomically write `meta.json` into an explicit run directory.
435///
436/// Callers that already know the directory should prefer this over
437/// [`write_meta`], which resolves it from the home directory - the daemon's
438/// recovery pass works from its configured `runs_dir` instead.
439pub(crate) fn write_meta_to(dir: &std::path::Path, meta: &RunMeta) -> anyhow::Result<()> {
440    let json =
441        serde_json::to_string_pretty(meta).expect("infallible: RunMeta always serializes to JSON");
442    write_private_atomic(&dir.join("meta.json"), &json)
443}
444
445/// Read run metadata for a given run ID.
446pub fn read_meta(run_id: &str) -> anyhow::Result<RunMeta> {
447    read_meta_from(&run_dir(run_id))
448}
449
450/// Read a run's final output, content included.
451///
452/// The descriptor in `meta.json` says whether there is one and how big it is;
453/// this fetches the bytes from the sidecar beside it. Returns `None` when the
454/// run produced no answer, or when the sidecar is missing (a run written by a
455/// build that stored the answer inline, or one whose directory was pruned).
456pub fn read_final_output(run_id: &str) -> Option<leviath_core::FinalOutput> {
457    let meta = read_meta(run_id).ok()?;
458    let descriptor = meta.final_output?;
459    let content = std::fs::read_to_string(final_output_path(&run_dir(run_id))).ok()?;
460    Some(leviath_core::FinalOutput {
461        content,
462        format: descriptor.format,
463        stage: descriptor.stage,
464        submitted_at: descriptor.submitted_at,
465        truncated: descriptor.truncated,
466        artifacts: descriptor.artifacts,
467    })
468}
469
470/// Where a run's answer lives, beside its `meta.json`.
471pub fn final_output_path(dir: &std::path::Path) -> PathBuf {
472    dir.join(leviath_core::FINAL_OUTPUT_FILE)
473}
474
475/// Write a run's answer to its sidecar, atomically.
476///
477/// Raw content with no wrapper: serving it is a read, and `lev result --raw` is
478/// a copy. The descriptor in `meta.json` is what says it exists.
479///
480/// Test-only; see [`write_context_snapshot`].
481#[cfg(test)]
482pub fn write_final_output(dir: &std::path::Path, content: &str) -> anyhow::Result<()> {
483    write_private_atomic(&final_output_path(dir), content)
484}
485
486/// Whether an on-disk run status means the run has finished and should be left
487/// alone. `Starting`/`Running`/`WaitingInput` are all "still going" as far as
488/// anything reading the runs dir is concerned.
489pub fn is_terminal_status(status: &RunStatus) -> bool {
490    matches!(
491        status,
492        RunStatus::Complete
493            | RunStatus::CompleteInteractive
494            | RunStatus::Error
495            | RunStatus::Cancelled
496    )
497}
498
499/// How long a run may claim to be live on disk, while the daemon is not holding
500/// it, before anything treats it as abandoned.
501///
502/// Comfortably longer than the persistence heartbeat, so a live-but-slow run (a
503/// long inference writes nothing else) is never mistaken for a dead one.
504pub const STALE_AFTER_SECS: i64 = 300;
505
506/// Whether a run that claims to be live on disk has nothing driving it: the
507/// daemon is not holding it *and* it has not moved in [`STALE_AFTER_SECS`].
508///
509/// `live` is the set of run ids the daemon reports hosting, or `None` when it
510/// gave no answer this poll. Both halves are needed and each is wrong on its
511/// own. An unreachable daemon reports an empty set, so the id check alone would
512/// condemn every healthy run the moment the daemon restarted. And a run parked
513/// on a long inference legitimately does not move for minutes, so the clock
514/// alone would condemn a run that is working. `None` therefore answers `false`
515/// for everything: no answer is not evidence.
516///
517/// Ages against `last_progress_at`, falling back to `updated_at` for runs
518/// written before that field existed. The fallback preserves the older, weaker
519/// behavior for old runs rather than declaring them all stale at once.
520///
521/// One definition, shared by the dashboard's STALE badge and by `lev ps --all`,
522/// so what an operator sees and what a harness reconciles against cannot drift.
523pub fn looks_abandoned(
524    meta: &RunMeta,
525    live: Option<&std::collections::HashSet<String>>,
526    now: i64,
527) -> bool {
528    let Some(live) = live else {
529        return false; // no answer from the daemon; assume nothing
530    };
531    if is_terminal_status(&meta.status) || live.contains(&meta.run_id) {
532        return false;
533    }
534    let moved_at = meta.last_progress_at.unwrap_or(meta.updated_at);
535    now.saturating_sub(moved_at) > STALE_AFTER_SECS
536}
537
538/// The outcome of forcing a run to a terminal state on disk.
539#[derive(Debug, Clone, Copy, PartialEq, Eq)]
540pub enum ForceCancelOutcome {
541    /// The run was live on disk and is now recorded terminal.
542    Terminated,
543    /// The run was already finished; nothing was written.
544    AlreadyTerminal,
545    /// No run directory with that id exists.
546    NoSuchRun,
547    /// The directory exists but its metadata could not be rewritten.
548    WriteFailed,
549}
550
551impl ForceCancelOutcome {
552    /// Whether the id named a run at all - i.e. whether the cancel had a target,
553    /// regardless of whether it needed to write anything.
554    pub fn found_run(&self) -> bool {
555        !matches!(self, Self::NoSuchRun)
556    }
557}
558
559/// Force a run's on-disk metadata to `Cancelled`, in the runs dir resolved from
560/// the environment. See [`force_cancel_in`].
561pub fn force_cancel(run_id: &str) -> ForceCancelOutcome {
562    force_cancel_in(&run_dir(run_id), now_secs())
563}
564
565/// Force the run in `run_dir` to `Cancelled`, stamping `updated_at` with `now`.
566///
567/// This is the floor under every kill path: it needs nothing but the filesystem,
568/// so it works for a run the daemon can't rebuild (blueprint deleted, metadata
569/// corrupt, died mid-spawn) and for a run whose daemon is gone entirely. Both
570/// the daemon's force-terminator seam and `lev cancel --force` route here so
571/// there is one definition of "terminated on disk".
572///
573/// A directory whose `meta.json` is missing or unparseable still gets a minimal
574/// `Cancelled` record written: such a run is otherwise skipped by `list_runs`,
575/// which makes it invisible *and* permanent.
576pub fn force_cancel_in(run_dir: &Path, now: i64) -> ForceCancelOutcome {
577    force_terminal_in(run_dir, RunStatus::Cancelled, None, now)
578}
579
580/// Force the run in `run_dir` to `Error` with `message`, stamping `updated_at`.
581///
582/// For the spawn that never became a run. The spawner stakes out the run
583/// directory and writes a `Starting` placeholder *before* building the agent, so
584/// a spawn that fails leaves something to diagnose - but `Starting` is not
585/// terminal, so that placeholder went on claiming the run was alive for ever,
586/// showing up in `lev ps` and the dashboard with nothing behind it (issue #190).
587/// Recording the failure where the placeholder is turns it into an answer.
588pub fn force_error_in(run_dir: &Path, message: &str, now: i64) -> ForceCancelOutcome {
589    force_terminal_in(run_dir, RunStatus::Error, Some(message.to_string()), now)
590}
591
592/// Rewrite the run in `run_dir` to a terminal `status`, attaching `error` when
593/// there is something to say. Shared by [`force_cancel_in`] and
594/// [`force_error_in`] so "terminated on disk" has one implementation.
595fn force_terminal_in(
596    run_dir: &Path,
597    status: RunStatus,
598    error: Option<String>,
599    now: i64,
600) -> ForceCancelOutcome {
601    if !run_dir.is_dir() {
602        return ForceCancelOutcome::NoSuchRun;
603    }
604    let run_id = run_dir
605        .file_name()
606        .map(|n| n.to_string_lossy().into_owned())
607        .unwrap_or_default();
608    let terminated = match read_meta_from(run_dir) {
609        Ok(meta) if is_terminal_status(&meta.status) => return ForceCancelOutcome::AlreadyTerminal,
610        Ok(meta) => RunMeta {
611            status,
612            updated_at: now,
613            // Keep whatever the run had already recorded when there is nothing
614            // new to say (the cancel path).
615            error: error.clone().or(meta.error),
616            ..meta
617        },
618        // Unreadable metadata: synthesize just enough to record the outcome. The
619        // run id is the directory name, which is the one field always recoverable.
620        Err(_) => RunMeta {
621            status,
622            updated_at: now,
623            error: Some(
624                error
625                    .clone()
626                    .unwrap_or_else(|| "run metadata was unreadable; cancelled".to_string()),
627            ),
628            ..RunMeta::new(
629                run_id.clone(),
630                run_id,
631                String::new(),
632                String::new(),
633                None,
634                String::new(),
635                0,
636            )
637        },
638    };
639    match write_meta_to(run_dir, &terminated) {
640        Ok(()) => ForceCancelOutcome::Terminated,
641        Err(e) => {
642            // Formatted outside the macro: a method call inside a `%field` is
643            // only evaluated when a subscriber visits the value, so it would go
644            // unexercised under the tests' no-op subscriber.
645            let path = run_dir.display().to_string();
646            tracing::warn!(
647                run_dir = %path,
648                error = %e,
649                "could not force a run to a terminal state on disk"
650            );
651            ForceCancelOutcome::WriteFailed
652        }
653    }
654}
655
656/// Read run metadata out of an explicit run directory (the daemon works from its
657/// own configured `runs_dir` rather than the home-resolved one).
658pub(crate) fn read_meta_from(dir: &std::path::Path) -> anyhow::Result<RunMeta> {
659    let path = dir.join("meta.json");
660    let json = std::fs::read_to_string(&path)?;
661    Ok(serde_json::from_str(&json)?)
662}
663
664/// Inner implementation of `list_runs`, parameterised so the early-return
665/// branch can be exercised in tests without deleting real on-disk state.
666fn list_runs_in_dir(dir: PathBuf) -> Vec<RunMeta> {
667    if !dir.exists() {
668        return Vec::new();
669    }
670
671    let mut runs = Vec::new();
672
673    if let Ok(entries) = std::fs::read_dir(&dir) {
674        for entry in entries.filter_map(|e| e.ok()) {
675            let meta_path = entry.path().join("meta.json");
676            if let Ok(json) = std::fs::read_to_string(&meta_path)
677                && let Ok(meta) = serde_json::from_str::<RunMeta>(&json)
678            {
679                runs.push(meta);
680            }
681        }
682    }
683
684    runs.sort_by_key(|r| std::cmp::Reverse(r.started_at));
685    runs
686}
687
688/// List all runs, sorted by started_at descending (most recent first).
689/// Silently skips any runs whose metadata cannot be read.
690pub fn list_runs() -> Vec<RunMeta> {
691    list_runs_in_dir(runs_dir())
692}
693
694/// [`list_runs`] through a [`StatCache`], for pollers: each `meta.json` is
695/// re-parsed only when its stat changes, and cache entries for deleted runs
696/// are dropped. Same ordering and skip-unreadable behavior as `list_runs`.
697pub fn list_runs_cached(cache: &mut StatCache<RunMeta>) -> Vec<Arc<RunMeta>> {
698    let dir = runs_dir();
699    let mut runs = Vec::new();
700    let mut live_dirs = std::collections::HashSet::new();
701    if let Ok(entries) = std::fs::read_dir(&dir) {
702        for entry in entries.filter_map(|e| e.ok()) {
703            live_dirs.insert(entry.path());
704            let meta_path = entry.path().join("meta.json");
705            if let Some(meta) = cache.get_with(&meta_path, |json| {
706                serde_json::from_str::<RunMeta>(json).ok()
707            }) {
708                runs.push(meta);
709            }
710        }
711    }
712    cache.retain_under(&live_dirs);
713    runs.sort_by_key(|r| std::cmp::Reverse(r.started_at));
714    runs
715}
716
717/// [`read_stages_index`] through a [`StatCache`], for pollers.
718pub fn read_stages_index_cached(
719    run_id: &str,
720    cache: &mut StatCache<Vec<StageRecord>>,
721) -> Vec<StageRecord> {
722    let path = run_dir(run_id).join("stages.json");
723    cache
724        .get_with(&path, |json| serde_json::from_str(json).ok())
725        .map(|records| records.as_ref().clone())
726        .unwrap_or_default()
727}
728
729/// [`read_context_snapshot`] through a [`StatCache`], for pollers. The
730/// snapshot is shared, not cloned: a context window is the largest thing in a
731/// run dir, and handing out copies per tick is the churn this cache removes.
732pub fn read_context_snapshot_cached(
733    run_id: &str,
734    cache: &mut StatCache<ContextSnapshot>,
735) -> Option<Arc<ContextSnapshot>> {
736    let path = run_dir(run_id).join("context.json");
737    cache.get_with(&path, |json| serde_json::from_str(json).ok())
738}
739
740/// Read the last `max_bytes` of any file on disk, returning UTF-8 text.
741/// If the file is smaller than `max_bytes` the whole file is returned.
742/// Partial UTF-8 at the truncation boundary is handled by skipping to the
743/// first newline.  Returns an empty string on any I/O error.
744pub fn tail_file(path: &std::path::Path, max_bytes: u64) -> String {
745    use std::io::{Read, Seek, SeekFrom};
746
747    let mut file = match std::fs::File::open(path) {
748        Ok(f) => f,
749        Err(_) => return String::new(),
750    };
751
752    // Use fstat on the open fd rather than a separate stat() call - avoids the
753    // TOCTOU window between existence check and metadata read. Falls back to 0
754    // (read everything) if fstat somehow fails on an already-open fd.
755    let file_size = file.metadata().map(|m| m.len()).unwrap_or(0);
756
757    if file_size <= max_bytes {
758        let mut buf = Vec::new();
759        let _ = file.read_to_end(&mut buf);
760        return String::from_utf8_lossy(&buf).to_string();
761    }
762
763    let offset = file_size - max_bytes;
764    let _ = file.seek(SeekFrom::Start(offset));
765
766    let mut buf = Vec::new();
767    let _ = file.read_to_end(&mut buf);
768
769    // Skip to the first newline so we don't emit a partial line at the start.
770    if let Some(nl) = buf.iter().position(|&b| b == b'\n') {
771        String::from_utf8_lossy(&buf[nl + 1..]).to_string()
772    } else {
773        String::from_utf8_lossy(&buf).to_string()
774    }
775}
776
777// ─── Per-stage persistence ────────────────────────────────────────────────────
778
779/// Directory for per-stage files within a run.
780pub fn stage_dir(run_id: &str, stage_idx: usize) -> PathBuf {
781    run_dir(run_id).join("stages").join(stage_idx.to_string())
782}
783
784/// Atomically write the stages index for a run.
785///
786/// Test-only; see [`write_context_snapshot`].
787#[cfg(test)]
788pub fn write_stages_index(run_id: &str, stages: &[StageRecord]) -> anyhow::Result<()> {
789    write_stages_index_to(&run_dir(run_id), stages)
790}
791
792#[cfg(test)]
793fn write_stages_index_to(dir: &std::path::Path, stages: &[StageRecord]) -> anyhow::Result<()> {
794    let json = serde_json::to_string_pretty(&stages)
795        .expect("infallible: StageRecord slice always serializes to JSON");
796    write_private_atomic(&dir.join("stages.json"), &json)
797}
798
799/// Read the stages index for a run, or return an empty vec on any error.
800pub fn read_stages_index(run_id: &str) -> Vec<StageRecord> {
801    let path = run_dir(run_id).join("stages.json");
802    let json = match std::fs::read_to_string(&path) {
803        Ok(j) => j,
804        Err(_) => return Vec::new(),
805    };
806    serde_json::from_str(&json).unwrap_or_default()
807}
808
809/// Ensure the per-stage directory exists (called before first write).
810#[cfg(test)]
811fn ensure_stage_dir(run_id: &str, stage_idx: usize) {
812    let dir = stage_dir(run_id, stage_idx);
813    let _ = leviath_sys::create_private_dir_all(&dir);
814}
815
816/// Append a line of readable agent output to the per-stage output log.
817///
818/// Test-only; see [`write_context_snapshot`].
819#[cfg(test)]
820pub fn append_stage_output(run_id: &str, stage_idx: usize, text: &str) {
821    use std::io::Write;
822    ensure_stage_dir(run_id, stage_idx);
823    let path = stage_dir(run_id, stage_idx).join("output.log");
824    if let Ok(mut file) = leviath_sys::open_private_append(&path) {
825        let _ = writeln!(file, "{}", text);
826    }
827}
828
829/// Append a line of operational/tool-activity log to the per-stage logs file.
830///
831/// Test-only; see [`write_context_snapshot`].
832#[cfg(test)]
833pub fn append_stage_log(run_id: &str, stage_idx: usize, text: &str) {
834    use std::io::Write;
835    ensure_stage_dir(run_id, stage_idx);
836    let path = stage_dir(run_id, stage_idx).join("logs.log");
837    if let Ok(mut file) = leviath_sys::open_private_append(&path) {
838        let _ = writeln!(file, "{}", text);
839    }
840}
841
842/// Atomically write a context snapshot for a specific stage.
843///
844/// Test-only; see [`write_context_snapshot`].
845#[cfg(test)]
846pub fn write_stage_context(
847    run_id: &str,
848    stage_idx: usize,
849    snap: &ContextSnapshot,
850) -> anyhow::Result<()> {
851    ensure_stage_dir(run_id, stage_idx);
852    write_context_snapshot_to(&stage_dir(run_id, stage_idx), snap)
853}
854
855/// Read the context snapshot for a specific stage, if present.
856pub fn read_stage_context(run_id: &str, stage_idx: usize) -> Option<ContextSnapshot> {
857    let path = stage_dir(run_id, stage_idx).join("context.json");
858    let json = std::fs::read_to_string(&path).ok()?;
859    serde_json::from_str(&json).ok()
860}
861
862/// Read the last `max_bytes` of the readable output log for a specific stage.
863pub fn tail_stage_output(run_id: &str, stage_idx: usize, max_bytes: u64) -> String {
864    tail_file(&stage_dir(run_id, stage_idx).join("output.log"), max_bytes)
865}
866
867/// Read the last `max_bytes` of the operational log for a specific stage.
868pub fn tail_stage_log(run_id: &str, stage_idx: usize, max_bytes: u64) -> String {
869    tail_file(&stage_dir(run_id, stage_idx).join("logs.log"), max_bytes)
870}
871
872/// Which stage's logs to read.
873#[derive(Debug, Clone, Copy, PartialEq, Eq)]
874pub enum StageSelector {
875    /// The stage the run is on now - the last entry in `stages.json`. What a
876    /// caller tailing a live run wants, and what `agent_result` already picked.
877    Current,
878    /// One specific stage by index.
879    Index(usize),
880    /// Every stage, oldest first, with a separator between them.
881    All,
882}
883
884/// Which of a stage's two logs to read.
885#[derive(Debug, Clone, Copy, PartialEq, Eq)]
886pub enum LogStream {
887    /// `output.log` - the assistant's readable output.
888    Output,
889    /// `logs.log` - operational lines: `[tool] …`, `[Tokens: …]`, `[error] …`.
890    Operational,
891}
892
893/// Read a run's logs, choosing the stage and the stream.
894///
895/// Exists because there were two answers in the codebase to "where is a run's
896/// output", and one of them was wrong: `GET /api/agents/{id}/logs` read
897/// `<run_dir>/output.log`, which nothing has ever written, so it returned an
898/// empty string for every run there has ever been. The real logs are per-stage,
899/// under `stages/<idx>/`. Routing both that handler and `agent_result` through
900/// here leaves one answer.
901///
902/// Stages come from `stages.json` rather than a `read_dir` of `stages/`, because
903/// that index is the record of which stages exist and in what order - the
904/// directory is just where their bytes landed.
905///
906/// `max_bytes` applies to what is returned, so for [`StageSelector::All`] it
907/// bounds the joined text rather than each stage separately: "the last N bytes
908/// of what you asked for" holds whatever the selector was.
909pub fn tail_run_logs(
910    run_id: &str,
911    selector: StageSelector,
912    stream: LogStream,
913    max_bytes: u64,
914) -> String {
915    let read = |idx: usize| match stream {
916        LogStream::Output => tail_stage_output(run_id, idx, max_bytes),
917        LogStream::Operational => tail_stage_log(run_id, idx, max_bytes),
918    };
919    let stages = read_stages_index(run_id);
920    match selector {
921        StageSelector::Index(idx) => read(idx),
922        StageSelector::Current => match stages.len().checked_sub(1) {
923            Some(last) => read(last),
924            // No stages recorded yet. Fall back to the legacy run-level file:
925            // nothing writes it today, but a run whose stage dirs were pruned
926            // still reads honestly instead of claiming it produced nothing.
927            None => tail_file(&run_dir(run_id).join("output.log"), max_bytes),
928        },
929        StageSelector::All => {
930            let joined = stages
931                .iter()
932                .map(|stage| {
933                    format!(
934                        "===== stage {}: {} =====\n{}",
935                        stage.index,
936                        stage.name,
937                        read(stage.index)
938                    )
939                })
940                .collect::<Vec<_>>()
941                .join("\n");
942            // Re-bound the join: each part was capped individually, so their
943            // concatenation can exceed the cap the caller asked for.
944            let start = leviath_core::text::floor_char_boundary(
945                &joined,
946                joined.len().saturating_sub(max_bytes as usize),
947            );
948            joined.split_at(start).1.to_string()
949        }
950    }
951}
952
953/// Build the isolated base directory for a run-state test and create its
954/// `runs/` subdir. Returned so the caller's closure can plant fixtures under it.
955///
956/// Rooted under `~/.leviath-test/rs-<hash>` rather than `std::env::temp_dir()`:
957/// some dashboard render tests display a real on-disk path inside a fixed-width
958/// terminal area and assert on a substring near its *end*, and macOS's real
959/// temp dir (`/var/folders/xy/.../T/`) is long enough to push realistic paths
960/// past the render width and truncate the asserted suffix. `unique` is hashed
961/// short for the same reason (test names run 60+ chars). `.leviath-test` is a
962/// sibling of `.leviath`, never read by `lev dash`/`lev serve`, so even if a
963/// killed test process skips cleanup it can't leak into the real dashboard.
964#[cfg(test)]
965fn make_runs_base_dir(unique: &str) -> std::path::PathBuf {
966    use std::hash::{Hash, Hasher};
967    let mut hasher = std::collections::hash_map::DefaultHasher::new();
968    unique.hash(&mut hasher);
969    let short = format!("{:x}", hasher.finish() & 0xffff_ffff);
970    let base_dir = dirs::home_dir()
971        .unwrap_or_default()
972        .join(".leviath-test")
973        .join(format!("rs-{short}"));
974    let _ = std::fs::create_dir_all(base_dir.join("runs"));
975    base_dir
976}
977
978/// The env overrides that point run-state I/O at `base_dir` instead of the
979/// real `~/.leviath/`. Handed to `temp_env` for scoped set-and-restore.
980#[cfg(test)]
981fn runs_dir_isolation_vars(
982    base_dir: &std::path::Path,
983) -> [(&'static str, Option<std::ffi::OsString>); 2] {
984    [
985        (
986            "LEVIATH_RUNS_DIR",
987            Some(base_dir.join("runs").into_os_string()),
988        ),
989        (
990            "LEVIATH_DASHBOARD_LOG_PATH",
991            Some(base_dir.join("dashboard.log").into_os_string()),
992        ),
993    ]
994}
995
996/// Runs `f` with `LEVIATH_RUNS_DIR`/`LEVIATH_DASHBOARD_LOG_PATH` pointed at a
997/// fresh isolated temp directory (passed to `f`), restoring them afterwards.
998/// Closure-scoped (not an RAII guard) because edition 2024 makes `set_var`
999/// `unsafe`, which the crate forbids; `temp_env` serializes it process-wide.
1000#[cfg(test)]
1001pub(crate) fn with_isolated_runs_dir<R>(unique: &str, f: impl FnOnce(&std::path::Path) -> R) -> R {
1002    let base_dir = make_runs_base_dir(unique);
1003    let result = temp_env::with_vars(runs_dir_isolation_vars(&base_dir), || f(&base_dir));
1004    let _ = std::fs::remove_dir_all(&base_dir);
1005    result
1006}
1007
1008/// Async counterpart of [`with_isolated_runs_dir`] for `#[tokio::test]`s.
1009#[cfg(test)]
1010pub(crate) async fn with_isolated_runs_dir_async<R, Fut>(
1011    unique: &str,
1012    f: impl FnOnce(std::path::PathBuf) -> Fut,
1013) -> R
1014where
1015    Fut: std::future::Future<Output = R>,
1016{
1017    let base_dir = make_runs_base_dir(unique);
1018    let result =
1019        temp_env::async_with_vars(runs_dir_isolation_vars(&base_dir), f(base_dir.clone())).await;
1020    let _ = std::fs::remove_dir_all(&base_dir);
1021    result
1022}
1023
1024#[cfg(test)]
1025mod tests {
1026    use super::*;
1027
1028    /// `run_id` arrives from URL segments on `GET /api/agents/{id}/logs` and
1029    /// friends. `Path::join` neither normalizes `..` nor resists an absolute
1030    /// path, so an unvalidated id read files anywhere. An unsafe one resolves to
1031    /// a name that cannot exist, giving the caller a plain miss.
1032    #[test]
1033    fn run_dir_refuses_an_unsafe_run_id() {
1034        crate::test_support::with_tracing(|| {
1035            for bad in ["../../etc", "/etc/passwd", "..", "a/b"] {
1036                let dir = run_dir(bad);
1037                let shown = dir.display().to_string();
1038                assert!(dir.ends_with("<invalid>"), "{bad} resolved to {shown}");
1039                assert!(!dir.exists(), "{bad} must not resolve to a real path");
1040            }
1041            // An ordinary id is untouched.
1042            assert!(run_dir("run-abc123").ends_with("run-abc123"));
1043        });
1044    }
1045
1046    // ─── looks_abandoned ────────────────────────────────────────────────────
1047
1048    /// A run claiming to be live on disk, last moved at 1000.
1049    fn live_on_disk(run_id: &str) -> RunMeta {
1050        let mut meta = RunMeta::new(
1051            run_id.to_string(),
1052            "coder".to_string(),
1053            "/agents/coder".to_string(),
1054            "t".to_string(),
1055            None,
1056            "/w".to_string(),
1057            1,
1058        );
1059        meta.status = RunStatus::Running;
1060        meta.updated_at = 1_000;
1061        meta.last_progress_at = Some(1_000);
1062        meta
1063    }
1064
1065    fn held(ids: &[&str]) -> std::collections::HashSet<String> {
1066        ids.iter().map(|s| (*s).to_string()).collect()
1067    }
1068
1069    /// The shape issue #202 reported: disk says running, the daemon is not
1070    /// hosting it, and it has not moved in a long time.
1071    #[test]
1072    fn a_run_nothing_is_driving_looks_abandoned() {
1073        let meta = live_on_disk("r1");
1074        assert!(looks_abandoned(
1075            &meta,
1076            Some(&held(&["other"])),
1077            1_000 + STALE_AFTER_SECS + 1
1078        ));
1079    }
1080
1081    /// The arm that decides whether a reconciler is safe to run at all. A daemon
1082    /// that is restarting gives no answer, which looks exactly like every run
1083    /// dying at once; anything that acted on it would cancel a whole factory.
1084    #[test]
1085    fn no_answer_from_the_daemon_condemns_nothing() {
1086        let meta = live_on_disk("r1");
1087        assert!(!looks_abandoned(
1088            &meta,
1089            None,
1090            1_000 + STALE_AFTER_SECS * 100
1091        ));
1092    }
1093
1094    #[test]
1095    fn a_run_the_daemon_is_hosting_is_never_abandoned() {
1096        let meta = live_on_disk("r1");
1097        assert!(!looks_abandoned(
1098            &meta,
1099            Some(&held(&["r1"])),
1100            1_000 + STALE_AFTER_SECS * 100
1101        ));
1102    }
1103
1104    /// A run parked on a long inference has not moved and is still working, so
1105    /// the window has to be wider than the persistence heartbeat.
1106    #[test]
1107    fn a_slow_run_inside_the_window_is_left_alone() {
1108        let meta = live_on_disk("r1");
1109        assert!(!looks_abandoned(
1110            &meta,
1111            Some(&held(&[])),
1112            1_000 + STALE_AFTER_SECS - 1
1113        ));
1114    }
1115
1116    /// A finished run is not abandoned, it is done. The daemon unloads it within
1117    /// seconds of it going terminal, so it is absent from the live set for the
1118    /// rest of time and would otherwise trip every other check here.
1119    #[test]
1120    fn a_finished_run_is_not_abandoned() {
1121        for status in [
1122            RunStatus::Complete,
1123            RunStatus::CompleteInteractive,
1124            RunStatus::Error,
1125            RunStatus::Cancelled,
1126        ] {
1127            let mut meta = live_on_disk("r1");
1128            meta.status = status.clone();
1129            assert!(
1130                !looks_abandoned(&meta, Some(&held(&[])), 1_000 + STALE_AFTER_SECS * 100),
1131                "{status} is finished, not abandoned"
1132            );
1133        }
1134    }
1135
1136    /// The progress stamp wins over the heartbeat. A wedged run keeps rewriting
1137    /// `updated_at` every 30 seconds, so judging on it would never age anything
1138    /// out, which is the reason issue #202 could not be fixed from meta.json
1139    /// before the stamp existed.
1140    #[test]
1141    fn a_fresh_heartbeat_does_not_rescue_a_run_that_stopped_moving() {
1142        let mut meta = live_on_disk("r1");
1143        let now = 1_000 + STALE_AFTER_SECS * 10;
1144        meta.updated_at = now; // the heartbeat, still beating
1145        meta.last_progress_at = Some(1_000); // but nothing has moved since 1000
1146        assert!(looks_abandoned(&meta, Some(&held(&[])), now));
1147    }
1148
1149    /// A run written before the stamp existed falls back to `updated_at`, so old
1150    /// runs keep the older, weaker behavior instead of all reading as stale.
1151    #[test]
1152    fn a_run_without_the_stamp_falls_back_to_updated_at() {
1153        let mut meta = live_on_disk("r1");
1154        meta.last_progress_at = None;
1155        meta.updated_at = 1_000;
1156        assert!(looks_abandoned(
1157            &meta,
1158            Some(&held(&[])),
1159            1_000 + STALE_AFTER_SECS + 1
1160        ));
1161        meta.updated_at = 1_000 + STALE_AFTER_SECS;
1162        assert!(!looks_abandoned(
1163            &meta,
1164            Some(&held(&[])),
1165            1_000 + STALE_AFTER_SECS + 1
1166        ));
1167    }
1168
1169    #[test]
1170    fn write_json_atomic_fs_write_failure() {
1171        // Drive the `std::fs::write(&tmp, json)?` error arm: writing the
1172        // `.json.tmp` sibling into a directory that does not exist fails.
1173        let path = std::path::Path::new("/nonexistent/leviath/runstate-cov/out.json");
1174        let result = write_private_atomic(path, "{}");
1175        assert!(result.is_err());
1176        assert!(!path.exists());
1177    }
1178
1179    // ─── RunStatus ──────────────────────────────────────────────────────────
1180
1181    #[test]
1182    fn run_status_serde_roundtrip() {
1183        for status in [
1184            RunStatus::Starting,
1185            RunStatus::Running,
1186            RunStatus::WaitingInput,
1187            RunStatus::Complete,
1188            RunStatus::CompleteInteractive,
1189            RunStatus::Paused,
1190            RunStatus::Error,
1191            RunStatus::Cancelled,
1192        ] {
1193            let json = serde_json::to_string(&status).unwrap();
1194            let back: RunStatus = serde_json::from_str(&json).unwrap();
1195            assert_eq!(status, back);
1196        }
1197    }
1198
1199    #[test]
1200    fn run_status_display() {
1201        assert_eq!(RunStatus::Starting.to_string(), "Starting");
1202        assert_eq!(RunStatus::Running.to_string(), "Running");
1203        assert_eq!(RunStatus::WaitingInput.to_string(), "WaitingInput");
1204        assert_eq!(RunStatus::Complete.to_string(), "Complete");
1205        assert_eq!(
1206            RunStatus::CompleteInteractive.to_string(),
1207            "CompleteInteractive"
1208        );
1209        assert_eq!(RunStatus::Paused.to_string(), "Paused");
1210        assert_eq!(RunStatus::Error.to_string(), "Error");
1211        assert_eq!(RunStatus::Cancelled.to_string(), "Cancelled");
1212    }
1213
1214    #[test]
1215    fn run_status_snake_case_serialization() {
1216        let json = serde_json::to_string(&RunStatus::WaitingInput).unwrap();
1217        assert_eq!(json, "\"waiting_input\"");
1218        let json = serde_json::to_string(&RunStatus::CompleteInteractive).unwrap();
1219        assert_eq!(json, "\"complete_interactive\"");
1220    }
1221
1222    // ─── StageRunStatus ─────────────────────────────────────────────────────
1223
1224    #[test]
1225    fn stage_run_status_serde_roundtrip() {
1226        for status in [
1227            StageRunStatus::Pending,
1228            StageRunStatus::Active,
1229            StageRunStatus::WaitingInput,
1230            StageRunStatus::Complete,
1231            StageRunStatus::Error,
1232        ] {
1233            let json = serde_json::to_string(&status).unwrap();
1234            let back: StageRunStatus = serde_json::from_str(&json).unwrap();
1235            assert_eq!(status, back);
1236        }
1237    }
1238
1239    #[test]
1240    fn stage_run_status_display() {
1241        assert_eq!(StageRunStatus::Pending.to_string(), "Pending");
1242        assert_eq!(StageRunStatus::Active.to_string(), "Active");
1243        assert_eq!(StageRunStatus::WaitingInput.to_string(), "WaitingInput");
1244        assert_eq!(StageRunStatus::Complete.to_string(), "Complete");
1245        assert_eq!(StageRunStatus::Error.to_string(), "Error");
1246    }
1247
1248    // ─── RunMeta ────────────────────────────────────────────────────────────
1249
1250    #[test]
1251    fn run_meta_new_defaults() {
1252        let meta = RunMeta::new(
1253            "run-1".into(),
1254            "agent".into(),
1255            "/path".into(),
1256            "do stuff".into(),
1257            Some("gpt-4".into()),
1258            "/work".into(),
1259            3,
1260        );
1261        assert_eq!(meta.run_id, "run-1");
1262        assert_eq!(meta.agent_name, "agent");
1263        assert_eq!(meta.task, "do stuff");
1264        assert_eq!(meta.model.as_deref(), Some("gpt-4"));
1265        assert_eq!(meta.num_stages, 3);
1266        assert_eq!(meta.status, RunStatus::Starting);
1267        assert_eq!(meta.pid, 0);
1268        assert_eq!(meta.stage_index, 0);
1269        assert!(meta.error.is_none());
1270        assert!(meta.title.is_none());
1271        assert!(meta.metadata.is_empty());
1272        assert!(meta.callback_url.is_none());
1273        assert!(meta.parent_run_id.is_none());
1274    }
1275
1276    #[test]
1277    fn run_meta_serde_roundtrip() {
1278        let meta = RunMeta::new(
1279            "test-run".into(),
1280            "test-agent".into(),
1281            "/agents/test".into(),
1282            "run tests".into(),
1283            None,
1284            "/tmp".into(),
1285            2,
1286        );
1287        let json = serde_json::to_string_pretty(&meta).unwrap();
1288        let back: RunMeta = serde_json::from_str(&json).unwrap();
1289        assert_eq!(back.run_id, "test-run");
1290        assert_eq!(back.agent_name, "test-agent");
1291        assert_eq!(back.num_stages, 2);
1292        assert!(back.model.is_none());
1293    }
1294
1295    #[test]
1296    fn run_meta_touch_updates_timestamp() {
1297        let mut meta = RunMeta::new(
1298            "r".into(),
1299            "a".into(),
1300            "/p".into(),
1301            "t".into(),
1302            None,
1303            "/w".into(),
1304            1,
1305        );
1306        let before = meta.updated_at;
1307        // Touch should update (or at least not decrease) updated_at
1308        meta.touch();
1309        assert!(meta.updated_at >= before);
1310    }
1311
1312    #[test]
1313    fn run_meta_optional_fields_deserialize() {
1314        // Simulate a meta.json without optional fields (e.g., from older version)
1315        let json = serde_json::json!({
1316            "run_id": "r1",
1317            "agent_name": "a",
1318            "agent_path": "/p",
1319            "task": "t",
1320            "model": null,
1321            "pid": 123,
1322            "status": "running",
1323            "current_stage": "init",
1324            "stage_index": 0,
1325            "num_stages": 1,
1326            "iteration": 0,
1327            "prompt_tokens": 0,
1328            "completion_tokens": 0,
1329            "workdir": "/w",
1330            "started_at": 1000,
1331            "updated_at": 1000,
1332            "error": null
1333        });
1334        let meta: RunMeta = serde_json::from_value(json).unwrap();
1335        assert_eq!(meta.cached_tokens, 0);
1336        assert!(meta.title.is_none());
1337        assert!(meta.metadata.is_empty());
1338        assert!(meta.callback_url.is_none());
1339        assert!(meta.parent_run_id.is_none());
1340        // A run written before the progress stamp existed has no answer, which is
1341        // why the field is an Option: `Some(0)` would read as "last moved in 1970"
1342        // and invite a reconciler to declare it abandoned.
1343        assert!(meta.last_progress_at.is_none());
1344    }
1345
1346    /// `pid` is written by every daemon there has ever been, and is always 0 in
1347    /// the shared world. A file that omits it entirely must still load, so the
1348    /// field can be dropped in a future major without stranding old runs.
1349    #[test]
1350    fn run_meta_without_a_pid_still_loads() {
1351        let json = serde_json::json!({
1352            "run_id": "r1",
1353            "agent_name": "a",
1354            "agent_path": "/p",
1355            "task": "t",
1356            "model": null,
1357            "status": "running",
1358            "current_stage": "init",
1359            "stage_index": 0,
1360            "num_stages": 1,
1361            "iteration": 0,
1362            "prompt_tokens": 0,
1363            "completion_tokens": 0,
1364            "workdir": "/w",
1365            "started_at": 1000,
1366            "updated_at": 1000,
1367            "error": null
1368        });
1369        let meta: RunMeta = serde_json::from_value(json).unwrap();
1370        assert_eq!(meta.pid, 0);
1371    }
1372
1373    // ─── StageRecord ────────────────────────────────────────────────────────
1374
1375    #[test]
1376    fn stage_record_new_defaults() {
1377        let rec = StageRecord::new("analyze".into(), 2);
1378        assert_eq!(rec.name, "analyze");
1379        assert_eq!(rec.index, 2);
1380        assert_eq!(rec.status, StageRunStatus::Pending);
1381        assert_eq!(rec.prompt_tokens, 0);
1382        assert_eq!(rec.completion_tokens, 0);
1383        assert_eq!(rec.cached_tokens, 0);
1384        assert!(rec.started_at.is_none());
1385        assert!(rec.ended_at.is_none());
1386    }
1387
1388    #[test]
1389    fn stage_record_serde_roundtrip() {
1390        let mut rec = StageRecord::new("build".into(), 0);
1391        rec.status = StageRunStatus::Complete;
1392        rec.prompt_tokens = 100;
1393        rec.started_at = Some(1000);
1394        rec.ended_at = Some(2000);
1395
1396        let json = serde_json::to_string(&rec).unwrap();
1397        let back: StageRecord = serde_json::from_str(&json).unwrap();
1398        assert_eq!(back.name, "build");
1399        assert_eq!(back.status, StageRunStatus::Complete);
1400        assert_eq!(back.prompt_tokens, 100);
1401        assert_eq!(back.started_at, Some(1000));
1402    }
1403
1404    // ─── RegionSnapshot / ContextSnapshot ───────────────────────────────────
1405
1406    #[test]
1407    fn region_snapshot_serde_roundtrip() {
1408        let snap = RegionSnapshot {
1409            name: "system".into(),
1410            kind: "pinned".into(),
1411            current_tokens: 100,
1412            max_tokens: 500,
1413            entries: vec![RegionEntrySnapshot {
1414                content: "You are helpful".into(),
1415                tokens: 3,
1416                kind: Default::default(),
1417                metadata: None,
1418                key: None,
1419                taint: Default::default(),
1420            }],
1421        };
1422        let json = serde_json::to_string(&snap).unwrap();
1423        let back: RegionSnapshot = serde_json::from_str(&json).unwrap();
1424        assert_eq!(back.name, "system");
1425        assert_eq!(back.entries.len(), 1);
1426        assert_eq!(back.entries[0].content, "You are helpful");
1427    }
1428
1429    #[test]
1430    fn region_snapshot_empty_entries_omitted() {
1431        let snap = RegionSnapshot {
1432            name: "empty".into(),
1433            kind: "temporary".into(),
1434            current_tokens: 0,
1435            max_tokens: 100,
1436            entries: vec![],
1437        };
1438        let json = serde_json::to_value(&snap).unwrap();
1439        assert!(json.get("entries").is_none());
1440    }
1441
1442    #[test]
1443    fn context_snapshot_serde_roundtrip() {
1444        let snap = ContextSnapshot {
1445            stage_name: "analyze".into(),
1446            total_tokens: 500,
1447            max_tokens: 8192,
1448            regions: vec![RegionSnapshot {
1449                name: "history".into(),
1450                kind: "sliding".into(),
1451                current_tokens: 300,
1452                max_tokens: 2000,
1453                entries: vec![],
1454            }],
1455        };
1456        let json = serde_json::to_string(&snap).unwrap();
1457        let back: ContextSnapshot = serde_json::from_str(&json).unwrap();
1458        assert_eq!(back.stage_name, "analyze");
1459        assert_eq!(back.total_tokens, 500);
1460        assert_eq!(back.regions.len(), 1);
1461    }
1462
1463    // ─── tail_file ──────────────────────────────────────────────────────────
1464
1465    #[test]
1466    fn tail_file_nonexistent_returns_empty() {
1467        let path = std::path::Path::new("/tmp/nonexistent-leviath-test-file.txt");
1468        assert_eq!(tail_file(path, 1024), "");
1469    }
1470
1471    #[test]
1472    fn tail_file_small_file_returns_all() {
1473        let dir = tempfile::tempdir().unwrap();
1474        let path = dir.path().join("small.txt");
1475        std::fs::write(&path, "line1\nline2\nline3\n").unwrap();
1476        let result = tail_file(&path, 1024);
1477        assert_eq!(result, "line1\nline2\nline3\n");
1478    }
1479
1480    #[test]
1481    fn tail_file_large_file_returns_tail() {
1482        let dir = tempfile::tempdir().unwrap();
1483        let path = dir.path().join("large.txt");
1484        let content = "abcdefghij\n".repeat(100); // 1100 bytes
1485        std::fs::write(&path, &content).unwrap();
1486        let result = tail_file(&path, 50);
1487        // Should be less than 50 bytes, starting from a line boundary
1488        assert!(result.len() <= 50);
1489        assert!(result.ends_with('\n'));
1490    }
1491
1492    // ─── read_final_output ──────────────────────────────────────────────────
1493
1494    /// The descriptor in `meta.json` and the sidecar beside it have to agree.
1495    /// Each way they can disagree reads as "no answer", which is the only safe
1496    /// reading: half an answer is worse than none.
1497    #[test]
1498    fn read_final_output_needs_both_the_descriptor_and_the_sidecar() {
1499        with_isolated_runs_dir("read-final-output", |_| {
1500            // No run at all.
1501            assert!(read_final_output("no-such-run").is_none());
1502
1503            // A run with no answer recorded.
1504            let meta = RunMeta::new(
1505                "run-silent".to_string(),
1506                "a".to_string(),
1507                "/p".to_string(),
1508                "t".to_string(),
1509                None,
1510                "/w".to_string(),
1511                1,
1512            );
1513            create_run(&meta).expect("run dir");
1514            assert!(read_final_output("run-silent").is_none());
1515
1516            // A descriptor saying there is one, with the sidecar missing: a run
1517            // written by a build that stored the answer inline, or one whose
1518            // directory was pruned.
1519            let answer = leviath_core::output::FinalOutput::new(
1520                "the answer",
1521                Some("markdown".to_string()),
1522                "present".to_string(),
1523                42,
1524            );
1525            let mut claimed = RunMeta::new(
1526                "run-claimed".to_string(),
1527                "a".to_string(),
1528                "/p".to_string(),
1529                "t".to_string(),
1530                None,
1531                "/w".to_string(),
1532                1,
1533            );
1534            claimed.final_output = Some(answer.descriptor());
1535            create_run(&claimed).expect("run dir");
1536            assert!(read_final_output("run-claimed").is_none());
1537
1538            // And both together: the answer comes back whole.
1539            write_final_output(&run_dir("run-claimed"), &answer.content).expect("sidecar");
1540            let read = read_final_output("run-claimed").expect("both halves are there");
1541            assert_eq!(read.content, "the answer");
1542            assert_eq!(read.format.as_deref(), Some("markdown"));
1543            assert_eq!(read.stage, "present");
1544        });
1545    }
1546
1547    // ─── new_run_id ─────────────────────────────────────────────────────────
1548
1549    #[test]
1550    fn new_run_id_contains_agent_name() {
1551        let id = new_run_id("my-agent");
1552        assert!(id.starts_with("my-agent-"));
1553    }
1554
1555    #[test]
1556    fn new_run_id_sanitizes_special_chars() {
1557        let id = new_run_id("agent with spaces!");
1558        assert!(!id.contains(' '));
1559        assert!(!id.contains('!'));
1560    }
1561
1562    /// The id becomes a directory name, and every reader resolves it through
1563    /// `is_safe_path_component`. A minted id that fails that check spawns a run
1564    /// the CLI can never read back, so the two rules have to agree whatever the
1565    /// blueprint calls itself.
1566    #[test]
1567    fn every_minted_run_id_is_a_safe_path_component() {
1568        for name in [
1569            "café",
1570            "日本語",
1571            "agent with spaces!",
1572            "../escape",
1573            "a/b",
1574            "..",
1575            "",
1576            "emoji-🚀-agent",
1577            "Ünïcödé",
1578        ] {
1579            let id = new_run_id(name);
1580            assert!(
1581                leviath_core::is_safe_path_component(&id),
1582                "agent {name:?} minted {id:?}, which run_dir resolves to <invalid>"
1583            );
1584        }
1585    }
1586
1587    #[test]
1588    fn new_run_id_is_unique_across_rapid_calls_in_same_second() {
1589        // `--count N` calls `new_run_id` N times in a tight loop, all within the
1590        // same wall-clock second.
1591        let ids: std::collections::HashSet<String> =
1592            (0..100).map(|_| new_run_id("same-agent")).collect();
1593        assert_eq!(ids.len(), 100);
1594    }
1595
1596    /// Split `<name>-<secs>-<hex>` from the right - the agent name itself may
1597    /// contain dashes.
1598    fn split_run_id(id: &str) -> (&str, &str) {
1599        let mut parts = id.rsplitn(3, '-');
1600        let suffix = parts.next().expect("run id has a suffix");
1601        let secs = parts.next().expect("run id has a timestamp");
1602        (secs, suffix)
1603    }
1604
1605    #[test]
1606    fn new_run_id_suffix_is_random_not_derived_from_the_clock() {
1607        // The collision this guards against is *across processes*: a suffix
1608        // derived as `(now ^ (now >> 16) ^ counter)` over a process-local
1609        // counter that every new process starts at 0 degenerates to a pure
1610        // function of the current second. Three concurrent `lev run`
1611        // invocations all mint `fetcher-1785127214-8b48` and silently share
1612        // one run directory. A fresh process has no state to vary, so the
1613        // property that has to hold is: IDs that share a timestamp still differ.
1614        let ids: Vec<String> = (0..200).map(|_| new_run_id("same-agent")).collect();
1615        let mut by_second: std::collections::HashMap<&str, Vec<&str>> =
1616            std::collections::HashMap::new();
1617        for id in &ids {
1618            let (secs, suffix) = split_run_id(id);
1619            by_second.entry(secs).or_default().push(suffix);
1620        }
1621        let mut largest = 0;
1622        for (secs, suffixes) in &by_second {
1623            let distinct: std::collections::HashSet<&&str> = suffixes.iter().collect();
1624            assert_eq!(
1625                distinct.len(),
1626                suffixes.len(),
1627                "two runs in second {secs} share a suffix: {suffixes:?}"
1628            );
1629            largest = largest.max(suffixes.len());
1630        }
1631        // 200 calls take microseconds, so they cannot all land in distinct
1632        // seconds - without this the assertion above would be vacuous.
1633        assert!(
1634            largest > 1,
1635            "expected IDs sharing a second, got {by_second:?}"
1636        );
1637    }
1638
1639    // ─── write_meta / read_meta roundtrip ───────────────────────────────────
1640
1641    #[test]
1642    fn write_and_read_meta_roundtrip() {
1643        // Isolated via `isolate_runs_dir_for_test` so write_meta/read_meta
1644        // never touch the real ~/.leviath/runs/ - the temp dir is removed
1645        // automatically when `_guard` drops, so no manual cleanup needed.
1646        with_isolated_runs_dir("write-and-read-meta-roundtrip", |_d| {
1647            let meta = RunMeta::new(
1648                "test-roundtrip-unit".into(),
1649                "test-agent".into(),
1650                "/agents/test".into(),
1651                "unit test".into(),
1652                Some("model-x".into()),
1653                "/tmp".into(),
1654                2,
1655            );
1656
1657            create_run(&meta).unwrap();
1658            let back = read_meta(&meta.run_id).unwrap();
1659            assert_eq!(back.run_id, "test-roundtrip-unit");
1660            assert_eq!(back.agent_name, "test-agent");
1661            assert_eq!(back.task, "unit test");
1662            assert_eq!(back.model.as_deref(), Some("model-x"));
1663        });
1664    }
1665
1666    #[test]
1667    fn read_meta_returns_err_on_corrupted_json() {
1668        // Exercises `read_meta_from`'s `serde_json::from_str(&json)?` Err
1669        // arm: a `meta.json` that exists but doesn't parse as a `RunMeta`.
1670        with_isolated_runs_dir("read-meta-returns-err-on-corrupted-json", |_d| {
1671            let run_id = "corrupted-meta-run";
1672            let dir = run_dir(run_id);
1673            std::fs::create_dir_all(&dir).unwrap();
1674            std::fs::write(dir.join("meta.json"), "not valid json").unwrap();
1675
1676            let result = read_meta(run_id);
1677            assert!(result.is_err());
1678        });
1679    }
1680
1681    // ─── write_stages_index / read_stages_index roundtrip ───────────────────
1682
1683    #[test]
1684    fn write_and_read_stages_index_roundtrip() {
1685        with_isolated_runs_dir("write-and-read-stages-index-roundtrip", |_d| {
1686            let run_id = "test-stages-idx-unit";
1687            let dir = run_dir(run_id);
1688            std::fs::create_dir_all(&dir).unwrap();
1689
1690            let stages = vec![
1691                StageRecord::new("init".into(), 0),
1692                StageRecord::new("process".into(), 1),
1693            ];
1694            write_stages_index(run_id, &stages).unwrap();
1695            let back = read_stages_index(run_id);
1696            assert_eq!(back.len(), 2);
1697            assert_eq!(back[0].name, "init");
1698            assert_eq!(back[1].name, "process");
1699        });
1700    }
1701
1702    #[test]
1703    fn read_stages_index_missing_returns_empty() {
1704        let back = read_stages_index("nonexistent-run-12345");
1705        assert!(back.is_empty());
1706    }
1707
1708    // ─── write/read context snapshot ────────────────────────────────────────
1709
1710    #[test]
1711    fn write_and_read_context_snapshot_roundtrip() {
1712        with_isolated_runs_dir("write-and-read-context-snapshot-roundtrip", |_d| {
1713            let run_id = "test-ctx-snap-unit";
1714            let dir = run_dir(run_id);
1715            std::fs::create_dir_all(&dir).unwrap();
1716
1717            let snap = ContextSnapshot {
1718                stage_name: "test".into(),
1719                total_tokens: 42,
1720                max_tokens: 8192,
1721                regions: vec![],
1722            };
1723            write_context_snapshot(run_id, &snap).unwrap();
1724            let back = read_context_snapshot(run_id).unwrap();
1725            assert_eq!(back.stage_name, "test");
1726            assert_eq!(back.total_tokens, 42);
1727        });
1728    }
1729
1730    #[test]
1731    fn read_context_snapshot_missing_returns_none() {
1732        assert!(read_context_snapshot("nonexistent-ctx-run").is_none());
1733    }
1734
1735    #[test]
1736    fn read_run_archive_roundtrips_and_context_history_replays() {
1737        with_isolated_runs_dir("read-run-archive-roundtrip", |_d| {
1738            use leviath_core::run_archive::{self, RunIdentity, RunRecord};
1739            let run_id = "archive-unit";
1740            std::fs::create_dir_all(run_dir(run_id)).unwrap();
1741            let mut buf = Vec::new();
1742            run_archive::write_archive_start(&mut buf, run_archive::RUN_ARCHIVE_VERSION).unwrap();
1743            let meta = RunMeta::new(
1744                run_id.to_string(),
1745                "a".to_string(),
1746                "/p".to_string(),
1747                "t".to_string(),
1748                None,
1749                "/w".to_string(),
1750                1,
1751            );
1752            run_archive::write_record(
1753                &mut buf,
1754                &RunRecord::Header {
1755                    identity: RunIdentity {
1756                        run_id: run_id.to_string(),
1757                        machine_id: "m".to_string(),
1758                        world_id: "w".to_string(),
1759                        created_at: 0,
1760                    },
1761                    meta: Box::new(meta),
1762                },
1763            )
1764            .unwrap();
1765            run_archive::write_record(
1766                &mut buf,
1767                &RunRecord::ContextCheckpoint {
1768                    snapshot: ContextSnapshot {
1769                        stage_name: "plan".to_string(),
1770                        total_tokens: 3,
1771                        max_tokens: 100,
1772                        regions: vec![],
1773                    },
1774                    at: 1,
1775                },
1776            )
1777            .unwrap();
1778            std::fs::write(run_dir(run_id).join("run.lvr"), &buf).unwrap();
1779
1780            let records = read_run_archive(run_id).expect("archive read");
1781            assert_eq!(records.len(), 2);
1782            let history = context_history(run_id);
1783            assert_eq!(history.len(), 1);
1784            assert_eq!(history[0].context.stage_name, "plan");
1785
1786            // The streaming visitors see the same journal without ever
1787            // materializing it.
1788            let mut streamed_points = Vec::new();
1789            visit_run_archive(run_id, &mut |p| {
1790                streamed_points.push((p.index, p.context.stage_name.to_string()));
1791                std::ops::ControlFlow::Continue(())
1792            })
1793            .expect("streamed replay");
1794            assert_eq!(streamed_points, vec![(0, "plan".to_string())]);
1795
1796            let mut streamed_records = 0usize;
1797            visit_run_records(run_id, &mut |_| {
1798                streamed_records += 1;
1799                std::ops::ControlFlow::Continue(())
1800            })
1801            .expect("streamed records");
1802            assert_eq!(streamed_records, 2);
1803
1804            // And a visitor can stop early.
1805            let mut first_only = 0usize;
1806            visit_run_records(run_id, &mut |_| {
1807                first_only += 1;
1808                std::ops::ControlFlow::Break(())
1809            })
1810            .expect("streamed records with break");
1811            assert_eq!(first_only, 1);
1812        });
1813    }
1814
1815    /// The stat cache's contract: parse once, serve from cache while the stat
1816    /// is unchanged, re-parse on change, cache negative results, and forget
1817    /// files that disappear.
1818    #[test]
1819    fn stat_cache_parses_once_per_stat_change() {
1820        let dir = tempfile::tempdir().unwrap();
1821        let path = dir.path().join("value.json");
1822        std::fs::write(&path, "41").unwrap();
1823        let mut cache: StatCache<i64> = StatCache::default();
1824        let mut parses = 0;
1825        let get = |cache: &mut StatCache<i64>, path: &std::path::Path, parses: &mut usize| {
1826            cache
1827                .get_with(path, |text| {
1828                    *parses += 1;
1829                    text.trim().parse().ok()
1830                })
1831                .map(|v| *v)
1832        };
1833
1834        assert_eq!(get(&mut cache, &path, &mut parses), Some(41));
1835        assert_eq!(get(&mut cache, &path, &mut parses), Some(41));
1836        assert_eq!(parses, 1, "the second read came from the cache");
1837
1838        // A same-length rewrite with a fresh mtime re-parses (the atomic-rename
1839        // writer always produces a new inode+mtime; simulate with a bumped
1840        // mtime via a rewrite of different content and length).
1841        std::fs::write(&path, "1234").unwrap();
1842        assert_eq!(get(&mut cache, &path, &mut parses), Some(1234));
1843        assert_eq!(parses, 2);
1844
1845        // Unparseable content is cached as a miss - one parse attempt, then
1846        // stat-only until the file changes again.
1847        std::fs::write(&path, "not a number").unwrap();
1848        assert_eq!(get(&mut cache, &path, &mut parses), None);
1849        assert_eq!(get(&mut cache, &path, &mut parses), None);
1850        assert_eq!(parses, 3, "the bad file was parsed once, not per tick");
1851
1852        // A deleted file is a miss and its entry is dropped.
1853        std::fs::remove_file(&path).unwrap();
1854        assert_eq!(get(&mut cache, &path, &mut parses), None);
1855        assert_eq!(parses, 3);
1856    }
1857
1858    #[test]
1859    fn stat_cache_retain_under_drops_dead_runs() {
1860        let dir = tempfile::tempdir().unwrap();
1861        let live = dir.path().join("live");
1862        let dead = dir.path().join("dead");
1863        std::fs::create_dir_all(&live).unwrap();
1864        std::fs::create_dir_all(&dead).unwrap();
1865        std::fs::write(live.join("meta.json"), "1").unwrap();
1866        std::fs::write(dead.join("meta.json"), "2").unwrap();
1867        let mut cache: StatCache<i64> = StatCache::default();
1868        cache.get_with(&live.join("meta.json"), |t| t.trim().parse().ok());
1869        cache.get_with(&dead.join("meta.json"), |t| t.trim().parse().ok());
1870        assert_eq!(cache.entries.len(), 2);
1871
1872        let keep: std::collections::HashSet<PathBuf> = [live.clone()].into_iter().collect();
1873        cache.retain_under(&keep);
1874        assert_eq!(cache.entries.len(), 1);
1875        assert!(cache.entries.contains_key(&live.join("meta.json")));
1876    }
1877
1878    /// The cached listing and per-run readers agree with their uncached
1879    /// counterparts, and serve repeat calls without re-parsing.
1880    #[test]
1881    fn cached_run_readers_match_the_uncached_ones() {
1882        with_isolated_runs_dir("cached-run-readers", |_d| {
1883            let meta = RunMeta::new(
1884                "cached-run".to_string(),
1885                "agent".to_string(),
1886                "/p".to_string(),
1887                "t".to_string(),
1888                None,
1889                "/w".to_string(),
1890                2,
1891            );
1892            create_run(&meta).unwrap();
1893            write_stages_index(
1894                "cached-run",
1895                &[leviath_core::run_meta::StageRecord::new(
1896                    "plan".to_string(),
1897                    0,
1898                )],
1899            )
1900            .unwrap();
1901            write_context_snapshot(
1902                "cached-run",
1903                &ContextSnapshot {
1904                    stage_name: "plan".to_string(),
1905                    total_tokens: 3,
1906                    max_tokens: 100,
1907                    regions: vec![],
1908                },
1909            )
1910            .unwrap();
1911
1912            let mut metas = StatCache::default();
1913            let mut stages = StatCache::default();
1914            let mut contexts = StatCache::default();
1915
1916            let listed = list_runs_cached(&mut metas);
1917            assert_eq!(listed.len(), 1);
1918            assert_eq!(listed[0].run_id, list_runs()[0].run_id);
1919
1920            let cached_stages = read_stages_index_cached("cached-run", &mut stages);
1921            let plain_stages = read_stages_index("cached-run");
1922            assert_eq!(cached_stages.len(), plain_stages.len());
1923            assert_eq!(cached_stages[0].name, plain_stages[0].name);
1924            let cached_ctx =
1925                read_context_snapshot_cached("cached-run", &mut contexts).expect("snapshot cached");
1926            assert_eq!(
1927                *cached_ctx,
1928                read_context_snapshot("cached-run").expect("snapshot read")
1929            );
1930            // A repeat serves the SAME Arc - the whole point of the cache.
1931            let again = read_context_snapshot_cached("cached-run", &mut contexts).unwrap();
1932            assert!(Arc::ptr_eq(&cached_ctx, &again));
1933
1934            // A second run makes the listing's ordering real: newest first,
1935            // same as the uncached listing.
1936            let mut second = RunMeta::new(
1937                "cached-run-2".to_string(),
1938                "agent".to_string(),
1939                "/p".to_string(),
1940                "t".to_string(),
1941                None,
1942                "/w".to_string(),
1943                1,
1944            );
1945            second.started_at += 100;
1946            create_run(&second).unwrap();
1947            let listed = list_runs_cached(&mut metas);
1948            assert_eq!(listed.len(), 2);
1949            assert_eq!(listed[0].run_id, "cached-run-2", "newest first");
1950
1951            // A run dir with a garbled meta.json is skipped, not fatal - and
1952            // skipped cheaply on every later tick (the negative result is
1953            // cached until the file changes).
1954            std::fs::create_dir_all(run_dir("garbled-run")).unwrap();
1955            std::fs::write(run_dir("garbled-run").join("meta.json"), "not json {{").unwrap();
1956            assert_eq!(list_runs_cached(&mut metas).len(), 2);
1957
1958            // A run whose dir disappears falls out of the cached listing.
1959            std::fs::remove_dir_all(run_dir("garbled-run")).unwrap();
1960            std::fs::remove_dir_all(run_dir("cached-run")).unwrap();
1961            std::fs::remove_dir_all(run_dir("cached-run-2")).unwrap();
1962            assert!(list_runs_cached(&mut metas).is_empty());
1963            assert!(read_stages_index_cached("cached-run", &mut stages).is_empty());
1964            assert!(read_context_snapshot_cached("cached-run", &mut contexts).is_none());
1965
1966            // And a missing runs DIRECTORY altogether lists nothing (the
1967            // read_dir-failed arm).
1968            std::fs::remove_dir_all(runs_dir()).unwrap();
1969            assert!(list_runs_cached(&mut metas).is_empty());
1970        });
1971    }
1972
1973    #[test]
1974    fn streaming_visitors_return_none_when_the_archive_is_missing() {
1975        with_isolated_runs_dir("streaming-visitors-missing", |_d| {
1976            // One visitor closure of each kind, shared across every call in
1977            // this test - the last pair of calls (on a real archive) executes
1978            // them, so a missing/invalid archive is proven by the counters
1979            // staying put, not by never-run closures.
1980            let points_seen = std::cell::Cell::new(0usize);
1981            let mut on_point = |_: leviath_core::run_archive::PointRef<'_>| {
1982                points_seen.set(points_seen.get() + 1);
1983                std::ops::ControlFlow::Continue(())
1984            };
1985            let records_seen = std::cell::Cell::new(0usize);
1986            let mut on_record = |_: &leviath_core::run_archive::RunRecord| {
1987                records_seen.set(records_seen.get() + 1);
1988                std::ops::ControlFlow::Continue(())
1989            };
1990
1991            assert!(visit_run_archive("no-such-run", &mut on_point).is_none());
1992            assert!(visit_run_records("no-such-run", &mut on_record).is_none());
1993            // A file that is not an archive fails the preamble check.
1994            let run_id = "bad-preamble";
1995            std::fs::create_dir_all(run_dir(run_id)).unwrap();
1996            std::fs::write(run_dir(run_id).join("run.lvr"), b"junk").unwrap();
1997            assert!(visit_run_archive(run_id, &mut on_point).is_none());
1998            assert!(visit_run_records(run_id, &mut on_record).is_none());
1999            assert_eq!((points_seen.get(), records_seen.get()), (0, 0));
2000
2001            // The same closures over a real archive do run.
2002            let real = "streaming-visitors-real";
2003            std::fs::create_dir_all(run_dir(real)).unwrap();
2004            write_minimal_archive(real);
2005            assert!(visit_run_archive(real, &mut on_point).is_some());
2006            assert!(visit_run_records(real, &mut on_record).is_some());
2007            assert_eq!(points_seen.get(), 1);
2008            assert_eq!(records_seen.get(), 2);
2009        });
2010    }
2011
2012    /// Write a two-record archive (Header + one ContextCheckpoint) for `run_id`.
2013    fn write_minimal_archive(run_id: &str) {
2014        use leviath_core::run_archive::{self, RunIdentity, RunRecord};
2015        let mut buf = Vec::new();
2016        run_archive::write_archive_start(&mut buf, run_archive::RUN_ARCHIVE_VERSION).unwrap();
2017        let meta = RunMeta::new(
2018            run_id.to_string(),
2019            "a".to_string(),
2020            "/p".to_string(),
2021            "t".to_string(),
2022            None,
2023            "/w".to_string(),
2024            1,
2025        );
2026        run_archive::write_record(
2027            &mut buf,
2028            &RunRecord::Header {
2029                identity: RunIdentity {
2030                    run_id: run_id.to_string(),
2031                    machine_id: "m".to_string(),
2032                    world_id: "w".to_string(),
2033                    created_at: 0,
2034                },
2035                meta: Box::new(meta),
2036            },
2037        )
2038        .unwrap();
2039        run_archive::write_record(
2040            &mut buf,
2041            &RunRecord::ContextCheckpoint {
2042                snapshot: ContextSnapshot {
2043                    stage_name: "plan".to_string(),
2044                    total_tokens: 3,
2045                    max_tokens: 100,
2046                    regions: vec![],
2047                },
2048                at: 1,
2049            },
2050        )
2051        .unwrap();
2052        std::fs::write(run_dir(run_id).join("run.lvr"), &buf).unwrap();
2053    }
2054
2055    /// The journal keeps `callback_secret` (the daemon re-signs webhooks for a
2056    /// run it reloads), so a replayed point carries it unless the reader strips
2057    /// it. `GET /api/agents/{id}/context/history` serves these points straight
2058    /// out, which handed the webhook signing key to any API token holder.
2059    ///
2060    /// Asserts against the *archive* as well as the history, so the test still
2061    /// means something if the journal ever stops storing the secret: were that
2062    /// to happen, the first assertion fails rather than the second silently
2063    /// passing on a field that is no longer there to leak.
2064    #[test]
2065    fn context_history_redacts_the_webhook_secret_the_journal_keeps() {
2066        with_isolated_runs_dir("context-history-redacts-secret", |_d| {
2067            use leviath_core::run_archive::{self, RunIdentity, RunRecord};
2068            let run_id = "archive-secret-unit";
2069            std::fs::create_dir_all(run_dir(run_id)).unwrap();
2070            let mut buf = Vec::new();
2071            run_archive::write_archive_start(&mut buf, run_archive::RUN_ARCHIVE_VERSION).unwrap();
2072            let mut meta = RunMeta::new(
2073                run_id.to_string(),
2074                "a".to_string(),
2075                "/p".to_string(),
2076                "t".to_string(),
2077                None,
2078                "/w".to_string(),
2079                1,
2080            );
2081            meta.callback_url = Some("https://example.invalid/hook".to_string());
2082            meta.callback_secret = Some("super-secret-signing-key".to_string());
2083            run_archive::write_record(
2084                &mut buf,
2085                &RunRecord::Header {
2086                    identity: RunIdentity {
2087                        run_id: run_id.to_string(),
2088                        machine_id: "m".to_string(),
2089                        world_id: "w".to_string(),
2090                        created_at: 0,
2091                    },
2092                    meta: Box::new(meta),
2093                },
2094            )
2095            .unwrap();
2096            run_archive::write_record(
2097                &mut buf,
2098                &RunRecord::ContextCheckpoint {
2099                    snapshot: ContextSnapshot {
2100                        stage_name: "plan".to_string(),
2101                        total_tokens: 3,
2102                        max_tokens: 100,
2103                        regions: vec![],
2104                    },
2105                    at: 1,
2106                },
2107            )
2108            .unwrap();
2109            std::fs::write(run_dir(run_id).join("run.lvr"), &buf).unwrap();
2110
2111            // The secret really is on disk, so redaction has work to do. Read
2112            // the raw bytes rather than matching over parsed records: a match
2113            // that stops at the Header leaves its other arm unreachable, and
2114            // this says the thing that actually matters anyway.
2115            let raw = std::fs::read(run_dir(run_id).join("run.lvr")).unwrap();
2116            assert!(String::from_utf8_lossy(&raw).contains("super-secret-signing-key"));
2117
2118            // What the reader hands out has it stripped, and keeps the rest.
2119            let history = context_history(run_id);
2120            assert_eq!(history.len(), 1);
2121            assert_eq!(history[0].meta.callback_secret, None);
2122            assert_eq!(
2123                history[0].meta.callback_url.as_deref(),
2124                Some("https://example.invalid/hook")
2125            );
2126            assert_eq!(history[0].context.stage_name, "plan");
2127        });
2128    }
2129
2130    #[test]
2131    fn read_run_archive_missing_or_corrupt_returns_none() {
2132        with_isolated_runs_dir("read-run-archive-corrupt", |_d| {
2133            // Missing archive.
2134            assert!(read_run_archive("no-such-archive-run").is_none());
2135            assert!(context_history("no-such-archive-run").is_empty());
2136            // Corrupt archive (bad magic) → None, not a panic.
2137            let run_id = "corrupt-archive-unit";
2138            std::fs::create_dir_all(run_dir(run_id)).unwrap();
2139            std::fs::write(run_dir(run_id).join("run.lvr"), b"not an archive").unwrap();
2140            assert!(read_run_archive(run_id).is_none());
2141            assert!(context_history(run_id).is_empty());
2142        });
2143    }
2144
2145    // ─── stage_dir / append_stage_output / append_stage_log ─────────────────
2146
2147    #[test]
2148    fn stage_dir_path_structure() {
2149        let path = stage_dir("run-abc", 2);
2150        assert!(path.ends_with("stages/2"));
2151        assert!(path.to_str().unwrap().contains("run-abc"));
2152    }
2153
2154    #[test]
2155    fn append_and_tail_stage_output() {
2156        with_isolated_runs_dir("append-and-tail-stage-output", |_d| {
2157            let run_id = "test-stage-output-unit";
2158            append_stage_output(run_id, 0, "line 1");
2159            append_stage_output(run_id, 0, "line 2");
2160            let output = tail_stage_output(run_id, 0, 4096);
2161            assert!(output.contains("line 1"));
2162            assert!(output.contains("line 2"));
2163        });
2164    }
2165
2166    #[test]
2167    fn append_and_tail_stage_log() {
2168        with_isolated_runs_dir("append-and-tail-stage-log", |_d| {
2169            let run_id = "test-stage-log-unit";
2170            append_stage_log(run_id, 0, "event A");
2171            append_stage_log(run_id, 0, "event B");
2172            let log = tail_stage_log(run_id, 0, 4096);
2173            assert!(log.contains("event A"));
2174            assert!(log.contains("event B"));
2175        });
2176    }
2177
2178    // ─── write/read stage context ───────────────────────────────────────────
2179
2180    #[test]
2181    fn write_and_read_stage_context_roundtrip() {
2182        with_isolated_runs_dir("write-and-read-stage-context-roundtrip", |_d| {
2183            let run_id = "test-stage-ctx-unit";
2184            let snap = ContextSnapshot {
2185                stage_name: "stage-0".into(),
2186                total_tokens: 100,
2187                max_tokens: 4096,
2188                regions: vec![],
2189            };
2190            write_stage_context(run_id, 0, &snap).unwrap();
2191            let back = read_stage_context(run_id, 0).unwrap();
2192            assert_eq!(back.stage_name, "stage-0");
2193        });
2194    }
2195
2196    #[test]
2197    fn read_stage_context_missing_returns_none() {
2198        assert!(read_stage_context("nonexistent-run", 99).is_none());
2199    }
2200
2201    // ─── append_dashboard_log ─────────────────────────────────────────────
2202
2203    #[test]
2204    fn append_dashboard_log_creates_log_file() {
2205        with_isolated_runs_dir("append-dashboard-log-creates-log-file", |_d| {
2206            append_dashboard_log("coverage-test-message");
2207            assert!(dashboard_log_path().exists());
2208        });
2209    }
2210
2211    #[test]
2212    fn append_dashboard_log_open_failure_is_silently_ignored() {
2213        // Covers the `if let Ok(mut file) = ... .open(&path)` pattern *not*
2214        // matching: pre-create the resolved log path as a directory, so
2215        // opening it for append fails with `IsADirectory` - the function
2216        // must swallow this silently (best-effort logging) rather than
2217        // panic.
2218        with_isolated_runs_dir("append-dashboard-log-open-failure", |_d| {
2219            let path = dashboard_log_path();
2220            std::fs::create_dir_all(&path).unwrap();
2221            append_dashboard_log("this should not panic");
2222            assert!(path.is_dir());
2223        });
2224    }
2225
2226    #[test]
2227    fn append_dashboard_log_path_with_no_parent_skips_create_dir_all() {
2228        // Every other test resolves `dashboard_log_path()` to a path with a
2229        // real parent component, leaving the `if let Some(parent) = ...`
2230        // pattern's `None` arm (root paths like "/" have no parent) never
2231        // exercised. `temp_env::with_var` points the override at "/" for the
2232        // closure's duration (serialized process-wide, then restored).
2233        temp_env::with_var("LEVIATH_DASHBOARD_LOG_PATH", Some("/"), || {
2234            assert!(dashboard_log_path().parent().is_none());
2235            append_dashboard_log("this should not panic even with no parent");
2236        });
2237    }
2238
2239    #[test]
2240    fn dashboard_log_rolls_once_over_cap() {
2241        // A tiny cap so a couple of lines trips the roll. The over-cap live file
2242        // is moved to `<name>.1` and a fresh live file is started.
2243        let dir = tempfile::tempdir().unwrap();
2244        let path = dir.path().join("dashboard.log");
2245        append_dashboard_log_capped(&path, "first line well over the tiny cap", 8);
2246        // First write created the file; it now exceeds the 8-byte cap.
2247        assert!(path.exists());
2248        assert!(!rolled_log_path(&path).exists());
2249        // Second write sees the file over cap → rolls it and restarts.
2250        append_dashboard_log_capped(&path, "second", 8);
2251        let rolled = rolled_log_path(&path);
2252        assert!(rolled.exists(), "previous generation rolled to <name>.1");
2253        assert!(
2254            std::fs::read_to_string(&rolled)
2255                .unwrap()
2256                .contains("first line")
2257        );
2258        // The live file was restarted with only the newest line.
2259        let live = std::fs::read_to_string(&path).unwrap();
2260        assert!(live.contains("second"));
2261        assert!(!live.contains("first line"));
2262    }
2263
2264    #[test]
2265    fn dashboard_log_does_not_roll_under_cap() {
2266        let dir = tempfile::tempdir().unwrap();
2267        let path = dir.path().join("dashboard.log");
2268        append_dashboard_log_capped(&path, "a", 1_000_000);
2269        append_dashboard_log_capped(&path, "b", 1_000_000);
2270        // Both lines are in the single live file; nothing was rolled.
2271        assert!(!rolled_log_path(&path).exists());
2272        let live = std::fs::read_to_string(&path).unwrap();
2273        assert!(live.contains("a") && live.contains("b"));
2274    }
2275
2276    // ─── dashboard_log_path ────────────────────────────────────────────────
2277
2278    #[test]
2279    fn dashboard_log_path_structure() {
2280        // Exercises the real (env-reading) `dashboard_log_path()` on its
2281        // fallback branch, so - like `runs_dir_structure` below - it forces
2282        // `LEVIATH_DASHBOARD_LOG_PATH` unset via `temp_env::with_var_unset`,
2283        // which also serializes against every other temp-env test so a
2284        // concurrently-isolated test can't race this assertion.
2285        temp_env::with_var_unset("LEVIATH_DASHBOARD_LOG_PATH", || {
2286            let path = dashboard_log_path();
2287            assert!(path.to_str().unwrap().contains(".leviath"));
2288            assert!(path.to_str().unwrap().ends_with("dashboard.log"));
2289        });
2290    }
2291
2292    /// With no `LEVIATH_DASHBOARD_LOG_PATH`, the dashboard log must follow
2293    /// `LEVIATH_HOME` like every other data path. Resolving through the raw
2294    /// OS home would leave a fully isolated test session still appending to
2295    /// the developer's real `~/.leviath/dashboard.log`.
2296    #[test]
2297    fn dashboard_log_path_honors_leviath_home() {
2298        temp_env::with_vars(
2299            [
2300                ("LEVIATH_DASHBOARD_LOG_PATH", None),
2301                ("LEVIATH_HOME", Some("/custom/home")),
2302            ],
2303            || {
2304                assert_eq!(
2305                    dashboard_log_path(),
2306                    PathBuf::from("/custom/home/.leviath/dashboard.log")
2307                );
2308            },
2309        );
2310    }
2311
2312    // ─── runs_dir / run_dir ────────────────────────────────────────────────
2313
2314    #[test]
2315    fn runs_dir_structure() {
2316        // See the comment on `dashboard_log_path_structure` above - same
2317        // race, same fix, for `LEVIATH_RUNS_DIR`.
2318        temp_env::with_var_unset("LEVIATH_RUNS_DIR", || {
2319            let path = runs_dir();
2320            assert!(path.to_str().unwrap().contains(".leviath"));
2321            assert!(path.to_str().unwrap().ends_with("runs"));
2322        });
2323    }
2324
2325    #[test]
2326    fn runs_dir_from_uses_override_when_provided() {
2327        let path = runs_dir_from(Some("/custom/leviath/runs"));
2328        assert_eq!(path, PathBuf::from("/custom/leviath/runs"));
2329    }
2330
2331    #[test]
2332    fn runs_dir_from_falls_back_to_home_when_none() {
2333        let path = runs_dir_from(None);
2334        #[cfg(unix)]
2335        assert!(path.ends_with(".leviath/runs"));
2336        #[cfg(windows)]
2337        assert!(path.ends_with(".leviath\\runs"));
2338    }
2339
2340    /// With no `LEVIATH_RUNS_DIR`, the runs dir must follow `LEVIATH_HOME` - the
2341    /// same home every other leviath path resolves through. Without this, setting
2342    /// `LEVIATH_HOME` isolates a test's config/socket/agents dir while its runs
2343    /// still land in the real `~/.leviath/runs`.
2344    #[test]
2345    fn runs_dir_follows_leviath_home() {
2346        temp_env::with_vars(
2347            [
2348                ("LEVIATH_RUNS_DIR", None::<&str>),
2349                ("LEVIATH_HOME", Some("/tmp/leviath-home-runs-test")),
2350            ],
2351            || {
2352                assert_eq!(
2353                    runs_dir(),
2354                    PathBuf::from("/tmp/leviath-home-runs-test")
2355                        .join(".leviath")
2356                        .join("runs")
2357                );
2358            },
2359        );
2360    }
2361
2362    #[test]
2363    fn dashboard_log_path_from_uses_override_when_provided() {
2364        let path = dashboard_log_path_from(Some("/custom/leviath/dashboard.log"));
2365        assert_eq!(path, PathBuf::from("/custom/leviath/dashboard.log"));
2366    }
2367
2368    #[test]
2369    fn dashboard_log_path_from_falls_back_to_home_when_none() {
2370        let path = dashboard_log_path_from(None);
2371        #[cfg(unix)]
2372        assert!(path.ends_with(".leviath/dashboard.log"));
2373        #[cfg(windows)]
2374        assert!(path.ends_with(".leviath\\dashboard.log"));
2375    }
2376
2377    #[test]
2378    fn run_dir_contains_run_id() {
2379        let path = run_dir("my-run-123");
2380        assert!(path.to_str().unwrap().contains("my-run-123"));
2381    }
2382
2383    // ─── with_isolated_runs_dir ─────────────────────────────────────────────
2384
2385    #[test]
2386    fn with_isolated_runs_dir_points_at_temp_dir_and_cleans_up_after() {
2387        // Deliberately avoids a racy before/after ambient comparison (a
2388        // concurrently-isolated test could own `LEVIATH_RUNS_DIR` just before
2389        // or after this closure's temp-env window): instead assert the helper's
2390        // own hash-derived path is live *inside* the closure and removed
2391        // afterward - a property no other test can perturb, since none
2392        // produces this exact path.
2393        let inside = with_isolated_runs_dir("helper-self-test", |base_dir| {
2394            let expected = base_dir.join("runs");
2395            assert_eq!(runs_dir(), expected);
2396            assert!(runs_dir().exists());
2397            assert_eq!(dashboard_log_path(), base_dir.join("dashboard.log"));
2398            expected
2399        });
2400        // Closure returned: the temp dir the helper created is gone.
2401        assert!(!inside.exists());
2402    }
2403
2404    // ─── tail_file edge cases ──────────────────────────────────────────────
2405
2406    #[test]
2407    fn tail_file_exact_size() {
2408        let dir = tempfile::tempdir().unwrap();
2409        let path = dir.path().join("exact.txt");
2410        std::fs::write(&path, "exactly").unwrap();
2411        // max_bytes == file size
2412        let result = tail_file(&path, 7);
2413        assert_eq!(result, "exactly");
2414    }
2415
2416    #[test]
2417    fn tail_file_tail_without_newline_returns_whole_window() {
2418        // When the last `max_bytes` window of a larger file contains no '\n'
2419        // at all (a single long line with no line breaks), `tail_file` cannot
2420        // skip to a newline boundary, so it falls through to the `else` arm and
2421        // returns the whole (newline-free) tail window verbatim. Bytes are
2422        // written raw (never via `writeln!`, which would append '\n') so that
2423        // on *every* OS the tail slice is guaranteed newline-free - on Windows
2424        // ordinary text output is `\r\n`-terminated, which would otherwise keep
2425        // a '\n' in the window and take the `if` arm instead.
2426        let dir = tempfile::tempdir().unwrap();
2427        let path = dir.path().join("no_newline.txt");
2428        // 100 raw bytes, no newline anywhere.
2429        let content = "a".repeat(100);
2430        std::fs::write(&path, content.as_bytes()).unwrap();
2431        // A 10-byte window is smaller than the file (100) and contains no '\n'.
2432        let result = tail_file(&path, 10);
2433        assert_eq!(result, "aaaaaaaaaa");
2434    }
2435
2436    // ─── RunMeta metadata and callback_url ─────────────────────────────────
2437
2438    #[test]
2439    fn run_meta_with_metadata() {
2440        let mut meta = RunMeta::new(
2441            "meta-run".into(),
2442            "agent".into(),
2443            "/p".into(),
2444            "task".into(),
2445            None,
2446            "/w".into(),
2447            1,
2448        );
2449        meta.metadata
2450            .insert("key1".to_string(), "value1".to_string());
2451        meta.callback_url = Some("https://example.com/hook".to_string());
2452        meta.parent_run_id = Some("parent-123".to_string());
2453
2454        let json = serde_json::to_string(&meta).unwrap();
2455        let back: RunMeta = serde_json::from_str(&json).unwrap();
2456        assert_eq!(back.metadata.get("key1").unwrap(), "value1");
2457        assert_eq!(
2458            back.callback_url.as_deref(),
2459            Some("https://example.com/hook")
2460        );
2461        assert_eq!(back.parent_run_id.as_deref(), Some("parent-123"));
2462    }
2463
2464    // ─── StageRecord modifications ─────────────────────────────────────────
2465
2466    #[test]
2467    fn stage_record_mutation() {
2468        let mut rec = StageRecord::new("test".into(), 0);
2469        rec.status = StageRunStatus::Active;
2470        rec.started_at = Some(1000);
2471        rec.prompt_tokens = 500;
2472        rec.completion_tokens = 200;
2473        rec.cached_tokens = 50;
2474
2475        assert_eq!(rec.status, StageRunStatus::Active);
2476        assert_eq!(rec.started_at, Some(1000));
2477        assert_eq!(rec.prompt_tokens, 500);
2478        assert_eq!(rec.completion_tokens, 200);
2479        assert_eq!(rec.cached_tokens, 50);
2480
2481        rec.status = StageRunStatus::Complete;
2482        rec.ended_at = Some(2000);
2483        assert_eq!(rec.status, StageRunStatus::Complete);
2484        assert_eq!(rec.ended_at, Some(2000));
2485    }
2486
2487    // ─── ContextSnapshot with entries ──────────────────────────────────────
2488
2489    #[test]
2490    fn context_snapshot_with_entries() {
2491        let snap = ContextSnapshot {
2492            stage_name: "main".into(),
2493            total_tokens: 1000,
2494            max_tokens: 8192,
2495            regions: vec![
2496                RegionSnapshot {
2497                    name: "system".into(),
2498                    kind: "pinned".into(),
2499                    current_tokens: 100,
2500                    max_tokens: 2000,
2501                    entries: vec![
2502                        RegionEntrySnapshot {
2503                            content: "You are helpful".into(),
2504                            tokens: 3,
2505                            kind: Default::default(),
2506                            metadata: None,
2507                            key: None,
2508                            taint: Default::default(),
2509                        },
2510                        RegionEntrySnapshot {
2511                            content: "Additional instruction".into(),
2512                            tokens: 5,
2513                            kind: Default::default(),
2514                            metadata: Some(serde_json::json!({"source": "user"})),
2515                            key: None,
2516                            taint: Default::default(),
2517                        },
2518                    ],
2519                },
2520                RegionSnapshot {
2521                    name: "conversation".into(),
2522                    kind: "sliding".into(),
2523                    current_tokens: 900,
2524                    max_tokens: 6000,
2525                    entries: vec![],
2526                },
2527            ],
2528        };
2529
2530        let json = serde_json::to_string_pretty(&snap).unwrap();
2531        let back: ContextSnapshot = serde_json::from_str(&json).unwrap();
2532        assert_eq!(back.regions.len(), 2);
2533        assert_eq!(back.regions[0].entries.len(), 2);
2534        assert_eq!(back.regions[0].entries[1].tokens, 5);
2535        assert!(back.regions[0].entries[1].metadata.is_some());
2536    }
2537
2538    // ─── RegionEntrySnapshot metadata ──────────────────────────────────────
2539
2540    #[test]
2541    fn region_entry_snapshot_metadata_omitted_when_none() {
2542        let entry = RegionEntrySnapshot {
2543            content: "test".into(),
2544            tokens: 1,
2545            kind: Default::default(),
2546            metadata: None,
2547            key: None,
2548            taint: Default::default(),
2549        };
2550        let json = serde_json::to_value(&entry).unwrap();
2551        assert!(json.get("metadata").is_none());
2552    }
2553
2554    // ─── Multiple stage output appends ─────────────────────────────────────
2555
2556    #[test]
2557    fn append_stage_output_multiple_stages() {
2558        with_isolated_runs_dir("append-stage-output-multiple-stages", |_d| {
2559            let run_id = "test-multi-stage-out";
2560            append_stage_output(run_id, 0, "stage 0 output");
2561            append_stage_output(run_id, 1, "stage 1 output");
2562            append_stage_output(run_id, 2, "stage 2 output");
2563
2564            let out0 = tail_stage_output(run_id, 0, 4096);
2565            let out1 = tail_stage_output(run_id, 1, 4096);
2566            let out2 = tail_stage_output(run_id, 2, 4096);
2567
2568            assert!(out0.contains("stage 0 output"));
2569            assert!(out1.contains("stage 1 output"));
2570            assert!(out2.contains("stage 2 output"));
2571            // Verify no cross-contamination
2572            assert!(!out0.contains("stage 1 output"));
2573        });
2574    }
2575
2576    // ─── list_runs ─────────────────────────────────────────────────────────
2577
2578    #[test]
2579    fn list_runs_returns_sorted() {
2580        with_isolated_runs_dir("list-runs-returns-sorted", |_d| {
2581            let meta1 = RunMeta::new(
2582                "test-list-run-a".into(),
2583                "agent".into(),
2584                "/p".into(),
2585                "task a".into(),
2586                None,
2587                "/w".into(),
2588                1,
2589            );
2590            let meta2 = RunMeta::new(
2591                "test-list-run-b".into(),
2592                "agent".into(),
2593                "/p".into(),
2594                "task b".into(),
2595                None,
2596                "/w".into(),
2597                1,
2598            );
2599
2600            let _ = create_run(&meta1);
2601            // Small delay to ensure different timestamps
2602            let _ = create_run(&meta2);
2603
2604            let runs = list_runs();
2605            // Both should appear in the list
2606            let ids: Vec<&str> = runs.iter().map(|r| r.run_id.as_str()).collect();
2607            assert!(ids.contains(&"test-list-run-a"));
2608            assert!(ids.contains(&"test-list-run-b"));
2609        });
2610    }
2611
2612    // ─── tail_stage_log / tail_stage_output empty ──────────────────────────
2613
2614    #[test]
2615    fn tail_stage_output_nonexistent_returns_empty() {
2616        assert_eq!(tail_stage_output("no-such-run-xyz", 0, 4096), "");
2617    }
2618
2619    #[test]
2620    fn tail_stage_log_nonexistent_returns_empty() {
2621        assert_eq!(tail_stage_log("no-such-run-xyz", 0, 4096), "");
2622    }
2623
2624    // ─── list_runs_in_dir ───────────────────────────────────────────────────
2625
2626    #[test]
2627    fn list_runs_in_dir_nonexistent_returns_empty() {
2628        let result = list_runs_in_dir(PathBuf::from("/nonexistent/leviath/runs/coverage-test"));
2629        assert!(result.is_empty());
2630    }
2631
2632    #[test]
2633    fn list_runs_in_dir_empty_dir_returns_empty() {
2634        let dir = tempfile::tempdir().unwrap();
2635        let result = list_runs_in_dir(dir.path().to_path_buf());
2636        assert!(result.is_empty());
2637    }
2638
2639    #[test]
2640    fn list_runs_in_dir_unreadable_dir_returns_empty() {
2641        // Covers the `if let Ok(entries) = std::fs::read_dir(&dir)` pattern
2642        // *not* matching: `dir.exists()` is true (so the earlier early-return
2643        // is skipped) but `read_dir` fails, so the whole block is silently
2644        // skipped. Pointing at a *file* makes `read_dir` fail on every platform.
2645        let dir = tempfile::tempdir().unwrap();
2646        let not_a_dir = dir.path().join("runs-is-a-file");
2647        std::fs::write(&not_a_dir, "not a dir").unwrap();
2648        let result = list_runs_in_dir(not_a_dir);
2649        assert!(result.is_empty());
2650    }
2651
2652    #[test]
2653    fn append_stage_output_open_failure_is_silently_skipped() {
2654        // When `output.log` already exists as a *directory*, `OpenOptions::open`
2655        // fails and the write is silently skipped (the `if let Ok(file)` false
2656        // path). Making the target a directory fails the open on every platform.
2657        crate::runstate::with_isolated_runs_dir("append_stage_output_open_failure", |_d| {
2658            let run_id = "append-out-openfail";
2659            ensure_stage_dir(run_id, 0);
2660            std::fs::create_dir_all(stage_dir(run_id, 0).join("output.log")).unwrap();
2661            append_stage_output(run_id, 0, "ignored"); // must not panic
2662        });
2663    }
2664
2665    #[test]
2666    fn append_stage_log_open_failure_is_silently_skipped() {
2667        // Same as above for `logs.log` in `append_stage_log`.
2668        crate::runstate::with_isolated_runs_dir("append_stage_log_open_failure", |_d| {
2669            let run_id = "append-log-openfail";
2670            ensure_stage_dir(run_id, 0);
2671            std::fs::create_dir_all(stage_dir(run_id, 0).join("logs.log")).unwrap();
2672            append_stage_log(run_id, 0, "ignored"); // must not panic
2673        });
2674    }
2675
2676    // ─── runs_dir / list_runs edge cases ────────────────────────────────────
2677
2678    #[test]
2679    fn runs_dir_with_override_set_returns_override() {
2680        let tmpdir = tempfile::tempdir().unwrap();
2681        temp_env::with_var("LEVIATH_RUNS_DIR", Some(tmpdir.path()), || {
2682            assert_eq!(runs_dir(), tmpdir.path());
2683        });
2684    }
2685
2686    #[test]
2687    fn runs_dir_without_override_falls_back_to_home() {
2688        temp_env::with_var_unset("LEVIATH_RUNS_DIR", || {
2689            let dir = runs_dir();
2690            #[cfg(unix)]
2691            assert!(dir.ends_with(".leviath/runs"));
2692            #[cfg(windows)]
2693            assert!(dir.ends_with(".leviath\\runs"));
2694        });
2695    }
2696
2697    #[test]
2698    fn list_runs_empty_when_runs_dir_missing_or_empty() {
2699        // Isolated via `isolate_runs_dir_for_test`, so this is a genuinely
2700        // empty runs dir (not "the real dir, which we hope has no entry with
2701        // this exact bogus id") - can assert real emptiness instead of just
2702        // absence of one specific id.
2703        with_isolated_runs_dir("list-runs-empty-when-runs-dir-missing-or-empty", |_d| {
2704            let runs = list_runs();
2705            assert!(runs.is_empty());
2706        });
2707    }
2708
2709    #[test]
2710    fn tail_file_nonexistent_path_returns_empty() {
2711        let path = std::path::Path::new("/nonexistent/path/to/a/file.log");
2712        assert_eq!(tail_file(path, 1024), "");
2713    }
2714
2715    #[test]
2716    fn tail_file_small_file_returns_whole_contents() {
2717        let dir = tempfile::tempdir().unwrap();
2718        let path = dir.path().join("small.log");
2719        std::fs::write(&path, "hello world").unwrap();
2720        assert_eq!(tail_file(&path, 1024), "hello world");
2721    }
2722
2723    #[test]
2724    fn tail_file_large_file_truncates_from_offset() {
2725        let dir = tempfile::tempdir().unwrap();
2726        let path = dir.path().join("big.log");
2727        let content = "a".repeat(100) + "\nTAIL_MARKER\n";
2728        std::fs::write(&path, &content).unwrap();
2729        let tailed = tail_file(&path, 20);
2730        assert!(tailed.contains("TAIL_MARKER"));
2731        assert!(tailed.len() < content.len());
2732    }
2733
2734    #[test]
2735    fn tail_file_directory_path_returns_empty() {
2736        // metadata() and File::open() both succeed on a directory (confirmed
2737        // empirically on macOS/Linux); it's read_to_end() that fails with
2738        // "Is a directory" - and that error is deliberately discarded (`let
2739        // _ = file.read_to_end(&mut buf);`), so this exercises the
2740        // graceful-empty-buffer fallback at the bottom of the function, not
2741        // either of the two `Err(_) => return String::new()` early returns.
2742        let dir = tempfile::tempdir().unwrap();
2743        assert_eq!(tail_file(dir.path(), 4), "");
2744    }
2745
2746    #[cfg(unix)]
2747    #[test]
2748    fn tail_file_open_permission_denied_returns_empty() {
2749        // A file with no permissions at all: `Path::exists()`/`fs::metadata()`
2750        // only need search (execute) permission on the *parent* directories
2751        // to stat a path, not read permission on the file itself - so both
2752        // succeed here. `std::fs::File::open()` in read mode, however,
2753        // genuinely fails with `PermissionDenied`. Unlike the metadata-error
2754        // arm (only reachable via a delete-between-calls race), this is a
2755        // deterministic way to exercise the `File::open` `Err(_)` arm.
2756        use std::os::unix::fs::PermissionsExt;
2757
2758        let dir = tempfile::tempdir().unwrap();
2759        let path = dir.path().join("no-permissions.log");
2760        // Content must exceed max_bytes so the "whole file" fast path
2761        // (`file_size <= max_bytes`) doesn't short-circuit before reaching
2762        // the `File::open` call under test.
2763        std::fs::write(&path, "x".repeat(100)).unwrap();
2764        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap();
2765
2766        assert_eq!(tail_file(&path, 4), "");
2767
2768        // Restore permissions so the tempdir can clean itself up on drop.
2769        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap();
2770    }
2771
2772    // ─── hermetic write/read coverage tests (use _to/_from/_in helpers) ───────
2773
2774    #[test]
2775    fn write_context_snapshot_to_hermetic() {
2776        let dir = tempfile::tempdir().unwrap();
2777        let snap = ContextSnapshot {
2778            stage_name: "cov-stage".into(),
2779            total_tokens: 42,
2780            max_tokens: 8192,
2781            regions: vec![],
2782        };
2783        write_context_snapshot_to(dir.path(), &snap).unwrap();
2784        let json = std::fs::read_to_string(dir.path().join("context.json")).unwrap();
2785        let back: ContextSnapshot = serde_json::from_str(&json).unwrap();
2786        assert_eq!(back.total_tokens, 42);
2787    }
2788
2789    #[test]
2790    fn write_context_snapshot_to_fails_without_dir() {
2791        let snap = ContextSnapshot {
2792            stage_name: "s".into(),
2793            total_tokens: 1,
2794            max_tokens: 100,
2795            regions: vec![],
2796        };
2797        let nonexistent = std::path::Path::new("/nonexistent-cov-dir-xyzzy-abc");
2798        let result = write_context_snapshot_to(nonexistent, &snap);
2799        assert!(result.is_err());
2800    }
2801
2802    #[test]
2803    fn write_context_snapshot_to_fails_when_rename_target_is_a_dir() {
2804        // Covers the `std::fs::rename(&tmp, &path)?` `Err` arm: the tmp file
2805        // write succeeds (its directory is writable), but the final rename
2806        // fails because `context.json` already exists as a *directory* --
2807        // `rename(2)` on POSIX refuses to replace a directory with a
2808        // regular file, unlike a plain overwrite of an existing file.
2809        let dir = tempfile::tempdir().unwrap();
2810        std::fs::create_dir(dir.path().join("context.json")).unwrap();
2811        let snap = ContextSnapshot {
2812            stage_name: "s".into(),
2813            total_tokens: 1,
2814            max_tokens: 100,
2815            regions: vec![],
2816        };
2817        let result = write_context_snapshot_to(dir.path(), &snap);
2818        assert!(result.is_err());
2819    }
2820
2821    #[test]
2822    fn create_run_in_hermetic() {
2823        let tmpdir = tempfile::tempdir().unwrap();
2824        let run_dir = tmpdir.path().join("cov-run");
2825        let meta = RunMeta::new(
2826            "cov-run".into(),
2827            "cov-agent".into(),
2828            "/agents/cov".into(),
2829            "cov task".into(),
2830            None,
2831            "/tmp".into(),
2832            1,
2833        );
2834        create_run_in(&run_dir, &meta).unwrap();
2835        let back = read_meta_from(&run_dir).unwrap();
2836        assert_eq!(back.run_id, "cov-run");
2837    }
2838
2839    #[test]
2840    fn create_run_in_fails_on_bad_parent() {
2841        // A hardcoded "/nonexistent-.../run" path isn't reliably bad across
2842        // platforms: on Windows CI runners (which typically have write
2843        // access to create directories at the drive root), that path
2844        // resolves under the current drive's root and create_dir_all
2845        // actually succeeds there, while on Unix it fails because writing
2846        // to the real filesystem root needs privileges the CI user lacks --
2847        // this passed locally but failed on Windows CI. Use a path with a
2848        // regular file as a parent component instead: create_dir_all can
2849        // never succeed under a file, on any platform or set of permissions.
2850        let dir = tempfile::tempdir().unwrap();
2851        let not_a_dir = dir.path().join("not-a-directory");
2852        std::fs::write(&not_a_dir, "x").unwrap();
2853        let bad = not_a_dir.join("run");
2854        let meta = RunMeta::new(
2855            "run".into(),
2856            "a".into(),
2857            "/".into(),
2858            "t".into(),
2859            None,
2860            "/tmp".into(),
2861            1,
2862        );
2863        let result = create_run_in(&bad, &meta);
2864        assert!(result.is_err());
2865    }
2866
2867    #[test]
2868    fn write_meta_to_hermetic() {
2869        let tmpdir = tempfile::tempdir().unwrap();
2870        let meta = RunMeta::new(
2871            "cov-write-meta".into(),
2872            "a".into(),
2873            "/".into(),
2874            "t".into(),
2875            None,
2876            "/tmp".into(),
2877            1,
2878        );
2879        write_meta_to(tmpdir.path(), &meta).unwrap();
2880        let back = read_meta_from(tmpdir.path()).unwrap();
2881        assert_eq!(back.run_id, "cov-write-meta");
2882    }
2883
2884    #[test]
2885    fn write_meta_to_fails_without_dir() {
2886        let meta = RunMeta::new(
2887            "cov-no-dir".into(),
2888            "a".into(),
2889            "/".into(),
2890            "t".into(),
2891            None,
2892            "/tmp".into(),
2893            1,
2894        );
2895        let bad = std::path::Path::new("/nonexistent-cov-write-meta-xyzzy");
2896        let result = write_meta_to(bad, &meta);
2897        assert!(result.is_err());
2898    }
2899
2900    #[test]
2901    fn write_meta_to_fails_when_rename_target_is_a_dir() {
2902        // See `write_context_snapshot_to_fails_when_rename_target_is_a_dir`:
2903        // same `std::fs::rename(&tmp_path, &final_path)?` `Err` arm, forced
2904        // by pre-creating `meta.json` as a directory.
2905        let dir = tempfile::tempdir().unwrap();
2906        std::fs::create_dir(dir.path().join("meta.json")).unwrap();
2907        let meta = RunMeta::new(
2908            "cov-rename-fail".into(),
2909            "a".into(),
2910            "/".into(),
2911            "t".into(),
2912            None,
2913            "/tmp".into(),
2914            1,
2915        );
2916        let result = write_meta_to(dir.path(), &meta);
2917        assert!(result.is_err());
2918    }
2919
2920    #[test]
2921    fn read_meta_from_fails_on_missing_file() {
2922        let tmpdir = tempfile::tempdir().unwrap();
2923        let result = read_meta_from(tmpdir.path());
2924        assert!(result.is_err());
2925    }
2926
2927    #[test]
2928    fn write_stages_index_to_hermetic() {
2929        let tmpdir = tempfile::tempdir().unwrap();
2930        let stages = vec![StageRecord::new("cov-stage".into(), 0)];
2931        write_stages_index_to(tmpdir.path(), &stages).unwrap();
2932        let json = std::fs::read_to_string(tmpdir.path().join("stages.json")).unwrap();
2933        let back: Vec<StageRecord> = serde_json::from_str(&json).unwrap();
2934        assert_eq!(back.len(), 1);
2935        assert_eq!(back[0].name, "cov-stage");
2936    }
2937
2938    #[test]
2939    fn write_stages_index_to_fails_without_dir() {
2940        let stages = vec![StageRecord::new("s".into(), 0)];
2941        let bad = std::path::Path::new("/nonexistent-cov-stages-xyzzy");
2942        let result = write_stages_index_to(bad, &stages);
2943        assert!(result.is_err());
2944    }
2945
2946    #[test]
2947    fn write_stages_index_to_fails_when_rename_target_is_a_dir() {
2948        // See `write_context_snapshot_to_fails_when_rename_target_is_a_dir`:
2949        // same `std::fs::rename(&tmp, &path)?` `Err` arm, forced by
2950        // pre-creating `stages.json` as a directory.
2951        let dir = tempfile::tempdir().unwrap();
2952        std::fs::create_dir(dir.path().join("stages.json")).unwrap();
2953        let stages = vec![StageRecord::new("s".into(), 0)];
2954        let result = write_stages_index_to(dir.path(), &stages);
2955        assert!(result.is_err());
2956    }
2957
2958    #[test]
2959    fn list_runs_in_dir_includes_valid_run() {
2960        let tmpdir = tempfile::tempdir().unwrap();
2961        let run_id = "cov-listed-run";
2962        let run_subdir = tmpdir.path().join(run_id);
2963        std::fs::create_dir_all(&run_subdir).unwrap();
2964        let meta = RunMeta::new(
2965            run_id.into(),
2966            "list-agent".into(),
2967            "/agents/list".into(),
2968            "list task".into(),
2969            None,
2970            "/tmp".into(),
2971            1,
2972        );
2973        let json = serde_json::to_string_pretty(&meta).unwrap();
2974        std::fs::write(run_subdir.join("meta.json"), &json).unwrap();
2975
2976        // list_runs_in_dir now reads meta.json directly from the dir, no env var needed
2977        let runs = list_runs_in_dir(tmpdir.path().to_path_buf());
2978        assert!(runs.iter().any(|r| r.run_id == run_id));
2979    }
2980
2981    #[test]
2982    fn list_runs_in_dir_skips_entry_with_corrupted_meta_json() {
2983        // Exercises the `if let Ok(meta) = serde_json::from_str::<RunMeta>(...)`
2984        // else arm: a subdirectory whose meta.json exists and is readable as
2985        // a string, but doesn't parse as a `RunMeta`, is silently skipped
2986        // rather than propagating an error.
2987        let tmpdir = tempfile::tempdir().unwrap();
2988        let good_run_id = "cov-listed-good-run";
2989        let bad_run_id = "cov-listed-corrupted-run";
2990
2991        let good_subdir = tmpdir.path().join(good_run_id);
2992        std::fs::create_dir_all(&good_subdir).unwrap();
2993        let meta = RunMeta::new(
2994            good_run_id.into(),
2995            "list-agent".into(),
2996            "/agents/list".into(),
2997            "list task".into(),
2998            None,
2999            "/tmp".into(),
3000            1,
3001        );
3002        let json = serde_json::to_string_pretty(&meta).unwrap();
3003        std::fs::write(good_subdir.join("meta.json"), &json).unwrap();
3004
3005        let bad_subdir = tmpdir.path().join(bad_run_id);
3006        std::fs::create_dir_all(&bad_subdir).unwrap();
3007        std::fs::write(bad_subdir.join("meta.json"), "not valid json").unwrap();
3008
3009        // A subdirectory with NO meta.json exercises the *other* skip branch:
3010        // the `if let Ok(json) = read_to_string(&meta_path)` else arm (the file
3011        // can't be read), distinct from the parse-fails arm above. Covering
3012        // both here keeps list_runs_in_dir at 100% on every OS deterministically.
3013        let no_meta_run_id = "cov-listed-no-meta-run";
3014        std::fs::create_dir_all(tmpdir.path().join(no_meta_run_id)).unwrap();
3015
3016        let runs = list_runs_in_dir(tmpdir.path().to_path_buf());
3017        assert!(runs.iter().any(|r| r.run_id == good_run_id));
3018        assert!(!runs.iter().any(|r| r.run_id == bad_run_id));
3019        assert!(!runs.iter().any(|r| r.run_id == no_meta_run_id));
3020    }
3021
3022    // ─── force_cancel_in: the floor under every kill path ───
3023
3024    /// Write a run dir with `status` and return its path.
3025    fn run_dir_with(base: &std::path::Path, run_id: &str, status: RunStatus) -> PathBuf {
3026        let dir = base.join(run_id);
3027        let meta = RunMeta {
3028            status,
3029            ..RunMeta::new(
3030                run_id.into(),
3031                "a".into(),
3032                "/p".into(),
3033                "t".into(),
3034                None,
3035                "/w".into(),
3036                1,
3037            )
3038        };
3039        create_run_in(&dir, &meta).unwrap();
3040        dir
3041    }
3042
3043    #[test]
3044    fn force_cancel_terminates_every_non_terminal_status() {
3045        let base = tempfile::tempdir().unwrap();
3046        for status in [
3047            RunStatus::Starting,
3048            RunStatus::Running,
3049            RunStatus::WaitingInput,
3050        ] {
3051            let dir = run_dir_with(base.path(), &format!("live-{status}"), status.clone());
3052            assert_eq!(force_cancel_in(&dir, 99), ForceCancelOutcome::Terminated);
3053            let meta = read_meta_from(&dir).unwrap();
3054            assert_eq!(meta.status, RunStatus::Cancelled, "{status} is killable");
3055            assert_eq!(meta.updated_at, 99, "the cancel is stamped");
3056        }
3057    }
3058
3059    #[test]
3060    fn force_cancel_leaves_a_finished_run_alone() {
3061        let base = tempfile::tempdir().unwrap();
3062        for status in [
3063            RunStatus::Complete,
3064            RunStatus::CompleteInteractive,
3065            RunStatus::Error,
3066            RunStatus::Cancelled,
3067        ] {
3068            let dir = run_dir_with(base.path(), &format!("done-{status}"), status.clone());
3069            assert_eq!(
3070                force_cancel_in(&dir, 99),
3071                ForceCancelOutcome::AlreadyTerminal,
3072                "{status} is already finished"
3073            );
3074            assert_eq!(read_meta_from(&dir).unwrap().status, status);
3075        }
3076    }
3077
3078    #[test]
3079    fn force_cancel_reports_no_such_run_for_a_missing_directory() {
3080        let base = tempfile::tempdir().unwrap();
3081        let outcome = force_cancel_in(&base.path().join("ghost"), 99);
3082        assert_eq!(outcome, ForceCancelOutcome::NoSuchRun);
3083        assert!(!outcome.found_run(), "nothing to cancel");
3084    }
3085
3086    /// A run dir whose metadata can't be parsed still gets terminated. Such a run
3087    /// is skipped by `list_runs`, so leaving it alone makes it both invisible and
3088    /// permanent - the one state from which there is no way back.
3089    #[test]
3090    fn force_cancel_writes_a_record_over_unreadable_metadata() {
3091        let base = tempfile::tempdir().unwrap();
3092        let dir = base.path().join("corrupt-run");
3093        std::fs::create_dir_all(&dir).unwrap();
3094        std::fs::write(dir.join("meta.json"), "{ not json").unwrap();
3095
3096        assert_eq!(force_cancel_in(&dir, 99), ForceCancelOutcome::Terminated);
3097        let meta = read_meta_from(&dir).expect("now parses");
3098        assert_eq!(meta.status, RunStatus::Cancelled);
3099        assert_eq!(meta.run_id, "corrupt-run", "recovered from the dir name");
3100        assert!(meta.error.is_some(), "records why it was synthesized");
3101    }
3102
3103    /// A directory that exists but can't be written still counts as "found" - the
3104    /// caller must not report "no such run" for a run that plainly exists.
3105    #[test]
3106    fn force_cancel_reports_a_write_failure_but_still_found_the_run() {
3107        crate::test_support::with_tracing(|| {
3108            let base = tempfile::tempdir().unwrap();
3109            let dir = base.path().join("blocked-run");
3110            std::fs::create_dir_all(&dir).unwrap();
3111            // A directory where `meta.json` must go: the rename can't succeed.
3112            std::fs::create_dir_all(dir.join("meta.json")).unwrap();
3113
3114            let outcome = force_cancel_in(&dir, 99);
3115            assert_eq!(outcome, ForceCancelOutcome::WriteFailed);
3116            assert!(outcome.found_run());
3117        });
3118    }
3119
3120    /// The spawn that never became a run: the placeholder is `Starting`, which
3121    /// is not terminal, so it has to be rewritten or it claims to be alive for
3122    /// ever (issue #190).
3123    #[test]
3124    fn force_error_records_the_failure_over_a_starting_placeholder() {
3125        let base = tempfile::tempdir().unwrap();
3126        let dir = base.path().join("stillborn-run");
3127        let meta = RunMeta::new(
3128            "stillborn-run".to_string(),
3129            "agent".to_string(),
3130            "/no/such/agent.leviath".to_string(),
3131            "t".to_string(),
3132            None,
3133            "/tmp".to_string(),
3134            0,
3135        );
3136        create_run_in(&dir, &meta).unwrap();
3137        assert_eq!(read_meta_from(&dir).unwrap().status, RunStatus::Starting);
3138
3139        assert_eq!(
3140            force_error_in(&dir, "blueprint not found", 99),
3141            ForceCancelOutcome::Terminated
3142        );
3143
3144        let written = read_meta_from(&dir).unwrap();
3145        assert_eq!(written.status, RunStatus::Error);
3146        assert_eq!(written.error.as_deref(), Some("blueprint not found"));
3147        assert_eq!(written.updated_at, 99);
3148        // The rest of the placeholder survives, so the run still explains itself.
3149        assert_eq!(written.task, "t");
3150    }
3151
3152    #[test]
3153    fn force_error_leaves_a_run_that_already_finished_alone() {
3154        let base = tempfile::tempdir().unwrap();
3155        let dir = base.path().join("done-run");
3156        let mut meta = RunMeta::new(
3157            "done-run".to_string(),
3158            "agent".to_string(),
3159            String::new(),
3160            "t".to_string(),
3161            None,
3162            "/tmp".to_string(),
3163            0,
3164        );
3165        meta.status = RunStatus::Complete;
3166        create_run_in(&dir, &meta).unwrap();
3167
3168        assert_eq!(
3169            force_error_in(&dir, "too late", 99),
3170            ForceCancelOutcome::AlreadyTerminal
3171        );
3172        assert_eq!(read_meta_from(&dir).unwrap().status, RunStatus::Complete);
3173    }
3174
3175    #[test]
3176    fn force_cancel_keeps_an_error_the_run_had_already_recorded() {
3177        // Cancelling passes no message of its own, so whatever the run managed
3178        // to say about itself before it was killed must survive.
3179        let base = tempfile::tempdir().unwrap();
3180        let dir = base.path().join("noisy-run");
3181        let mut meta = RunMeta::new(
3182            "noisy-run".to_string(),
3183            "agent".to_string(),
3184            String::new(),
3185            "t".to_string(),
3186            None,
3187            "/tmp".to_string(),
3188            0,
3189        );
3190        meta.error = Some("a provider hiccup".to_string());
3191        create_run_in(&dir, &meta).unwrap();
3192
3193        assert_eq!(force_cancel_in(&dir, 99), ForceCancelOutcome::Terminated);
3194        let written = read_meta_from(&dir).unwrap();
3195        assert_eq!(written.status, RunStatus::Cancelled);
3196        assert_eq!(written.error.as_deref(), Some("a provider hiccup"));
3197    }
3198
3199    #[test]
3200    fn force_error_writes_its_message_over_unreadable_metadata() {
3201        let base = tempfile::tempdir().unwrap();
3202        let dir = base.path().join("corrupt-stillborn");
3203        std::fs::create_dir_all(&dir).unwrap();
3204        std::fs::write(dir.join("meta.json"), "{ not json").unwrap();
3205
3206        assert_eq!(
3207            force_error_in(&dir, "blueprint not found", 99),
3208            ForceCancelOutcome::Terminated
3209        );
3210        let written = read_meta_from(&dir).expect("now parses");
3211        assert_eq!(written.status, RunStatus::Error);
3212        assert_eq!(written.error.as_deref(), Some("blueprint not found"));
3213    }
3214
3215    #[test]
3216    fn append_dashboard_log_writes_message() {
3217        // Exercises the create_dir_all branch and writeln! branch via a unique marker.
3218        with_isolated_runs_dir("append-dashboard-log-writes-message", |_d| {
3219            let unique = format!("cov-dashboard-log-{}", std::process::id());
3220            append_dashboard_log(&unique);
3221            let content = std::fs::read_to_string(dashboard_log_path()).unwrap_or_default();
3222            assert!(content.contains(&unique));
3223        });
3224    }
3225}