Skip to main content

leviath_cli/
runstate.rs

1//! On-disk run state for background agent executions.
2//!
3//! Each run lives under `~/.leviath/runs/<run-id>/` with:
4//! - `meta.json`    - run metadata, updated atomically (tmp + rename)
5//! - `output.log`  - append-only combined worker stdout (legacy/fallback)
6//! - `stages.json` - index of per-stage records
7//! - `stages/<idx>/output.log` - readable agent output for that stage
8//! - `stages/<idx>/logs.log`   - operational events + tool activity
9//! - `stages/<idx>/context.json` - context snapshot for that stage
10//!
11//! The dashboard's activity log is persisted separately at:
12//! - `~/.leviath/dashboard.log` - never cleared, appended across sessions
13
14use std::path::{Path, PathBuf};
15use std::time::{SystemTime, UNIX_EPOCH};
16
17// The plain run-state data types (RunMeta, RunStatus, the snapshot structs, and
18// the per-stage records) live in `leviath_core::run_meta`. Re-exported here so
19// `crate::runstate::RunMeta` / `runstate::RunMeta` call sites across the cli
20// resolve. All on-disk IO for these types remains in this module.
21pub use leviath_core::run_meta::{
22    ContextSnapshot, RegionEntrySnapshot, RegionSnapshot, RunMeta, RunStatus, StageRecord,
23    StageRunStatus,
24};
25
26/// Atomically write a context snapshot for the run.
27pub fn write_context_snapshot(run_id: &str, snap: &ContextSnapshot) -> anyhow::Result<()> {
28    write_context_snapshot_to(&run_dir(run_id), snap)
29}
30
31/// Atomically write pre-serialized `json` to `path` (via a `.json.tmp`
32/// sibling + rename).
33///
34/// Non-generic (takes an already-serialized string) so it has a single
35/// monomorphization and every region - including the `std::fs` error `?`
36/// arms - is exercised by real tests. Serialization is performed by the
37/// callers, whose concrete production types
38/// (`ContextSnapshot`/`RunMeta`/`&[StageRecord]`) are provably infallible to
39/// serialize (see the `.expect` sites).
40fn write_json_atomic(path: &std::path::Path, json: &str) -> anyhow::Result<()> {
41    let tmp = path.with_extension("json.tmp");
42    // `write_private`: these files carry the run's full task prompt,
43    // conversation and tool output - and `meta.json` carries the webhook
44    // signing secret. They were written with a plain `fs::write` at the umask
45    // default (typically 0644), protected only by the 0700 on the enclosing run
46    // directory. That is one `chmod` away from being readable, and defence in
47    // depth is the whole point of a mode on the file itself.
48    leviath_sys::write_private(&tmp, json.as_bytes())?;
49    std::fs::rename(&tmp, path)?;
50    Ok(())
51}
52
53fn write_context_snapshot_to(dir: &std::path::Path, snap: &ContextSnapshot) -> anyhow::Result<()> {
54    let json = serde_json::to_string_pretty(snap)
55        .expect("infallible: ContextSnapshot always serializes to JSON");
56    write_json_atomic(&dir.join("context.json"), &json)
57}
58
59/// Read the context snapshot for a run, if present.
60pub fn read_context_snapshot(run_id: &str) -> Option<ContextSnapshot> {
61    let path = run_dir(run_id).join("context.json");
62    let json = std::fs::read_to_string(&path).ok()?;
63    serde_json::from_str(&json).ok()
64}
65
66/// Read + parse a run's portable archive (`<run_dir>/run.lvr`), returning its
67/// records, or `None` if the archive is missing or unreadable.
68pub fn read_run_archive(run_id: &str) -> Option<Vec<leviath_core::run_archive::RunRecord>> {
69    let path = run_dir(run_id).join("run.lvr");
70    let bytes = std::fs::read(&path).ok()?;
71    leviath_core::run_archive::read_archive(&mut bytes.as_slice())
72        .ok()
73        .map(|(_version, records)| records)
74}
75
76/// A run's context-window history: the full window (+ metadata) at each recorded
77/// point over time, oldest first. Empty when there's no readable archive.
78pub fn context_history(run_id: &str) -> Vec<leviath_core::run_archive::RunPoint> {
79    read_run_archive(run_id)
80        .map(|records| leviath_core::run_archive::replay_points(&records))
81        .unwrap_or_default()
82}
83
84fn now_secs() -> i64 {
85    SystemTime::now()
86        .duration_since(UNIX_EPOCH)
87        .map(|d| d.as_secs() as i64)
88        .unwrap_or(0)
89}
90
91/// Inner implementation of `runs_dir`, parameterised so it can be tested
92/// without touching the process-global env. All callers go through `runs_dir`.
93///
94/// The fallback resolves through [`crate::config::leviath_home_dir`], not
95/// `dirs::home_dir` directly, so `LEVIATH_HOME` redirects the runs dir like it
96/// redirects the config, the control socket and the agents dir. With the raw
97/// OS home instead, a test that sets `LEVIATH_HOME` would be isolated
98/// everywhere *except* here and still write runs into the developer's real
99/// `~/.leviath/runs`. `LEVIATH_RUNS_DIR` wins over both.
100fn runs_dir_from(env_override: Option<&str>) -> PathBuf {
101    if let Some(dir) = env_override {
102        return PathBuf::from(dir);
103    }
104    leviath_core::paths::data_dir()
105        .unwrap_or_default()
106        .join("runs")
107}
108
109/// Directory where all run state is stored.
110pub fn runs_dir() -> PathBuf {
111    runs_dir_from(std::env::var("LEVIATH_RUNS_DIR").ok().as_deref())
112}
113
114/// Directory for a specific run.
115///
116/// A `run_id` that is not a single safe path component resolves to
117/// `<runs_dir>/<invalid>`, a name that cannot exist - so a caller that passes an
118/// attacker-supplied id gets a miss rather than a traversal. `run_id` reaches
119/// this from URL segments on `GET /api/agents/{id}/logs` and friends, where
120/// `Path::join` would otherwise happily accept `../../` or an absolute path.
121///
122/// Returning a definitely-missing path rather than an `Option` keeps every
123/// caller's "no such run" branch as the single failure path, instead of adding a
124/// second one that all of them would have to handle identically.
125pub fn run_dir(run_id: &str) -> PathBuf {
126    if !leviath_core::is_safe_path_component(run_id) {
127        tracing::warn!(run_id = %run_id, "rejected an unsafe run id");
128        return runs_dir().join("<invalid>");
129    }
130    runs_dir().join(run_id)
131}
132
133/// Inner implementation of `dashboard_log_path`, parameterised so it can be
134/// tested without touching the process-global env. All callers go through
135/// `dashboard_log_path`.
136fn dashboard_log_path_from(env_override: Option<&str>) -> PathBuf {
137    if let Some(path) = env_override {
138        return PathBuf::from(path);
139    }
140    leviath_core::paths::data_dir()
141        .unwrap_or_default()
142        .join("dashboard.log")
143}
144
145/// Path to the persistent dashboard activity log (~/.leviath/dashboard.log).
146///
147/// Honours the `LEVIATH_DASHBOARD_LOG_PATH` override when set (tests use it via
148/// `isolate_runs_dir_for_test`); otherwise resolves the real home-relative
149/// path. This function only *computes* a `PathBuf` - it never writes - so both
150/// arms are safe to exercise directly in tests. The write side
151/// ([`append_dashboard_log`] and `Dashboard::add_log`) is what must stay off
152/// the user's real log in tests: `append_dashboard_log`'s own tests set the
153/// override, and `Dashboard` carries an injected log path (a temp dir under
154/// `make_test_dashboard`) so no dashboard-input test ever appends to the real
155/// `~/.leviath/dashboard.log`.
156pub fn dashboard_log_path() -> PathBuf {
157    match std::env::var("LEVIATH_DASHBOARD_LOG_PATH") {
158        Ok(path) => dashboard_log_path_from(Some(&path)),
159        Err(_) => dashboard_log_path_from(None),
160    }
161}
162
163/// Append a timestamped line to the persistent dashboard activity log at the
164/// default [`dashboard_log_path`]. Silently ignores I/O errors - best-effort.
165pub fn append_dashboard_log(msg: &str) {
166    append_dashboard_log_to(&dashboard_log_path(), msg);
167}
168
169/// Append a timestamped line to the dashboard activity log at an explicit
170/// `path`. Silently ignores I/O errors - the dashboard log is best-effort.
171///
172/// The path is a parameter so `Dashboard` can inject a test-isolated log
173/// location, guaranteeing no dashboard-input test appends to the user's real
174/// `~/.leviath/dashboard.log` (see [`dashboard_log_path`]).
175pub fn append_dashboard_log_to(path: &Path, msg: &str) {
176    append_dashboard_log_capped(path, msg, DASHBOARD_LOG_MAX_BYTES);
177}
178
179/// The dashboard log is capped at this size; once the live file reaches it, the
180/// file is rolled (see [`roll_log_if_over_cap`]) so it can't grow without bound
181/// across a long-lived daemon's lifetime.
182const DASHBOARD_LOG_MAX_BYTES: u64 = 5 * 1024 * 1024;
183
184/// Append with an explicit cap (the public entry points use
185/// [`DASHBOARD_LOG_MAX_BYTES`]; tests pass a small cap to exercise rolling).
186fn append_dashboard_log_capped(path: &Path, msg: &str, max_bytes: u64) {
187    use std::io::Write;
188    // Ensure the parent directory exists (first-run case).
189    if let Some(parent) = path.parent() {
190        let _ = std::fs::create_dir_all(parent);
191    }
192    roll_log_if_over_cap(path, max_bytes);
193    if let Ok(mut file) = std::fs::OpenOptions::new()
194        .create(true)
195        .append(true)
196        .open(path)
197    {
198        let timestamp = chrono::Local::now().format("%Y-%m-%d %H:%M:%S");
199        let _ = writeln!(file, "{} {}", timestamp, msg);
200    }
201}
202
203/// The path the rolled (previous-generation) log is moved to: `<name>.1`.
204fn rolled_log_path(path: &Path) -> PathBuf {
205    let mut name = path.as_os_str().to_owned();
206    name.push(".1");
207    PathBuf::from(name)
208}
209
210/// Roll the live log to `<name>.1` once it reaches `max_bytes`, replacing any
211/// existing rolled file, so the live file restarts empty and at most one
212/// previous generation is retained (bounded ~2×cap on disk). Best-effort - a
213/// failed rename just leaves the log to keep growing rather than erroring.
214fn roll_log_if_over_cap(path: &Path, max_bytes: u64) {
215    let over = std::fs::metadata(path)
216        .map(|m| m.len() >= max_bytes)
217        .unwrap_or(false);
218    if over {
219        let _ = std::fs::rename(path, rolled_log_path(path));
220    }
221}
222
223/// How many random bits go in a run ID's suffix, rendered as 12 hex digits.
224/// Collisions only matter within one wall-clock second for one agent name, so 48
225/// bits is many orders of magnitude more than needed while staying short enough
226/// to read in `lev ps` and the dashboard.
227const RUN_ID_ENTROPY_BITS: u32 = 48;
228
229/// Generate a unique run ID: `<agent_name>-<timestamp>-<random>`.
230///
231/// The suffix is **random**, not derived. A derived suffix like
232/// `(now ^ (now >> 16) ^ counter)` over a process-local counter defends a
233/// `lev run --count N` batch inside one process but degenerates to a pure
234/// function of the current second across separate processes: three concurrent
235/// `lev run` invocations all mint `fetcher-1785127214-8b48` and silently share
236/// one run directory. Nothing downstream detects that - `create_dir_all` is a
237/// no-op on an existing directory and the persistence worker then
238/// last-writer-wins over `meta.json` / `context.json` / `run.lvr`, interleaving
239/// two runs' state irrecoverably.
240///
241/// The `<name>-<secs>-<hex>` shape is preserved: the timestamp keeps IDs sorting
242/// and reading chronologically, and the dashboard's short-ID display
243/// (`split('-').next_back()`) still lands on the unique component.
244pub fn new_run_id(agent_name: &str) -> String {
245    use rand::RngExt as _;
246    let entropy: u64 = rand::rng().random::<u64>() >> (u64::BITS - RUN_ID_ENTROPY_BITS);
247    let safe_name = agent_name.replace(|c: char| !c.is_alphanumeric() && c != '-', "-");
248    format!("{}-{}-{:012x}", safe_name, now_secs(), entropy)
249}
250
251/// Create the run directory and write initial metadata.
252pub fn create_run(meta: &RunMeta) -> anyhow::Result<()> {
253    create_run_in(&run_dir(&meta.run_id), meta)
254}
255
256/// Create an explicit run directory and write initial metadata into it.
257///
258/// Callers that already know the directory should prefer this over
259/// [`create_run`], which resolves it from the home directory - the daemon's
260/// spawner stakes out the run dir under its own configured `runs_dir`.
261pub(crate) fn create_run_in(dir: &std::path::Path, meta: &RunMeta) -> anyhow::Result<()> {
262    std::fs::create_dir_all(dir)?;
263
264    // Restrict the run directory to owner-only (no-op on non-Unix).
265    let _ = leviath_sys::secure_dir_perms(dir);
266
267    write_meta_to(dir, meta)
268}
269
270/// Atomically write run metadata (write to tmp, then rename).
271pub fn write_meta(meta: &RunMeta) -> anyhow::Result<()> {
272    write_meta_to(&run_dir(&meta.run_id), meta)
273}
274
275/// Atomically write `meta.json` into an explicit run directory.
276///
277/// Callers that already know the directory should prefer this over
278/// [`write_meta`], which resolves it from the home directory - the daemon's
279/// recovery pass works from its configured `runs_dir` instead.
280pub(crate) fn write_meta_to(dir: &std::path::Path, meta: &RunMeta) -> anyhow::Result<()> {
281    let json =
282        serde_json::to_string_pretty(meta).expect("infallible: RunMeta always serializes to JSON");
283    write_json_atomic(&dir.join("meta.json"), &json)
284}
285
286/// Read run metadata for a given run ID.
287pub fn read_meta(run_id: &str) -> anyhow::Result<RunMeta> {
288    read_meta_from(&run_dir(run_id))
289}
290
291/// Whether an on-disk run status means the run has finished and should be left
292/// alone. `Starting`/`Running`/`WaitingInput` are all "still going" as far as
293/// anything reading the runs dir is concerned.
294pub fn is_terminal_status(status: &RunStatus) -> bool {
295    matches!(
296        status,
297        RunStatus::Complete
298            | RunStatus::CompleteInteractive
299            | RunStatus::Error
300            | RunStatus::Cancelled
301    )
302}
303
304/// How long a run may claim to be live on disk, while the daemon is not holding
305/// it, before anything treats it as abandoned.
306///
307/// Comfortably longer than the persistence heartbeat, so a live-but-slow run (a
308/// long inference writes nothing else) is never mistaken for a dead one.
309pub const STALE_AFTER_SECS: i64 = 300;
310
311/// Whether a run that claims to be live on disk has nothing driving it: the
312/// daemon is not holding it *and* it has not moved in [`STALE_AFTER_SECS`].
313///
314/// `live` is the set of run ids the daemon reports hosting, or `None` when it
315/// gave no answer this poll. Both halves are needed and each is wrong on its
316/// own. An unreachable daemon reports an empty set, so the id check alone would
317/// condemn every healthy run the moment the daemon restarted. And a run parked
318/// on a long inference legitimately does not move for minutes, so the clock
319/// alone would condemn a run that is working. `None` therefore answers `false`
320/// for everything: no answer is not evidence.
321///
322/// Ages against `last_progress_at`, falling back to `updated_at` for runs
323/// written before that field existed. The fallback preserves the older, weaker
324/// behavior for old runs rather than declaring them all stale at once.
325///
326/// One definition, shared by the dashboard's STALE badge and by `lev ps --all`,
327/// so what an operator sees and what a harness reconciles against cannot drift.
328pub fn looks_abandoned(
329    meta: &RunMeta,
330    live: Option<&std::collections::HashSet<String>>,
331    now: i64,
332) -> bool {
333    let Some(live) = live else {
334        return false; // no answer from the daemon; assume nothing
335    };
336    if is_terminal_status(&meta.status) || live.contains(&meta.run_id) {
337        return false;
338    }
339    let moved_at = meta.last_progress_at.unwrap_or(meta.updated_at);
340    now.saturating_sub(moved_at) > STALE_AFTER_SECS
341}
342
343/// The outcome of forcing a run to a terminal state on disk.
344#[derive(Debug, Clone, Copy, PartialEq, Eq)]
345pub enum ForceCancelOutcome {
346    /// The run was live on disk and is now recorded terminal.
347    Terminated,
348    /// The run was already finished; nothing was written.
349    AlreadyTerminal,
350    /// No run directory with that id exists.
351    NoSuchRun,
352    /// The directory exists but its metadata could not be rewritten.
353    WriteFailed,
354}
355
356impl ForceCancelOutcome {
357    /// Whether the id named a run at all - i.e. whether the cancel had a target,
358    /// regardless of whether it needed to write anything.
359    pub fn found_run(&self) -> bool {
360        !matches!(self, Self::NoSuchRun)
361    }
362}
363
364/// Force a run's on-disk metadata to `Cancelled`, in the runs dir resolved from
365/// the environment. See [`force_cancel_in`].
366pub fn force_cancel(run_id: &str) -> ForceCancelOutcome {
367    force_cancel_in(&run_dir(run_id), now_secs())
368}
369
370/// Force the run in `run_dir` to `Cancelled`, stamping `updated_at` with `now`.
371///
372/// This is the floor under every kill path: it needs nothing but the filesystem,
373/// so it works for a run the daemon can't rebuild (blueprint deleted, metadata
374/// corrupt, died mid-spawn) and for a run whose daemon is gone entirely. Both
375/// the daemon's force-terminator seam and `lev cancel --force` route here so
376/// there is one definition of "terminated on disk".
377///
378/// A directory whose `meta.json` is missing or unparseable still gets a minimal
379/// `Cancelled` record written: such a run is otherwise skipped by `list_runs`,
380/// which makes it invisible *and* permanent.
381pub fn force_cancel_in(run_dir: &Path, now: i64) -> ForceCancelOutcome {
382    force_terminal_in(run_dir, RunStatus::Cancelled, None, now)
383}
384
385/// Force the run in `run_dir` to `Error` with `message`, stamping `updated_at`.
386///
387/// For the spawn that never became a run. The spawner stakes out the run
388/// directory and writes a `Starting` placeholder *before* building the agent, so
389/// a spawn that fails leaves something to diagnose - but `Starting` is not
390/// terminal, so that placeholder went on claiming the run was alive for ever,
391/// showing up in `lev ps` and the dashboard with nothing behind it (issue #190).
392/// Recording the failure where the placeholder is turns it into an answer.
393pub fn force_error_in(run_dir: &Path, message: &str, now: i64) -> ForceCancelOutcome {
394    force_terminal_in(run_dir, RunStatus::Error, Some(message.to_string()), now)
395}
396
397/// Rewrite the run in `run_dir` to a terminal `status`, attaching `error` when
398/// there is something to say. Shared by [`force_cancel_in`] and
399/// [`force_error_in`] so "terminated on disk" has one implementation.
400fn force_terminal_in(
401    run_dir: &Path,
402    status: RunStatus,
403    error: Option<String>,
404    now: i64,
405) -> ForceCancelOutcome {
406    if !run_dir.is_dir() {
407        return ForceCancelOutcome::NoSuchRun;
408    }
409    let run_id = run_dir
410        .file_name()
411        .map(|n| n.to_string_lossy().into_owned())
412        .unwrap_or_default();
413    let terminated = match read_meta_from(run_dir) {
414        Ok(meta) if is_terminal_status(&meta.status) => return ForceCancelOutcome::AlreadyTerminal,
415        Ok(meta) => RunMeta {
416            status,
417            updated_at: now,
418            // Keep whatever the run had already recorded when there is nothing
419            // new to say (the cancel path).
420            error: error.clone().or(meta.error),
421            ..meta
422        },
423        // Unreadable metadata: synthesize just enough to record the outcome. The
424        // run id is the directory name, which is the one field always recoverable.
425        Err(_) => RunMeta {
426            status,
427            updated_at: now,
428            error: Some(
429                error
430                    .clone()
431                    .unwrap_or_else(|| "run metadata was unreadable; cancelled".to_string()),
432            ),
433            ..RunMeta::new(
434                run_id.clone(),
435                run_id,
436                String::new(),
437                String::new(),
438                None,
439                String::new(),
440                0,
441            )
442        },
443    };
444    match write_meta_to(run_dir, &terminated) {
445        Ok(()) => ForceCancelOutcome::Terminated,
446        Err(e) => {
447            // Formatted outside the macro: a method call inside a `%field` is
448            // only evaluated when a subscriber visits the value, so it would go
449            // unexercised under the tests' no-op subscriber.
450            let path = run_dir.display().to_string();
451            tracing::warn!(
452                run_dir = %path,
453                error = %e,
454                "could not force a run to a terminal state on disk"
455            );
456            ForceCancelOutcome::WriteFailed
457        }
458    }
459}
460
461/// Read run metadata out of an explicit run directory (the daemon works from its
462/// own configured `runs_dir` rather than the home-resolved one).
463pub(crate) fn read_meta_from(dir: &std::path::Path) -> anyhow::Result<RunMeta> {
464    let path = dir.join("meta.json");
465    let json = std::fs::read_to_string(&path)?;
466    Ok(serde_json::from_str(&json)?)
467}
468
469/// Inner implementation of `list_runs`, parameterised so the early-return
470/// branch can be exercised in tests without deleting real on-disk state.
471fn list_runs_in_dir(dir: PathBuf) -> Vec<RunMeta> {
472    if !dir.exists() {
473        return Vec::new();
474    }
475
476    let mut runs = Vec::new();
477
478    if let Ok(entries) = std::fs::read_dir(&dir) {
479        for entry in entries.filter_map(|e| e.ok()) {
480            let meta_path = entry.path().join("meta.json");
481            if let Ok(json) = std::fs::read_to_string(&meta_path)
482                && let Ok(meta) = serde_json::from_str::<RunMeta>(&json)
483            {
484                runs.push(meta);
485            }
486        }
487    }
488
489    runs.sort_by_key(|r| std::cmp::Reverse(r.started_at));
490    runs
491}
492
493/// List all runs, sorted by started_at descending (most recent first).
494/// Silently skips any runs whose metadata cannot be read.
495pub fn list_runs() -> Vec<RunMeta> {
496    list_runs_in_dir(runs_dir())
497}
498
499/// Read the last `max_bytes` of any file on disk, returning UTF-8 text.
500/// If the file is smaller than `max_bytes` the whole file is returned.
501/// Partial UTF-8 at the truncation boundary is handled by skipping to the
502/// first newline.  Returns an empty string on any I/O error.
503pub fn tail_file(path: &std::path::Path, max_bytes: u64) -> String {
504    use std::io::{Read, Seek, SeekFrom};
505
506    let mut file = match std::fs::File::open(path) {
507        Ok(f) => f,
508        Err(_) => return String::new(),
509    };
510
511    // Use fstat on the open fd rather than a separate stat() call - avoids the
512    // TOCTOU window between existence check and metadata read. Falls back to 0
513    // (read everything) if fstat somehow fails on an already-open fd.
514    let file_size = file.metadata().map(|m| m.len()).unwrap_or(0);
515
516    if file_size <= max_bytes {
517        let mut buf = Vec::new();
518        let _ = file.read_to_end(&mut buf);
519        return String::from_utf8_lossy(&buf).to_string();
520    }
521
522    let offset = file_size - max_bytes;
523    let _ = file.seek(SeekFrom::Start(offset));
524
525    let mut buf = Vec::new();
526    let _ = file.read_to_end(&mut buf);
527
528    // Skip to the first newline so we don't emit a partial line at the start.
529    if let Some(nl) = buf.iter().position(|&b| b == b'\n') {
530        String::from_utf8_lossy(&buf[nl + 1..]).to_string()
531    } else {
532        String::from_utf8_lossy(&buf).to_string()
533    }
534}
535
536// ─── Per-stage persistence ────────────────────────────────────────────────────
537
538/// Directory for per-stage files within a run.
539pub fn stage_dir(run_id: &str, stage_idx: usize) -> PathBuf {
540    run_dir(run_id).join("stages").join(stage_idx.to_string())
541}
542
543/// Atomically write the stages index for a run.
544pub fn write_stages_index(run_id: &str, stages: &[StageRecord]) -> anyhow::Result<()> {
545    write_stages_index_to(&run_dir(run_id), stages)
546}
547
548fn write_stages_index_to(dir: &std::path::Path, stages: &[StageRecord]) -> anyhow::Result<()> {
549    let json = serde_json::to_string_pretty(&stages)
550        .expect("infallible: StageRecord slice always serializes to JSON");
551    write_json_atomic(&dir.join("stages.json"), &json)
552}
553
554/// Read the stages index for a run, or return an empty vec on any error.
555pub fn read_stages_index(run_id: &str) -> Vec<StageRecord> {
556    let path = run_dir(run_id).join("stages.json");
557    let json = match std::fs::read_to_string(&path) {
558        Ok(j) => j,
559        Err(_) => return Vec::new(),
560    };
561    serde_json::from_str(&json).unwrap_or_default()
562}
563
564/// Ensure the per-stage directory exists (called before first write).
565fn ensure_stage_dir(run_id: &str, stage_idx: usize) {
566    let dir = stage_dir(run_id, stage_idx);
567    let _ = std::fs::create_dir_all(&dir);
568}
569
570/// Append a line of readable agent output to the per-stage output log.
571pub fn append_stage_output(run_id: &str, stage_idx: usize, text: &str) {
572    use std::io::Write;
573    ensure_stage_dir(run_id, stage_idx);
574    let path = stage_dir(run_id, stage_idx).join("output.log");
575    if let Ok(mut file) = std::fs::OpenOptions::new()
576        .create(true)
577        .append(true)
578        .open(&path)
579    {
580        let _ = writeln!(file, "{}", text);
581    }
582}
583
584/// Append a line of operational/tool-activity log to the per-stage logs file.
585pub fn append_stage_log(run_id: &str, stage_idx: usize, text: &str) {
586    use std::io::Write;
587    ensure_stage_dir(run_id, stage_idx);
588    let path = stage_dir(run_id, stage_idx).join("logs.log");
589    if let Ok(mut file) = std::fs::OpenOptions::new()
590        .create(true)
591        .append(true)
592        .open(&path)
593    {
594        let _ = writeln!(file, "{}", text);
595    }
596}
597
598/// Atomically write a context snapshot for a specific stage.
599pub fn write_stage_context(
600    run_id: &str,
601    stage_idx: usize,
602    snap: &ContextSnapshot,
603) -> anyhow::Result<()> {
604    ensure_stage_dir(run_id, stage_idx);
605    write_context_snapshot_to(&stage_dir(run_id, stage_idx), snap)
606}
607
608/// Read the context snapshot for a specific stage, if present.
609pub fn read_stage_context(run_id: &str, stage_idx: usize) -> Option<ContextSnapshot> {
610    let path = stage_dir(run_id, stage_idx).join("context.json");
611    let json = std::fs::read_to_string(&path).ok()?;
612    serde_json::from_str(&json).ok()
613}
614
615/// Read the last `max_bytes` of the readable output log for a specific stage.
616pub fn tail_stage_output(run_id: &str, stage_idx: usize, max_bytes: u64) -> String {
617    tail_file(&stage_dir(run_id, stage_idx).join("output.log"), max_bytes)
618}
619
620/// Read the last `max_bytes` of the operational log for a specific stage.
621pub fn tail_stage_log(run_id: &str, stage_idx: usize, max_bytes: u64) -> String {
622    tail_file(&stage_dir(run_id, stage_idx).join("logs.log"), max_bytes)
623}
624
625/// Build the isolated base directory for a run-state test and create its
626/// `runs/` subdir. Returned so the caller's closure can plant fixtures under it.
627///
628/// Rooted under `~/.leviath-test/rs-<hash>` rather than `std::env::temp_dir()`:
629/// some dashboard render tests display a real on-disk path inside a fixed-width
630/// terminal area and assert on a substring near its *end*, and macOS's real
631/// temp dir (`/var/folders/xy/.../T/`) is long enough to push realistic paths
632/// past the render width and truncate the asserted suffix. `unique` is hashed
633/// short for the same reason (test names run 60+ chars). `.leviath-test` is a
634/// sibling of `.leviath`, never read by `lev dash`/`lev serve`, so even if a
635/// killed test process skips cleanup it can't leak into the real dashboard.
636#[cfg(test)]
637fn make_runs_base_dir(unique: &str) -> std::path::PathBuf {
638    use std::hash::{Hash, Hasher};
639    let mut hasher = std::collections::hash_map::DefaultHasher::new();
640    unique.hash(&mut hasher);
641    let short = format!("{:x}", hasher.finish() & 0xffff_ffff);
642    let base_dir = dirs::home_dir()
643        .unwrap_or_default()
644        .join(".leviath-test")
645        .join(format!("rs-{short}"));
646    let _ = std::fs::create_dir_all(base_dir.join("runs"));
647    base_dir
648}
649
650/// The env overrides that point run-state I/O at `base_dir` instead of the
651/// real `~/.leviath/`. Handed to `temp_env` for scoped set-and-restore.
652#[cfg(test)]
653fn runs_dir_isolation_vars(
654    base_dir: &std::path::Path,
655) -> [(&'static str, Option<std::ffi::OsString>); 2] {
656    [
657        (
658            "LEVIATH_RUNS_DIR",
659            Some(base_dir.join("runs").into_os_string()),
660        ),
661        (
662            "LEVIATH_DASHBOARD_LOG_PATH",
663            Some(base_dir.join("dashboard.log").into_os_string()),
664        ),
665    ]
666}
667
668/// Runs `f` with `LEVIATH_RUNS_DIR`/`LEVIATH_DASHBOARD_LOG_PATH` pointed at a
669/// fresh isolated temp directory (passed to `f`), restoring them afterwards.
670/// Closure-scoped (not an RAII guard) because edition 2024 makes `set_var`
671/// `unsafe`, which the crate forbids; `temp_env` serializes it process-wide.
672#[cfg(test)]
673pub(crate) fn with_isolated_runs_dir<R>(unique: &str, f: impl FnOnce(&std::path::Path) -> R) -> R {
674    let base_dir = make_runs_base_dir(unique);
675    let result = temp_env::with_vars(runs_dir_isolation_vars(&base_dir), || f(&base_dir));
676    let _ = std::fs::remove_dir_all(&base_dir);
677    result
678}
679
680/// Async counterpart of [`with_isolated_runs_dir`] for `#[tokio::test]`s.
681#[cfg(test)]
682pub(crate) async fn with_isolated_runs_dir_async<R, Fut>(
683    unique: &str,
684    f: impl FnOnce(std::path::PathBuf) -> Fut,
685) -> R
686where
687    Fut: std::future::Future<Output = R>,
688{
689    let base_dir = make_runs_base_dir(unique);
690    let result =
691        temp_env::async_with_vars(runs_dir_isolation_vars(&base_dir), f(base_dir.clone())).await;
692    let _ = std::fs::remove_dir_all(&base_dir);
693    result
694}
695
696#[cfg(test)]
697mod tests {
698    use super::*;
699
700    /// `run_id` arrives from URL segments on `GET /api/agents/{id}/logs` and
701    /// friends. `Path::join` neither normalizes `..` nor resists an absolute
702    /// path, so an unvalidated id read files anywhere. An unsafe one resolves to
703    /// a name that cannot exist, giving the caller a plain miss.
704    #[test]
705    fn run_dir_refuses_an_unsafe_run_id() {
706        crate::test_support::with_tracing(|| {
707            for bad in ["../../etc", "/etc/passwd", "..", "a/b"] {
708                let dir = run_dir(bad);
709                let shown = dir.display().to_string();
710                assert!(dir.ends_with("<invalid>"), "{bad} resolved to {shown}");
711                assert!(!dir.exists(), "{bad} must not resolve to a real path");
712            }
713            // An ordinary id is untouched.
714            assert!(run_dir("run-abc123").ends_with("run-abc123"));
715        });
716    }
717
718    // ─── looks_abandoned ────────────────────────────────────────────────────
719
720    /// A run claiming to be live on disk, last moved at 1000.
721    fn live_on_disk(run_id: &str) -> RunMeta {
722        let mut meta = RunMeta::new(
723            run_id.to_string(),
724            "coder".to_string(),
725            "/agents/coder".to_string(),
726            "t".to_string(),
727            None,
728            "/w".to_string(),
729            1,
730        );
731        meta.status = RunStatus::Running;
732        meta.updated_at = 1_000;
733        meta.last_progress_at = Some(1_000);
734        meta
735    }
736
737    fn held(ids: &[&str]) -> std::collections::HashSet<String> {
738        ids.iter().map(|s| (*s).to_string()).collect()
739    }
740
741    /// The shape issue #202 reported: disk says running, the daemon is not
742    /// hosting it, and it has not moved in a long time.
743    #[test]
744    fn a_run_nothing_is_driving_looks_abandoned() {
745        let meta = live_on_disk("r1");
746        assert!(looks_abandoned(
747            &meta,
748            Some(&held(&["other"])),
749            1_000 + STALE_AFTER_SECS + 1
750        ));
751    }
752
753    /// The arm that decides whether a reconciler is safe to run at all. A daemon
754    /// that is restarting gives no answer, which looks exactly like every run
755    /// dying at once; anything that acted on it would cancel a whole factory.
756    #[test]
757    fn no_answer_from_the_daemon_condemns_nothing() {
758        let meta = live_on_disk("r1");
759        assert!(!looks_abandoned(
760            &meta,
761            None,
762            1_000 + STALE_AFTER_SECS * 100
763        ));
764    }
765
766    #[test]
767    fn a_run_the_daemon_is_hosting_is_never_abandoned() {
768        let meta = live_on_disk("r1");
769        assert!(!looks_abandoned(
770            &meta,
771            Some(&held(&["r1"])),
772            1_000 + STALE_AFTER_SECS * 100
773        ));
774    }
775
776    /// A run parked on a long inference has not moved and is still working, so
777    /// the window has to be wider than the persistence heartbeat.
778    #[test]
779    fn a_slow_run_inside_the_window_is_left_alone() {
780        let meta = live_on_disk("r1");
781        assert!(!looks_abandoned(
782            &meta,
783            Some(&held(&[])),
784            1_000 + STALE_AFTER_SECS - 1
785        ));
786    }
787
788    /// A finished run is not abandoned, it is done. The daemon unloads it within
789    /// seconds of it going terminal, so it is absent from the live set for the
790    /// rest of time and would otherwise trip every other check here.
791    #[test]
792    fn a_finished_run_is_not_abandoned() {
793        for status in [
794            RunStatus::Complete,
795            RunStatus::CompleteInteractive,
796            RunStatus::Error,
797            RunStatus::Cancelled,
798        ] {
799            let mut meta = live_on_disk("r1");
800            meta.status = status.clone();
801            assert!(
802                !looks_abandoned(&meta, Some(&held(&[])), 1_000 + STALE_AFTER_SECS * 100),
803                "{status} is finished, not abandoned"
804            );
805        }
806    }
807
808    /// The progress stamp wins over the heartbeat. A wedged run keeps rewriting
809    /// `updated_at` every 30 seconds, so judging on it would never age anything
810    /// out, which is the reason issue #202 could not be fixed from meta.json
811    /// before the stamp existed.
812    #[test]
813    fn a_fresh_heartbeat_does_not_rescue_a_run_that_stopped_moving() {
814        let mut meta = live_on_disk("r1");
815        let now = 1_000 + STALE_AFTER_SECS * 10;
816        meta.updated_at = now; // the heartbeat, still beating
817        meta.last_progress_at = Some(1_000); // but nothing has moved since 1000
818        assert!(looks_abandoned(&meta, Some(&held(&[])), now));
819    }
820
821    /// A run written before the stamp existed falls back to `updated_at`, so old
822    /// runs keep the older, weaker behavior instead of all reading as stale.
823    #[test]
824    fn a_run_without_the_stamp_falls_back_to_updated_at() {
825        let mut meta = live_on_disk("r1");
826        meta.last_progress_at = None;
827        meta.updated_at = 1_000;
828        assert!(looks_abandoned(
829            &meta,
830            Some(&held(&[])),
831            1_000 + STALE_AFTER_SECS + 1
832        ));
833        meta.updated_at = 1_000 + STALE_AFTER_SECS;
834        assert!(!looks_abandoned(
835            &meta,
836            Some(&held(&[])),
837            1_000 + STALE_AFTER_SECS + 1
838        ));
839    }
840
841    #[test]
842    fn write_json_atomic_fs_write_failure() {
843        // Drive the `std::fs::write(&tmp, json)?` error arm: writing the
844        // `.json.tmp` sibling into a directory that does not exist fails.
845        let path = std::path::Path::new("/nonexistent/leviath/runstate-cov/out.json");
846        let result = write_json_atomic(path, "{}");
847        assert!(result.is_err());
848        assert!(!path.exists());
849    }
850
851    // ─── RunStatus ──────────────────────────────────────────────────────────
852
853    #[test]
854    fn run_status_serde_roundtrip() {
855        for status in [
856            RunStatus::Starting,
857            RunStatus::Running,
858            RunStatus::WaitingInput,
859            RunStatus::Complete,
860            RunStatus::CompleteInteractive,
861            RunStatus::Paused,
862            RunStatus::Error,
863            RunStatus::Cancelled,
864        ] {
865            let json = serde_json::to_string(&status).unwrap();
866            let back: RunStatus = serde_json::from_str(&json).unwrap();
867            assert_eq!(status, back);
868        }
869    }
870
871    #[test]
872    fn run_status_display() {
873        assert_eq!(RunStatus::Starting.to_string(), "Starting");
874        assert_eq!(RunStatus::Running.to_string(), "Running");
875        assert_eq!(RunStatus::WaitingInput.to_string(), "WaitingInput");
876        assert_eq!(RunStatus::Complete.to_string(), "Complete");
877        assert_eq!(
878            RunStatus::CompleteInteractive.to_string(),
879            "CompleteInteractive"
880        );
881        assert_eq!(RunStatus::Paused.to_string(), "Paused");
882        assert_eq!(RunStatus::Error.to_string(), "Error");
883        assert_eq!(RunStatus::Cancelled.to_string(), "Cancelled");
884    }
885
886    #[test]
887    fn run_status_snake_case_serialization() {
888        let json = serde_json::to_string(&RunStatus::WaitingInput).unwrap();
889        assert_eq!(json, "\"waiting_input\"");
890        let json = serde_json::to_string(&RunStatus::CompleteInteractive).unwrap();
891        assert_eq!(json, "\"complete_interactive\"");
892    }
893
894    // ─── StageRunStatus ─────────────────────────────────────────────────────
895
896    #[test]
897    fn stage_run_status_serde_roundtrip() {
898        for status in [
899            StageRunStatus::Pending,
900            StageRunStatus::Active,
901            StageRunStatus::WaitingInput,
902            StageRunStatus::Complete,
903            StageRunStatus::Error,
904        ] {
905            let json = serde_json::to_string(&status).unwrap();
906            let back: StageRunStatus = serde_json::from_str(&json).unwrap();
907            assert_eq!(status, back);
908        }
909    }
910
911    #[test]
912    fn stage_run_status_display() {
913        assert_eq!(StageRunStatus::Pending.to_string(), "Pending");
914        assert_eq!(StageRunStatus::Active.to_string(), "Active");
915        assert_eq!(StageRunStatus::WaitingInput.to_string(), "WaitingInput");
916        assert_eq!(StageRunStatus::Complete.to_string(), "Complete");
917        assert_eq!(StageRunStatus::Error.to_string(), "Error");
918    }
919
920    // ─── RunMeta ────────────────────────────────────────────────────────────
921
922    #[test]
923    fn run_meta_new_defaults() {
924        let meta = RunMeta::new(
925            "run-1".into(),
926            "agent".into(),
927            "/path".into(),
928            "do stuff".into(),
929            Some("gpt-4".into()),
930            "/work".into(),
931            3,
932        );
933        assert_eq!(meta.run_id, "run-1");
934        assert_eq!(meta.agent_name, "agent");
935        assert_eq!(meta.task, "do stuff");
936        assert_eq!(meta.model.as_deref(), Some("gpt-4"));
937        assert_eq!(meta.num_stages, 3);
938        assert_eq!(meta.status, RunStatus::Starting);
939        assert_eq!(meta.pid, 0);
940        assert_eq!(meta.stage_index, 0);
941        assert!(meta.error.is_none());
942        assert!(meta.title.is_none());
943        assert!(meta.metadata.is_empty());
944        assert!(meta.callback_url.is_none());
945        assert!(meta.parent_run_id.is_none());
946    }
947
948    #[test]
949    fn run_meta_serde_roundtrip() {
950        let meta = RunMeta::new(
951            "test-run".into(),
952            "test-agent".into(),
953            "/agents/test".into(),
954            "run tests".into(),
955            None,
956            "/tmp".into(),
957            2,
958        );
959        let json = serde_json::to_string_pretty(&meta).unwrap();
960        let back: RunMeta = serde_json::from_str(&json).unwrap();
961        assert_eq!(back.run_id, "test-run");
962        assert_eq!(back.agent_name, "test-agent");
963        assert_eq!(back.num_stages, 2);
964        assert!(back.model.is_none());
965    }
966
967    #[test]
968    fn run_meta_touch_updates_timestamp() {
969        let mut meta = RunMeta::new(
970            "r".into(),
971            "a".into(),
972            "/p".into(),
973            "t".into(),
974            None,
975            "/w".into(),
976            1,
977        );
978        let before = meta.updated_at;
979        // Touch should update (or at least not decrease) updated_at
980        meta.touch();
981        assert!(meta.updated_at >= before);
982    }
983
984    #[test]
985    fn run_meta_optional_fields_deserialize() {
986        // Simulate a meta.json without optional fields (e.g., from older version)
987        let json = serde_json::json!({
988            "run_id": "r1",
989            "agent_name": "a",
990            "agent_path": "/p",
991            "task": "t",
992            "model": null,
993            "pid": 123,
994            "status": "running",
995            "current_stage": "init",
996            "stage_index": 0,
997            "num_stages": 1,
998            "iteration": 0,
999            "prompt_tokens": 0,
1000            "completion_tokens": 0,
1001            "workdir": "/w",
1002            "started_at": 1000,
1003            "updated_at": 1000,
1004            "error": null
1005        });
1006        let meta: RunMeta = serde_json::from_value(json).unwrap();
1007        assert_eq!(meta.cached_tokens, 0);
1008        assert!(meta.title.is_none());
1009        assert!(meta.metadata.is_empty());
1010        assert!(meta.callback_url.is_none());
1011        assert!(meta.parent_run_id.is_none());
1012        // A run written before the progress stamp existed has no answer, which is
1013        // why the field is an Option: `Some(0)` would read as "last moved in 1970"
1014        // and invite a reconciler to declare it abandoned.
1015        assert!(meta.last_progress_at.is_none());
1016    }
1017
1018    /// `pid` is written by every daemon there has ever been, and is always 0 in
1019    /// the shared world. A file that omits it entirely must still load, so the
1020    /// field can be dropped in a future major without stranding old runs.
1021    #[test]
1022    fn run_meta_without_a_pid_still_loads() {
1023        let json = serde_json::json!({
1024            "run_id": "r1",
1025            "agent_name": "a",
1026            "agent_path": "/p",
1027            "task": "t",
1028            "model": null,
1029            "status": "running",
1030            "current_stage": "init",
1031            "stage_index": 0,
1032            "num_stages": 1,
1033            "iteration": 0,
1034            "prompt_tokens": 0,
1035            "completion_tokens": 0,
1036            "workdir": "/w",
1037            "started_at": 1000,
1038            "updated_at": 1000,
1039            "error": null
1040        });
1041        let meta: RunMeta = serde_json::from_value(json).unwrap();
1042        assert_eq!(meta.pid, 0);
1043    }
1044
1045    // ─── StageRecord ────────────────────────────────────────────────────────
1046
1047    #[test]
1048    fn stage_record_new_defaults() {
1049        let rec = StageRecord::new("analyze".into(), 2);
1050        assert_eq!(rec.name, "analyze");
1051        assert_eq!(rec.index, 2);
1052        assert_eq!(rec.status, StageRunStatus::Pending);
1053        assert_eq!(rec.prompt_tokens, 0);
1054        assert_eq!(rec.completion_tokens, 0);
1055        assert_eq!(rec.cached_tokens, 0);
1056        assert!(rec.started_at.is_none());
1057        assert!(rec.ended_at.is_none());
1058    }
1059
1060    #[test]
1061    fn stage_record_serde_roundtrip() {
1062        let mut rec = StageRecord::new("build".into(), 0);
1063        rec.status = StageRunStatus::Complete;
1064        rec.prompt_tokens = 100;
1065        rec.started_at = Some(1000);
1066        rec.ended_at = Some(2000);
1067
1068        let json = serde_json::to_string(&rec).unwrap();
1069        let back: StageRecord = serde_json::from_str(&json).unwrap();
1070        assert_eq!(back.name, "build");
1071        assert_eq!(back.status, StageRunStatus::Complete);
1072        assert_eq!(back.prompt_tokens, 100);
1073        assert_eq!(back.started_at, Some(1000));
1074    }
1075
1076    // ─── RegionSnapshot / ContextSnapshot ───────────────────────────────────
1077
1078    #[test]
1079    fn region_snapshot_serde_roundtrip() {
1080        let snap = RegionSnapshot {
1081            name: "system".into(),
1082            kind: "pinned".into(),
1083            current_tokens: 100,
1084            max_tokens: 500,
1085            entries: vec![RegionEntrySnapshot {
1086                content: "You are helpful".into(),
1087                tokens: 3,
1088                kind: Default::default(),
1089                metadata: None,
1090                key: None,
1091                taint: Default::default(),
1092            }],
1093        };
1094        let json = serde_json::to_string(&snap).unwrap();
1095        let back: RegionSnapshot = serde_json::from_str(&json).unwrap();
1096        assert_eq!(back.name, "system");
1097        assert_eq!(back.entries.len(), 1);
1098        assert_eq!(back.entries[0].content, "You are helpful");
1099    }
1100
1101    #[test]
1102    fn region_snapshot_empty_entries_omitted() {
1103        let snap = RegionSnapshot {
1104            name: "empty".into(),
1105            kind: "temporary".into(),
1106            current_tokens: 0,
1107            max_tokens: 100,
1108            entries: vec![],
1109        };
1110        let json = serde_json::to_value(&snap).unwrap();
1111        assert!(json.get("entries").is_none());
1112    }
1113
1114    #[test]
1115    fn context_snapshot_serde_roundtrip() {
1116        let snap = ContextSnapshot {
1117            stage_name: "analyze".into(),
1118            total_tokens: 500,
1119            max_tokens: 8192,
1120            regions: vec![RegionSnapshot {
1121                name: "history".into(),
1122                kind: "sliding".into(),
1123                current_tokens: 300,
1124                max_tokens: 2000,
1125                entries: vec![],
1126            }],
1127        };
1128        let json = serde_json::to_string(&snap).unwrap();
1129        let back: ContextSnapshot = serde_json::from_str(&json).unwrap();
1130        assert_eq!(back.stage_name, "analyze");
1131        assert_eq!(back.total_tokens, 500);
1132        assert_eq!(back.regions.len(), 1);
1133    }
1134
1135    // ─── tail_file ──────────────────────────────────────────────────────────
1136
1137    #[test]
1138    fn tail_file_nonexistent_returns_empty() {
1139        let path = std::path::Path::new("/tmp/nonexistent-leviath-test-file.txt");
1140        assert_eq!(tail_file(path, 1024), "");
1141    }
1142
1143    #[test]
1144    fn tail_file_small_file_returns_all() {
1145        let dir = tempfile::tempdir().unwrap();
1146        let path = dir.path().join("small.txt");
1147        std::fs::write(&path, "line1\nline2\nline3\n").unwrap();
1148        let result = tail_file(&path, 1024);
1149        assert_eq!(result, "line1\nline2\nline3\n");
1150    }
1151
1152    #[test]
1153    fn tail_file_large_file_returns_tail() {
1154        let dir = tempfile::tempdir().unwrap();
1155        let path = dir.path().join("large.txt");
1156        let content = "abcdefghij\n".repeat(100); // 1100 bytes
1157        std::fs::write(&path, &content).unwrap();
1158        let result = tail_file(&path, 50);
1159        // Should be less than 50 bytes, starting from a line boundary
1160        assert!(result.len() <= 50);
1161        assert!(result.ends_with('\n'));
1162    }
1163
1164    // ─── new_run_id ─────────────────────────────────────────────────────────
1165
1166    #[test]
1167    fn new_run_id_contains_agent_name() {
1168        let id = new_run_id("my-agent");
1169        assert!(id.starts_with("my-agent-"));
1170    }
1171
1172    #[test]
1173    fn new_run_id_sanitizes_special_chars() {
1174        let id = new_run_id("agent with spaces!");
1175        assert!(!id.contains(' '));
1176        assert!(!id.contains('!'));
1177    }
1178
1179    #[test]
1180    fn new_run_id_is_unique_across_rapid_calls_in_same_second() {
1181        // `--count N` calls `new_run_id` N times in a tight loop, all within the
1182        // same wall-clock second.
1183        let ids: std::collections::HashSet<String> =
1184            (0..100).map(|_| new_run_id("same-agent")).collect();
1185        assert_eq!(ids.len(), 100);
1186    }
1187
1188    /// Split `<name>-<secs>-<hex>` from the right - the agent name itself may
1189    /// contain dashes.
1190    fn split_run_id(id: &str) -> (&str, &str) {
1191        let mut parts = id.rsplitn(3, '-');
1192        let suffix = parts.next().expect("run id has a suffix");
1193        let secs = parts.next().expect("run id has a timestamp");
1194        (secs, suffix)
1195    }
1196
1197    #[test]
1198    fn new_run_id_suffix_is_random_not_derived_from_the_clock() {
1199        // The collision this guards against is *across processes*: a suffix
1200        // derived as `(now ^ (now >> 16) ^ counter)` over a process-local
1201        // counter that every new process starts at 0 degenerates to a pure
1202        // function of the current second. Three concurrent `lev run`
1203        // invocations all mint `fetcher-1785127214-8b48` and silently share
1204        // one run directory. A fresh process has no state to vary, so the
1205        // property that has to hold is: IDs that share a timestamp still differ.
1206        let ids: Vec<String> = (0..200).map(|_| new_run_id("same-agent")).collect();
1207        let mut by_second: std::collections::HashMap<&str, Vec<&str>> =
1208            std::collections::HashMap::new();
1209        for id in &ids {
1210            let (secs, suffix) = split_run_id(id);
1211            by_second.entry(secs).or_default().push(suffix);
1212        }
1213        let mut largest = 0;
1214        for (secs, suffixes) in &by_second {
1215            let distinct: std::collections::HashSet<&&str> = suffixes.iter().collect();
1216            assert_eq!(
1217                distinct.len(),
1218                suffixes.len(),
1219                "two runs in second {secs} share a suffix: {suffixes:?}"
1220            );
1221            largest = largest.max(suffixes.len());
1222        }
1223        // 200 calls take microseconds, so they cannot all land in distinct
1224        // seconds - without this the assertion above would be vacuous.
1225        assert!(
1226            largest > 1,
1227            "expected IDs sharing a second, got {by_second:?}"
1228        );
1229    }
1230
1231    // ─── write_meta / read_meta roundtrip ───────────────────────────────────
1232
1233    #[test]
1234    fn write_and_read_meta_roundtrip() {
1235        // Isolated via `isolate_runs_dir_for_test` so write_meta/read_meta
1236        // never touch the real ~/.leviath/runs/ - the temp dir is removed
1237        // automatically when `_guard` drops, so no manual cleanup needed.
1238        with_isolated_runs_dir("write-and-read-meta-roundtrip", |_d| {
1239            let meta = RunMeta::new(
1240                "test-roundtrip-unit".into(),
1241                "test-agent".into(),
1242                "/agents/test".into(),
1243                "unit test".into(),
1244                Some("model-x".into()),
1245                "/tmp".into(),
1246                2,
1247            );
1248
1249            create_run(&meta).unwrap();
1250            let back = read_meta(&meta.run_id).unwrap();
1251            assert_eq!(back.run_id, "test-roundtrip-unit");
1252            assert_eq!(back.agent_name, "test-agent");
1253            assert_eq!(back.task, "unit test");
1254            assert_eq!(back.model.as_deref(), Some("model-x"));
1255        });
1256    }
1257
1258    #[test]
1259    fn read_meta_returns_err_on_corrupted_json() {
1260        // Exercises `read_meta_from`'s `serde_json::from_str(&json)?` Err
1261        // arm: a `meta.json` that exists but doesn't parse as a `RunMeta`.
1262        with_isolated_runs_dir("read-meta-returns-err-on-corrupted-json", |_d| {
1263            let run_id = "corrupted-meta-run";
1264            let dir = run_dir(run_id);
1265            std::fs::create_dir_all(&dir).unwrap();
1266            std::fs::write(dir.join("meta.json"), "not valid json").unwrap();
1267
1268            let result = read_meta(run_id);
1269            assert!(result.is_err());
1270        });
1271    }
1272
1273    // ─── write_stages_index / read_stages_index roundtrip ───────────────────
1274
1275    #[test]
1276    fn write_and_read_stages_index_roundtrip() {
1277        with_isolated_runs_dir("write-and-read-stages-index-roundtrip", |_d| {
1278            let run_id = "test-stages-idx-unit";
1279            let dir = run_dir(run_id);
1280            std::fs::create_dir_all(&dir).unwrap();
1281
1282            let stages = vec![
1283                StageRecord::new("init".into(), 0),
1284                StageRecord::new("process".into(), 1),
1285            ];
1286            write_stages_index(run_id, &stages).unwrap();
1287            let back = read_stages_index(run_id);
1288            assert_eq!(back.len(), 2);
1289            assert_eq!(back[0].name, "init");
1290            assert_eq!(back[1].name, "process");
1291        });
1292    }
1293
1294    #[test]
1295    fn read_stages_index_missing_returns_empty() {
1296        let back = read_stages_index("nonexistent-run-12345");
1297        assert!(back.is_empty());
1298    }
1299
1300    // ─── write/read context snapshot ────────────────────────────────────────
1301
1302    #[test]
1303    fn write_and_read_context_snapshot_roundtrip() {
1304        with_isolated_runs_dir("write-and-read-context-snapshot-roundtrip", |_d| {
1305            let run_id = "test-ctx-snap-unit";
1306            let dir = run_dir(run_id);
1307            std::fs::create_dir_all(&dir).unwrap();
1308
1309            let snap = ContextSnapshot {
1310                stage_name: "test".into(),
1311                total_tokens: 42,
1312                max_tokens: 8192,
1313                regions: vec![],
1314            };
1315            write_context_snapshot(run_id, &snap).unwrap();
1316            let back = read_context_snapshot(run_id).unwrap();
1317            assert_eq!(back.stage_name, "test");
1318            assert_eq!(back.total_tokens, 42);
1319        });
1320    }
1321
1322    #[test]
1323    fn read_context_snapshot_missing_returns_none() {
1324        assert!(read_context_snapshot("nonexistent-ctx-run").is_none());
1325    }
1326
1327    #[test]
1328    fn read_run_archive_roundtrips_and_context_history_replays() {
1329        with_isolated_runs_dir("read-run-archive-roundtrip", |_d| {
1330            use leviath_core::run_archive::{self, RunIdentity, RunRecord};
1331            let run_id = "archive-unit";
1332            std::fs::create_dir_all(run_dir(run_id)).unwrap();
1333            let mut buf = Vec::new();
1334            run_archive::write_archive_start(&mut buf, run_archive::RUN_ARCHIVE_VERSION).unwrap();
1335            let meta = RunMeta::new(
1336                run_id.to_string(),
1337                "a".to_string(),
1338                "/p".to_string(),
1339                "t".to_string(),
1340                None,
1341                "/w".to_string(),
1342                1,
1343            );
1344            run_archive::write_record(
1345                &mut buf,
1346                &RunRecord::Header {
1347                    identity: RunIdentity {
1348                        run_id: run_id.to_string(),
1349                        machine_id: "m".to_string(),
1350                        world_id: "w".to_string(),
1351                        created_at: 0,
1352                    },
1353                    meta: Box::new(meta),
1354                },
1355            )
1356            .unwrap();
1357            run_archive::write_record(
1358                &mut buf,
1359                &RunRecord::ContextCheckpoint {
1360                    snapshot: ContextSnapshot {
1361                        stage_name: "plan".to_string(),
1362                        total_tokens: 3,
1363                        max_tokens: 100,
1364                        regions: vec![],
1365                    },
1366                    at: 1,
1367                },
1368            )
1369            .unwrap();
1370            std::fs::write(run_dir(run_id).join("run.lvr"), &buf).unwrap();
1371
1372            let records = read_run_archive(run_id).expect("archive read");
1373            assert_eq!(records.len(), 2);
1374            let history = context_history(run_id);
1375            assert_eq!(history.len(), 1);
1376            assert_eq!(history[0].context.stage_name, "plan");
1377        });
1378    }
1379
1380    #[test]
1381    fn read_run_archive_missing_or_corrupt_returns_none() {
1382        with_isolated_runs_dir("read-run-archive-corrupt", |_d| {
1383            // Missing archive.
1384            assert!(read_run_archive("no-such-archive-run").is_none());
1385            assert!(context_history("no-such-archive-run").is_empty());
1386            // Corrupt archive (bad magic) → None, not a panic.
1387            let run_id = "corrupt-archive-unit";
1388            std::fs::create_dir_all(run_dir(run_id)).unwrap();
1389            std::fs::write(run_dir(run_id).join("run.lvr"), b"not an archive").unwrap();
1390            assert!(read_run_archive(run_id).is_none());
1391            assert!(context_history(run_id).is_empty());
1392        });
1393    }
1394
1395    // ─── stage_dir / append_stage_output / append_stage_log ─────────────────
1396
1397    #[test]
1398    fn stage_dir_path_structure() {
1399        let path = stage_dir("run-abc", 2);
1400        assert!(path.ends_with("stages/2"));
1401        assert!(path.to_str().unwrap().contains("run-abc"));
1402    }
1403
1404    #[test]
1405    fn append_and_tail_stage_output() {
1406        with_isolated_runs_dir("append-and-tail-stage-output", |_d| {
1407            let run_id = "test-stage-output-unit";
1408            append_stage_output(run_id, 0, "line 1");
1409            append_stage_output(run_id, 0, "line 2");
1410            let output = tail_stage_output(run_id, 0, 4096);
1411            assert!(output.contains("line 1"));
1412            assert!(output.contains("line 2"));
1413        });
1414    }
1415
1416    #[test]
1417    fn append_and_tail_stage_log() {
1418        with_isolated_runs_dir("append-and-tail-stage-log", |_d| {
1419            let run_id = "test-stage-log-unit";
1420            append_stage_log(run_id, 0, "event A");
1421            append_stage_log(run_id, 0, "event B");
1422            let log = tail_stage_log(run_id, 0, 4096);
1423            assert!(log.contains("event A"));
1424            assert!(log.contains("event B"));
1425        });
1426    }
1427
1428    // ─── write/read stage context ───────────────────────────────────────────
1429
1430    #[test]
1431    fn write_and_read_stage_context_roundtrip() {
1432        with_isolated_runs_dir("write-and-read-stage-context-roundtrip", |_d| {
1433            let run_id = "test-stage-ctx-unit";
1434            let snap = ContextSnapshot {
1435                stage_name: "stage-0".into(),
1436                total_tokens: 100,
1437                max_tokens: 4096,
1438                regions: vec![],
1439            };
1440            write_stage_context(run_id, 0, &snap).unwrap();
1441            let back = read_stage_context(run_id, 0).unwrap();
1442            assert_eq!(back.stage_name, "stage-0");
1443        });
1444    }
1445
1446    #[test]
1447    fn read_stage_context_missing_returns_none() {
1448        assert!(read_stage_context("nonexistent-run", 99).is_none());
1449    }
1450
1451    // ─── append_dashboard_log ─────────────────────────────────────────────
1452
1453    #[test]
1454    fn append_dashboard_log_creates_log_file() {
1455        with_isolated_runs_dir("append-dashboard-log-creates-log-file", |_d| {
1456            append_dashboard_log("coverage-test-message");
1457            assert!(dashboard_log_path().exists());
1458        });
1459    }
1460
1461    #[test]
1462    fn append_dashboard_log_open_failure_is_silently_ignored() {
1463        // Covers the `if let Ok(mut file) = ... .open(&path)` pattern *not*
1464        // matching: pre-create the resolved log path as a directory, so
1465        // opening it for append fails with `IsADirectory` - the function
1466        // must swallow this silently (best-effort logging) rather than
1467        // panic.
1468        with_isolated_runs_dir("append-dashboard-log-open-failure", |_d| {
1469            let path = dashboard_log_path();
1470            std::fs::create_dir_all(&path).unwrap();
1471            append_dashboard_log("this should not panic");
1472            assert!(path.is_dir());
1473        });
1474    }
1475
1476    #[test]
1477    fn append_dashboard_log_path_with_no_parent_skips_create_dir_all() {
1478        // Every other test resolves `dashboard_log_path()` to a path with a
1479        // real parent component, leaving the `if let Some(parent) = ...`
1480        // pattern's `None` arm (root paths like "/" have no parent) never
1481        // exercised. `temp_env::with_var` points the override at "/" for the
1482        // closure's duration (serialized process-wide, then restored).
1483        temp_env::with_var("LEVIATH_DASHBOARD_LOG_PATH", Some("/"), || {
1484            assert!(dashboard_log_path().parent().is_none());
1485            append_dashboard_log("this should not panic even with no parent");
1486        });
1487    }
1488
1489    #[test]
1490    fn dashboard_log_rolls_once_over_cap() {
1491        // A tiny cap so a couple of lines trips the roll. The over-cap live file
1492        // is moved to `<name>.1` and a fresh live file is started.
1493        let dir = tempfile::tempdir().unwrap();
1494        let path = dir.path().join("dashboard.log");
1495        append_dashboard_log_capped(&path, "first line well over the tiny cap", 8);
1496        // First write created the file; it now exceeds the 8-byte cap.
1497        assert!(path.exists());
1498        assert!(!rolled_log_path(&path).exists());
1499        // Second write sees the file over cap → rolls it and restarts.
1500        append_dashboard_log_capped(&path, "second", 8);
1501        let rolled = rolled_log_path(&path);
1502        assert!(rolled.exists(), "previous generation rolled to <name>.1");
1503        assert!(
1504            std::fs::read_to_string(&rolled)
1505                .unwrap()
1506                .contains("first line")
1507        );
1508        // The live file was restarted with only the newest line.
1509        let live = std::fs::read_to_string(&path).unwrap();
1510        assert!(live.contains("second"));
1511        assert!(!live.contains("first line"));
1512    }
1513
1514    #[test]
1515    fn dashboard_log_does_not_roll_under_cap() {
1516        let dir = tempfile::tempdir().unwrap();
1517        let path = dir.path().join("dashboard.log");
1518        append_dashboard_log_capped(&path, "a", 1_000_000);
1519        append_dashboard_log_capped(&path, "b", 1_000_000);
1520        // Both lines are in the single live file; nothing was rolled.
1521        assert!(!rolled_log_path(&path).exists());
1522        let live = std::fs::read_to_string(&path).unwrap();
1523        assert!(live.contains("a") && live.contains("b"));
1524    }
1525
1526    // ─── dashboard_log_path ────────────────────────────────────────────────
1527
1528    #[test]
1529    fn dashboard_log_path_structure() {
1530        // Exercises the real (env-reading) `dashboard_log_path()` on its
1531        // fallback branch, so - like `runs_dir_structure` below - it forces
1532        // `LEVIATH_DASHBOARD_LOG_PATH` unset via `temp_env::with_var_unset`,
1533        // which also serializes against every other temp-env test so a
1534        // concurrently-isolated test can't race this assertion.
1535        temp_env::with_var_unset("LEVIATH_DASHBOARD_LOG_PATH", || {
1536            let path = dashboard_log_path();
1537            assert!(path.to_str().unwrap().contains(".leviath"));
1538            assert!(path.to_str().unwrap().ends_with("dashboard.log"));
1539        });
1540    }
1541
1542    /// With no `LEVIATH_DASHBOARD_LOG_PATH`, the dashboard log must follow
1543    /// `LEVIATH_HOME` like every other data path. Resolving through the raw
1544    /// OS home would leave a fully isolated test session still appending to
1545    /// the developer's real `~/.leviath/dashboard.log`.
1546    #[test]
1547    fn dashboard_log_path_honors_leviath_home() {
1548        temp_env::with_vars(
1549            [
1550                ("LEVIATH_DASHBOARD_LOG_PATH", None),
1551                ("LEVIATH_HOME", Some("/custom/home")),
1552            ],
1553            || {
1554                assert_eq!(
1555                    dashboard_log_path(),
1556                    PathBuf::from("/custom/home/.leviath/dashboard.log")
1557                );
1558            },
1559        );
1560    }
1561
1562    // ─── runs_dir / run_dir ────────────────────────────────────────────────
1563
1564    #[test]
1565    fn runs_dir_structure() {
1566        // See the comment on `dashboard_log_path_structure` above - same
1567        // race, same fix, for `LEVIATH_RUNS_DIR`.
1568        temp_env::with_var_unset("LEVIATH_RUNS_DIR", || {
1569            let path = runs_dir();
1570            assert!(path.to_str().unwrap().contains(".leviath"));
1571            assert!(path.to_str().unwrap().ends_with("runs"));
1572        });
1573    }
1574
1575    #[test]
1576    fn runs_dir_from_uses_override_when_provided() {
1577        let path = runs_dir_from(Some("/custom/leviath/runs"));
1578        assert_eq!(path, PathBuf::from("/custom/leviath/runs"));
1579    }
1580
1581    #[test]
1582    fn runs_dir_from_falls_back_to_home_when_none() {
1583        let path = runs_dir_from(None);
1584        #[cfg(unix)]
1585        assert!(path.ends_with(".leviath/runs"));
1586        #[cfg(windows)]
1587        assert!(path.ends_with(".leviath\\runs"));
1588    }
1589
1590    /// With no `LEVIATH_RUNS_DIR`, the runs dir must follow `LEVIATH_HOME` - the
1591    /// same home every other leviath path resolves through. Without this, setting
1592    /// `LEVIATH_HOME` isolates a test's config/socket/agents dir while its runs
1593    /// still land in the real `~/.leviath/runs`.
1594    #[test]
1595    fn runs_dir_follows_leviath_home() {
1596        temp_env::with_vars(
1597            [
1598                ("LEVIATH_RUNS_DIR", None::<&str>),
1599                ("LEVIATH_HOME", Some("/tmp/leviath-home-runs-test")),
1600            ],
1601            || {
1602                assert_eq!(
1603                    runs_dir(),
1604                    PathBuf::from("/tmp/leviath-home-runs-test")
1605                        .join(".leviath")
1606                        .join("runs")
1607                );
1608            },
1609        );
1610    }
1611
1612    #[test]
1613    fn dashboard_log_path_from_uses_override_when_provided() {
1614        let path = dashboard_log_path_from(Some("/custom/leviath/dashboard.log"));
1615        assert_eq!(path, PathBuf::from("/custom/leviath/dashboard.log"));
1616    }
1617
1618    #[test]
1619    fn dashboard_log_path_from_falls_back_to_home_when_none() {
1620        let path = dashboard_log_path_from(None);
1621        #[cfg(unix)]
1622        assert!(path.ends_with(".leviath/dashboard.log"));
1623        #[cfg(windows)]
1624        assert!(path.ends_with(".leviath\\dashboard.log"));
1625    }
1626
1627    #[test]
1628    fn run_dir_contains_run_id() {
1629        let path = run_dir("my-run-123");
1630        assert!(path.to_str().unwrap().contains("my-run-123"));
1631    }
1632
1633    // ─── with_isolated_runs_dir ─────────────────────────────────────────────
1634
1635    #[test]
1636    fn with_isolated_runs_dir_points_at_temp_dir_and_cleans_up_after() {
1637        // Deliberately avoids a racy before/after ambient comparison (a
1638        // concurrently-isolated test could own `LEVIATH_RUNS_DIR` just before
1639        // or after this closure's temp-env window): instead assert the helper's
1640        // own hash-derived path is live *inside* the closure and removed
1641        // afterward - a property no other test can perturb, since none
1642        // produces this exact path.
1643        let inside = with_isolated_runs_dir("helper-self-test", |base_dir| {
1644            let expected = base_dir.join("runs");
1645            assert_eq!(runs_dir(), expected);
1646            assert!(runs_dir().exists());
1647            assert_eq!(dashboard_log_path(), base_dir.join("dashboard.log"));
1648            expected
1649        });
1650        // Closure returned: the temp dir the helper created is gone.
1651        assert!(!inside.exists());
1652    }
1653
1654    // ─── tail_file edge cases ──────────────────────────────────────────────
1655
1656    #[test]
1657    fn tail_file_exact_size() {
1658        let dir = tempfile::tempdir().unwrap();
1659        let path = dir.path().join("exact.txt");
1660        std::fs::write(&path, "exactly").unwrap();
1661        // max_bytes == file size
1662        let result = tail_file(&path, 7);
1663        assert_eq!(result, "exactly");
1664    }
1665
1666    #[test]
1667    fn tail_file_tail_without_newline_returns_whole_window() {
1668        // When the last `max_bytes` window of a larger file contains no '\n'
1669        // at all (a single long line with no line breaks), `tail_file` cannot
1670        // skip to a newline boundary, so it falls through to the `else` arm and
1671        // returns the whole (newline-free) tail window verbatim. Bytes are
1672        // written raw (never via `writeln!`, which would append '\n') so that
1673        // on *every* OS the tail slice is guaranteed newline-free - on Windows
1674        // ordinary text output is `\r\n`-terminated, which would otherwise keep
1675        // a '\n' in the window and take the `if` arm instead.
1676        let dir = tempfile::tempdir().unwrap();
1677        let path = dir.path().join("no_newline.txt");
1678        // 100 raw bytes, no newline anywhere.
1679        let content = "a".repeat(100);
1680        std::fs::write(&path, content.as_bytes()).unwrap();
1681        // A 10-byte window is smaller than the file (100) and contains no '\n'.
1682        let result = tail_file(&path, 10);
1683        assert_eq!(result, "aaaaaaaaaa");
1684    }
1685
1686    // ─── RunMeta metadata and callback_url ─────────────────────────────────
1687
1688    #[test]
1689    fn run_meta_with_metadata() {
1690        let mut meta = RunMeta::new(
1691            "meta-run".into(),
1692            "agent".into(),
1693            "/p".into(),
1694            "task".into(),
1695            None,
1696            "/w".into(),
1697            1,
1698        );
1699        meta.metadata
1700            .insert("key1".to_string(), "value1".to_string());
1701        meta.callback_url = Some("https://example.com/hook".to_string());
1702        meta.parent_run_id = Some("parent-123".to_string());
1703
1704        let json = serde_json::to_string(&meta).unwrap();
1705        let back: RunMeta = serde_json::from_str(&json).unwrap();
1706        assert_eq!(back.metadata.get("key1").unwrap(), "value1");
1707        assert_eq!(
1708            back.callback_url.as_deref(),
1709            Some("https://example.com/hook")
1710        );
1711        assert_eq!(back.parent_run_id.as_deref(), Some("parent-123"));
1712    }
1713
1714    // ─── StageRecord modifications ─────────────────────────────────────────
1715
1716    #[test]
1717    fn stage_record_mutation() {
1718        let mut rec = StageRecord::new("test".into(), 0);
1719        rec.status = StageRunStatus::Active;
1720        rec.started_at = Some(1000);
1721        rec.prompt_tokens = 500;
1722        rec.completion_tokens = 200;
1723        rec.cached_tokens = 50;
1724
1725        assert_eq!(rec.status, StageRunStatus::Active);
1726        assert_eq!(rec.started_at, Some(1000));
1727        assert_eq!(rec.prompt_tokens, 500);
1728        assert_eq!(rec.completion_tokens, 200);
1729        assert_eq!(rec.cached_tokens, 50);
1730
1731        rec.status = StageRunStatus::Complete;
1732        rec.ended_at = Some(2000);
1733        assert_eq!(rec.status, StageRunStatus::Complete);
1734        assert_eq!(rec.ended_at, Some(2000));
1735    }
1736
1737    // ─── ContextSnapshot with entries ──────────────────────────────────────
1738
1739    #[test]
1740    fn context_snapshot_with_entries() {
1741        let snap = ContextSnapshot {
1742            stage_name: "main".into(),
1743            total_tokens: 1000,
1744            max_tokens: 8192,
1745            regions: vec![
1746                RegionSnapshot {
1747                    name: "system".into(),
1748                    kind: "pinned".into(),
1749                    current_tokens: 100,
1750                    max_tokens: 2000,
1751                    entries: vec![
1752                        RegionEntrySnapshot {
1753                            content: "You are helpful".into(),
1754                            tokens: 3,
1755                            kind: Default::default(),
1756                            metadata: None,
1757                            key: None,
1758                            taint: Default::default(),
1759                        },
1760                        RegionEntrySnapshot {
1761                            content: "Additional instruction".into(),
1762                            tokens: 5,
1763                            kind: Default::default(),
1764                            metadata: Some(serde_json::json!({"source": "user"})),
1765                            key: None,
1766                            taint: Default::default(),
1767                        },
1768                    ],
1769                },
1770                RegionSnapshot {
1771                    name: "conversation".into(),
1772                    kind: "sliding".into(),
1773                    current_tokens: 900,
1774                    max_tokens: 6000,
1775                    entries: vec![],
1776                },
1777            ],
1778        };
1779
1780        let json = serde_json::to_string_pretty(&snap).unwrap();
1781        let back: ContextSnapshot = serde_json::from_str(&json).unwrap();
1782        assert_eq!(back.regions.len(), 2);
1783        assert_eq!(back.regions[0].entries.len(), 2);
1784        assert_eq!(back.regions[0].entries[1].tokens, 5);
1785        assert!(back.regions[0].entries[1].metadata.is_some());
1786    }
1787
1788    // ─── RegionEntrySnapshot metadata ──────────────────────────────────────
1789
1790    #[test]
1791    fn region_entry_snapshot_metadata_omitted_when_none() {
1792        let entry = RegionEntrySnapshot {
1793            content: "test".into(),
1794            tokens: 1,
1795            kind: Default::default(),
1796            metadata: None,
1797            key: None,
1798            taint: Default::default(),
1799        };
1800        let json = serde_json::to_value(&entry).unwrap();
1801        assert!(json.get("metadata").is_none());
1802    }
1803
1804    // ─── Multiple stage output appends ─────────────────────────────────────
1805
1806    #[test]
1807    fn append_stage_output_multiple_stages() {
1808        with_isolated_runs_dir("append-stage-output-multiple-stages", |_d| {
1809            let run_id = "test-multi-stage-out";
1810            append_stage_output(run_id, 0, "stage 0 output");
1811            append_stage_output(run_id, 1, "stage 1 output");
1812            append_stage_output(run_id, 2, "stage 2 output");
1813
1814            let out0 = tail_stage_output(run_id, 0, 4096);
1815            let out1 = tail_stage_output(run_id, 1, 4096);
1816            let out2 = tail_stage_output(run_id, 2, 4096);
1817
1818            assert!(out0.contains("stage 0 output"));
1819            assert!(out1.contains("stage 1 output"));
1820            assert!(out2.contains("stage 2 output"));
1821            // Verify no cross-contamination
1822            assert!(!out0.contains("stage 1 output"));
1823        });
1824    }
1825
1826    // ─── list_runs ─────────────────────────────────────────────────────────
1827
1828    #[test]
1829    fn list_runs_returns_sorted() {
1830        with_isolated_runs_dir("list-runs-returns-sorted", |_d| {
1831            let meta1 = RunMeta::new(
1832                "test-list-run-a".into(),
1833                "agent".into(),
1834                "/p".into(),
1835                "task a".into(),
1836                None,
1837                "/w".into(),
1838                1,
1839            );
1840            let meta2 = RunMeta::new(
1841                "test-list-run-b".into(),
1842                "agent".into(),
1843                "/p".into(),
1844                "task b".into(),
1845                None,
1846                "/w".into(),
1847                1,
1848            );
1849
1850            let _ = create_run(&meta1);
1851            // Small delay to ensure different timestamps
1852            let _ = create_run(&meta2);
1853
1854            let runs = list_runs();
1855            // Both should appear in the list
1856            let ids: Vec<&str> = runs.iter().map(|r| r.run_id.as_str()).collect();
1857            assert!(ids.contains(&"test-list-run-a"));
1858            assert!(ids.contains(&"test-list-run-b"));
1859        });
1860    }
1861
1862    // ─── tail_stage_log / tail_stage_output empty ──────────────────────────
1863
1864    #[test]
1865    fn tail_stage_output_nonexistent_returns_empty() {
1866        assert_eq!(tail_stage_output("no-such-run-xyz", 0, 4096), "");
1867    }
1868
1869    #[test]
1870    fn tail_stage_log_nonexistent_returns_empty() {
1871        assert_eq!(tail_stage_log("no-such-run-xyz", 0, 4096), "");
1872    }
1873
1874    // ─── list_runs_in_dir ───────────────────────────────────────────────────
1875
1876    #[test]
1877    fn list_runs_in_dir_nonexistent_returns_empty() {
1878        let result = list_runs_in_dir(PathBuf::from("/nonexistent/leviath/runs/coverage-test"));
1879        assert!(result.is_empty());
1880    }
1881
1882    #[test]
1883    fn list_runs_in_dir_empty_dir_returns_empty() {
1884        let dir = tempfile::tempdir().unwrap();
1885        let result = list_runs_in_dir(dir.path().to_path_buf());
1886        assert!(result.is_empty());
1887    }
1888
1889    #[test]
1890    fn list_runs_in_dir_unreadable_dir_returns_empty() {
1891        // Covers the `if let Ok(entries) = std::fs::read_dir(&dir)` pattern
1892        // *not* matching: `dir.exists()` is true (so the earlier early-return
1893        // is skipped) but `read_dir` fails, so the whole block is silently
1894        // skipped. Pointing at a *file* makes `read_dir` fail on every platform.
1895        let dir = tempfile::tempdir().unwrap();
1896        let not_a_dir = dir.path().join("runs-is-a-file");
1897        std::fs::write(&not_a_dir, "not a dir").unwrap();
1898        let result = list_runs_in_dir(not_a_dir);
1899        assert!(result.is_empty());
1900    }
1901
1902    #[test]
1903    fn append_stage_output_open_failure_is_silently_skipped() {
1904        // When `output.log` already exists as a *directory*, `OpenOptions::open`
1905        // fails and the write is silently skipped (the `if let Ok(file)` false
1906        // path). Making the target a directory fails the open on every platform.
1907        crate::runstate::with_isolated_runs_dir("append_stage_output_open_failure", |_d| {
1908            let run_id = "append-out-openfail";
1909            ensure_stage_dir(run_id, 0);
1910            std::fs::create_dir_all(stage_dir(run_id, 0).join("output.log")).unwrap();
1911            append_stage_output(run_id, 0, "ignored"); // must not panic
1912        });
1913    }
1914
1915    #[test]
1916    fn append_stage_log_open_failure_is_silently_skipped() {
1917        // Same as above for `logs.log` in `append_stage_log`.
1918        crate::runstate::with_isolated_runs_dir("append_stage_log_open_failure", |_d| {
1919            let run_id = "append-log-openfail";
1920            ensure_stage_dir(run_id, 0);
1921            std::fs::create_dir_all(stage_dir(run_id, 0).join("logs.log")).unwrap();
1922            append_stage_log(run_id, 0, "ignored"); // must not panic
1923        });
1924    }
1925
1926    // ─── runs_dir / list_runs edge cases ────────────────────────────────────
1927
1928    #[test]
1929    fn runs_dir_with_override_set_returns_override() {
1930        let tmpdir = tempfile::tempdir().unwrap();
1931        temp_env::with_var("LEVIATH_RUNS_DIR", Some(tmpdir.path()), || {
1932            assert_eq!(runs_dir(), tmpdir.path());
1933        });
1934    }
1935
1936    #[test]
1937    fn runs_dir_without_override_falls_back_to_home() {
1938        temp_env::with_var_unset("LEVIATH_RUNS_DIR", || {
1939            let dir = runs_dir();
1940            #[cfg(unix)]
1941            assert!(dir.ends_with(".leviath/runs"));
1942            #[cfg(windows)]
1943            assert!(dir.ends_with(".leviath\\runs"));
1944        });
1945    }
1946
1947    #[test]
1948    fn list_runs_empty_when_runs_dir_missing_or_empty() {
1949        // Isolated via `isolate_runs_dir_for_test`, so this is a genuinely
1950        // empty runs dir (not "the real dir, which we hope has no entry with
1951        // this exact bogus id") - can assert real emptiness instead of just
1952        // absence of one specific id.
1953        with_isolated_runs_dir("list-runs-empty-when-runs-dir-missing-or-empty", |_d| {
1954            let runs = list_runs();
1955            assert!(runs.is_empty());
1956        });
1957    }
1958
1959    #[test]
1960    fn tail_file_nonexistent_path_returns_empty() {
1961        let path = std::path::Path::new("/nonexistent/path/to/a/file.log");
1962        assert_eq!(tail_file(path, 1024), "");
1963    }
1964
1965    #[test]
1966    fn tail_file_small_file_returns_whole_contents() {
1967        let dir = tempfile::tempdir().unwrap();
1968        let path = dir.path().join("small.log");
1969        std::fs::write(&path, "hello world").unwrap();
1970        assert_eq!(tail_file(&path, 1024), "hello world");
1971    }
1972
1973    #[test]
1974    fn tail_file_large_file_truncates_from_offset() {
1975        let dir = tempfile::tempdir().unwrap();
1976        let path = dir.path().join("big.log");
1977        let content = "a".repeat(100) + "\nTAIL_MARKER\n";
1978        std::fs::write(&path, &content).unwrap();
1979        let tailed = tail_file(&path, 20);
1980        assert!(tailed.contains("TAIL_MARKER"));
1981        assert!(tailed.len() < content.len());
1982    }
1983
1984    #[test]
1985    fn tail_file_directory_path_returns_empty() {
1986        // metadata() and File::open() both succeed on a directory (confirmed
1987        // empirically on macOS/Linux); it's read_to_end() that fails with
1988        // "Is a directory" - and that error is deliberately discarded (`let
1989        // _ = file.read_to_end(&mut buf);`), so this exercises the
1990        // graceful-empty-buffer fallback at the bottom of the function, not
1991        // either of the two `Err(_) => return String::new()` early returns.
1992        let dir = tempfile::tempdir().unwrap();
1993        assert_eq!(tail_file(dir.path(), 4), "");
1994    }
1995
1996    #[cfg(unix)]
1997    #[test]
1998    fn tail_file_open_permission_denied_returns_empty() {
1999        // A file with no permissions at all: `Path::exists()`/`fs::metadata()`
2000        // only need search (execute) permission on the *parent* directories
2001        // to stat a path, not read permission on the file itself - so both
2002        // succeed here. `std::fs::File::open()` in read mode, however,
2003        // genuinely fails with `PermissionDenied`. Unlike the metadata-error
2004        // arm (only reachable via a delete-between-calls race), this is a
2005        // deterministic way to exercise the `File::open` `Err(_)` arm.
2006        use std::os::unix::fs::PermissionsExt;
2007
2008        let dir = tempfile::tempdir().unwrap();
2009        let path = dir.path().join("no-permissions.log");
2010        // Content must exceed max_bytes so the "whole file" fast path
2011        // (`file_size <= max_bytes`) doesn't short-circuit before reaching
2012        // the `File::open` call under test.
2013        std::fs::write(&path, "x".repeat(100)).unwrap();
2014        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o000)).unwrap();
2015
2016        assert_eq!(tail_file(&path, 4), "");
2017
2018        // Restore permissions so the tempdir can clean itself up on drop.
2019        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap();
2020    }
2021
2022    // ─── hermetic write/read coverage tests (use _to/_from/_in helpers) ───────
2023
2024    #[test]
2025    fn write_context_snapshot_to_hermetic() {
2026        let dir = tempfile::tempdir().unwrap();
2027        let snap = ContextSnapshot {
2028            stage_name: "cov-stage".into(),
2029            total_tokens: 42,
2030            max_tokens: 8192,
2031            regions: vec![],
2032        };
2033        write_context_snapshot_to(dir.path(), &snap).unwrap();
2034        let json = std::fs::read_to_string(dir.path().join("context.json")).unwrap();
2035        let back: ContextSnapshot = serde_json::from_str(&json).unwrap();
2036        assert_eq!(back.total_tokens, 42);
2037    }
2038
2039    #[test]
2040    fn write_context_snapshot_to_fails_without_dir() {
2041        let snap = ContextSnapshot {
2042            stage_name: "s".into(),
2043            total_tokens: 1,
2044            max_tokens: 100,
2045            regions: vec![],
2046        };
2047        let nonexistent = std::path::Path::new("/nonexistent-cov-dir-xyzzy-abc");
2048        let result = write_context_snapshot_to(nonexistent, &snap);
2049        assert!(result.is_err());
2050    }
2051
2052    #[test]
2053    fn write_context_snapshot_to_fails_when_rename_target_is_a_dir() {
2054        // Covers the `std::fs::rename(&tmp, &path)?` `Err` arm: the tmp file
2055        // write succeeds (its directory is writable), but the final rename
2056        // fails because `context.json` already exists as a *directory* --
2057        // `rename(2)` on POSIX refuses to replace a directory with a
2058        // regular file, unlike a plain overwrite of an existing file.
2059        let dir = tempfile::tempdir().unwrap();
2060        std::fs::create_dir(dir.path().join("context.json")).unwrap();
2061        let snap = ContextSnapshot {
2062            stage_name: "s".into(),
2063            total_tokens: 1,
2064            max_tokens: 100,
2065            regions: vec![],
2066        };
2067        let result = write_context_snapshot_to(dir.path(), &snap);
2068        assert!(result.is_err());
2069    }
2070
2071    #[test]
2072    fn create_run_in_hermetic() {
2073        let tmpdir = tempfile::tempdir().unwrap();
2074        let run_dir = tmpdir.path().join("cov-run");
2075        let meta = RunMeta::new(
2076            "cov-run".into(),
2077            "cov-agent".into(),
2078            "/agents/cov".into(),
2079            "cov task".into(),
2080            None,
2081            "/tmp".into(),
2082            1,
2083        );
2084        create_run_in(&run_dir, &meta).unwrap();
2085        let back = read_meta_from(&run_dir).unwrap();
2086        assert_eq!(back.run_id, "cov-run");
2087    }
2088
2089    #[test]
2090    fn create_run_in_fails_on_bad_parent() {
2091        // A hardcoded "/nonexistent-.../run" path isn't reliably bad across
2092        // platforms: on Windows CI runners (which typically have write
2093        // access to create directories at the drive root), that path
2094        // resolves under the current drive's root and create_dir_all
2095        // actually succeeds there, while on Unix it fails because writing
2096        // to the real filesystem root needs privileges the CI user lacks --
2097        // this passed locally but failed on Windows CI. Use a path with a
2098        // regular file as a parent component instead: create_dir_all can
2099        // never succeed under a file, on any platform or set of permissions.
2100        let dir = tempfile::tempdir().unwrap();
2101        let not_a_dir = dir.path().join("not-a-directory");
2102        std::fs::write(&not_a_dir, "x").unwrap();
2103        let bad = not_a_dir.join("run");
2104        let meta = RunMeta::new(
2105            "run".into(),
2106            "a".into(),
2107            "/".into(),
2108            "t".into(),
2109            None,
2110            "/tmp".into(),
2111            1,
2112        );
2113        let result = create_run_in(&bad, &meta);
2114        assert!(result.is_err());
2115    }
2116
2117    #[test]
2118    fn write_meta_to_hermetic() {
2119        let tmpdir = tempfile::tempdir().unwrap();
2120        let meta = RunMeta::new(
2121            "cov-write-meta".into(),
2122            "a".into(),
2123            "/".into(),
2124            "t".into(),
2125            None,
2126            "/tmp".into(),
2127            1,
2128        );
2129        write_meta_to(tmpdir.path(), &meta).unwrap();
2130        let back = read_meta_from(tmpdir.path()).unwrap();
2131        assert_eq!(back.run_id, "cov-write-meta");
2132    }
2133
2134    #[test]
2135    fn write_meta_to_fails_without_dir() {
2136        let meta = RunMeta::new(
2137            "cov-no-dir".into(),
2138            "a".into(),
2139            "/".into(),
2140            "t".into(),
2141            None,
2142            "/tmp".into(),
2143            1,
2144        );
2145        let bad = std::path::Path::new("/nonexistent-cov-write-meta-xyzzy");
2146        let result = write_meta_to(bad, &meta);
2147        assert!(result.is_err());
2148    }
2149
2150    #[test]
2151    fn write_meta_to_fails_when_rename_target_is_a_dir() {
2152        // See `write_context_snapshot_to_fails_when_rename_target_is_a_dir`:
2153        // same `std::fs::rename(&tmp_path, &final_path)?` `Err` arm, forced
2154        // by pre-creating `meta.json` as a directory.
2155        let dir = tempfile::tempdir().unwrap();
2156        std::fs::create_dir(dir.path().join("meta.json")).unwrap();
2157        let meta = RunMeta::new(
2158            "cov-rename-fail".into(),
2159            "a".into(),
2160            "/".into(),
2161            "t".into(),
2162            None,
2163            "/tmp".into(),
2164            1,
2165        );
2166        let result = write_meta_to(dir.path(), &meta);
2167        assert!(result.is_err());
2168    }
2169
2170    #[test]
2171    fn read_meta_from_fails_on_missing_file() {
2172        let tmpdir = tempfile::tempdir().unwrap();
2173        let result = read_meta_from(tmpdir.path());
2174        assert!(result.is_err());
2175    }
2176
2177    #[test]
2178    fn write_stages_index_to_hermetic() {
2179        let tmpdir = tempfile::tempdir().unwrap();
2180        let stages = vec![StageRecord::new("cov-stage".into(), 0)];
2181        write_stages_index_to(tmpdir.path(), &stages).unwrap();
2182        let json = std::fs::read_to_string(tmpdir.path().join("stages.json")).unwrap();
2183        let back: Vec<StageRecord> = serde_json::from_str(&json).unwrap();
2184        assert_eq!(back.len(), 1);
2185        assert_eq!(back[0].name, "cov-stage");
2186    }
2187
2188    #[test]
2189    fn write_stages_index_to_fails_without_dir() {
2190        let stages = vec![StageRecord::new("s".into(), 0)];
2191        let bad = std::path::Path::new("/nonexistent-cov-stages-xyzzy");
2192        let result = write_stages_index_to(bad, &stages);
2193        assert!(result.is_err());
2194    }
2195
2196    #[test]
2197    fn write_stages_index_to_fails_when_rename_target_is_a_dir() {
2198        // See `write_context_snapshot_to_fails_when_rename_target_is_a_dir`:
2199        // same `std::fs::rename(&tmp, &path)?` `Err` arm, forced by
2200        // pre-creating `stages.json` as a directory.
2201        let dir = tempfile::tempdir().unwrap();
2202        std::fs::create_dir(dir.path().join("stages.json")).unwrap();
2203        let stages = vec![StageRecord::new("s".into(), 0)];
2204        let result = write_stages_index_to(dir.path(), &stages);
2205        assert!(result.is_err());
2206    }
2207
2208    #[test]
2209    fn list_runs_in_dir_includes_valid_run() {
2210        let tmpdir = tempfile::tempdir().unwrap();
2211        let run_id = "cov-listed-run";
2212        let run_subdir = tmpdir.path().join(run_id);
2213        std::fs::create_dir_all(&run_subdir).unwrap();
2214        let meta = RunMeta::new(
2215            run_id.into(),
2216            "list-agent".into(),
2217            "/agents/list".into(),
2218            "list task".into(),
2219            None,
2220            "/tmp".into(),
2221            1,
2222        );
2223        let json = serde_json::to_string_pretty(&meta).unwrap();
2224        std::fs::write(run_subdir.join("meta.json"), &json).unwrap();
2225
2226        // list_runs_in_dir now reads meta.json directly from the dir, no env var needed
2227        let runs = list_runs_in_dir(tmpdir.path().to_path_buf());
2228        assert!(runs.iter().any(|r| r.run_id == run_id));
2229    }
2230
2231    #[test]
2232    fn list_runs_in_dir_skips_entry_with_corrupted_meta_json() {
2233        // Exercises the `if let Ok(meta) = serde_json::from_str::<RunMeta>(...)`
2234        // else arm: a subdirectory whose meta.json exists and is readable as
2235        // a string, but doesn't parse as a `RunMeta`, is silently skipped
2236        // rather than propagating an error.
2237        let tmpdir = tempfile::tempdir().unwrap();
2238        let good_run_id = "cov-listed-good-run";
2239        let bad_run_id = "cov-listed-corrupted-run";
2240
2241        let good_subdir = tmpdir.path().join(good_run_id);
2242        std::fs::create_dir_all(&good_subdir).unwrap();
2243        let meta = RunMeta::new(
2244            good_run_id.into(),
2245            "list-agent".into(),
2246            "/agents/list".into(),
2247            "list task".into(),
2248            None,
2249            "/tmp".into(),
2250            1,
2251        );
2252        let json = serde_json::to_string_pretty(&meta).unwrap();
2253        std::fs::write(good_subdir.join("meta.json"), &json).unwrap();
2254
2255        let bad_subdir = tmpdir.path().join(bad_run_id);
2256        std::fs::create_dir_all(&bad_subdir).unwrap();
2257        std::fs::write(bad_subdir.join("meta.json"), "not valid json").unwrap();
2258
2259        // A subdirectory with NO meta.json exercises the *other* skip branch:
2260        // the `if let Ok(json) = read_to_string(&meta_path)` else arm (the file
2261        // can't be read), distinct from the parse-fails arm above. Covering
2262        // both here keeps list_runs_in_dir at 100% on every OS deterministically.
2263        let no_meta_run_id = "cov-listed-no-meta-run";
2264        std::fs::create_dir_all(tmpdir.path().join(no_meta_run_id)).unwrap();
2265
2266        let runs = list_runs_in_dir(tmpdir.path().to_path_buf());
2267        assert!(runs.iter().any(|r| r.run_id == good_run_id));
2268        assert!(!runs.iter().any(|r| r.run_id == bad_run_id));
2269        assert!(!runs.iter().any(|r| r.run_id == no_meta_run_id));
2270    }
2271
2272    // ─── force_cancel_in: the floor under every kill path ───
2273
2274    /// Write a run dir with `status` and return its path.
2275    fn run_dir_with(base: &std::path::Path, run_id: &str, status: RunStatus) -> PathBuf {
2276        let dir = base.join(run_id);
2277        let meta = RunMeta {
2278            status,
2279            ..RunMeta::new(
2280                run_id.into(),
2281                "a".into(),
2282                "/p".into(),
2283                "t".into(),
2284                None,
2285                "/w".into(),
2286                1,
2287            )
2288        };
2289        create_run_in(&dir, &meta).unwrap();
2290        dir
2291    }
2292
2293    #[test]
2294    fn force_cancel_terminates_every_non_terminal_status() {
2295        let base = tempfile::tempdir().unwrap();
2296        for status in [
2297            RunStatus::Starting,
2298            RunStatus::Running,
2299            RunStatus::WaitingInput,
2300        ] {
2301            let dir = run_dir_with(base.path(), &format!("live-{status}"), status.clone());
2302            assert_eq!(force_cancel_in(&dir, 99), ForceCancelOutcome::Terminated);
2303            let meta = read_meta_from(&dir).unwrap();
2304            assert_eq!(meta.status, RunStatus::Cancelled, "{status} is killable");
2305            assert_eq!(meta.updated_at, 99, "the cancel is stamped");
2306        }
2307    }
2308
2309    #[test]
2310    fn force_cancel_leaves_a_finished_run_alone() {
2311        let base = tempfile::tempdir().unwrap();
2312        for status in [
2313            RunStatus::Complete,
2314            RunStatus::CompleteInteractive,
2315            RunStatus::Error,
2316            RunStatus::Cancelled,
2317        ] {
2318            let dir = run_dir_with(base.path(), &format!("done-{status}"), status.clone());
2319            assert_eq!(
2320                force_cancel_in(&dir, 99),
2321                ForceCancelOutcome::AlreadyTerminal,
2322                "{status} is already finished"
2323            );
2324            assert_eq!(read_meta_from(&dir).unwrap().status, status);
2325        }
2326    }
2327
2328    #[test]
2329    fn force_cancel_reports_no_such_run_for_a_missing_directory() {
2330        let base = tempfile::tempdir().unwrap();
2331        let outcome = force_cancel_in(&base.path().join("ghost"), 99);
2332        assert_eq!(outcome, ForceCancelOutcome::NoSuchRun);
2333        assert!(!outcome.found_run(), "nothing to cancel");
2334    }
2335
2336    /// A run dir whose metadata can't be parsed still gets terminated. Such a run
2337    /// is skipped by `list_runs`, so leaving it alone makes it both invisible and
2338    /// permanent - the one state from which there is no way back.
2339    #[test]
2340    fn force_cancel_writes_a_record_over_unreadable_metadata() {
2341        let base = tempfile::tempdir().unwrap();
2342        let dir = base.path().join("corrupt-run");
2343        std::fs::create_dir_all(&dir).unwrap();
2344        std::fs::write(dir.join("meta.json"), "{ not json").unwrap();
2345
2346        assert_eq!(force_cancel_in(&dir, 99), ForceCancelOutcome::Terminated);
2347        let meta = read_meta_from(&dir).expect("now parses");
2348        assert_eq!(meta.status, RunStatus::Cancelled);
2349        assert_eq!(meta.run_id, "corrupt-run", "recovered from the dir name");
2350        assert!(meta.error.is_some(), "records why it was synthesized");
2351    }
2352
2353    /// A directory that exists but can't be written still counts as "found" - the
2354    /// caller must not report "no such run" for a run that plainly exists.
2355    #[test]
2356    fn force_cancel_reports_a_write_failure_but_still_found_the_run() {
2357        crate::test_support::with_tracing(|| {
2358            let base = tempfile::tempdir().unwrap();
2359            let dir = base.path().join("blocked-run");
2360            std::fs::create_dir_all(&dir).unwrap();
2361            // A directory where `meta.json` must go: the rename can't succeed.
2362            std::fs::create_dir_all(dir.join("meta.json")).unwrap();
2363
2364            let outcome = force_cancel_in(&dir, 99);
2365            assert_eq!(outcome, ForceCancelOutcome::WriteFailed);
2366            assert!(outcome.found_run());
2367        });
2368    }
2369
2370    /// The spawn that never became a run: the placeholder is `Starting`, which
2371    /// is not terminal, so it has to be rewritten or it claims to be alive for
2372    /// ever (issue #190).
2373    #[test]
2374    fn force_error_records_the_failure_over_a_starting_placeholder() {
2375        let base = tempfile::tempdir().unwrap();
2376        let dir = base.path().join("stillborn-run");
2377        let meta = RunMeta::new(
2378            "stillborn-run".to_string(),
2379            "agent".to_string(),
2380            "/no/such/agent.leviath".to_string(),
2381            "t".to_string(),
2382            None,
2383            "/tmp".to_string(),
2384            0,
2385        );
2386        create_run_in(&dir, &meta).unwrap();
2387        assert_eq!(read_meta_from(&dir).unwrap().status, RunStatus::Starting);
2388
2389        assert_eq!(
2390            force_error_in(&dir, "blueprint not found", 99),
2391            ForceCancelOutcome::Terminated
2392        );
2393
2394        let written = read_meta_from(&dir).unwrap();
2395        assert_eq!(written.status, RunStatus::Error);
2396        assert_eq!(written.error.as_deref(), Some("blueprint not found"));
2397        assert_eq!(written.updated_at, 99);
2398        // The rest of the placeholder survives, so the run still explains itself.
2399        assert_eq!(written.task, "t");
2400    }
2401
2402    #[test]
2403    fn force_error_leaves_a_run_that_already_finished_alone() {
2404        let base = tempfile::tempdir().unwrap();
2405        let dir = base.path().join("done-run");
2406        let mut meta = RunMeta::new(
2407            "done-run".to_string(),
2408            "agent".to_string(),
2409            String::new(),
2410            "t".to_string(),
2411            None,
2412            "/tmp".to_string(),
2413            0,
2414        );
2415        meta.status = RunStatus::Complete;
2416        create_run_in(&dir, &meta).unwrap();
2417
2418        assert_eq!(
2419            force_error_in(&dir, "too late", 99),
2420            ForceCancelOutcome::AlreadyTerminal
2421        );
2422        assert_eq!(read_meta_from(&dir).unwrap().status, RunStatus::Complete);
2423    }
2424
2425    #[test]
2426    fn force_cancel_keeps_an_error_the_run_had_already_recorded() {
2427        // Cancelling passes no message of its own, so whatever the run managed
2428        // to say about itself before it was killed must survive.
2429        let base = tempfile::tempdir().unwrap();
2430        let dir = base.path().join("noisy-run");
2431        let mut meta = RunMeta::new(
2432            "noisy-run".to_string(),
2433            "agent".to_string(),
2434            String::new(),
2435            "t".to_string(),
2436            None,
2437            "/tmp".to_string(),
2438            0,
2439        );
2440        meta.error = Some("a provider hiccup".to_string());
2441        create_run_in(&dir, &meta).unwrap();
2442
2443        assert_eq!(force_cancel_in(&dir, 99), ForceCancelOutcome::Terminated);
2444        let written = read_meta_from(&dir).unwrap();
2445        assert_eq!(written.status, RunStatus::Cancelled);
2446        assert_eq!(written.error.as_deref(), Some("a provider hiccup"));
2447    }
2448
2449    #[test]
2450    fn force_error_writes_its_message_over_unreadable_metadata() {
2451        let base = tempfile::tempdir().unwrap();
2452        let dir = base.path().join("corrupt-stillborn");
2453        std::fs::create_dir_all(&dir).unwrap();
2454        std::fs::write(dir.join("meta.json"), "{ not json").unwrap();
2455
2456        assert_eq!(
2457            force_error_in(&dir, "blueprint not found", 99),
2458            ForceCancelOutcome::Terminated
2459        );
2460        let written = read_meta_from(&dir).expect("now parses");
2461        assert_eq!(written.status, RunStatus::Error);
2462        assert_eq!(written.error.as_deref(), Some("blueprint not found"));
2463    }
2464
2465    #[test]
2466    fn append_dashboard_log_writes_message() {
2467        // Exercises the create_dir_all branch and writeln! branch via a unique marker.
2468        with_isolated_runs_dir("append-dashboard-log-writes-message", |_d| {
2469            let unique = format!("cov-dashboard-log-{}", std::process::id());
2470            append_dashboard_log(&unique);
2471            let content = std::fs::read_to_string(dashboard_log_path()).unwrap_or_default();
2472            assert!(content.contains(&unique));
2473        });
2474    }
2475}