oxo-flow-cli 0.23.2

use crate::commands::{print_banner, resolve_workflow, samples};
use anyhow::{Context, Result};
use colored::Colorize;
use oxo_flow_core::config::WorkflowConfig;
use oxo_flow_core::config_impact::{ConfigChangeReport, config_value_string};
use oxo_flow_core::dag::WorkflowDag;
use oxo_flow_core::executor::{
    CheckpointState, ExecutorConfig, LocalExecutor, WorkdirLock, output_tail_bytes_from_env,
};
use oxo_flow_core::rule::{Rule, parse_duration_secs};
use oxo_flow_core::wildcard;
use std::collections::{HashMap, HashSet};
use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::sync::Mutex;

/// Flatten workflow config values into the `{config.key}` placeholder map
/// used for path expansion (DAG edge matching, run-time rendering, …).
/// Default targets (issue #469): rules marked `target = true` are what
/// `run`/`dry-run` build when neither `-t` nor `--module` selected
/// anything — Snakemake's rule-all semantics. No marked rule → the full
/// DAG (backward compatible); an explicit `-t`/`--module` always wins.
fn resolve_default_targets(config: &WorkflowConfig, target: &[String]) -> Vec<String> {
    if !target.is_empty() {
        return target.to_vec();
    }
    let marked: Vec<String> = config
        .rules
        .iter()
        .filter(|r| r.target)
        .map(|r| r.name.clone())
        .collect();
    if marked.is_empty() {
        target.to_vec()
    } else {
        marked
    }
}

/// Render one engine-sampled benchmark record as JSON for the rule's
/// declared `benchmark` path (issue #469). JSON keeps the fields
/// self-describing and re-parseable; values mirror the checkpoint's
/// BenchmarkRecord exactly.
fn render_benchmark_json(b: &oxo_flow_core::executor::checkpoint::BenchmarkRecord) -> String {
    serde_json::json!({
        "rule": b.rule,
        "wall_time_secs": b.wall_time_secs,
        "max_memory_mb": b.max_memory_mb,
        "memory_limit_mb": b.memory_limit_mb,
        "cpu_seconds": b.cpu_seconds,
        "retries": b.retries,
    })
    .to_string()
}

#[cfg(test)]
mod default_target_tests {
    use super::*;

    #[test]
    fn resolve_default_targets_prefers_explicit_selection() {
        let config = WorkflowConfig::parse(
            r#"
[workflow]
name = "t"

[[rules]]
name = "a"
target = true
shell = "true"

[[rules]]
name = "b"
shell = "true"
"#,
        )
        .unwrap();

        // Explicit -t always wins.
        assert_eq!(
            resolve_default_targets(&config, &["b".to_string()]),
            vec!["b".to_string()]
        );
        // No explicit selection → the marked rule.
        assert_eq!(resolve_default_targets(&config, &[]), vec!["a".to_string()]);
        // --module already selected something → unchanged.
        assert_eq!(
            resolve_default_targets(&config, &["m_rule".to_string()]),
            vec!["m_rule".to_string()]
        );
    }

    #[test]
    fn resolve_default_targets_full_dag_without_marks() {
        let config = WorkflowConfig::parse(
            r#"
[workflow]
name = "t"

[[rules]]
name = "a"
shell = "true"
"#,
        )
        .unwrap();
        assert!(resolve_default_targets(&config, &[]).is_empty());
    }

    #[test]
    fn render_benchmark_json_carries_sampled_fields() {
        let b = oxo_flow_core::executor::checkpoint::BenchmarkRecord {
            rule: "fastqc".into(),
            wall_time_secs: 2.75,
            max_memory_mb: Some(512),
            memory_limit_mb: Some(1024),
            cpu_seconds: Some(2.1),
            retries: 0,
            recorded_as: None,
        };
        let json = render_benchmark_json(&b);
        assert!(json.contains(r#""rule":"fastqc""#), "{json}");
        assert!(json.contains(r#""wall_time_secs":2.75"#), "{json}");
        assert!(json.contains(r#""max_memory_mb":512"#), "{json}");
    }
}

pub(crate) fn config_placeholder_values(
    config: &HashMap<String, toml::Value>,
) -> HashMap<String, String> {
    oxo_flow_core::config_impact::config_placeholder_map(config)
}

/// The active run log, shared so spawned rule tasks can write their own
/// narrative lines — the tee's file side is private to `crate::logging`, and
/// the guard cannot move into a spawned task while the loop still needs it.
type SharedRunLog = Arc<std::sync::Mutex<Option<crate::logging::RunLogGuard>>>;

/// Emit one progress narrative line exactly TWICE in total — once on stderr
/// (the console surface) and once in the archived run log (issue #194 B1).
/// The historical form routed the log copy through the tracing stream, whose
/// writer also targets stderr, so every line printed twice and a leading
/// blank line in the message surfaced as an empty INFO row (audit P4-11).
/// Colors live in the console form; the log side gets plain text.
///
/// Background runs hold no run log (their stderr already lands in it), so
/// only the console copy is written.
/// Append one plain-text line to the active run log (the shared guard slot).
/// Failures degrade to a debug trace — the log is best-effort and the console
/// copy has already gone out.
fn write_run_log_line(run_log: &SharedRunLog, text: &str) {
    let mut slot = run_log
        .lock()
        .unwrap_or_else(|poisoned| poisoned.into_inner());
    if let Some(log) = slot.as_mut() {
        let line = format!("{}\n", text.trim_start());
        if let Err(e) = log.write_all(line.as_bytes()) {
            tracing::debug!(error = %e, "narrative line could not be written to the run log");
        }
    }
}

fn progress_narrate(msg: std::fmt::Arguments<'_>, run_log: &SharedRunLog) {
    // --quiet silences the console copy only: the run log is an artifact the
    // report/web UI reads, not terminal noise.
    if !crate::commands::is_quiet() {
        eprintln!("{msg}");
    }
    write_run_log_line(run_log, &strip_ansi(&msg.to_string()));
}

/// Emit one audit-trail diagnostic line to stderr AND the run log: adoption
/// notes, config-change summaries, and invalidation cascades explain WHY
/// checkpoint credit was lost, and nohup/CI users only keep the run-log file
/// (issue #484). Unlike [`progress_narrate`] the console copy is NOT gated on
/// --quiet — these lines print unconditionally today and operators parse them.
/// Callers outside the run path (dry-run, status) pass `None` and keep the
/// historic stderr-only behavior; background runs also pass a file-less slot
/// (their stderr already lands in the log), so nothing is written twice.
fn diagnostic_narrate(msg: std::fmt::Arguments<'_>, run_log: Option<&SharedRunLog>) {
    eprintln!("{msg}");
    if let Some(run_log) = run_log {
        write_run_log_line(run_log, &strip_ansi(&msg.to_string()));
    }
}

/// Narrate the #837 workdir source-linking pass: what was linked, what was
/// converted from a duplicate copy, and which references need operator
/// attention (conflicts, undeclared shell paths, filesystem failures).
fn narrate_source_link_report(
    report: &crate::commands::workdir_sources::SourceLinkReport,
    run_log: Option<&SharedRunLog>,
) {
    if let Some(note) = &report.skipped_note {
        diagnostic_narrate(format_args!("workdir source linking: {}", note), run_log);
        return;
    }
    if report.is_empty() {
        return;
    }
    if !report.linked.is_empty() {
        diagnostic_narrate(
            format_args!(
                "workdir source linking: created {} symlink(s) into the workdir (repo stays the single source of truth): {}",
                report.linked.len(),
                report.linked.join(", ")
            ),
            run_log,
        );
    }
    if !report.sidecars.is_empty() {
        diagnostic_narrate(
            format_args!(
                "workdir source linking: linked {} tool-convention index sidecar(s) next to linked file(s) (bwa/samtools/picard auto-discovery — not spelled out in any shell command): {}",
                report.sidecars.len(),
                report.sidecars.join(", ")
            ),
            run_log,
        );
    }
    if !report.converted.is_empty() {
        diagnostic_narrate(
            format_args!(
                "workdir source linking: replaced {} byte-identical workdir duplicate(s) with symlink(s) into the repo: {}",
                report.converted.len(),
                report.converted.join(", ")
            ),
            run_log,
        );
    }
    for path in &report.conflicts {
        diagnostic_narrate(
            format_args!(
                "{} workdir source linking: '{}' differs from the workflow-repo file — keeping the workdir copy for this run (remove the copy or update the repo to link it)",
                "Warning:".yellow(),
                path
            ),
            run_log,
        );
    }
    if !report.undeclared.is_empty() {
        diagnostic_narrate(
            format_args!(
                "{} workdir source linking: shell command(s) reference workflow-repo file(s) that no rule declares as inputs — they were NOT linked (declaring the path as a rule input links it automatically):\n  {}",
                "Note:".yellow(),
                report.undeclared.join("\n  ")
            ),
            run_log,
        );
    }
    for err in &report.errors {
        diagnostic_narrate(
            format_args!("{} workdir source linking: {}", "Warning:".yellow(), err),
            run_log,
        );
    }
}

// ── Spawn watchdog (issue #685) ─────────────────────────────────────────────
// A rule can end up marked Running in the scheduler while its task never
// reaches a child spawn (lost wakeup in the wait/dispatch path). The engine
// then wedges: nothing completes, nothing fails, the process idles in a
// futex wait. The watchdog compares the scheduler's running set against the
// executor's live child pids on a fixed tick and reports stragglers. Since
// #847, rules queued in the resource pool are reported as benign waits
// (WARN) — only unqueued rules keep the #685 ERROR.

/// How often the watchdog compares running rules against live pids.
const SPAWN_WATCHDOG_TICK_SECS: u64 = 30;
/// Default age at which a Running rule with no live child process is
/// reported. Generous: legitimate env setup can take tens of minutes, and
/// the watchdog only logs — it never kills.
const DEFAULT_SPAWN_WATCHDOG_SECS: u64 = 600;
const SPAWN_WATCHDOG_ENV: &str = "OXO_FLOW_SPAWN_WATCHDOG_SECS";

/// How often the #843 disk-pressure monitor re-measures free space while the
/// run is parked under Hold (nothing in flight) or between completion batches.
const DISK_PRESSURE_TICK_SECS: u64 = 60;
/// Consecutive parked Hold rounds (nothing in flight, nothing reclaimable)
/// after which the run aborts: the disk is not recovering on its own.
const DISK_HOLD_ABORT_ROUNDS: u32 = 2;
/// Environment overrides for the two constants above (review: shared-HPC
/// scratch may want a longer grace than 2×60s). Same convention as
/// `OXO_FLOW_SPAWN_WATCHDOG_SECS`: unparseable or `0` falls back to the
/// default — a typo can never disable the abort (it would hang instead).
const DISK_TICK_ENV: &str = "OXO_FLOW_DISK_TICK_SECS";
const DISK_HOLD_ROUNDS_ENV: &str = "OXO_FLOW_DISK_HOLD_ROUNDS";

fn disk_tick_secs() -> u64 {
    parse_disk_tick_secs(std::env::var(DISK_TICK_ENV).ok().as_deref())
}

fn parse_disk_tick_secs(raw: Option<&str>) -> u64 {
    raw.and_then(|v| v.parse::<u64>().ok())
        .filter(|&s| s > 0)
        .unwrap_or(DISK_PRESSURE_TICK_SECS)
}

fn disk_hold_abort_rounds() -> u32 {
    parse_disk_hold_abort_rounds(std::env::var(DISK_HOLD_ROUNDS_ENV).ok().as_deref())
}

fn parse_disk_hold_abort_rounds(raw: Option<&str>) -> u32 {
    raw.and_then(|v| v.parse::<u32>().ok())
        .filter(|&s| s > 0)
        .unwrap_or(DISK_HOLD_ABORT_ROUNDS)
}

/// Parse the watchdog threshold from the environment. Extracted as a pure
/// function of the env so tests exercise it without process-global mutation.
fn spawn_watchdog_threshold() -> std::time::Duration {
    parse_spawn_watchdog_threshold(std::env::var(SPAWN_WATCHDOG_ENV).ok().as_deref())
}

/// Pure derivation: `None`/unparseable/`0` fall back to the default — `0`
/// matches the `OXO_ENV_LOCK_TIMEOUT_SECS` convention rather than meaning
/// "disable", so a typo can never silently turn the watchdog off.
fn parse_spawn_watchdog_threshold(raw: Option<&str>) -> std::time::Duration {
    raw.and_then(|v| v.parse::<u64>().ok())
        .filter(|&s| s > 0)
        .map(std::time::Duration::from_secs)
        .unwrap_or(std::time::Duration::from_secs(DEFAULT_SPAWN_WATCHDOG_SECS))
}

/// Strip ANSI escape sequences: run-log files are plain text. Mirrors the
/// file-side filter of `crate::logging`, which exposes no public writer.
fn strip_ansi(text: &str) -> String {
    let bytes = text.as_bytes();
    let mut plain = String::with_capacity(text.len());
    let mut i = 0;
    while i < bytes.len() {
        if bytes[i] == 0x1b {
            if i + 1 < bytes.len() && bytes[i + 1] == b'[' {
                // CSI sequence: ESC '[' parameters final-byte.
                i += 2;
                while i < bytes.len() && !(0x40..=0x7e).contains(&bytes[i]) {
                    i += 1;
                }
                if i < bytes.len() {
                    i += 1;
                }
            } else {
                // Stray escape.
                i += 1;
            }
        } else {
            let start = i;
            while i < bytes.len() && bytes[i] != 0x1b {
                i += 1;
            }
            // 0x1b never appears inside a multi-byte UTF-8 sequence.
            plain.push_str(&text[start..i]);
        }
    }
    plain
}

/// Emit a structured
/// [`ExecutionEvent`](oxo_flow_core::executor::ExecutionEvent) as one JSON
/// line in the tracing
/// stream (issue #194 B3): the event schema stops being dead code and the
/// run log gains a machine-readable execution timeline alongside the prose.
/// Background runs skip it for the same reason as [`progress_narrate`] —
/// the redirected stderr already carries the line exactly once.
fn emit_execution_event(event: oxo_flow_core::executor::ExecutionEvent) {
    if std::env::var_os("OXO_FLOW_STDERR_ALREADY_REDIRECTED").is_none() {
        tracing::info!(target: "execution_event", "{}", event.to_json_log());
    }
}

/// A termination signal `run` handles (audit C14), with the shell exit
/// convention `128 + signum`: SIGINT → 130, SIGTERM → 143, SIGHUP → 129.
#[derive(Debug, Clone, Copy)]
pub struct InterruptSignal {
    /// Signal name, for the operator-facing message.
    pub name: &'static str,
    /// Raw signal number.
    pub signum: i32,
}

impl InterruptSignal {
    /// Process exit status for a run stopped by this signal.
    pub fn exit_code(&self) -> i32 {
        128 + self.signum
    }
}

/// Resolve when a termination signal arrives (audit C14). Installs handlers
/// for SIGINT, SIGTERM, and SIGHUP; `None` where they cannot be installed.
async fn wait_for_termination_signal() -> Option<InterruptSignal> {
    #[cfg(unix)]
    {
        use tokio::signal::unix::{SignalKind, signal};
        let mut sigint = signal(SignalKind::interrupt()).ok()?;
        let mut sigterm = signal(SignalKind::terminate()).ok()?;
        let mut sighup = signal(SignalKind::hangup()).ok()?;
        tokio::select! {
            _ = sigint.recv() => Some(InterruptSignal { name: "SIGINT", signum: 2 }),
            _ = sigterm.recv() => Some(InterruptSignal { name: "SIGTERM", signum: 15 }),
            _ = sighup.recv() => Some(InterruptSignal { name: "SIGHUP", signum: 1 }),
        }
    }
    #[cfg(not(unix))]
    {
        tokio::signal::ctrl_c().await.ok()?;
        Some(InterruptSignal {
            name: "SIGINT",
            signum: 2,
        })
    }
}

/// Tear the run down on a termination signal (audit C14): signal every
/// in-flight rule's process tree (the default disposition would orphan them
/// with PPID 1), record the interrupted rules in the checkpoint, persist it,
/// and exit `128 + signum`. Never returns.
#[allow(clippy::too_many_arguments)] // the teardown needs the full teardown context
async fn terminate_on_signal(
    signal: InterruptSignal,
    executor: &LocalExecutor,
    join_set: &mut tokio::task::JoinSet<(
        String,
        oxo_flow_core::executor::JobStatus,
        oxo_flow_core::executor::JobRecord,
    )>,
    in_flight: impl Iterator<Item = &String>,
    checkpoint: &Arc<Mutex<CheckpointState>>,
    checkpoint_path: &Path,
    // #540: `std::process::exit` bypassed the RunJsonSummary Drop backstop
    // AND skipped the terminal webhook — --json consumers got zero bytes
    // and webhook subscribers never saw the run end. Emit both here.
    json_summary: &mut RunJsonSummary,
    config: &WorkflowConfig,
) -> ! {
    eprintln!(
        "\n{} {} received — stopping in-flight rules and saving the checkpoint",
        "Interrupted:".yellow().bold(),
        signal.name
    );
    // Same teardown as the first-failure abort (issue #131): release queued
    // resource waiters, then signal each live child's whole subtree.
    executor.clear_resource_waiters().await;
    // Blocking-pool sweep + a post-abort_all second pass — same shape and
    // rationale as the first-failure abort (#524).
    let snapshot = executor.active_pids();
    tokio::task::spawn_blocking(move || {
        for (rule, pid) in snapshot {
            if let Err(e) = oxo_flow_core::executor::timeout::kill_process_tree(pid) {
                tracing::warn!(rule = %rule, pid, error = %e, "failed to signal in-flight rule on {}", signal.name);
            }
        }
    })
    .await
    .ok();
    join_set.abort_all();
    let late = executor.active_pids();
    if !late.is_empty() {
        tokio::task::spawn_blocking(move || {
            for (rule, pid) in late {
                if let Err(e) = oxo_flow_core::executor::timeout::kill_process_tree(pid) {
                    tracing::warn!(rule = %rule, pid, error = %e, "failed to signal late-spawned rule on {}", signal.name);
                }
            }
        })
        .await
        .ok();
    }

    let mut ck = checkpoint.lock().await;
    for rule in in_flight {
        // #525: the engine killed this rule — record the signal evidence,
        // never a fabricated exit code. `exit_code: Some(130)` read as a
        // real process exit in rule_runs/reports (the exact inverse of the
        // #498 evidence model, and undemotable).
        let record = oxo_flow_core::executor::JobRecord {
            signal: Some(15), // SIGTERM — what kill_process_tree sends first
            rule: rule.clone(),
            status: oxo_flow_core::executor::JobStatus::Failed,
            enqueued_at: None,
            started_at: None,
            finished_at: Some(chrono::Utc::now()),
            exit_code: None,
            stdout: None,
            stderr: None,
            command: None,
            retries: 0,
            skip_reason: Some(format!("interrupted by {}", signal.name)),
            max_rss_mb: None,
            cpu_seconds: None,
            caption: None,
        };
        ck.record_run(&record, output_tail_bytes_from_env());
        ck.mark_failed(rule);
    }
    if let Err(e) = ck.save_to_file(checkpoint_path) {
        tracing::warn!(error = %e, "failed to save checkpoint after interruption");
    }
    drop(ck);
    json_summary.emit("failed", &RunCounts::default(), vec![]);
    notify_webhook(
        config,
        oxo_flow_core::webhook::WebhookEvent::WorkflowFailed,
        None,
    )
    .await;
    std::process::exit(signal.exit_code());
}

/// #843: reclaim one candidate's outputs mid-run — the runtime counterpart
/// of the post-run temporary-cleanup sweep. Same persist-before-unlink audit
/// rule (issue #315 F2): tombstone + checksum migration are persisted first,
/// and the unlink is skipped when persistence fails, so a crash can never
/// leave an output gone-but-not-tombstoned. Returns the number of unlinked
/// paths so the caller can narrate reclaimed bytes.
async fn reclaim_rule_outputs(
    candidate: &oxo_flow_core::scheduler::ReclaimCandidate,
    workdir: &std::path::Path,
    checkpoint: &Arc<Mutex<CheckpointState>>,
    checkpoint_path: &Path,
    run_log: &SharedRunLog,
) -> anyhow::Result<usize> {
    let mut paths = Vec::new();
    {
        let mut ck = checkpoint.lock().await;
        for path in &candidate.paths {
            let resolved = workdir.join(path);
            if !resolved.exists() {
                continue;
            }
            if let Some(sha) = ck.checksums.remove(path) {
                ck.record_cleaned_checksum(path, sha);
            }
            paths.push(path.clone());
        }
        if paths.is_empty() {
            return Ok(0);
        }
        ck.tombstones.insert(candidate.rule.clone(), paths.clone());
        if let Err(e) = ck.save_to_file_async(checkpoint_path).await {
            // Roll back the in-memory migration; the outputs stay on disk.
            ck.tombstones.remove(&candidate.rule);
            for path in &paths {
                if let Some(sha) = ck.cleaned_checksums.remove(path) {
                    ck.checksums.insert(path.clone(), sha);
                }
            }
            return Err(anyhow::anyhow!(
                "persisting tombstone failed: {e} — outputs kept"
            ));
        }
    }
    let mut unlinked = 0usize;
    for path in &paths {
        let resolved = workdir.join(path);
        // Symlink-aware classification (review): `is_dir` follows links, so
        // a symlinked output directory would hit remove_dir_all's
        // not-a-directory failure. A symlink is always unlinked as the link
        // itself — never the bytes it points at.
        let ok = match std::fs::symlink_metadata(&resolved) {
            Ok(md) if md.file_type().is_symlink() => std::fs::remove_file(&resolved).is_ok(),
            Ok(md) if md.is_dir() => std::fs::remove_dir_all(&resolved).is_ok(),
            Ok(_) => std::fs::remove_file(&resolved).is_ok(),
            Err(_) => false,
        };
        if ok {
            unlinked += 1;
        } else {
            diagnostic_narrate(
                format_args!(
                    "  {} reclaim unlink failed for '{}' path '{path}'",
                    "!".yellow(),
                    candidate.rule
                ),
                Some(run_log),
            );
        }
    }
    diagnostic_narrate(
        format_args!(
            "  {} reclaimed temporary outputs of '{}' ({} file(s)) — regenerated on demand via cascade-up",
            "⊘".dimmed(),
            candidate.rule,
            unlinked
        ),
        Some(run_log),
    );
    Ok(unlinked)
}

/// Print the config-change impact summary (issue #62).
///
/// Distinguishes the full invalidation set (checkpoint mutation; includes
/// rules outside this run's targets/samples) from the rules that will
/// actually re-execute in THIS run (intersection with `order`). Under
/// `--rerun` everything is forced anyway, so the summary is suppressed —
/// the snapshot/fingerprint refresh still happened silently.
#[allow(clippy::too_many_arguments)]
/// Values of `[config_meta.*] sensitive = true` keys, for runner-side
/// output masking (issue #99 B1). Shared by the local run, the verbose
/// plan print, and the cluster path.
fn sensitive_values_of(config: &WorkflowConfig) -> Vec<String> {
    config
        .config_meta
        .iter()
        .filter(|(_, def)| def.sensitive)
        .filter_map(|(key, _)| config.config.get(key))
        .map(oxo_flow_core::config_impact::config_value_string)
        .collect()
}

/// Adopt a loaded checkpoint for the workflow about to run (audit B8).
///
/// A checkpoint lives at `<workdir>/.oxo-flow/checkpoint.json`, so two
/// workflows pointed at the same directory read each other's records and
/// would reuse one another's artifacts: the freshness gate only compares
/// paths, mtimes, and content hashes, never workflow identity. A checkpoint
/// recorded under a different workflow path therefore has its reuse records
/// dropped (every rule re-executes), and one that predates workflow tracking
/// keeps the historical behavior with a note.
fn adopt_checkpoint_for_workflow(
    ck: &mut CheckpointState,
    workflow: &Path,
    run_log: Option<&SharedRunLog>,
) {
    match ck.foreign_workflow(workflow) {
        Some(true) => {
            diagnostic_narrate(
                format_args!(
                    "  {} checkpoint belongs to a different workflow — ignoring its \
                     completed/failed records so nothing is reused (every rule re-executes)",
                    "Note:".yellow()
                ),
                run_log,
            );
            ck.invalidate_reuse_records();
        }
        Some(false) => {}
        None => {
            tracing::debug!(
                "checkpoint predates workflow-identity tracking; reuse checks stay \
                 path/mtime/fingerprint based"
            );
        }
    }
}

fn print_truncated_list(label: &str, names: &[String]) -> String {
    // Truncate the list — a cohort-wide shell edit can mismatch hundreds of
    // expanded rule instances (e.g. step5 × 100 samples).
    let shown: Vec<&str> = names.iter().take(3).map(String::as_str).collect();
    let extra = names.len().saturating_sub(3);
    let suffix = if extra > 0 {
        format!(", … (+{extra} more)")
    } else {
        String::new()
    };
    format!("  {label} {}{}", shown.join(", "), suffix)
}

#[allow(clippy::too_many_arguments)] // report + snapshot + config + log plumbing; all needed
fn print_config_change_summary(
    report: &ConfigChangeReport,
    old_snapshot: &std::collections::BTreeMap<String, String>,
    config: &WorkflowConfig,
    sensitive_keys: &HashSet<String>,
    order: &[String],
    completed_in_run: usize,
    rerun: bool,
    run_log: Option<&SharedRunLog>,
) {
    if rerun {
        return;
    }
    if report.is_legacy {
        diagnostic_narrate(
            format_args!(
                "  {} checkpoint predates config tracking: recorded a baseline snapshot; \
                 future config changes will invalidate affected rules automatically",
                "Note:".yellow()
            ),
            run_log,
        );
        return;
    }
    if report.changed_keys.is_empty()
        && report.added_keys.is_empty()
        && report.removed_keys.is_empty()
        && report.fingerprint_mismatches.is_empty()
        && report.when_flip_invalidated.is_empty()
    {
        return;
    }

    let mut lines: Vec<String> = vec!["Config change:".bold().cyan().to_string()];
    for key in &report.changed_keys {
        if sensitive_keys.contains(key) {
            lines.push(format!("  {key}: **** → ****"));
        } else {
            let old = old_snapshot.get(key).map(String::as_str).unwrap_or("?");
            let new = config
                .config
                .get(key)
                .map(config_value_string)
                .unwrap_or_else(|| "?".to_string());
            lines.push(format!("  {key}: {old} → {new}"));
        }
    }
    for key in &report.added_keys {
        lines.push(format!("  {key}: (new key)"));
    }
    for key in &report.removed_keys {
        lines.push(format!("  {key}: (removed)"));
    }
    if !report.fingerprint_mismatches.is_empty() {
        lines.push(print_truncated_list(
            "rule definition changed:",
            &report.fingerprint_mismatches,
        ));
    }

    if !report.when_flip_invalidated.is_empty() {
        // Truncate like fingerprint lists — flips can cascade across many
        // expanded instances.
        lines.push(print_truncated_list(
            "when-condition flipped:",
            &report.when_flip_invalidated,
        ));
    }
    if !report.when_gate_exempt.is_empty() {
        // Issue #198: gates whose truth value survived the toggle keep their
        // checkpoint entries — say so, or the reuse looks like a bug.
        lines.push(print_truncated_list(
            "when-condition unchanged, reused:",
            &report.when_gate_exempt,
        ));
    }

    let order_set: HashSet<&str> = order.iter().map(String::as_str).collect();
    let rerun_this_run = report
        .invalidated
        .iter()
        .filter(|name| order_set.contains(name.as_str()))
        .count();
    lines.push(format!(
        "  → invalidated {} ({} directly affected), re-running {}/{} this run, skipping {}",
        report.invalidated.len(),
        report.directly_affected.len(),
        rerun_this_run,
        order.len(),
        completed_in_run,
    ));
    // The summary is multi-line; narrate it as one block so the run log keeps
    // the same grouping the console shows.
    diagnostic_narrate(format_args!("{}", lines.join("\n")), run_log);
}

/// Resolve a possibly-relative path against the current directory without
/// resolving symlinks. Checkpoint records use this so `resume` works from
/// any invocation directory (issue #68).
fn absolutize(path: &Path) -> Result<PathBuf> {
    if path.is_absolute() {
        Ok(path.to_path_buf())
    } else {
        Ok(std::env::current_dir()?.join(path))
    }
}

/// Lexically collapse `.`/`..` segments from an absolutized path
/// (`/wf/./data` → `/wf/data`) without touching the filesystem. The
/// workflow-dir join in `parent_dir` leaves a `.` segment for bare
/// filenames; anchored {input}/{log} strings inherit it (issue #427).
fn normalize_dots(path: PathBuf) -> PathBuf {
    let mut out = PathBuf::with_capacity(path.as_os_str().len());
    for comp in path.components() {
        match comp {
            std::path::Component::CurDir => {}
            std::path::Component::ParentDir => {
                // Only reachable for the leading segments of a relative
                // input — absolutize already stripped interior `..` against
                // a real base. Keep it verbatim rather than guessing.
                out.push(comp);
            }
            other => out.push(other),
        }
    }
    if out.as_os_str().is_empty() {
        PathBuf::from(".")
    } else {
        out
    }
}

/// Long-form names of `oxo-flow run` flags (plus clap's global --help/
/// --version). Keep in sync with the `Run` variant in main.rs.
///
/// clap's trailing `config_overrides` positional (`allow_hyphen_values`)
/// cannot distinguish `--json` the run flag from `--json` a hyphen-value
/// once positional overrides start, so flags typed after `KEY=VALUE` land
/// in the override list (issue #71). Matching them here turns a confusing
/// "invalid config flag" error — or a silently wrong override — into
/// actionable guidance.
const RUN_FLAG_NAMES: &[&str] = &[
    "jobs",
    "keep-going",
    "workdir",
    "target",
    "module",
    "retry",
    "timeout",
    "log-file",
    "resume-failed",
    "profile",
    "max-threads",
    "max-memory",
    "max-submitted",
    "skip-env-setup",
    "skip-ref-build",
    "cache-dir",
    "provenance",
    "bundle",
    "yes",
    "arg",
    "ai-recover",
    "ai-max-retries",
    "samples",
    "rerun",
    "no-report-snapshot",
    "background",
    "json",
    "quiet",
    "verbose",
    "no-color",
    "help",
    "version",
];

/// Short-form run flag names (same contract as [`RUN_FLAG_NAMES`]).
const RUN_SHORT_FLAGS: &[&str] = &["j", "k", "d", "t", "r", "h", "V"];

/// Run flags that `dry-run` also accepts as real command flags (issue
/// #441: `--cache-dir` was added for run→dry-run transcription parity).
/// When the swallowed-flag parser runs under `dry-run`, these names read
/// as a misordered command flag, not "not supported".
const DRY_RUN_SHARED_FLAGS: &[&str] = &["cache-dir"];

/// Flags removed from `run`, mapped to their replacement.
///
/// They must NOT live in [`RUN_FLAG_NAMES`]: that list names real command
/// flags, and a workflow declaring a config key with one of these names
/// (`sample = {...}`) would have its `--sample VALUE` override rejected as a
/// command flag (audit finding). Reaching the unknown-argument path instead
/// still gets a migration hint.
const REMOVED_RUN_FLAGS: &[(&str, &str)] = &[("sample", "--samples")];

/// Validate a config value against its ConfigDef declaration.
///
/// `from_cli` distinguishes an EXPLICIT user-provided value from a
/// workflow-declared default: an empty string from the CLI is a legitimate
/// derive-if-empty override (issue #430) and skips the path checks, while
/// defaults stay strict.
fn validate_config_value(
    name: &str,
    value: &str,
    def: &oxo_flow_core::config::ConfigDef,
    from_cli: bool,
) -> anyhow::Result<()> {
    // ── choices ──
    if let Some(ref choices) = def.choices
        && !choices.iter().any(|c| c == value)
    {
        anyhow::bail!(
            "invalid value '{}' for config '{}'. Allowed: [{}]\n  {}",
            value,
            name,
            choices.join(", "),
            def.help.as_deref().unwrap_or("")
        );
    }

    // ── type validation ──
    if let Some(ref type_) = def.type_ {
        match type_.as_str() {
            "int" => {
                value.parse::<i64>().map_err(|_| {
                    anyhow::anyhow!(
                        "config '{}' expects an integer, got '{}'\n  {}",
                        name,
                        value,
                        def.help.as_deref().unwrap_or("")
                    )
                })?;
            }
            "float" => {
                value.parse::<f64>().map_err(|_| {
                    anyhow::anyhow!(
                        "config '{}' expects a float, got '{}'\n  {}",
                        name,
                        value,
                        def.help.as_deref().unwrap_or("")
                    )
                })?;
            }
            "bool" => {
                if value != "true" && value != "false" {
                    anyhow::bail!(
                        "config '{}' expects a boolean (true/false), got '{}'\n  {}",
                        name,
                        value,
                        def.help.as_deref().unwrap_or("")
                    );
                }
            }
            "path" => {
                // issue #430: an explicitly-passed EMPTY value is a
                // legitimate override — pipelines branch on
                // `[ -n "{config.x}" ]` for derive-if-empty behavior, so
                // `--gene_bed ""` selects "derive from fasta+gtf" and must
                // bypass both the empty-path check and `must_exist` (there
                // is no file to exist yet). Only DEFAULTS keep the strict
                // check: a workflow author's default must point at a real
                // file.
                if value.is_empty() && from_cli {
                    return Ok(());
                }
                if value.is_empty() {
                    anyhow::bail!(
                        "config '{}' expects a path, got empty string\n  {}",
                        name,
                        def.help.as_deref().unwrap_or("")
                    );
                }
                // must_exist check
                if def.must_exist {
                    let p = std::path::Path::new(value);
                    if !p.exists() {
                        anyhow::bail!(
                            "config '{}' path does not exist: '{}'\n  {}",
                            name,
                            value,
                            def.help.as_deref().unwrap_or("")
                        );
                    }
                }
            }
            _ => { /* "string" or unknown — no validation */ }
        }
    }

    // ── range validation (requires type = int or float) ──
    if let Some(ref range) = def.range
        && let Some((min_str, max_str)) = range.split_once("..")
    {
        let min: f64 = min_str.trim().parse().map_err(|_| {
            anyhow::anyhow!("invalid range min '{}' for config '{}'", min_str, name)
        })?;
        let max: f64 = max_str.trim().parse().map_err(|_| {
            anyhow::anyhow!("invalid range max '{}' for config '{}'", max_str, name)
        })?;
        let val: f64 = value.parse().map_err(|_| {
            anyhow::anyhow!(
                "config '{}' has range '{range}' but value '{}' is not numeric\n  {}",
                name,
                value,
                def.help.as_deref().unwrap_or("")
            )
        })?;
        if val < min || val > max {
            anyhow::bail!(
                "config '{}' value {} is outside range {}..{}\n  {}",
                name,
                val,
                min,
                max,
                def.help.as_deref().unwrap_or("")
            );
        }
    }

    Ok(())
}

/// Remove entries in the environment cache that have not been touched for
/// `max_age_days` (issue #75; recursive since #194 C1 — subdirectories
/// were previously never aged out). Returns the number of entries removed.
fn cleanup_cache_dir(cache_dir: &std::path::Path, max_age_days: u64) -> usize {
    let max_age = std::time::Duration::from_secs(max_age_days * 24 * 3600);
    let now = std::time::SystemTime::now();
    let mut removed = 0;
    let mut stack = vec![cache_dir.to_path_buf()];
    while let Some(dir) = stack.pop() {
        let Ok(entries) = std::fs::read_dir(&dir) else {
            continue;
        };
        for entry in entries.flatten() {
            let path = entry.path();
            let is_file = path.is_file();
            if !is_file && !path.is_dir() {
                continue;
            }
            let Ok(metadata) = std::fs::metadata(&path) else {
                continue;
            };
            let Ok(modified) = metadata.modified() else {
                continue;
            };
            let Ok(age) = now.duration_since(modified) else {
                continue;
            };
            if age <= max_age {
                // Young entries may still contain aged descendants; only
                // descend into directories that are themselves young.
                if !is_file {
                    stack.push(path);
                }
                continue;
            }
            let ok = if is_file {
                std::fs::remove_file(&path).is_ok()
            } else {
                std::fs::remove_dir_all(&path).is_ok()
            };
            if ok {
                removed += 1;
            }
        }
    }
    removed
}

/// Attempt count for a single-shot AI operation in `run`/`dry-run`.
///
/// `--ai-max-retries N` bounds how many times a FAILED AI call is attempted
/// (1 when the flag is absent — the pre-flag behavior); a successful call is
/// never repeated. The workflow's `[ai] max_retries` is not consulted here:
/// that budget drives the agentic orchestrator used by `template`, which
/// neither the recovery diagnosis nor the dry-run analysis goes through
/// (audit finding: the flag was accepted and discarded).
fn ai_attempts(cli_max_retries: Option<u32>) -> u32 {
    cli_max_retries.unwrap_or(1).max(1)
}

/// Pull the real exit code out of a failure reason ("exit code 137 — OOM …").
/// Panics and aborts have no code in the reason and keep -1.
fn failure_exit_code(failure_reason: &str) -> i32 {
    failure_reason
        .lines()
        .next()
        .and_then(|line| line.split("exit code ").nth(1))
        .and_then(|rest| {
            rest.split_whitespace()
                .next()
                .and_then(|tok| tok.parse().ok())
        })
        .unwrap_or(-1)
}

#[allow(clippy::too_many_arguments)]
/// Parse CLI config overrides — the SAME accepted forms for `run` and
/// `dry-run` (issue #77 parity):
///   KEY=VALUE            direct positional form
///   --KEY=VALUE          long-flag form
///   --KEY VALUE          long-flag form with separate value (only for
///                        config keys the workflow DECLARES — an unknown
///                        `--token` is a typo'd command flag and must not
///                        be silently swallowed as an override, issue #71)
///   --arg KEY=VALUE      legacy `--arg` form (backward compatible)
///
/// Override keys are canonicalized onto their declared `[config]` spelling:
/// `--api-token=x` names the declared key `api_token` (hyphen/underscore
/// variants resolve when the normalization is unambiguous, issue #794).
/// This matters most for `sensitive` keys — validation, masking, and the
/// checkpoint snapshot are keyed on the declared spelling — but it applies
/// to every declared key. Keys that match nothing declared are untouched:
/// `KEY=VALUE` keeps its issue #62 injection semantics, and an unknown
/// `--` token is rejected (issue #71).
///
/// `command` names the invoking subcommand ("run", "dry-run", …) so a
/// swallowed run flag under a command that does not support it says so
/// (issue #430): under `dry-run`, `-j` reads "not supported by dry-run",
/// not the run-ordering hint.
pub(crate) fn parse_cli_overrides(
    cli_args: Vec<String>,
    declared_config_keys: &std::collections::HashSet<String>,
    command: &str,
) -> anyhow::Result<std::collections::HashMap<String, String>> {
    let mut cli_arg_values: std::collections::HashMap<String, String> =
        std::collections::HashMap::new();
    let mut cli_args_iter = cli_args.into_iter().peekable();
    while let Some(arg_str) = cli_args_iter.next() {
        // issue #71: run flags swallowed by the trailing overrides positional
        // get a targeted error instead of a confusing parse failure.
        let swallowed_flag = arg_str
            .strip_prefix("--")
            .map(|flag| flag.split('=').next().unwrap_or(flag))
            .filter(|name| RUN_FLAG_NAMES.contains(name))
            .map(|name| format!("--{name}"))
            .or_else(|| {
                arg_str
                    .strip_prefix('-')
                    .filter(|rest| !rest.starts_with('-'))
                    .map(|flag| flag.split('=').next().unwrap_or(flag))
                    .filter(|name| RUN_SHORT_FLAGS.contains(name))
                    .map(|name| format!("-{name}"))
            });
        if let Some(flag) = swallowed_flag {
            // issue #430: dry-run (and every other command sharing this
            // parser) accepts only a subset of run's flags — a token like
            // `-j` under `dry-run` is not a misordered flag but one the
            // command does not support at all, so the message is
            // command-aware.
            let flag_name = flag.trim_start_matches('-');
            let is_run_flag = RUN_FLAG_NAMES.contains(&flag_name)
                || (flag_name.len() == 2 && RUN_SHORT_FLAGS.contains(&&flag_name[1..]));
            let hint = if command == "run" {
                format!(
                    "'{flag}' is a command flag, not a config override.\n  \
                     Command flags must come before KEY=VALUE overrides, e.g.:\n  \
                     oxo-flow run <workflow.oxoflow> --json min_quality=30\n  \
                     For a config key that itself starts with dashes, use --arg KEY=VALUE\n  \
                     (also placed before positional overrides)."
                )
            } else if is_run_flag {
                // Issue #441: dry-run shares some run flags for real
                // (`--cache-dir`) — for those, a swallowed token is a
                // misordered command flag, not an unsupported one.
                let shared = command == "dry-run" && DRY_RUN_SHARED_FLAGS.contains(&flag_name);
                if shared {
                    format!(
                        "'{flag}' is a command flag, not a config override.\n  \
                         Command flags must come before KEY=VALUE overrides, e.g.:\n  \
                         oxo-flow {command} <workflow.oxoflow> --cache-dir .env-cache min_quality=30"
                    )
                } else {
                    format!(
                        "'{flag}' is a run flag and is not supported by {command}.\n  \
                         Run flags only work on `oxo-flow run`; {command} accepts its own \
                         flags before KEY=VALUE overrides, e.g.:\n  \
                         oxo-flow {command} <workflow.oxoflow> --target my_rule min_quality=30"
                    )
                }
            } else {
                format!(
                    "'{flag}' is not supported by {command} and is not a config override.\n  \
                     Config overrides take KEY=VALUE, e.g.:\n  \
                     oxo-flow {command} <workflow.oxoflow> min_quality=30"
                )
            };
            anyhow::bail!("{hint}");
        }
        let (mut k, v) = if let Some(eq) = arg_str.find('=') {
            let k = arg_str[..eq].trim_start_matches('-').to_string();
            (k, arg_str[eq + 1..].to_string())
        } else if let Some(k) = arg_str.strip_prefix("--") {
            // Space form: the declared check accepts hyphen-spelled keys that
            // canonicalize onto a declared key (`--api-token VALUE` names
            // `api_token`, issue #794). Unknown tokens still get the issue
            // #71 rejection — canonicalization never widens that gate.
            let space_form_key = if declared_config_keys.contains(k) {
                Some(k.to_string())
            } else {
                canonicalize_override_key(k, declared_config_keys)
            };
            if let Some(k) = space_form_key {
                // `--KEY VALUE` — consume the next argument as the value.
                // A following token that starts with `--` is a command flag
                // the user meant to pass, never a config value: swallowing
                // it silently disabled the flag (`dry-run wf --mode --json`
                // ran without JSON output, audit finding). The `=` form
                // already errors on the unknown token, so point there.
                if let Some(next) = cli_args_iter.peek()
                    && next.starts_with("--")
                {
                    anyhow::bail!(
                        "config flag '--{k}' expects a value, but the next argument '{next}' \
                         is a command flag.\n  Use the '=' form (--{k}=<value>) or place the \
                         override after the command flags as {k}=<value>."
                    );
                }
                let v = cli_args_iter.next().with_context(|| {
                    format!(
                        "invalid config flag: '{arg_str}' — expected --KEY=VALUE or --KEY VALUE"
                    )
                })?;
                (k.to_string(), v)
            } else {
                // issue #71 follow-up: a `--` token that is neither a
                // registered flag (caught above) nor a declared config key
                // is almost certainly a typo'd command flag (e.g.
                // `--config x`). Never silently swallow it as an override.
                let rename = REMOVED_RUN_FLAGS
                    .iter()
                    .find(|(removed, _)| *removed == k)
                    .map(|(removed, replacement)| {
                        format!("\n  '--{removed}' was renamed to {replacement}.")
                    })
                    .unwrap_or_default();
                anyhow::bail!(
                    "unknown argument '{arg_str}' — did you mean KEY=VALUE overrides?{rename}\n  \
                     Config overrides take KEY=VALUE (e.g. threads=8) or --KEY=VALUE, e.g.:\n  \
                     oxo-flow {command} <workflow.oxoflow> threads=8; \
                     for a config key that itself starts with dashes, use --arg KEY=VALUE"
                );
            }
        } else {
            anyhow::bail!(
                "invalid config value format: '{arg_str}' — expected KEY=VALUE, --KEY=VALUE, or --KEY VALUE"
            );
        };
        if k.is_empty() {
            anyhow::bail!("invalid config value format: '{arg_str}' — KEY must be non-empty");
        }
        // issue #794: canonicalize onto the declared spelling so a hyphen-
        // spelled override (`--api-token=x` for declared `api_token`) lands
        // on the key the workflow declared — inheriting its validation,
        // sensitive masking, and space-form eligibility. Without this the
        // override is stored under the dashed alias, the declared key keeps
        // its default, and the value leaks into the checkpoint unmasked.
        if !declared_config_keys.contains(&k)
            && let Some(canonical) = canonicalize_override_key(&k, declared_config_keys)
        {
            k = canonical;
        }
        // issue #430: an EMPTY VALUE is a legitimate override — pipelines
        // branch on `[ -n "{config.x}" ]` for derive-if-empty behavior
        // (rnaseq gene_bed/chrom_sizes/transcript_fasta), and the CLI must
        // be able to select that branch (`gene_bed=`, `--gene_bed ""`).
        // Only an empty KEY is a syntax error.
        cli_arg_values.insert(k, v);
    }
    Ok(cli_arg_values)
}

/// Resolve an override key that does not exactly match a declared `[config]`
/// key by normalizing hyphens to underscores (issue #794). Returns the
/// declared key only when the normalization is unambiguous — a normalized
/// form that would map to two declared keys is left alone (the caller then
/// applies issue #62/#71 semantics to the literal spelling). Callers invoke
/// this only after an exact-match check, and a key reaching here always
/// contains a hyphen, so the result never equals the input.
fn canonicalize_override_key(
    key: &str,
    declared_config_keys: &std::collections::HashSet<String>,
) -> Option<String> {
    if !key.contains('-') {
        return None;
    }
    let normalized = key.replace('-', "_");
    if declared_config_keys.contains(&normalized) {
        return Some(normalized);
    }
    let matches: Vec<&String> = declared_config_keys
        .iter()
        .filter(|d| d.replace('-', "_") == normalized)
        .collect();
    match matches.as_slice() {
        [only] => Some((*only).clone()),
        _ => None,
    }
}

/// Apply parsed overrides and the defaults of declarative config entries
/// (`key = { default, required, … }` in `[config]`). Shared by `run` and
/// `dry-run` so preview and execution validate identically.
pub(crate) fn apply_cli_overrides(
    config: &mut WorkflowConfig,
    cli_arg_values: &std::collections::HashMap<String, String>,
) -> anyhow::Result<()> {
    for (name, cfg_def) in &config.config_meta {
        let effective_value = if let Some(val) = cli_arg_values.get(name) {
            validate_config_value(name, val, cfg_def, true)?;
            config
                .config
                .insert(name.clone(), toml::Value::String(val.clone()));
            val.clone()
        } else if let Some(ref default) = cfg_def.default {
            validate_config_value(name, default, cfg_def, false)?;
            config
                .config
                .entry(name.clone())
                .or_insert_with(|| toml::Value::String(default.clone()));
            default.clone()
        } else if cfg_def.required {
            let help_suffix = cfg_def
                .help
                .as_deref()
                .map(|h| format!("\n  {h}"))
                .unwrap_or_default();
            let display_name = if cfg_def.sensitive {
                format!("{name} (sensitive)")
            } else {
                name.clone()
            };
            anyhow::bail!(
                "required config '{display_name}' not set. Use --{name} <value>{help_suffix}"
            );
        } else {
            continue;
        };
        // Mask sensitive values in all non-execution output
        if cfg_def.sensitive {
            tracing::info!("config '{}' = **** (sensitive)", name);
        } else {
            tracing::debug!("config '{}' = '{}'", name, effective_value);
        }
    }
    // Inject undeclared KEY=VALUE values as {config.xxx} — a CLI override
    // must win over the workflow's [config] table (issue #62).
    for (k, v) in cli_arg_values {
        config
            .config
            .insert(k.clone(), toml::Value::String(v.clone()));
    }
    Ok(())
}

/// Warn about `KEY=VALUE` overrides the workflow never uses.
///
/// Undeclared override keys are a documented escape hatch (issue #62), so
/// this warns rather than fails — but a TYPO (`samplename` for
/// `sample_name`) used to pass in total silence and exit 0 having changed
/// nothing (audit finding). A key counts as used when `[config]` declares it
/// or the workflow references `{config.KEY}` anywhere; the closest declared
/// key is named when it is within a two-edit typo.
pub(crate) fn warn_unknown_override_keys(
    overrides: &HashMap<String, String>,
    declared: &HashSet<String>,
    workflow: &Path,
) {
    let unknown: Vec<&String> = overrides
        .keys()
        .filter(|key| !declared.contains(*key))
        .collect();
    if unknown.is_empty() {
        return;
    }
    // The workflow parsed a moment ago; if it cannot be re-read the
    // reference scan is skipped and every undeclared key warns.
    let text = std::fs::read_to_string(workflow).unwrap_or_default();
    for key in unknown {
        if text.contains(&format!("config.{key}")) {
            continue;
        }
        let hint = closest_declared_key(key, declared)
            .map(|candidate| format!(" — did you mean '{candidate}'?"))
            .unwrap_or_default();
        eprintln!(
            "{} override '{key}' is not a declared [config] key and the workflow never \
             references {{config.{key}}}{hint} It changes nothing.",
            "Warning:".bold().yellow(),
        );
    }
}

/// Nearest declared config key within a two-edit typo of `key` (the same
/// heuristic `oxo-flow-core`'s unknown-key check applies).
fn closest_declared_key<'a>(key: &str, declared: &'a HashSet<String>) -> Option<&'a str> {
    let lowered = key.to_ascii_lowercase();
    declared
        .iter()
        .map(|candidate| {
            (
                edit_distance(&lowered, &candidate.to_ascii_lowercase(), 2),
                candidate,
            )
        })
        .filter(|(distance, _)| *distance <= 2)
        .min_by_key(|(distance, candidate)| (*distance, candidate.as_str()))
        .map(|(_, candidate)| candidate.as_str())
}

/// Levenshtein edit distance, capped so unrelated pairs exit early.
fn edit_distance(a: &str, b: &str, cap: usize) -> usize {
    let a: Vec<char> = a.chars().collect();
    let b: Vec<char> = b.chars().collect();
    if a.len().abs_diff(b.len()) > cap {
        return cap + 1;
    }
    let mut prev: Vec<usize> = (0..=b.len()).collect();
    let mut cur = vec![0usize; b.len() + 1];
    for (i, ca) in a.iter().enumerate() {
        cur[0] = i + 1;
        for (j, cb) in b.iter().enumerate() {
            let cost = usize::from(ca != cb);
            cur[j + 1] = (prev[j + 1] + 1).min(cur[j] + 1).min(prev[j] + cost);
        }
        std::mem::swap(&mut prev, &mut cur);
    }
    prev[b.len()]
}

/// Quote a path for safe embedding in a reference build command (executed
/// via `bash -c` / `sh -c`): wrap in single quotes and escape embedded
/// quotes with the POSIX `'\''` sequence — the same idiom the environment
/// wrapper uses when embedding commands into `sh -c '...'` and the cluster
/// renderer uses for its `cd '...'`. Without the quoting, a source path
/// containing spaces or shell metacharacters would be spliced bare into the
/// command (issue #136 tier-2 audit).
fn quote_shell_path(path: &str) -> String {
    format!("'{}'", path.replace('\'', "'\\''"))
}

/// Substitute the reference builder's `{source}` placeholder with the
/// expanded source path, shell-quoted so the path survives as one argument
/// (issue #136 tier-2 audit — the raw splice broke on spaces/metacharacters).
fn substitute_source_placeholder(build_cmd: &str, expanded_source: &str) -> String {
    build_cmd.replace("{source}", &quote_shell_path(expanded_source))
}

/// Render the "known modules" list for an unknown-module error. `run` and
/// `dry-run` share the phrasing — in particular the explicit empty-list
/// hint, so a workflow without includes cannot print a trailing "known
/// modules: " (issue #136 tier-2 audit).
fn known_modules_hint(module_rules: &std::collections::HashMap<String, Vec<String>>) -> String {
    if module_rules.is_empty() {
        "(none — no [[include]] modules)".to_string()
    } else {
        module_rules
            .keys()
            .map(|k| k.as_str())
            .collect::<Vec<_>>()
            .join(", ")
    }
}

/// Issue #142 H1 gate: an unknown `{config.*}` placeholder used to expand to
/// literal text while the run exited 0 — silent wrong outputs. This is the
/// same E005 detector `validate` uses, applied as a hard gate by both `run`
/// and `dry-run`, so the three surfaces can never disagree about a typo'd
/// key. Returns the human-readable findings (rule, key, fix) or empty.
fn undefined_config_findings(config: &WorkflowConfig) -> Vec<String> {
    config
        .rules
        .iter()
        .flat_map(|rule| oxo_flow_core::format::undefined_config_refs(rule, config))
        .map(|d| {
            format!(
                "rule '{}': {} ({})",
                d.rule.as_deref().unwrap_or("<unknown>"),
                d.message,
                d.suggestion.unwrap_or_default()
            )
        })
        .collect()
}

/// Fail-fast gate for EXTERNAL source inputs: a required input pattern that
/// resolves to concrete missing files AND has no producer rule in the DAG can
/// never be regenerated by this run. Previously such a run could complete
/// silently when every rule already held checkpoint credit — the manifest
/// detector flagged the missing source, but the cascade-up found no completed
/// producer (external files have none) and the run exited 0 claiming
/// success while its declared inputs no longer existed on disk.
///
/// Scoped to no-producer patterns only: generated intermediates have
/// producers and stay owned by the checkpoint cascade path, which handles
/// tombstones and recorded when-verdicts (this gate runs before the
/// checkpoint is even loaded, so it must not duplicate that logic).
/// Optional rules, when-gated-off rules, and scatter rules with zero
/// fan-out can legitimately have absent inputs and are skipped — the same
/// stances the validate/quality gates take (issues #493, #616).
///
/// Two further engine-tolerated shapes must not be flagged:
/// - `[[references]]` outputs: the reference-build machinery creates them
///   during the run (after this gate) and they are not DAG nodes;
/// - inputs living in the `sample_pattern` discovery directory: bare
///   relative inputs resolve from the shell cwd (= workdir) at execution,
///   but sample data legitimately lives where sample_pattern found it.
///
/// Returns human-readable findings, or empty when every required external
/// source is present.
fn missing_source_findings(
    config: &WorkflowConfig,
    dag: &WorkflowDag,
    order: &[String],
    workdir: &Path,
    workflow_dir: &Path,
    wildcard_values: &HashMap<String, String>,
) -> Vec<String> {
    // Outputs the reference-build machinery will create during this run —
    // absent from the DAG by design, so a missing one is not a dead source.
    let reference_outputs: HashSet<String> = config
        .references
        .iter()
        .map(|r| {
            oxo_flow_core::executor::checkpoint::expand_config_in_path(&r.output, wildcard_values)
        })
        .collect();
    // Directory sample_pattern discovery scanned (parse.rs anchoring):
    // absolute patterns anchor at their own parent, relative ones at the
    // workflow file's parent. Bare relative inputs may live here instead
    // of the workdir.
    let sample_search_dir = config.workflow.sample_pattern.as_ref().and_then(|pattern| {
        let expanded =
            oxo_flow_core::executor::checkpoint::expand_config_in_path(pattern, wildcard_values);
        let path = Path::new(&expanded);
        if path.is_absolute() {
            path.parent().map(|p| p.to_path_buf())
        } else {
            // Relative pattern: parse.rs anchors it at the workflow file's
            // parent (mirror of the sample_pattern split there).
            let joined = workflow_dir.join(path);
            Some(
                joined
                    .parent()
                    .map(|p| p.to_path_buf())
                    .unwrap_or_else(|| workflow_dir.to_path_buf()),
            )
        }
    });
    let mut findings = Vec::new();
    for name in order {
        let Some(rule) = config.get_rule(name) else {
            continue;
        };
        // A `when`-gated-off rule never runs in this configuration — its
        // inputs are not required to exist (issue #493 stance).
        if crate::commands::run_preview::when_condition_false(rule, config, wildcard_values) {
            continue;
        }
        // A scatter rule with zero fan-out never instantiates (issue #616).
        if rule.scatter.as_ref().is_some_and(|scatter| {
            if !scatter.values.is_empty() {
                return false;
            }
            match scatter.values_from.as_deref() {
                None => true,
                Some(values_from) => config
                    .resolve_config_list(values_from)
                    .is_some_and(|values| values.is_empty()),
            }
        }) {
            continue;
        }
        // Optional rules may be absent by design.
        if rule.optional.is_optional() {
            continue;
        }
        let missing = oxo_flow_core::executor::checkpoint::missing_input_patterns(
            rule,
            workdir,
            wildcard_values,
        );
        for pattern in missing {
            if reference_outputs.contains(&pattern) {
                // Built by the [[references]] machinery later in this run.
                continue;
            }
            // Only external sources gate here — a missing pattern WITH a
            // producer belongs to the manifest/cascade path below.
            if dag.producer_of(&pattern).is_some() {
                continue;
            }
            // A bare relative input may resolve inside the sample_pattern
            // discovery directory rather than the workdir.
            let absolute = Path::new(&pattern).is_absolute();
            if !absolute
                && sample_search_dir
                    .as_ref()
                    .is_some_and(|dir| dir.join(&pattern).exists())
            {
                continue;
            }
            findings.push(format!(
                "rule '{name}': input '{pattern}' does not exist and no rule produces it"
            ));
        }
    }
    findings
}

/// Per-rule listing for the loud empty-cascade warning: names each affected
/// rule and the concrete input patterns that cannot be resolved. Extracted
/// so tests can assert the message content without driving a full run.
fn missing_source_lines(
    missing_inputs: &HashSet<String>,
    config: &WorkflowConfig,
    workdir: &Path,
    wildcard_values: &HashMap<String, String>,
) -> Vec<String> {
    let mut names: Vec<&String> = missing_inputs.iter().collect();
    names.sort();
    names
        .into_iter()
        .map(|name| {
            let patterns = config
                .get_rule(name)
                .map(|rule| {
                    oxo_flow_core::executor::checkpoint::missing_input_patterns(
                        rule,
                        workdir,
                        wildcard_values,
                    )
                })
                .unwrap_or_default();
            let listing = if patterns.is_empty() {
                "(unresolvable)".to_string()
            } else {
                patterns.join(", ")
            };
            format!("    {name} — missing: {listing}")
        })
        .collect()
}

#[allow(clippy::too_many_arguments)]
pub async fn run_command(
    workflow: Option<PathBuf>,
    jobs: usize,
    keep_going: bool,
    workdir: Option<PathBuf>,
    log_file: Option<PathBuf>,
    target: Vec<String>,
    module: Vec<String>,
    retry: u32,
    timeout: String,
    resume_failed: bool,
    profile: Option<String>,
    max_threads: u32,
    max_memory: u64,
    skip_env_setup: bool,
    skip_ref_build: bool,
    cache_dir: Option<PathBuf>,
    provenance: bool,
    json: bool,
    cli_args: Vec<String>,
    ai_recover: bool,
    ai_max_retries: Option<u32>,
    samples_filter: Vec<String>,
    rerun: bool,
    no_report_snapshot: bool,
    max_submitted: Option<usize>,
    // #538: `resume` forwards the user-supplied checkpoint path — without
    // this, run_command re-derived <workdir>/.oxo-flow/checkpoint.json and
    // an explicit `resume <path>` silently resumed from a DIFFERENT (or
    // empty) checkpoint while the banner claimed otherwise.
    checkpoint_override: Option<PathBuf>,
) -> Result<()> {
    print_banner();

    // `--json` consumers get the summary document on every exit path — the
    // guard emits it from `Drop` when a path returns before the explicit
    // emission below (audit finding: preflight/parse aborts exited 1 with
    // zero bytes on stdout).
    let mut json_summary = RunJsonSummary::new(json, workflow.as_deref());
    // A detached background child removes the pid file its launcher wrote
    // when this run finishes; a foreground run gets an inert guard.
    let _background_pid_file = crate::background::BackgroundPidFileGuard::from_env();

    // `-j 0` means "no explicit concurrency limit": clamp once at the
    // boundary so every downstream consumer — the scheduler submit cap,
    // the run-loop semaphore, and the executor's own semaphore (which
    // would otherwise be a zero-permit gate that hangs the first
    // submission) — sees a consistent value (issue #136 fix 1).
    let jobs = jobs.max(1);

    // ── Repository-URL workflows (nextflow-style `run <repo>`) ──────────
    // `oxo-flow run gh:owner/repo[@ref]` / `owner/repo[@ref]` (or a *.git
    // URL / local repo dir) checks out into <cwd>/.oxo-flow/repos/<owner>-<name>
    // (reused on later runs).
    // For run, @ref is a git branch/tag — never a Release asset (that is
    // pull's bundle namespace). Outputs/checkpoint default to the CURRENT
    // directory: data belongs outside the clone.
    let (workflow, from_repo) = match workflow {
        Some(w) => {
            let text = w.to_string_lossy();
            match crate::commands::pull::classify_run_source(&text) {
                Some(crate::commands::pull::RunSource::Repo { url, git_ref }) => {
                    let cache = crate::commands::pull::repo_cache_dir(&url, git_ref.as_deref())?;
                    let wf = crate::commands::pull::checkout_repo_workflow(
                        &url,
                        git_ref.as_deref(),
                        &cache,
                    )
                    .await?;
                    (wf, true)
                }
                None => (w, false),
            }
        }
        None => (resolve_workflow(None)?, false),
    };
    json_summary.set_workflow(&workflow);
    // Issue #427: everything downstream — base_dir, workflow_root,
    // runtime_workdir comparison in expand.rs, reference anchoring, env
    // resolution — compares against this directory. A bare `wf.oxoflow`
    // resolves its parent to ".", so workflow_root = Some(".") anchors
    // inputs to "./data/…" which still fails from the run workdir.
    // Absolutize once; relative workflow_dir only resolves from the
    // invocation cwd anyway.
    let workflow_dir = normalize_dots(absolutize(oxo_flow_core::parent_dir(&workflow))?);
    // Workdir default: the workflow's own directory, EXCEPT for repository
    // runs, where the current directory holds the user's data (the clone is
    // a read-only cache).
    let workdir_default: PathBuf = if from_repo {
        std::env::current_dir().context("cannot determine current directory")?
    } else {
        workflow_dir.clone()
    };

    // ── Workdir lock (issue #70) ────────────────────────────────────────
    // Concurrent runs on the same workdir would race on
    // .oxo-flow/checkpoint.json (last-writer-wins). The exclusive lock is
    // acquired as early as possible and held for the whole run; the OS
    // releases it automatically if this process exits or crashes, so there
    // are no stale locks.
    // Issue #427: absolutize the effective workdir once — the lock file
    // path, workflow_root comparison, and expand.rs's runtime_workdir
    // comparison all want canonical forms.
    let workdir_effective =
        normalize_dots(absolutize(workdir.as_ref().unwrap_or(&workdir_default))?);
    // Surface the lock error's suggestion (issue #158): without it a user
    // hitting a busy workdir learns WHAT is wrong but not that the lock
    // auto-releases when the other process exits.
    let _workdir_lock =
        WorkdirLock::acquire(&workdir_effective).map_err(|e| match e.suggestion() {
            Some(s) => anyhow::anyhow!("{e}\n  hint: {s}"),
            None => anyhow::anyhow!("{e}"),
        })?;

    // Issue #751: bundle runs extract the workflow into a temp dir, so
    // relative sample_pattern/pairs_pattern scans anchor inside the
    // extraction and never see data kept under --workdir. When the workflow
    // file lives in the extraction AND the effective workdir differs from
    // it, hand the workdir to the parser as a discovery fallback. (Without
    // an explicit --workdir, main.rs passes the extraction dir itself, so
    // the two are equal and the fallback is a no-op.)
    let discovery_fallback =
        if workflow.starts_with(std::env::temp_dir()) && workdir_effective != workflow_dir {
            Some(workdir_effective.as_path())
        } else {
            None
        };
    let mut config =
        WorkflowConfig::from_file_with_discovery_fallback(&workflow, discovery_fallback)
            .with_context(|| format!("failed to parse {}", workflow.display()))?;

    // ── Parse and merge CLI config overrides (shared with dry-run) ─────
    // The workflow's own [config] keys gate the `--KEY VALUE` space form:
    // unknown `--` tokens are rejected as typos instead of silently
    // swallowed. config.config covers BOTH plain values (`key = "v"`) and
    // declarative entries (`key = { default = ... }`) — config_meta alone
    // only sees the latter.
    let declared_config_keys: std::collections::HashSet<String> =
        config.config.keys().cloned().collect();
    let cli_arg_values = parse_cli_overrides(cli_args, &declared_config_keys, "run")?;

    apply_cli_overrides(&mut config, &cli_arg_values)?;
    // A typo'd override key must not pass silently (audit finding): warn
    // when it is neither declared nor referenced anywhere in the workflow.
    warn_unknown_override_keys(&cli_arg_values, &declared_config_keys, &workflow);

    // ── Filter to a sample subset (--samples @path / first:N / names / ready) ────
    // Runs after CLI overrides so `ready` resolution sees the final config
    // values in `{config.x}` paths (issue #63).
    if !samples_filter.is_empty()
        && let Some(readiness) = samples::apply_samples_filter(
            &mut config,
            &samples_filter,
            true,
            workdir.as_deref().unwrap_or(&workdir_default),
        )?
    {
        samples::print_readiness_section(&readiness);
    }

    // Load profile if specified and merge config values — the SAME shared
    // helper dry-run uses, so preview and execution can never drift apart.
    // Merged BEFORE apply_defaults so a profile's `[defaults]` table
    // actually reaches the rules (apply_defaults consumes self.defaults).
    if let Some(ref profile_name) = profile {
        crate::commands::run_preview::merge_profile(&mut config, profile_name, &workflow_dir)?;
    }

    config.apply_defaults();
    // Issue #427: when the run workdir differs from the workflow directory,
    // tell expansion so relative input_groups disk-scan patterns emit
    // workflow-absolute paths — rule shells run with cwd = workdir, and
    // workflow-relative `{input}` would not resolve from there. Every
    // other expand caller (preview, validate, cluster, web) leaves this
    // unset and keeps raw emission. Absolutized so the expand.rs comparison
    // `runtime_workdir != base_dir` sees canonical forms on both sides.
    if let Some(effective_workdir) = workdir.as_ref() {
        config.runtime_workdir = Some(normalize_dots(absolutize(effective_workdir)?));
    }
    config
        .expand_wildcards()
        .context("failed to expand wildcard rules")?;

    // ── Undefined `{config.*}` gate (issue #142 H1) ───────────────────────
    // Runs AFTER expansion so engine-generated rules are covered too, and
    // before any file is touched — a typo'd key must fail the run, never
    // produce literal-placeholder outputs with exit 0.
    let e005 = undefined_config_findings(&config);
    if !e005.is_empty() {
        json_summary.emit("failed", &RunCounts::default(), vec![]);
        return Err(anyhow::anyhow!(
            "workflow references undefined config variable(s) — fix before running:\n  {}",
            e005.join("\n  ")
        ));
    }

    let mut dag = WorkflowDag::from_rules_with_config(
        &config.rules,
        &config_placeholder_values(&config.config),
    )
    .context("failed to build workflow DAG")?;

    // --module partial runs (issue #112 elasticity): each module name
    // resolves to its rules plus the host producers of its declared
    // concrete inputs; upstream DAG dependents come via the target
    // machinery below.
    let mut target = target;
    for m in &module {
        match config.module_closure(m) {
            Some(names) => target.extend(names),
            None => {
                // Pre-execution abort: nothing ran, but the summary
                // contract still holds for --json (issue #142 H6).
                json_summary.emit("failed", &RunCounts::default(), vec![]);
                return Err(anyhow::anyhow!(
                    "unknown module '{m}' — known modules: {}",
                    known_modules_hint(&config.module_rules)
                ));
            }
        }
    }
    // ── Run log + workflow provenance (issue #115 pillar 1) ────────────
    // Every run (and `resume`, which re-enters this function) archives its
    // own log under the workdir with numbered rotation; the header names
    // the exact workflow version (name, version, git HEAD) that produced
    // this record. Best-effort: a logging failure never fails the run.
    // Sensitive values must be computed BEFORE the run-log header is built:
    // the header embeds the raw command line, and `--arg KEY=secret` would
    // otherwise land in plaintext in the run log and its rotated backups
    // (issue #99 B1 / #136 fix 3). Computed once here; the executor's
    // capture masking below reuses the same list.
    let sensitive_values = sensitive_values_of(&config);
    let workflow_abs = absolutize(&workflow)?;
    let workflow_git_sha =
        oxo_flow_core::executor::checkpoint::CheckpointState::workflow_git_sha(&workflow_abs);
    let effective_workdir_log = workdir.as_ref().unwrap_or(&workdir_default);
    let run_log_path = match &log_file {
        // Relative --log-file paths resolve against the workdir, matching
        // the default log location (issue #136 fix 4).
        Some(p) if p.is_absolute() => p.clone(),
        Some(p) => effective_workdir_log.join(p),
        None => effective_workdir_log.join(".oxo-flow/logs/oxo-flow.log"),
    };
    let run_log_header = format!(
        "oxo-flow run log\nstarted_at: {}\noxo-flow: v{}\ncommand: {}\nworkflow: {}\nworkflow_name: {}\nworkflow_version: {}\ngit_sha: {}\nworkdir: {}\n\n",
        chrono::Local::now().to_rfc3339(),
        env!("CARGO_PKG_VERSION"),
        oxo_flow_core::executor::process::mask_sensitive(
            &std::env::args().collect::<Vec<_>>().join(" "),
            &sensitive_values,
        ),
        workflow_abs.display(),
        config.workflow.name,
        config.workflow.version,
        workflow_git_sha
            .as_deref()
            .unwrap_or("(not inside a git repository)"),
        effective_workdir_log.display(),
    );
    // Background runs (issue #194 A3): `spawn_detached` already redirected
    // this process's stdout/stderr onto the run log, so arming the tracing
    // tee here would write every event TWICE into the same file. In that
    // mode the header is printed to stderr (→ the redirect) and the tee is
    // skipped entirely; foreground runs arm the tee as before.
    let run_log: SharedRunLog = Arc::new(std::sync::Mutex::new(
        if std::env::var_os("OXO_FLOW_STDERR_ALREADY_REDIRECTED").is_some() {
            eprintln!("{run_log_header}");
            None
        } else {
            match crate::logging::activate_run_log(&run_log_path, &run_log_header) {
                Ok(guard) => Some(guard),
                Err(e) => {
                    tracing::warn!(error = %e, path = %run_log_path.display(), "failed to open run log; continuing without file logging");
                    None
                }
            }
        },
    ));
    tracing::info!(
        workflow = %config.workflow.name,
        version = %config.workflow.version,
        git_sha = workflow_git_sha.as_deref().unwrap_or("(none)"),
        workdir = %effective_workdir_log.display(),
        "workflow run started (log: {})",
        run_log_path.display()
    );

    // Targeted runs (-t) close over the INSTANTIATED DAG (issue #247):
    // instances whose `when` evaluates false under the effective config
    // never enter the execution set, and the closure does not expand
    // through them. Prefix matching still resolves the entries.
    let when_false_rules: std::collections::HashSet<String> = if target.is_empty() {
        std::collections::HashSet::new()
    } else {
        let wildcard_values: HashMap<String, String> = config_placeholder_values(&config.config);
        config
            .rules
            .iter()
            .filter(|rule| {
                crate::commands::run_preview::when_condition_false(rule, &config, &wildcard_values)
            })
            .map(|rule| rule.name.clone())
            .collect()
    };

    // Default targets (issue #469): `target = true` rules seed the build
    // when nothing explicit was selected. Explicit -t/--module wins.
    let target = resolve_default_targets(&config, &target);
    let mut order = if target.is_empty() {
        dag.execution_order()?
    } else {
        let target_refs: Vec<&str> = target.iter().map(String::as_str).collect();
        let (filtered_order, skipped_targets) = dag
            .execution_order_for_targets_skipping_with_source_check(
                &target_refs,
                &when_false_rules,
                // Pre-built inputs (atacseq bwa_mem finding): an input whose
                // concrete file already exists in the run workdir is
                // satisfiable — a when-false producer must not gate it.
                &|path| run_workdir_check(Some(workdir.as_ref().unwrap_or(&workdir_default)), path),
            )
            .with_context(|| "failed to resolve target rules")?;
        for skipped in &skipped_targets {
            print_target_skipped_note(skipped, &when_false_rules, Some(&run_log));
        }
        if filtered_order.is_empty() {
            json_summary.emit("failed", &RunCounts::default(), vec![]);
            return Err(anyhow::anyhow!(
                "all requested targets are when-gated false — nothing to run"
            ));
        }
        filtered_order
    };
    // ── Workdir source linking (issue #837) ────────────────────────────
    // In the two-directory mode (shared workflow repo + separate workdir)
    // rules run with CWD = workdir, so relative paths resolve against the
    // WORKDIR copy of a repo file — which operators used to hand-sync with
    // nothing verifying the copies. The engine now materializes referenced
    // repo files in the workdir as SYMLINKS: zero duplicated content, the
    // repo stays the single source of truth. This runs BEFORE the
    // missing-source gate so a fresh workdir is satisfied by the freshly
    // created links instead of hard-failing on absent source inputs.
    let link_report = crate::commands::workdir_sources::link_workflow_sources(
        &config,
        &dag,
        &order,
        &workflow_dir,
        workdir.as_ref().unwrap_or(&workdir_default),
        &config_placeholder_values(&config.config),
    );
    narrate_source_link_report(&link_report, Some(&run_log));
    // ── Missing external source gate ───────────────────────────────────
    // A required input with no producer in the DAG is a source file — if it
    // is absent NOW the run can never succeed, so fail before scheduling
    // instead of exiting 0 with stale outputs. Generated intermediates are
    // excluded (they have producers and are owned by the manifest cascade).
    let missing_sources = missing_source_findings(
        &config,
        &dag,
        &order,
        workdir.as_ref().unwrap_or(&workdir_default),
        &workflow_dir,
        &config_placeholder_values(&config.config),
    );
    if !missing_sources.is_empty() {
        json_summary.emit("failed", &RunCounts::default(), vec![]);
        return Err(anyhow::anyhow!(
            "workflow is missing required source input(s) — no rule produces them:\n  {}",
            missing_sources.join("\n  ")
        ));
    }
    emit_execution_event(oxo_flow_core::executor::ExecutionEvent::WorkflowStarted {
        workflow_name: config.workflow.name.clone(),
        total_rules: order.len(),
    });
    notify_webhook(
        &config,
        oxo_flow_core::webhook::WebhookEvent::WorkflowStarted,
        None,
    )
    .await;
    if !crate::commands::is_quiet() {
        eprintln!(
            "{} {} rules in execution order",
            "DAG:".bold().green(),
            order.len()
        );

        for (i, rule_name) in order.iter().enumerate() {
            eprintln!("  {}. {}", i + 1, rule_name);
        }
    }

    // ── Load checkpoint + config-change impact analysis (issue #62) ─────
    // Detection must happen AFTER overrides/profile merges and wildcard
    // expansion (config.config is final) and BEFORE ExecutorConfig
    // construction (the invalidation set must reach the executor's
    // freshness gate, which would otherwise silently skip re-submitted
    // rules with stale outputs).
    // An explicit checkpoint path (resume) wins over the workdir-derived
    // default (#538); load AND save both use it.
    let checkpoint_path = checkpoint_override.unwrap_or_else(|| {
        workdir
            .as_ref()
            .unwrap_or(&workdir_default)
            .join(".oxo-flow/checkpoint.json")
    });
    // Retention for failed-output aside files (issue #194 C2): stale
    // `.oxo-failed` evidence ages out at run start instead of accumulating
    // forever. Best-effort — cleanup must never block the run.
    let stale_asides = oxo_flow_core::executor::output_invalidation::cleanup_stale_failed_asides(
        workdir.as_ref().unwrap_or(&workdir_default),
        oxo_flow_core::executor::output_invalidation::OXOX_FAILED_RETENTION_DAYS,
    );
    if stale_asides > 0 {
        tracing::info!(
            count = stale_asides,
            "removed stale .oxo-failed aside files (retention)"
        );
    }

    // Issue #432(a): a corrupt checkpoint must never degrade silently into
    // an empty state — that silently re-executes every completed rule, which
    // costs hours on real data. Fail loudly BEFORE any work is scheduled so
    // the user can back up / repair the file rather than lose the credit.
    let mut loaded_checkpoint = if checkpoint_path.exists() {
        match CheckpointState::load_from_file(&checkpoint_path) {
            Ok(ck) => ck,
            Err(e) => {
                return Err(anyhow::anyhow!(
                    "checkpoint at {} is corrupt and cannot be loaded ({}); \
                     refusing to start — every completed rule would silently re-run. \
                     Fix or remove the file, then retry.",
                    checkpoint_path.display(),
                    e
                ));
            }
        }
    } else {
        CheckpointState::default()
    };
    adopt_checkpoint_for_workflow(&mut loaded_checkpoint, &workflow_abs, Some(&run_log));
    // Stale running-set cleanup (issue #685): persisted `running` entries
    // mean the previous run died (crash / signal) between spawn and any
    // terminal state. No task is executing those rules now — clear them
    // once so the reuse block and resume accounting stay honest, loudly.
    let stale_running: Vec<String> = loaded_checkpoint.running_rules().iter().cloned().collect();
    if !stale_running.is_empty() {
        for rule in &stale_running {
            loaded_checkpoint.clear_running(rule);
        }
        diagnostic_narrate(
            format_args!(
                "  {} checkpoint lists {} rule(s) as running from a previous run \
                 ({}): they died with it and will re-run.",
                "!".yellow(),
                stale_running.len(),
                stale_running.join(", ")
            ),
            Some(&run_log),
        );
        if let Err(e) = loaded_checkpoint.save_to_file(&checkpoint_path) {
            tracing::warn!("Failed to persist cleaned checkpoint: {e}");
        }
    }
    let checkpoint: Arc<Mutex<CheckpointState>> = Arc::new(Mutex::new(loaded_checkpoint));

    // Sensitive keys are needed by the snapshot-drift warning below and by
    // config-change detection.
    let sensitive_keys: std::collections::HashSet<String> = config
        .config_meta
        .iter()
        .filter(|(_, def)| def.sensitive)
        .map(|(key, _)| key.clone())
        .collect();

    // ── Snapshot drift warning (issue #243) ──────────────────────────────
    // The checkpoint records the EFFECTIVE config of the original run
    // (workflow defaults + its CLI overrides). This invocation re-parsed
    // the workflow with THIS invocation's overrides — `resume` passes none,
    // so plan-time gates (pair/instance `when`, `{meta.*}` baking) re-judge
    // on defaults and the instance set can drift from the original run
    // before any invalidation machinery sees it. Invalidation (below)
    // covers shell interpolation and gate flips; it cannot resurrect
    // instances expansion already dropped. Warn loudly, naming every key
    // that differs — the operator decides whether to re-pass the overrides.
    {
        let ck = checkpoint.lock().await;
        let drifted: Vec<String> = ck
            .config_snapshot
            .keys()
            .filter(|key| {
                !oxo_flow_core::config_impact::is_engine_injected_key(key)
                    && match config.config.get(key.as_str()) {
                        Some(value) => {
                            let new = oxo_flow_core::config_impact::snapshot_value(
                                value,
                                sensitive_keys.contains(key.as_str()),
                            );
                            &new != ck.config_snapshot.get(*key).unwrap()
                        }
                        None => true,
                    }
            })
            .cloned()
            .collect();
        if !drifted.is_empty() && !ck.completed_rules.is_empty() {
            eprintln!(
                "{} effective config differs from the original run's snapshot — \
                 plan-time gated instances (pair/sample `when`, `{{meta.*}}`) may \
                 drift from the recorded run. Re-pass the original overrides to \
                 resume the same instance set:",
                "Config drift:".bold().yellow()
            );
            for key in &drifted {
                match config.config.get(key.as_str()) {
                    Some(value) => {
                        let new = oxo_flow_core::config_impact::snapshot_value(
                            value,
                            sensitive_keys.contains(key.as_str()),
                        );
                        let old = ck.config_snapshot.get(key.as_str()).unwrap();
                        let (old, new) = if old.starts_with("sha256:") || new.starts_with("sha256:")
                        {
                            ("****".to_string(), "****".to_string())
                        } else {
                            (old.clone(), new)
                        };
                        eprintln!("  {key}: {old} → {new}");
                    }
                    None => eprintln!("  {key}: (removed from this invocation)"),
                }
            }
        }
    }

    // Store workflow path and working directory in the checkpoint so
    // `resume` re-runs from the same place (issue #68). Both are made
    // absolute — a raw relative path would only resolve from the original
    // invocation directory.
    {
        let mut ck = checkpoint.lock().await;
        ck.set_workflow_path(&workflow_abs);
        // Record the workflow repository's HEAD SHA for provenance
        // (issue #115 pillar 1) — best-effort, never fails the run.
        if let Some(sha) = &workflow_git_sha {
            ck.set_workflow_git_sha(sha.clone());
        }
        ck.set_workdir(&absolutize(workdir.as_deref().unwrap_or(&workdir_default))?);
    }
    // Same absolutized run workdir, reused by the config-change replay below
    // (issue #282): runtime file functions in `when` resolve their relative
    // paths against it so threshold changes get real verdicts.
    let run_workdir = absolutize(workdir.as_deref().unwrap_or(&workdir_default))?;

    // Changed config keys → only rules referencing them (plus DAG downstream)
    // are invalidated; edited rule definitions are caught by fingerprints.
    // Engine-injected keys (samples_list, samples_<group>) are excluded:
    // their --samples churn must not invalidate everything.
    let old_snapshot = {
        let ck = checkpoint.lock().await;
        ck.config_snapshot.clone()
    };
    let (change_report, mut force_rules, completed_in_run) = {
        let mut ck = checkpoint.lock().await;
        let report = oxo_flow_core::config_impact::detect_config_changes_with_replay(
            &mut ck,
            &config.rules,
            &dag,
            &config.config,
            &sensitive_keys,
            &config.workflow.interpreter_map,
            config.defaults.shell_prelude.as_deref(),
            Some(&workflow_dir),
            Some(&run_workdir),
            Some(&config),
        );
        let force_rules: std::collections::HashSet<String> =
            report.invalidated.iter().cloned().collect();
        let completed_in_run = order
            .iter()
            .filter(|name| ck.completed_rules.contains(*name))
            .count();
        // Eager save: the invalidation is correct regardless of whether this
        // run proceeds (budget bail, crash, a `when` condition flipping to
        // false). Persisting now makes change detection idempotent across
        // aborted runs and stabilizes the `when`-false state on disk.
        if let Err(e) = ck.save_to_file(&checkpoint_path) {
            tracing::warn!(error = %e, "failed to save checkpoint after config-change detection");
        }
        (report, force_rules, completed_in_run)
    };
    print_config_change_summary(
        &change_report,
        &old_snapshot,
        &config,
        &sensitive_keys,
        &order,
        completed_in_run,
        rerun,
        Some(&run_log),
    );

    // Issue #142 M1: rules whose fingerprint differed only in the
    // sample-derived input list are NOT invalidated — toggling --samples
    // must not re-run (and overwrite) cohort-level gather outputs. The
    // input-manifest check re-verifies set + content below, so a genuine
    // input edit still invalidates there.
    if !change_report.sample_selection_exempt.is_empty() {
        diagnostic_narrate(
            format_args!(
                "  {} skipped {} rule(s) whose definition only changed with the --samples selection: {} — outputs still cover the previous run's full sample set; use --rerun to regenerate with the new selection",
                "⚠".yellow(),
                change_report.sample_selection_exempt.len(),
                change_report.sample_selection_exempt.join(", ")
            ),
            Some(&run_log),
        );
    }

    // All config values (including CLI --arg overrides) become {config.key} in templates.
    let wildcard_values: Arc<HashMap<String, String>> =
        Arc::new(config_placeholder_values(&config.config));
    let workdir_actual = Arc::new(workdir.as_ref().unwrap_or(&workdir_default).clone());

    // ── Pre-flight disk check (issue #75, wired 2026-08-23) ────────────────
    // ENOSPC mid-run is the classic silent-loss incident (live: TMPDIR full
    // killed a scatter campaign half-way). Warn when either the workdir or
    // the temp dir has less than 1 GB free BEFORE any rule runs.
    for (label, path) in [
        ("workdir", workdir_actual.as_path()),
        ("temp dir", std::env::temp_dir().as_path()),
    ] {
        if let Some(free_kb) = free_kilobytes(path) {
            let free_gb = free_kb as f64 / 1024.0 / 1024.0;
            if free_gb < 1.0 {
                tracing::warn!(
                    label,
                    path = %path.display(),
                    free_gb = format!("{free_gb:.1}"),
                    "less than 1 GB free disk space — the run may fail with ENOSPC mid-way"
                );
            }
        }
    }

    // ── Input manifest comparison (issue #72) ──────────────────────────────
    // A completed rule is reused only when the file set its inputs resolved
    // to at completion time (paths + size + mtime) still matches. Globs and
    // Dir inputs detect added/removed files; plain files detect edits. A
    // mismatch invalidates the rule and its DAG downstream (the same cascade
    // as config changes); legacy checkpoints (no manifest recorded) adopt
    // the current set as a one-time baseline — the same policy as the config
    // snapshot (issue #62).
    {
        let mut ck = checkpoint.lock().await;
        // Shared with dry-run's read-only preview (issue #66) — keep the
        // detection semantics in ONE place.
        let (mismatched, missing_inputs, baselined, sample_selection_driven) =
            crate::commands::run_preview::detect_input_manifest_invalidations(
                &mut ck,
                &config,
                &dag,
                &order,
                workdir_actual.as_ref(),
                &wildcard_values,
            );
        // Issue #142 M1: a rule whose only input-manifest change is the
        // engine-injected sample list is NOT invalidated — toggling
        // --samples must not overwrite cohort-level gather outputs with a
        // subset table. Emit the documented warning; the detection function
        // already proved (re-expansion + content-identity) the change is
        // sample-selection-only.
        if !sample_selection_driven.is_empty() {
            let names: Vec<&str> = sample_selection_driven.iter().map(String::as_str).collect();
            diagnostic_narrate(
                format_args!(
                    "  {} skipped {} rule(s) whose inputs only changed with the --samples selection: {} — outputs still cover the previous run's full sample set; use --rerun to regenerate with the new selection",
                    "⚠".yellow(),
                    names.len(),
                    names.join(", ")
                ),
                Some(&run_log),
            );
        }
        // Cascade-up: missing inputs (tombstoned temporaries) re-run their
        // completed producers first — the same semantics the preview shows.
        // Empty upstream means NO completed producer exists for the missing
        // inputs (e.g. a deleted external source file with no producer rule
        // at all) — re-running "0 producers" silently and exiting 0 would
        // claim success from stale outputs, so surface each affected rule's
        // missing patterns loudly instead (never "re-running 0 producer(s)").
        if !missing_inputs.is_empty() {
            let upstream = crate::commands::run_preview::cascade_up(&mut ck, &dag, &missing_inputs);
            force_rules.extend(upstream.iter().cloned());
            if upstream.is_empty() {
                let lines = missing_source_lines(
                    &missing_inputs,
                    &config,
                    workdir_actual.as_ref(),
                    &wildcard_values,
                );
                diagnostic_narrate(
                    format_args!(
                        "  {} required input(s) missing and cannot be regenerated in this run — affected rule(s):\n{}",
                        "⚠".yellow(),
                        lines.join("\n")
                    ),
                    Some(&run_log),
                );
            } else {
                diagnostic_narrate(
                    format_args!(
                        "  {} missing intermediate inputs — re-running {} producer rule(s): {}",
                        "↻".yellow(),
                        upstream.len(),
                        upstream.join(", ")
                    ),
                    Some(&run_log),
                );
            }
        }
        if !mismatched.is_empty() {
            let invalidated = crate::commands::run_preview::invalidate_with_downstream(
                &mut ck,
                &dag,
                &mismatched,
            );
            force_rules.extend(invalidated.iter().cloned());
            // Issue #432(a): say WHY completed credit was lost when the
            // likely cause is a prior failed run's side effects — the issue
            // reports users reading this as "restarts lose credit" when it
            // is deterministic invalidation (shared-dir side effects shift
            // glob/dir input sets, which cascades downstream).
            let prior_failures = !ck.failed_rules.is_empty();
            diagnostic_narrate(
                format_args!(
                    "  {} input changes invalidated {} rule(s): {}{}",
                    "↻".yellow(),
                    invalidated.len(),
                    invalidated.join(", "),
                    if prior_failures {
                        " — the previous run failed after some rules wrote into shared input dirs (logs/ QC dirs); their inputs changed on disk, so affected rules re-run. This is expected invalidation, not lost credit"
                    } else {
                        ""
                    }
                ),
                Some(&run_log),
            );
        }
        if baselined > 0 {
            diagnostic_narrate(
                format_args!(
                    "  Note: checkpoint predates input tracking: recorded baseline input manifests for {} completed rule(s); future input changes will invalidate them automatically",
                    baselined
                ),
                Some(&run_log),
            );
        }
        if (!mismatched.is_empty() || baselined > 0)
            && let Err(e) = ck.save_to_file(&checkpoint_path)
        {
            tracing::warn!(error = %e, "failed to save checkpoint after input-manifest detection");
        }
    }

    // Tombstone-aware skip (temporary rules): a tombstoned rule whose
    // outputs are deleted stays SKIPPED while no dependent needs them;
    // when a dependent will run, cascade-up has already removed it from
    // completed so it re-executes first. Mirrored by the dry-run preview.
    let tombstone_keep: std::collections::HashSet<String> = {
        let ck = checkpoint.lock().await;
        ck.tombstones
            .keys()
            .filter(|rule| {
                dag.dependents(rule)
                    .map(|deps| deps.iter().all(|d| ck.completed_rules.contains(d)))
                    .unwrap_or(true)
            })
            .cloned()
            .collect()
    };

    // Lazy regeneration (temporary rules): any rule that will execute may
    // consume the tombstoned outputs of completed producers — cascade up so
    // those producers re-execute first. Mirrored by the dry-run preview.
    if !rerun {
        let will_run_seeds: std::collections::HashSet<String> = {
            let ck = checkpoint.lock().await;
            order
                .iter()
                .filter(|name| {
                    !ck.completed_rules.contains(*name)
                        || (!tombstone_keep.contains(*name)
                            && !config.get_rule(name).is_some_and(|rule| {
                                crate::commands::run_preview::when_condition_false(
                                    rule,
                                    &config,
                                    &wildcard_values,
                                )
                            })
                            && config.get_rule(name).is_some_and(|rule| {
                                !crate::commands::run_preview::rule_outputs_exist(
                                    rule,
                                    workdir_actual.as_ref(),
                                    &wildcard_values,
                                )
                            }))
                })
                .cloned()
                .collect()
        };
        let upstream = {
            let mut ck = checkpoint.lock().await;
            crate::commands::run_preview::cascade_up_tombstoned(&mut ck, &dag, &will_run_seeds)
        };
        if !upstream.is_empty() {
            force_rules.extend(upstream.iter().cloned());
            diagnostic_narrate(
                format_args!(
                    "  {} temporary outputs needed again — re-running {} producer rule(s): {}",
                    "↻".yellow(),
                    upstream.len(),
                    upstream.join(", ")
                ),
                Some(&run_log),
            );
        }
    }

    // indicatif's stderr draw target auto-hides when stderr is not a terminal,
    // which makes every per-rule progress message silently disappear under pipes,
    // redirects, nohup, CI, or schedulers. When that happens, fall back to plain
    // eprintln lines so the run is never silent.
    let is_tty = std::io::IsTerminal::is_terminal(&std::io::stderr());
    // Background runs redirect stderr onto the run log (issue #194 A3):
    // the bar's redraw sequences would land in the file as ANSI garbage.
    let stderr_redirected = std::env::var_os("OXO_FLOW_STDERR_ALREADY_REDIRECTED").is_some();

    let progress = indicatif::ProgressBar::new(order.len() as u64);
    progress.set_style(
        indicatif::ProgressStyle::default_bar()
            .template(
                "{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ETA:{eta} ({msg})",
            )?
            .progress_chars("#>-"),
    );
    if !is_tty || stderr_redirected {
        // Draw into the void instead of stderr: no bar output, but every
        // set_message/set_position call stays valid (message lines are
        // printed separately via progress_narrate when !is_tty).
        progress.set_draw_target(indicatif::ProgressDrawTarget::hidden());
    }

    let timeout_secs: u64 = if timeout == "0" {
        0
    } else if let Ok(n) = timeout.parse::<u64>() {
        n
    } else {
        // A typo'd --timeout must fail fast: silently disabling the timeout
        // turns "kill the job after 30m" into "let it run forever", which is
        // the opposite of what the flag promises (audit finding).
        parse_duration_secs(&timeout).ok_or_else(|| {
            anyhow::anyhow!(
                "invalid --timeout value '{timeout}'.\n  \
                 Expected seconds (e.g. 3600) or a duration like 1h / 30m / 2d."
            )
        })?
    };

    // ── Cluster path (issue #74 phase 2) ───────────────────────────────────
    // A `[cluster]` block in effect plus a named profile routes execution to
    // a scheduler instead of the local executor. Both conditions are
    // required: a workflow carrying `[cluster]` keeps running locally until
    // the user opts in with `--profile`, so no existing workflow starts
    // submitting jobs because of this change.
    if profile.is_some()
        && let Some(cluster_profile) = config.cluster.clone()
    {
        let summary = crate::commands::run_cluster::run_on_cluster(
            &cluster_profile,
            crate::commands::run_cluster::ClusterRunArgs {
                config: &config,
                dag: &dag,
                order: &order,
                checkpoint: &checkpoint,
                checkpoint_path: &checkpoint_path,
                workdir: workdir_actual.as_ref(),
                wildcard_values: wildcard_values.as_ref(),
                sensitive_keys: &sensitive_keys,
                sensitive_values: &sensitive_values,
                force_rules: &force_rules,
                max_submitted,
                rerun,
                resume_failed,
                // The cluster path honors --cache-dir (shared env cache);
                // --skip-env-setup/--ai-recover are unsupported there and
                // warned about at submit time (issue #136 tier-2 audit).
                cache_dir: cache_dir.clone(),
                skip_env_setup,
                ai_recover,
            },
        )
        .await?;
        // The cluster path returns before the common summary emission, so
        // it routes through the same `--json` contract itself (issue #142
        // H6): the document must appear on both outcomes.
        json_summary.emit(
            if summary.is_success() {
                "completed"
            } else {
                "failed"
            },
            &RunCounts {
                succeeded: summary.succeeded,
                skipped: summary.skipped,
                failed: summary.failed,
                non_required_failed: summary.non_required_failed,
                blocked: 0,
            },
            // Cluster per-rule resources come from sacct accounting at
            // report time, not the live checkpoint — empty here.
            vec![],
        );
        notify_webhook(
            &config,
            if summary.is_success() {
                oxo_flow_core::webhook::WebhookEvent::WorkflowCompleted
            } else {
                oxo_flow_core::webhook::WebhookEvent::WorkflowFailed
            },
            Some((summary.succeeded, summary.failed, summary.skipped)),
        )
        .await;
        if !summary.is_success() {
            return Err(anyhow::anyhow!("workflow execution failed"));
        }
        // Report snapshot parity with the local path (issue #83 P1-14):
        // a cluster run leaves the same `.oxo-flow/reports/` artifacts as a
        // local one. Errors are warnings, exactly like the local call site.
        if !no_report_snapshot {
            let ck = checkpoint.lock().await;
            if let Err(e) =
                crate::commands::output::snapshot_report(&workflow, &workdir_actual, &ck)
            {
                eprintln!("  {} Report snapshot failed: {e}", "⚠".yellow());
            }
        }
        return Ok(());
    }

    let exec_config = ExecutorConfig {
        max_jobs: jobs,
        dry_run: false,
        // Issue #427: absolutized — the executor's rule shells run with
        // cwd = workdir, and anchored {input}/{log} rendering assumes a
        // usable base. A raw relative value would only resolve from the
        // invocation directory.
        workdir: match workdir.as_ref() {
            Some(p) => normalize_dots(absolutize(p)?),
            None => workdir_default.clone(),
        },
        sensitive_values: sensitive_values.clone(),
        shell_prelude: config.defaults.shell_prelude.clone(),
        keep_going,
        retry_count: retry,
        timeout: if timeout_secs > 0 {
            Some(std::time::Duration::from_secs(timeout_secs))
        } else {
            None
        },
        max_threads: if max_threads > 0 {
            Some(max_threads)
        } else {
            None
        },
        max_memory_mb: if max_memory > 0 {
            Some(max_memory)
        } else {
            None
        },
        force_rerun: rerun,
        // Rules invalidated by config-change analysis bypass the executor's
        // mtime freshness gate so their stale outputs are actually rebuilt.
        force_rules: force_rules.clone(),
        resource_groups: config
            .resource_groups
            .iter()
            .map(|(k, v)| (k.clone(), v.max))
            .collect(),
        skip_env_setup,
        cache_dir: cache_dir.clone(),
        interpreter_map: config.workflow.interpreter_map.clone(),
        // Unconstrained wildcards fall back to the safe default charset
        // inside validate_wildcard_injection (issue #203).
        wildcard_constraints: config.wildcard_constraints.clone(),
        // `file_exists(...)` in `when` conditions resolves relative paths
        // against the workflow root (issue #241) — same as plan time.
        base_dir: Some(workflow_dir.clone()),
        // Issue #427: file-backed env specs and workflow-shipped rule
        // inputs/scripts anchor to the workflow directory whenever the
        // run workdir differs from it — rule shells run with cwd = workdir.
        // workdir_effective is a CLI value here or workflow_dir itself;
        // both sides of the comparison are absolute (workflow_dir was
        // absolutized at the top), so a relative --workdir can't fool it.
        workflow_root: if *workdir_effective != workflow_dir {
            Some(workflow_dir.clone())
        } else {
            None
        },
        // Shared with the manifest snapshot resolver so staging and
        // invalidation always see the same backends (issue #80 item 2).
        storage_resolver: crate::commands::run_preview::storage_resolver(),
        // Checkpoint excerpt size, honoring OXO_FLOW_OUTPUT_TAIL_BYTES (also
        // read by the signal handler and cluster paths, which have no
        // ExecutorConfig here).
        output_tail_bytes: output_tail_bytes_from_env(),
        // The freshness gate reads recorded provenance checksums from the
        // live checkpoint (issue #194 B2).
        checkpoint: Some(checkpoint.clone()),
    };

    // Fail fast if any rule's declared request can never fit an explicit
    // --max-memory / --max-threads cap. Otherwise the run would execute earlier
    // rules and only discover the impossible one mid-pipeline.
    let scheduled: Vec<&oxo_flow_core::rule::Rule> = order
        .iter()
        .filter_map(|name| config.get_rule(name))
        .collect();
    let breaches = oxo_flow_core::scheduler::check_budget_feasibility(
        &scheduled,
        exec_config.max_threads,
        exec_config.max_memory_mb,
    );
    if !breaches.is_empty() {
        progress.finish_and_clear();
        let detail = breaches
            .iter()
            .map(|b| format!("  - {b}"))
            .collect::<Vec<_>>()
            .join("\n");
        // Pre-execution abort: nothing ran — the summary still reports
        // the failed run for --json consumers (issue #142 H6).
        json_summary.emit("failed", &RunCounts::default(), vec![]);
        return Err(anyhow::anyhow!(
            "resource budget too small for {} rule(s); no rules were run:\n{}",
            breaches.len(),
            detail
        ));
    }

    // Disk pre-flight (issue #75): warn when a rule's declared
    // `resources.disk` exceeds the free space in the workdir — a long run
    // should not discover this mid-pipeline.
    let disk_warnings = oxo_flow_core::scheduler::validate_disk_requirements(
        &scheduled.iter().map(|r| (*r).clone()).collect::<Vec<_>>(),
        workdir_actual.as_ref(),
    );
    for warning in &disk_warnings {
        eprintln!("  {} {}", "Warning:".bold().yellow(), warning);
    }

    // #843 runtime disk-pressure monitor pre-flight: when a floor is
    // configured and the workdir already sits below it, say so before the
    // first rule dispatches. On non-Unix the monitor cannot measure free
    // space (it fails open to Normal), so a configured floor would be a
    // silent no-op — warn once instead.
    if config
        .engine
        .as_ref()
        .is_some_and(|e| e.min_free_disk.is_some())
    {
        #[cfg(not(unix))]
        {
            eprintln!(
                "  {} [engine] min_free_disk is configured but free-space monitoring is not supported on this platform — the monitor will stay inactive.",
                "Warning:".bold().yellow()
            );
        }
        #[cfg(unix)]
        {
            if let Some((min_mb, _reclaim_mb)) =
                config.engine.as_ref().and_then(|e| e.disk_thresholds())
                && let Some(start_mb) =
                    oxo_flow_core::scheduler::check_available_disk_mb(workdir_actual.as_ref())
                && start_mb < min_mb
            {
                eprintln!(
                    "  {} workdir starts below [engine] min_free_disk: {}MB free, threshold {}MB — the run will hold dispatches until space is freed.",
                    "Warning:".bold().yellow(),
                    start_mb,
                    min_mb
                );
            }
        }
    }

    // Backend availability pre-flight: a pending rule declaring an
    // environment whose backend binary is missing used to surface only
    // mid-run as the backend's own "command not found" buried in that
    // rule's stderr log (live: a missing pixi discovered hours into a
    // cohort run). Reuse the exact validator `env check` uses and fail
    // before executing anything when a pending rule's backend is unusable
    // — mirroring the budget pre-flight above. Skipped/cached rules do not
    // block: this run never invokes their backend.
    let pending_envs: Vec<(String, oxo_flow_core::rule::EnvironmentSpec)> = {
        let ck = checkpoint.lock().await;
        let preview = crate::commands::run_preview::preview_run_plan(
            &ck,
            &config,
            &dag,
            &order,
            workdir_actual.as_ref(),
            &wildcard_values,
            &sensitive_keys,
            &config.workflow.interpreter_map,
            &checkpoint_path,
            rerun,
            resume_failed,
        );
        preview
            .plan
            .iter()
            .filter(|entry| !entry.status.is_skip())
            .filter_map(|entry| {
                config.get_rule(&entry.name).map(|rule| {
                    // Resolve against the workflow dir — same base the
                    // executor uses (issue #848): a manifest declared
                    // relative to the workflow file must be found from any CWD.
                    (
                        entry.name.clone(),
                        rule.environment.resolve_relative_paths(Some(&workflow_dir)),
                    )
                })
            })
            .filter(|(_, env)| !env.is_empty())
            .collect()
    };
    let env_failures = backend_preflight_failures(
        &pending_envs,
        &oxo_flow_core::environment::EnvironmentResolver::new(),
    );
    if !env_failures.is_empty() {
        progress.finish_and_clear();
        let detail = env_failures
            .iter()
            .map(|f| format!("  - {f}"))
            .collect::<Vec<_>>()
            .join("\n");
        // Pre-execution abort: nothing ran — the summary still reports
        // the failed run for --json consumers (issue #142 H6).
        json_summary.emit("failed", &RunCounts::default(), vec![]);
        return Err(anyhow::anyhow!(
            "{} environment backend(s) required by pending rules are unavailable; no rules were run:\n{}",
            env_failures.len(),
            detail
        ));
    }

    // Same cache dir the executor's EnvironmentResolver uses — reference
    // builds that declare an environment share the env cache with rules.
    let env_cache_dir = exec_config
        .cache_dir
        .clone()
        .unwrap_or_else(|| workdir_actual.as_ref().join(".oxo-flow").join("env-cache"));
    let executor = Arc::new(LocalExecutor::new(exec_config));
    let success_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
    let fail_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
    // Checksums recorded under --provenance across the whole run: when the
    // flag was set but nothing landed, the summary must say so loudly
    // instead of leaving `provenance verify` to puzzle the user (issue #276).
    let provenance_recorded = Arc::new(std::sync::atomic::AtomicUsize::new(0));
    // Failures of `required = false` rules (issue #99 B2): counted
    // separately, surfaced in the summary, but exempt from failing the run.
    let non_required_fail_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
    let skipped_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
    let failed_rules_set: Arc<Mutex<std::collections::HashSet<String>>> =
        Arc::new(Mutex::new(std::collections::HashSet::new()));
    // Rules skipped at execution time without producing outputs (e.g.
    // "optional inputs missing"). Non-optional dependents are blocked from
    // running on them — the declared input files can never exist, so
    // executing them would just produce a missing-file shell error
    // (issue #200, live: eager's samtools_flagstat after a skipped filter).
    let skipped_no_output: Arc<Mutex<std::collections::HashSet<String>>> =
        Arc::new(Mutex::new(std::collections::HashSet::new()));
    let failures: Arc<Mutex<Vec<(String, String)>>> = Arc::new(Mutex::new(Vec::new()));
    // Rules that never ran because an upstream dependency failed, paired with the
    // dependency that blocked them. Reported separately from genuine failures so
    // the root cause stays distinguishable from the fallout.
    let blocked: Arc<Mutex<Vec<(String, String)>>> = Arc::new(Mutex::new(Vec::new()));

    // When --resume-failed is set, clear failed rules from checkpoint so they re-execute.
    if resume_failed && checkpoint_path.exists() {
        let mut ck = checkpoint.lock().await;
        let failed_count = ck.failed_rules.len();
        let completed_count = ck.completed_rules.len();
        ck.failed_rules.clear();
        eprintln!(
            "{} Resuming {} completed, re-running {} failed rules",
            "Resume:".bold().cyan(),
            completed_count,
            failed_count
        );
    }

    // ── Auto-build references (indexes, data files) ──────────────────────

    if !skip_ref_build && !config.references.is_empty() {
        let ref_workdir = workdir.as_ref().unwrap_or(&workdir_default);
        let ref_checkpoint = checkpoint_path
            .parent()
            .unwrap_or(Path::new("."))
            .join("reference-checkpoint.json");
        // Migration: versions before this fix wrote the state one level
        // deeper (the checkpoint parent already ends in `.oxo-flow`, so
        // the old join produced `.oxo-flow/.oxo-flow/`). Move the old file
        // over once so stored fingerprints survive the upgrade.
        let legacy_ref_checkpoint = ref_checkpoint
            .parent()
            .unwrap_or(Path::new("."))
            .join(".oxo-flow/reference-checkpoint.json");
        if !ref_checkpoint.exists() && legacy_ref_checkpoint.exists() {
            let _ = std::fs::rename(&legacy_ref_checkpoint, &ref_checkpoint);
            // The rename runs under the workdir lock (acquired above); the
            // now-empty nested directory is removed so no shadow path stays
            // behind for a future reader to mistake for the real one.
            if let Some(parent) = legacy_ref_checkpoint.parent() {
                let _ = std::fs::remove_dir(parent);
            }
        }
        // name → fingerprint. Legacy checkpoints store a plain JSON array of
        // names; those entries are adopted with the current fingerprint
        // (no rebuild) — the same one-time window as rule config snapshots.
        let mut ref_state: HashMap<String, RefRecord> = if ref_checkpoint.exists() {
            let raw = std::fs::read_to_string(&ref_checkpoint).unwrap_or_default();
            serde_json::from_str::<HashMap<String, RefRecord>>(&raw)
                .ok()
                .or_else(|| {
                    // Legacy formats: a plain name→fingerprint map, or a
                    // bare name list — adopted with no content signature.
                    serde_json::from_str::<HashMap<String, String>>(&raw)
                        .ok()
                        .map(|m| {
                            m.into_iter()
                                .map(|(k, v)| {
                                    (
                                        k,
                                        RefRecord {
                                            fingerprint: v,
                                            content_sig: None,
                                            source_path: None,
                                        },
                                    )
                                })
                                .collect()
                        })
                })
                .or_else(|| {
                    serde_json::from_str::<Vec<String>>(&raw)
                        .map(|names| {
                            names
                                .into_iter()
                                .map(|n| {
                                    (
                                        n,
                                        RefRecord {
                                            fingerprint: String::new(),
                                            content_sig: None,
                                            source_path: None,
                                        },
                                    )
                                })
                                .collect::<HashMap<String, RefRecord>>()
                        })
                        .ok()
                })
                .unwrap_or_default()
        } else {
            HashMap::new()
        };

        // Artifacts actually rebuilt in THIS run — consumers (rules whose
        // declared inputs match) are invalidated afterwards.
        let mut rebuilt_outputs: Vec<String> = Vec::new();

        for ref_def in &config.references {
            let output_path = oxo_flow_core::executor::checkpoint::expand_config_in_path(
                &ref_def.output,
                &wildcard_values,
            );
            let output_full = ref_workdir.join(&output_path);
            // Issue #427: the SOURCE is a workflow-shipped file — anchor it
            // to the workflow directory, not the run workdir, so existence
            // probes, fingerprints, and content signatures all see the
            // same file regardless of --workdir. (Outputs stay anchored to
            // the run workdir: they are run artifacts.)
            let resolved_source = ref_def.source.as_ref().map(|source| {
                let expanded = oxo_flow_core::executor::checkpoint::expand_config_in_path(
                    source,
                    &wildcard_values,
                );
                let path = std::path::Path::new(&expanded);
                if path.is_absolute() {
                    path.to_path_buf()
                } else {
                    workflow_dir.join(path)
                }
            });
            let current_fp = oxo_flow_core::config_impact::reference_fingerprint(
                ref_def,
                &config.config,
                resolved_source.as_deref(),
            );
            let stored: Option<RefRecord> = ref_state.get(&ref_def.name).cloned();
            // Content signature of the CURRENT source (size + full content
            // hash for small files). Equality with the stored signature
            // means the source is a byte-identical copy at a new path — the
            // fingerprint differs only in the path string, so a rebuild
            // would reproduce identical output (issue #142 follow-up).
            let current_sig = resolved_source.as_deref().and_then(source_content_sig);

            // Decide whether the artifact must be (re)built. The freshness
            // check comes FIRST so an mtime-only touch reports the accurate
            // reason; a fingerprint mismatch then covers definition edits,
            // referenced-config changes, and content changes the freshness
            // check cannot see (same-path, timestamp-preserving rewrites).
            let source_newer = resolved_source.as_deref().is_some_and(|p| {
                p.exists() && oxo_flow_core::executor::checkpoint::file_is_newer(p, &output_full)
            });
            let rebuild_reason = if !output_full.exists() {
                Some("output missing")
            } else if stored.as_ref().is_some_and(|r| r.fingerprint.is_empty()) {
                // Legacy entry: adopt the current fingerprint without
                // rebuilding — and PERSIST the adoption. An in-memory-only
                // adopt would leave the entry "" forever: every future run
                // would silently re-adopt the CURRENT source state and the
                // content guard could never engage.
                ref_state.insert(
                    ref_def.name.clone(),
                    RefRecord {
                        fingerprint: current_fp.clone(),
                        content_sig: current_sig.clone(),
                        source_path: ref_def.source.clone(),
                    },
                );
                if let Some(parent) = ref_checkpoint.parent() {
                    let _ = std::fs::create_dir_all(parent);
                }
                let _ = std::fs::write(
                    &ref_checkpoint,
                    serde_json::to_string(&ref_state).unwrap_or_default(),
                );
                None
            } else if source_newer {
                Some("source is newer than output")
            } else if stored.as_ref().is_some_and(|r| r.fingerprint != current_fp) {
                // The fingerprint changed. One benign cause: a pure PATH
                // migration — the same byte content now lives at a new
                // location (live: --arg index=/new/path where /new/path is
                // a content copy). PROOF of purity: re-fingerprinting the
                // definition with the STORED source path must reproduce the
                // stored fingerprint (so the build/output/config did NOT
                // change), AND the content signatures must match with a
                // full hash component (size-only is too weak).
                let pure_path_migration = (|| {
                    let rec = stored.as_ref()?;
                    let old_path_str = rec.source_path.as_deref()?;
                    let old_def = oxo_flow_core::config::ReferenceDef {
                        source: Some(old_path_str.to_string()),
                        ..ref_def.clone()
                    };
                    let fp_at_old_path = oxo_flow_core::config_impact::reference_fingerprint(
                        &old_def,
                        &config.config,
                        Some(ref_workdir.join(old_path_str)).as_deref(),
                    );
                    if fp_at_old_path != rec.fingerprint {
                        return None; // definition or config really changed
                    }
                    let cur = current_sig.as_deref()?;
                    if !cur.contains("|hash:") {
                        return None; // no strong hash — never exempt
                    }
                    (rec.content_sig.as_deref() == Some(cur)).then_some(())
                })()
                .is_some();
                if pure_path_migration {
                    tracing::info!(
                        reference = %ref_def.name,
                        "reference source moved to a new path with identical content — keeping the existing artifact"
                    );
                    ref_state.insert(
                        ref_def.name.clone(),
                        RefRecord {
                            fingerprint: current_fp.clone(),
                            content_sig: current_sig.clone(),
                            source_path: ref_def.source.clone(),
                        },
                    );
                    if let Some(parent) = ref_checkpoint.parent() {
                        let _ = std::fs::create_dir_all(parent);
                    }
                    let _ = std::fs::write(
                        &ref_checkpoint,
                        serde_json::to_string(&ref_state).unwrap_or_default(),
                    );
                    None
                } else {
                    Some("definition, source content, or referenced config changed")
                }
            } else {
                None
            };

            if let Some(reason) = rebuild_reason {
                // Issue #427: the synthetic `{input}` is the reference
                // source — a workflow-shipped file. When the run workdir
                // differs from the workflow dir, render it workflow-absolute
                // so the build shell (cwd = ref_workdir) can read it.
                let source_rendered: Option<String> = ref_def.source.as_ref().map(|source| {
                    let expanded = oxo_flow_core::executor::checkpoint::expand_config_in_path(
                        source,
                        &wildcard_values,
                    );
                    if std::path::Path::new(&expanded).is_absolute() || workflow_dir == *ref_workdir
                    {
                        expanded
                    } else {
                        workflow_dir.join(expanded).to_string_lossy().into_owned()
                    }
                });
                let synthetic_rule = oxo_flow_core::rule::Rule {
                    name: format!("ref:{}", ref_def.name),
                    // `{input}` is the renderer's alias for the reference
                    // source (documented in config_impact::reference_fingerprint).
                    input: source_rendered
                        .as_deref()
                        .map(|s| vec![s.to_string()].into())
                        .unwrap_or_default(),
                    output: vec![output_path.clone()].into(),
                    ..Default::default()
                };
                let mut build_cmd = oxo_flow_core::executor::process::render_shell_command(
                    &ref_def.build,
                    &synthetic_rule,
                    &wildcard_values,
                    oxo_flow_core::scheduler::detect_system_limits(),
                );
                // `{source}` is the builder-template spelling of the same
                // thing; render it too (live evidence: tcasia's STAR
                // genomeGenerate died with 'could not open genomeFastaFile:
                // {source}' — the placeholder was never substituted). The
                // path is shell-quoted — a bare splice breaks reference
                // builds whose source path contains spaces (issue #136
                // tier-2 audit).
                if let Some(expanded) = source_rendered.as_deref() {
                    build_cmd = substitute_source_placeholder(&build_cmd, expanded);
                }
                // References take the same shell prelude as rules (issue #92),
                // before the environment wrapper resolves.
                build_cmd = config.defaults.apply_shell_prelude(&build_cmd);
                // References that need workflow tools (bowtie2-build, STAR
                // genomeGenerate, …) declare an `environment` — same spec as
                // `[rules.environment]`. The env is created on first use and
                // the build command runs inside it; without one, the build
                // runs in the bare system shell as before.
                if let Some(ref env) = ref_def.environment
                    && !env.is_empty()
                {
                    let resolver = oxo_flow_core::environment::EnvironmentResolver::with_cache_dir(
                        &env_cache_dir,
                    );
                    // Issue #427: file-backed env specs (`conda = "envs/x.yaml"`)
                    // resolve against the workflow directory — the build shell's
                    // cwd is the ref workdir, so a workflow-relative spec would
                    // not resolve from there. One resolved spec feeds cache_key,
                    // setup, and wrap so all three agree on the same environment.
                    let resolved_env = env.resolve_relative_paths(Some(&workflow_dir));
                    let key = resolver.cache_key(&resolved_env);
                    if !resolver.cache_is_ready(&key).await {
                        let setup = resolver.setup_command(&resolved_env)?;
                        let out = tokio::process::Command::new("sh")
                            .arg("-c")
                            .arg(&setup)
                            .current_dir(ref_workdir)
                            .output()
                            .await;
                        match out {
                            Ok(o) if o.status.success() => {
                                resolver.cache_mark_ready(&key).await;
                            }
                            Ok(o) => {
                                let stderr = String::from_utf8_lossy(&o.stderr).into_owned();
                                // Pre-execution abort — the summary still
                                // reports the failed run (issue #142 H6).
                                json_summary.emit("failed", &RunCounts::default(), vec![]);
                                return Err(anyhow::anyhow!(
                                    "failed to set up the environment for reference '{}': {}",
                                    ref_def.name,
                                    stderr.trim()
                                ));
                            }
                            Err(e) => {
                                json_summary.emit("failed", &RunCounts::default(), vec![]);
                                return Err(anyhow::anyhow!(
                                    "failed to run the environment setup for reference '{}': {e}",
                                    ref_def.name
                                ));
                            }
                        }
                    }
                    build_cmd =
                        resolver.wrap_command(&build_cmd, &resolved_env, None, ref_workdir)?;
                }
                eprintln!(
                    "  {} Building {}: {} ({})",
                    "⚙".cyan().bold(),
                    ref_def.name,
                    ref_def.description.as_deref().unwrap_or(&ref_def.output),
                    reason
                );
                // bash first, sh fallback — mirrors the rule executor
                // (spawn_rule_shell): `set -o pipefail` in a shell prelude
                // is invalid under dash, which /bin/sh is on Debian-family
                // systems (review finding on issue #92).
                let spawn_ref_build = |shell: &str| {
                    std::process::Command::new(shell)
                        .arg("-c")
                        .arg(&build_cmd)
                        .current_dir(ref_workdir)
                        .status()
                };
                let status = match spawn_ref_build("bash") {
                    Err(e) if e.kind() == std::io::ErrorKind::NotFound => spawn_ref_build("sh"),
                    other => other,
                };
                match status {
                    Ok(s) if s.success() => {
                        // Store the POST-build fingerprint: the decision
                        // fingerprint above was computed before the build
                        // ran, and a build that creates or rewrites its own
                        // source (download-then-index) changes the source
                        // state DURING the build — storing the pre-build
                        // state would mismatch on every subsequent run and
                        // rebuild forever.
                        let post_fp = oxo_flow_core::config_impact::reference_fingerprint(
                            ref_def,
                            &config.config,
                            resolved_source.as_deref(),
                        );
                        ref_state.insert(
                            ref_def.name.clone(),
                            RefRecord {
                                fingerprint: post_fp,
                                content_sig: resolved_source
                                    .as_deref()
                                    .and_then(source_content_sig),
                                source_path: ref_def.source.clone(),
                            },
                        );
                        rebuilt_outputs.push(output_path.clone());
                        if let Some(parent) = ref_checkpoint.parent() {
                            let _ = std::fs::create_dir_all(parent);
                        }
                        let _ = std::fs::write(
                            &ref_checkpoint,
                            serde_json::to_string(&ref_state).unwrap_or_default(),
                        );
                        eprintln!("    {} {} built", "✓".green(), ref_def.name);
                    }
                    Ok(s) => {
                        anyhow::bail!(
                            "failed to build reference '{}' (exit {}). Command: {}",
                            ref_def.name,
                            s.code().unwrap_or(-1),
                            oxo_flow_core::executor::process::mask_sensitive(
                                &build_cmd,
                                &sensitive_values
                            )
                        );
                    }
                    Err(e) => {
                        anyhow::bail!("failed to run build command for '{}': {}", ref_def.name, e);
                    }
                }
            } else if stored.is_none() {
                // Output exists but not tracked — adopt as built.
                ref_state.insert(
                    ref_def.name.clone(),
                    RefRecord {
                        fingerprint: current_fp,
                        content_sig: current_sig,
                        source_path: ref_def.source.clone(),
                    },
                );
                if let Some(parent) = ref_checkpoint.parent() {
                    let _ = std::fs::create_dir_all(parent);
                }
                let _ = std::fs::write(
                    &ref_checkpoint,
                    serde_json::to_string(&ref_state).unwrap_or_default(),
                );
            }
        }

        // A rebuilt reference invalidates the rules that consume it through
        // declared inputs (plus their DAG downstream). The checkpoint
        // pre-process would otherwise skip them as "already completed"
        // without consulting input freshness; after invalidation the
        // executor's mtime gate sees the rebuilt artifact and re-runs them.
        // Shell-only reads of a reference cannot be tracked — declare
        // reference artifacts in a rule's `input` list.
        if !rebuilt_outputs.is_empty() {
            let mut consumers: HashSet<String> = HashSet::new();
            for rule in &config.rules {
                let inputs: Vec<String> = rule
                    .input
                    .to_vec()
                    .into_iter()
                    .map(|i| {
                        oxo_flow_core::executor::checkpoint::expand_config_in_path(
                            &i,
                            &wildcard_values,
                        )
                    })
                    .collect();
                if inputs.iter().any(|input| {
                    !input.contains('{') && rebuilt_outputs.iter().any(|rebuilt| rebuilt == input)
                }) {
                    consumers.insert(rule.name.clone());
                }
            }
            if !consumers.is_empty() {
                let mut ck = checkpoint.lock().await;
                let invalidated = crate::commands::run_preview::invalidate_with_downstream(
                    &mut ck, &dag, &consumers,
                );
                if let Err(e) = ck.save_to_file(&checkpoint_path) {
                    tracing::warn!(error = %e, "failed to save checkpoint after reference rebuild");
                }
                eprintln!(
                    "  {} reference rebuild invalidated {} rule(s): {}",
                    "↻".yellow(),
                    invalidated.len(),
                    invalidated.join(", ")
                );
            }
        }
    }

    // Event-driven fine-grained scheduler.
    //
    // Instead of group barriers (wait for ALL rules at depth N before starting
    // ANY at depth N+1), each rule is submitted as soon as all its individual
    // dependencies complete.  SchedulerState tracks per-rule status and
    // re-evaluates readiness after every completion event.  This eliminates the
    // "tail latency" problem where a single slow rule in a parallel group delays
    // downstream rules that only depend on already-finished fast rules.
    //
    // Concurrency is still bounded by -j (tokio Semaphore).  ResourcePool
    // (threads/memory/groups) is checked per-rule inside execute_rule_with_config.

    // ── Checkpoint re-entry replay (issue #78 P3) ────────────────────────
    // Re-apply recorded re-entries whose checkpoint rule still stands, then
    // rebuild the DAG so a resume reconstructs the same static plan a fresh
    // run would. Invalidated checkpoint rules are revoked: their samples
    // leave the plan until the rule re-runs and re-records.
    {
        let ck = checkpoint.lock().await;
        if !ck.reentries.is_empty() {
            let valid: std::collections::HashSet<String> = config
                .rules
                .iter()
                .filter(|rule| {
                    ck.is_completed(&rule.name)
                        && !rerun
                        && (tombstone_keep.contains(&rule.name)
                            || crate::commands::run_preview::rule_outputs_exist(
                                rule,
                                workdir_actual.as_ref(),
                                &wildcard_values,
                            ))
                })
                .map(|r| r.name.clone())
                .collect();
            let replayed =
                oxo_flow_core::reentry::replay_valid_reentries(&mut config, &ck.reentries, &valid)?;
            tracing::info!(count = replayed.len(), "replayed checkpoint re-entries");
            dag = WorkflowDag::from_rules_with_config(
                &config.rules,
                &config_placeholder_values(&config.config),
            )
            .context("failed to rebuild workflow DAG after re-entry replay")?;
            order = if target.is_empty() {
                dag.execution_order()?
            } else {
                let target_refs: Vec<&str> = target.iter().map(String::as_str).collect();
                let (filtered, skipped_targets) = dag
                    .execution_order_for_targets_skipping_with_source_check(
                        &target_refs,
                        &when_false_rules,
                        &|path| {
                            run_workdir_check(
                                Some(workdir.as_ref().unwrap_or(&workdir_default)),
                                path,
                            )
                        },
                    )
                    .with_context(|| "failed to resolve target rules")?;
                for skipped in &skipped_targets {
                    print_target_skipped_note(skipped, &when_false_rules, Some(&run_log));
                }
                filtered
            };
        }
    }

    // ── Runtime-discovered output_pattern replay (issue #227 item 5) ──────
    // Persisted producer domains replay the deferred-consumer fan-out on
    // resume: consumers instantiate from the recorded domain WITHOUT
    // re-running the producer (its instances are already-completed, handled
    // by the skip preprocessing below). A fresh run replays an empty record
    // and adds nothing; `--rerun` deliberately skips it so the producer
    // re-discovers the domain live.
    // Gate (issue #296 follow-up): instantiate only when EVERY producer
    // instance is already completed. A producer that still has to re-run
    // keeps its consumers pending — the runtime discovery pass then
    // instantiates them from the final domain union, so the final consumer
    // set is the union of all per-instance slices (a partial replay would
    // drain the pending set and silently drop the later slices).
    {
        let ck = checkpoint.lock().await;
        if !ck.output_pattern_domains.is_empty() && !rerun {
            config.discovered_output_patterns = ck.output_pattern_domains.clone();
            let all_producers_complete = ck.output_pattern_domains.keys().all(|template| {
                config.rules.iter().all(|r| {
                    r.output_pattern.is_none()
                        || (config.expansion_templates.get(&r.name).map(String::as_str)
                            != Some(template.as_str())
                            && r.name != *template)
                        || ck.completed_rules.contains(&r.name)
                })
            });
            let replayed = if all_producers_complete {
                config.expand_output_pattern_consumers()?
            } else {
                tracing::info!(
                    "deferred output_pattern consumers stay pending: a producer instance \
                     is not completed yet — the runtime discovery pass will instantiate them"
                );
                Vec::new()
            };
            tracing::info!(
                count = replayed.len(),
                "replayed runtime-discovered output_pattern domains"
            );
            if !replayed.is_empty() {
                dag = WorkflowDag::from_rules_with_config(
                    &config.rules,
                    &config_placeholder_values(&config.config),
                )
                .context("failed to rebuild workflow DAG after output_pattern replay")?;
                order = if target.is_empty() {
                    dag.execution_order()?
                } else {
                    let target_refs: Vec<&str> = target.iter().map(String::as_str).collect();
                    let (filtered, skipped_targets) = dag
                        .execution_order_for_targets_skipping_with_source_check(
                            &target_refs,
                            &when_false_rules,
                            &|path| {
                                run_workdir_check(
                                    Some(workdir.as_ref().unwrap_or(&workdir_default)),
                                    path,
                                )
                            },
                        )
                        .with_context(|| "failed to resolve target rules")?;
                    for skipped in &skipped_targets {
                        print_target_skipped_note(skipped, &when_false_rules, Some(&run_log));
                    }
                    filtered
                };
            }
        }
    }

    let rule_names: Vec<String> = order.to_vec();
    let rule_name_refs: Vec<&str> = rule_names.iter().map(String::as_str).collect();
    let mut sched = oxo_flow_core::scheduler::SchedulerState::new(&rule_name_refs);
    let mut order_set: std::collections::HashSet<String> = order.iter().cloned().collect();
    let run_started = std::time::Instant::now();
    let semaphore = Arc::new(tokio::sync::Semaphore::new(jobs.max(1)));
    let mut join_set = tokio::task::JoinSet::new();
    let mut submitted: std::collections::HashSet<String> = std::collections::HashSet::new();
    // Task id → rule name for panic attribution (issue #136 fix 2): a
    // panicked task leaves the JoinSet without its payload, so the id is
    // the only way to learn which rule died and release its scheduler slot.
    let mut task_rule: std::collections::HashMap<tokio::task::Id, String> =
        std::collections::HashMap::new();

    // Pre-process checkpoint-completed rules so they are never re-submitted.
    {
        let ck = checkpoint.lock().await;
        for rule_name in &order {
            // A rule still marked running (issue #685) is never "already
            // completed": the previous run died with it mid-flight, its
            // outputs may be half-written, and treating it as done would
            // silently skip real work.
            if ck.is_running(rule_name) {
                continue;
            }
            if ck.is_completed(rule_name)
                && !rerun
                && order_set.contains(rule_name.as_str())
                && !submitted.contains(rule_name.as_str())
            {
                // Shared with dry-run's read-only preview (issue #66).
                // Tombstoned rules count as up to date while no dependent
                // needs their outputs.
                let outputs_ok = tombstone_keep.contains(rule_name)
                    || config
                        .get_rule(rule_name)
                        .map(|rule| {
                            crate::commands::run_preview::rule_outputs_exist(
                                rule,
                                workdir_actual.as_ref(),
                                &wildcard_values,
                            )
                        })
                        .unwrap_or(true);
                if outputs_ok {
                    submitted.insert(rule_name.clone());
                    sched.mark_completed(oxo_flow_core::executor::JobRecord {
                        signal: None,
                        rule: rule_name.clone(),
                        status: oxo_flow_core::executor::JobStatus::Success,
                        enqueued_at: None,
                        started_at: None,
                        finished_at: None,
                        exit_code: Some(0),
                        stdout: None,
                        stderr: None,
                        command: None,
                        retries: 0,
                        skip_reason: None,
                        max_rss_mb: None,
                        cpu_seconds: None,
                        caption: config.get_rule(rule_name).and_then(|r| {
                            oxo_flow_core::executor::process::rule_report_caption(
                                r,
                                workdir_actual.as_ref(),
                            )
                        }),
                    });
                    skipped_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                    if !is_tty {
                        diagnostic_narrate(
                            format_args!("  {} {} (already completed)", "⊝".dimmed(), rule_name),
                            Some(&run_log),
                        );
                    }
                    progress.inc(1);
                }
            }
        }
    }

    // Ready-list ordering (age_ready_list below, called every dispatch
    // round): the scheduler's ready rules are filtered to this run's order
    // set minus already-submitted rules, then sorted by EFFECTIVE priority
    // — declared priority plus the rounds a rule has waited ready-but-not-
    // submitted (the aging counter of issue #123, cluster-side issue #134)
    // — descending, ties broken by name for determinism. Priorities are
    // captured ONCE up front: the config mutates mid-run across checkpoint
    // re-entry (issue #78 P3), and re-reading priorities per round would
    // drift the ordering.
    let priority_map: std::collections::HashMap<String, i32> = config
        .rules
        .iter()
        .map(|r| (r.name.clone(), r.priority))
        .collect();
    // Fair-dispatch aging (issue #123): every dispatch round a rule spends
    // ready-but-not-submitted adds AGING_STEP to its effective priority, so
    // high-priority rules that keep failing/re-occupying slots cannot starve
    // their lower-priority producers forever (live: auto-sra dumps at p10
    // starved behind merges at p20 sharing the limit_merge group). Aging
    // only matters under contention — with free slots everything submits
    // immediately and the relative order is unchanged.
    const AGING_STEP: i32 = 1;
    let mut waited_rounds: std::collections::HashMap<String, i32> = Default::default();

    // Spawn watchdog (issue #685): wall-clock time each rule was marked
    // running, used to detect rules stuck in Running without ever spawning
    // a child process (lost wakeup wedge). Entries are removed in the
    // common completion funnel below; cleanup on every terminal path.
    let mut running_since: std::collections::HashMap<String, std::time::Instant> =
        Default::default();
    // Rules the watchdog has already reported — one report per rule per run
    // (WARN for benign resource-pool queues, ERROR for genuine no-child
    // stragglers; issue #847).
    let mut watchdog_reported: std::collections::HashSet<String> = std::collections::HashSet::new();
    let watchdog_threshold = spawn_watchdog_threshold();
    let mut watchdog_tick =
        tokio::time::interval(std::time::Duration::from_secs(SPAWN_WATCHDOG_TICK_SECS));

    // #843 disk-pressure monitor: level thresholds resolved once from the
    // validated config (None = feature off — zero monitoring overhead).
    let disk_thresholds = config.engine.as_ref().and_then(|e| e.disk_thresholds());
    let mut disk_monitor = oxo_flow_core::scheduler::DiskPressureMonitor::new(
        disk_thresholds.map_or(0, |t| t.0),
        disk_thresholds.map_or(0, |t| t.1),
    );
    let mut disk_tick = tokio::time::interval(std::time::Duration::from_secs(disk_tick_secs()));
    // Grace rounds accumulated while parked under Hold with nothing in
    // flight; disk recovery resets it.
    let mut disk_hold_rounds: u32 = 0;
    let disk_hold_abort_rounds = disk_hold_abort_rounds();

    // ---- main event loop -------------------------------------------------

    // Termination signals (audit C14): SIGINT/SIGTERM/SIGHUP stop the run
    // through `terminate_on_signal` instead of killing the process outright.
    // A watch channel keeps the wait re-armable across loop iterations, the
    // way a oneshot receiver could only be awaited once.
    let (signal_tx, mut signal_rx) = tokio::sync::watch::channel(None::<InterruptSignal>);
    tokio::spawn(async move {
        if let Some(signal) = wait_for_termination_signal().await
            && signal_tx.send(Some(signal)).is_err()
        {
            // The run loop is gone but the handlers are still installed, so
            // a late signal must still end the process.
            std::process::exit(signal.exit_code());
        }
    });

    loop {
        // #843 disk-pressure gate — measure BEFORE any dispatch decision so
        // a floor crossing narrates once (crossing dedup inside the monitor)
        // and enforces its policy this round.
        if disk_thresholds.is_some() {
            let mut n = |args: std::fmt::Arguments<'_>| {
                diagnostic_narrate(args, Some(&run_log));
            };
            let level = disk_monitor.observe(workdir_actual.as_ref(), &mut n);
            let hold = level >= oxo_flow_core::scheduler::DiskPressureLevel::Hold;

            // Reclaim BEFORE the Hold-park decision (review F1): a run that
            // starts below the floor — or falls through to Hold within a
            // single tick — must still get a reclaim pass before parking,
            // or the "nothing left to reclaim" abort below is untrue and
            // reclaimable intermediates are never released.
            if level >= oxo_flow_core::scheduler::DiskPressureLevel::Reclaim {
                // Completed temporary rules whose dependents are all done
                // no longer need their outputs.
                let wildcards = wildcard_values.clone();
                let candidates = {
                    let ck = checkpoint.lock().await;
                    oxo_flow_core::scheduler::reclaim_candidates(
                        &config.rules,
                        &dag,
                        &ck,
                        workdir_actual.as_ref(),
                        &wildcards,
                        &config.expansion_templates,
                    )
                };
                for candidate in candidates {
                    if let Err(reclaim_err) = reclaim_rule_outputs(
                        &candidate,
                        workdir_actual.as_ref(),
                        &checkpoint,
                        &checkpoint_path,
                        &run_log,
                    )
                    .await
                    {
                        diagnostic_narrate(
                            format_args!(
                                "  {} reclaim of '{}' failed: {reclaim_err}",
                                "!".yellow(),
                                candidate.rule
                            ),
                            Some(&run_log),
                        );
                    }
                }
            }
            if hold && !sched.is_complete() && sched.running_count() == 0 && join_set.is_empty() {
                // Nothing in flight and nothing dispatchable: park for a
                // tick (external space may free up) and re-measure. After
                // disk_hold_abort_rounds consecutive parked rounds below the
                // floor, abort budget-abort style — reclaim has already fired
                // above, before the park decision; if the disk is still full,
                // no progress is possible.
                if disk_hold_rounds >= disk_hold_abort_rounds {
                    {
                        let ck = checkpoint.lock().await;
                        ck.save_to_file_async(&checkpoint_path).await.ok();
                    }
                    progress.finish_and_clear();
                    json_summary.emit("failed", &RunCounts::default(), vec![]);
                    anyhow::bail!(
                        "disk pressure: {}MB free is below min_free_disk ({}MB) with no rules \
                         in flight and nothing left to reclaim — the run cannot make progress. \
                         Free space manually (delete intermediates or run `oxo-flow clean`), \
                         then resume from the checkpoint, or lower [engine] min_free_disk.",
                        oxo_flow_core::scheduler::check_available_disk_mb(workdir_actual.as_ref())
                            .unwrap_or(0),
                        disk_thresholds.map_or(0, |t| t.0),
                    );
                }
                disk_hold_rounds += 1;
                tokio::time::sleep(std::time::Duration::from_secs(disk_tick_secs())).await;
                continue;
            }
            // Any pass that is not a parked hold clears the grace counter.
            disk_hold_rounds = 0;
        }

        // Check deadlock before each scheduling round.
        if !sched.is_complete() && sched.running_count() == 0 && join_set.is_empty() {
            sched.check_deadlock(&dag)?;
        }

        // Submit every rule whose dependencies are now satisfied — capped to
        // the free -j slots so the semaphore queue never builds, and the
        // ready list (aged by age_ready_list) decides dispatch order fairly.
        let ready = age_ready_list(
            sched.ready_rules(&dag)?,
            &order_set,
            &submitted,
            &priority_map,
            &waited_rounds,
        );
        // jobs was clamped to >= 1 at run_command entry, so a raw 0 can
        // never silently suppress all submissions (issue #136 fix 1).
        // Under #843 Hold, new dispatches are withheld entirely — in-flight
        // rules continue to completion.
        let available = if disk_thresholds.is_some()
            && disk_monitor.last_level() >= oxo_flow_core::scheduler::DiskPressureLevel::Hold
        {
            0
        } else {
            jobs.saturating_sub(sched.running_count())
        };
        let to_submit: Vec<String> = ready.iter().take(available).cloned().collect();
        for name in ready.iter().skip(available) {
            *waited_rounds.entry(name.clone()).or_insert(0) += AGING_STEP;
        }
        for rule_name in &to_submit {
            // Skip rules blocked by a failed — or no-output-skipped — upstream
            // dependency. Optional rules are exempt: they evaluate their own
            // inputs and skip gracefully when they're absent (issue #200).
            let rule_is_optional = config
                .get_rule(rule_name)
                .is_some_and(|r| r.optional.is_optional());
            let blocked_by = if rule_is_optional {
                None
            } else {
                let frs = failed_rules_set.lock().await;
                let sno = skipped_no_output.lock().await;
                dag.dependencies(rule_name).ok().and_then(|deps| {
                    deps.into_iter()
                        .find(|d| frs.contains(d.as_str()) || sno.contains(d.as_str()))
                })
            };
            if let Some(dep) = blocked_by {
                skipped_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                // Propagate transitively: this rule produced nothing, so its
                // own dependents are blocked the same way.
                let dep_skipped = {
                    let sno = skipped_no_output.lock().await;
                    sno.contains(dep.as_str())
                };
                if dep_skipped {
                    let mut sno = skipped_no_output.lock().await;
                    sno.insert(rule_name.clone());
                } else {
                    let mut frs = failed_rules_set.lock().await;
                    frs.insert(rule_name.clone());
                }
                blocked.lock().await.push((rule_name.clone(), dep));
                submitted.insert(rule_name.clone());
                sched.mark_completed(oxo_flow_core::executor::JobRecord {
                    signal: None,
                    rule: rule_name.clone(),
                    status: oxo_flow_core::executor::JobStatus::Skipped,
                    enqueued_at: None,
                    started_at: None,
                    finished_at: None,
                    exit_code: None,
                    stdout: None,
                    stderr: None,
                    command: None,
                    retries: 0,
                    skip_reason: Some(
                        if dep_skipped {
                            "blocked by skipped upstream dependency"
                        } else {
                            "blocked by failed upstream dependency"
                        }
                        .into(),
                    ),
                    max_rss_mb: None,
                    cpu_seconds: None,
                    caption: config.get_rule(rule_name).and_then(|r| {
                        oxo_flow_core::executor::process::rule_report_caption(
                            r,
                            workdir_actual.as_ref(),
                        )
                    }),
                });
                if !is_tty {
                    eprintln!(
                        "  {} {} (blocked by {} dependency)",
                        "⊘".yellow(),
                        rule_name,
                        if dep_skipped { "skipped" } else { "failed" }
                    );
                }
                progress.inc(1);
                continue;
            }

            submitted.insert(rule_name.clone());
            sched.mark_running(rule_name);
            // Spawn-time persistence (issue #685, ask 3): the checkpoint's
            // running set is written when the rule actually enters flight,
            // so a crashed run leaves an honest record for the next start.
            {
                let mut ck = checkpoint.lock().await;
                ck.mark_running(rule_name);
                if let Err(e) = ck.save_to_file_async(&checkpoint_path).await {
                    tracing::warn!("Failed to save checkpoint (running mark): {e}");
                }
            }
            running_since.insert(rule_name.clone(), std::time::Instant::now());

            // ---- spawn task (identical logic to pre-scheduler version) -----
            let Some(rule) = config.get_rule(rule_name).cloned() else {
                anyhow::bail!("rule '{rule_name}' not found in workflow");
            };
            // Per-instance wildcard bindings for the substitution floor
            // (issue #276): sample names / [[values]] / pair metadata are
            // baked into the instance's shell text at expansion time and
            // never travel in the `config.*` map — hand them to the
            // executor explicitly. Empty for non-fan-out rules.
            let instance_bindings = config.instance_bindings(rule_name);
            let rule_name = rule_name.clone();
            let executor = executor.clone();
            let checkpoint = checkpoint.clone();
            let checkpoint_path = checkpoint_path.clone();
            let failed_rules_set = failed_rules_set.clone();
            let skipped_no_output = skipped_no_output.clone();
            let failures = failures.clone();
            let success_count = success_count.clone();
            let fail_count = fail_count.clone();
            let non_required_fail_count = non_required_fail_count.clone();
            let sensitive_values = sensitive_values.clone();
            let skipped_count = skipped_count.clone();
            let wildcard_values = wildcard_values.clone();
            let provenance_recorded = provenance_recorded.clone();
            let workdir_actual = workdir_actual.clone();
            let semaphore = semaphore.clone();
            let progress = progress.clone();

            progress.set_message(format!("executing {}", rule_name));
            if !is_tty {
                progress_narrate(
                    format_args!("  {} {}", "Running:".bold().cyan(), rule_name),
                    &run_log,
                );
            }
            emit_execution_event(oxo_flow_core::executor::ExecutionEvent::RuleStarted {
                rule: rule_name.clone(),
                command: None,
            });

            let typed_config = config.config.clone();
            // Resolve the rule's report caption BEFORE the spawn: the closure
            // must not capture `config` (owned by this loop), only the already
            // resolved value (issue #281).
            let rule_caption = config.get_rule(&rule_name).and_then(|r| {
                oxo_flow_core::executor::process::rule_report_caption(r, workdir_actual.as_ref())
            });
            // When-gate verdict BEFORE the spawn (issue #757 part 3): the
            // input snapshot used to run before when-evaluation, so a
            // gated-off rule emitted spurious "cannot stat input" warnings
            // for files absent by design (and an empty-config path even
            // rendered as a double slash). Same evaluator + raw-config
            // `{` guard as readiness/deep-check.
            let when_gated_off = rule
                .when
                .as_deref()
                .is_some_and(|when| {
                    !when.contains('{')
                        && !oxo_flow_core::executor::process::evaluate_condition_with_wildcards_and_base_dir(
                            when,
                            &config.config,
                            &wildcard_values,
                            Some(workdir_actual.as_ref().as_path()),
                        )
                });
            let instance_bindings = instance_bindings.clone();
            // Register the task name BEFORE the spawn: the closure moves
            // `rule_name` in, so the id→name map needs its own clone.
            let task_rule_name = rule_name.clone();
            let task_run_log = Arc::clone(&run_log);
            let handle = join_set.spawn(async move {
                let _permit = semaphore.acquire().await;

                // Snapshot the input file set BEFORE execution (issue #72):
                // the manifest records what the rule is about to consume, so
                // files added mid-run are not silently baked into the
                // recorded baseline.
                // Remote inputs record (scheme, key, size, etag) when a cloud
                // backend is registered (issue #78 P2); without one they
                // degrade gracefully (warning + entry skipped).
                // A failed snapshot is narrated, not swallowed (issue #633):
                // with no manifest recorded, every later run re-invalidates
                // this rule — the least the user can expect is to be told.
                let input_manifest = if when_gated_off {
                    // Gated-off rule: it will be skipped this run — its
                    // inputs are absent by design, and snapshotting them
                    // only emits spurious warnings (#757 part 3).
                    None
                } else {
                    match oxo_flow_core::executor::checkpoint::snapshot_input_manifest_async(
                        &rule,
                        workdir_actual.as_ref(),
                        &wildcard_values,
                        &crate::commands::run_preview::storage_resolver(),
                    )
                    .await
                    {
                        Ok(manifest) => manifest,
                        Err(e) => {
                            let msg = format!("could not snapshot inputs: {e}");
                            diagnostic_narrate(
                                format_args!(
                                    "  {} could not snapshot inputs of rule '{}' — {} (no manifest recorded; later runs may re-execute it)",
                                    "⚠".yellow(),
                                    rule_name,
                                    oxo_flow_core::executor::process::mask_sensitive(
                                        &msg,
                                        &sensitive_values,
                                    )
                                ),
                                Some(&task_run_log),
                            );
                            None
                        }
                    }
                };

                let result = executor
                    .execute_rule_with_bindings(&rule, &wildcard_values, &typed_config, &instance_bindings)
                    .await;

                match result {
                    Ok(mut record) => {
                        let duration = record
                            .finished_at
                            .and_then(|f| record.started_at.map(|s| f.signed_duration_since(s)))
                            .map(|d| d.num_milliseconds() as f64 / 1000.0)
                            .unwrap_or(0.0);

                        if record.status == oxo_flow_core::executor::JobStatus::Success {
                            success_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                            emit_execution_event(oxo_flow_core::executor::ExecutionEvent::RuleCompleted {
                                rule: rule_name.clone(),
                                status: oxo_flow_core::executor::JobStatus::Success,
                                duration_ms: (duration * 1000.0) as u64,
                            });
                            if !is_tty {
                                if record.skip_reason.as_deref()
                                    == Some("restored from content cache")
                                {
                                    progress_narrate(
                                        format_args!(
                                            "  {} {} (restored from content cache)",
                                            "↺".cyan(),
                                            rule_name,
                                        ),
                                        &task_run_log,
                                    );
                                } else {
                                    progress_narrate(
                                        format_args!("  {} {} ({:.1}s)", "✓".green(), rule_name, duration),
                                        &task_run_log,
                                    );
                                }
                            }
                            // Issue #832: a succeeded rule whose declared
                            // `log` path stayed empty (0-byte file or empty
                            // dir) while BOTH captured tails are empty too
                            // almost certainly self-logs somewhere the
                            // workflow did not declare (dnbc4tools writes
                            // `scrna/count/Pt01/logs/<date>.txt`). One Info
                            // hint, and only when all three are empty — a
                            // quiet-but-correct tool with content in any of
                            // them stays silent. The hint never gates the
                            // rule; captured tails are persisted in the
                            // checkpoint job records regardless (issue #692).
                            if let Some(log_pattern) = &rule.log {
                                let expanded = oxo_flow_core::executor::checkpoint::expand_config_in_path(
                                    log_pattern,
                                    &wildcard_values,
                                );
                                if !expanded.contains('{') {
                                    let log_path = workdir_actual.as_ref().join(&expanded);
                                    let log_empty = if log_path.is_dir() {
                                        std::fs::read_dir(&log_path)
                                            .map(|mut entries| entries.next().is_none())
                                            .unwrap_or(false)
                                    } else if log_path.is_file() {
                                        std::fs::metadata(&log_path)
                                            .map(|m| m.len() == 0)
                                            .unwrap_or(false)
                                    } else {
                                        // Never created: the rule's `2> {log}`
                                        // redirect never ran — a different
                                        // signal than "declared log stayed
                                        // empty", so stay silent here.
                                        false
                                    };
                                    let tails_empty = record
                                        .stdout
                                        .as_deref()
                                        .map(str::trim)
                                        .unwrap_or("")
                                        .is_empty()
                                        && record
                                            .stderr
                                            .as_deref()
                                            .map(str::trim)
                                            .unwrap_or("")
                                            .is_empty();
                                    if log_empty && tails_empty {
                                        diagnostic_narrate(
                                            format_args!(
                                                "  {} rule '{}' succeeded but its declared log '{}' is empty — the tool likely writes its own log elsewhere (declare that path in `log`, or check the tool's default log location); captured stdout/stderr tails are persisted in the checkpoint job records regardless",
                                                "ℹ".dimmed(),
                                                rule_name,
                                                log_path.display()
                                            ),
                                            Some(&task_run_log),
                                        );
                                    }
                                }
                            }
                            let benchmark =
                                oxo_flow_core::executor::checkpoint::BenchmarkRecord {
                                    rule: rule_name.clone(),
                                    wall_time_secs: duration,
                                    max_memory_mb: record.max_rss_mb,
                                    memory_limit_mb: rule
                                        .effective_memory()
                                        .and_then(oxo_flow_core::scheduler::parse_memory_mb),
                                    cpu_seconds: record.cpu_seconds,
                                    retries: record.retries,
                                    recorded_as: None,
                                };
                            // `benchmark` declaration (issue #469): write the
                            // engine-sampled metrics to the declared path,
                            // per instance (the declaration may carry
                            // `{sample}`-style wildcards). Best-effort — a
                            // benchmark write failure must never fail a
                            // completed rule.
                            if let Some(bench_decl) = &rule.benchmark {
                                let bench_path = oxo_flow_core::executor::checkpoint::expand_config_in_path(bench_decl, &wildcard_values);
                                if !bench_path.contains('{') {
                                    let rendered = render_benchmark_json(&benchmark);
                                    let dest = workdir_actual.as_ref().join(&bench_path);
                                    if let Some(parent) = dest.parent() {
                                        let _ = std::fs::create_dir_all(parent);
                                    }
                                    if let Err(e) = std::fs::write(&dest, rendered) {
                                        tracing::warn!(
                                            rule = %rule_name,
                                            path = %dest.display(),
                                            error = %e,
                                            "failed to write benchmark file"
                                        );
                                    }
                                }
                            }
                            // Post-run manifest re-verification
                            // (issue #194 §2.10): an external writer
                            // touching an input mid-run is invisible to the
                            // pre-run snapshot alone. Re-snapshot now; on
                            // any change, warn loudly and keep the PRE-run
                            // manifest recorded — the next run's comparison
                            // then sees the mismatch and re-executes this
                            // rule instead of serving outputs built from a
                            // mixed file state.
                            let post_result =
                                oxo_flow_core::executor::checkpoint::snapshot_input_manifest_async(
                                    &rule,
                                    workdir_actual.as_ref(),
                                    &wildcard_values,
                                    &crate::commands::run_preview::storage_resolver(),
                                )
                                .await;
                            let post_manifest = match post_result {
                                Ok(manifest) => manifest,
                                Err(e) => {
                                    let msg = format!("could not re-snapshot inputs: {e}");
                                    diagnostic_narrate(
                                        format_args!(
                                            "  {} could not re-snapshot inputs of rule '{}' — {} (mid-run input changes cannot be detected for it)",
                                            "⚠".yellow(),
                                            rule_name,
                                            oxo_flow_core::executor::process::mask_sensitive(
                                                &msg,
                                                &sensitive_values,
                                            )
                                        ),
                                        Some(&task_run_log),
                                    );
                                    None
                                }
                            };
                            if let (Some(pre), Some(post)) = (&input_manifest, &post_manifest) {
                                let changes =
                                    oxo_flow_core::executor::checkpoint::manifest_changes(
                                        pre, post,
                                    );
                                if !changes.is_empty() {
                                    let note = format!(
                                        "\n[oxo-flow] WARNING: an input changed while this rule was running ({}); outputs may be inconsistent — the next run will re-execute this rule",
                                        changes.join(", ")
                                    );
                                    tracing::warn!(
                                        rule = %rule_name,
                                        changes = ?changes,
                                        "input changed while the rule was running; outputs may be inconsistent"
                                    );
                                    if let Some(ref mut stderr) = record.stderr {
                                        stderr.push_str(&oxo_flow_core::executor::process::mask_sensitive(
                                            &note,
                                            &sensitive_values,
                                        ));
                                    } else {
                                        record.stderr = Some(
                                            oxo_flow_core::executor::process::mask_sensitive(
                                                &note,
                                                &sensitive_values,
                                            ),
                                        );
                                    }
                                }
                            }

                            let mut ck = checkpoint.lock().await;
                            ck.record_run(&record, output_tail_bytes_from_env());
                            ck.mark_completed(&rule_name, benchmark);
                            // #843: a rule re-executed via cascade-up has
                            // just regenerated its outputs — the old
                            // tombstone (from reclaim or the post-run sweep)
                            // is stale and would make the post-run sweep
                            // skip re-deleting them. Clear it here; the
                            // sweep re-tombstones below.
                            ck.tombstones.remove(&rule_name);
                            if let Some(ref manifest) = input_manifest {
                                ck.record_input_manifest(&rule_name, manifest.clone());
                            }
                            if provenance {
                                // Expand `{config.*}` (and wildcards) exactly
                                // like the output-verification pass does —
                                // recording the UNEXPANDED template stored a
                                // path that never existed, so `--provenance`
                                // silently recorded nothing on the default
                                // `init` scaffold and any templated-output
                                // workflow (issue #276). Keys stay consistent
                                // with `should_skip_rule_with_checksums`,
                                // which looks up EXPANDED paths.
                                let mut recorded_here = 0usize;
                                for output in &rule.output {
                                    let expanded = oxo_flow_core::executor::checkpoint::expand_config_in_path(output, &wildcard_values);
                                    if expanded.contains('{') {
                                        continue; // per-instance wildcard: resolved only at fan-out
                                    }
                                    let output_path = workdir_actual.join(&expanded);
                                    if output_path.exists()
                                        && let Ok(checksum) =
                                            oxo_flow_core::executor::checkpoint::compute_file_checksum(&output_path)
                                    {
                                        ck.record_checksum(&expanded, checksum);
                                        recorded_here += 1;
                                    }
                                }
                                provenance_recorded.fetch_add(recorded_here, std::sync::atomic::Ordering::Relaxed);
                            }
                            if let Err(e) = ck.save_to_file_async(&checkpoint_path).await {
                                tracing::warn!("Failed to save checkpoint: {e}");
                            }
                            (rule_name, oxo_flow_core::executor::JobStatus::Success, record)
                        } else if record.status == oxo_flow_core::executor::JobStatus::Skipped {
                            skipped_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                            // Issue #747: a Skipped record is terminal. Drop
                            // the spawn-time running mark regardless of which
                            // skip path produced it (when-gate false skips it
                            // in the executor; the other terminal skip
                            // reasons clear it here). Otherwise the persisted
                            // running set outlives the run and the web status
                            // poll renders phantom Running nodes.
                            if record.skip_reason.as_deref() != Some("outputs up-to-date") {
                                let mut ck = checkpoint.lock().await;
                                ck.clear_running(&rule_name);
                                if let Err(e) = ck.save_to_file_async(&checkpoint_path).await {
                                    tracing::warn!("Failed to save checkpoint (skip): {e}");
                                }
                            }
                            if record.skip_reason.as_deref() == Some("optional inputs missing") {
                                // No outputs will exist — block non-optional
                                // dependents instead of letting them run into
                                // a missing-file shell error (issue #200).
                                let mut sno = skipped_no_output.lock().await;
                                sno.insert(rule_name.clone());
                            }
                            if record.skip_reason.as_deref() == Some("outputs up-to-date") {
                                // Up-to-date outputs are valid completions
                                // (issue #324 F-1): a rule interrupted
                                // mid-run can leave outputs behind (its
                                // shell kept running after a SIGKILL), and
                                // resume verdicts it up-to-date. Recording
                                // it prevents re-submission on every later
                                // run and keeps the audit trail aligned
                                // with the disk.
                                // Record a synthetic benchmark only when the
                                // rule has no real one (e.g. first adoption
                                // of a pre-existing run). A zero-wall
                                // placeholder must never overwrite measured
                                // wall-time history — the Dashboard's
                                // "Total Runtime" and Prometheus
                                // oxo_flow_rule_duration_seconds sum these
                                // records (issue #481).
                                let mut ck = checkpoint.lock().await;
                                if !ck.benchmarks.contains_key(&rule_name) {
                                    let benchmark =
                                        oxo_flow_core::executor::checkpoint::BenchmarkRecord {
                                            rule: rule_name.clone(),
                                            wall_time_secs: 0.0,
                                            max_memory_mb: None,
                                            memory_limit_mb: rule
                                                .effective_memory()
                                                .and_then(oxo_flow_core::scheduler::parse_memory_mb),
                                            cpu_seconds: None,
                                            retries: 0,
                                            recorded_as: Some("outputs up-to-date".to_string()),
                                        };
                                    ck.mark_completed(&rule_name, benchmark);
                                } else {
                                    ck.mark_completed_quiet(&rule_name);
                                }
                                if let Err(e) = ck.save_to_file_async(&checkpoint_path).await {
                                    tracing::warn!("Failed to save checkpoint: {e}");
                                }
                            }
                            // Surface the executor's skip reason (condition
                            // false, optional inputs missing, outputs
                            // up-to-date) — otherwise a submitted rule that
                            // never executes looks indistinguishable from one
                            // that ran.
                            if !is_tty {
                                let reason = record
                                    .skip_reason
                                    .as_deref()
                                    .unwrap_or("skipped");
                                eprintln!("  {} {} ({reason})", "⊝".dimmed(), rule_name);
                            }
                            (rule_name, oxo_flow_core::executor::JobStatus::Skipped, record)
                        } else {
                            // A failed `required = false` rule is recorded and
                            // blocks its dependents, but does not fail the
                            // run (issue #99 B2).
                            if rule.required {
                                fail_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                            } else {
                                non_required_fail_count
                                    .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                            }
                            {
                                let mut frs = failed_rules_set.lock().await;
                                frs.insert(rule_name.clone());
                            }
                            let mut ck = checkpoint.lock().await;
                            ck.record_run(&record, output_tail_bytes_from_env());
                            ck.mark_failed(&rule_name);
                            if let Err(e) = ck.save_to_file_async(&checkpoint_path).await {
                                tracing::warn!("Failed to save checkpoint: {e}");
                            }
                            let mut err_msg = format!("rule '{}' failed", rule_name);
                            if let Some(ref stderr) = record.stderr {
                                let trimmed = stderr.trim();
                                if !trimmed.is_empty() {
                                    err_msg.push_str(&format!("\nstderr: {}", trimmed));
                                }
                            }
                            if let Some(code) = record.exit_code {
                                err_msg.push_str(&format!("\nexit code: {}", code));
                            }
                            progress_narrate(
                            format_args!("  {} {}", "✗".red(), err_msg),
                            &task_run_log,
                        );
                            // Always record: keep_going needs the final
                            // listing, non-required failures are listed in
                            // either mode, and the AI recovery path must be
                            // able to find the ABORTING rule by name.
                            {
                                let mut reason = String::new();
                                if let Some(code) = record.exit_code {
                                    reason.push_str(&format!("exit code {}", code));
                                }
                                if let Some(ref stderr) = record.stderr
                                    && let Some(last) = stderr
                                        .trim()
                                        .lines()
                                        .next_back()
                                        .filter(|l| !l.is_empty())
                                {
                                    if !reason.is_empty() {
                                        reason.push_str(" — ");
                                    }
                                    reason.push_str(last);
                                }
                                if reason.is_empty() {
                                    reason.push_str("failed");
                                }
                                // Retry visibility (audit #276 P4-3): the
                                // executor retries silently by design, but the
                                // failure summary must say the rule was
                                // retried N times so automation and humans can
                                // distinguish flaky from immediate failures.
                                if record.retries > 0 {
                                    reason.push_str(&format!(
                                        " (after {} retr{})",
                                        record.retries,
                                        if record.retries == 1 { "y" } else { "ies" }
                                    ));
                                }
                                if !rule.required {
                                    reason.push_str(" (non-required)");
                                }
                                let mut f = failures.lock().await;
                                f.push((rule_name.clone(), reason));
                            }
                            (
                                rule_name,
                                record.status, // Failed or TimedOut
                                record,
                            )
                        }
                    }
                    Err(e) => {
                        if rule.required {
                            fail_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                        } else {
                            non_required_fail_count
                                .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                        }
                        {
                            let mut frs = failed_rules_set.lock().await;
                            frs.insert(rule_name.clone());
                        }
                        // Build the failure record first so the checkpoint
                        // captures the same diagnostics the report will show
                        // (issue #83 WS2).
                        let record = oxo_flow_core::executor::JobRecord {
                            signal: None,
                            rule: rule_name.clone(),
                            status: oxo_flow_core::executor::JobStatus::Failed,
                            enqueued_at: None,
                            started_at: None,
                            finished_at: None,
                            exit_code: Some(-1),
                            stdout: None,
                            // Staging/security errors embed config-expanded
                            // URIs and full commands — mask them like every
                            // other captured surface (issue #99 B1).
                            stderr: Some(oxo_flow_core::executor::process::mask_sensitive(
                                &e.to_string(),
                                &sensitive_values,
                            )),
                            command: None,
                            retries: 0,
                            skip_reason: None,
                            max_rss_mb: None,
                            cpu_seconds: None,
                            caption: rule_caption,
                        };
                        emit_execution_event(oxo_flow_core::executor::ExecutionEvent::RuleCompleted {
                            rule: rule_name.clone(),
                            status: oxo_flow_core::executor::JobStatus::Failed,
                            duration_ms: 0,
                        });
                        let mut ck = checkpoint.lock().await;
                        ck.record_run(&record, output_tail_bytes_from_env());
                        ck.mark_failed(&rule_name);
                        if let Err(e) = ck.save_to_file_async(&checkpoint_path).await {
                            tracing::warn!("Failed to save checkpoint: {e}");
                        }
                        if !keep_going {
                            progress_narrate(
                                format_args!(
                                    "  {} rule '{}' failed: {}",
                                    "✗".red(),
                                    rule_name,
                                    e
                                ),
                                &task_run_log,
                            );
                        }
                        // Always record: keep_going needs the final
                        // listing, non-required failures are listed in
                        // either mode, and the AI recovery path must be able
                        // to find the ABORTING rule by name (a non-required
                        // failure recorded earlier must not shadow it).
                        let mut f = failures.lock().await;
                        let reason = if rule.required {
                            e.to_string()
                        } else {
                            format!("{} (non-required)", e)
                        };
                        f.push((rule_name.clone(), reason));
                        (rule_name, oxo_flow_core::executor::JobStatus::Failed, record)
                    }
                }
            });
            task_rule.insert(handle.id(), task_rule_name);
        }

        // ---- wait for completions -----------------------------------------

        // If nothing is in-flight, we're done (all rules completed, failed, or blocked).
        if join_set.is_empty() {
            break;
        }

        // Wait for the next rule to finish — or for a termination signal
        // (audit C14): the in-flight rules must be torn down and the
        // checkpoint persisted before the process exits, instead of the
        // default disposition killing oxo-flow outright and orphaning every
        // child (PPID 1) with a half-written checkpoint.
        let join_result = tokio::select! {
            // `biased` with join-next first: a result that is already
            // reapable is ALWAYS drained before the signal branch can be
            // taken. Without this, a signal landing in the same poll
            // cycle as a completion races it at random — the teardown
            // would then see the finished rule's id still in `task_rule`
            // and record it Failed in the checkpoint.
            biased;
            result = join_set.join_next_with_id() => Some(result),
            received = async {
                match signal_rx.changed().await {
                    Ok(()) => *signal_rx.borrow_and_update(),
                    // The watcher died before the run did; the run then
                    // keeps the signals' default dispositions.
                    Err(_) => None,
                }
            } => match received {
                Some(signal) => {
                    terminate_on_signal(
                        signal,
                        &executor,
                        &mut join_set,
                        task_rule.values(),
                        &checkpoint,
                        &checkpoint_path,
                        &mut json_summary,
                        &config,
                    )
                    .await
                }
                // The watcher died before the run did; the run then keeps
                // the signals' default dispositions.
                None => None,
            },
            // Spawn watchdog (issue #685): a rule stuck in Running without
            // ever spawning a child process wedged the engine in a futex
            // wait (empty checkpoint running-set, zero completions). Every
            // SPAWN_WATCHDOG_TICK_SECS, compare the scheduler's running set
            // with the executor's live child pids: any rule running longer
            // than the threshold with NO pid at all gets one report naming
            // it. Issue #847 splits the report: a rule still queued in the
            // resource pool is a benign wait — the scheduler's own 60s
            // blockage WARNs already narrate holders and FIFO position — so
            // it only earns a louder one-shot WARN here. The #685 ERROR is
            // reserved for rules NOT queued in the pool (env-setup stragglers
            // have their own throttled WARN; anything else is the genuine
            // lost-wakeup signature).
            _ = watchdog_tick.tick() => {
                let live_rules: std::collections::HashSet<String> = executor
                    .active_pids()
                    .into_iter()
                    .map(|(rule, _)| rule)
                    .collect();
                let queued: std::collections::HashSet<String> =
                    executor.resource_waiters().await.into_iter().collect();
                let n = |args: std::fmt::Arguments<'_>| {
                    diagnostic_narrate(args, Some(&run_log));
                };
                for rule in sched.running_rules() {
                    if live_rules.contains(&rule) || watchdog_reported.contains(&rule) {
                        continue;
                    }
                    let Some(started) = running_since.get(&rule) else {
                        continue;
                    };
                    if started.elapsed() < watchdog_threshold {
                        continue;
                    }
                    watchdog_reported.insert(rule.clone());
                    let running_secs = started.elapsed().as_secs();
                    if queued.contains(&rule) {
                        n(format_args!(
                            "{} rule '{rule}' has been Running for {running_secs}s while \
                             queued in the resource pool — benign wait, the scheduler \
                             reports blockage details every 60s (issue #847). The engine \
                             will keep waiting.",
                            "Watchdog:".yellow().bold()
                        ));
                    } else {
                        n(format_args!(
                            "{} rule '{rule}' has been Running for {running_secs}s but never \
                             spawned a child process — likely causes: environment setup \
                             still in progress, or a lost scheduler wakeup (issue #685). \
                             The engine will keep waiting; if the rule never starts, \
                             capture the run log and file an issue.",
                            "Watchdog ERROR:".yellow().bold()
                        ));
                        tracing::error!(
                            rule = %rule,
                            running_secs,
                            "rule has been Running for over {}s but never spawned a child \
                             process (issue #685)",
                            watchdog_threshold.as_secs()
                        );
                    }
                }
                None
            }
            // #843 disk-pressure tick: while rules are in flight, re-measure
            // free space periodically so Reclaim fires even without a
            // completion boundary (e.g. a single long-running rule writing
            // hundreds of GB). Expression is None — the measured state is
            // enforced at the top of the next loop iteration.
            _ = disk_tick.tick() => {
                // F4: skip the statfs entirely while the feature is off —
                // otherwise a disabled config still pays one measurement
                // per tick for nothing.
                if disk_thresholds.is_some() {
                    let mut n = |args: std::fmt::Arguments<'_>| {
                        diagnostic_narrate(args, Some(&run_log));
                    };
                    disk_monitor.observe(workdir_actual.as_ref(), &mut n);
                }
                None
            }
        };
        let Some(join_result) = join_result else {
            continue;
        };
        let (completed_rule, status, record) = match join_result {
            // Reaped: drop the id→rule link so a later signal teardown
            // cannot record this rule Failed. (Only the panic branch
            // used to remove the entry — every success or rule-level
            // failure lingered here, so Ctrl-C marked ALL finished
            // rules Failed in the checkpoint.)
            Some(Ok((id, v))) => {
                task_rule.remove(&id);
                v
            }
            Some(Err(e)) => {
                if e.is_panic() {
                    tracing::error!("Task panicked: {e}");
                }
                // A panicked task is an ENGINE fault, not a rule-level
                // best-effort failure — always counts as required (the run
                // fails), regardless of the rule's `required` flag.
                // Its pool registration (if it was waiting) must not hold
                // the FIFO line hostage — live waiters re-register.
                executor.clear_resource_waiters().await;
                fail_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                // The task left the JoinSet without its payload, so the id
                // is the only link back to the rule (issue #136 fix 2).
                // Without marking it completed the rule would stay in the
                // scheduler's running set forever, permanently shrinking
                // the submit cap — with -j 1 every remaining rule would
                // silently never run. Fall through to the common failure
                // bookkeeping instead of `continue`, so dependents are
                // blocked, the checkpoint records the engine fault, and
                // the standard abort path fails the run loudly.
                let Some(rule_name) = task_rule.remove(&e.id()) else {
                    // Internal invariant: every spawned task registers its
                    // name at spawn. Fail the run rather than leak the cap.
                    // The engine fault counts as a failure in the summary
                    // (issue #142 H6).
                    json_summary.emit(
                        "failed",
                        &RunCounts {
                            succeeded: success_count.load(std::sync::atomic::Ordering::Relaxed),
                            skipped: skipped_count.load(std::sync::atomic::Ordering::Relaxed),
                            failed: fail_count.load(std::sync::atomic::Ordering::Relaxed),
                            non_required_failed: non_required_fail_count
                                .load(std::sync::atomic::Ordering::Relaxed),
                            blocked: blocked.lock().await.len(),
                        },
                        vec![],
                    );
                    return Err(anyhow::anyhow!(
                        "internal error: task {} has no recorded rule",
                        e.id()
                    ));
                };
                {
                    let mut frs = failed_rules_set.lock().await;
                    frs.insert(rule_name.clone());
                }
                let record = oxo_flow_core::executor::JobRecord {
                    signal: None,
                    rule: rule_name.clone(),
                    status: oxo_flow_core::executor::JobStatus::Failed,
                    enqueued_at: None,
                    started_at: None,
                    finished_at: None,
                    exit_code: Some(-1),
                    stdout: None,
                    stderr: Some(oxo_flow_core::executor::process::mask_sensitive(
                        &e.to_string(),
                        &sensitive_values,
                    )),
                    command: None,
                    retries: 0,
                    skip_reason: None,
                    max_rss_mb: None,
                    cpu_seconds: None,
                    caption: config.get_rule(&rule_name).and_then(|r| {
                        oxo_flow_core::executor::process::rule_report_caption(
                            r,
                            workdir_actual.as_ref(),
                        )
                    }),
                };
                let mut ck = checkpoint.lock().await;
                ck.record_run(&record, output_tail_bytes_from_env());
                ck.mark_failed(&rule_name);
                if let Err(save_err) = ck.save_to_file_async(&checkpoint_path).await {
                    tracing::warn!("Failed to save checkpoint: {save_err}");
                }
                let mut f = failures.lock().await;
                f.push((rule_name.clone(), format!("task panicked: {e}")));
                (
                    rule_name,
                    oxo_flow_core::executor::JobStatus::Failed,
                    record,
                )
            }
            None => break,
        };

        sched.mark_completed(record);
        running_since.remove(&completed_rule);
        progress.inc(1);

        // ── Checkpoint re-entry processing (issue #78 P3) ───────────────────
        // A successful checkpoint rule may declare new samples in its manifest;
        // they merge into the plan and execute in this same run.
        if status == oxo_flow_core::executor::JobStatus::Success
            && let Some(cp_rule) = config.get_rule(&completed_rule).cloned()
            && cp_rule.checkpoint
            && let Err(e) = process_reentry(
                &mut config,
                &cp_rule,
                workdir_actual.as_ref(),
                &wildcard_values,
                &mut sched,
                &mut order,
                &mut order_set,
                &mut dag,
                &checkpoint,
            )
            .await
        {
            tracing::error!(rule = %completed_rule, error = %e, "checkpoint re-entry failed");
            // Fail the checkpoint rule and propagate like any failure.
            // Build the record first so the checkpoint captures the same
            // diagnostics the report will show (issue #83 WS2).
            let record = oxo_flow_core::executor::JobRecord {
                signal: None,
                rule: completed_rule.clone(),
                status: oxo_flow_core::executor::JobStatus::Failed,
                enqueued_at: None,
                started_at: None,
                finished_at: None,
                exit_code: Some(1),
                stdout: None,
                stderr: None,
                command: None,
                retries: 0,
                skip_reason: Some(format!("re-entry manifest: {e}")),
                max_rss_mb: None,
                cpu_seconds: None,
                caption: config.get_rule(&completed_rule).and_then(|r| {
                    oxo_flow_core::executor::process::rule_report_caption(
                        r,
                        workdir_actual.as_ref(),
                    )
                }),
            };
            {
                let mut frs = failed_rules_set.lock().await;
                frs.insert(completed_rule.clone());
            }
            {
                let mut ck = checkpoint.lock().await;
                ck.record_run(&record, output_tail_bytes_from_env());
                ck.mark_failed(&completed_rule);
                if let Err(save_err) = ck.save_to_file_async(&checkpoint_path).await {
                    tracing::warn!("Failed to save checkpoint: {save_err}");
                }
            }
            {
                let mut f = failures.lock().await;
                let reason = if cp_rule.required {
                    format!("re-entry manifest: {e}")
                } else {
                    format!("re-entry manifest: {e} (non-required)")
                };
                f.push((completed_rule.clone(), reason));
            }
            if cp_rule.required {
                fail_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
            } else {
                non_required_fail_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
            }
            sched.mark_completed(record);
        }

        // ── Runtime-discovered output_pattern processing (issue #227 item 5) ─
        // A producer instance succeeded: scan its output_pattern domain,
        // persist it, and — once ALL of the producer's instances are
        // completed (the domain union is final) — instantiate the deferred
        // consumers and insert them into the remaining plan. A discovery
        // failure fails the producer like a re-entry failure: the run aborts
        // with attribution and the next run re-executes the producer.
        if status == oxo_flow_core::executor::JobStatus::Success
            && config
                .get_rule(&completed_rule)
                .is_some_and(|r| r.output_pattern.is_some())
            && let Err(e) = process_output_pattern_discovery(
                &mut config,
                &completed_rule,
                workdir_actual.as_ref(),
                &mut sched,
                &mut order,
                &mut order_set,
                &mut dag,
                &checkpoint,
                &checkpoint_path,
            )
            .await
        {
            tracing::error!(rule = %completed_rule, error = %e, "output_pattern discovery failed");
            let record = oxo_flow_core::executor::JobRecord {
                signal: None,
                rule: completed_rule.clone(),
                status: oxo_flow_core::executor::JobStatus::Failed,
                enqueued_at: None,
                started_at: None,
                finished_at: None,
                exit_code: Some(1),
                stdout: None,
                stderr: None,
                command: None,
                retries: 0,
                skip_reason: Some(format!("output_pattern discovery: {e}")),
                max_rss_mb: None,
                cpu_seconds: None,
                caption: config.get_rule(&completed_rule).and_then(|r| {
                    oxo_flow_core::executor::process::rule_report_caption(
                        r,
                        workdir_actual.as_ref(),
                    )
                }),
            };
            {
                let mut frs = failed_rules_set.lock().await;
                frs.insert(completed_rule.clone());
            }
            {
                let mut ck = checkpoint.lock().await;
                ck.record_run(&record, output_tail_bytes_from_env());
                ck.mark_failed(&completed_rule);
                if let Err(save_err) = ck.save_to_file_async(&checkpoint_path).await {
                    tracing::warn!("Failed to save checkpoint: {save_err}");
                }
            }
            {
                let mut f = failures.lock().await;
                f.push((
                    completed_rule.clone(),
                    format!("output_pattern discovery: {e}"),
                ));
            }
            fail_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
            sched.mark_completed(record);
        }

        // Failure attribution for output_pattern producers (issue #227
        // item 5): a producer that failed leaves its deferred consumers
        // uninstantiated — name them so the failure is attributable, and
        // record them for the failure summary.
        if (status == oxo_flow_core::executor::JobStatus::Failed
            || status == oxo_flow_core::executor::JobStatus::TimedOut
            || status == oxo_flow_core::executor::JobStatus::Cancelled)
            && let Some(producer_template) = config.output_pattern_template_of(&completed_rule)
        {
            let consumers = config.pending_output_pattern_consumers_of(&producer_template);
            if !consumers.is_empty() {
                eprintln!(
                    "{} {} not instantiated — output_pattern producer '{}' failed",
                    "Warning:".bold().yellow(),
                    consumers.join(", "),
                    producer_template
                );
                let mut f = failures.lock().await;
                for c in consumers {
                    f.push((
                        c,
                        format!(
                            "not instantiated — output_pattern producer '{}' failed",
                            producer_template
                        ),
                    ));
                }
            }
        }

        // Skipped producers (issue #297 item 4): an outputs-up-to-date or
        // condition-false skip never reaches discovery, so the producer's
        // deferred consumers silently vanish from THIS run's plan. A
        // checkpointed domain from an earlier success is replayed on resume,
        // but a skip with no recorded domain leaves them uninstantiated for
        // good — name both sides so the disappearance is attributable. A
        // skip is not a failure, so unlike the block above this stays a
        // warning and does not enter the failure summary.
        if status == oxo_flow_core::executor::JobStatus::Skipped
            && let Some(producer_template) = config.output_pattern_template_of(&completed_rule)
        {
            let consumers = config.pending_output_pattern_consumers_of(&producer_template);
            if !consumers.is_empty() {
                eprintln!(
                    "{} {} not instantiated — output_pattern producer '{}' was skipped",
                    "Warning:".bold().yellow(),
                    consumers.join(", "),
                    producer_template
                );
                tracing::warn!(
                    producer = %producer_template,
                    consumers = ?consumers,
                    "skipped output_pattern producer leaves deferred consumers uninstantiated"
                );
            }
        }

        // Abort on first failure when not in keep_going mode.
        let fc = fail_count.load(std::sync::atomic::Ordering::Relaxed);
        if fc > 0
            && !keep_going
            && (status == oxo_flow_core::executor::JobStatus::Failed
                || status == oxo_flow_core::executor::JobStatus::TimedOut
                || status == oxo_flow_core::executor::JobStatus::Cancelled)
        {
            progress.finish_and_clear();
            // Kill in-flight rule processes before cancelling their tasks —
            // `abort_all()` alone orphans the OS children, which keep
            // consuming the machine after the run exits (issue #131).
            // Clear the pool's FIFO waiter queue first so cancelled waiters
            // cannot hold the line hostage (issue #123 100% guarantee).
            executor.clear_resource_waiters().await;
            // The grace poll inside kill_process_tree sleeps up to 10 s —
            // run the sweep on the blocking pool, not a runtime worker
            // (#524; the timeout path already follows this rule). The
            // snapshot also records WHICH rules the abort actually killed:
            // the #498 demotion below keys on that evidence, not on bare
            // signal deaths — a sibling the kernel OOM-killer shot carries
            // the same signal signature and must keep its failed record
            // (#525).
            let snapshot = executor.active_pids();
            let mut abort_killed: std::collections::HashSet<String> =
                snapshot.iter().map(|(r, _)| r.clone()).collect();
            tokio::task::spawn_blocking(move || {
                for (rule_name, pid) in snapshot {
                    if let Err(e) = oxo_flow_core::executor::timeout::kill_process_tree(pid) {
                        tracing::warn!(
                            rule = %rule_name,
                            pid,
                            error = %e,
                            "failed to signal in-flight rule process during abort"
                        );
                    }
                }
            })
            .await
            .ok();
            join_set.abort_all();
            // Re-snapshot and sweep again: a retry task could wake during
            // the first sweep and spawn its next attempt; those late
            // children escape the stale first snapshot (#524).
            let late = executor.active_pids();
            abort_killed.extend(late.iter().map(|(r, _)| r.clone()));
            if !late.is_empty() {
                tokio::task::spawn_blocking(move || {
                    for (rule_name, pid) in late {
                        if let Err(e) = oxo_flow_core::executor::timeout::kill_process_tree(pid) {
                            tracing::warn!(
                                rule = %rule_name,
                                pid,
                                error = %e,
                                "failed to signal late-spawned rule process during abort"
                            );
                        }
                    }
                })
                .await
                .ok();
            }

            // The killed siblings would otherwise surface as bare
            // "✗ rule 'X' failed" with no exit code and an empty stderr
            // tail: the process-tree kill makes their shells die by signal,
            // so their command loops record exit_code None and no
            // diagnostics. Record them as honest Cancelled entries instead
            // — never in failed_rules, they are collateral cancellations,
            // not failures — and narrate them like skips.
            {
                let verb = if status == oxo_flow_core::executor::JobStatus::Cancelled {
                    "cancelled"
                } else {
                    "failed"
                };
                let killed: Vec<String> = task_rule
                    .values()
                    .filter(|name| *name != &completed_rule)
                    .cloned()
                    .collect();
                if !killed.is_empty() {
                    // A rule that already finished its closure (result still
                    // unreaped in the JoinSet) has written its own record —
                    // never overwrite a genuine Success/Failed. BUT a failure
                    // whose only evidence is a signal death with no exit code
                    // IS the abort's kill, not a self-caused failure: demote
                    // it out of failed_rules/metrics (issue #498). Pure
                    // signal deaths with no exit code are exactly what the
                    // process-tree kill produces; a rule that failed on its
                    // own (exit code, or a pre-abort signal like OOM-kill
                    // whose result was reaped first) keeps its record.
                    let already_failed = failed_rules_set.lock().await.clone();
                    let mut ck = checkpoint.lock().await;
                    for rule_name in &killed {
                        if ck.is_completed(rule_name) {
                            continue;
                        }
                        // The kill-snapshot membership is the demotion
                        // evidence (#525): a signal death alone cannot
                        // distinguish the abort's kill from a kernel
                        // OOM-kill that merely reaped around the same time.
                        // A rule that already failed on its own and was NOT
                        // in the kill snapshots keeps its genuine failure
                        // record untouched — never demoted, never
                        // overwritten with a cancellation.
                        if already_failed.contains(rule_name) && !abort_killed.contains(rule_name) {
                            continue;
                        }
                        if already_failed.contains(rule_name) {
                            let skip_reason = format!(
                                "run aborted before this rule finished — required rule \
'{completed_rule}' {verb}"
                            );
                            if ck.demote_failed_to_cancelled(rule_name, skip_reason) {
                                let required =
                                    config.get_rule(rule_name).is_some_and(|r| r.required);
                                if required {
                                    fail_count.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
                                } else {
                                    non_required_fail_count
                                        .fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
                                }
                                failures.lock().await.retain(|(n, _)| n != rule_name);
                                if !is_tty {
                                    diagnostic_narrate(
                                        format_args!(
                                            "  {} {} (run aborted — required rule '{}' {})",
                                            "⊝".dimmed(),
                                            rule_name,
                                            completed_rule,
                                            verb
                                        ),
                                        Some(&run_log),
                                    );
                                }
                            }
                            continue;
                        }
                        let record = oxo_flow_core::executor::JobRecord {
                            signal: None,
                            rule: rule_name.clone(),
                            status: oxo_flow_core::executor::JobStatus::Cancelled,
                            enqueued_at: None,
                            started_at: None,
                            finished_at: Some(chrono::Utc::now()),
                            exit_code: None,
                            stdout: None,
                            stderr: None,
                            command: None,
                            retries: 0,
                            skip_reason: Some(format!(
                                "run aborted before this rule finished — required rule \
'{completed_rule}' {verb}"
                            )),
                            max_rss_mb: None,
                            cpu_seconds: None,
                            caption: None,
                        };
                        ck.record_run(&record, output_tail_bytes_from_env());
                        if !is_tty {
                            diagnostic_narrate(
                                format_args!(
                                    "  {} {} (run aborted — required rule '{}' {})",
                                    "⊝".dimmed(),
                                    rule_name,
                                    completed_rule,
                                    verb
                                ),
                                Some(&run_log),
                            );
                        }
                    }
                    if let Err(e) = ck.save_to_file_async(&checkpoint_path).await {
                        tracing::warn!(error = %e, "failed to save checkpoint after abort");
                    }
                }
            }

            // AI error recovery
            let should_recover =
                ai_recover || crate::commands::ai_template::should_use_ai(Some(&workflow), false);
            if should_recover {
                let failures_guard = failures.lock().await;
                // Diagnose the rule that actually aborted the run — a
                // non-required failure recorded earlier must not shadow it.
                let failure = failures_guard
                    .iter()
                    .find(|(name, _)| name == &completed_rule)
                    .or_else(|| failures_guard.first());
                if let Some((rule, error)) = failure
                    && let Some(provider) =
                        crate::commands::ai_template::try_resolve_ai(Some(&workflow), true)
                {
                    // `--ai-max-retries` bounds the recovery attempts: a
                    // failed diagnosis call is retried up to the flag's
                    // value, a successful one is never repeated.
                    let attempts = ai_attempts(ai_max_retries);
                    let mut result: Result<crate::commands::ai_recover::DiagnoseResult> =
                        Err(anyhow::anyhow!("AI recovery did not run"));
                    for attempt in 1..=attempts {
                        // Feed the real exit code when the failure record has
                        // one: the banner previously always showed -1, and the
                        // prompt's exit-code→category triage (137=OOM,
                        // 139=segfault, 127=command not found) needs the true
                        // value to classify correctly. Panics/aborts keep -1.
                        let exit_code = failure_exit_code(error);
                        result = crate::commands::ai_recover::diagnose_failure(
                            &workflow, rule, exit_code, error, &provider,
                        )
                        .await;
                        if result.is_ok() || attempt == attempts {
                            break;
                        }
                        if let Err(e) = &result {
                            eprintln!(
                                "  AI diagnosis attempt {attempt}/{attempts} failed: {e} — retrying"
                            );
                        }
                    }
                    match result {
                        Ok(diag) => {
                            if let Some(ref toml) = diag.modified_toml {
                                // Only write AI-generated TOML when the model itself
                                // assessed it safe AND it parses as a valid workflow.
                                // Anything else is shown, never auto-applied — the
                                // content can carry injected instructions just like
                                // any other model output.
                                if !diag.safe_to_auto_apply {
                                    eprintln!(
                                        "  AI fix not auto-applied: flagged unsafe by the \
                                         safety assessment. Review the suggestion manually."
                                    );
                                } else if let Err(e) = oxo_flow_core::WorkflowConfig::parse(toml) {
                                    eprintln!(
                                        "  AI fix not auto-applied: generated TOML failed \
                                         validation ({e}). Review the suggestion manually."
                                    );
                                } else if let Err(e) = crate::commands::ai_recover::apply_fix(
                                    &workflow, toml, "recovery",
                                ) {
                                    eprintln!("  Failed to apply AI fix: {e}");
                                }
                            }
                        }
                        Err(e) => eprintln!("  AI diagnosis failed: {e}"),
                    }
                }
            }

            // The plain-failure abort: emit the summary BEFORE returning,
            // mirroring the keep-going path's document (issue #142 H6).
            let ck = checkpoint.lock().await;
            json_summary.emit(
                "failed",
                &RunCounts {
                    succeeded: success_count.load(std::sync::atomic::Ordering::Relaxed),
                    skipped: skipped_count.load(std::sync::atomic::Ordering::Relaxed),
                    failed: fail_count.load(std::sync::atomic::Ordering::Relaxed),
                    non_required_failed: non_required_fail_count
                        .load(std::sync::atomic::Ordering::Relaxed),
                    blocked: blocked.lock().await.len(),
                },
                rule_resource_rows(&ck),
            );
            // on_error fires on the plain-failure abort too (issue #227).
            run_terminal_hook(
                &config,
                &workdir_actual,
                success_count.load(std::sync::atomic::Ordering::Relaxed),
                fail_count.load(std::sync::atomic::Ordering::Relaxed),
                skipped_count.load(std::sync::atomic::Ordering::Relaxed),
            )
            .await;
            notify_webhook(
                &config,
                oxo_flow_core::webhook::WebhookEvent::WorkflowFailed,
                Some((
                    success_count.load(std::sync::atomic::Ordering::Relaxed),
                    fail_count.load(std::sync::atomic::Ordering::Relaxed),
                    skipped_count.load(std::sync::atomic::Ordering::Relaxed),
                )),
            )
            .await;
            return Err(anyhow::anyhow!("workflow execution failed"));
        }

        // Propagate failure transitively: every rule that (transitively)
        // depends on a failed rule is marked as skipped/blocked. Uses a
        // worklist to ensure transitive propagation (grandchild of a
        // failed rule is also blocked). Runs when the loop continues past
        // failures: keep_going with any failure, or a non-required failure
        // in either mode (issue #99 B2).
        let nrf = non_required_fail_count.load(std::sync::atomic::Ordering::Relaxed);
        if (fc > 0 && keep_going) || nrf > 0 {
            // Seed the worklist with every rule that directly depends on a
            // failed rule and has not been submitted yet.
            let frs = failed_rules_set.lock().await;
            let mut worklist: Vec<String> = Vec::new();
            for rule_name in &rule_names {
                if submitted.contains(rule_name) {
                    continue;
                }
                let deps = dag.dependencies(rule_name).unwrap_or_default();
                if deps.iter().any(|d| frs.contains(d.as_str())) {
                    worklist.push(rule_name.to_string());
                }
            }
            drop(frs);

            // Process transitively: when a rule is blocked, its own dependents
            // may also become blockable.
            let mut idx = 0;
            while idx < worklist.len() {
                let name = &worklist[idx];
                if submitted.contains(name.as_str()) {
                    idx += 1;
                    continue;
                }
                submitted.insert(name.clone());
                skipped_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                {
                    let mut frs = failed_rules_set.lock().await;
                    frs.insert(name.clone());
                }
                sched.mark_completed(oxo_flow_core::executor::JobRecord {
                    signal: None,
                    rule: name.clone(),
                    status: oxo_flow_core::executor::JobStatus::Skipped,
                    enqueued_at: None,
                    started_at: None,
                    finished_at: None,
                    exit_code: None,
                    stdout: None,
                    stderr: None,
                    command: None,
                    retries: 0,
                    skip_reason: Some("blocked by failed upstream dependency".into()),
                    max_rss_mb: None,
                    cpu_seconds: None,
                    caption: config.get_rule(name).and_then(|r| {
                        oxo_flow_core::executor::process::rule_report_caption(
                            r,
                            workdir_actual.as_ref(),
                        )
                    }),
                });
                progress.inc(1);
                if !is_tty {
                    eprintln!("  {} {} (blocked by failed dependency)", "⊘".yellow(), name);
                }

                // Find transitive dependents of this newly-blocked rule.
                {
                    let frs = failed_rules_set.lock().await;
                    for other in &rule_names {
                        if submitted.contains(other) || worklist.contains(other) {
                            continue;
                        }
                        let deps = dag.dependencies(other).unwrap_or_default();
                        if deps.iter().any(|d| frs.contains(d.as_str())) {
                            worklist.push(other.to_string());
                        }
                    }
                }

                idx += 1;
            }

            // Record blocked rules for final summary
            {
                let frs = failed_rules_set.lock().await;
                for name in &worklist {
                    let deps = dag.dependencies(name).unwrap_or_default();
                    if let Some(dep) = deps.into_iter().find(|d| frs.contains(d.as_str())) {
                        blocked.lock().await.push((name.clone(), dep));
                    }
                }
            }
        }
    }

    // #540: the run loop is done — drop the receiver so a Ctrl-C landing
    // in the post-run work (report snapshot, webhook POST, AI report)
    // makes the watcher's `send` fail and its process-exit fallback
    // engage. With the receiver alive but never polled again, the late
    // signal was swallowed: nothing happened for potentially minutes.
    drop(signal_rx);

    let success_count = success_count.load(std::sync::atomic::Ordering::Relaxed);
    let fail_count = fail_count.load(std::sync::atomic::Ordering::Relaxed);
    let non_required_fail_count =
        non_required_fail_count.load(std::sync::atomic::Ordering::Relaxed);
    let skipped_count = skipped_count.load(std::sync::atomic::Ordering::Relaxed);
    let mut checkpoint = checkpoint.lock().await;
    let failures = failures.lock().await;
    let blocked = blocked.lock().await;

    progress.finish_and_clear();

    // Environment-cache aging (issue #75): --cache-dir would grow without
    // bound; files untouched beyond the age limit are removed after the
    // run so the next one starts clean. Default 90 days, overridable with
    // the `cache_max_age_days` workflow config key (0 disables aging).
    const DEFAULT_CACHE_MAX_AGE_DAYS: u64 = 90;
    let cache_max_age_days: u64 = config
        .config
        .get("cache_max_age_days")
        .and_then(toml::Value::as_integer)
        .filter(|d| *d >= 0)
        .map(|d| d as u64)
        .unwrap_or(DEFAULT_CACHE_MAX_AGE_DAYS);
    if cache_max_age_days > 0
        && let Some(ref cache_dir) = cache_dir
    {
        let removed = cleanup_cache_dir(cache_dir, cache_max_age_days);
        if removed > 0 {
            tracing::info!(
                removed,
                cache = %cache_dir.display(),
                "aged environment-cache files removed"
            );
        }
    }

    emit_execution_event(oxo_flow_core::executor::ExecutionEvent::WorkflowCompleted {
        total_duration_ms: run_started.elapsed().as_millis() as u64,
        succeeded: success_count,
        failed: fail_count,
        skipped: skipped_count,
    });
    if non_required_fail_count > 0 {
        progress_narrate(
            format_args!(
                "\n{} {} succeeded, {} skipped, {} failed, {} non-required failed",
                "Done:".bold(),
                success_count,
                skipped_count,
                fail_count,
                non_required_fail_count
            ),
            &run_log,
        );
    } else {
        progress_narrate(
            format_args!(
                "\n{} {} succeeded, {} skipped, {} failed",
                "Done:".bold(),
                success_count,
                skipped_count,
                fail_count
            ),
            &run_log,
        );
    }

    // Automatic report snapshot (issue #83 P1-14): capture the final
    // checkpoint as a JSON report plus an index.json entry. One call site
    // covers every terminal path of the run loop (and resume, which shares
    // this summary via run_command); dry-run never reaches here. Snapshot
    // errors are warnings — a reporting hiccup must never fail the run.
    if !no_report_snapshot
        && let Err(e) =
            crate::commands::output::snapshot_report(&workflow, &workdir_actual, &checkpoint)
    {
        eprintln!("  {} Report snapshot failed: {e}", "⚠".yellow());
    }

    // --provenance recorded nothing: say so loudly. The historical failure
    // mode was silent — the user asked for tracking, got an empty checksum
    // map, and `provenance verify` later reported "Run with --provenance"
    // after they had done exactly that (issue #276).
    if provenance {
        let recorded = provenance_recorded.load(std::sync::atomic::Ordering::Relaxed);
        if recorded == 0 {
            eprintln!(
                "  {} --provenance recorded 0 checksums — no literal outputs completed \
                 (templated or wildcard outputs are tracked per instance only after fan-out); \
                 'provenance verify' will find nothing to check",
                "⚠".yellow()
            );
        }
    }

    // Workflow-level terminal hooks (issue #227 item 1): on_complete /
    // on_error fire on every terminal path of the run loop, and the
    // configured `[webhook]` endpoint receives the matching event.
    run_terminal_hook(
        &config,
        &workdir_actual,
        success_count,
        fail_count,
        skipped_count,
    )
    .await;
    notify_webhook(
        &config,
        if fail_count > 0 {
            oxo_flow_core::webhook::WebhookEvent::WorkflowFailed
        } else {
            oxo_flow_core::webhook::WebhookEvent::WorkflowCompleted
        },
        Some((success_count, fail_count, skipped_count)),
    )
    .await;

    // With --keep-going, execution continues past failures, so list every failed
    // rule (and why) in one place rather than making the user hunt for them.
    if !failures.is_empty() {
        eprintln!("\n{}", "Failed rules:".bold().red());
        for (name, reason) in failures.iter() {
            eprintln!("  {} {} — {}", "✗".red(), name.bold(), reason);
        }
    }

    // Blocked rules did not run and produced no outputs. Listing them keeps the
    // fallout of a failure visible without disguising it as a genuine failure.
    if !blocked.is_empty() {
        eprintln!("\n{}", "Blocked rules (did not run):".bold().yellow());
        for (name, dep) in blocked.iter() {
            eprintln!("  {} {} — depends on '{}'", "⊘".yellow(), name.bold(), dep);
        }
    }

    // Verify output files exist for completed rules
    if success_count > 0 {
        let workdir_actual = workdir.as_ref().unwrap_or(&workdir_default);
        let mut missing_outputs = Vec::new();
        let mut verified = 0usize;
        let mut total_size: u64 = 0;
        for rule_name in &order {
            if checkpoint.is_completed(rule_name)
                && let Some(rule) = config.get_rule(rule_name)
            {
                for output in &rule.output {
                    if !output.contains('{') {
                        let expanded = oxo_flow_core::executor::checkpoint::expand_config_in_path(
                            output,
                            &wildcard_values,
                        );
                        let resolved = workdir_actual.join(&expanded);
                        if resolved.exists() {
                            verified += 1;
                            if let Ok(meta) = std::fs::metadata(&resolved) {
                                total_size += meta.len();
                            }
                        } else if !workdir_actual.join(output).exists() {
                            missing_outputs.push(format!("  {}: {}", rule_name, output));
                        } else {
                            verified += 1;
                        }
                    }
                }
            }
        }
        if verified > 0 {
            let size_str = if total_size > 1_073_741_824 {
                format!("{:.1}GB", total_size as f64 / 1_073_741_824.0)
            } else if total_size > 1_048_576 {
                format!("{:.1}MB", total_size as f64 / 1_048_576.0)
            } else {
                format!("{}B", total_size)
            };
            eprintln!(
                "{} {} output files verified ({} total)",
                "✓".green(),
                verified,
                size_str
            );
        }
        if !missing_outputs.is_empty() {
            eprintln!(
                "{} {} output file(s) were not found:",
                "⚠".yellow(),
                missing_outputs.len()
            );
            for m in &missing_outputs {
                eprintln!("{}", m.dimmed());
            }
        }
    }

    // Transform chunk cleanup (transform.cleanup = true): delete the chunk
    // inputs of completed combine rules once the whole run has finished
    // successfully. Deferred to here (instead of per-rule success) so that
    // a failed run keeps its chunks for debugging, and a resumed run never
    // regenerates chunks while the combine stays pre-skipped (which would
    // orphan the freshly created chunk files).
    if fail_count == 0 {
        let workdir_actual = workdir.as_ref().unwrap_or(&workdir_default);
        for rule in &config.rules {
            if rule.cleanup_chunks && checkpoint.is_completed(&rule.name) {
                // Migrate the chunk checksums BEFORE deleting the files
                // (issue #315 F2): a checkpoint that lists already-deleted
                // files as ordinary outputs would be an audit-trail lie —
                // verify would report them missing. If the migration
                // cannot be persisted, keep the files and skip the
                // deletion entirely.
                //
                // Checksums only exist under `--provenance`. With no
                // recorded checksum there is nothing that can lie, so the
                // deletion proceeds without a persistence round-trip
                // (issue #324 F-2) — previously the empty migration made
                // cleanup a silent no-op on every default run.
                let mut migrated: Vec<String> = Vec::new();
                for chunk in rule.input.iter() {
                    if let Some(sha) = checkpoint.checksums.remove(&chunk.to_string()) {
                        checkpoint.record_cleaned_checksum(&chunk.to_string(), sha);
                        migrated.push(chunk.to_string());
                    }
                }
                if !migrated.is_empty()
                    && let Err(e) = checkpoint.save_to_file(&checkpoint_path)
                {
                    tracing::warn!(
                        error = %e,
                        "failed to persist chunk migration — keeping chunk files"
                    );
                    // Roll the in-memory state back so a later save (e.g.
                    // tombstoning) does not persist the migration without
                    // the deletion.
                    for path in &migrated {
                        if let Some(sha) = checkpoint.cleaned_checksums.remove(path) {
                            checkpoint.checksums.insert(path.clone(), sha);
                        }
                    }
                    continue;
                }
                oxo_flow_core::executor::checkpoint::cleanup_transform_chunks(rule, workdir_actual)
                    .await;
            }
        }

        // Temporary rules: after a FULLY successful run, delete their
        // outputs once every dependent is complete and record a tombstone.
        // A future run regenerates them via cascade-up (missing inputs
        // re-run the completed producer). Leaf rules keep their outputs.
        let mut new_tombstones: Vec<(String, Vec<String>)> = Vec::new();
        for rule in &config.rules {
            if !rule.temporary || !checkpoint.is_completed(&rule.name) {
                continue;
            }
            // #843: mid-run reclamation may already have tombstoned this
            // rule — re-deleting would double-unlink. Skip only while the
            // reclaimed outputs are genuinely gone; a tombstoned rule whose
            // outputs are back on disk (restored by hand) must be
            // re-deleted and its tombstone refreshed. Regeneration clears
            // the tombstone at execution time (mark_completed site).
            if checkpoint
                .tombstones
                .get(&rule.name)
                .is_some_and(|paths| paths.iter().all(|p| !workdir_actual.join(p).exists()))
            {
                continue;
            }
            let dependents = dag.dependents(&rule.name).unwrap_or_default();
            // Issue #844: output_pattern producers register no DAG edges by
            // design (dag.rs excludes pattern-claimed outputs from template
            // matching), so `dependents` alone misclassifies them as leaves.
            // Detect consumers structurally instead: compile the producer's
            // config-expanded output_pattern and match every other rule's
            // config-expanded input paths against it. Consumer instantiation
            // bakes instance inputs ("chunks/1.txt"), which match
            // `^chunks/(?P<i>\S+)\.txt$`; consumer TEMPLATES still carry a
            // `{` placeholder and are skipped so they cannot falsely match.
            let mut consumers: HashSet<String> = dependents.iter().cloned().collect();
            if let Some(pattern) = &rule.output_pattern {
                let expanded_pattern = oxo_flow_core::executor::checkpoint::expand_config_in_path(
                    pattern,
                    &wildcard_values,
                );
                if let Ok(re) = wildcard::pattern_to_regex(&expanded_pattern) {
                    for other in &config.rules {
                        if other.name == rule.name {
                            continue;
                        }
                        for input in other.input.iter() {
                            let expanded =
                                oxo_flow_core::executor::checkpoint::expand_config_in_path(
                                    input,
                                    &wildcard_values,
                                );
                            if !expanded.contains('{') && re.is_match(&expanded) {
                                consumers.insert(other.name.clone());
                                break;
                            }
                        }
                    }
                }
            }
            if consumers.is_empty() {
                eprintln!(
                    "  {} temporary rule '{}' is a leaf — outputs kept",
                    "ℹ".dimmed(),
                    rule.name
                );
                continue;
            }
            if !consumers.iter().all(|d| checkpoint.is_completed(d)) {
                let pending: Vec<&str> = consumers
                    .iter()
                    .filter(|d| !checkpoint.is_completed(d))
                    .map(String::as_str)
                    .collect();
                diagnostic_narrate(
                    format_args!(
                        "  {} temporary rule '{}' kept: dependents not complete yet ({})",
                        "!".yellow(),
                        rule.name,
                        pending.join(", ")
                    ),
                    Some(&run_log),
                );
                continue;
            }
            let mut deleted = Vec::new();
            // Output patterns that still contain `{` after config
            // expansion and resolved to no concrete instance path
            // (issue #844 diagnostic) — collected during the loop below
            // so a pattern that DID expand is never falsely reported.
            let mut skipped: Vec<String> = Vec::new();
            // Instance bindings recorded for this producer template by the
            // output_pattern discovery pass (issue #844): template → one
            // combo per completed instance. Empty for plain [[values]]/
            // [[pairs]] fan-outs (their instances run under their own names
            // and expand their outputs at plan time) and for legacy
            // checkpoints.
            let template = config.output_pattern_template_of(&rule.name);
            let domains = template
                .as_deref()
                .and_then(|t| checkpoint.output_pattern_domains.get(t))
                .cloned()
                .unwrap_or_default();
            // #844: an output_pattern producer has an EMPTY `output` list —
            // the pattern itself is the deletion target, expanded per
            // recorded instance combo in the loop below. Chain it in so the
            // same expansion logic covers both declaration styles.
            for output in rule.output.iter().chain(rule.output_pattern.iter()) {
                let expanded = oxo_flow_core::executor::checkpoint::expand_config_in_path(
                    output,
                    &wildcard_values,
                );
                if !expanded.contains('{') {
                    if oxo_flow_core::executor::output_invalidation::is_protected_path(
                        rule,
                        &wildcard_values,
                        &expanded,
                    ) {
                        // #843: protected outputs are user-declared
                        // keep-forever files — the same gate reclamation
                        // applies, so the sweep must not delete what the
                        // monitor wouldn't.
                        continue;
                    }
                    deleted.push(expanded);
                    continue;
                }
                // #844: a pattern-level path still containing `{` after
                // config expansion is a wildcard output (e.g. per-sample
                // BAMs). Delete only the concrete instance paths that
                // actually ran — expand every recorded combo, never the
                // pattern itself.
                let mut resolved_any = false;
                for combo in &domains {
                    let instance = wildcard::expand_pattern(&expanded, combo)
                        .unwrap_or_else(|_| expanded.clone());
                    if instance.contains('{') {
                        continue;
                    }
                    resolved_any = true;
                    if oxo_flow_core::executor::output_invalidation::is_protected_path(
                        rule,
                        &wildcard_values,
                        &instance,
                    ) {
                        continue;
                    }
                    if !deleted.contains(&instance) {
                        deleted.push(instance);
                    }
                }
                if !resolved_any {
                    // Expansion failure only — instances that resolved but
                    // matched `protected_output` are intentionally kept,
                    // not skipped.
                    if domains.is_empty() {
                        skipped.push(format!(
                            "{output} (no recorded instance bindings — legacy checkpoint or non-output_pattern fan-out; use `oxo-flow clean`)"
                        ));
                    } else {
                        skipped.push(format!(
                            "{expanded} (wildcard did not resolve against recorded combos)"
                        ));
                    }
                }
            }
            if !skipped.is_empty() {
                // Diagnostic (issue #844): a silent no-op is the worst
                // outcome — name the rule and the patterns that could not
                // be expanded to instance paths.
                diagnostic_narrate(
                    format_args!(
                        "  {} temporary wildcard outputs of '{}' left in place: instance paths could not be expanded: {}",
                        "!".yellow(),
                        rule.name,
                        skipped.join("; ")
                    ),
                    Some(&run_log),
                );
            }
            if !deleted.is_empty() {
                new_tombstones.push((rule.name.clone(), deleted));
            }
        }
        if !new_tombstones.is_empty() {
            // Same audit rule as chunk cleanup (issue #315 F2): persist
            // tombstones + the cleaned migration BEFORE deleting, and skip
            // the deletion when the persistence fails.
            let mut rollback: Vec<String> = Vec::new();
            for (rule, paths) in &new_tombstones {
                checkpoint.tombstones.insert(rule.clone(), paths.clone());
                for path in paths {
                    if let Some(sha) = checkpoint.checksums.remove(path) {
                        checkpoint.record_cleaned_checksum(path, sha);
                        rollback.push(path.clone());
                    }
                }
            }
            if let Err(e) = checkpoint.save_to_file(&checkpoint_path) {
                tracing::warn!(
                    error = %e,
                    "failed to persist tombstone migration — keeping temporary outputs"
                );
                for (rule, _) in &new_tombstones {
                    checkpoint.tombstones.remove(rule);
                }
                for path in &rollback {
                    if let Some(sha) = checkpoint.cleaned_checksums.remove(path) {
                        checkpoint.checksums.insert(path.clone(), sha);
                    }
                }
            } else {
                for (rule, paths) in &new_tombstones {
                    for expanded in paths {
                        let resolved = workdir_actual.join(expanded);
                        // Symlink-aware classification (review): a symlink is
                        // always unlinked as the link itself — never the
                        // bytes it points at.
                        let _ = match std::fs::symlink_metadata(&resolved) {
                            Ok(md) if md.file_type().is_symlink() => {
                                std::fs::remove_file(&resolved)
                            }
                            Ok(md) if md.is_dir() => std::fs::remove_dir_all(&resolved),
                            Ok(_) => std::fs::remove_file(&resolved),
                            Err(_) => Ok(()),
                        };
                    }
                    eprintln!(
                        "  {} temporary outputs deleted for '{}' ({} file(s), regenerated on demand)",
                        "⊘".dimmed(),
                        rule,
                        paths.len()
                    );
                }
            }
        }
    }

    // Pilot summary: after a --samples subset run, report what the pilot
    // implies for the full cohort (simple linear projection). With [ai]
    // enabled, append a plain-language interpretation.
    if !samples_filter.is_empty() {
        let pilot_count = oxo_flow_core::scientific_preflight::count_samples(&config);
        let total_count = WorkflowConfig::from_file(&workflow)
            .map(|c| oxo_flow_core::scientific_preflight::count_samples(&c))
            .unwrap_or(pilot_count);
        let elapsed = run_started.elapsed();
        let elapsed_secs = elapsed.as_secs_f64();
        let per_sample = if pilot_count > 0 {
            elapsed_secs / pilot_count as f64
        } else {
            0.0
        };
        let projected_secs = per_sample * total_count as f64;

        eprintln!("\n{}", "Pilot summary:".bold().cyan());
        eprintln!(
            "  Samples: {}/{} (pilot) | Wall time: {:.1}s | Per-sample: {:.1}s | Projected full run: ~{:.0}s",
            pilot_count, total_count, elapsed_secs, per_sample, projected_secs
        );
        let scientific =
            oxo_flow_core::scientific_preflight::analyze_scientific_constraints(&config);
        if !scientific.is_empty() {
            // Issue #441: in the run's post-expansion config each template
            // appears once per sample instance, so a 1-template finding
            // printed as "3 finding(s)" for a 3-sample pilot. Group by the
            // finding's content — identical (code, message, suggestion)
            // across instances is ONE distinct finding with M instances.
            type WarningGroup<'a> = (
                (&'a str, &'a str, &'a str),
                Vec<&'a oxo_flow_core::scientific_preflight::ScientificWarning>,
            );
            let mut distinct: Vec<WarningGroup> = Vec::new();
            for w in &scientific {
                let key = (w.code.as_str(), w.message.as_str(), w.suggestion.as_str());
                match distinct.iter_mut().find(|(k, _)| *k == key) {
                    Some((_, group)) => group.push(w),
                    None => distinct.push((key, vec![w])),
                }
            }
            eprintln!(
                "  {} distinct scientific preflight finding(s) ({} instance(s) across sample rules) — see 'oxo-flow dry-run --samples ...'",
                distinct.len(),
                scientific.len()
            );
            for (key, group) in &distinct {
                eprintln!("    ⚠ [{}] {} (×{})", key.0, group[0].rule, group.len());
            }
        }

        // AI interpretation when the workflow opts in via [ai].enabled.
        if let Some(provider) = crate::commands::ai_template::try_resolve_ai(Some(&workflow), false)
        {
            let warnings_text = scientific
                .iter()
                .map(|w| format!("- [{}] {}: {}", w.code, w.rule, w.message))
                .collect::<Vec<_>>()
                .join("\n");
            let prompt = format!(
                "A pilot run of {} sample(s) out of {} completed in {:.1}s ({} succeeded, {} failed).\n\
                 Per-sample wall time: {:.1}s; projected full run: ~{:.0}s.\n\
                 Scientific preflight findings:\n{}\n\n\
                 Write a short pilot report (3-5 sentences): whether the pilot results are\n\
                 healthy, whether scaling up is advisable, and what to fix first.",
                pilot_count,
                total_count,
                elapsed_secs,
                success_count,
                fail_count,
                per_sample,
                projected_secs,
                if warnings_text.is_empty() {
                    "none"
                } else {
                    &warnings_text
                }
            );
            match provider
                .chat(
                    "You are a bioinformatics workflow consultant summarizing a pilot run.",
                    &prompt,
                )
                .await
            {
                Ok(response) => {
                    eprintln!();
                    eprintln!("{}", "Pilot report (AI):".bold().green());
                    eprintln!("{response}");
                }
                Err(e) => eprintln!("  AI pilot report failed: {e}"),
            }
        }
    }

    // JSON output mode (issue #142 H6): the summary is emitted on EVERY
    // path — completed, failed, and aborted — so `--json` consumers can
    // rely on the document regardless of how the run ended.
    json_summary.emit(
        if fail_count > 0 {
            "failed"
        } else {
            "completed"
        },
        &RunCounts {
            succeeded: success_count,
            skipped: skipped_count,
            failed: fail_count,
            non_required_failed: non_required_fail_count,
            blocked: blocked.len(),
        },
        // `checkpoint` here is the MutexGuard locked above (line ~2415);
        // Deref coercion hands rule_resource_rows the underlying state.
        rule_resource_rows(&checkpoint),
    );

    // The verdict is independent of --keep-going (issue #133): keep-going
    // changes SCHEDULING (failures don't stop the run), never the verdict —
    // a run with required failures exits non-zero so scripts and the web
    // delegator (which classifies by exit code) see the truth.
    // `required = false` failures keep exit 0 (issue #99 B2).
    if fail_count > 0 {
        return Err(anyhow::anyhow!("workflow execution failed"));
    }

    Ok(())
}

/// Run the workflow-level terminal hook (`on_complete` after a fully
/// successful run, `on_error` after any failure) on every terminal path of
/// the run loop. Rendered with `{config.*}` plus the run counters and
/// `{workdir}`; best-effort by design — a failing hook warns, never
/// changes the run status (issue #227 item 1).
async fn run_terminal_hook(
    config: &WorkflowConfig,
    workdir: &std::path::Path,
    succeeded: usize,
    failed: usize,
    skipped: usize,
) {
    let hook_cmd = if failed > 0 {
        config.workflow.on_error.as_deref()
    } else {
        config.workflow.on_complete.as_deref()
    };
    let Some(hook_cmd) = hook_cmd else {
        return;
    };
    let mut values: std::collections::HashMap<String, String> = config
        .config
        .iter()
        .map(|(k, v)| (format!("config.{k}"), config_value_string(v)))
        .collect();
    values.insert("succeeded".to_string(), succeeded.to_string());
    values.insert("failed".to_string(), failed.to_string());
    values.insert("skipped".to_string(), skipped.to_string());
    values.insert("workdir".to_string(), workdir.display().to_string());
    let rendered = oxo_flow_core::executor::process::expand_placeholders(hook_cmd, &values);
    oxo_flow_core::executor::process::run_workflow_hook(
        &rendered,
        workdir,
        config.defaults.shell_prelude.as_deref(),
    )
    .await;
}

/// Workflow-level webhook notification (issue #227 item 1): posts the
/// `workflow_started` / `workflow_completed` / `workflow_failed` event to
/// the `[webhook]` endpoint. Best-effort by design — a failing endpoint
/// logs a warning and never changes the run status. Awaited (not spawned)
/// so the process cannot exit before the request lands; the client's own
/// timeout bounds the wait.
async fn notify_webhook(
    config: &WorkflowConfig,
    event: oxo_flow_core::webhook::WebhookEvent,
    counts: Option<(usize, usize, usize)>,
) {
    let Some(ref webhook) = config.webhook else {
        return;
    };
    let (succeeded, failed, skipped) = counts.unwrap_or_default();
    let payload = oxo_flow_core::webhook::WebhookPayload {
        event,
        workflow_name: config.workflow.name.clone(),
        timestamp: chrono::Utc::now().to_rfc3339(),
        data: oxo_flow_core::webhook::WebhookData {
            total_rules: None,
            succeeded: Some(succeeded),
            failed: Some(failed),
            skipped: Some(skipped),
            duration_ms: None,
            rule: None,
            exit_code: None,
            error: if failed > 0 {
                Some(format!("{failed} rule(s) failed"))
            } else {
                None
            },
        },
        version: env!("CARGO_PKG_VERSION").to_string(),
    };
    let client = oxo_flow_core::webhook::WebhookClient::new(webhook.clone());
    if let Err(e) = client.send(&payload).await {
        tracing::warn!(error = %e, event = %event, "webhook notification failed (run continues)");
    }
}

/// Terminal run counters shared by the `--json` summary and the workflow
/// terminal hooks. Grouped so the summary emitter stays under clippy's
/// argument-count lint.
#[derive(Default)]
struct RunCounts {
    succeeded: usize,
    skipped: usize,
    failed: usize,
    non_required_failed: usize,
    blocked: usize,
}

/// The `--json` summary for one `run`/`resume` invocation, emitted at most
/// once in the canonical `{"command":"run",...}` shape.
///
/// The document is documented as appearing on EVERY exit path
/// (docs/guide/src/commands/run.md): the explicit [`Self::emit`] calls carry
/// the terminal counts, and the `Drop` fallback covers the early aborts that
/// never reach one (workflow parse errors, preflight failures, repository
/// checkout errors) — those used to exit non-zero with zero bytes on stdout.
struct RunJsonSummary {
    json: bool,
    workflow: PathBuf,
    emitted: bool,
}

impl RunJsonSummary {
    fn new(json: bool, workflow: Option<&Path>) -> Self {
        Self {
            json,
            // Until the workflow is resolved the CLI argument is the best
            // name available; `set_workflow` replaces it with the resolved
            // path (a repository spec resolves to a checkout path).
            workflow: workflow.unwrap_or_else(|| Path::new("")).to_path_buf(),
            emitted: false,
        }
    }

    /// Point the summary at the resolved workflow file.
    fn set_workflow(&mut self, workflow: &Path) {
        self.workflow = workflow.to_path_buf();
    }

    fn emit(&mut self, status: &str, counts: &RunCounts, resources: Vec<serde_json::Value>) {
        if !self.json || self.emitted {
            return;
        }
        self.emitted = true;
        let output = serde_json::json!({
            "command": "run",
            "status": status,
            "workflow": self.workflow.to_string_lossy(),
            "results": serde_json::json!({
                "succeeded": counts.succeeded,
                "skipped": counts.skipped,
                "failed": counts.failed,
                "non_required_failed": counts.non_required_failed,
                "blocked": counts.blocked,
            }),
            "resources": resources,
        });
        println!("{}", serde_json::to_string_pretty(&output).unwrap());
    }
}

impl Drop for RunJsonSummary {
    fn drop(&mut self) {
        // Only the failure paths return without an explicit emission, so an
        // abort that never reached one still reports `status: "failed"`.
        self.emit("failed", &RunCounts::default(), vec![]);
    }
}

/// Per-rule resource rows for the `--json` summary (issue #163): the same
/// data the report's Benchmarks table shows — wall time, sampled peak RSS,
/// sampled CPU seconds, retries — keyed by rule, deterministic order.
/// Empty for aborted paths where no checkpoint summary is meaningful.
fn rule_resource_rows(
    checkpoint: &oxo_flow_core::executor::checkpoint::CheckpointState,
) -> Vec<serde_json::Value> {
    let mut names: Vec<&String> = checkpoint.benchmarks.keys().collect();
    names.sort_unstable();
    names
        .into_iter()
        .map(|name| {
            let b = &checkpoint.benchmarks[name];
            // Issue #690: a stale failed_rules entry with a recorded
            // when-false verdict was gated off, not failed.
            let status = if checkpoint.completed_rules.contains(name) {
                "completed"
            } else if checkpoint.failed_rules.contains(name) {
                if checkpoint.when_verdicts.get(name) == Some(&false) {
                    "skipped_by_when"
                } else {
                    "failed"
                }
            } else {
                "running"
            };
            serde_json::json!({
                "rule": name,
                "status": status,
                "wall_time_secs": b.wall_time_secs,
                // Synthetic up-to-date records (issue #324 F-1) carry a 0.0
                // placeholder wall time — JSON consumers can distinguish
                // them from real measurements via this marker.
                "recorded_as": b.recorded_as,
                "peak_rss_mb": b.max_memory_mb,
                "cpu_seconds": b.cpu_seconds,
                "retries": b.retries,
            })
        })
        .collect()
}

/// Stored reference state: the decision fingerprint plus the source's
/// content signature (size + full content hash for small files) at the
/// time it was recorded. The signature enables the path-migration
/// exemption — identical content at a new path skips the rebuild.
#[derive(serde::Serialize, serde::Deserialize, Clone, Default)]
struct RefRecord {
    fingerprint: String,
    content_sig: Option<String>,
    /// The source path (config-expanded, workdir-relative) this record was
    /// computed against — the exemption re-fingerprints with it to prove a
    /// fingerprint change is a PURE path migration.
    source_path: Option<String>,
}

/// Content signature of a reference SOURCE: `size:<bytes>` plus
/// `|hash:<sha256>` for files under the manifest hash cap. Directories
/// and large files return `None` — the path-migration exemption requires
/// the strong hash component, so anything weaker never skips a rebuild.
fn source_content_sig(path: &std::path::Path) -> Option<String> {
    let md = std::fs::metadata(path).ok()?;
    if !md.is_file() {
        return None;
    }
    let mut sig = format!("size:{}", md.len());
    if let Some(hash) = oxo_flow_core::executor::checkpoint::content_hash_if_small(path, &md) {
        sig.push_str(&format!("|hash:{hash}"));
    }
    Some(sig)
}

/// Free disk space in KiB on the filesystem holding `path`, read from
/// `df -Pk` (the portable POSIX form; parses the Available column of the
/// mount line). `None` when df is unavailable or the output is unparsable —
/// the pre-flight degrades to no check rather than blocking the run.
fn free_kilobytes(path: &std::path::Path) -> Option<u64> {
    let out = std::process::Command::new("df")
        .args(["-Pk", path.to_str()?])
        .output()
        .ok()?;
    if !out.status.success() {
        return None;
    }
    let stdout = String::from_utf8_lossy(&out.stdout);
    // The last line is the mount that actually holds the path.
    let line = stdout.lines().next_back()?;
    line.split_whitespace().nth(3)?.parse().ok()
}

/// Sorts the scheduler's ready list by effective priority: declared priority
/// plus rounds already waited (the aging counter of issue #123), ties broken
/// by name. Pure and unit-tested — the dispatch loop feeds it the live ready
/// list every round, so passed-over rules gain priority until submitted.
fn age_ready_list(
    mut ready: Vec<String>,
    order_set: &std::collections::HashSet<String>,
    submitted: &std::collections::HashSet<String>,
    priority_map: &std::collections::HashMap<String, i32>,
    waited: &std::collections::HashMap<String, i32>,
) -> Vec<String> {
    ready.retain(|name| order_set.contains(name) && !submitted.contains(name));
    ready.sort_by(|a, b| {
        let pa = priority_map.get(a).copied().unwrap_or(0) + waited.get(a).copied().unwrap_or(0);
        let pb = priority_map.get(b).copied().unwrap_or(0) + waited.get(b).copied().unwrap_or(0);
        pb.cmp(&pa).then_with(|| a.cmp(b))
    });
    ready
}

/// Suggest a `-j` (max concurrent jobs) value for the workflow on this
/// machine, evaluated **per parallel group** (issue #361).
///
/// For every wave: the wider of (wave width) and (cores ÷ heaviest rule in
/// that wave) is the number of jobs that wave can sustain. The suggestion
/// is the maximum across waves, so a light wide wave is never starved by
/// an unrelated heavy wave elsewhere in the DAG — the old global
/// `cores / max_threads_per_rule` formula measured 1.7–3.7× makespan loss
/// on mixed DAGs. The engine's resource pool already enforces declared
/// threads, so a generous suggestion is safe: this is a performance hint,
/// not an oversubscription gate.
fn suggest_jobs(config: &WorkflowConfig, groups: &[Vec<String>], system_threads: u32) -> u32 {
    let system_threads = system_threads.max(1);
    let wave_capacity = |names: &[String]| -> u32 {
        let max_threads = names
            .iter()
            .filter_map(|n| config.get_rule(n))
            .map(|r| r.effective_threads())
            .max()
            .unwrap_or(1)
            .max(1);
        (system_threads / max_threads)
            .min(names.len() as u32)
            .max(1)
    };
    groups
        .iter()
        .map(|g| wave_capacity(g))
        .max()
        .unwrap_or(1)
        .clamp(1, system_threads)
}

#[allow(clippy::too_many_arguments)]
pub async fn dry_run_command(
    workflow: Option<PathBuf>,
    target: Vec<String>,
    module: Vec<String>,
    verbose: bool,
    json: bool,
    ai: bool,
    ai_max_retries: Option<u32>,
    samples_filter: Vec<String>,
    workdir: Option<PathBuf>,
    profile: Option<String>,
    skip_ref_build: bool,
    _cache_dir: Option<PathBuf>,
    cli_args: Vec<String>,
    rerun: bool,
    resume_failed: bool,
) -> Result<()> {
    print_banner();
    let workflow = resolve_workflow(workflow)?;
    let workflow_dir = oxo_flow_core::parent_dir(&workflow).to_path_buf();
    // Path resolution base: --workdir wins, default is the workflow file's
    // directory — the same base the executor uses for run (issue #68).
    let base_dir = workdir.as_deref().unwrap_or(&workflow_dir);

    // Issue #751: same discovery fallback as run — bundle extractions live
    // under the temp dir, and --workdir may hold the data. The fallback is
    // a no-op unless the two roots differ.
    let discovery_fallback =
        if workflow.starts_with(std::env::temp_dir()) && base_dir != workflow_dir {
            Some(base_dir)
        } else {
            None
        };
    let mut config =
        WorkflowConfig::from_file_with_discovery_fallback(&workflow, discovery_fallback)
            .with_context(|| format!("failed to parse {}", workflow.display()))?;

    // ── CLI config overrides (shared with run, issue #77) ─────
    // Applied in run's exact order: overrides first, then
    // the --samples subset filter — so the preview config is structurally
    // identical to what `run` would execute. The workflow's own [config]
    // keys gate the `--KEY VALUE` space form (same rule as run, issue #71).
    let declared_config_keys: std::collections::HashSet<String> =
        config.config.keys().cloned().collect();
    let cli_arg_values = parse_cli_overrides(cli_args, &declared_config_keys, "dry-run")?;
    apply_cli_overrides(&mut config, &cli_arg_values)?;
    // Same typo guard as `run` (shared override machinery).
    warn_unknown_override_keys(&cli_arg_values, &declared_config_keys, &workflow);

    // ── Filter to a sample subset (--samples first:N / names / ready) ──
    // `ready` resolves against a scratch expanded clone; the report covers
    // the full cohort so the readiness section stays informative even when
    // the listing is filtered (issue #63).
    let ready_report = if !samples_filter.is_empty() {
        samples::apply_samples_filter(&mut config, &samples_filter, false, base_dir)?
    } else {
        None
    };

    // ── Execution profile (shared with run) ──────────────────────────────
    // The SAME merge helper `run` uses: preview and execution can never
    // drift apart on profile semantics.
    if let Some(ref profile_name) = profile {
        crate::commands::run_preview::merge_profile(&mut config, profile_name, &workflow_dir)?;
    }

    // ── Scientific preflight (deterministic, evidence-backed) ────────────
    // Printed for every dry-run; with --ai the findings are also passed to
    // the model for a plain-language explanation.
    let scientific = oxo_flow_core::scientific_preflight::analyze_scientific_constraints(&config);
    let scientific_context = if scientific.is_empty() {
        String::new()
    } else {
        let mut text = String::new();
        for w in &scientific {
            text.push_str(&format!(
                "- [{}] {}: {}\n  Fix: {}\n",
                w.code, w.rule, w.message, w.suggestion
            ));
        }
        text
    };

    // AI: auto-detect from workflow [ai] or explicit --ai flag. Suppressed
    // under --json (#539): same stdout-contract reasoning as validate/lint.
    if !json
        && let Some(provider) = crate::commands::ai_template::try_resolve_ai(Some(&workflow), ai)
    {
        // `--ai-max-retries` bounds the attempts, same contract as `run`'s
        // recovery (only a FAILED analysis call is retried).
        let attempts = ai_attempts(ai_max_retries);
        let mut result = Err(anyhow::anyhow!("AI analysis did not run"));
        for attempt in 1..=attempts {
            result = crate::commands::ai_check::analyze_workflow(
                &workflow,
                &provider,
                "dry-run",
                &scientific_context,
            )
            .await;
            if result.is_ok() || attempt == attempts {
                break;
            }
            if let Err(e) = &result {
                eprintln!("  AI analysis attempt {attempt}/{attempts} failed: {e} — retrying");
            }
        }
        result?;
        println!();
    }

    if !scientific.is_empty() {
        eprintln!("\n{}", "Scientific preflight:".bold().yellow());
        for w in &scientific {
            eprintln!("  ⚠ [{}] {}: {}", w.code, w.rule, w.message);
            eprintln!("    {} {}", "→".bold(), w.suggestion);
        }
    }

    config.apply_defaults();
    config
        .expand_wildcards()
        .context("failed to expand wildcard rules")?;

    // ── Undefined `{config.*}` gate (issue #142 H1) — the same gate run
    // applies: preview must refuse a typo'd key exactly like execution.
    let e005 = undefined_config_findings(&config);
    if !e005.is_empty() {
        return Err(anyhow::anyhow!(
            "workflow references undefined config variable(s) — fix before running:\n  {}",
            e005.join("\n  ")
        ));
    }

    let mut dag = WorkflowDag::from_rules_with_config(
        &config.rules,
        &config_placeholder_values(&config.config),
    )
    .context("failed to build workflow DAG")?;

    // --module partial runs (issue #112 elasticity) — same resolution as
    // `run`.
    let mut target = target;
    for m in &module {
        match config.module_closure(m) {
            Some(names) => target.extend(names),
            None => {
                return Err(anyhow::anyhow!(
                    "unknown module '{m}' — known modules: {}",
                    known_modules_hint(&config.module_rules)
                ));
            }
        }
    }
    // Default targets (issue #469): same seeding as `run`.
    let target = resolve_default_targets(&config, &target);
    // Targeted dry-runs close over the INSTANTIATED DAG (issue #247) —
    // the same when-gate filter `run` applies, so the preview cannot list
    // instances that would never execute.
    let dry_wildcard_values: HashMap<String, String> = config_placeholder_values(&config.config);
    let when_false_rules: std::collections::HashSet<String> = if target.is_empty() {
        std::collections::HashSet::new()
    } else {
        config
            .rules
            .iter()
            .filter(|rule| {
                crate::commands::run_preview::when_condition_false(
                    rule,
                    &config,
                    &dry_wildcard_values,
                )
            })
            .map(|rule| rule.name.clone())
            .collect()
    };

    // Compute the execution set (respects --target), then display it in
    // parallel-group order — the same grouping `run` uses for scheduling.
    // Independent rules at the same level appear adjacent, so the listing
    // reflects actual concurrency instead of an arbitrary topological order.
    let mut order = if target.is_empty() {
        let order_set: std::collections::HashSet<String> =
            dag.execution_order()?.into_iter().collect();
        let mut ordered: Vec<String> = Vec::new();
        for group in dag.parallel_groups()? {
            let mut level: Vec<String> = group
                .into_iter()
                .filter(|name| order_set.contains(name))
                .collect();
            level.sort();
            ordered.extend(level);
        }
        ordered
    } else {
        let target_refs: Vec<&str> = target.iter().map(String::as_str).collect();
        let (filtered, skipped_targets) = dag
            .execution_order_for_targets_skipping_with_source_check(
                &target_refs,
                &when_false_rules,
                // Pre-built inputs (atacseq bwa_mem finding): see the main
                // target-closure call — existence in the dry-run base dir
                // keeps the consumer plannable.
                &|path| run_workdir_check(Some(base_dir), path),
            )
            .with_context(|| "failed to resolve target rules")?;
        for skipped in &skipped_targets {
            print_target_skipped_note(skipped, &when_false_rules, None);
        }
        filtered
    };

    // All config values (including CLI --arg overrides) become {config.key} in templates.
    let mut wildcard_values: HashMap<String, String> = HashMap::new();
    for (key, value) in &config.config {
        let string_val = match value {
            toml::Value::String(s) => s.clone(),
            other => other.to_string(),
        };
        wildcard_values.insert(format!("config.{key}"), string_val);
    }

    // ── Checkpoint-aware rerun preview (issue #66) ────────────────────────
    // Read-only: the preview classifies every rule in the execution set
    // exactly as `run` would — invalidation sources, downstream cascade,
    // and the protected remainder — without touching the checkpoint on disk.
    let checkpoint_path = base_dir.join(".oxo-flow/checkpoint.json");
    // The preview is ALWAYS computed — with an empty state when no
    // checkpoint exists. `when` conditions can skip rules even on a fresh
    // run, so "every rule will execute" would be wrong without it.
    let mut checkpoint_state =
        match oxo_flow_core::executor::checkpoint::CheckpointState::load_from_file(&checkpoint_path)
        {
            Ok(ck) => ck,
            Err(_) => {
                eprintln!(
                    "{} no checkpoint at {} — treating every rule as never completed",
                    "⚠".yellow(),
                    checkpoint_path.display()
                );
                oxo_flow_core::executor::checkpoint::CheckpointState::default()
            }
        };
    // Same identity gate `run` applies (audit B8): another workflow's
    // artifacts must not be reported as up to date in the plan either.
    adopt_checkpoint_for_workflow(&mut checkpoint_state, &workflow, None);
    let sensitive_keys: std::collections::HashSet<String> = config
        .config_meta
        .iter()
        .filter(|(_, def)| def.sensitive)
        .map(|(key, _)| key.clone())
        .collect();
    // The verbose plan print must not leak sensitive values (issue #99 B1).
    let sensitive_values = sensitive_values_of(&config);
    let interpreter_map = config.workflow.interpreter_map.clone();
    let mut preview = crate::commands::run_preview::preview_run_plan(
        &checkpoint_state,
        &config,
        &dag,
        &order,
        base_dir,
        &wildcard_values,
        &sensitive_keys,
        &interpreter_map,
        &checkpoint_path,
        rerun,
        resume_failed,
    );

    // ── Checkpoint re-entry replay (issue #78 P3) ────────────────────────
    // The round-0 preview determines which checkpoint rules are up-to-date
    // (Skipped); their recorded re-entries replay and the preview recomputes
    // on the re-expanded plan — the same static plan a run would execute.
    if !checkpoint_state.reentries.is_empty() {
        use crate::commands::run_preview::RuleStatus;
        let valid: std::collections::HashSet<String> = preview
            .plan
            .iter()
            .filter(|p| p.status == RuleStatus::Skipped)
            .map(|p| p.name.clone())
            .collect();
        let replayed = oxo_flow_core::reentry::replay_valid_reentries(
            &mut config,
            &checkpoint_state.reentries,
            &valid,
        )?;
        if !replayed.is_empty() {
            tracing::info!(
                count = replayed.len(),
                "dry-run replayed checkpoint re-entries"
            );
            dag = WorkflowDag::from_rules_with_config(
                &config.rules,
                &config_placeholder_values(&config.config),
            )
            .context("failed to rebuild workflow DAG after re-entry replay")?;
            let new_order: Vec<String> = if target.is_empty() {
                let order_set: std::collections::HashSet<String> =
                    dag.execution_order()?.into_iter().collect();
                let mut ordered: Vec<String> = Vec::new();
                for group in dag.parallel_groups()? {
                    let mut level: Vec<String> = group
                        .into_iter()
                        .filter(|n| order_set.contains(n))
                        .collect();
                    level.sort();
                    ordered.extend(level);
                }
                ordered
            } else {
                let target_refs: Vec<&str> = target.iter().map(String::as_str).collect();
                let (filtered, _) = dag
                    .execution_order_for_targets_skipping_with_source_check(
                        &target_refs,
                        &when_false_rules,
                        &|path| run_workdir_check(Some(base_dir), path),
                    )
                    .with_context(|| "failed to resolve target rules")?;
                filtered
            };
            let old_order_len = order.len();
            order = new_order;
            if order.len() != old_order_len {
                eprintln!(
                    "  {} re-entry replay: plan grew from {} to {} rules",
                    "Replay:".bold().cyan(),
                    old_order_len,
                    order.len()
                );
            }
            preview = crate::commands::run_preview::preview_run_plan(
                &checkpoint_state,
                &config,
                &dag,
                &order,
                base_dir,
                &wildcard_values,
                &sensitive_keys,
                &interpreter_map,
                &checkpoint_path,
                rerun,
                resume_failed,
            );
        }
    }
    // Pixi-manifest preflight (issue #848): mirror run's pending_envs —
    // every non-skip rule's env spec, resolved against the workflow dir.
    // plan/dry-run previously passed while a declared manifest was missing
    // and only the run failed after the plan was already committed to.
    let env_manifest_issues: Vec<(String, String)> = {
        let mut issues = Vec::new();
        for entry in preview.plan.iter().filter(|entry| !entry.status.is_skip()) {
            let Some(rule) = config.get_rule(&entry.name) else {
                continue;
            };
            let resolved = rule.environment.resolve_relative_paths(Some(&workflow_dir));
            if let Some(spec) = resolved.pixi
                && oxo_flow_core::environment::validate_pixi_manifest(&spec).is_err()
            {
                issues.push((entry.name.clone(), spec));
            }
        }
        issues
    };
    {
        let p = &preview;
        let modified = p
            .checkpoint_modified
            .map(|t| {
                chrono::DateTime::<chrono::Local>::from(t)
                    .format("%Y-%m-%d %H:%M:%S")
                    .to_string()
            })
            .unwrap_or_else(|| "unknown time".to_string());
        eprintln!(
            "{} {} (modified {})",
            "Checkpoint:".bold().cyan(),
            p.checkpoint_path.display(),
            modified.dimmed()
        );
        // Issue #432(b): the headline must answer "what happens next", not
        // "how big is the DAG". `order.len()` is DAG size — shown, but
        // labelled as such and secondary to the run/skip/completed split.
        let will_run = p.plan.len() - p.will_skip;
        eprintln!(
            "{} would run: {} | skip: {} | completed: {} (DAG size: {})",
            "Plan:".bold().yellow(),
            will_run.to_string().green(),
            p.will_skip.to_string().dimmed(),
            p.completed_total.to_string().cyan(),
            order.len()
        );
        for chain in &p.cascade_chains {
            eprintln!("  {} {}", "rerun cascade:".yellow(), chain.join(" → "));
        }
    }

    // ── Reference auto-builds (mirrors run's --skip-ref-build semantics) ──
    // run builds any reference whose output is missing before scheduling;
    // the preview surfaces that cost instead of hiding it.
    let reference_builds: Vec<String> = if skip_ref_build {
        Vec::new()
    } else {
        config
            .references
            .iter()
            .filter(|r| !base_dir.join(&r.output).exists())
            .map(|r| r.name.clone())
            .collect()
    };
    if !reference_builds.is_empty() {
        eprintln!(
            "{} {} reference build(s) would run: {}",
            "References:".bold().cyan(),
            reference_builds.len(),
            reference_builds.join(", ")
        );
        eprintln!(
            "  {} pass --skip-ref-build to assume they are pre-built.",
            "ℹ".dimmed()
        );
    }

    // Sample readiness (issue #63). With `--samples ready` the report covers
    // the full cohort (computed pre-filter); otherwise it is computed on the
    // (possibly filtered) expanded config.
    let readiness_report = match ready_report {
        Some(report) => Some(report),
        None => Some(oxo_flow_core::readiness::compute_readiness(
            &config, base_dir,
        )),
    };
    if let Some(report) = &readiness_report {
        samples::print_readiness_section(report);
    }

    for (i, rule_name) in order.iter().enumerate() {
        let rule = config
            .get_rule(rule_name)
            .ok_or_else(|| anyhow::anyhow!("rule '{}' not found", rule_name))?;
        let status_text = preview
            .plan
            .iter()
            .find(|r| r.name == *rule_name)
            .map(|r| match &r.status {
                crate::commands::run_preview::RuleStatus::NeverCompleted => {
                    format!("{}", "[run: never completed]".bold())
                }
                crate::commands::run_preview::RuleStatus::ConfigInvalidated => {
                    format!("{}", "[run: config changed]".bold())
                }
                crate::commands::run_preview::RuleStatus::InputInvalidated => {
                    format!("{}", "[run: input changed]".bold().yellow())
                }
                crate::commands::run_preview::RuleStatus::OutputsMissing => {
                    format!("{}", "[run: outputs missing]".bold().yellow())
                }
                crate::commands::run_preview::RuleStatus::Cascaded { from } => {
                    format!(
                        "{}",
                        format!("[rerun: downstream of {from}]").bold().yellow()
                    )
                }
                crate::commands::run_preview::RuleStatus::Skipped => {
                    format!("{}", "[skip: up to date]".dimmed())
                }
                crate::commands::run_preview::RuleStatus::SkippedByWhen => {
                    format!("{}", "[skip: when condition false]".dimmed())
                }
                crate::commands::run_preview::RuleStatus::CascadedUpstream { from } => {
                    format!("{}", format!("[rerun: upstream of {from}]").bold().yellow())
                }
                crate::commands::run_preview::RuleStatus::Forced => {
                    format!("{}", "[run: forced (--rerun)]".bold())
                }
                crate::commands::run_preview::RuleStatus::SkippedFresh => {
                    format!("{}", "[skip: outputs up-to-date]".dimmed())
                }
            })
            .unwrap_or_default();
        eprintln!("  {}. {}  {}", i + 1, rule_name.bold().cyan(), status_text);

        let threads = rule.effective_threads();
        eprintln!("     threads={}", threads);

        if !rule.environment.is_empty() {
            eprintln!("     env={}", rule.environment.kind());
        }

        if let Some(ref mem) = rule.effective_memory() {
            eprintln!("     memory={}", mem);
        }

        if rule.checkpoint {
            eprintln!("     checkpoint=true");
        }

        if !rule.output.is_empty() {
            let expanded_outputs: Vec<String> = rule
                .output
                .iter()
                .map(|o| {
                    oxo_flow_core::executor::checkpoint::expand_config_in_path(o, &wildcard_values)
                })
                .collect();
            eprintln!("     outputs: {:?}", expanded_outputs);
        }

        if let Some(ref cmd) = rule.shell {
            let expanded = oxo_flow_core::executor::process::render_shell_command(
                cmd,
                rule,
                &wildcard_values,
                oxo_flow_core::scheduler::detect_system_limits(),
            );
            // Mask sensitive values (issue #99 B1): the plan print is a log
            // surface like any other.
            let expanded =
                oxo_flow_core::executor::process::mask_sensitive(&expanded, &sensitive_values);
            // Workdir-aware blocking preview (issue #428): E011 only applies
            // when the deletion target lies outside the run workdir, so an
            // absolute {config.out_dir} cleanup is not labeled "blocked".
            let blocked_label = if let Some((_category, description)) =
                oxo_flow_core::format::shell_blocking_pattern(&expanded)
            {
                match oxo_flow_core::executor::validate_shell_safety_in_workdir(&expanded, base_dir)
                {
                    Err(e) => Some(e.to_string()),
                    Ok(()) => None,
                }
                .map(|_| format!("(blocked: E011 — {description})"))
            } else {
                None
            };
            if let Some(label) = blocked_label {
                eprintln!("     command: {}  {}", expanded, label.red().bold());
            } else {
                eprintln!("     command: {}", expanded);
            }
        }

        // Show input file status for concrete (non-wildcard) paths.
        // Relative paths resolve against the workflow's directory — the same
        // place the executor runs shells from and readiness checks (issue #63
        // follow-up: CWD-based checks contradicted the readiness report when
        // invoked from another directory).
        for inp in &rule.input {
            let s = inp.to_string();
            if !s.contains('{') && !s.contains('*') && !s.starts_with('/') {
                let exists = base_dir.join(&s).exists();
                let icon = if exists { "✓" } else { "✗" };
                eprintln!("     input {}: {}", icon, s);
            }
        }

        if verbose {
            // Additional verbose info
        }
    }

    // Fail-fast pixi-manifest block (issue #848): printed after the rule
    // listing so the plan context stays visible, before the JSON document.
    if !env_manifest_issues.is_empty() {
        eprintln!(
            "\n  {} {} pixi manifest(s) declared by pending rule(s) do not exist:",
            "✗".red().bold(),
            env_manifest_issues.len()
        );
        for (rule, spec) in &env_manifest_issues {
            eprintln!("    - {rule}: {spec} (resolved relative to the workflow directory)");
        }
        eprintln!(
            "  {} create the manifest at the resolved path or correct the rule's `pixi` path",
            "hint:".yellow().bold()
        );
    }

    // Resource summary
    let total_threads: u32 = config.rules.iter().map(|r| r.effective_threads()).sum();
    let max_threads: u32 = config
        .rules
        .iter()
        .map(|r| r.effective_threads())
        .max()
        .unwrap_or(1);
    let memory_values: Vec<&str> = config
        .rules
        .iter()
        .filter_map(|r| r.effective_memory())
        .collect();
    eprintln!();
    eprintln!(
        "{} {} rules, total {} threads declared, max {} threads/rule",
        "Summary:".bold(),
        order.len(),
        total_threads,
        max_threads,
    );
    if !memory_values.is_empty() {
        eprintln!(
            "         {} rule(s) with memory requirements",
            memory_values.len()
        );
    }
    if !config.sample_groups.is_empty() {
        eprintln!(
            "         {} sample group(s), {} pair(s)",
            config.sample_groups.len(),
            config.pairs.len()
        );
    }

    // Suggest -j per parallel group (issue #361): each wave is evaluated
    // against its own heaviest rule, so a light wide wave is not starved
    // by an unrelated heavy wave elsewhere in the DAG. The engine's
    // resource pool enforces declared threads, so this is a performance
    // hint, not an oversubscription gate.
    let system_threads = std::thread::available_parallelism()
        .map(|n| n.get() as u32)
        .unwrap_or(4);
    let groups = dag.parallel_groups().unwrap_or_default();
    let suggested_jobs = suggest_jobs(&config, &groups, system_threads).to_string();
    eprintln!(
        "\n{}  oxo-flow run {} -j {}",
        "To execute:".bold().cyan(),
        workflow.display(),
        suggested_jobs
    );

    // JSON output mode
    if json {
        let order_list = order.clone();
        let rule_list: Vec<serde_json::Value> = order
            .iter()
            .filter_map(|name| config.get_rule(name))
            .map(|r| {
                serde_json::json!({
                    "name": r.name,
                    "threads": r.effective_threads(),
                    "environment": r.environment.kind(),
                    "memory": r.effective_memory(),
                    "checkpoint": r.checkpoint,
                })
            })
            .collect();

        // Machine-readable sample readiness (issue #63).
        let samples_block = readiness_report.as_ref().map(|report| {
            let ready_names: Vec<&str> = report.ready.iter().map(|s| s.name.as_str()).collect();
            let waiting: Vec<serde_json::Value> = report
                .waiting
                .iter()
                .map(|s| {
                    serde_json::json!({
                        "name": s.name,
                        "missing": s.missing,
                    })
                })
                .collect();
            serde_json::json!({
                "total": report.total,
                "ready": report.ready.len(),
                "waiting_count": report.waiting.len(),
                "ready_names": ready_names,
                "waiting": waiting,
                "missing_global": report.missing_global,
            })
        });

        // Machine-readable checkpoint preview (issue #66): the predicted
        // execution plan with per-rule status + reason.
        let checkpoint_block = Some(&preview).map(|p| {
            let plan: Vec<serde_json::Value> = p
                .plan
                .iter()
                .map(|r| {
                    let (status, cascaded_from) = match &r.status {
                        crate::commands::run_preview::RuleStatus::NeverCompleted => {
                            ("run-never-completed", None)
                        }
                        crate::commands::run_preview::RuleStatus::ConfigInvalidated => {
                            ("run-config-changed", None)
                        }
                        crate::commands::run_preview::RuleStatus::InputInvalidated => {
                            ("run-input-changed", None)
                        }
                        crate::commands::run_preview::RuleStatus::OutputsMissing => {
                            ("run-outputs-missing", None)
                        }
                        crate::commands::run_preview::RuleStatus::Cascaded { from } => {
                            ("run-cascaded", Some(from.clone()))
                        }
                        crate::commands::run_preview::RuleStatus::Skipped => ("skip", None),
                        crate::commands::run_preview::RuleStatus::SkippedByWhen => {
                            ("skip-when-condition", None)
                        }
                        crate::commands::run_preview::RuleStatus::CascadedUpstream { from } => {
                            ("run-cascaded-upstream", Some(from.clone()))
                        }
                        crate::commands::run_preview::RuleStatus::Forced => ("run-forced", None),
                        crate::commands::run_preview::RuleStatus::SkippedFresh => {
                            ("skip-fresh", None)
                        }
                    };
                    serde_json::json!({
                        "name": r.name,
                        "status": status,
                        "cascaded_from": cascaded_from,
                    })
                })
                .collect();
            serde_json::json!({
                "path": p.checkpoint_path.display().to_string(),
                "modified": p.checkpoint_modified.map(|t| {
                    chrono::DateTime::<chrono::Local>::from(t)
                        .format("%Y-%m-%d %H:%M:%S")
                        .to_string()
                }),
                "completed_total": p.completed_total,
                "summary": {
                    "will_run": p.plan.len() - p.will_skip,
                    "will_skip": p.will_skip,
                    "protected_outside": p.protected_outside,
                },
                "plan": plan,
                "cascade_chains": p.cascade_chains,
            })
        });

        // Stable per-rule plan (schema_version 1 — the first release to
        // ship this surface): status is the same bracket word the human
        // listing prints ([run:/[skip:/[rerun:), reason is the human
        // status text after the colon — so the JSON is a 1:1 structured
        // mirror of the stderr plan the ecosystem greps.
        let plan_entries: Vec<serde_json::Value> = order
            .iter()
            .filter_map(|name| {
                let rule = config.get_rule(name)?;
                let (status, reason, cascaded_from) = match preview
                    .plan
                    .iter()
                    .find(|r| r.name == *name)
                    .map(|r| &r.status)
                {
                    Some(crate::commands::run_preview::RuleStatus::NeverCompleted) => {
                        ("run", "never completed".to_string(), None)
                    }
                    Some(crate::commands::run_preview::RuleStatus::ConfigInvalidated) => {
                        ("run", "config changed".to_string(), None)
                    }
                    Some(crate::commands::run_preview::RuleStatus::InputInvalidated) => {
                        ("run", "input changed".to_string(), None)
                    }
                    Some(crate::commands::run_preview::RuleStatus::OutputsMissing) => {
                        ("run", "outputs missing".to_string(), None)
                    }
                    Some(crate::commands::run_preview::RuleStatus::Cascaded { from }) => (
                        "rerun",
                        format!("downstream of {from}"),
                        Some(from.clone()),
                    ),
                    Some(crate::commands::run_preview::RuleStatus::Skipped) => {
                        ("skip", "up to date".to_string(), None)
                    }
                    Some(crate::commands::run_preview::RuleStatus::SkippedByWhen) => {
                        ("skip", "when condition false".to_string(), None)
                    }
                    Some(crate::commands::run_preview::RuleStatus::CascadedUpstream { from }) => (
                        "rerun",
                        format!("upstream of {from}"),
                        Some(from.clone()),
                    ),
                    Some(crate::commands::run_preview::RuleStatus::Forced) => {
                        ("run", "forced (--rerun)".to_string(), None)
                    }
                    Some(crate::commands::run_preview::RuleStatus::SkippedFresh) => {
                        ("skip", "outputs up-to-date".to_string(), None)
                    }
                    // The preview classifies every rule in the execution
                    // set, so this branch is unreachable; keep it loud.
                    None => ("unknown", "unknown".to_string(), None),
                };
                let expanded_shell = rule.shell.as_ref().map(|cmd| {
                    oxo_flow_core::executor::process::render_shell_command(
                        cmd,
                        rule,
                        &wildcard_values,
                        oxo_flow_core::scheduler::detect_system_limits(),
                    )
                });
                // Declared patterns stay raw in inputs/outputs (the stable
                // contract); the expanded variants give consumers the exact
                // per-instance paths the engine will touch — the same
                // {config.x} expansion the human stderr listing shows.
                let inputs_expanded: Vec<String> = rule
                    .input
                    .to_vec()
                    .iter()
                    .map(|p| {
                        oxo_flow_core::executor::checkpoint::expand_config_in_path(
                            p,
                            &wildcard_values,
                        )
                    })
                    .collect();
                let outputs_expanded: Vec<String> = rule
                    .output
                    .to_vec()
                    .iter()
                    .map(|p| {
                        oxo_flow_core::executor::checkpoint::expand_config_in_path(
                            p,
                            &wildcard_values,
                        )
                    })
                    .collect();
                Some(serde_json::json!({
                    "name": rule.name,
                    "status": status,
                    "reason": reason,
                    "cascaded_from": cascaded_from,
                    "threads": rule.effective_threads(),
                    "memory": rule.effective_memory(),
                    "environment": rule.environment.kind(),
                    "command": expanded_shell,
                    "inputs": serde_json::to_value(&rule.input).unwrap_or(serde_json::Value::Null),
                    "outputs": serde_json::to_value(&rule.output).unwrap_or(serde_json::Value::Null),
                    "inputs_expanded": inputs_expanded,
                    "outputs_expanded": outputs_expanded,
                }))
            })
            .collect();

        let output = serde_json::json!({
            "command": "dry-run",
            "schema_version": 1,
            "workflow": workflow.display().to_string(),
            "total_rules": order.len(),
            "execution_order": order_list,
            "rules": rule_list,
            "plan": plan_entries,
            "summary": {
                "total_threads": total_threads,
                "max_threads_per_rule": max_threads,
                "memory_rules": memory_values.len(),
                "would_execute": preview.plan.len() - preview.will_skip,
                "will_skip": preview.will_skip,
                "total_rules": order.len(),
            },
            "sample_groups": config
                .sample_groups
                .iter()
                .map(|g| {
                    serde_json::json!({
                        "name": g.name,
                        "samples": g.samples,
                    })
                })
                .collect::<Vec<_>>(),
            "pairs": config
                .pairs
                .iter()
                .map(|p| {
                    serde_json::json!({
                        "pair_id": p.pair_id,
                        "experiment": p.experiment,
                        "control": p.control,
                    })
                })
                .collect::<Vec<_>>(),
            "suggested_jobs": suggested_jobs,
            "samples": samples_block,
            "checkpoint_preview": checkpoint_block,
            "reentry": {
                // Recorded re-entries (checkpoint.json) — replayed when their
                // checkpoint rule is up-to-date; revoked otherwise (issue #78 P3).
                "recorded": checkpoint_state.reentries,
                // Checkpoint rules that may add instances at runtime.
                "possible": config
                    .rules
                    .iter()
                    .filter(|r| r.checkpoint)
                    .map(|r| serde_json::json!({
                        "rule": r.name,
                        "manifest": r.checkpoint_manifest,
                    }))
                    .collect::<Vec<_>>(),
            },
            "profile": profile,
            "reference_builds": reference_builds,
            // Issue #848: declared pixi manifests that do not exist (resolved
            // against the workflow directory). Empty unless a pending rule's
            // manifest is missing — the command still exits non-zero in that
            // case after the document is printed.
            "env_manifest_issues": env_manifest_issues
                .iter()
                .map(|(rule, spec)| serde_json::json!({"rule": rule, "spec": spec}))
                .collect::<Vec<_>>(),
        });
        println!("{}", serde_json::to_string_pretty(&output).unwrap());
    }

    // Bail AFTER the JSON document so callers still get machine-readable
    // output on this failure path (same contract as `validate --json`).
    if !env_manifest_issues.is_empty() {
        anyhow::bail!(
            "{} pixi manifest(s) declared by pending rule(s) do not exist — \
             create the manifest at the resolved path or correct the rule's \
             `pixi` path (resolved relative to the workflow directory)",
            env_manifest_issues.len()
        );
    }

    Ok(())
}

pub async fn debug_command(workflow: PathBuf, rule_name: Option<String>, ai: bool) -> Result<()> {
    print_banner();
    let mut config = WorkflowConfig::from_file(&workflow)
        .with_context(|| format!("failed to parse {}", workflow.display()))?;

    config.apply_defaults();
    config
        .expand_wildcards()
        .context("failed to expand wildcard rules")?;

    // AI: auto-detect from workflow [ai] or explicit --ai flag
    if let Some(provider) = crate::commands::ai_template::try_resolve_ai(Some(&workflow), ai) {
        // Build wildcard values from config so template variables expand
        let mut debug_wildcards = std::collections::HashMap::new();
        for (key, value) in &config.config {
            let string_val = match value {
                toml::Value::String(s) => s.clone(),
                other => other.to_string(),
            };
            debug_wildcards.insert(format!("config.{key}"), string_val);
        }
        let expanded: Vec<String> = config
            .rules
            .iter()
            .map(|r| {
                let shell_cmd = r.shell.as_deref().unwrap_or("(no shell command)");
                let shell = oxo_flow_core::executor::process::render_shell_command(
                    shell_cmd,
                    r,
                    &debug_wildcards,
                    oxo_flow_core::scheduler::detect_system_limits(),
                );
                format!(
                    "## {}\nthreads={}, memory={}, env={:?}\n```bash\n{}\n```",
                    r.name,
                    r.resources.threads,
                    r.resources.memory.as_deref().unwrap_or("1G"),
                    r.environment,
                    shell
                )
            })
            .collect();
        let joined = expanded.join("\n\n");

        let system = r#"## Role
You are a bioinformatics pipeline debugger. Explain each expanded shell command in plain language,
flagging potential issues: parameter misuse, resource mismatch, missing required flags, or
logical errors. Output format per rule:

**rule_name**: <what this command does — 1 sentence>
⚠ <any issues found — 1 line per issue, or "No issues" if clean>
"#
        .to_string();
        let user = format!(
            "## Workflow: {}\n\n## Expanded Commands\n\n{joined}\n\n## Task\nExplain each command and flag issues.",
            workflow.display()
        );

        println!("{}", "  Analyzing commands...".bold().cyan());
        match provider.chat(&system, &user).await {
            Ok(response) => {
                println!("\n{}\n{response}", "AI Analysis".bold().green().underline());
                return Ok(());
            }
            Err(e) => eprintln!("  AI analysis failed: {e}"),
        }
    }

    let dag = WorkflowDag::from_rules_with_config(
        &config.rules,
        &config_placeholder_values(&config.config),
    )
    .context("failed to build workflow DAG")?;

    let rules_to_show: Vec<&oxo_flow_core::rule::Rule> = if let Some(ref name) = rule_name {
        match config.rules.iter().find(|r| r.name == *name) {
            Some(r) => vec![r],
            None => {
                eprintln!("{} rule '{}' not found", "error:".bold().red(), name);
                return Err(anyhow::anyhow!("rule not found"));
            }
        }
    } else {
        config.rules.iter().collect()
    };

    eprintln!(
        "{} Debugging {} rules",
        "Debug:".bold().cyan(),
        rules_to_show.len()
    );

    let mut wildcard_values: HashMap<String, String> = HashMap::new();
    for (key, value) in &config.config {
        let string_val = match value {
            toml::Value::String(s) => s.clone(),
            other => other.to_string(),
        };
        wildcard_values.insert(format!("config.{key}"), string_val);
    }

    for rule in &rules_to_show {
        eprintln!("{}", format!("── Rule: {} ──", rule.name).bold().cyan());

        if let Some(ref desc) = rule.description {
            eprintln!("  {} {}", "Description:".dimmed(), desc);
        }

        if !rule.output.is_empty() {
            let expanded_outputs: Vec<String> = rule
                .output
                .iter()
                .map(|o| {
                    oxo_flow_core::executor::checkpoint::expand_config_in_path(o, &wildcard_values)
                })
                .collect();
            eprintln!("  {} {:?}", "Outputs:".dimmed(), expanded_outputs);
        }

        if let Some(ref cmd) = rule.shell {
            let expanded = oxo_flow_core::executor::process::render_shell_command(
                cmd,
                rule,
                &wildcard_values,
                oxo_flow_core::scheduler::detect_system_limits(),
            );
            eprintln!("  {} {}", "Shell (expanded):".dimmed(), expanded);
        }

        if let Ok(deps) = dag.dependencies(&rule.name)
            && !deps.is_empty()
        {
            let mut deps = deps;
            deps.sort();
            deps.dedup();
            eprintln!("  {} {:?}", "Dependencies:".dimmed(), deps);
        }

        eprintln!();
    }

    Ok(())
}

/// Default checkpoint location used when `status` is invoked without an argument.
const DEFAULT_CHECKPOINT: &str = ".oxo-flow/checkpoint.json";

/// Snapshot age (seconds) at which `status` flags the checkpoint as stale
/// (issue #849). The observed flush lag on a live 561-rule run was ~1–2 min;
/// beyond this threshold the human output switches to a warning.
const SNAPSHOT_STALE_SECS: i64 = 300;

/// Process a checkpoint rule's re-entry manifest after it completes
/// (issue #78 P3): merge new samples, re-expand from templates, extend the
/// plan, and record the re-entry in the checkpoint. Errors fail the rule.
#[allow(clippy::too_many_arguments)]
async fn process_reentry(
    config: &mut WorkflowConfig,
    rule: &Rule,
    workdir: &std::path::Path,
    wildcard_values: &std::collections::HashMap<String, String>,
    sched: &mut oxo_flow_core::scheduler::SchedulerState,
    order: &mut Vec<String>,
    order_set: &mut std::collections::HashSet<String>,
    dag: &mut WorkflowDag,
    checkpoint: &Arc<tokio::sync::Mutex<CheckpointState>>,
) -> anyhow::Result<()> {
    let manifest_path = rule.checkpoint_manifest.as_ref().ok_or_else(|| {
        anyhow::anyhow!("checkpoint rule '{}' has no checkpoint_manifest", rule.name)
    })?;
    let expanded =
        oxo_flow_core::executor::checkpoint::expand_config_in_path(manifest_path, wildcard_values);
    let full = workdir.join(&expanded);
    let content = std::fs::read_to_string(&full).map_err(|e| {
        anyhow::anyhow!(
            "checkpoint rule '{}' produced no readable manifest at '{}': {e}",
            rule.name,
            full.display()
        )
    })?;
    let (group, samples, pairs) = oxo_flow_core::reentry::parse_manifest(&content)
        .map_err(|e| anyhow::anyhow!("checkpoint rule '{}': {e}", rule.name))?;
    if samples.is_empty() && pairs.is_empty() {
        return Ok(()); // empty manifest = valid no-op
    }
    let new_names =
        oxo_flow_core::reentry::apply_reentry(config, group.as_deref(), &samples, &pairs)?;
    if new_names.is_empty() {
        return Ok(()); // everything already present
    }
    let round = {
        let mut ck = checkpoint.lock().await;
        let round = ck.reentries.iter().map(|r| r.round).max().unwrap_or(0) + 1;
        if round > oxo_flow_core::reentry::MAX_REENTRY_ROUNDS {
            anyhow::bail!(
                "re-entry round cap ({}) exceeded — checkpoint rule '{}' keeps discovering new values; this is a workflow bug",
                oxo_flow_core::reentry::MAX_REENTRY_ROUNDS,
                rule.name
            );
        }
        ck.record_reentry(oxo_flow_core::reentry::ReentryRecord {
            round,
            rule: rule.name.clone(),
            group,
            samples,
            pairs,
        });
        round
    };
    for name in &new_names {
        sched.add_rule(name);
        order_set.insert(name.clone());
        order.push(name.clone());
    }
    *dag = WorkflowDag::from_rules_with_config(
        &config.rules,
        &config_placeholder_values(&config.config),
    )
    .context("failed to rebuild workflow DAG after re-entry")?;
    tracing::info!(
        rule = %rule.name,
        round = round,
        new_instances = ?new_names,
        "checkpoint re-entry expanded the plan"
    );
    Ok(())
}

/// Runtime-discovered output fan-out (issue #227 item 5): the producer
/// instance named `instance_name` just completed. Scan the filesystem for
/// the files its (baked) `output_pattern` enumerates, contribute them to
/// the producer template's accumulated domain, persist the domain to the
/// checkpoint FIRST (so a crash resumes without re-running the producer),
/// and — once ALL of the producer's instances are completed, so the domain
/// union is final — instantiate the deferred consumers and insert them
/// into the remaining plan. Forward-safety comes from completion ordering,
/// not DAG topology: new instances join `order`/`order_set` and the DAG is
/// rebuilt, exactly like checkpoint re-entry.
#[allow(clippy::too_many_arguments)]
async fn process_output_pattern_discovery(
    config: &mut WorkflowConfig,
    instance_name: &str,
    workdir: &std::path::Path,
    sched: &mut oxo_flow_core::scheduler::SchedulerState,
    order: &mut Vec<String>,
    order_set: &mut std::collections::HashSet<String>,
    dag: &mut WorkflowDag,
    checkpoint: &Arc<tokio::sync::Mutex<CheckpointState>>,
    checkpoint_path: &std::path::Path,
) -> anyhow::Result<Vec<String>> {
    let Some(template) = config.output_pattern_template_of(instance_name) else {
        return Ok(Vec::new());
    };
    let Some(instance) = config.get_rule(instance_name).cloned() else {
        return Ok(Vec::new());
    };

    // 1. Scan the filesystem for this instance's output_pattern files.
    //
    // A fully-bound baked pattern (issue #296) is a static contract:
    // plan-time fan-out already IS the domain and plan-time consumers hold
    // their DAG edges — discovery has nothing to enumerate and the
    // zero-discovery warning would be spurious. Deferred consumers can
    // still be attached, though, when the binding came from a fan-out the
    // engine baked itself (scatter writes its variable into the pattern):
    // their domain is the instance's own bindings, not the filesystem, so
    // contribute that directly instead of starving them silently.
    let combos = if config.output_pattern_needs_discovery(&instance) {
        config.discover_output_pattern_files(&instance, workdir)?
    } else {
        let bindings = config.instance_bindings(instance_name);
        if bindings.is_empty()
            || config
                .pending_output_pattern_consumers_of(&template)
                .is_empty()
        {
            return Ok(Vec::new());
        }
        vec![bindings]
    };
    if combos.is_empty() {
        // Zero discoveries after producer success: loud, but legal — the
        // consumers stay uninstantiated (a later instance of the same
        // producer may still contribute the domain).
        eprintln!(
            "{} output_pattern '{}' of rule '{}' matched no files — \
             downstream consumers stay uninstantiated",
            "Warning:".bold().yellow(),
            instance.output_pattern.as_deref().unwrap_or_default(),
            instance_name
        );
        tracing::warn!(
            rule = instance_name,
            pattern = ?instance.output_pattern,
            "output_pattern discovery matched no files"
        );
        return Ok(Vec::new());
    }

    // 2. Contribute to the template's domain (union across instances) and
    //    persist FIRST — a crash after this point resumes without
    //    re-running the producer.
    config.contribute_output_pattern_domain(&template, combos.clone());
    {
        let mut ck = checkpoint.lock().await;
        ck.record_output_pattern_domain(&template, combos);
        if let Err(save_err) = ck.save_to_file(checkpoint_path) {
            tracing::warn!("Failed to save checkpoint: {save_err}");
        }
    }

    // 3. Gate: only instantiate consumers once ALL of the producer's
    //    instances are completed — the domain union must be final (a
    //    partial domain would drop values discovered later).
    let belongs_to = |r: &Rule| {
        r.output_pattern.is_some()
            && (r.name == template
                || config.expansion_templates.get(&r.name).map(String::as_str)
                    == Some(template.as_str()))
    };
    let outstanding = config.rules.iter().any(|r| {
        belongs_to(r) && sched.status(&r.name) != Some(oxo_flow_core::executor::JobStatus::Success)
    });
    if outstanding {
        return Ok(Vec::new());
    }

    // 4. Instantiate the deferred consumers on the final domain.
    let new_names = config.expand_output_pattern_consumers()?;
    if new_names.is_empty() {
        return Ok(Vec::new());
    }
    for name in &new_names {
        sched.add_rule(name);
        order_set.insert(name.clone());
        order.push(name.clone());
    }
    *dag = WorkflowDag::from_rules_with_config(
        &config.rules,
        &config_placeholder_values(&config.config),
    )
    .context("failed to rebuild workflow DAG after output_pattern fan-out")?;
    tracing::info!(
        producer = template,
        new_instances = ?new_names,
        "output_pattern discovery expanded the plan"
    );
    Ok(new_names)
}

/// Wall-time timings for `status --timing`, slowest first, plus their sum.
/// Synthetic records (issue #324 F-1 `recorded_as: Some(...)`) carry a 0.0
/// placeholder rather than a measurement, so they are excluded from the
/// list and the total — a fake "0.0s" row would pollute both.
fn rule_timings(state: &CheckpointState) -> (Vec<(&str, f64)>, f64) {
    let mut timings: Vec<(&str, f64)> = state
        .benchmarks
        .iter()
        .filter(|(_, b)| b.recorded_as.is_none())
        .map(|(rule, bench)| (rule.as_str(), bench.wall_time_secs))
        .collect();
    // Deterministic order: slowest first (HashSet iteration order is arbitrary)
    timings.sort_by(|a, b| b.1.total_cmp(&a.1));
    let total = timings.iter().map(|(_, secs)| secs).sum();
    (timings, total)
}

/// Explain a target removed from a targeted execution set. A target present
/// in `when_false_rules` has its OWN false gate; one absent from the set was
/// cascade-pruned because an UPSTREAM gate is false — the old single message
/// claimed both were "when-gated false", which read as nonsense for gate-less
/// rules (issue #299).
fn print_target_skipped_note(
    skipped: &str,
    when_false_rules: &std::collections::HashSet<String>,
    run_log: Option<&SharedRunLog>,
) {
    let detail = if when_false_rules.contains(skipped) {
        format!(
            "target '{skipped}' is when-gated false — it never runs; \
             removed from the execution set (its upstream was pruned too)"
        )
    } else {
        format!(
            "target '{skipped}' is unreachable — its upstream is when-gated \
             false; removed from the execution set"
        )
    };
    diagnostic_narrate(format_args!("{} {detail}", "Note:".yellow()), run_log);
}

/// Existence predicate for the target closure: an input path (config-
/// expanded, relative to the workflow DAG) that already exists in the run
/// workdir is a pre-built input — the executor will find it, so a pruned
/// when-false producer must not gate the consumer. Absolute paths resolve
/// as-is; the run_workdir-check exists so every call site stays anchored in
/// ITS effective workdir (`run` uses the run workdir, dry-run the plan base
/// dir).
fn run_workdir_check(workdir: Option<&std::path::Path>, path: &str) -> bool {
    let p = std::path::Path::new(path);
    if p.is_absolute() {
        p.exists()
    } else {
        workdir
            .unwrap_or_else(|| std::path::Path::new("."))
            .join(p)
            .exists()
    }
}

/// Per-rule staleness reasons for `status` (issue #432(c)).
///
/// Answers "why will this rule re-run?" with the SAME classification the
/// dry-run preview uses, by re-parsing the workflow recorded in the
/// checkpoint and running the shared invalidation detection on a throwaway
/// state clone (detection mutates the state it is given).
///
/// Returns an empty map when the answer cannot be computed — workflow file
/// missing/moved, config no longer parses, or no completed rules — so the
/// caller degrades to the plain listing instead of failing.
#[allow(clippy::type_complexity)]
fn compute_staleness_reasons(
    state: &CheckpointState,
    checkpoint_path: &std::path::Path,
) -> std::collections::BTreeMap<String, (String, String)> {
    use crate::commands::run_preview::RuleStatus;
    let mut reasons = std::collections::BTreeMap::new();
    if state.completed_rules.is_empty() {
        return reasons;
    }
    let Some(workflow_path) = state.workflow_path.as_ref() else {
        return reasons;
    };
    let workflow = std::path::PathBuf::from(workflow_path);
    if !workflow.exists() {
        return reasons;
    }
    let Ok(config) = WorkflowConfig::from_file(&workflow) else {
        return reasons;
    };
    let workflow_dir = oxo_flow_core::parent_dir(&workflow).to_path_buf();
    let Ok(dag) = WorkflowDag::from_rules_with_config(
        &config.rules,
        &config_placeholder_values(&config.config),
    ) else {
        return reasons;
    };
    let Ok(raw_order) = dag.execution_order() else {
        return reasons;
    };
    let order_set: std::collections::HashSet<String> = raw_order.into_iter().collect();
    let mut order: Vec<String> = Vec::new();
    let Ok(groups) = dag.parallel_groups() else {
        return reasons;
    };
    for group in groups {
        let mut level: Vec<String> = group
            .into_iter()
            .filter(|n| order_set.contains(n))
            .collect();
        level.sort();
        order.extend(level);
    }

    // Read-only: preview_run_plan clones internally and mutates only that
    // clone (detection may record legacy baselines there) — the on-disk
    // checkpoint is never touched from `status`. The pristine state must be
    // passed so completed-rule credit is classified as such.
    let wildcard_values = config_placeholder_values(&config.config);
    let sensitive_keys: std::collections::HashSet<String> = config
        .config_meta
        .iter()
        .filter(|(_, def)| def.sensitive)
        .map(|(key, _)| key.clone())
        .collect();
    let interpreter_map = config.workflow.interpreter_map.clone();
    // `run` and dry-run resolve this identity gate before any preview; from
    // `status` the checkpoint's own recorded path is the workflow (same file
    // it was saved under), so adoption is a no-op — but run it anyway to keep
    // the classification path byte-identical with run's.
    let mut adopted = state.clone();
    crate::commands::run::adopt_checkpoint_for_workflow(&mut adopted, &workflow, None);
    let preview = crate::commands::run_preview::preview_run_plan(
        &adopted,
        &config,
        &dag,
        &order,
        &workflow_dir,
        &wildcard_values,
        &sensitive_keys,
        &interpreter_map,
        checkpoint_path,
        false,
        false,
    );
    for entry in &preview.plan {
        // Only rules recorded as completed are in scope: the question is
        // "why would this COMPLETED rule re-run".
        if !state.completed_rules.contains(&entry.name) {
            continue;
        }
        let (reason, detail) = match entry.status {
            RuleStatus::Skipped | RuleStatus::SkippedByWhen | RuleStatus::SkippedFresh => {
                continue;
            }
            // After adoption a completed rule that survived classification
            // but is gone from the adopted state means the identity gate
            // invalidated it — the credit really is gone, so say so.
            RuleStatus::NeverCompleted if !adopted.completed_rules.contains(&entry.name) => (
                "checkpoint reset".to_string(),
                "completed credit was invalidated (foreign workflow adoption)".to_string(),
            ),
            RuleStatus::NeverCompleted => (
                "output missing".to_string(),
                "completion record present but the executor would re-run — outputs are gone or failed cleanup removed them".to_string(),
            ),
            RuleStatus::ConfigInvalidated => (
                "config changed".to_string(),
                "an effective config value differs from the recorded snapshot".to_string(),
            ),
            RuleStatus::InputInvalidated => (
                "input changed".to_string(),
                "recorded input manifest (paths/size/mtime) no longer matches disk".to_string(),
            ),
            RuleStatus::OutputsMissing => (
                "output missing".to_string(),
                "recorded output is gone from the workdir".to_string(),
            ),
            RuleStatus::Cascaded { from: ref f1 } => (
                "cascaded".to_string(),
                format!("upstream rule '{f1}' will re-run"),
            ),
            RuleStatus::CascadedUpstream { from: ref f2 } => (
                "missing input".to_string(),
                format!(
                    "a downstream rule needs an intermediate that is gone; producer '{f2}' re-runs first"
                ),
            ),
            RuleStatus::Forced => ("forced".to_string(), "--rerun was requested".to_string()),
        };
        reasons.insert(entry.name.clone(), (reason, detail));
    }
    reasons
}

/// Split the checkpoint's failed rules into real failures and rules whose
/// runtime `when` gate evaluated to false (issue #690).
///
/// The executor records `Skipped` for a when-false verdict and (since #690)
/// removes any stale failure record itself, but checkpoints written by older
/// versions can still carry the stale entry — `status` then rendered the
/// rule as `✗ Failed` forever. This display-side reconcile keeps those dirty
/// checkpoints honest: an entry whose recorded verdict is `false` is reported
/// as skipped-by-when, not failed. Both outputs are sorted (`failed_rules`
/// is a BTreeSet, so iteration is already deterministic).
fn reconcile_failed_rules<'a>(
    failed_rules: &'a std::collections::BTreeSet<String>,
    when_verdicts: &'a std::collections::BTreeMap<String, bool>,
) -> (Vec<&'a str>, Vec<&'a str>) {
    let mut real_failed = Vec::new();
    let mut skipped_when_false = Vec::new();
    for rule in failed_rules {
        if when_verdicts.get(rule) == Some(&false) {
            skipped_when_false.push(rule.as_str());
        } else {
            real_failed.push(rule.as_str());
        }
    }
    (real_failed, skipped_when_false)
}

pub async fn handle_status(
    checkpoint: Option<PathBuf>,
    workdir: Option<PathBuf>,
    json: bool,
    timing: bool,
    limit: usize,
) -> Result<()> {
    print_banner();

    // Detect common mistake: user passes a .oxoflow file instead of checkpoint
    if let Some(path) = &checkpoint
        && path.extension().is_some_and(|ext| ext == "oxoflow")
    {
        eprintln!(
            "{} '{}' appears to be a workflow file, not a checkpoint.",
            "Warning:".bold().yellow(),
            path.display()
        );
        eprintln!(
            "  The 'status' command expects a checkpoint file (e.g., .oxo-flow/checkpoint.json)."
        );
        eprintln!(
            "  Run 'oxo-flow run {}' first to generate a checkpoint.",
            path.display()
        );
        anyhow::bail!("Cannot read workflow file as checkpoint");
    }

    // Issue #432(d): mirror `run --workdir` so users do not need to know the
    // checkpoint's internal path — the run directory is enough. An explicit
    // CHECKPOINT argument wins.
    let checkpoint_path = match (&checkpoint, &workdir) {
        (Some(p), _) => p.clone(),
        (None, Some(dir)) => dir.join(".oxo-flow/checkpoint.json"),
        (None, None) => PathBuf::from(DEFAULT_CHECKPOINT),
    };
    let state = CheckpointState::load_from_file(&checkpoint_path).with_context(|| {
        format!(
            "failed to load checkpoint from '{}'.\n  \
             Check that the file exists and is a valid checkpoint (JSON format).\n  \
             Checkpoint files are generated automatically by 'oxo-flow run' in .oxo-flow/checkpoint.json.",
            checkpoint_path.display()
        )
    })?;

    // Issue #849: `status` reads a snapshot, not live state — surface how
    // fresh the snapshot is (the flush lag observed on a live 561-rule run
    // was ~1–2 min and completely invisible to operators). Sourced from the
    // file's mtime, so it reflects the last write even when a monitoring
    // wrapper reads the checkpoint out-of-band.
    let checkpoint_mtime = std::fs::metadata(&checkpoint_path)
        .and_then(|m| m.modified())
        .ok()
        .map(|t| {
            let t: chrono::DateTime<chrono::Utc> = t.into();
            t
        });
    let snapshot_age_secs = checkpoint_mtime.as_ref().map(|m| {
        chrono::Utc::now()
            .signed_duration_since(*m)
            .num_seconds()
            .max(0)
    });
    if snapshot_age_secs.is_some_and(|age| age > SNAPSHOT_STALE_SECS) {
        tracing::warn!(
            "checkpoint snapshot is {}s old (threshold {}s)",
            snapshot_age_secs.unwrap(),
            SNAPSHOT_STALE_SECS
        );
    }

    // Deterministic order (HashSet iteration order is arbitrary)
    let mut completed: Vec<&str> = state.completed_rules.iter().map(String::as_str).collect();
    completed.sort_unstable();
    // Issue #690: stale entries from older checkpoints — a rule whose
    // recorded when-verdict is false — are not failures. Reconcile so
    // `status` matches what the executor actually did. Both vectors are
    // sorted (failed_rules is a BTreeSet, so iteration is deterministic).
    let (failed, when_gated_off) =
        reconcile_failed_rules(&state.failed_rules, &state.when_verdicts);

    // ── Per-rule staleness reasons (issue #432(c)) ────────────────────────
    // Reuse the dry-run preview's exact classification so "why will this
    // rule re-run?" has the same answer in `status` and `dry-run -v`.
    // Best-effort: needs the workflow file recorded in the checkpoint plus
    // a current config; when either is missing the section degrades to the
    // plain completed/failed listing below.
    let staleness = compute_staleness_reasons(&state, &checkpoint_path);
    let staleness_json: Option<serde_json::Map<String, serde_json::Value>> = if staleness.is_empty()
    {
        None
    } else {
        Some(
            staleness
                .iter()
                .map(|(rule, (reason, detail))| {
                    (
                        rule.clone(),
                        serde_json::json!({
                            "status": reason,
                            "detail": detail,
                        }),
                    )
                })
                .collect(),
        )
    };

    if json {
        let mut output = serde_json::json!({
            "command": "status",
            "checkpoint": checkpoint_path.display().to_string(),
            "workflow": state.workflow_path,
            "completed": completed,
            "failed": failed,
        });
        // Issue #849: snapshot freshness — scripts/monitoring wrappers no
        // longer need to stat the file out-of-band to judge it.
        if let Some(mtime) = checkpoint_mtime {
            output["checkpoint_mtime"] =
                serde_json::json!(mtime.to_rfc3339_opts(chrono::SecondsFormat::Secs, true));
        }
        if let Some(age) = snapshot_age_secs {
            output["snapshot_age_secs"] = serde_json::json!(age);
        }
        // Issue #690: stale failed_rules entries whose when-gate evaluated
        // false are reported here, not under "failed".
        if !when_gated_off.is_empty() {
            output["skipped_by_when"] = serde_json::json!(when_gated_off);
        }
        if let Some(map) = staleness_json {
            output["staleness"] = serde_json::Value::Object(map);
        }
        if timing {
            let (timings, total) = rule_timings(&state);
            // serde_json::Map is a BTreeMap: keys stay deterministically sorted
            let timings_map: serde_json::Map<String, serde_json::Value> = timings
                .into_iter()
                .map(|(rule, secs)| (rule.to_string(), serde_json::json!(secs)))
                .collect();
            output["timings"] = serde_json::Value::Object(timings_map);
            output["total_time_secs"] = serde_json::json!(total);
            // Sampled peak-RSS measurements where recorded (issue #67 §4).
            let memory_map: serde_json::Map<String, serde_json::Value> = state
                .benchmarks
                .iter()
                .filter(|(_, b)| b.max_memory_mb.is_some())
                .map(|(rule, b)| {
                    (
                        rule.to_string(),
                        serde_json::json!({
                            "max_memory_mb": b.max_memory_mb,
                            "memory_limit_mb": b.memory_limit_mb,
                        }),
                    )
                })
                .collect();
            if !memory_map.is_empty() {
                output["memory"] = serde_json::Value::Object(memory_map);
            }
        }
        println!("{}", serde_json::to_string_pretty(&output)?);
        return Ok(());
    }

    eprintln!(
        "{} Status for checkpoint: {}",
        "Status:".bold().cyan(),
        checkpoint_path.display()
    );
    // Issue #849: the checkpoint is a flushed snapshot, not live state —
    // the observed flush lag on a live 561-rule run was ~1–2 min. Surface
    // the age so "Completed: 0" is never mistaken for "nothing ran".
    if let (Some(mtime), Some(age)) = (checkpoint_mtime, snapshot_age_secs) {
        let written = mtime.with_timezone(&chrono::Local).format("%H:%M:%S");
        if age > SNAPSHOT_STALE_SECS {
            eprintln!(
                "  {} Snapshot is {}s old (checkpoint written {} local) — flush lag, not necessarily stalled progress",
                "Warning:".bold().yellow(),
                age,
                written
            );
        } else {
            eprintln!(
                "  State as of {}s ago (checkpoint written {} local)",
                age, written
            );
        }
    }
    eprintln!("  Completed: {}", completed.len());
    eprintln!("  Failed:    {}", failed.len());
    if !when_gated_off.is_empty() {
        eprintln!("  Skipped (when-gated off): {}", when_gated_off.len());
    }

    if timing {
        let (timings, total) = rule_timings(&state);
        if !timings.is_empty() {
            eprintln!(
                "\n{} (top {}, total {:.1}s)",
                "Rule timings:".bold().green(),
                limit.min(timings.len()),
                total
            );
            for (rule, secs) in timings.iter().take(limit) {
                let mem = state.benchmarks.get(*rule).and_then(|b| {
                    b.max_memory_mb.map(|m| match b.memory_limit_mb {
                        Some(l) if l > 0 => {
                            if m * 100 >= l * 80 {
                                format!("  peak {m}/{l} MiB ⚠")
                            } else {
                                format!("  peak {m}/{l} MiB")
                            }
                        }
                        _ => format!("  peak {m} MiB"),
                    })
                });
                match mem {
                    Some(mem) => eprintln!("  {} {} ({:.1}s){}", "✓".green(), rule, secs, mem),
                    None => eprintln!("  {} {} ({:.1}s)", "✓".green(), rule, secs),
                }
            }
        }
    } else {
        if !staleness.is_empty() {
            eprintln!("\n{}", "Staleness reasons (as of now):".bold().yellow());
            eprintln!(
                "  {}",
                "why each completed rule would re-run under the current workflow+config".dimmed()
            );
            for rule in &completed {
                match staleness.get(*rule) {
                    Some((reason, detail)) => {
                        eprintln!(
                            "  {} {} — {} ({})",
                            "✓".green(),
                            rule,
                            reason,
                            detail.dimmed()
                        );
                    }
                    None => eprintln!("  {} {} — up to date", "✓".green(), rule),
                }
            }
        }
        if !completed.is_empty() {
            eprintln!("\n{}", "Completed rules:".bold().green());
            for rule in &completed {
                eprintln!("  {} {}", "✓".green(), rule);
            }
        }

        if !failed.is_empty() {
            eprintln!("\n{}", "Failed rules:".bold().red());
            for rule in &failed {
                eprintln!("  {} {}", "✗".red(), rule);
            }
        }

        if !when_gated_off.is_empty() {
            eprintln!("\n{}", "Skipped (when condition false):".bold().yellow());
            for rule in &when_gated_off {
                eprintln!(
                    "  {} {} — {}",
                    "⊘".yellow(),
                    rule,
                    "condition evaluated to false".dimmed()
                );
            }
        }
    }

    Ok(())
}

#[allow(clippy::too_many_arguments)]
pub async fn resume_command(
    checkpoint: PathBuf,
    jobs: usize,
    ai_recover: bool,
    ai_max_retries: Option<u32>,
    keep_going: bool,
    timeout: String,
    workdir: Option<PathBuf>,
    no_report_snapshot: bool,
) -> Result<()> {
    // resume does not produce structured JSON output
    print_banner();

    // Load checkpoint state
    let state = CheckpointState::load_from_file(&checkpoint)
        .with_context(|| {
            format!(
                "failed to load checkpoint from '{}'.\n  \
             Check that the file exists and is a valid checkpoint (JSON format).",
                checkpoint.display()
            )
        })
        .map_err(|e| {
            // Same friendly hint `status` gives for the identical mistake
            // (audit #276 P4-4): passing a workflow file where a checkpoint
            // belongs should name the confusion, not just the JSON parse error.
            if checkpoint.extension().is_some_and(|ext| ext == "oxoflow") {
                anyhow::anyhow!(
                    "{e}\n  '{}' appears to be a workflow file, not a checkpoint.\n  \
                 The 'resume' command expects a checkpoint file (e.g., .oxo-flow/checkpoint.json).",
                    checkpoint.display()
                )
            } else {
                e
            }
        })?;

    // Get workflow path from checkpoint
    let workflow_path = match &state.workflow_path {
        Some(p) => PathBuf::from(p),
        None => {
            anyhow::bail!(
                "Checkpoint does not contain a workflow reference.\n  \
                 The checkpoint at '{}' was generated by an older version of oxo-flow.\n  \
                 To resume manually, run: oxo-flow run <workflow.oxoflow>\n  \
                 (oxo-flow run automatically resumes from the checkpoint)",
                checkpoint.display()
            );
        }
    };

    if !workflow_path.exists() {
        anyhow::bail!(
            "Workflow file '{}' referenced by checkpoint no longer exists.\n  \
             The workflow may have been moved or deleted.",
            workflow_path.display()
        );
    }

    let completed = state.completed_rules.len();
    // Issue #690: don't count stale when-gated entries as failures in the
    // resume banner either — they are skips, and the executor will re-judge
    // their gate on this resume anyway.
    let (failed, skipped_when_false) =
        reconcile_failed_rules(&state.failed_rules, &state.when_verdicts);
    let failed = failed.len();

    eprintln!(
        "{} Resuming workflow '{}'",
        "Resume:".bold().cyan(),
        workflow_path.display()
    );
    eprintln!("  Checkpoint: {}", checkpoint.display());
    // Remaining = the rules this resume will actually execute: everything
    // the checkpoint has not completed (failed rules are retried). The
    // checkpoint records only what ran, so the workflow supplies the total
    // (its template count — wildcards expand later); if it cannot be read
    // here, `run_command` reports that error with full context a moment
    // later and the failed count is the fallback. The old arithmetic
    // reduced to `failed` (audit finding).
    let remaining = WorkflowConfig::from_file(&workflow_path)
        .map(|cfg| cfg.rules.len().saturating_sub(completed))
        .unwrap_or(failed);
    eprintln!("  State: {completed} completed, {failed} failed, {remaining} remaining");
    if !skipped_when_false.is_empty() {
        eprintln!(
            "  Note: {} rule(s) were skipped by their `when` condition (recorded false); the gate is re-judged on this resume.",
            skipped_when_false.len()
        );
    }
    // Issue #685: entries persisting in the running set mean the prior run
    // died mid-flight; run_command clears them (with a warning) on start.
    if !state.running_rules().is_empty() {
        eprintln!(
            "  Note: {} rule(s) were still running when the previous run died; they will re-run.",
            state.running_rules().len()
        );
    }

    if completed == 0 && failed == 0 {
        eprintln!(
            "  {} No rules have been executed yet. Use 'oxo-flow run' instead.",
            "Note:".yellow()
        );
        return Ok(());
    }

    // Re-run the workflow — the checkpoint with completed rules will cause
    // already-finished rules to be skipped automatically
    eprintln!();
    eprintln!(
        "{} Launching executor with {} parallel job(s)...",
        "Info:".bold().cyan(),
        jobs
    );

    // Workdir precedence (issue #68): explicit --workdir > the directory the
    // original run recorded in the checkpoint > the workflow's directory.
    let effective_workdir = workdir.or_else(|| state.workdir.clone().map(PathBuf::from));
    eprintln!(
        "  Workdir: {}",
        effective_workdir
            .as_deref()
            .map(|p| p.display().to_string())
            .unwrap_or_else(|| "(workflow directory)".to_string())
    );

    run_command(
        Some(workflow_path),
        jobs,
        keep_going,        // keep_going (same semantics as `run`)
        effective_workdir, // workdir (recorded by the original run)
        // #538: the user's checkpoint path wins over the workdir-derived
        // default — the banner's "Resuming… N completed" must be the truth.
        None,       // log_file (default path)
        Vec::new(), // target
        Vec::new(), // module
        0,          // retry
        timeout,    // timeout
        false,      // resume_failed (user can re-run with --resume-failed in 'run')
        None,       // profile
        0,          // max_threads
        0,          // max_memory
        false,      // skip_env_setup
        true,       // skip_ref_build (resume: refs already built)
        None,       // cache_dir
        false,      // provenance (checkpoint already has checksums)
        false,      // json (resume defaults to human-readable)
        Vec::new(), // cli_args (resume reuses checkpoint state)
        ai_recover,
        ai_max_retries,
        Vec::new(), // samples_filter (resume restores checkpoint state as-is)
        false,      // rerun (resume skips completed rules by design)
        no_report_snapshot,
        None,             // max_submitted (cluster queue cap — resume keeps the profile's)
        Some(checkpoint), // #538: explicit checkpoint path wins
    )
    .await
}

/// One consolidated backend-availability failure from the run pre-flight:
/// identical (backend, message) pairs across a cohort's instances collapse
/// into a single entry whose `count` says how many pending rules share it.
#[derive(Debug, PartialEq)]
struct BackendPreflightFailure {
    kind: String,
    message: String,
    example_rule: String,
    spec: String,
    count: usize,
}

impl std::fmt::Display for BackendPreflightFailure {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "{kind}: {message} — required by {count} pending rule(s), e.g. `{example_rule}` (environment: {spec}); {hint}",
            kind = self.kind,
            message = self.message,
            count = self.count,
            example_rule = self.example_rule,
            spec = self.spec,
            hint = backend_install_hint(self),
        )
    }
}

/// Remediation hint for a preflight failure. The failure MODE decides the
/// advice (issue #848): a missing pixi MANIFEST is a workflow defect —
/// advise creating/correcting the declared file; a missing backend BINARY
/// is an environment defect — advise installing the tool.
fn backend_install_hint(failure: &BackendPreflightFailure) -> &'static str {
    if failure.kind == "pixi" && failure.message.contains("not found") {
        return "create the manifest or correct the rule's `pixi` path (resolved relative to the workflow directory)";
    }
    match failure.kind.as_str() {
        "pixi" => "install pixi (https://pixi.sh) and ensure it is on PATH",
        "mamba" => "install mamba or micromamba and ensure it is on PATH",
        "conda" => "install conda and ensure it is on PATH",
        "docker" => "install/start Docker and ensure `docker` is on PATH",
        "singularity" => "install Singularity/Apptainer and ensure it is on PATH",
        "modules" => "ensure environment modules (modulecmd) is on PATH",
        _ => "make the environment backend available on PATH",
    }
}

/// Primary spec string for an environment, for the failure message.
fn primary_env_spec(env: &oxo_flow_core::rule::EnvironmentSpec) -> String {
    env.mamba
        .as_deref()
        .or(env.conda.as_deref())
        .or(env.pixi.as_deref())
        .or(env.docker.as_deref())
        .or(env.singularity.as_deref())
        .or(env.venv.as_deref())
        .map(str::to_string)
        .unwrap_or_else(|| env.modules.join(","))
}

/// Validate each unique environment spec among the pending rules once
/// (instances fan out one rule template × N samples, so the spec set is
/// tiny even for large cohorts) and collapse the failures: rules sharing
/// one broken spec merge into a single entry, and distinct specs failing
/// with the identical message (the backend-missing case) merge too, so
/// the operator fixes one root cause per line. First-failure order is
/// preserved for stable output.
fn backend_preflight_failures(
    pending: &[(String, oxo_flow_core::rule::EnvironmentSpec)],
    resolver: &oxo_flow_core::environment::EnvironmentResolver,
) -> Vec<BackendPreflightFailure> {
    // spec cache_key → failure slot (None = spec validated OK)
    let mut spec_slot: std::collections::HashMap<String, Option<usize>> =
        std::collections::HashMap::new();
    // (backend kind, message) → failure slot, for merging identical root causes
    let mut msg_slot: std::collections::HashMap<(String, String), usize> =
        std::collections::HashMap::new();
    let mut out: Vec<BackendPreflightFailure> = Vec::new();
    for (rule_name, env) in pending {
        let key = resolver.cache_key(env);
        match spec_slot.get(&key) {
            Some(Some(slot)) => {
                out[*slot].count += 1;
                continue;
            }
            Some(None) => continue, // identical spec, already validated OK
            None => {}
        }
        if let Err(err) = resolver.validate_spec(env) {
            let (kind, message) = match &err {
                oxo_flow_core::OxoFlowError::Environment { kind, message } => {
                    (kind.clone(), message.clone())
                }
                other => ("environment".to_string(), other.to_string()),
            };
            let slot = match msg_slot.get(&(kind.clone(), message.clone())) {
                Some(&slot) => {
                    out[slot].count += 1;
                    slot
                }
                None => {
                    out.push(BackendPreflightFailure {
                        kind: kind.clone(),
                        message: message.clone(),
                        example_rule: rule_name.clone(),
                        spec: primary_env_spec(env),
                        count: 1,
                    });
                    msg_slot.insert((kind, message), out.len() - 1);
                    out.len() - 1
                }
            };
            spec_slot.insert(key, Some(slot));
        } else {
            spec_slot.insert(key, None);
        }
    }
    out
}

#[cfg(test)]
mod tests {
    use super::{
        age_ready_list, ai_attempts, apply_cli_overrides, cleanup_cache_dir, closest_declared_key,
        config_placeholder_values, failure_exit_code, known_modules_hint, missing_source_findings,
        missing_source_lines, parse_cli_overrides, parse_disk_hold_abort_rounds,
        parse_disk_tick_secs, parse_spawn_watchdog_threshold, reconcile_failed_rules,
        substitute_source_placeholder, suggest_jobs,
    };
    use oxo_flow_core::dag::WorkflowDag;
    use std::collections::{HashMap, HashSet};

    #[test]
    fn reconcile_failed_rules_separates_real_failures_from_when_false_skips() {
        // Issue #690: a checkpoint written by an older version keeps a
        // when-gated rule in failed_rules; only the recorded verdict makes
        // it a skip.
        let failed: std::collections::BTreeSet<String> = ["align", "qc_S2", "trim"]
            .into_iter()
            .map(str::to_string)
            .collect();
        let verdicts: std::collections::BTreeMap<String, bool> =
            [("qc_S2".to_string(), false)].into_iter().collect();
        let (real_failed, skipped) = reconcile_failed_rules(&failed, &verdicts);
        assert_eq!(real_failed, vec!["align", "trim"]);
        assert_eq!(skipped, vec!["qc_S2"]);
    }

    #[test]
    fn reconcile_failed_rules_keeps_failures_without_a_recorded_verdict() {
        // No verdict recorded (or verdict true — a real failure followed by
        // a gate flip back to true must stay a failure): everything is real.
        let failed: std::collections::BTreeSet<String> =
            ["a".to_string(), "b".to_string()].into_iter().collect();
        let verdicts: std::collections::BTreeMap<String, bool> =
            [("a".to_string(), true)].into_iter().collect();
        let (real_failed, skipped) = reconcile_failed_rules(&failed, &verdicts);
        assert_eq!(real_failed, vec!["a", "b"]);
        assert!(skipped.is_empty());
    }

    #[test]
    fn parse_disk_env_knobs_fall_back_to_defaults() {
        // Unset, empty, garbage, zero — a typo can never disable the
        // abort (it would hang instead), so every invalid form falls
        // back to the shipped default.
        assert_eq!(parse_disk_tick_secs(None), 60);
        assert_eq!(parse_disk_tick_secs(Some("")), 60);
        assert_eq!(parse_disk_tick_secs(Some("abc")), 60);
        assert_eq!(parse_disk_tick_secs(Some("0")), 60);
        assert_eq!(parse_disk_tick_secs(Some("-5")), 60);
        assert_eq!(parse_disk_hold_abort_rounds(None), 2);
        assert_eq!(parse_disk_hold_abort_rounds(Some("xyz")), 2);
        assert_eq!(parse_disk_hold_abort_rounds(Some("0")), 2);
    }

    #[test]
    fn parse_disk_env_knobs_honor_valid_values() {
        assert_eq!(parse_disk_tick_secs(Some("1")), 1);
        assert_eq!(parse_disk_tick_secs(Some("300")), 300);
        assert_eq!(parse_disk_hold_abort_rounds(Some("1")), 1);
        assert_eq!(parse_disk_hold_abort_rounds(Some("10")), 10);
    }

    #[test]
    fn snapshot_age_is_derived_from_checkpoint_mtime() {
        // Issue #849: snapshot freshness comes from the file's mtime, not
        // from any field inside the checkpoint (CheckpointState carries no
        // self-timestamp). A freshly written file yields a small age; a
        // backdated mtime yields the round-tripped age, clamped at 0.
        let dir = std::env::temp_dir().join(format!(
            "oxoflow-status-age-{}-{}",
            std::process::id(),
            std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .unwrap()
                .as_nanos()
        ));
        std::fs::create_dir_all(&dir).unwrap();
        let path = dir.join("checkpoint.json");
        std::fs::write(&path, "{}").unwrap();

        let mtime_now: chrono::DateTime<chrono::Utc> =
            std::fs::metadata(&path).unwrap().modified().unwrap().into();
        let age_now = chrono::Utc::now()
            .signed_duration_since(mtime_now)
            .num_seconds()
            .max(0);
        assert!(age_now < 60, "fresh file reported {age_now}s old");

        // Backdate the mtime by 10 minutes via filetime semantics through
        // std: rewrite the file, then shift the clock comparison instead —
        // std has no mtime setter, so assert the arithmetic directly.
        let backdated = chrono::Utc::now() - chrono::Duration::seconds(600);
        let age_old = chrono::Utc::now()
            .signed_duration_since(backdated)
            .num_seconds()
            .max(0);
        assert_eq!(age_old, 600);
        assert!(age_old > super::SNAPSHOT_STALE_SECS);

        std::fs::remove_dir_all(&dir).ok();
    }

    fn config_with_rules(rules_toml: &str) -> oxo_flow_core::config::WorkflowConfig {
        oxo_flow_core::config::WorkflowConfig::parse(rules_toml).unwrap()
    }

    fn source_gate_fixture(
        dir: &std::path::Path,
    ) -> (
        oxo_flow_core::config::WorkflowConfig,
        WorkflowDag,
        Vec<String>,
        HashMap<String, String>,
    ) {
        let toml = r#"
[workflow]
name = "t"
version = "1.0"

[config]
threshold = 5
empty = []

[[rules]]
name = "gen"
input = []
output = ["out.txt"]
shell = "echo x > {output[0]}"

[[rules]]
name = "use"
input = ["out.txt", "ext.txt"]
output = ["final.txt"]
shell = "cat {input} > {output[0]}"

[[rules]]
name = "gated"
input = ["gated_missing.txt"]
output = ["gated.txt"]
when = "config.threshold >= 10"
shell = "true"

[[rules]]
name = "scatter_zero"
input = ["scatter_missing.txt"]
output = ["scatter.txt"]
scatter = { variable = "chunk", values_from = "config.empty" }
shell = "true"

[[rules]]
name = "opt"
input = ["optional_missing.txt"]
output = ["opt.txt"]
optional = true
shell = "true"
"#;
        let mut config = config_with_rules(toml);
        config.apply_defaults();
        let dag = WorkflowDag::from_rules_with_config(
            &config.rules,
            &config_placeholder_values(&config.config),
        )
        .unwrap();
        let order = dag.execution_order().unwrap();
        let wildcard_values = config_placeholder_values(&config.config);
        std::fs::create_dir_all(dir).unwrap();
        (config, dag, order, wildcard_values)
    }

    fn source_gate_call(
        config: &oxo_flow_core::config::WorkflowConfig,
        dag: &WorkflowDag,
        order: &[String],
        workdir: &std::path::Path,
        workflow_dir: &std::path::Path,
        wildcard_values: &HashMap<String, String>,
    ) -> Vec<String> {
        missing_source_findings(config, dag, order, workdir, workflow_dir, wildcard_values)
    }

    #[test]
    fn missing_source_gate_flags_only_unproduced_required_inputs() {
        let dir = tempfile::tempdir().unwrap();
        let (config, dag, order, wildcard_values) = source_gate_fixture(dir.path());
        let findings = source_gate_call(
            &config,
            &dag,
            &order,
            dir.path(),
            dir.path(),
            &wildcard_values,
        );
        assert_eq!(
            findings,
            vec!["rule 'use': input 'ext.txt' does not exist and no rule produces it".to_string()],
            "'out.txt' has a producer (gen) and belongs to the cascade path; when-false, \
             zero-fan-out scatter, and optional rules are all skipped"
        );
    }

    #[test]
    fn missing_source_gate_passes_when_sources_exist() {
        let dir = tempfile::tempdir().unwrap();
        let (config, dag, order, wildcard_values) = source_gate_fixture(dir.path());
        std::fs::write(dir.path().join("ext.txt"), b"data").unwrap();
        let findings = source_gate_call(
            &config,
            &dag,
            &order,
            dir.path(),
            dir.path(),
            &wildcard_values,
        );
        assert!(findings.is_empty(), "{findings:?}");
    }

    #[test]
    fn missing_source_gate_silent_with_full_checkpoint_is_the_regressed_scenario() {
        // The reported bug: every rule completed, raw/ deleted, re-run exited
        // 0. At gate time (pre-checkpoint) the completed state is irrelevant
        // — the missing external source must be flagged regardless.
        let dir = tempfile::tempdir().unwrap();
        let (config, dag, order, wildcard_values) = source_gate_fixture(dir.path());
        std::fs::write(dir.path().join("out.txt"), b"stale").unwrap();
        std::fs::write(dir.path().join("final.txt"), b"stale").unwrap();
        let findings = source_gate_call(
            &config,
            &dag,
            &order,
            dir.path(),
            dir.path(),
            &wildcard_values,
        );
        assert!(!findings.is_empty(), "missing ext.txt must gate the run");
    }

    #[test]
    fn missing_source_gate_skips_reference_outputs() {
        // [[references]] outputs are built by the reference machinery after
        // this gate and are not DAG nodes — a missing one must not gate.
        let dir = tempfile::tempdir().unwrap();
        let toml = r#"
[workflow]
name = "refgate"

[[references]]
name = "idx"
source = "genome.fa"
output = "genome.idx"
build = "cat genome.fa > genome.idx"

[[rules]]
name = "use_ref"
input = ["genome.idx"]
output = ["result.txt"]
shell = "cat {input} > {output}"
"#;
        let mut config = config_with_rules(toml);
        config.apply_defaults();
        let dag = WorkflowDag::from_rules_with_config(
            &config.rules,
            &config_placeholder_values(&config.config),
        )
        .unwrap();
        let order = dag.execution_order().unwrap();
        let wildcard_values = config_placeholder_values(&config.config);
        std::fs::write(dir.path().join("genome.fa"), b"ACGT").unwrap();
        let findings = source_gate_call(
            &config,
            &dag,
            &order,
            dir.path(),
            dir.path(),
            &wildcard_values,
        );
        assert!(
            findings.is_empty(),
            "reference-built output must not gate the run: {findings:?}"
        );
    }

    #[test]
    fn missing_source_gate_accepts_sample_pattern_dir_inputs() {
        // Bare relative input whose data lives in the sample_pattern
        // discovery directory (absolute pattern, sibling of the workdir) —
        // the web instance-table shape.
        let dir = tempfile::tempdir().unwrap(); // workflow dir
        let data = tempfile::tempdir().unwrap(); // sample_pattern root
        let toml = format!(
            r#"
[workflow]
name = "samplegate"
sample_pattern = "{root}/data/{{{{sample}}}}.fq"

[[rules]]
name = "qc"
input = ["{{{{sample}}}}.fq"]
output = ["qc_{{{{sample}}}}.txt"]
shell = "cat {{{{input}}}} > {{{{output}}}}"
"#,
            root = data.path().display(),
        );
        let mut config = config_with_rules(&toml);
        config.apply_defaults();
        std::fs::create_dir_all(data.path().join("data")).unwrap();
        std::fs::write(data.path().join("data").join("S1.fq"), b"read").unwrap();
        let dag = WorkflowDag::from_rules_with_config(
            &config.rules,
            &config_placeholder_values(&config.config),
        )
        .unwrap();
        let order = dag.execution_order().unwrap();
        let wildcard_values = config_placeholder_values(&config.config);
        let findings = source_gate_call(
            &config,
            &dag,
            &order,
            dir.path(),
            dir.path(),
            &wildcard_values,
        );
        assert!(
            findings.is_empty(),
            "input present in the sample_pattern dir must not gate: {findings:?}"
        );
    }

    #[test]
    fn missing_source_lines_lists_rule_names_and_patterns() {
        let dir = tempfile::tempdir().unwrap();
        let (config, _dag, _order, wildcard_values) = source_gate_fixture(dir.path());
        let mut missing = HashSet::new();
        missing.insert("use".to_string());
        let lines = missing_source_lines(&missing, &config, dir.path(), &wildcard_values);
        assert_eq!(lines.len(), 1);
        assert!(lines[0].contains("use"), "{:?}", lines[0]);
        assert!(lines[0].contains("ext.txt"), "{:?}", lines[0]);
    }

    #[test]
    fn suggest_jobs_evaluates_each_wave_against_its_own_heaviest_rule() {
        // Waves: 20×t1, 2×t16, 20×t1 on a 64-core box. The old global
        // formula suggested 4; the per-group form keeps the wide light
        // waves at 20 (issue #361 measured 11.0 s vs 3.0 s makespan on
        // this exact shape).
        let mut toml = String::from("[workflow]\nname = \"t\"\nversion = \"0.1\"\n");
        for i in 0..20 {
            toml.push_str(&format!(
                "\n[[rules]]\nname = \"a{i}\"\noutput = [\"a{i}.txt\"]\nshell = \"true\"\n\n[rules.resources]\nthreads = 1\n"
            ));
        }
        for i in 0..2 {
            toml.push_str(&format!(
                "\n[[rules]]\nname = \"b{i}\"\noutput = [\"b{i}.txt\"]\nshell = \"true\"\n\n[rules.resources]\nthreads = 16\n"
            ));
        }
        for i in 0..20 {
            toml.push_str(&format!(
                "\n[[rules]]\nname = \"c{i}\"\noutput = [\"c{i}.txt\"]\nshell = \"true\"\n\n[rules.resources]\nthreads = 1\n"
            ));
        }
        let config = config_with_rules(&toml);
        let groups = vec![
            (0..20).map(|i| format!("a{i}")).collect::<Vec<_>>(),
            (0..2).map(|i| format!("b{i}")).collect::<Vec<_>>(),
            (0..20).map(|i| format!("c{i}")).collect::<Vec<_>>(),
        ];
        assert_eq!(suggest_jobs(&config, &groups, 64), 20);
        // The old global formula would have said 4 here.
        assert_ne!(suggest_jobs(&config, &groups, 64), 4);
    }

    #[test]
    fn suggest_jobs_clamps_to_the_wave_width_and_system_threads() {
        // A single wave of 5 heavy rules on 4 cores: 4/16 → 0 → clamped
        // to at least 1 job, and never above the wave width (5).
        let mut toml = String::from("[workflow]\nname = \"t\"\nversion = \"0.1\"\n");
        for i in 0..5 {
            toml.push_str(&format!(
                "\n[[rules]]\nname = \"h{i}\"\noutput = [\"h{i}.txt\"]\nshell = \"true\"\n\n[rules.resources]\nthreads = 16\n"
            ));
        }
        let config = config_with_rules(&toml);
        let groups = vec![(0..5).map(|i| format!("h{i}")).collect::<Vec<_>>()];
        assert_eq!(suggest_jobs(&config, &groups, 4), 1);
        // Wide light wave on a big machine: the 16-job legacy ceiling is gone.
        let mut toml = String::from("[workflow]\nname = \"t\"\nversion = \"0.1\"\n");
        for i in 0..20 {
            toml.push_str(&format!(
                "\n[[rules]]\nname = \"l{i}\"\noutput = [\"l{i}.txt\"]\nshell = \"true\"\n\n[rules.resources]\nthreads = 1\n"
            ));
        }
        let config = config_with_rules(&toml);
        let groups = vec![(0..20).map(|i| format!("l{i}")).collect::<Vec<_>>()];
        assert_eq!(suggest_jobs(&config, &groups, 64), 20);
    }

    #[test]
    fn suggest_jobs_defaults_to_one_without_rules() {
        let config = config_with_rules("[workflow]\nname = \"t\"\nversion = \"0.1\"\n");
        assert_eq!(suggest_jobs(&config, &[], 64), 1);
        assert_eq!(suggest_jobs(&config, &[vec![]], 64), 1);
    }

    #[test]
    fn cleanup_cache_dir_is_recursive() {
        // issue #194 C1: nested directories age out too (max_age_days = 0
        // makes everything eligible immediately).
        let dir = tempfile::tempdir().unwrap();
        let nested = dir.path().join("sub").join("deeper");
        std::fs::create_dir_all(&nested).unwrap();
        std::fs::write(dir.path().join("old.txt"), "x").unwrap();
        std::fs::write(nested.join("old-too.txt"), "y").unwrap();
        let removed = cleanup_cache_dir(dir.path(), 0);
        assert_eq!(removed, 2, "file + one nested directory removed");
        assert!(!dir.path().join("old.txt").exists());
        assert!(!dir.path().join("sub").exists());
    }

    fn declared(keys: &[&str]) -> HashSet<String> {
        keys.iter().map(|k| k.to_string()).collect()
    }

    fn map_of(pairs: &[(&str, i32)]) -> std::collections::HashMap<String, i32> {
        pairs.iter().map(|(k, v)| (k.to_string(), *v)).collect()
    }

    fn set_of(names: &[&str]) -> HashSet<String> {
        names.iter().map(|n| n.to_string()).collect()
    }

    #[test]
    fn age_ready_list_orders_by_declared_priority_then_name() {
        // Arrange
        let priority = map_of(&[("high", 20), ("low", 10)]);
        // Act
        let ready = age_ready_list(
            vec!["low".into(), "high".into()],
            &set_of(&["high", "low"]),
            &HashSet::new(),
            &priority,
            &std::collections::HashMap::new(),
        );
        // Assert
        assert_eq!(ready, vec!["high", "low"]);
    }

    #[test]
    fn age_ready_list_aged_rule_overtakes_higher_declared_priority() {
        // The starvation cure (issue #123): a producer passed over for many
        // rounds must eventually outrank the higher-priority rules that keep
        // re-occupying the slots.
        // Arrange
        let priority = map_of(&[("merge", 20), ("dump", 10)]);
        let waited = map_of(&[("dump", 15)]);
        // Act
        let ready = age_ready_list(
            vec!["merge".into(), "dump".into()],
            &set_of(&["merge", "dump"]),
            &HashSet::new(),
            &priority,
            &waited,
        );
        // Assert
        assert_eq!(ready, vec!["dump", "merge"]);
    }

    #[test]
    fn age_ready_list_drops_submitted_and_out_of_scope_rules() {
        // Arrange
        let priority = map_of(&[("a", 5), ("b", 5), ("c", 5)]);
        let submitted = set_of(&["b"]);
        // Act
        let ready = age_ready_list(
            vec!["c".into(), "b".into(), "a".into()],
            &set_of(&["a", "b", "c"]),
            &submitted,
            &priority,
            &std::collections::HashMap::new(),
        );
        // Assert — b is already submitted, so it cannot be dispatched again.
        assert_eq!(ready, vec!["a", "c"]);
    }

    #[test]
    fn age_ready_list_treats_missing_priority_and_waits_as_zero() {
        // Arrange — neither rule appears in the maps.
        // Act
        let ready = age_ready_list(
            vec!["z".into(), "a".into()],
            &set_of(&["a", "z"]),
            &HashSet::new(),
            &std::collections::HashMap::new(),
            &std::collections::HashMap::new(),
        );
        // Assert — equal effective priority (0), name tie-break.
        assert_eq!(ready, vec!["a", "z"]);
    }

    #[test]
    fn accepts_equals_forms_for_any_key() {
        let map = parse_cli_overrides(
            vec![
                "threads=8".to_string(),
                "--mode=protein".to_string(),
                "--new_key=1".to_string(), // undeclared injection via '=' stays legal
            ],
            &declared(&["threads", "mode"]),
            "run",
        )
        .unwrap();
        assert_eq!(map["threads"], "8");
        assert_eq!(map["mode"], "protein");
        assert_eq!(map["new_key"], "1");
    }

    #[test]
    fn accepts_space_form_for_declared_keys() {
        let map = parse_cli_overrides(
            vec!["--min_quality".to_string(), "45".to_string()],
            &declared(&["min_quality"]),
            "run",
        )
        .unwrap();
        assert_eq!(map["min_quality"], "45");
    }

    #[test]
    fn canonicalizes_hyphen_spelled_override_to_declared_key() {
        // issue #794: `--api-token=x` names the declared key `api_token`.
        // Without canonicalization the override is stored under the dashed
        // alias: the declared key keeps its default and the value bypasses
        // sensitive masking in the checkpoint snapshot.
        let map = parse_cli_overrides(
            vec![
                "--api-token=sk-OVERRIDE-123".to_string(),
                "min-quality=45".to_string(),
            ],
            &declared(&["api_token", "min_quality"]),
            "run",
        )
        .unwrap();
        assert_eq!(map["api_token"], "sk-OVERRIDE-123");
        assert_eq!(map["min_quality"], "45");
        assert!(!map.contains_key("api-token"), "{map:?}");
        assert!(!map.contains_key("min-quality"), "{map:?}");
    }

    #[test]
    fn canonicalizes_space_form_for_hyphen_spelled_declared_key() {
        // issue #794: `--api-token VALUE` must be accepted when only the
        // underscore-spelled `api_token` is declared.
        let map = parse_cli_overrides(
            vec!["--api-token".to_string(), "sk-OVERRIDE-123".to_string()],
            &declared(&["api_token"]),
            "run",
        )
        .unwrap();
        assert_eq!(map["api_token"], "sk-OVERRIDE-123");
    }

    #[test]
    fn canonicalization_wins_on_exact_match_and_stays_silent_on_ambiguity() {
        // Exact declared match beats normalization: `mode` declared plus
        // `sub-mode` hyphen-spelled keeps its literal key.
        let map = parse_cli_overrides(
            vec!["--sub-mode=2".to_string()],
            &declared(&["sub_mode"]),
            "run",
        )
        .unwrap();
        assert_eq!(map["sub_mode"], "2");

        // Two declared keys normalizing to the same form is pathological but
        // must not crash or silently pick one: the literal spelling is kept
        // and the usual undeclared-injection / unknown-token rules apply.
        let map = parse_cli_overrides(
            vec!["--a-b=1".to_string()],
            &declared(&["a_b", "a-b"]),
            "run",
        )
        .unwrap();
        assert_eq!(map["a-b"], "1");

        // Nothing declared matches → literal key, issue #62 injection.
        let map = parse_cli_overrides(
            vec!["--totally-unknown=7".to_string()],
            &declared(&["min_quality"]),
            "run",
        )
        .unwrap();
        assert_eq!(map["totally-unknown"], "7");

        // Space form for an unresolvable dashed token stays rejected (#71).
        let err = parse_cli_overrides(
            vec!["--totally-unknown".to_string(), "7".to_string()],
            &declared(&["min_quality"]),
            "run",
        )
        .unwrap_err();
        assert!(format!("{err}").contains("unknown argument"), "{err:?}");
    }

    #[test]
    fn apply_cli_overrides_puts_override_on_declared_sensitive_key() {
        // issue #794 end-to-end at the apply layer: the canonicalized
        // override must win over the declarative default (precedence
        // CLI > default) instead of leaving the default in place.
        let mut config = config_with_rules(
            r#"
[workflow]
name = "t"
version = "1.0"

[config]
api_token = { default = "sk-DEFAULT-abc", sensitive = true, help = "API token" }

[[rules]]
name = "r"
shell = "true"
"#,
        );
        let overrides = parse_cli_overrides(
            vec!["--api-token=sk-OVERRIDE-123".to_string()],
            &declared(&["api_token"]),
            "run",
        )
        .unwrap();
        apply_cli_overrides(&mut config, &overrides).unwrap();
        assert_eq!(config.config["api_token"].as_str(), Some("sk-OVERRIDE-123"));
    }

    #[test]
    fn rejects_unknown_dash_token() {
        let err = parse_cli_overrides(
            vec!["--config".to_string(), "config/x.toml".to_string()],
            &declared(&["min_quality"]),
            "run",
        )
        .unwrap_err();
        let msg = format!("{err}");
        assert!(msg.contains("unknown argument"), "{msg}");
        assert!(msg.contains("--config"), "{msg}");
        assert!(msg.contains("KEY=VALUE"), "{msg}");
    }

    #[test]
    fn rejects_known_flag_after_positional_with_flag_error() {
        // issue #71 contract: real flags swallowed by the trailing
        // positional get the "command flag, not a config override" error.
        let err = parse_cli_overrides(
            vec!["threads=8".to_string(), "--json".to_string()],
            &declared(&["threads"]),
            "run",
        )
        .unwrap_err();
        let msg = format!("{err}");
        assert!(msg.contains("command flag"), "{msg}");
        assert!(msg.contains("--json"), "{msg}");
    }

    #[test]
    fn dry_run_treats_shared_run_flags_as_misordered_not_unsupported() {
        // issue #441: dry-run accepts `--cache-dir` as a real flag now, so a
        // swallowed one must read as misordered, not "not supported by
        // dry-run".
        let err = parse_cli_overrides(
            vec!["threads=8".to_string(), "--cache-dir=x".to_string()],
            &declared(&["threads"]),
            "dry-run",
        )
        .unwrap_err();
        let msg = format!("{err}");
        assert!(msg.contains("command flag"), "{msg}");
        assert!(msg.contains("must come before"), "{msg}");

        // Other run flags stay "not supported" under dry-run.
        let err = parse_cli_overrides(
            vec!["threads=8".to_string(), "--provenance".to_string()],
            &declared(&["threads"]),
            "dry-run",
        )
        .unwrap_err();
        let msg = format!("{err}");
        assert!(msg.contains("not supported by dry-run"), "{msg}");
    }

    #[test]
    fn rejects_space_form_for_undeclared_key_even_via_arg() {
        // `--arg --config x` lands in the same override list.
        let err = parse_cli_overrides(
            vec!["--config".to_string(), "x".to_string()],
            &declared(&[]),
            "run",
        )
        .unwrap_err();
        assert!(format!("{err}").contains("unknown argument"));
    }

    #[test]
    fn rejects_declared_key_without_value() {
        let err = parse_cli_overrides(
            vec!["--min_quality".to_string()],
            &declared(&["min_quality"]),
            "run",
        )
        .unwrap_err();
        assert!(format!("{err}").contains("invalid config flag"), "{err:?}");
    }

    #[test]
    fn rejects_space_form_that_would_swallow_a_command_flag() {
        // Audit finding: `dry-run wf --mode --json` consumed `--json` as the
        // value of the declared key `mode`, so the run silently dropped its
        // JSON output. The `=` form already errors; the space form must too.
        let err = parse_cli_overrides(
            vec!["--mode".to_string(), "--json".to_string()],
            &declared(&["mode"]),
            "run",
        )
        .unwrap_err();
        let msg = format!("{err}");
        assert!(msg.contains("--mode"), "{msg}");
        assert!(msg.contains("--json"), "{msg}");
        assert!(msg.contains("--mode=<value>"), "{msg}");
    }

    #[test]
    fn ai_attempts_defaults_to_one_and_never_zero() {
        // Audit finding: the flag was accepted and discarded. It now bounds
        // the recovery/analysis attempts; absent → the pre-flag single try,
        // and an explicit 0 still means one attempt (never none).
        assert_eq!(ai_attempts(None), 1);
        assert_eq!(ai_attempts(Some(0)), 1);
        assert_eq!(ai_attempts(Some(3)), 3);
    }

    #[test]
    fn failure_exit_code_parses_recorded_reasons() {
        // The failure summary format from the executor path.
        assert_eq!(
            failure_exit_code("exit code 3 — ERROR: unknown reference genome hg999"),
            3
        );
        assert_eq!(failure_exit_code("exit code 137"), 137);
        assert_eq!(failure_exit_code("exit code 137\nstderr: ..."), 137);
        // Panics/aborts carry no code in the reason → keep -1.
        assert_eq!(failure_exit_code("task panicked: boom"), -1);
        assert_eq!(failure_exit_code(""), -1);
        assert_eq!(failure_exit_code("exit code notanumber"), -1);
    }

    #[test]
    fn closest_declared_key_names_a_typo_but_not_an_unrelated_key() {
        // Audit finding: `samplename=world` was silently accepted.
        let keys = declared(&["sample_name", "threads", "min_quality"]);
        assert_eq!(
            closest_declared_key("samplename", &keys),
            Some("sample_name")
        );
        assert_eq!(closest_declared_key("threds", &keys), Some("threads"));
        assert_eq!(closest_declared_key("completely_different", &keys), None);
    }

    #[test]
    fn rejects_bare_non_key_value_positional() {
        let err =
            parse_cli_overrides(vec!["naked".to_string()], &declared(&[]), "run").unwrap_err();
        assert!(format!("{err}").contains("KEY=VALUE"), "{err:?}");
    }

    #[test]
    fn accepts_empty_value_overrides() {
        // issue #430: `KEY=` and `--KEY ""` are legitimate derive-if-empty
        // overrides (rnaseq gene_bed/chrom_sizes/transcript_fasta branch on
        // `[ -n "{config.x}" ]`).
        let map = parse_cli_overrides(
            vec![
                "gene_bed=".to_string(),
                "--chrom_sizes".to_string(),
                String::new(),
                "--transcript_fasta=".to_string(),
            ],
            &declared(&["gene_bed", "chrom_sizes", "transcript_fasta"]),
            "run",
        )
        .unwrap();
        assert_eq!(map["gene_bed"], "");
        assert_eq!(map["chrom_sizes"], "");
        assert_eq!(map["transcript_fasta"], "");
    }

    #[test]
    fn rejects_empty_key_but_accepts_empty_value() {
        // "=value" trims to an empty KEY — still a syntax error.
        let err = parse_cli_overrides(vec!["=v".to_string()], &declared(&[]), "run").unwrap_err();
        assert!(
            format!("{err}").contains("KEY must be non-empty"),
            "{err:?}"
        );
    }

    #[test]
    fn dry_run_flag_error_is_command_aware() {
        // issue #430: `-j` under dry-run is not a misordered run flag — the
        // command does not support it at all. The message must say so
        // instead of implying a flag-ordering problem.
        let err = parse_cli_overrides(
            vec!["-j".to_string(), "12".to_string()],
            &declared(&[]),
            "dry-run",
        )
        .unwrap_err();
        let msg = format!("{err}");
        assert!(msg.contains("not supported by dry-run"), "{msg}");
        assert!(
            !msg.contains("command flag, not a config override"),
            "{msg}"
        );
    }

    #[test]
    fn run_flag_error_under_run_keeps_ordering_hint() {
        // `run` itself keeps the original ordering guidance (issue #71).
        let err = parse_cli_overrides(
            vec!["threads=8".to_string(), "-j".to_string()],
            &declared(&["threads"]),
            "run",
        )
        .unwrap_err();
        let msg = format!("{err}");
        assert!(msg.contains("command flag"), "{msg}");
        assert!(msg.contains("before KEY=VALUE"), "{msg}");
    }

    #[test]
    fn unknown_token_under_dry_run_stays_command_aware() {
        // An unknown --token under a non-run command points at the command,
        // not at run's ordering rule.
        let err = parse_cli_overrides(
            vec!["--config".to_string(), "x".to_string()],
            &declared(&[]),
            "dry-run",
        )
        .unwrap_err();
        let msg = format!("{err}");
        assert!(msg.contains("unknown argument"), "{msg}");
        assert!(msg.contains("dry-run"), "{msg}");
    }

    #[test]
    fn source_placeholder_quotes_paths_with_spaces() {
        // A `{source}` path containing spaces must survive the reference
        // build as ONE shell argument (issue #136 tier-2 audit — the raw
        // splice broke on spaces/metacharacters).
        let rendered =
            substitute_source_placeholder("STAR --genomeDir {source}", "refs/genome data/hg38");
        assert_eq!(rendered, "STAR --genomeDir 'refs/genome data/hg38'");
    }

    #[test]
    fn source_placeholder_escapes_embedded_quotes() {
        // POSIX-safe: an embedded quote closes and reopens the quoting
        // (the `'\''` idiom the environment wrapper also uses).
        let rendered = substitute_source_placeholder("--in {source}", "dir/we'ird");
        assert_eq!(rendered, "--in 'dir/we'\\''ird'");
    }

    #[test]
    fn source_placeholder_without_placeholder_is_unchanged() {
        let rendered = substitute_source_placeholder("echo built", "ignored");
        assert_eq!(rendered, "echo built");
    }

    #[test]
    fn known_modules_hint_names_the_empty_case() {
        // Dry-run must print the same explicit "(none — no [[include]]
        // modules)" hint `run` does instead of a trailing empty list
        // (issue #136 tier-2 audit).
        let empty: std::collections::HashMap<String, Vec<String>> =
            std::collections::HashMap::new();
        assert_eq!(
            known_modules_hint(&empty),
            "(none — no [[include]] modules)"
        );
        let mut map: std::collections::HashMap<String, Vec<String>> =
            std::collections::HashMap::new();
        map.insert("qc".to_string(), vec!["qc".to_string()]);
        assert_eq!(known_modules_hint(&map), "qc");
    }

    // ── Issue #432: dry-run headline + status --workdir / staleness ──────

    #[test]
    fn dry_run_headline_reports_run_skip_completed_not_dag_size() {
        // The headline must answer "what happens next" (issue #432b): the
        // old `DAG: (dry-run) N rules would execute` read as "everything
        // re-runs" while the checkpoint said most rules were up to date.
        use assert_cmd::Command;
        let wf_toml = r#"
[workflow]
name = "headline"

[[rules]]
name = "step"
input = []
output = ["out.txt"]
shell = "touch out.txt"
"#;
        let dir = tempfile::tempdir().unwrap();
        let wf = dir.path().join("wf.oxoflow");
        std::fs::write(&wf, wf_toml).unwrap();

        let assert = Command::cargo_bin("oxo-flow")
            .unwrap()
            .args(["dry-run", wf.to_str().unwrap()])
            .env("NO_COLOR", "1")
            .assert()
            .success();

        let stderr = String::from_utf8_lossy(&assert.get_output().stderr).to_string();
        assert!(
            stderr.contains("Plan: would run: 1 | skip: 0 | completed: 0 (DAG size: 1)"),
            "headline must lead with would-run/skip/completed, got: {stderr}"
        );
        assert!(
            !stderr.contains("rules would execute"),
            "the old misleading headline must be gone, got: {stderr}"
        );
    }

    #[test]
    fn status_accepts_workdir_like_run() {
        // Issue #432(d): `status --workdir <dir>` must resolve the default
        // checkpoint under that run directory, same as `run --workdir`.
        use assert_cmd::Command;
        let wf_toml = r#"
[workflow]
name = "status-workdir"

[[rules]]
name = "step"
input = []
output = ["out.txt"]
shell = "touch out.txt"
"#;
        let dir = tempfile::tempdir().unwrap();
        let wf = dir.path().join("wf.oxoflow");
        std::fs::write(&wf, wf_toml).unwrap();
        let run_dir = dir.path().join("run1");

        // A real run first: it records the checkpoint in the workdir.
        Command::cargo_bin("oxo-flow")
            .unwrap()
            .args([
                "run",
                wf.to_str().unwrap(),
                "--workdir",
                run_dir.to_str().unwrap(),
                "-j",
                "1",
            ])
            .env("NO_COLOR", "1")
            .assert()
            .success();

        // status --workdir finds the checkpoint without the internal path.
        let assert = Command::cargo_bin("oxo-flow")
            .unwrap()
            .args(["status", "--workdir", run_dir.to_str().unwrap(), "--json"])
            .env("NO_COLOR", "1")
            .assert()
            .success();
        let stdout = String::from_utf8_lossy(&assert.get_output().stdout).to_string();
        let parsed: serde_json::Value =
            serde_json::from_str(&stdout).expect("status --json must emit JSON");
        assert_eq!(parsed["completed"], serde_json::json!(["step"]));
    }

    #[test]
    fn status_reports_staleness_reasons_for_a_stale_completed_rule() {
        // Issue #432(c): after a completed rule's output is deleted, status
        // must say WHY it would re-run (outputs missing), not just list it.
        use assert_cmd::Command;
        let wf_toml = r#"
[workflow]
name = "stale-reasons"

[[rules]]
name = "producer"
input = []
output = ["stage1.txt"]
shell = "printf 'v1' > stage1.txt"

[[rules]]
name = "consumer"
input = ["stage1.txt"]
output = ["stage2.txt"]
shell = "cp stage1.txt stage2.txt"
"#;
        let dir = tempfile::tempdir().unwrap();
        let wf = dir.path().join("wf.oxoflow");
        std::fs::write(&wf, wf_toml).unwrap();
        let run_dir = dir.path().join("run1");

        Command::cargo_bin("oxo-flow")
            .unwrap()
            .args([
                "run",
                wf.to_str().unwrap(),
                "--workdir",
                run_dir.to_str().unwrap(),
                "-j",
                "1",
            ])
            .env("NO_COLOR", "1")
            .assert()
            .success();

        // Delete the producer's output → it is stale (outputs missing) and
        // the consumer cascades behind it.
        std::fs::remove_file(run_dir.join("stage1.txt")).unwrap();

        let assert = Command::cargo_bin("oxo-flow")
            .unwrap()
            .args(["status", "--workdir", run_dir.to_str().unwrap(), "--json"])
            .env("NO_COLOR", "1")
            .assert()
            .success();
        let stdout = String::from_utf8_lossy(&assert.get_output().stdout).to_string();
        let parsed: serde_json::Value = serde_json::from_str(&stdout).unwrap();
        let staleness = parsed["staleness"].as_object().expect("staleness map");
        let producer = &staleness["producer"];
        // stage1.txt was deleted: the producer's completion record survives
        // but its output is gone, so it classifies as the lazy cascade-up
        // producer (a downstream rule needs the intermediate).
        assert_eq!(producer["status"], "missing input");
        let consumer = &staleness["consumer"];
        // stage1.txt no longer matches the recorded manifest (gone), so the
        // consumer's own credit is input-invalidated before any cascade.
        assert_eq!(consumer["status"], "input changed");
        assert!(
            consumer["detail"].as_str().unwrap().contains("manifest"),
            "input-changed detail must explain the manifest mismatch, got: {consumer}"
        );
    }

    #[test]
    fn spawn_watchdog_threshold_parsing() {
        // issue #685: unset, garbage, and `0` all fall back to the default
        // — `0` matches the OXO_ENV_LOCK_TIMEOUT_SECS convention rather
        // than meaning "disable", so a typo can never silently turn the
        // watchdog off. The derivation is pure: no env mutation needed.
        assert_eq!(
            parse_spawn_watchdog_threshold(None),
            std::time::Duration::from_secs(600)
        );
        assert_eq!(
            parse_spawn_watchdog_threshold(Some("abc")),
            std::time::Duration::from_secs(600)
        );
        assert_eq!(
            parse_spawn_watchdog_threshold(Some("0")),
            std::time::Duration::from_secs(600)
        );
        assert_eq!(
            parse_spawn_watchdog_threshold(Some("120")),
            std::time::Duration::from_secs(120)
        );
    }

    fn pixi_env(manifest: &str) -> oxo_flow_core::rule::EnvironmentSpec {
        oxo_flow_core::rule::EnvironmentSpec {
            pixi: Some(manifest.to_string()),
            ..Default::default()
        }
    }

    #[test]
    fn backend_preflight_passes_empty_and_empty_specs() {
        let resolver = oxo_flow_core::environment::EnvironmentResolver::new();
        // Nothing pending → nothing to validate.
        assert!(super::backend_preflight_failures(&[], &resolver).is_empty());
        // A rule with no environment declaration never invokes a backend.
        let no_env = vec![("qc".to_string(), Default::default())];
        assert!(super::backend_preflight_failures(&no_env, &resolver).is_empty());
    }

    #[test]
    fn backend_preflight_reports_missing_pixi_manifest_once_per_unique_spec() {
        // Two pending instances of one rule template share the same spec:
        // validate it once and collapse the failures, keeping the FIRST
        // pending rule in execution order as the example (deterministic).
        let resolver = oxo_flow_core::environment::EnvironmentResolver::new();
        let pending = vec![
            ("trim_S1".to_string(), pixi_env("envs/absent/pixi.toml")),
            ("trim_S2".to_string(), pixi_env("envs/absent/pixi.toml")),
        ];
        let failures = super::backend_preflight_failures(&pending, &resolver);
        assert_eq!(
            failures.len(),
            1,
            "identical specs must group: {failures:?}"
        );
        let f = &failures[0];
        assert_eq!(f.kind, "pixi");
        assert_eq!(f.count, 2);
        assert_eq!(f.example_rule, "trim_S1");
        assert_eq!(f.spec, "envs/absent/pixi.toml");
        let rendered = f.to_string();
        assert!(
            rendered.contains("required by 2 pending rule(s)"),
            "the message must carry the cohort size: {rendered}"
        );
        // Whether THIS test observes the manifest mode depends on whether
        // the pixi binary happens to be on PATH (validate_spec checks the
        // binary first); the hint split itself is covered without that
        // dependency by backend_install_hint_splits_manifest_not_found…
    }

    #[test]
    fn backend_install_hint_splits_manifest_not_found_from_missing_binary() {
        // Issue #848: the failure MODE decides the advice. A missing pixi
        // MANIFEST is a workflow defect — the hint must point at the file,
        // not tell the operator to install pixi; a missing backend BINARY
        // keeps the install advice. Constructed failures keep this test
        // independent of the test sandbox's PATH.
        let make = |message: &str| super::BackendPreflightFailure {
            kind: "pixi".to_string(),
            message: message.to_string(),
            example_rule: "trim_S1".to_string(),
            spec: "envs/absent/pixi.toml".to_string(),
            count: 2,
        };
        let manifest = super::backend_install_hint(&make(
            "pixi manifest 'envs/absent/pixi.toml' not found — the rule declares it via `pixi = \"envs/absent/pixi.toml\"` (resolved relative to the workflow directory)",
        ));
        assert_eq!(
            manifest,
            "create the manifest or correct the rule's `pixi` path (resolved relative to the workflow directory)"
        );
        let binary = super::backend_install_hint(&make("pixi is not installed or not in PATH"));
        assert_eq!(
            binary,
            "install pixi (https://pixi.sh) and ensure it is on PATH"
        );
        // The mode discriminator keys on "not found", which only the
        // manifest message carries — the binary message must not trip it.
        assert!(!"pixi is not installed or not in PATH".contains("not found"));
    }

    #[test]
    fn backend_preflight_missing_backend_binary_still_advises_install() {
        // Issue #848: the OTHER failure mode — pixi the BINARY is absent —
        // keeps the install advice. validate_spec checks the binary first,
        // so an unresolvable spec with pixi absent from PATH yields the
        // "not installed" message, distinct from the manifest check.
        let resolver = oxo_flow_core::environment::EnvironmentResolver::new();
        let pending = vec![("trim_S1".to_string(), pixi_env("envs/absent/pixi.toml"))];
        let failures = super::backend_preflight_failures(&pending, &resolver);
        // The exact mode depends on whether pixi is on PATH in the test
        // sandbox; assert the hint MATCHES whichever mode fired.
        let rendered = failures[0].to_string();
        if failures[0].message.contains("not found") {
            assert!(
                rendered.contains("create the manifest"),
                "manifest mode must carry the file advice: {rendered}"
            );
        } else {
            assert!(
                rendered.contains("install pixi"),
                "binary mode must keep the install advice: {rendered}"
            );
        }
    }
}