orchestratectl 0.1.6

//! Read-time stall detection for an undriven `--kind orchestrate` driver run.
//!
//! A `--kind orchestrate` supervisor only *adopts* children — it never drives
//! the fan-out itself; the orchestrator agent runs in the user's main
//! conversation and is what spawns children (issue
//! `peculiarly-muddled-caption`). If that agent never runs its drive loop (or
//! dies immediately after `run create`), the driver node `n-0001` sits
//! `pending` with zero children and no fresh events forever, supervisor alive —
//! indistinguishable at a glance from a healthy long-running campaign (a real
//! reproduction sat this way for 15 hours).
//!
//! `stalled` is a **computed** hint, not a persisted status: it touches no
//! event-append / reducer / schema path (state-integrity invariants 1–3 are not
//! in play). It is derived purely from the driver node's existing timestamp +
//! status + children counter, read under the same shared lock the caller
//! already holds for the manifest. Terminal-status semantics are untouched — a
//! stalled run is still `pending`; the flag only says "pending, but visibly not
//! progressing".
//!
//! Scope: this catches the *specific* zombie in the issue — a driver that was
//! **never driven** (still `pending`, zero children). It deliberately does NOT
//! try to detect every stalled shape (a driver that spawned one child then
//! died, or transitioned to `running` and then stopped emitting events): those
//! need a real orchestrator liveness/heartbeat signal, tracked as the follow-up
//! `peculiarly-cheerful-mine`. The hint is a heuristic, not a liveness proof —
//! hence the human output says "verify" before prescribing a cancel.

use chrono::{DateTime, Duration, Utc};

use octl_core::{Kind, Node, Status};

/// Grace window an undriven orchestrate driver node may sit `pending` with zero
/// children before `stalled` trips. Chosen to comfortably exceed the time a
/// genuinely-driven orchestrator takes to spawn its first child (planning +
/// `run create` of the first ready feature) while still catching a zombie
/// within a useful window. The 15h real reproduction dwarfs any reasonable
/// value here.
pub const STALL_GRACE: Duration = Duration::minutes(12);

/// The driver node's `n-0001` — the single fan-out driver of an `orchestrate`
/// run. Mirrors `run show`'s `DEFAULT_NODE_ID`.
pub const DRIVER_NODE_ID: &str = "n-0001";

/// Compute the `stalled` hint for a run from its manifest status, kind, and
/// driver node.
///
/// Returns `true` only for a `pending` `--kind orchestrate` run whose driver
/// node is itself still `pending`, has spawned **zero** children, and has not
/// been touched for longer than [`STALL_GRACE`] — the exact signature of a
/// driver that was created but never driven. Any of these disqualifies it:
///
/// - a non-`pending` run (a `done`/`failed`/`cancelled`/`running` manifest is
///   not a zombie — a terminal run whose `n-0001` projection stayed `pending`
///   must never be flagged, or `run list` would print `done (stalled)` and the
///   remediation would tell the user to cancel an already-terminal run);
/// - a non-`orchestrate` kind (only the orchestrate driver has the
///   "supervisor adopts but does not drive" shape);
/// - a driver node that reached a non-`pending` status (it is running,
///   terminal, or otherwise progressing);
/// - a driver node with ≥1 child (the orchestrator agent *is* driving);
/// - a driver node touched within the grace window. The node projection's
///   `updated_at` is bumped by exactly the events that mark driver progress —
///   `node.status` / `node.report` / `node.retry` / `child.spawned` (verified
///   against the reducer). Discussion / spinoff / supervisor events bump the
///   *manifest* timestamp, not the node's, so `node.updated_at` is a precise
///   "the driver made progress" proxy — deliberately narrower than
///   `manifest.updated_at`, which unrelated supervisor churn would keep falsely
///   fresh. The narrow cost: a driver that only opens a discussion (e.g. asks
///   the user a question) without spawning a child is still counted idle, which
///   is why the hint is advisory, not authoritative;
/// - a missing driver node (a half-initialized run — not assessable, so not
///   flagged rather than falsely alarmed).
///
/// `now` is injected so the decision is deterministic in tests.
#[must_use]
pub fn is_stalled(
    run_status: Status,
    kind: Kind,
    driver: Option<&Node>,
    now: DateTime<Utc>,
) -> bool {
    if run_status != Status::Pending {
        return false;
    }
    if kind != Kind::Orchestrate {
        return false;
    }
    let Some(node) = driver else {
        return false;
    };
    if node.status != Status::Pending {
        return false;
    }
    if !node.children.is_empty() {
        return false;
    }
    now.signed_duration_since(node.updated_at) > STALL_GRACE
}

/// Detect a *stillborn* run: created successfully, but its supervisor died
/// before ever spawning the first worker node — so the run can never make
/// progress and will otherwise sit `pending` until a caller's timeout expires
/// (issue `run-wait-stillborn-run-not-detected`; a real incident blocked
/// `run wait` for ~6h).
///
/// Returns `true` only for the exact "never started" signature:
///
/// - `status == Pending` — the run never advanced past creation. A terminal or
///   `running` manifest is not stillborn (it started).
/// - the supervisor is **not alive** — the actor that would create `n-0001` and
///   roll the run up is dead (or was never recorded). This is the crucial
///   difference from [`is_stalled`]: there the supervisor is *alive* but idle,
///   so a grace window is needed to tell "slow" from "dead"; here the
///   supervisor is confirmed dead, which is unambiguous and needs no grace.
/// - `node_count == 0` — not a single worker node was ever created. This also
///   makes the check kind-agnostic: a `--kind orchestrate` run whose driver
///   node was never even created is stillborn by the same logic, while a run
///   that got as far as `n-0001` is excluded (it started).
/// - `updated_at == created_at` — no manifest-bumping event has been applied
///   since creation, so there has been zero forward progress.
///
/// # Why the timestamp guard is sound (not the fragile check it looks like)
///
/// A reasonable worry is that `supervisor.started` (emitted during supervisor
/// boot, before `run create` returns) would bump `manifest.updated_at` and make
/// this a common false negative. It does not: `supervisor.started` has **no
/// reducer arm** — it folds through the catch-all to a no-op that emits zero
/// projection ops, so it never touches `manifest.updated_at` (verified against
/// `octl-core::reducer`). The first event that bumps the manifest clock on a
/// fresh run is `node.created`, which *also* increments `node_count`. So on a
/// zero-node run `node_count == 0` and `updated_at == created_at` move in
/// lockstep — the guard is redundant-but-robust confirmation, and matches the
/// incident manifest exactly. The `alive` check dominates the healthy path
/// regardless: during the (up to ~90s) create window between `run.created` and
/// `node.created`, the supervisor is alive, so a healthy run is never flagged.
///
/// Residual limitation: a human manually opening a discussion/spinoff on a
/// never-started run *would* bump `updated_at` and defeat the guard — the run
/// then degrades to the old timeout behavior (no new harm). Runs orphaned
/// *after* creating `n-0001` (a supervisor that died mid-run, `node_count > 0`)
/// are handled by the sibling [`is_orphaned`], which needs a grace window
/// because a `node_count > 0` pending/running run is also the shape of a
/// healthy working run.
///
/// Like [`is_stalled`], this is a **computed** read-time hint over the manifest
/// (plus a single-file supervisor-pid probe) — it touches no
/// event-append / reducer / schema path.
#[must_use]
pub fn is_stillborn(
    run_status: Status,
    supervisor_alive: bool,
    node_count: u32,
    created_at: DateTime<Utc>,
    updated_at: DateTime<Utc>,
) -> bool {
    run_status == Status::Pending
        && !supervisor_alive
        && node_count == 0
        && updated_at == created_at
}

/// Grace window a `node_count > 0` run may sit idle with a dead supervisor
/// before [`is_orphaned`] trips. Mirrors the supervisor's own in-process
/// `NO_WORKER_GRACE` (15 min, `supervise/mod.rs`): the *alive* supervisor waits
/// that long before terminalizing a stuck run, so a read-time orphan verdict
/// uses the same budget. Long enough that a supervisor briefly between a
/// reattach/restart handoff (its pid file momentarily reads dead while the
/// manifest clock is still fresh) is never misjudged; short enough to catch a
/// genuinely stranded run well inside a caller's default 6h `run wait` timeout.
pub const ORPHAN_GRACE: Duration = Duration::minutes(15);

/// Detect an *orphaned* run: its supervisor created ≥1 worker node and then died
/// mid-run, leaving the run `pending`/`running` with no actor able to roll it up
/// to a terminal status (issue `run-wait-still`). This is the sibling case the
/// stillborn fix (`run-wait-stillborn-run-not-detected`) deliberately scoped
/// out — [`is_stillborn`] handles `node_count == 0` (the supervisor died
/// *before* starting any work); this handles `node_count > 0` (it died *after*).
///
/// Returns `true` only when every part of the stranded signature holds:
///
/// - `status in {Pending, Running}` — a non-terminal run. Terminal runs
///   (`Done`/`Failed`/`Cancelled`) already settled; `Blocked` is a deliberate
///   human-action handoff (a blocked `node.report`), not a stranded supervisor,
///   so it is excluded.
/// - the supervisor is **not alive** — the actor that would roll the run up is
///   gone (or was never recorded). This is the crux: a `node_count > 0`
///   pending/running run with a *live* supervisor is the NORMAL shape of a
///   healthy, heads-down worker, so the liveness probe is what separates
///   "stranded" from "still working" (issue's "why it's harder than stillborn").
/// - `node_count > 0` — at least one node was created. The `== 0` case is
///   [`is_stillborn`] (unambiguous, no grace needed); this one is not.
/// - idle for longer than [`ORPHAN_GRACE`] — `manifest.updated_at` is the last
///   time ANY manifest-bumping event was applied. A dead supervisor stops
///   producing them, so a stale manifest clock alongside a dead supervisor is
///   the stranded signature. The grace window is the crucial guard against a
///   transient dead-read: a supervisor caught mid-reattach/restart (pid file
///   momentarily absent) whose clock is still fresh is NOT flagged, mirroring
///   the orchestrate-stall grace ([`is_stalled`]). Unlike [`is_stillborn`],
///   which can key off the exact `updated_at == created_at` never-progressed
///   signature and needs no grace, a mid-run orphan has a moving clock and so
///   REQUIRES the idle window to tell "just now" from "long dead".
///
/// A **computed** read-time hint like its siblings — no event-append / reducer /
/// schema path is touched (state-integrity invariants 1–3 are not in play). It
/// reads only fields already held under the caller's shared lock (the manifest)
/// plus the single-file supervisor-pid probe. `now` is injected so the decision
/// is deterministic in tests.
///
/// Clock skew fails closed: a `updated_at` in the future yields a negative
/// `signed_duration_since`, which is never `> ORPHAN_GRACE`, so a skewed clock
/// suppresses the verdict rather than raising a false orphan alarm — the run
/// degrades to the old timeout behavior, no new harm. (The residual weakness is
/// a genuinely-dead supervisor whose transient dead-read coincides with a
/// heads-down worker that has legitimately emitted no manifest event for the
/// grace window; hardening that needs a supervisor heartbeat/lease, tracked as a
/// follow-up. The hint stays advisory — it points at the non-destructive
/// `run reattach`, never a destructive action.)
#[must_use]
pub fn is_orphaned(
    run_status: Status,
    supervisor_alive: bool,
    node_count: u32,
    updated_at: DateTime<Utc>,
    now: DateTime<Utc>,
) -> bool {
    matches!(run_status, Status::Pending | Status::Running)
        && !supervisor_alive
        && node_count > 0
        && now.signed_duration_since(updated_at) > ORPHAN_GRACE
}

/// Which read-time "cannot progress on its own" shape a run matches, if any.
/// Both variants are supervisor-dead orphans, distinguished only by how far the
/// run got before the supervisor died. Callers settle the wait identically for
/// either but phrase a slightly different remediation hint per variant.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StallKind {
    /// Supervisor died *before* creating any worker node (`node_count == 0`):
    /// the run never started. See [`is_stillborn`].
    Stillborn,
    /// Supervisor died *mid-run*, after creating ≥1 node (`node_count > 0`):
    /// the run started but its work is now stranded. See [`is_orphaned`].
    Orphaned,
}

/// Combined read-time stall verdict over one consistent manifest + supervisor
/// snapshot: the run is [`Stillborn`] (never started), [`Orphaned`] (started,
/// then stranded), or neither. Both callers (`run wait`, `run show`) evaluate
/// this under the same shared lock they already hold for the manifest, so the
/// verdict, the run's `status`, and the remediation it prints all come from one
/// view that cannot straddle a reducer write.
///
/// Stillborn is checked first: the two are mutually exclusive on `node_count`
/// (`== 0` vs `> 0`), so the order only formalizes that a zero-node run can
/// never be orphaned.
///
/// [`Stillborn`]: StallKind::Stillborn
/// [`Orphaned`]: StallKind::Orphaned
#[must_use]
pub fn stall_kind(
    run_status: Status,
    supervisor_alive: bool,
    node_count: u32,
    created_at: DateTime<Utc>,
    updated_at: DateTime<Utc>,
    now: DateTime<Utc>,
) -> Option<StallKind> {
    if is_stillborn(
        run_status,
        supervisor_alive,
        node_count,
        created_at,
        updated_at,
    ) {
        Some(StallKind::Stillborn)
    } else if is_orphaned(run_status, supervisor_alive, node_count, updated_at, now) {
        Some(StallKind::Orphaned)
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use octl_core::{NodeId, RunId};

    fn node(status: Status, children: usize, updated_at: DateTime<Utc>) -> Node {
        Node {
            schema_version: 1,
            node_id: NodeId::parse_str("n-0001").unwrap(),
            run_id: RunId::parse_str("01arz3ndektsv4rrffq69g5fav").unwrap(),
            parent_node_id: None,
            kind: Kind::Orchestrate,
            status,
            task: None,
            worktree_path: None,
            branch: None,
            base_sha: None,
            tmux_window: None,
            tmux_identity: None,
            agent_pid: None,
            agent_pid_start_time: None,
            supervisor_pid: None,
            children: (0..children)
                .map(|i| octl_core::ChildRef {
                    run_id: RunId::parse_str("01arz3ndektsv4rrffq69g5fav").unwrap(),
                    node_id: NodeId::parse_str(&format!("n-{:04}", i + 2)).unwrap(),
                })
                .collect(),
            started_at: None,
            updated_at,
            last_report: None,
            last_processed_report_seq_by_child: serde_json::Map::new(),
            retry_attempts: 0,
        }
    }

    fn now() -> DateTime<Utc> {
        "2026-08-06T12:00:00Z".parse().unwrap()
    }

    /// (a) An undriven orchestrate driver past the grace window is stalled.
    #[test]
    fn undriven_driver_past_grace_is_stalled() {
        let created = now() - STALL_GRACE - Duration::seconds(1);
        let n = node(Status::Pending, 0, created);
        assert!(is_stalled(
            Status::Pending,
            Kind::Orchestrate,
            Some(&n),
            now()
        ));
    }

    /// (b1) A driver that has spawned a child is being driven — not stalled,
    /// even long past the grace window.
    #[test]
    fn driver_with_child_is_not_stalled() {
        let created = now() - STALL_GRACE - Duration::hours(1);
        let n = node(Status::Pending, 1, created);
        assert!(!is_stalled(
            Status::Pending,
            Kind::Orchestrate,
            Some(&n),
            now()
        ));
    }

    /// (b2) A driver whose node was touched recently (fresh events) is not
    /// stalled, even with zero children yet.
    #[test]
    fn driver_with_recent_activity_is_not_stalled() {
        let recent = now() - Duration::minutes(1);
        let n = node(Status::Pending, 0, recent);
        assert!(!is_stalled(
            Status::Pending,
            Kind::Orchestrate,
            Some(&n),
            now()
        ));
    }

    /// (c) Within the grace window, an undriven driver is not yet stalled.
    #[test]
    fn within_grace_window_is_not_stalled() {
        let created = now() - STALL_GRACE + Duration::seconds(1);
        let n = node(Status::Pending, 0, created);
        assert!(!is_stalled(
            Status::Pending,
            Kind::Orchestrate,
            Some(&n),
            now()
        ));
    }

    /// Exactly at the grace boundary is not yet stalled (strict `>`).
    #[test]
    fn exactly_at_grace_boundary_is_not_stalled() {
        let created = now() - STALL_GRACE;
        let n = node(Status::Pending, 0, created);
        assert!(!is_stalled(
            Status::Pending,
            Kind::Orchestrate,
            Some(&n),
            now()
        ));
    }

    /// A terminal (or otherwise non-`pending`) *manifest* is never flagged, even
    /// when its driver projection stayed `pending` with 0 children and is stale
    /// — a cancelled/done run is not a zombie, and flagging it would tell the
    /// user to cancel an already-terminal run (the review's top finding).
    #[test]
    fn non_pending_run_status_is_never_stalled() {
        let created = now() - STALL_GRACE - Duration::hours(1);
        let n = node(Status::Pending, 0, created);
        for run_status in [
            Status::Running,
            Status::Blocked,
            Status::Done,
            Status::Failed,
            Status::Cancelled,
        ] {
            assert!(
                !is_stalled(run_status, Kind::Orchestrate, Some(&n), now()),
                "run status {run_status:?} must not stall"
            );
        }
    }

    /// A non-`orchestrate` kind is never flagged, however idle — other kinds do
    /// not have the "supervisor adopts but does not drive" shape.
    #[test]
    fn non_orchestrate_kind_is_never_stalled() {
        let created = now() - STALL_GRACE - Duration::hours(1);
        let n = node(Status::Pending, 0, created);
        for k in [Kind::Spinoff, Kind::FanOut, Kind::Orchestrated, Kind::Code] {
            assert!(
                !is_stalled(Status::Pending, k, Some(&n), now()),
                "kind {k:?} must not stall"
            );
        }
    }

    /// A driver node that reached a non-`pending` status (running / blocked /
    /// terminal) is progressing, so it is never flagged even if idle.
    #[test]
    fn non_pending_driver_is_not_stalled() {
        let created = now() - STALL_GRACE - Duration::hours(1);
        for s in [
            Status::Running,
            Status::Blocked,
            Status::Done,
            Status::Failed,
            Status::Cancelled,
        ] {
            let n = node(s, 0, created);
            assert!(
                !is_stalled(Status::Pending, Kind::Orchestrate, Some(&n), now()),
                "driver status {s:?} must not stall"
            );
        }
    }

    /// A run with no driver node yet cannot be judged — not stalled.
    #[test]
    fn missing_driver_node_is_not_stalled() {
        assert!(!is_stalled(Status::Pending, Kind::Orchestrate, None, now()));
    }

    fn created() -> DateTime<Utc> {
        "2026-08-06T11:00:00Z".parse().unwrap()
    }

    /// The exact stillborn signature: pending, dead supervisor, zero nodes, no
    /// forward progress since creation.
    #[test]
    fn stillborn_signature_is_detected() {
        assert!(is_stillborn(
            Status::Pending,
            false,
            0,
            created(),
            created()
        ));
    }

    /// An alive supervisor is a run that is (or may still be) starting — never
    /// stillborn, however fresh.
    #[test]
    fn alive_supervisor_is_not_stillborn() {
        assert!(!is_stillborn(
            Status::Pending,
            true,
            0,
            created(),
            created()
        ));
    }

    /// A run that created its first node started — not stillborn, even with a
    /// dead supervisor (that is an orphaned-but-started run, a different shape).
    #[test]
    fn nonzero_node_count_is_not_stillborn() {
        assert!(!is_stillborn(
            Status::Pending,
            false,
            1,
            created(),
            created()
        ));
    }

    /// Any forward progress (`updated_at` past `created_at`) means the
    /// supervisor did something before dying — not the never-started shape.
    #[test]
    fn forward_progress_is_not_stillborn() {
        let updated = created() + Duration::seconds(1);
        assert!(!is_stillborn(Status::Pending, false, 0, created(), updated));
    }

    /// A non-`pending` run started (and possibly finished) — never stillborn,
    /// whatever the counters say.
    #[test]
    fn non_pending_run_is_not_stillborn() {
        for s in [
            Status::Running,
            Status::Blocked,
            Status::Done,
            Status::Failed,
            Status::Cancelled,
        ] {
            assert!(
                !is_stillborn(s, false, 0, created(), created()),
                "status {s:?} must not be stillborn"
            );
        }
    }

    // ── is_orphaned: supervisor died mid-run (node_count > 0) ──────────────

    /// The core orphan signature: a `pending` run with ≥1 node, a dead
    /// supervisor, and a manifest clock idle past the grace window — the exact
    /// "supervisor died mid-run, work stranded" shape (issue `run-wait-still`).
    #[test]
    fn pending_dead_supervisor_past_grace_is_orphaned() {
        let idle = now() - ORPHAN_GRACE - Duration::seconds(1);
        assert!(is_orphaned(Status::Pending, false, 1, idle, now()));
    }

    /// A `running` run (a node reached `running` before the supervisor died) is
    /// orphaned by the same logic — the issue scopes in both pending and running.
    #[test]
    fn running_dead_supervisor_past_grace_is_orphaned() {
        let idle = now() - ORPHAN_GRACE - Duration::minutes(30);
        assert!(is_orphaned(Status::Running, false, 3, idle, now()));
    }

    /// The grace-window guard: a dead supervisor with a still-fresh manifest
    /// clock is NOT orphaned. This is the transient-state protection — a
    /// supervisor caught mid-reattach/restart must not be misread as stranded,
    /// and it is why the existing `run wait` timeout test (a freshly-noded
    /// pending run) keeps blocking rather than settling early.
    #[test]
    fn recently_active_dead_supervisor_is_not_orphaned() {
        let recent = now() - Duration::minutes(1);
        assert!(!is_orphaned(Status::Pending, false, 1, recent, now()));
    }

    /// Exactly at the grace boundary is not yet orphaned (strict `>`), matching
    /// the orchestrate-stall boundary convention.
    #[test]
    fn exactly_at_orphan_grace_boundary_is_not_orphaned() {
        let boundary = now() - ORPHAN_GRACE;
        assert!(!is_orphaned(Status::Pending, false, 1, boundary, now()));
    }

    /// A live supervisor is a normal heads-down worker, never an orphan — this
    /// is the distinction the liveness probe buys over the plain "pending run
    /// with nodes" shape, however long it has been idle.
    #[test]
    fn alive_supervisor_is_never_orphaned() {
        let idle = now() - ORPHAN_GRACE - Duration::hours(2);
        assert!(!is_orphaned(Status::Pending, true, 1, idle, now()));
    }

    /// Zero nodes is the stillborn case, not the orphan case — `is_orphaned`
    /// must not fire on it (they are mutually exclusive on `node_count`).
    #[test]
    fn zero_nodes_is_not_orphaned() {
        let idle = now() - ORPHAN_GRACE - Duration::hours(1);
        assert!(!is_orphaned(Status::Pending, false, 0, idle, now()));
    }

    /// A terminal (or `blocked`) run is never orphaned: terminal runs settled,
    /// and a `blocked` run is a deliberate human-action handoff, not a stranded
    /// supervisor.
    #[test]
    fn terminal_or_blocked_run_is_not_orphaned() {
        let idle = now() - ORPHAN_GRACE - Duration::hours(1);
        for s in [
            Status::Blocked,
            Status::Done,
            Status::Failed,
            Status::Cancelled,
        ] {
            assert!(
                !is_orphaned(s, false, 1, idle, now()),
                "status {s:?} must not be orphaned"
            );
        }
    }

    // ── stall_kind: the combined verdict the callers act on ────────────────

    /// A zero-node never-progressed run classifies as `Stillborn`.
    #[test]
    fn stall_kind_classifies_stillborn() {
        assert_eq!(
            stall_kind(Status::Pending, false, 0, created(), created(), now()),
            Some(StallKind::Stillborn)
        );
    }

    /// A ≥1-node run idle past the grace with a dead supervisor classifies as
    /// `Orphaned`.
    #[test]
    fn stall_kind_classifies_orphaned() {
        let idle = now() - ORPHAN_GRACE - Duration::seconds(1);
        assert_eq!(
            stall_kind(Status::Pending, false, 1, created(), idle, now()),
            Some(StallKind::Orphaned)
        );
    }

    /// A healthy run (live supervisor, or fresh clock) classifies as neither.
    #[test]
    fn stall_kind_healthy_is_none() {
        // Live supervisor with nodes: still working.
        assert_eq!(
            stall_kind(Status::Running, true, 2, created(), now(), now()),
            None
        );
        // Dead supervisor but within the grace window: transient, not yet judged.
        let recent = now() - Duration::minutes(1);
        assert_eq!(
            stall_kind(Status::Pending, false, 1, created(), recent, now()),
            None
        );
    }
}