Skip to main content

fno_agents/
loop_megawalk.rs

1//! Megawalk driver: MegawalkQueue + MegawalkDispatcher + the `loop run --driver megawalk` arm.
2//!
3//! ## What this module does
4//!
5//! MegawalkQueue shells `fno backlog next` to dequeue work items and
6//! `fno backlog done` to close them.  Claims (ab-7303e5d7) are acquired via
7//! `fno claim acquire` so a second walker cannot pick up the same node, and
8//! released on every close path including error paths.
9//!
10//! MegawalkDispatcher wraps ShelloutDispatcher, injecting per-unit env vars
11//! (`CONTINUE_PROMPT`, `TARGET_SESSION_ID`) so the worker session knows which
12//! node it is driving and the loopcheck termination event carries the matching
13//! session_key.
14//!
15//! ## Cross-cwd termination event delivery (ab-7303e5d7)
16//!
17//! Worker sessions dispatched by the walker run `/target` in their OWN
18//! conductor worktrees (the /target location hard-gate moves them), so their
19//! termination events land in the WORKTREE's events.jsonl AND in the global
20//! `~/.fno/events.jsonl` mirror (via loopcheck's emit_to_both).  The
21//! walker's project journal lives at the walker cwd.  Journal::find_termination
22//! (extended in loop_runtime.rs) scans the project journal first, then falls
23//! back to the global mirror so the walker always finds the event.
24//!
25//! ## Claim idempotency (verified)
26//!
27//! The walker acquires `node:<id>` with holder `target-session:<session_key>`.
28//! When the worker's init-target-state.sh fires (with TARGET_SESSION_ID set),
29//! it calls `fno claim acquire node:<id> --holder target-session:<session_key>`.
30//! core.py:acquire_claim line 209: if `existing.holder == holder`, the re-acquire
31//! is idempotent (refreshes pid/host/acquired_at, emits claim_idempotent_reacquired).
32//! The holder strings match exactly because the walker pre-assigns the session_key
33//! and passes it via TARGET_SESSION_ID, so init-target-state.sh uses the same value.
34//! The re-acquire is NOT blocked by PID mismatch (the refresh accepts any pid).
35//!
36//! ## Module naming
37//!
38//! The module name starts with `loop` so that the LOC-ratchet glob
39//! `crates/fno-agents/src/loop*` counts this file's LOC toward the ratchet
40//! budget deliberately (alongside loopcheck.rs).
41
42use crate::loop_dispatch::ShelloutDispatcher;
43use crate::loop_runtime::{
44    CloseOutcome, DispatchCtx, Dispatcher, Evidence, Journal, LoopError, Queue, Session, Unit,
45};
46use crate::loopcheck::TerminationReason;
47use std::collections::HashMap;
48use std::path::PathBuf;
49use std::process::Command;
50
51// ── Multi-PR umbrella verification (grilled decision 9, simplicity) ──────────
52//
53// Spec B.5 asks: does a walked group-child close prematurely complete the
54// epic, and does the epic surface again via `fno backlog next` after all
55// children are ready?
56//
57// Verified (2026-06-06): NO premature epic completion.
58//
59// Evidence:
60//   cli/src/fno/graph/statuses.py:recompute_statuses derives _status:done
61//   from completed_at on the INDIVIDUAL node only (line 81). There is no
62//   parent-cascade: walking a group-child through cmd_done sets
63//   `completed_at` on that child; the epic's completed_at remains None until
64//   cmd_done is called explicitly on the epic itself. The children summary
65//   index (store.py:_compute_children) is read-only metadata for the Kanban
66//   board; it does not influence _status.
67//
68//   fno.plan._stamp: group-aware graduation writes expected_url_count
69//   on the shared epic doc (first-writer-wins, fno plan set-expected, ab-9e864e42).
70//   The epic backlog node is NOT automatically marked done by the stamp module; the
71//   walker must call fno backlog done on the epic node separately after all
72//   group children are confirmed done. This is correct: the epic node is still
73//   visible in `fno backlog next` until its own completed_at is set.
74//
75// Consequence: no speculative walker-side umbrella machinery is needed here.
76// The current walker closes group-child nodes one at a time; the epic parent
77// continues to surface in `fno backlog next` as a separate ready node. This is
78// deliberate simplicity (grilled decision 9). No gap was found; no test added.
79
80// ── Event-kind prune ledger (Claude's Discretion 2) ──────────────────────────
81//
82// The legacy megawalk.py emits ~29 kinds via _emit_event() into
83// megawalk-events.jsonl. Task 2.4 deletes megawalk.py; the writer dies with
84// it. This table records the fate of each legacy kind so auditors can trace
85// the transition.
86//
87// Legacy kind                  -> Fate
88// ──────────────────────────── ─────────────────────────────────────────────
89// node_complete                -> node_closed (close="closed")
90// node_parked                  -> node_closed (close="parked")
91// walker_paused                -> walk_paused
92// node_failed                  -> node_failed (runtime, unchanged)
93// node_help_requested          -> absorbed: cv-d3943d2a
94//                                 No typed help event post-wedge; a
95//                                 help-stuck session exits without a
96//                                 termination event -> per-unit cap parks it
97//                                 -> streak counts it. Stdout parsing is
98//                                 forbidden at this altitude (locked decision 9).
99// merge_attempt                -> deleted (reconcile + backlog-done cover residue)
100// pr_externally_merged         -> deleted (reconcile covers residue)
101// reconcile_started            -> deleted
102// reconcile_completed          -> deleted
103// reconcile_failed             -> deleted
104// stuck_detection_failed       -> deleted (detector era removed in wedge PR)
105// activity_read_failed         -> deleted (detector era)
106// worktree_stuck_checkin       -> deleted (detector era)
107// walker_started               -> deleted (preflight print replaces this)
108// walker_completed             -> loop_terminated (runtime)
109// walker_aborted               -> loop_terminated (runtime)
110// node_started                 -> loop_unit_dispatched (runtime, unchanged)
111// node_dispatched              -> loop_unit_dispatched (runtime, unchanged)
112// node_error                   -> node_failed (runtime, unchanged)
113// node_skipped                 -> deleted (claim-filter in fno backlog next covers)
114// backlog_empty                -> loop_terminated{reason:NoWork} (runtime)
115// iteration_limit_reached      -> loop_terminated{reason:Budget} (runtime)
116// consecutive_failures_paused  -> walk_paused{policy:consecutive_failures}
117// p0_failure_paused            -> walk_paused{policy:p0_failed}
118//
119// The legacy WRITER (megawalk.py) is deleted in task 2.4. Do not touch
120// megawalk.py in this task.
121
122// ── parallel-cap helper ───────────────────────────────────────────────────────
123
124/// Clamp a parallel-cap value to at least 1.
125///
126/// A cap of 0 (or negative, if caller uses signed math) would prohibit all
127/// dispatch. We clamp to 1 and log a warning so the flag is accepted but
128/// explicit: the caller is responsible for printing the clamp message.
129///
130/// Note: group-2 always executes sequentially (run_loop is single-threaded).
131/// When cap > 1 the verb glue prints one honest line explaining that execution
132/// is still sequential (collision-conservative default, Claude's Discretion 3).
133pub fn clamp_parallel_cap(cap: u64) -> u64 {
134    if cap < 1 {
135        eprintln!(
136            "loop-megawalk: WARNING: --parallel-cap {cap} < 1; clamped to 1 \
137             (boundary: cap must be >= 1)"
138        );
139        1
140    } else {
141        cap
142    }
143}
144
145// ── MegawalkPolicyQueue ───────────────────────────────────────────────────────
146
147/// Per-unit state stored in the policy queue.
148struct PolicyUnitEntry {
149    unit: Unit,
150}
151
152// ── shared predicate ──────────────────────────────────────────────────────────
153
154/// Returns true when the termination reason is a successful close
155/// (DonePRGreen or DoneAdvisory). Used in three policy sites; extracted to
156/// avoid triplication (sigma-review finding 4).
157fn is_done_reason(r: &TerminationReason) -> bool {
158    matches!(
159        r,
160        TerminationReason::DonePRGreen | TerminationReason::DoneAdvisory
161    )
162}
163
164/// Walk policy state tracker for the megawalk driver.
165///
166/// This struct is NOT a full Queue impl by itself; it is the policy layer that
167/// wraps or extends the shell-based MegawalkQueue. For unit tests it acts as a
168/// standalone Queue: `push_unit` / `next` / `close` with full policy tracking.
169///
170/// ## Consecutive-failure pause (3)
171///
172/// A unit is a "failure" when close() is called with evidence.reason NOT in
173/// {DonePRGreen, DoneAdvisory}. A successful close resets the streak. When the
174/// streak reaches 3, the next next() call returns
175/// Err(LoopError::Pause{policy:"consecutive_failures", detail}).
176///
177/// ## p0 immediate pause
178///
179/// If a p0 unit fails (close called with non-Done reason and is_p0=true in the
180/// per-unit state), the NEXT next() call returns
181/// Err(LoopError::Pause{policy:"p0_failed", detail:uid}) immediately, regardless
182/// of streak.
183///
184/// ## Park-on-help absorption
185///
186/// No typed help event exists post-wedge (verified: only mission-emit.sh prints
187/// a help tag to stderr; loopcheck emits none). A help-stuck session exits
188/// without a termination event -> node_failed -> per-unit cap parks it ->
189/// streak counts it. See carveout cv-d3943d2a (typed help source = future work).
190/// Locked decision 9 forbids stdout parsing at this altitude.
191pub struct MegawalkPolicyQueue {
192    /// Units queued for dispatch (push_unit adds here; next consumes from front).
193    pending: std::collections::VecDeque<PolicyUnitEntry>,
194    /// Whether a p0 failure happened; set in record_close for is_p0 failures.
195    p0_failure: Option<String>, // unit id of the failed p0 unit
196    /// Consecutive failure streak counter.
197    consecutive_failures: usize,
198    /// Unit IDs involved in the current streak (for Pause detail).
199    streak_ids: Vec<String>,
200}
201
202impl MegawalkPolicyQueue {
203    /// Construct an empty policy queue.
204    pub fn new() -> Self {
205        Self {
206            pending: std::collections::VecDeque::new(),
207            p0_failure: None,
208            consecutive_failures: 0,
209            streak_ids: vec![],
210        }
211    }
212
213    /// Add a unit to the back of the queue.
214    ///
215    /// `is_p0`: when true, a failure on this unit triggers an immediate pause.
216    pub fn push_unit(&mut self, unit: Unit, is_p0: bool) {
217        // is_p0 is passed to record_close separately; the entry only holds the unit.
218        let _ = is_p0;
219        self.pending.push_back(PolicyUnitEntry { unit });
220    }
221
222    /// Record the outcome of a close() call for policy tracking.
223    ///
224    /// Call this AFTER queue.close() returns, with the same evidence passed to close.
225    /// `is_p0`: whether the unit that was closed had p0 priority.
226    ///
227    /// This is a test-accessible hook. In run_loop the policy tracking is done
228    /// via the Queue::close() implementation (which calls record_close internally).
229    pub fn record_close(&mut self, unit: &Unit, evidence: &Evidence, is_p0: bool) {
230        let is_success = is_done_reason(&evidence.reason);
231
232        if is_success {
233            // Success: reset consecutive-failure streak AND p0_failure.
234            // p0_failure was never cleared here before (sigma-review finding 3);
235            // omitting the reset caused spurious pauses after a successful
236            // recovery unit followed a failed p0 unit.
237            self.consecutive_failures = 0;
238            self.streak_ids.clear();
239            self.p0_failure = None;
240        } else {
241            // Failure: increment streak.
242            self.consecutive_failures += 1;
243            self.streak_ids.push(unit.id.clone());
244            // p0 failure: record for immediate pause on next next() call.
245            if is_p0 {
246                self.p0_failure = Some(unit.id.clone());
247            }
248        }
249    }
250
251    /// Check whether walk policy requires a pause.
252    ///
253    /// Returns `Some((policy, detail))` if a pause is warranted; `None` otherwise.
254    /// Called at the top of next() before dequeuing.
255    pub fn should_pause(&self) -> Option<(String, String)> {
256        // p0 failure takes precedence.
257        if let Some(ref uid) = self.p0_failure {
258            return Some(("p0_failed".to_string(), uid.clone()));
259        }
260        // Consecutive-failure streak.
261        if self.consecutive_failures >= 3 {
262            let detail = self.streak_ids.join(" ");
263            return Some(("consecutive_failures".to_string(), detail));
264        }
265        None
266    }
267}
268
269impl Default for MegawalkPolicyQueue {
270    fn default() -> Self {
271        Self::new()
272    }
273}
274
275impl Queue for MegawalkPolicyQueue {
276    fn next(&mut self) -> Result<Option<Unit>, LoopError> {
277        // Check policy before dequeuing.
278        if let Some((policy, detail)) = self.should_pause() {
279            return Err(LoopError::Pause { policy, detail });
280        }
281        // Dequeue the next unit.
282        match self.pending.pop_front() {
283            None => Ok(None),
284            Some(entry) => Ok(Some(entry.unit)),
285        }
286    }
287
288    fn close(&mut self, unit: &Unit, evidence: &Evidence) -> Result<CloseOutcome, LoopError> {
289        // Determine if this unit was p0 (we no longer have the PolicyUnitEntry
290        // since it was consumed by next(); for test use, is_p0 is always false
291        // unless record_close is called separately).
292        // For production use, the MegawalkQueue wraps this and passes is_p0.
293        // For unit tests that call record_close() separately, close() just parks.
294        let is_success = is_done_reason(&evidence.reason);
295        if is_success {
296            // Success: reset streak AND p0_failure (sigma-review finding 3).
297            self.consecutive_failures = 0;
298            self.streak_ids.clear();
299            self.p0_failure = None;
300        } else {
301            self.consecutive_failures += 1;
302            self.streak_ids.push(unit.id.clone());
303        }
304
305        let outcome = if is_success {
306            CloseOutcome::Closed
307        } else {
308            CloseOutcome::Parked(format!("policy-park: {:?}", evidence.reason))
309        };
310        Ok(outcome)
311    }
312}
313
314// ── constants ─────────────────────────────────────────────────────────────────
315
316/// Maximum number of claim-held skips inside a single next() call before
317/// giving up and returning an error.  Prevents infinite loops when every
318/// ready node is claimed by live sessions.
319const MAX_CLAIM_RETRIES: usize = 5;
320
321// ── helper: run fno sub-command ───────────────────────────────────────────────
322
323/// Build a Command for the `fno` binary.
324///
325/// Binary resolution: `abi_bin` (the path/name given at construction time,
326/// overridden by `$FNO_BIN` for tests).  If `FNO_BIN` is set and non-empty,
327/// it wins; otherwise `abi_bin` is used as-is (callers pass "fno" for
328/// production and a tempdir stub path for tests).
329pub(crate) fn abi_cmd(abi_bin: &str) -> Command {
330    let binary = std::env::var("FNO_BIN")
331        .ok()
332        .filter(|s| !s.is_empty())
333        .unwrap_or_else(|| abi_bin.to_string());
334    Command::new(binary)
335}
336
337/// Run a spawn closure, retrying briefly on ETXTBSY ("Text file busy", os
338/// error 26). The spawned file is the `fno` / `fno-agents` binary: a
339/// concurrent `fno update` relinks it in place, and under `cargo test` a
340/// sibling thread that just wrote+exec'd a stub leaves a transient write-fd
341/// open in another thread's fork window - either way the kernel can refuse
342/// the exec with ETXTBSY. The condition clears within microseconds once the
343/// writing fd closes, so a bounded retry turns a hard spawn failure into a
344/// short wait. Any other error - and the successful value - passes through
345/// unchanged.
346pub(crate) fn retry_etxtbsy<T>(
347    mut spawn: impl FnMut() -> std::io::Result<T>,
348) -> std::io::Result<T> {
349    const MAX_RETRIES: u32 = 5;
350    let mut attempt: u32 = 0;
351    loop {
352        match spawn() {
353            Err(e) if e.raw_os_error() == Some(libc::ETXTBSY) && attempt < MAX_RETRIES => {
354                attempt += 1;
355                std::thread::sleep(std::time::Duration::from_millis(2 * u64::from(attempt)));
356            }
357            other => return other,
358        }
359    }
360}
361
362/// Run `fno doctor --json` best-effort and return whether the output indicates
363/// a stale installation.  Any failure (I/O, parse) is treated as "unknown"
364/// and does not append the staleness hint.
365fn is_abi_stale(abi_bin: &str) -> bool {
366    let out = match abi_cmd(abi_bin).args(["doctor", "--json"]).output() {
367        Ok(o) => o,
368        Err(_) => return false,
369    };
370    let stdout = String::from_utf8_lossy(&out.stdout);
371    // Expect {"status":"stale",...} - check the status field.
372    if let Ok(v) = serde_json::from_str::<serde_json::Value>(stdout.trim()) {
373        return v["status"].as_str() == Some("stale");
374    }
375    false
376}
377
378/// Append a staleness hint to an error message if `fno doctor` says stale.
379pub(crate) fn maybe_stale_hint(msg: String, abi_bin: &str) -> String {
380    if is_abi_stale(abi_bin) {
381        format!("{msg}; installed fno may be stale - run `fno update`")
382    } else {
383        msg
384    }
385}
386
387// ── session key generation ────────────────────────────────────────────────────
388
389/// Generate a unique session key in the same shape used by init-target-state.sh:
390/// `{utc %Y%m%dT%H%M%SZ}-{infix}{pid}-{6 hex}`.
391///
392/// The infix distinguishes the assigning driver at a glance in logs:
393/// "mw" = megawalk, "mt" = megatron (group 3).
394pub(crate) fn gen_session_key_with_infix(infix: &str) -> String {
395    let ts = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string();
396    let pid = std::process::id();
397    // 3 random bytes -> 6 hex chars.
398    let entropy: u32 = {
399        let mut buf = [0u8; 3];
400        // Best-effort: use /dev/urandom bytes; fall back to a mix of pid+time.
401        if let Ok(mut f) = std::fs::File::open("/dev/urandom") {
402            use std::io::Read;
403            let _ = f.read_exact(&mut buf);
404        } else {
405            buf[0] = (pid & 0xFF) as u8;
406            buf[1] = ((pid >> 8) & 0xFF) as u8;
407            buf[2] = (chrono::Utc::now().timestamp_subsec_nanos() & 0xFF) as u8;
408        }
409        u32::from_le_bytes([buf[0], buf[1], buf[2], 0])
410    };
411    format!("{ts}-{infix}{pid}-{entropy:06x}")
412}
413
414/// Megawalk-assigned session key (the "mw" infix).
415fn gen_session_key() -> String {
416    gen_session_key_with_infix("mw")
417}
418
419// ── mission env extraction ────────────────────────────────────────────────────
420
421/// Extract TARGET_MISSION_* env vars from a `_node_summary` JSON value.
422///
423/// Mirrors Python `extract_mission_env()` in megawalk.py:75-117:
424///   - If `mission_id` is null/absent: returns Ok(vec![]) (non-fleet node).
425///   - If `mission_id` is set: all four vars are required.
426///     - `mission_wave` null/absent -> Err(Queue) naming node + field.
427///     - `mission_slug` null/absent -> Err(Queue) naming node + field.
428///     - `mission_from_msg_id` null -> maps to "" (Python line 117 behavior).
429///
430/// Returns `Err(LoopError::Queue(...))` on corrupted fleet metadata so the
431/// commander is not silently stranded.
432fn extract_mission_env(
433    v: &serde_json::Value,
434    node_id: &str,
435) -> Result<Vec<(String, String)>, LoopError> {
436    let mission_id = match v["mission_id"].as_str() {
437        Some(s) if !s.is_empty() => s.to_string(),
438        _ => {
439            // null, absent, or empty string: non-fleet node.
440            return Ok(vec![]);
441        }
442    };
443
444    // mission_wave: must be present and coercible to a string.
445    let mission_wave = match &v["mission_wave"] {
446        serde_json::Value::Null => {
447            return Err(LoopError::Queue(format!(
448                "node {node_id:?} has mission_id={mission_id:?} but mission_wave is missing; \
449                 dispatcher bug (corrupted fleet metadata)"
450            )));
451        }
452        w => w.to_string().trim_matches('"').to_string(),
453    };
454    if mission_wave.is_empty() || mission_wave == "null" {
455        return Err(LoopError::Queue(format!(
456            "node {node_id:?} has mission_id={mission_id:?} but mission_wave is null; \
457             dispatcher bug (corrupted fleet metadata)"
458        )));
459    }
460
461    // mission_slug: must be a non-empty string.
462    let mission_slug = match v["mission_slug"].as_str() {
463        Some(s) if !s.is_empty() => s.to_string(),
464        _ => {
465            return Err(LoopError::Queue(format!(
466                "node {node_id:?} has mission_id={mission_id:?} but mission_slug is missing \
467                 or empty; dispatcher bug (corrupted fleet metadata)"
468            )));
469        }
470    };
471
472    // mission_from_msg_id: null -> "" (Python line 117).
473    let mission_from_msg_id = v["mission_from_msg_id"].as_str().unwrap_or("").to_string();
474
475    Ok(vec![
476        ("TARGET_MISSION_ID".to_string(), mission_id),
477        ("TARGET_MISSION_WAVE".to_string(), mission_wave),
478        ("TARGET_MISSION_SLUG".to_string(), mission_slug),
479        (
480            "TARGET_MISSION_FROM_MSG_ID".to_string(),
481            mission_from_msg_id,
482        ),
483    ])
484}
485
486// ── MegawalkQueue ─────────────────────────────────────────────────────────────
487
488/// Per-unit claim state stored in `MegawalkQueue::active_claims`.
489struct ClaimEntry {
490    /// The session key used for the node claim (matched by close() for release).
491    session_key: String,
492    /// Whether this unit has p0 priority. A p0 failure triggers an immediate
493    /// walk pause regardless of the consecutive-failure streak.
494    is_p0: bool,
495}
496
497/// A Queue that shells `fno backlog next` / `fno backlog done` and coordinates
498/// node claims so two walkers never dispatch the same node simultaneously.
499///
500/// ## Walk policy (folded in, not a separate wrapper)
501///
502/// Policy state lives directly in `MegawalkQueue` so the production path uses
503/// the same policy as the test path - no separate wrapper needed. The policy
504/// is the same as `MegawalkPolicyQueue`:
505///   - consecutive-failure streak of 3 -> Pause{consecutive_failures}
506///   - p0 unit failure -> Pause{p0_failed} (immediate, no streak needed)
507///   - Success (DonePRGreen | DoneAdvisory) resets the streak.
508///
509/// The `priority` field from the backlog-next JSON is stored per-unit in
510/// `active_claims` so `close()` can check it for the p0 rule.
511///
512/// ## --max-units N (once-mode)
513///
514/// When `max_units` is `Some(N)`, `next()` returns `None` (Drained) after N
515/// units have been closed by `close()`. This maps the `/megawalk once` modifier
516/// (task 2.4 uses `--max-units 1`) and general "execute at most N units then
517/// stop" semantics. `N` must be >= 1; the CLI gate enforces N > 0 (exit 2).
518/// `units_closed` is incremented at the END of each `close()` call so the cap
519/// fires on the NEXT `next()` call, giving the correct semantics: close the
520/// N-th unit, then the outer loop calls `next()` which returns None -> NoWork.
521pub struct MegawalkQueue {
522    /// Path or name of the fno binary.  `$FNO_BIN` env overrides for tests.
523    abi_bin: String,
524    /// Optional `--project <name>` filter.
525    project: Option<String>,
526    /// When true, pass `--all` to `fno backlog next`.
527    all: bool,
528    /// Map from node id -> ClaimEntry for active dispatches.
529    /// Stores session_key (for claim release/hold) and is_p0 (for policy).
530    /// On Parked/Refused outcomes the entry is KEPT (park-exclusion hold).
531    /// On Closed the entry is removed and the claim is released.
532    active_claims: HashMap<String, ClaimEntry>,
533    // ── policy state ──────────────────────────────────────────────────────────
534    /// Pending p0 failure: the unit id of a p0 unit that failed. When set,
535    /// the NEXT next() call returns a pause immediately.
536    policy_p0_failure: Option<String>,
537    /// Consecutive failure streak counter. A "failure" is any close with
538    /// evidence.reason NOT in {DonePRGreen, DoneAdvisory}.
539    policy_consecutive_failures: usize,
540    /// Unit IDs involved in the current streak (for Pause detail string).
541    policy_streak_ids: Vec<String>,
542    // ── max-units cap (once-mode) ─────────────────────────────────────────────
543    /// Optional cap on total units closed. When set, next() returns None
544    /// (Drained) after `units_closed` reaches this value.
545    max_units: Option<u64>,
546    /// Count of units closed so far (incremented at end of close()).
547    units_closed: u64,
548    /// Optional `--mission <id>` selection filter (group 3, ab-9fd662c6).
549    /// A megatron child walk passes this so the walk works ONLY the
550    /// mission's nodes and never drifts into the project's general backlog.
551    mission: Option<String>,
552}
553
554impl MegawalkQueue {
555    /// Construct a MegawalkQueue.
556    ///
557    /// `abi_bin`: "fno" in production; a stub path in tests (or override via
558    ///   `$FNO_BIN`).
559    /// `project`: optional project filter passed as `--project <p>` to `fno backlog next`.
560    /// `all`: when true, pass `--all` instead of `--project`.
561    pub fn new(abi_bin: String, project: Option<String>, all: bool) -> Self {
562        Self::new_with_max_units(abi_bin, project, all, None)
563    }
564
565    /// Construct a MegawalkQueue with an optional max-units cap.
566    ///
567    /// `max_units`: when `Some(N)`, next() returns None (Drained) after N units
568    /// have been closed. Maps the `--max-units` CLI flag / `/megawalk once`.
569    /// Must be >= 1; callers are responsible for rejecting 0 before calling
570    /// (the verb glue in loop_target.rs exits 2 on N == 0).
571    pub fn new_with_max_units(
572        abi_bin: String,
573        project: Option<String>,
574        all: bool,
575        max_units: Option<u64>,
576    ) -> Self {
577        Self {
578            abi_bin,
579            project,
580            all,
581            active_claims: HashMap::new(),
582            policy_p0_failure: None,
583            policy_consecutive_failures: 0,
584            policy_streak_ids: vec![],
585            max_units,
586            units_closed: 0,
587            mission: None,
588        }
589    }
590
591    /// Builder: set the `--mission <id>` selection filter (megatron child walks).
592    pub fn with_mission(mut self, mission: Option<String>) -> Self {
593        self.mission = mission;
594        self
595    }
596
597    /// Check whether walk policy requires a pause. Returns a typed
598    /// `LoopError::Pause { policy, detail }` when a pause is warranted, `None`
599    /// otherwise. Called at the top of `next()` before shelling out.
600    fn policy_check(&self) -> Option<LoopError> {
601        // p0 failure takes precedence over streak.
602        if let Some(ref uid) = self.policy_p0_failure {
603            return Some(LoopError::Pause {
604                policy: "p0_failed".to_string(),
605                detail: uid.clone(),
606            });
607        }
608        if self.policy_consecutive_failures >= 3 {
609            let detail = self.policy_streak_ids.join(" ");
610            return Some(LoopError::Pause {
611                policy: "consecutive_failures".to_string(),
612                detail,
613            });
614        }
615        None
616    }
617
618    /// Update policy state after a close() call.
619    fn policy_record_close(&mut self, unit_id: &str, is_success: bool, is_p0: bool) {
620        if is_success {
621            // Success: reset streak AND p0_failure (sigma-review finding 3).
622            // p0_failure was not cleared here before, causing spurious pauses
623            // after a recovery unit succeeded following a failed p0 unit.
624            self.policy_consecutive_failures = 0;
625            self.policy_streak_ids.clear();
626            self.policy_p0_failure = None;
627        } else {
628            // Failure: increment streak.
629            self.policy_consecutive_failures += 1;
630            self.policy_streak_ids.push(unit_id.to_string());
631            if is_p0 {
632                self.policy_p0_failure = Some(unit_id.to_string());
633            }
634        }
635    }
636}
637
638impl Queue for MegawalkQueue {
639    /// Dequeue the next ready backlog node.
640    ///
641    /// Algorithm:
642    /// 0. Check walk policy (consecutive-failure streak / p0 failure).
643    ///    Returns Err(LoopError::Pause{policy, detail}) when policy requires a pause.
644    /// 1. Shell `fno backlog next [--project P | --all]`.
645    /// 2. Empty output (literal "null") -> return Ok(None).
646    /// 3. Parse JSON as `_node_summary` shape (captures `priority` for p0 check).
647    /// 4. Generate a unique session_key.
648    /// 5. Shell `fno claim acquire node:<id> --holder target-session:<session_key>
649    ///    --ttl 2h --reason "megawalk walker dispatch"`.
650    ///    Exit 0 -> record claim, return the Unit.
651    ///    Exit 1 (held by other) -> loop back to step 1 (the live-claim filter
652    ///    in _live_claimed_node_ids excludes it on the next call).
653    /// 6. Bound retries at MAX_CLAIM_RETRIES; on exhaustion return LoopError::Queue.
654    ///
655    /// Malformed JSON or verb failure -> LoopError::Queue with staleness hint.
656    fn next(&mut self) -> Result<Option<Unit>, LoopError> {
657        // ── max-units cap check (step -1) ─────────────────────────────────
658        // When max_units is set and units_closed has reached the cap, signal
659        // Drained so the outer loop terminates with NoWork.
660        if let Some(cap) = self.max_units {
661            if self.units_closed >= cap {
662                return Ok(None);
663            }
664        }
665
666        // ── walk policy check (step 0) ────────────────────────────────────
667        if let Some(err) = self.policy_check() {
668            return Err(err);
669        }
670
671        for attempt in 0..MAX_CLAIM_RETRIES {
672            let _ = attempt; // retry count tracked implicitly
673
674            // ── shell fno backlog next ────────────────────────────────────────
675            let mut cmd = abi_cmd(&self.abi_bin);
676            cmd.args(["backlog", "next"]);
677            if self.all {
678                cmd.arg("--all");
679            } else if let Some(ref p) = self.project {
680                cmd.args(["--project", p]);
681            }
682            if let Some(ref m) = self.mission {
683                cmd.args(["--mission", m]);
684            }
685
686            let out = retry_etxtbsy(|| cmd.output()).map_err(|e| {
687                LoopError::Queue(maybe_stale_hint(
688                    format!("fno backlog next: spawn failed: {e}"),
689                    &self.abi_bin,
690                ))
691            })?;
692
693            if !out.status.success() {
694                let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();
695                return Err(LoopError::Queue(maybe_stale_hint(
696                    format!("fno backlog next: exit {}: {stderr}", out.status),
697                    &self.abi_bin,
698                )));
699            }
700
701            let stdout = String::from_utf8_lossy(&out.stdout).trim().to_string();
702
703            // ── null -> empty backlog ─────────────────────────────────────────
704            if stdout == "null" || stdout.is_empty() {
705                return Ok(None);
706            }
707
708            // ── parse _node_summary JSON ──────────────────────────────────────
709            let v: serde_json::Value = serde_json::from_str(&stdout).map_err(|e| {
710                LoopError::Queue(maybe_stale_hint(
711                    format!("fno backlog next: JSON parse error: {e} (stdout: {stdout:?})"),
712                    &self.abi_bin,
713                ))
714            })?;
715
716            let id = match v["id"].as_str() {
717                Some(s) if !s.is_empty() => s.to_string(),
718                _ => {
719                    return Err(LoopError::Queue(maybe_stale_hint(
720                        format!("fno backlog next: missing or empty 'id' field in: {stdout:?}"),
721                        &self.abi_bin,
722                    )));
723                }
724            };
725
726            let title = v["title"].as_str().unwrap_or("(untitled)").to_string();
727            let plan_path = v["plan_path"].as_str().map(|s| s.to_string());
728            // Extract priority for p0 policy. The backlog-next JSON includes a
729            // "priority" field (p0/p1/p2/p3). Treat anything other than "p0"
730            // (or missing) as non-p0; best-effort (a corrupt field is non-p0).
731            let is_p0 = v["priority"].as_str() == Some("p0");
732
733            // ── extract mission env (fleet nodes) ─────────────────────────────
734            // Mirrors Python extract_mission_env(): if mission_id is set, all
735            // four TARGET_MISSION_* vars are required (wave + slug must be
736            // present; from_msg_id maps to "" when null). Corrupted metadata
737            // is a loud Queue error naming the node and missing field.
738            let extra_env = extract_mission_env(&v, &id)?;
739
740            let session_key = gen_session_key();
741
742            // ── acquire the node claim ────────────────────────────────────────
743            let claim_key = format!("node:{id}");
744            let claim_holder = format!("target-session:{session_key}");
745
746            let claim_out = retry_etxtbsy(|| {
747                abi_cmd(&self.abi_bin)
748                    .args([
749                        "claim",
750                        "acquire",
751                        &claim_key,
752                        "--holder",
753                        &claim_holder,
754                        "--ttl",
755                        "2h",
756                        "--reason",
757                        "megawalk walker dispatch",
758                    ])
759                    .env(
760                        "FNO_CLAIMS_ROOT",
761                        std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()),
762                    )
763                    .output()
764            })
765            .map_err(|e| {
766                LoopError::Queue(maybe_stale_hint(
767                    format!("fno claim acquire: spawn failed: {e}"),
768                    &self.abi_bin,
769                ))
770            })?;
771
772            if claim_out.status.success() {
773                // Claim acquired; store session_key + is_p0 for close().
774                self.active_claims.insert(
775                    id.clone(),
776                    ClaimEntry {
777                        session_key: session_key.clone(),
778                        is_p0,
779                    },
780                );
781                return Ok(Some(Unit {
782                    id,
783                    title,
784                    session_key,
785                    plan_path,
786                    extra_env,
787                }));
788            }
789
790            // Branch on the exit code (sigma-review finding 1 - exit-code collapse fix).
791            //
792            // The claim CLI contract (cli/src/fno/claims/cli.py header):
793            //   exit 1 = ClaimHeldByOther  -> retry (live-claims filter on next call)
794            //   exit 2 = validation error  -> surface immediately; do NOT loop
795            //   exit 3 = ClaimCorrupted / ClaimGoneAway -> surface immediately
796            //   other  -> unexpected; surface immediately
797            //
798            // Before this fix every non-zero exit was treated as "held" and the
799            // loop continued silently, hiding validation and corruption errors.
800            match claim_out.status.code() {
801                Some(1) => {
802                    // Held by another session; let the live-claim filter in
803                    // `fno backlog next` exclude this node on the next call.
804                    // Continue the retry loop.
805                }
806                _ => {
807                    // Validation error, corruption, or unexpected exit.
808                    // Surface immediately as a Queue error.
809                    let stderr = String::from_utf8_lossy(&claim_out.stderr)
810                        .trim()
811                        .to_string();
812                    let code = claim_out.status.code().unwrap_or(-1);
813                    return Err(LoopError::Queue(maybe_stale_hint(
814                        format!("fno claim acquire {claim_key}: exit {code}: {stderr}"),
815                        &self.abi_bin,
816                    )));
817                }
818            }
819        }
820
821        // Exhausted MAX_CLAIM_RETRIES picks without finding a claimable node.
822        Err(LoopError::Queue(maybe_stale_hint(
823            format!(
824                "fno backlog next: exhausted {MAX_CLAIM_RETRIES} attempts; every ready node \
825                 is claimed by another session (last node may be stuck)"
826            ),
827            &self.abi_bin,
828        )))
829    }
830
831    /// Mark a unit as closed.
832    ///
833    /// DonePRGreen | DoneAdvisory -> shell `fno backlog done <id>`.
834    ///   Exit 0 -> CloseOutcome::Closed.
835    ///   Nonzero -> CloseOutcome::Parked(stderr tail).
836    ///
837    /// Any other reason -> CloseOutcome::Parked(reason description).
838    ///   Does NOT call `fno backlog done` (task 2.2 handles refusal paths).
839    ///
840    /// Updates walk policy state (consecutive-failure streak / p0 flag) so
841    /// the NEXT next() call can return a pause when policy warrants it.
842    ///
843    /// ## Park-exclusion (AC2-EDGE): hold claim on Parked/Refused
844    ///
845    /// ONLY releases the node claim when the outcome is CloseOutcome::Closed.
846    /// For Parked and Refused outcomes, the claim is HELD so the live-claims
847    /// selection filter in `_live_claimed_node_ids` (cli/src/fno/graph/cli.py:43-65)
848    /// continues to exclude this node and `fno backlog next` moves on to other
849    /// ready work instead of re-picking the same busted node.
850    ///
851    /// ## Claim TTL refresh after park (same-holder re-acquire finding)
852    ///
853    /// core.py:acquire_claim line 209: a same-holder re-acquire refreshes
854    /// `pid/host/acquired_at` (idempotent). The worker session's init-target-state.sh
855    /// calls `fno claim acquire node:<id> --holder target-session:<session_key>`
856    /// which, being the SAME holder as the walker's claim, is an idempotent
857    /// re-acquire - it rewrites `acquired_at` and `pid` (the WORKER's pid, not
858    /// the walker's). After the worker exits the claim record reflects the
859    /// worker's (now-dead) pid. For a TTL-liveness claim this is fine because
860    /// the filter reads `expires_at` not pid, but the `acquired_at` reset means
861    /// the TTL window is measured from the WORKER's re-acquire time, not the
862    /// walker's original acquire.  To ensure the claim stays LIVE through the
863    /// next walk iteration (while the walker processes the next unit), the
864    /// walker calls `fno claim acquire` again immediately after a park to
865    /// refresh `acquired_at` with the current time and reset the TTL window.
866    fn close(&mut self, unit: &Unit, evidence: &Evidence) -> Result<CloseOutcome, LoopError> {
867        let should_done = is_done_reason(&evidence.reason);
868
869        let outcome = if should_done {
870            let done_out = retry_etxtbsy(|| {
871                abi_cmd(&self.abi_bin)
872                    .args(["backlog", "done", &unit.id])
873                    .output()
874            })
875            .map_err(|e| LoopError::Queue(format!("fno backlog done: spawn failed: {e}")))?;
876
877            if done_out.status.success() {
878                CloseOutcome::Closed
879            } else {
880                let stderr = String::from_utf8_lossy(&done_out.stderr).trim().to_string();
881                CloseOutcome::Parked(if stderr.is_empty() {
882                    format!(
883                        "fno backlog done {} failed (exit {})",
884                        unit.id, done_out.status
885                    )
886                } else {
887                    stderr
888                })
889            }
890        } else {
891            // Append evidence.message when non-empty so synthesized diagnostics
892            // (e.g. "no termination event after N dispatch(es)") are not lost
893            // (sigma-review finding 2).
894            let detail = if evidence.message.is_empty() {
895                format!("session terminated: {:?}", evidence.reason)
896            } else {
897                format!(
898                    "session terminated: {:?}: {}",
899                    evidence.reason, evidence.message
900                )
901            };
902            CloseOutcome::Parked(detail)
903        };
904
905        // ── update walk policy state ──────────────────────────────────────────
906        // Determine is_p0 from the stored claim entry (recorded at dequeue time).
907        // If the entry is gone (e.g. the unit was never in active_claims - only
908        // possible in tests that bypass next()), default to non-p0.
909        let is_p0 = self
910            .active_claims
911            .get(&unit.id)
912            .map(|e| e.is_p0)
913            .unwrap_or(false);
914        self.policy_record_close(&unit.id, should_done, is_p0);
915
916        // ── claim release vs. hold (park-exclusion) ───────────────────────────
917        match &outcome {
918            CloseOutcome::Closed => {
919                // Success: release the claim so the node is no longer live-claimed.
920                let session_key = self
921                    .active_claims
922                    .remove(&unit.id)
923                    .map(|e| e.session_key)
924                    .unwrap_or_else(|| unit.session_key.clone());
925                let claim_key = format!("node:{}", unit.id);
926                let claim_holder = format!("target-session:{session_key}");
927
928                let release_result = retry_etxtbsy(|| {
929                    abi_cmd(&self.abi_bin)
930                        .args(["claim", "release", &claim_key, "--holder", &claim_holder])
931                        .env(
932                            "FNO_CLAIMS_ROOT",
933                            std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()),
934                        )
935                        .output()
936                });
937
938                match release_result {
939                    Ok(o) if !o.status.success() => {
940                        eprintln!(
941                            "loop-megawalk: WARNING: claim release {} failed (exit {}): {}",
942                            claim_key,
943                            o.status,
944                            String::from_utf8_lossy(&o.stderr).trim()
945                        );
946                    }
947                    Err(e) => {
948                        eprintln!(
949                            "loop-megawalk: WARNING: claim release {} spawn failed: {e}",
950                            claim_key
951                        );
952                    }
953                    Ok(_) => {}
954                }
955            }
956            CloseOutcome::Parked(_) | CloseOutcome::Refused(_) => {
957                // Park-exclusion: HOLD the claim so the live-claims filter keeps
958                // skipping this node. The active_claims entry is NOT removed.
959                //
960                // Also refresh the TTL via a same-holder re-acquire. The worker's
961                // idempotent re-acquire during init-target-state.sh rewrites
962                // acquired_at with the worker's pid; after the worker exits the
963                // TTL window is measured from the worker's re-acquire time.
964                // Re-acquiring here refreshes acquired_at to NOW so the claim
965                // stays live through the next walk iteration.
966                let session_key = self
967                    .active_claims
968                    .get(&unit.id)
969                    .map(|e| e.session_key.clone())
970                    .unwrap_or_else(|| unit.session_key.clone());
971                let claim_key = format!("node:{}", unit.id);
972                let claim_holder = format!("target-session:{session_key}");
973
974                let refresh_result = retry_etxtbsy(|| {
975                    abi_cmd(&self.abi_bin)
976                        .args([
977                            "claim",
978                            "acquire",
979                            &claim_key,
980                            "--holder",
981                            &claim_holder,
982                            "--ttl",
983                            "2h",
984                            "--reason",
985                            "megawalk park-exclusion hold",
986                        ])
987                        .env(
988                            "FNO_CLAIMS_ROOT",
989                            std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()),
990                        )
991                        .output()
992                });
993
994                match refresh_result {
995                    Ok(o) if !o.status.success() => {
996                        eprintln!(
997                            "loop-megawalk: WARNING: claim refresh (park-hold) {} failed (exit {}): {}",
998                            claim_key,
999                            o.status,
1000                            String::from_utf8_lossy(&o.stderr).trim()
1001                        );
1002                    }
1003                    Err(e) => {
1004                        eprintln!(
1005                            "loop-megawalk: WARNING: claim refresh (park-hold) {} spawn failed: {e}",
1006                            claim_key
1007                        );
1008                    }
1009                    Ok(_) => {}
1010                }
1011            }
1012        }
1013
1014        // Increment the closed-units counter (used by max_units cap).
1015        //
1016        // Intentional: Parked and Refused outcomes count toward --max-units just
1017        // like Closed outcomes. The semantics of --max-units N are "process N
1018        // units, whatever the outcome" (once-mode). A parked unit consumed one
1019        // dispatch slot; counting it avoids an unbounded walk when every unit parks.
1020        self.units_closed += 1;
1021
1022        Ok(outcome)
1023    }
1024}
1025
1026// ── MegawalkDispatcher ────────────────────────────────────────────────────────
1027
1028/// A Dispatcher that wraps ShelloutDispatcher with per-unit env injection.
1029///
1030/// For each Dispatcher::run call, builds env = static env +
1031/// `CONTINUE_PROMPT="/target no-merge <unit.id>"` (or `/target <unit.id>`
1032/// when allow_merge) + `TARGET_SESSION_ID=<unit.session_key>`.
1033///
1034/// This injects TARGET_SESSION_ID into the worker session so
1035/// init-target-state.sh uses it verbatim (Task 4: the override path in
1036/// init-target-state.sh).  The claim re-acquire in init-target-state.sh then
1037/// has `holder = target-session:<session_key>` matching the walker's claim,
1038/// making it idempotent (see module doc).
1039pub struct MegawalkDispatcher {
1040    driver_lib: PathBuf,
1041    static_env: Vec<(String, String)>,
1042    cwd: PathBuf,
1043    /// Reserved for future use (e.g. fno claim refresh per dispatch).
1044    _abi_bin: String,
1045    allow_merge: bool,
1046}
1047
1048impl MegawalkDispatcher {
1049    pub fn new(
1050        driver_lib: PathBuf,
1051        static_env: Vec<(String, String)>,
1052        cwd: PathBuf,
1053        abi_bin: String,
1054        allow_merge: bool,
1055    ) -> Self {
1056        Self {
1057            driver_lib,
1058            static_env,
1059            // Root dispatched target workers at canonical main: megawalk is
1060            // single-repo target-class, so a walk launched from a linked
1061            // worktree must not start each worker in that worktree (the shared
1062            // .fno/ session-state collision, ab-77b691dc). canonical_repo_root
1063            // is a no-op when already canonical and falls back to the given cwd
1064            // when resolution is ambiguous (git missing / bare).
1065            cwd: crate::paths::canonical_repo_root(&cwd).unwrap_or(cwd),
1066            _abi_bin: abi_bin,
1067            allow_merge,
1068        }
1069    }
1070}
1071
1072impl Dispatcher for MegawalkDispatcher {
1073    fn run(&self, unit: &Unit, ctx: &DispatchCtx) -> Result<Box<dyn Session>, LoopError> {
1074        let continue_prompt = if self.allow_merge {
1075            format!("/target {}", unit.id)
1076        } else {
1077            format!("/target no-merge {}", unit.id)
1078        };
1079
1080        // Build merged env: static + per-unit overrides.
1081        let mut env = self.static_env.clone();
1082        // Override CONTINUE_PROMPT with the per-unit prompt.
1083        // Remove any existing CONTINUE_PROMPT from the static list to avoid
1084        // duplicates (the last value wins in most shells, but explicit removal
1085        // is cleaner).
1086        env.retain(|(k, _)| k != "CONTINUE_PROMPT" && k != "TARGET_SESSION_ID");
1087        env.push(("CONTINUE_PROMPT".to_string(), continue_prompt));
1088        env.push(("TARGET_SESSION_ID".to_string(), unit.session_key.clone()));
1089
1090        // Inject driver-specific extra env (e.g. TARGET_MISSION_* for fleet nodes).
1091        // These come after the static env so they take precedence over any
1092        // identically-named static values (last write wins in most shells).
1093        env.extend(unit.extra_env.iter().cloned());
1094
1095        // Construct a ShelloutDispatcher with the merged env per-unit.
1096        // ShelloutDispatcher is cheap to construct (no I/O at construction time).
1097        let dispatcher = ShelloutDispatcher::new(self.driver_lib.clone(), env, self.cwd.clone());
1098        dispatcher.run(unit, ctx)
1099    }
1100}
1101
1102// ── walk-as-unit termination emission (group 3, ab-9fd662c6) ──────────────────
1103
1104/// Emit a `termination` event for the WALK itself, keyed by `session_key`.
1105///
1106/// A parent loop (megatron) that dispatched this walk as a unit awaits a
1107/// `termination` event matching the unit's session_key - the same contract a
1108/// target session's loop-check satisfies one altitude down. Reuses the
1109/// existing `termination` event kind (loopcheck's), so no new kind and no
1110/// 4-place lockstep edit; `Journal::append` mirrors to the global journal,
1111/// which is how the parent (running in a different cwd) finds it.
1112///
1113/// The reason string is the serde serialization of the walk-level
1114/// TerminationReason (e.g. "NoWork"), the same spelling
1115/// `parse_termination_reason` round-trips.
1116pub fn emit_walk_termination(
1117    journal: &Journal,
1118    session_key: &str,
1119    reason: &TerminationReason,
1120    iterations_used: u64,
1121    units_closed: usize,
1122) -> Result<(), crate::loop_runtime::LoopError> {
1123    // Loud on a non-string serialization: a Debug-format fallback would emit
1124    // a spelling parse_termination_reason cannot match, turning a future
1125    // enum-shape change into a silent find_termination miss at the parent
1126    // (sigma-review). All current variants are unit -> always strings.
1127    let reason_str = match serde_json::to_value(reason) {
1128        Ok(serde_json::Value::String(s)) => s,
1129        other => {
1130            return Err(crate::loop_runtime::LoopError::Journal(format!(
1131                "walk termination reason did not serialize to a string \
1132                 (got {other:?}); refusing to journal an unparseable reason"
1133            )));
1134        }
1135    };
1136    journal.append(
1137        "termination",
1138        serde_json::json!({
1139            "session_id": session_key,
1140            "reason": reason_str,
1141            "message": format!(
1142                "megawalk walk terminated: {reason_str} ({iterations_used} iterations, {units_closed} units closed)"
1143            ),
1144        }),
1145    )
1146}
1147
1148// ── verb glue: pub fn run() ────────────────────────────────────────────────────
1149
1150/// Entry point for `fno-agents loop run --driver megawalk ...`.
1151///
1152/// Called from loop_target.rs when --driver megawalk is specified.
1153///
1154/// Exit codes:
1155/// - 0: NoWork | DonePRGreen | DoneAdvisory (walk completed or backlog empty)
1156/// - 1: Budget | NoProgress | Aborted (walk hit ceiling or failed)
1157/// - 2: usage / configuration error
1158/// - 77: driver binary missing from PATH (preflight failure)
1159/// - 130: Interrupted (SIGINT)
1160#[allow(clippy::too_many_arguments)]
1161pub fn run(
1162    // Parsed flags forwarded from loop_target.rs run_loop_verb_inner.
1163    dispatcher_name: &str,
1164    max_iterations: Option<u64>,
1165    max_turns: u64,
1166    budget_usd: f64,
1167    model: Option<&str>,
1168    prompt_file: Option<&str>,
1169    cli_alias: Option<&str>,
1170    driver_lib_dir: Option<PathBuf>,
1171    cwd: PathBuf,
1172    project: Option<String>,
1173    all: bool,
1174    allow_merge: bool,
1175    parallel_cap: Option<u64>,
1176    max_units: Option<u64>,
1177    mission: Option<String>,
1178    termination_key: Option<String>,
1179) -> i32 {
1180    match run_inner(
1181        dispatcher_name,
1182        max_iterations,
1183        max_turns,
1184        budget_usd,
1185        model,
1186        prompt_file,
1187        cli_alias,
1188        driver_lib_dir,
1189        cwd,
1190        project,
1191        all,
1192        allow_merge,
1193        parallel_cap,
1194        max_units,
1195        mission,
1196        termination_key,
1197    ) {
1198        Ok(code) => code,
1199        Err(e) => {
1200            eprintln!("fno-agents loop megawalk: {e}");
1201            2
1202        }
1203    }
1204}
1205
1206#[allow(clippy::too_many_arguments)]
1207fn run_inner(
1208    dispatcher_name: &str,
1209    max_iterations: Option<u64>,
1210    max_turns: u64,
1211    budget_usd: f64,
1212    model: Option<&str>,
1213    prompt_file: Option<&str>,
1214    cli_alias: Option<&str>,
1215    driver_lib_dir: Option<PathBuf>,
1216    cwd: PathBuf,
1217    project: Option<String>,
1218    all: bool,
1219    allow_merge: bool,
1220    parallel_cap: Option<u64>,
1221    max_units: Option<u64>,
1222    mission: Option<String>,
1223    termination_key: Option<String>,
1224) -> Result<i32, Box<dyn std::error::Error>> {
1225    use crate::loop_dispatch::{driver_default_max, preflight, resolve_driver_binary};
1226    use crate::loop_runtime::{
1227        run_loop, GlobalJournalPath, Journal, LoopBudget, ProjectJournalPath,
1228    };
1229    use crate::loop_target::{exit_code_for_reason, install_sigint_handler, SIGINT_RECEIVED};
1230    use std::sync::atomic::Ordering;
1231
1232    // ── resolve driver-lib-dir ────────────────────────────────────────────────
1233    let lib_dir = match driver_lib_dir {
1234        Some(d) => d,
1235        None => {
1236            if let Ok(env_dir) = std::env::var("FNO_DRIVER_LIB_DIR") {
1237                PathBuf::from(env_dir)
1238            } else {
1239                let candidate = cwd.join("scripts").join("lib");
1240                if candidate.is_dir() {
1241                    candidate
1242                } else {
1243                    eprintln!(
1244                        "fno-agents loop megawalk: cannot resolve driver lib directory. \
1245                         Pass --driver-lib-dir <path> or set FNO_DRIVER_LIB_DIR env."
1246                    );
1247                    return Ok(2);
1248                }
1249            }
1250        }
1251    };
1252
1253    // ── preflight: driver whitelist, lib file, binary ─────────────────────────
1254    let lib_path = match preflight(dispatcher_name, &lib_dir, cli_alias) {
1255        Ok(p) => p,
1256        Err(crate::loop_runtime::LoopError::Dispatch(msg)) => {
1257            eprintln!("fno-agents loop megawalk: {msg}");
1258            return Ok(77);
1259        }
1260        Err(e) => {
1261            eprintln!("fno-agents loop megawalk: {e}");
1262            return Ok(2);
1263        }
1264    };
1265
1266    // ── resolve max_iterations ────────────────────────────────────────────────
1267    let max_iters = match max_iterations {
1268        Some(n) => n,
1269        None => match driver_default_max(&lib_path) {
1270            Ok(n) => n,
1271            Err(e) => {
1272                eprintln!(
1273                    "fno-agents loop megawalk: could not query driver_default_max: {e}; \
1274                     pass --max-iterations explicitly"
1275                );
1276                return Ok(2);
1277            }
1278        },
1279    };
1280
1281    // ── acquire walker singleton claim ────────────────────────────────────────
1282    // Prevents two concurrent megawalk processes from racing on the same backlog.
1283    // Key the singleton on the CANONICAL repo root, matching the canonical cwd
1284    // each dispatched worker is rooted at (MegawalkDispatcher). Otherwise two
1285    // walkers launched from two linked worktrees of the same repo would take
1286    // different `walker:<worktree>` keys yet dispatch workers into the same
1287    // canonical .fno/ state, recreating the collision this change removes
1288    // (codex P2). No-op when already canonical; falls back to cwd when ambiguous.
1289    let walker_root = crate::paths::canonical_repo_root(&cwd).unwrap_or_else(|| cwd.clone());
1290    let walker_key = format!("walker:{}", walker_root.display());
1291    let walker_holder = format!("megawalk-loop:{}", std::process::id());
1292    let abi_bin = std::env::var("FNO_BIN").unwrap_or_else(|_| "fno".to_string());
1293
1294    let walker_claim_result = abi_cmd(&abi_bin)
1295        .args([
1296            "claim",
1297            "acquire",
1298            &walker_key,
1299            "--holder",
1300            &walker_holder,
1301            "--ttl",
1302            "24h",
1303            "--reason",
1304            "megawalk walker singleton",
1305        ])
1306        .output();
1307
1308    match walker_claim_result {
1309        Ok(o) if !o.status.success() => {
1310            let stderr = String::from_utf8_lossy(&o.stderr).trim().to_string();
1311            eprintln!(
1312                "fno-agents loop megawalk: walker singleton already running: {stderr}; \
1313                 another megawalk is active for this project (holder in claim file)"
1314            );
1315            return Ok(1);
1316        }
1317        Err(e) => {
1318            // If fno is not available, warn and continue (best-effort singleton).
1319            eprintln!(
1320                "fno-agents loop megawalk: WARNING: walker claim acquire failed: {e} (continuing)"
1321            );
1322        }
1323        Ok(_) => {}
1324    }
1325
1326    // ── build static env ──────────────────────────────────────────────────────
1327    let abilities_dir = cwd.join(".fno");
1328    let output_file = abilities_dir.join("target-last-output.txt");
1329    let history_file = abilities_dir.join("target-history.txt");
1330    let signal_file = abilities_dir.join("target-promise.signal");
1331
1332    let mut env: Vec<(String, String)> = vec![
1333        (
1334            "OUTPUT_FILE".to_string(),
1335            output_file.to_str().unwrap_or("").to_string(),
1336        ),
1337        (
1338            "HISTORY_FILE".to_string(),
1339            history_file.to_str().unwrap_or("").to_string(),
1340        ),
1341        (
1342            "SIGNAL_FILE".to_string(),
1343            signal_file.to_str().unwrap_or("").to_string(),
1344        ),
1345        ("MAX_TURNS".to_string(), max_turns.to_string()),
1346        ("BUDGET_USD".to_string(), format!("{budget_usd}")),
1347        // CONTINUE_PROMPT is set per-unit by MegawalkDispatcher.
1348        ("CONTINUE_PROMPT".to_string(), String::new()),
1349    ];
1350
1351    if let Some(m) = model {
1352        env.push(("MODEL_FLAG".to_string(), format!("--model {m}")));
1353    } else {
1354        env.push(("MODEL_FLAG".to_string(), String::new()));
1355    }
1356
1357    if let Some(pf) = prompt_file {
1358        env.push(("PROMPT_FILE".to_string(), pf.to_string()));
1359    }
1360
1361    if let Some(cli) = cli_alias {
1362        env.push(("CLI".to_string(), cli.to_string()));
1363    }
1364
1365    env.push((
1366        "FNO_CWD".to_string(),
1367        cwd.to_str().unwrap_or(".").to_string(),
1368    ));
1369
1370    // ── SIGINT handler ────────────────────────────────────────────────────────
1371    install_sigint_handler();
1372
1373    // ── build journal ─────────────────────────────────────────────────────────
1374    let project_events = abilities_dir.join("events.jsonl");
1375    let home_dir = std::env::var("HOME")
1376        .map(PathBuf::from)
1377        .unwrap_or_else(|_| PathBuf::from("/tmp"));
1378    let global_events = home_dir.join(".fno").join("events.jsonl");
1379    let journal = Journal::new(
1380        ProjectJournalPath(project_events),
1381        GlobalJournalPath(global_events),
1382    );
1383
1384    // ── print header ──────────────────────────────────────────────────────────
1385    let binary_name = resolve_driver_binary(dispatcher_name, cli_alias);
1386    let scope = if all {
1387        "all projects".to_string()
1388    } else if let Some(ref p) = project {
1389        format!("project={p}")
1390    } else {
1391        "auto-detected project".to_string()
1392    };
1393    println!("fno-agents loop megawalk");
1394    println!("  driver:     megawalk");
1395    println!("  dispatcher: {dispatcher_name} (binary: {binary_name})");
1396    println!("  scope:      {scope}");
1397    println!("  iterations: {max_iters} max");
1398    println!("  budget:     ${budget_usd} USD");
1399
1400    // ── resume narration (AC4-UI) ─────────────────────────────────────────────
1401    // Shell `fno claim list --prefix node: --include-stale --json` and print
1402    // one header line when stale node claims exist. A stale claim means a prior
1403    // walk was interrupted mid-unit; the walker will re-acquire on contact and
1404    // recover the work. Best-effort only: if the command fails for any reason,
1405    // skip silently (claim narration is informational, not a gate).
1406    {
1407        let claim_out = abi_cmd(&abi_bin)
1408            .args([
1409                "claim",
1410                "list",
1411                "--prefix",
1412                "node:",
1413                "--include-stale",
1414                "--json",
1415            ])
1416            .output();
1417        if let Ok(o) = claim_out {
1418            if o.status.success() {
1419                let stdout = String::from_utf8_lossy(&o.stdout);
1420                // JSON output is a list of claim objects with a "status" field.
1421                // Count entries where status == "stale".
1422                if let Ok(arr) = serde_json::from_str::<serde_json::Value>(stdout.trim()) {
1423                    let stale_count = arr
1424                        .as_array()
1425                        .map(|a| {
1426                            a.iter()
1427                                .filter(|v| v["status"].as_str() == Some("stale"))
1428                                .count()
1429                        })
1430                        .unwrap_or(0);
1431                    if stale_count > 0 {
1432                        println!(
1433                            "resume: {stale_count} stale node claim(s) from a prior walk \
1434                             will be recovered on contact"
1435                        );
1436                    }
1437                }
1438            }
1439        }
1440    }
1441
1442    // ── build queue and dispatcher ────────────────────────────────────────────
1443    let mut queue = MegawalkQueue::new_with_max_units(abi_bin.clone(), project, all, max_units)
1444        .with_mission(mission.clone());
1445    let dispatcher =
1446        MegawalkDispatcher::new(lib_path, env, cwd.clone(), abi_bin.clone(), allow_merge);
1447
1448    // ── parallel-cap notice ───────────────────────────────────────────────────
1449    // Group-2 ships the conservative sequential default (Claude's Discretion 3).
1450    // run_loop is single-threaded; when cap > 1, print one honest line so the
1451    // flag is accepted-but-explicit, never a silent drop.
1452    if let Some(cap) = parallel_cap {
1453        if cap > 1 {
1454            println!(
1455                "megawalk: --parallel-cap {cap} accepted; execution is SEQUENTIAL \
1456                 (collision-conservative default; group-2 serializes regardless of cap)"
1457            );
1458        }
1459    }
1460
1461    // ── max-units notice ──────────────────────────────────────────────────────
1462    if let Some(n) = max_units {
1463        println!("megawalk: --max-units {n} (walk stops after {n} unit(s) closed)");
1464    }
1465
1466    // ── walker-claim release helper ───────────────────────────────────────────
1467    // Called on every early-return path after claim acquisition so the
1468    // documented contract ("releases on all exit paths") is met. TTL/PID
1469    // makes leaked claims recoverable, but explicit release is cleaner and
1470    // makes tests deterministic (Gemini HIGH finding).
1471    let release_walker_claim = || {
1472        let _ = abi_cmd(&abi_bin)
1473            .args(["claim", "release", &walker_key, "--holder", &walker_holder])
1474            .output();
1475    };
1476
1477    // ── build budget ──────────────────────────────────────────────────────────
1478    let budget = match LoopBudget::new(max_iters) {
1479        Ok(b) => b,
1480        Err(e) => {
1481            eprintln!("fno-agents loop megawalk: {e}");
1482            release_walker_claim();
1483            return Ok(2);
1484        }
1485    };
1486
1487    // ── cancel closure ────────────────────────────────────────────────────────
1488    let cancel_file = cwd.join(".fno").join(".target-cancelled");
1489    let cancel = move || SIGINT_RECEIVED.load(Ordering::SeqCst) || cancel_file.exists();
1490
1491    // ── run the loop ──────────────────────────────────────────────────────────
1492    // Per-unit dispatch cap (plan Failure Mode: a session that dies without a
1493    // TerminationReason event is synthesized as node_failed and must count
1494    // toward the consecutive-failure pause). Re-dispatch is the NORMAL
1495    // continuation mechanism for multi-session work, so the cap is generous:
1496    // 15 sessions x MAX_TURNS turns is ample for an L-sized node, while a
1497    // crash-looping driver parks after 15 fast failures (close(NoProgress) ->
1498    // streak) instead of burning the whole walk budget on one unit.
1499    const PER_UNIT_MAX_DISPATCHES: u64 = 15;
1500    let outcome = match run_loop(
1501        &mut queue,
1502        &dispatcher,
1503        &budget,
1504        &journal,
1505        &cancel,
1506        Some(PER_UNIT_MAX_DISPATCHES),
1507    ) {
1508        Ok(o) => o,
1509        Err(e) => {
1510            eprintln!("fno-agents loop megawalk: fatal loop error: {e}");
1511            release_walker_claim();
1512            return Ok(2);
1513        }
1514    };
1515
1516    // ── release walker singleton claim ────────────────────────────────────────
1517    release_walker_claim();
1518
1519    // ── walk-as-unit termination event (group 3) ──────────────────────────────
1520    // When a parent loop (megatron) dispatched this walk with a session key,
1521    // journal the walk's own termination so the parent's find_termination
1522    // observes it (via the global mirror when cwds differ). Fatal on project-
1523    // journal failure, consistent with every other journal.append here.
1524    if let Some(ref key) = termination_key {
1525        if let Err(e) = emit_walk_termination(
1526            &journal,
1527            key,
1528            &outcome.reason,
1529            outcome.iterations_used,
1530            outcome.units.len(),
1531        ) {
1532            eprintln!("fno-agents loop megawalk: failed to journal walk termination: {e}");
1533            return Ok(2);
1534        }
1535    }
1536
1537    // ── report outcome ────────────────────────────────────────────────────────
1538    let exit_code = exit_code_for_reason(&outcome.reason);
1539    println!(
1540        "megawalk: {:?} ({} iterations used, {} units closed)",
1541        outcome.reason,
1542        outcome.iterations_used,
1543        outcome.units.len()
1544    );
1545    for unit_result in &outcome.units {
1546        println!(
1547            "  unit {}: {:?} ({:?})",
1548            unit_result.unit_id, unit_result.evidence.reason, unit_result.close
1549        );
1550    }
1551
1552    Ok(exit_code)
1553}
1554
1555#[cfg(test)]
1556mod fresh_tests {
1557    //! ab-77b691dc: a megawalk worker is rooted at canonical main, so a walk
1558    //! launched from a linked worktree does not start each target worker in that
1559    //! worktree (the shared .fno/ session-state collision).
1560    use super::*;
1561
1562    fn git(dir: &std::path::Path, args: &[&str]) -> bool {
1563        std::process::Command::new("git")
1564            .arg("-C")
1565            .arg(dir)
1566            .args(args)
1567            .output()
1568            .map(|o| o.status.success())
1569            .unwrap_or(false)
1570    }
1571
1572    #[test]
1573    fn dispatcher_roots_worker_cwd_at_canonical_from_worktree() {
1574        // Skip when git is unavailable (mirrors the Python skipif(no git)).
1575        if std::process::Command::new("git")
1576            .arg("--version")
1577            .output()
1578            .is_err()
1579        {
1580            return;
1581        }
1582        let tmp = tempfile::tempdir().unwrap();
1583        let main = tmp.path().join("main");
1584        std::fs::create_dir(&main).unwrap();
1585        assert!(git(&main, &["init", "-q"]));
1586        assert!(git(&main, &["config", "user.email", "t@t"]));
1587        assert!(git(&main, &["config", "user.name", "t"]));
1588        assert!(git(&main, &["commit", "-q", "--allow-empty", "-m", "init"]));
1589        let linked = tmp.path().join("wt");
1590        assert!(git(
1591            &main,
1592            &[
1593                "worktree",
1594                "add",
1595                "-q",
1596                linked.to_str().unwrap(),
1597                "-b",
1598                "feat"
1599            ]
1600        ));
1601
1602        let d = MegawalkDispatcher::new(
1603            std::path::PathBuf::from("/driver/lib.sh"),
1604            vec![],
1605            linked.clone(),
1606            "fno".to_string(),
1607            false,
1608        );
1609        let want = std::fs::canonicalize(&main).unwrap();
1610        assert_eq!(
1611            d.cwd, want,
1612            "megawalk worker cwd must be rooted at canonical main, not the worktree"
1613        );
1614    }
1615
1616    #[test]
1617    fn dispatcher_keeps_cwd_when_not_a_worktree() {
1618        // A non-git cwd -> canonical resolution returns None -> keep the given
1619        // cwd (the safe-side fallback; also the already-canonical no-op case).
1620        let tmp = tempfile::tempdir().unwrap();
1621        let d = MegawalkDispatcher::new(
1622            std::path::PathBuf::from("/driver/lib.sh"),
1623            vec![],
1624            tmp.path().to_path_buf(),
1625            "fno".to_string(),
1626            false,
1627        );
1628        assert_eq!(d.cwd, tmp.path());
1629    }
1630
1631    #[test]
1632    fn retry_etxtbsy_passes_success_through_without_retry() {
1633        let mut calls = 0u32;
1634        let r: std::io::Result<u8> = retry_etxtbsy(|| {
1635            calls += 1;
1636            Ok(7)
1637        });
1638        assert_eq!(r.unwrap(), 7);
1639        assert_eq!(calls, 1, "a successful spawn must not retry");
1640    }
1641
1642    #[test]
1643    fn retry_etxtbsy_retries_then_succeeds() {
1644        // Simulate ETXTBSY clearing after a couple of attempts.
1645        let mut calls = 0u32;
1646        let r: std::io::Result<u8> = retry_etxtbsy(|| {
1647            calls += 1;
1648            if calls < 3 {
1649                Err(std::io::Error::from_raw_os_error(libc::ETXTBSY))
1650            } else {
1651                Ok(42)
1652            }
1653        });
1654        assert_eq!(r.unwrap(), 42);
1655        assert_eq!(calls, 3, "must retry past transient ETXTBSY");
1656    }
1657
1658    #[test]
1659    fn retry_etxtbsy_does_not_swallow_other_errors() {
1660        // A non-ETXTBSY error returns immediately, no retry.
1661        let mut calls = 0u32;
1662        let r: std::io::Result<u8> = retry_etxtbsy(|| {
1663            calls += 1;
1664            Err(std::io::Error::from_raw_os_error(libc::ENOENT))
1665        });
1666        assert_eq!(r.unwrap_err().raw_os_error(), Some(libc::ENOENT));
1667        assert_eq!(calls, 1, "a non-ETXTBSY error must not retry");
1668    }
1669
1670    #[test]
1671    fn retry_etxtbsy_gives_up_after_max_retries() {
1672        // Persistent ETXTBSY surfaces after the bounded retry budget (1 initial
1673        // + 5 retries = 6 calls) rather than spinning forever.
1674        let mut calls = 0u32;
1675        let r: std::io::Result<u8> = retry_etxtbsy(|| {
1676            calls += 1;
1677            Err(std::io::Error::from_raw_os_error(libc::ETXTBSY))
1678        });
1679        assert_eq!(r.unwrap_err().raw_os_error(), Some(libc::ETXTBSY));
1680        assert_eq!(calls, 6, "1 initial attempt + MAX_RETRIES(5)");
1681    }
1682}