fno_agents/loop_megawalk.rs
1//! Megawalk driver: MegawalkQueue + MegawalkDispatcher + the `loop run --driver megawalk` arm.
2//!
3//! ## What this module does
4//!
5//! MegawalkQueue shells `fno backlog next` to dequeue work items and
6//! `fno backlog done` to close them. Claims (ab-7303e5d7) are acquired via
7//! `fno claim acquire` so a second walker cannot pick up the same node, and
8//! released on every close path including error paths.
9//!
10//! MegawalkDispatcher wraps ShelloutDispatcher, injecting per-unit env vars
11//! (`CONTINUE_PROMPT`, `TARGET_SESSION_ID`) so the worker session knows which
12//! node it is driving and the loopcheck termination event carries the matching
13//! session_key.
14//!
15//! ## Cross-cwd termination event delivery (ab-7303e5d7)
16//!
17//! Worker sessions dispatched by the walker run `/target` in their OWN
18//! conductor worktrees (the /target location hard-gate moves them), so their
19//! termination events land in the WORKTREE's events.jsonl AND in the global
20//! `~/.fno/events.jsonl` mirror (via loopcheck's emit_to_both). The
21//! walker's project journal lives at the walker cwd. Journal::find_termination
22//! (extended in loop_runtime.rs) scans the project journal first, then falls
23//! back to the global mirror so the walker always finds the event.
24//!
25//! ## Claim idempotency (verified)
26//!
27//! The walker acquires `node:<id>` with holder `target-session:<session_key>`.
28//! When the worker's init-target-state.sh fires (with TARGET_SESSION_ID set),
29//! it calls `fno claim acquire node:<id> --holder target-session:<session_key>`.
30//! core.py:acquire_claim line 209: if `existing.holder == holder`, the re-acquire
31//! is idempotent (refreshes pid/host/acquired_at, emits claim_idempotent_reacquired).
32//! The holder strings match exactly because the walker pre-assigns the session_key
33//! and passes it via TARGET_SESSION_ID, so init-target-state.sh uses the same value.
34//! The re-acquire is NOT blocked by PID mismatch (the refresh accepts any pid).
35//!
36//! ## Module naming
37//!
38//! The module name starts with `loop` so that the LOC-ratchet glob
39//! `crates/fno-agents/src/loop*` counts this file's LOC toward the ratchet
40//! budget deliberately (alongside loopcheck.rs).
41
42use crate::loop_dispatch::ShelloutDispatcher;
43use crate::loop_runtime::{
44 CloseOutcome, DispatchCtx, Dispatcher, Evidence, Journal, LoopError, Queue, Session, Unit,
45};
46use crate::loopcheck::TerminationReason;
47use std::collections::HashMap;
48use std::path::PathBuf;
49use std::process::Command;
50
51// ── Multi-PR umbrella verification (grilled decision 9, simplicity) ──────────
52//
53// Spec B.5 asks: does a walked group-child close prematurely complete the
54// epic, and does the epic surface again via `fno backlog next` after all
55// children are ready?
56//
57// Verified (2026-06-06): NO premature epic completion.
58//
59// Evidence:
60// cli/src/fno/graph/statuses.py:recompute_statuses derives _status:done
61// from completed_at on the INDIVIDUAL node only (line 81). There is no
62// parent-cascade: walking a group-child through cmd_done sets
63// `completed_at` on that child; the epic's completed_at remains None until
64// cmd_done is called explicitly on the epic itself. The children summary
65// index (store.py:_compute_children) is read-only metadata for the Kanban
66// board; it does not influence _status.
67//
68// fno.plan._stamp: group-aware graduation writes expected_url_count
69// on the shared epic doc (first-writer-wins, fno plan set-expected, ab-9e864e42).
70// The epic backlog node is NOT automatically marked done by the stamp module; the
71// walker must call fno backlog done on the epic node separately after all
72// group children are confirmed done. This is correct: the epic node is still
73// visible in `fno backlog next` until its own completed_at is set.
74//
75// Consequence: no speculative walker-side umbrella machinery is needed here.
76// The current walker closes group-child nodes one at a time; the epic parent
77// continues to surface in `fno backlog next` as a separate ready node. This is
78// deliberate simplicity (grilled decision 9). No gap was found; no test added.
79
80// ── Event-kind prune ledger (Claude's Discretion 2) ──────────────────────────
81//
82// The legacy megawalk.py emits ~29 kinds via _emit_event() into
83// megawalk-events.jsonl. Task 2.4 deletes megawalk.py; the writer dies with
84// it. This table records the fate of each legacy kind so auditors can trace
85// the transition.
86//
87// Legacy kind -> Fate
88// ──────────────────────────── ─────────────────────────────────────────────
89// node_complete -> node_closed (close="closed")
90// node_parked -> node_closed (close="parked")
91// walker_paused -> walk_paused
92// node_failed -> node_failed (runtime, unchanged)
93// node_help_requested -> absorbed: cv-d3943d2a
94// No typed help event post-wedge; a
95// help-stuck session exits without a
96// termination event -> per-unit cap parks it
97// -> streak counts it. Stdout parsing is
98// forbidden at this altitude (locked decision 9).
99// merge_attempt -> deleted (reconcile + backlog-done cover residue)
100// pr_externally_merged -> deleted (reconcile covers residue)
101// reconcile_started -> deleted
102// reconcile_completed -> deleted
103// reconcile_failed -> deleted
104// stuck_detection_failed -> deleted (detector era removed in wedge PR)
105// activity_read_failed -> deleted (detector era)
106// worktree_stuck_checkin -> deleted (detector era)
107// walker_started -> deleted (preflight print replaces this)
108// walker_completed -> loop_terminated (runtime)
109// walker_aborted -> loop_terminated (runtime)
110// node_started -> loop_unit_dispatched (runtime, unchanged)
111// node_dispatched -> loop_unit_dispatched (runtime, unchanged)
112// node_error -> node_failed (runtime, unchanged)
113// node_skipped -> deleted (claim-filter in fno backlog next covers)
114// backlog_empty -> loop_terminated{reason:NoWork} (runtime)
115// iteration_limit_reached -> loop_terminated{reason:Budget} (runtime)
116// consecutive_failures_paused -> walk_paused{policy:consecutive_failures}
117// p0_failure_paused -> walk_paused{policy:p0_failed}
118//
119// The legacy WRITER (megawalk.py) is deleted in task 2.4. Do not touch
120// megawalk.py in this task.
121
122// ── parallel-cap helper ───────────────────────────────────────────────────────
123
124/// Clamp a parallel-cap value to at least 1.
125///
126/// A cap of 0 (or negative, if caller uses signed math) would prohibit all
127/// dispatch. We clamp to 1 and log a warning so the flag is accepted but
128/// explicit: the caller is responsible for printing the clamp message.
129///
130/// Note: group-2 always executes sequentially (run_loop is single-threaded).
131/// When cap > 1 the verb glue prints one honest line explaining that execution
132/// is still sequential (collision-conservative default, Claude's Discretion 3).
133pub fn clamp_parallel_cap(cap: u64) -> u64 {
134 if cap < 1 {
135 eprintln!(
136 "loop-megawalk: WARNING: --parallel-cap {cap} < 1; clamped to 1 \
137 (boundary: cap must be >= 1)"
138 );
139 1
140 } else {
141 cap
142 }
143}
144
145// ── MegawalkPolicyQueue ───────────────────────────────────────────────────────
146
147/// Per-unit state stored in the policy queue.
148struct PolicyUnitEntry {
149 unit: Unit,
150}
151
152// ── shared predicate ──────────────────────────────────────────────────────────
153
154/// Returns true when the termination reason is a successful close
155/// (DonePRGreen or DoneAdvisory). Used in three policy sites; extracted to
156/// avoid triplication (sigma-review finding 4).
157fn is_done_reason(r: &TerminationReason) -> bool {
158 matches!(
159 r,
160 TerminationReason::DonePRGreen | TerminationReason::DoneAdvisory
161 )
162}
163
164/// Walk policy state tracker for the megawalk driver.
165///
166/// This struct is NOT a full Queue impl by itself; it is the policy layer that
167/// wraps or extends the shell-based MegawalkQueue. For unit tests it acts as a
168/// standalone Queue: `push_unit` / `next` / `close` with full policy tracking.
169///
170/// ## Consecutive-failure pause (3)
171///
172/// A unit is a "failure" when close() is called with evidence.reason NOT in
173/// {DonePRGreen, DoneAdvisory}. A successful close resets the streak. When the
174/// streak reaches 3, the next next() call returns
175/// Err(LoopError::Pause{policy:"consecutive_failures", detail}).
176///
177/// ## p0 immediate pause
178///
179/// If a p0 unit fails (close called with non-Done reason and is_p0=true in the
180/// per-unit state), the NEXT next() call returns
181/// Err(LoopError::Pause{policy:"p0_failed", detail:uid}) immediately, regardless
182/// of streak.
183///
184/// ## Park-on-help absorption
185///
186/// No typed help event exists post-wedge (verified: only mission-emit.sh prints
187/// a help tag to stderr; loopcheck emits none). A help-stuck session exits
188/// without a termination event -> node_failed -> per-unit cap parks it ->
189/// streak counts it. See carveout cv-d3943d2a (typed help source = future work).
190/// Locked decision 9 forbids stdout parsing at this altitude.
191pub struct MegawalkPolicyQueue {
192 /// Units queued for dispatch (push_unit adds here; next consumes from front).
193 pending: std::collections::VecDeque<PolicyUnitEntry>,
194 /// Whether a p0 failure happened; set in record_close for is_p0 failures.
195 p0_failure: Option<String>, // unit id of the failed p0 unit
196 /// Consecutive failure streak counter.
197 consecutive_failures: usize,
198 /// Unit IDs involved in the current streak (for Pause detail).
199 streak_ids: Vec<String>,
200}
201
202impl MegawalkPolicyQueue {
203 /// Construct an empty policy queue.
204 pub fn new() -> Self {
205 Self {
206 pending: std::collections::VecDeque::new(),
207 p0_failure: None,
208 consecutive_failures: 0,
209 streak_ids: vec![],
210 }
211 }
212
213 /// Add a unit to the back of the queue.
214 ///
215 /// `is_p0`: when true, a failure on this unit triggers an immediate pause.
216 pub fn push_unit(&mut self, unit: Unit, is_p0: bool) {
217 // is_p0 is passed to record_close separately; the entry only holds the unit.
218 let _ = is_p0;
219 self.pending.push_back(PolicyUnitEntry { unit });
220 }
221
222 /// Record the outcome of a close() call for policy tracking.
223 ///
224 /// Call this AFTER queue.close() returns, with the same evidence passed to close.
225 /// `is_p0`: whether the unit that was closed had p0 priority.
226 ///
227 /// This is a test-accessible hook. In run_loop the policy tracking is done
228 /// via the Queue::close() implementation (which calls record_close internally).
229 pub fn record_close(&mut self, unit: &Unit, evidence: &Evidence, is_p0: bool) {
230 let is_success = is_done_reason(&evidence.reason);
231
232 if is_success {
233 // Success: reset consecutive-failure streak AND p0_failure.
234 // p0_failure was never cleared here before (sigma-review finding 3);
235 // omitting the reset caused spurious pauses after a successful
236 // recovery unit followed a failed p0 unit.
237 self.consecutive_failures = 0;
238 self.streak_ids.clear();
239 self.p0_failure = None;
240 } else {
241 // Failure: increment streak.
242 self.consecutive_failures += 1;
243 self.streak_ids.push(unit.id.clone());
244 // p0 failure: record for immediate pause on next next() call.
245 if is_p0 {
246 self.p0_failure = Some(unit.id.clone());
247 }
248 }
249 }
250
251 /// Check whether walk policy requires a pause.
252 ///
253 /// Returns `Some((policy, detail))` if a pause is warranted; `None` otherwise.
254 /// Called at the top of next() before dequeuing.
255 pub fn should_pause(&self) -> Option<(String, String)> {
256 // p0 failure takes precedence.
257 if let Some(ref uid) = self.p0_failure {
258 return Some(("p0_failed".to_string(), uid.clone()));
259 }
260 // Consecutive-failure streak.
261 if self.consecutive_failures >= 3 {
262 let detail = self.streak_ids.join(" ");
263 return Some(("consecutive_failures".to_string(), detail));
264 }
265 None
266 }
267}
268
269impl Default for MegawalkPolicyQueue {
270 fn default() -> Self {
271 Self::new()
272 }
273}
274
275impl Queue for MegawalkPolicyQueue {
276 fn next(&mut self) -> Result<Option<Unit>, LoopError> {
277 // Check policy before dequeuing.
278 if let Some((policy, detail)) = self.should_pause() {
279 return Err(LoopError::Pause { policy, detail });
280 }
281 // Dequeue the next unit.
282 match self.pending.pop_front() {
283 None => Ok(None),
284 Some(entry) => Ok(Some(entry.unit)),
285 }
286 }
287
288 fn close(&mut self, unit: &Unit, evidence: &Evidence) -> Result<CloseOutcome, LoopError> {
289 // Determine if this unit was p0 (we no longer have the PolicyUnitEntry
290 // since it was consumed by next(); for test use, is_p0 is always false
291 // unless record_close is called separately).
292 // For production use, the MegawalkQueue wraps this and passes is_p0.
293 // For unit tests that call record_close() separately, close() just parks.
294 let is_success = is_done_reason(&evidence.reason);
295 if is_success {
296 // Success: reset streak AND p0_failure (sigma-review finding 3).
297 self.consecutive_failures = 0;
298 self.streak_ids.clear();
299 self.p0_failure = None;
300 } else {
301 self.consecutive_failures += 1;
302 self.streak_ids.push(unit.id.clone());
303 }
304
305 let outcome = if is_success {
306 CloseOutcome::Closed
307 } else {
308 CloseOutcome::Parked(format!("policy-park: {:?}", evidence.reason))
309 };
310 Ok(outcome)
311 }
312}
313
314// ── constants ─────────────────────────────────────────────────────────────────
315
316/// Maximum number of claim-held skips inside a single next() call before
317/// giving up and returning an error. Prevents infinite loops when every
318/// ready node is claimed by live sessions.
319const MAX_CLAIM_RETRIES: usize = 5;
320
321// ── helper: run fno sub-command ───────────────────────────────────────────────
322
323/// Build a Command for the `fno` binary.
324///
325/// Binary resolution: `abi_bin` (the path/name given at construction time,
326/// overridden by `$FNO_BIN` for tests). If `FNO_BIN` is set and non-empty,
327/// it wins; otherwise `abi_bin` is used as-is (callers pass "fno" for
328/// production and a tempdir stub path for tests).
329pub(crate) fn abi_cmd(abi_bin: &str) -> Command {
330 let binary = std::env::var("FNO_BIN")
331 .ok()
332 .filter(|s| !s.is_empty())
333 .unwrap_or_else(|| abi_bin.to_string());
334 Command::new(binary)
335}
336
337/// Run a spawn closure, retrying briefly on ETXTBSY ("Text file busy", os
338/// error 26). The spawned file is the `fno` / `fno-agents` binary: a
339/// concurrent `fno update` relinks it in place, and under `cargo test` a
340/// sibling thread that just wrote+exec'd a stub leaves a transient write-fd
341/// open in another thread's fork window - either way the kernel can refuse
342/// the exec with ETXTBSY. The condition clears within microseconds once the
343/// writing fd closes, so a bounded retry turns a hard spawn failure into a
344/// short wait. Any other error - and the successful value - passes through
345/// unchanged.
346pub(crate) fn retry_etxtbsy<T>(
347 mut spawn: impl FnMut() -> std::io::Result<T>,
348) -> std::io::Result<T> {
349 const MAX_RETRIES: u32 = 5;
350 let mut attempt: u32 = 0;
351 loop {
352 match spawn() {
353 Err(e) if e.raw_os_error() == Some(libc::ETXTBSY) && attempt < MAX_RETRIES => {
354 attempt += 1;
355 std::thread::sleep(std::time::Duration::from_millis(2 * u64::from(attempt)));
356 }
357 other => return other,
358 }
359 }
360}
361
362/// Run `fno doctor --json` best-effort and return whether the output indicates
363/// a stale installation. Any failure (I/O, parse) is treated as "unknown"
364/// and does not append the staleness hint.
365fn is_abi_stale(abi_bin: &str) -> bool {
366 let out = match abi_cmd(abi_bin).args(["doctor", "--json"]).output() {
367 Ok(o) => o,
368 Err(_) => return false,
369 };
370 let stdout = String::from_utf8_lossy(&out.stdout);
371 // Expect {"status":"stale",...} - check the status field.
372 if let Ok(v) = serde_json::from_str::<serde_json::Value>(stdout.trim()) {
373 return v["status"].as_str() == Some("stale");
374 }
375 false
376}
377
378/// Append a staleness hint to an error message if `fno doctor` says stale.
379pub(crate) fn maybe_stale_hint(msg: String, abi_bin: &str) -> String {
380 if is_abi_stale(abi_bin) {
381 format!("{msg}; installed fno may be stale - run `fno update`")
382 } else {
383 msg
384 }
385}
386
387// ── session key generation ────────────────────────────────────────────────────
388
389/// Generate a unique session key in the same shape used by init-target-state.sh:
390/// `{utc %Y%m%dT%H%M%SZ}-{infix}{pid}-{6 hex}`.
391///
392/// The infix distinguishes the assigning driver at a glance in logs:
393/// "mw" = megawalk, "mt" = megatron (group 3).
394pub(crate) fn gen_session_key_with_infix(infix: &str) -> String {
395 let ts = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string();
396 let pid = std::process::id();
397 // 3 random bytes -> 6 hex chars.
398 let entropy: u32 = {
399 let mut buf = [0u8; 3];
400 // Best-effort: use /dev/urandom bytes; fall back to a mix of pid+time.
401 if let Ok(mut f) = std::fs::File::open("/dev/urandom") {
402 use std::io::Read;
403 let _ = f.read_exact(&mut buf);
404 } else {
405 buf[0] = (pid & 0xFF) as u8;
406 buf[1] = ((pid >> 8) & 0xFF) as u8;
407 buf[2] = (chrono::Utc::now().timestamp_subsec_nanos() & 0xFF) as u8;
408 }
409 u32::from_le_bytes([buf[0], buf[1], buf[2], 0])
410 };
411 format!("{ts}-{infix}{pid}-{entropy:06x}")
412}
413
414/// Megawalk-assigned session key (the "mw" infix).
415fn gen_session_key() -> String {
416 gen_session_key_with_infix("mw")
417}
418
419// ── mission env extraction ────────────────────────────────────────────────────
420
421/// Extract TARGET_MISSION_* env vars from a `_node_summary` JSON value.
422///
423/// Mirrors Python `extract_mission_env()` in megawalk.py:75-117:
424/// - If `mission_id` is null/absent: returns Ok(vec![]) (non-fleet node).
425/// - If `mission_id` is set: all four vars are required.
426/// - `mission_wave` null/absent -> Err(Queue) naming node + field.
427/// - `mission_slug` null/absent -> Err(Queue) naming node + field.
428/// - `mission_from_msg_id` null -> maps to "" (Python line 117 behavior).
429///
430/// Returns `Err(LoopError::Queue(...))` on corrupted fleet metadata so the
431/// commander is not silently stranded.
432fn extract_mission_env(
433 v: &serde_json::Value,
434 node_id: &str,
435) -> Result<Vec<(String, String)>, LoopError> {
436 let mission_id = match v["mission_id"].as_str() {
437 Some(s) if !s.is_empty() => s.to_string(),
438 _ => {
439 // null, absent, or empty string: non-fleet node.
440 return Ok(vec![]);
441 }
442 };
443
444 // mission_wave: must be present and coercible to a string.
445 let mission_wave = match &v["mission_wave"] {
446 serde_json::Value::Null => {
447 return Err(LoopError::Queue(format!(
448 "node {node_id:?} has mission_id={mission_id:?} but mission_wave is missing; \
449 dispatcher bug (corrupted fleet metadata)"
450 )));
451 }
452 w => w.to_string().trim_matches('"').to_string(),
453 };
454 if mission_wave.is_empty() || mission_wave == "null" {
455 return Err(LoopError::Queue(format!(
456 "node {node_id:?} has mission_id={mission_id:?} but mission_wave is null; \
457 dispatcher bug (corrupted fleet metadata)"
458 )));
459 }
460
461 // mission_slug: must be a non-empty string.
462 let mission_slug = match v["mission_slug"].as_str() {
463 Some(s) if !s.is_empty() => s.to_string(),
464 _ => {
465 return Err(LoopError::Queue(format!(
466 "node {node_id:?} has mission_id={mission_id:?} but mission_slug is missing \
467 or empty; dispatcher bug (corrupted fleet metadata)"
468 )));
469 }
470 };
471
472 // mission_from_msg_id: null -> "" (Python line 117).
473 let mission_from_msg_id = v["mission_from_msg_id"].as_str().unwrap_or("").to_string();
474
475 Ok(vec![
476 ("TARGET_MISSION_ID".to_string(), mission_id),
477 ("TARGET_MISSION_WAVE".to_string(), mission_wave),
478 ("TARGET_MISSION_SLUG".to_string(), mission_slug),
479 (
480 "TARGET_MISSION_FROM_MSG_ID".to_string(),
481 mission_from_msg_id,
482 ),
483 ])
484}
485
486// ── MegawalkQueue ─────────────────────────────────────────────────────────────
487
488/// Per-unit claim state stored in `MegawalkQueue::active_claims`.
489struct ClaimEntry {
490 /// The session key used for the node claim (matched by close() for release).
491 session_key: String,
492 /// Whether this unit has p0 priority. A p0 failure triggers an immediate
493 /// walk pause regardless of the consecutive-failure streak.
494 is_p0: bool,
495}
496
497/// A Queue that shells `fno backlog next` / `fno backlog done` and coordinates
498/// node claims so two walkers never dispatch the same node simultaneously.
499///
500/// ## Walk policy (folded in, not a separate wrapper)
501///
502/// Policy state lives directly in `MegawalkQueue` so the production path uses
503/// the same policy as the test path - no separate wrapper needed. The policy
504/// is the same as `MegawalkPolicyQueue`:
505/// - consecutive-failure streak of 3 -> Pause{consecutive_failures}
506/// - p0 unit failure -> Pause{p0_failed} (immediate, no streak needed)
507/// - Success (DonePRGreen | DoneAdvisory) resets the streak.
508///
509/// The `priority` field from the backlog-next JSON is stored per-unit in
510/// `active_claims` so `close()` can check it for the p0 rule.
511///
512/// ## --max-units N (once-mode)
513///
514/// When `max_units` is `Some(N)`, `next()` returns `None` (Drained) after N
515/// units have been closed by `close()`. This maps the `/megawalk once` modifier
516/// (task 2.4 uses `--max-units 1`) and general "execute at most N units then
517/// stop" semantics. `N` must be >= 1; the CLI gate enforces N > 0 (exit 2).
518/// `units_closed` is incremented at the END of each `close()` call so the cap
519/// fires on the NEXT `next()` call, giving the correct semantics: close the
520/// N-th unit, then the outer loop calls `next()` which returns None -> NoWork.
521pub struct MegawalkQueue {
522 /// Path or name of the fno binary. `$FNO_BIN` env overrides for tests.
523 abi_bin: String,
524 /// Optional `--project <name>` filter.
525 project: Option<String>,
526 /// When true, pass `--all` to `fno backlog next`.
527 all: bool,
528 /// Map from node id -> ClaimEntry for active dispatches.
529 /// Stores session_key (for claim release/hold) and is_p0 (for policy).
530 /// On Parked/Refused outcomes the entry is KEPT (park-exclusion hold).
531 /// On Closed the entry is removed and the claim is released.
532 active_claims: HashMap<String, ClaimEntry>,
533 // ── policy state ──────────────────────────────────────────────────────────
534 /// Pending p0 failure: the unit id of a p0 unit that failed. When set,
535 /// the NEXT next() call returns a pause immediately.
536 policy_p0_failure: Option<String>,
537 /// Consecutive failure streak counter. A "failure" is any close with
538 /// evidence.reason NOT in {DonePRGreen, DoneAdvisory}.
539 policy_consecutive_failures: usize,
540 /// Unit IDs involved in the current streak (for Pause detail string).
541 policy_streak_ids: Vec<String>,
542 // ── max-units cap (once-mode) ─────────────────────────────────────────────
543 /// Optional cap on total units closed. When set, next() returns None
544 /// (Drained) after `units_closed` reaches this value.
545 max_units: Option<u64>,
546 /// Count of units closed so far (incremented at end of close()).
547 units_closed: u64,
548 /// Optional `--mission <id>` selection filter (group 3, ab-9fd662c6).
549 /// A megatron child walk passes this so the walk works ONLY the
550 /// mission's nodes and never drifts into the project's general backlog.
551 mission: Option<String>,
552}
553
554impl MegawalkQueue {
555 /// Construct a MegawalkQueue.
556 ///
557 /// `abi_bin`: "fno" in production; a stub path in tests (or override via
558 /// `$FNO_BIN`).
559 /// `project`: optional project filter passed as `--project <p>` to `fno backlog next`.
560 /// `all`: when true, pass `--all` instead of `--project`.
561 pub fn new(abi_bin: String, project: Option<String>, all: bool) -> Self {
562 Self::new_with_max_units(abi_bin, project, all, None)
563 }
564
565 /// Construct a MegawalkQueue with an optional max-units cap.
566 ///
567 /// `max_units`: when `Some(N)`, next() returns None (Drained) after N units
568 /// have been closed. Maps the `--max-units` CLI flag / `/megawalk once`.
569 /// Must be >= 1; callers are responsible for rejecting 0 before calling
570 /// (the verb glue in loop_target.rs exits 2 on N == 0).
571 pub fn new_with_max_units(
572 abi_bin: String,
573 project: Option<String>,
574 all: bool,
575 max_units: Option<u64>,
576 ) -> Self {
577 Self {
578 abi_bin,
579 project,
580 all,
581 active_claims: HashMap::new(),
582 policy_p0_failure: None,
583 policy_consecutive_failures: 0,
584 policy_streak_ids: vec![],
585 max_units,
586 units_closed: 0,
587 mission: None,
588 }
589 }
590
591 /// Builder: set the `--mission <id>` selection filter (megatron child walks).
592 pub fn with_mission(mut self, mission: Option<String>) -> Self {
593 self.mission = mission;
594 self
595 }
596
597 /// Check whether walk policy requires a pause. Returns a typed
598 /// `LoopError::Pause { policy, detail }` when a pause is warranted, `None`
599 /// otherwise. Called at the top of `next()` before shelling out.
600 fn policy_check(&self) -> Option<LoopError> {
601 // p0 failure takes precedence over streak.
602 if let Some(ref uid) = self.policy_p0_failure {
603 return Some(LoopError::Pause {
604 policy: "p0_failed".to_string(),
605 detail: uid.clone(),
606 });
607 }
608 if self.policy_consecutive_failures >= 3 {
609 let detail = self.policy_streak_ids.join(" ");
610 return Some(LoopError::Pause {
611 policy: "consecutive_failures".to_string(),
612 detail,
613 });
614 }
615 None
616 }
617
618 /// Update policy state after a close() call.
619 fn policy_record_close(&mut self, unit_id: &str, is_success: bool, is_p0: bool) {
620 if is_success {
621 // Success: reset streak AND p0_failure (sigma-review finding 3).
622 // p0_failure was not cleared here before, causing spurious pauses
623 // after a recovery unit succeeded following a failed p0 unit.
624 self.policy_consecutive_failures = 0;
625 self.policy_streak_ids.clear();
626 self.policy_p0_failure = None;
627 } else {
628 // Failure: increment streak.
629 self.policy_consecutive_failures += 1;
630 self.policy_streak_ids.push(unit_id.to_string());
631 if is_p0 {
632 self.policy_p0_failure = Some(unit_id.to_string());
633 }
634 }
635 }
636}
637
638impl Queue for MegawalkQueue {
639 /// Dequeue the next ready backlog node.
640 ///
641 /// Algorithm:
642 /// 0. Check walk policy (consecutive-failure streak / p0 failure).
643 /// Returns Err(LoopError::Pause{policy, detail}) when policy requires a pause.
644 /// 1. Shell `fno backlog next [--project P | --all]`.
645 /// 2. Empty output (literal "null") -> return Ok(None).
646 /// 3. Parse JSON as `_node_summary` shape (captures `priority` for p0 check).
647 /// 4. Generate a unique session_key.
648 /// 5. Shell `fno claim acquire node:<id> --holder target-session:<session_key>
649 /// --ttl 2h --reason "megawalk walker dispatch"`.
650 /// Exit 0 -> record claim, return the Unit.
651 /// Exit 1 (held by other) -> loop back to step 1 (the live-claim filter
652 /// in _live_claimed_node_ids excludes it on the next call).
653 /// 6. Bound retries at MAX_CLAIM_RETRIES; on exhaustion return LoopError::Queue.
654 ///
655 /// Malformed JSON or verb failure -> LoopError::Queue with staleness hint.
656 fn next(&mut self) -> Result<Option<Unit>, LoopError> {
657 // ── max-units cap check (step -1) ─────────────────────────────────
658 // When max_units is set and units_closed has reached the cap, signal
659 // Drained so the outer loop terminates with NoWork.
660 if let Some(cap) = self.max_units {
661 if self.units_closed >= cap {
662 return Ok(None);
663 }
664 }
665
666 // ── walk policy check (step 0) ────────────────────────────────────
667 if let Some(err) = self.policy_check() {
668 return Err(err);
669 }
670
671 for attempt in 0..MAX_CLAIM_RETRIES {
672 let _ = attempt; // retry count tracked implicitly
673
674 // ── shell fno backlog next ────────────────────────────────────────
675 let mut cmd = abi_cmd(&self.abi_bin);
676 cmd.args(["backlog", "next"]);
677 if self.all {
678 cmd.arg("--all");
679 } else if let Some(ref p) = self.project {
680 cmd.args(["--project", p]);
681 }
682 if let Some(ref m) = self.mission {
683 cmd.args(["--mission", m]);
684 }
685
686 let out = retry_etxtbsy(|| cmd.output()).map_err(|e| {
687 LoopError::Queue(maybe_stale_hint(
688 format!("fno backlog next: spawn failed: {e}"),
689 &self.abi_bin,
690 ))
691 })?;
692
693 if !out.status.success() {
694 let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();
695 return Err(LoopError::Queue(maybe_stale_hint(
696 format!("fno backlog next: exit {}: {stderr}", out.status),
697 &self.abi_bin,
698 )));
699 }
700
701 let stdout = String::from_utf8_lossy(&out.stdout).trim().to_string();
702
703 // ── null -> empty backlog ─────────────────────────────────────────
704 if stdout == "null" || stdout.is_empty() {
705 return Ok(None);
706 }
707
708 // ── parse _node_summary JSON ──────────────────────────────────────
709 let v: serde_json::Value = serde_json::from_str(&stdout).map_err(|e| {
710 LoopError::Queue(maybe_stale_hint(
711 format!("fno backlog next: JSON parse error: {e} (stdout: {stdout:?})"),
712 &self.abi_bin,
713 ))
714 })?;
715
716 let id = match v["id"].as_str() {
717 Some(s) if !s.is_empty() => s.to_string(),
718 _ => {
719 return Err(LoopError::Queue(maybe_stale_hint(
720 format!("fno backlog next: missing or empty 'id' field in: {stdout:?}"),
721 &self.abi_bin,
722 )));
723 }
724 };
725
726 let title = v["title"].as_str().unwrap_or("(untitled)").to_string();
727 let plan_path = v["plan_path"].as_str().map(|s| s.to_string());
728 // Extract priority for p0 policy. The backlog-next JSON includes a
729 // "priority" field (p0/p1/p2/p3). Treat anything other than "p0"
730 // (or missing) as non-p0; best-effort (a corrupt field is non-p0).
731 let is_p0 = v["priority"].as_str() == Some("p0");
732
733 // ── extract mission env (fleet nodes) ─────────────────────────────
734 // Mirrors Python extract_mission_env(): if mission_id is set, all
735 // four TARGET_MISSION_* vars are required (wave + slug must be
736 // present; from_msg_id maps to "" when null). Corrupted metadata
737 // is a loud Queue error naming the node and missing field.
738 let extra_env = extract_mission_env(&v, &id)?;
739
740 let session_key = gen_session_key();
741
742 // ── acquire the node claim ────────────────────────────────────────
743 let claim_key = format!("node:{id}");
744 let claim_holder = format!("target-session:{session_key}");
745
746 let claim_out = retry_etxtbsy(|| {
747 abi_cmd(&self.abi_bin)
748 .args([
749 "claim",
750 "acquire",
751 &claim_key,
752 "--holder",
753 &claim_holder,
754 "--ttl",
755 "2h",
756 "--reason",
757 "megawalk walker dispatch",
758 ])
759 .env(
760 "FNO_CLAIMS_ROOT",
761 std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()),
762 )
763 .output()
764 })
765 .map_err(|e| {
766 LoopError::Queue(maybe_stale_hint(
767 format!("fno claim acquire: spawn failed: {e}"),
768 &self.abi_bin,
769 ))
770 })?;
771
772 if claim_out.status.success() {
773 // Claim acquired; store session_key + is_p0 for close().
774 self.active_claims.insert(
775 id.clone(),
776 ClaimEntry {
777 session_key: session_key.clone(),
778 is_p0,
779 },
780 );
781 return Ok(Some(Unit {
782 id,
783 title,
784 session_key,
785 plan_path,
786 extra_env,
787 }));
788 }
789
790 // Branch on the exit code (sigma-review finding 1 - exit-code collapse fix).
791 //
792 // The claim CLI contract (cli/src/fno/claims/cli.py header):
793 // exit 1 = ClaimHeldByOther -> retry (live-claims filter on next call)
794 // exit 2 = validation error -> surface immediately; do NOT loop
795 // exit 3 = ClaimCorrupted / ClaimGoneAway -> surface immediately
796 // other -> unexpected; surface immediately
797 //
798 // Before this fix every non-zero exit was treated as "held" and the
799 // loop continued silently, hiding validation and corruption errors.
800 match claim_out.status.code() {
801 Some(1) => {
802 // Held by another session; let the live-claim filter in
803 // `fno backlog next` exclude this node on the next call.
804 // Continue the retry loop.
805 }
806 _ => {
807 // Validation error, corruption, or unexpected exit.
808 // Surface immediately as a Queue error.
809 let stderr = String::from_utf8_lossy(&claim_out.stderr)
810 .trim()
811 .to_string();
812 let code = claim_out.status.code().unwrap_or(-1);
813 return Err(LoopError::Queue(maybe_stale_hint(
814 format!("fno claim acquire {claim_key}: exit {code}: {stderr}"),
815 &self.abi_bin,
816 )));
817 }
818 }
819 }
820
821 // Exhausted MAX_CLAIM_RETRIES picks without finding a claimable node.
822 Err(LoopError::Queue(maybe_stale_hint(
823 format!(
824 "fno backlog next: exhausted {MAX_CLAIM_RETRIES} attempts; every ready node \
825 is claimed by another session (last node may be stuck)"
826 ),
827 &self.abi_bin,
828 )))
829 }
830
831 /// Mark a unit as closed.
832 ///
833 /// DonePRGreen | DoneAdvisory -> shell `fno backlog done <id>`.
834 /// Exit 0 -> CloseOutcome::Closed.
835 /// Nonzero -> CloseOutcome::Parked(stderr tail).
836 ///
837 /// Any other reason -> CloseOutcome::Parked(reason description).
838 /// Does NOT call `fno backlog done` (task 2.2 handles refusal paths).
839 ///
840 /// Updates walk policy state (consecutive-failure streak / p0 flag) so
841 /// the NEXT next() call can return a pause when policy warrants it.
842 ///
843 /// ## Park-exclusion (AC2-EDGE): hold claim on Parked/Refused
844 ///
845 /// ONLY releases the node claim when the outcome is CloseOutcome::Closed.
846 /// For Parked and Refused outcomes, the claim is HELD so the live-claims
847 /// selection filter in `_live_claimed_node_ids` (cli/src/fno/graph/cli.py:43-65)
848 /// continues to exclude this node and `fno backlog next` moves on to other
849 /// ready work instead of re-picking the same busted node.
850 ///
851 /// ## Claim TTL refresh after park (same-holder re-acquire finding)
852 ///
853 /// core.py:acquire_claim line 209: a same-holder re-acquire refreshes
854 /// `pid/host/acquired_at` (idempotent). The worker session's init-target-state.sh
855 /// calls `fno claim acquire node:<id> --holder target-session:<session_key>`
856 /// which, being the SAME holder as the walker's claim, is an idempotent
857 /// re-acquire - it rewrites `acquired_at` and `pid` (the WORKER's pid, not
858 /// the walker's). After the worker exits the claim record reflects the
859 /// worker's (now-dead) pid. For a TTL-liveness claim this is fine because
860 /// the filter reads `expires_at` not pid, but the `acquired_at` reset means
861 /// the TTL window is measured from the WORKER's re-acquire time, not the
862 /// walker's original acquire. To ensure the claim stays LIVE through the
863 /// next walk iteration (while the walker processes the next unit), the
864 /// walker calls `fno claim acquire` again immediately after a park to
865 /// refresh `acquired_at` with the current time and reset the TTL window.
866 fn close(&mut self, unit: &Unit, evidence: &Evidence) -> Result<CloseOutcome, LoopError> {
867 let should_done = is_done_reason(&evidence.reason);
868
869 let outcome = if should_done {
870 let done_out = retry_etxtbsy(|| {
871 abi_cmd(&self.abi_bin)
872 .args(["backlog", "done", &unit.id])
873 .output()
874 })
875 .map_err(|e| LoopError::Queue(format!("fno backlog done: spawn failed: {e}")))?;
876
877 if done_out.status.success() {
878 CloseOutcome::Closed
879 } else {
880 let stderr = String::from_utf8_lossy(&done_out.stderr).trim().to_string();
881 CloseOutcome::Parked(if stderr.is_empty() {
882 format!(
883 "fno backlog done {} failed (exit {})",
884 unit.id, done_out.status
885 )
886 } else {
887 stderr
888 })
889 }
890 } else {
891 // Append evidence.message when non-empty so synthesized diagnostics
892 // (e.g. "no termination event after N dispatch(es)") are not lost
893 // (sigma-review finding 2).
894 let detail = if evidence.message.is_empty() {
895 format!("session terminated: {:?}", evidence.reason)
896 } else {
897 format!(
898 "session terminated: {:?}: {}",
899 evidence.reason, evidence.message
900 )
901 };
902 CloseOutcome::Parked(detail)
903 };
904
905 // ── update walk policy state ──────────────────────────────────────────
906 // Determine is_p0 from the stored claim entry (recorded at dequeue time).
907 // If the entry is gone (e.g. the unit was never in active_claims - only
908 // possible in tests that bypass next()), default to non-p0.
909 let is_p0 = self
910 .active_claims
911 .get(&unit.id)
912 .map(|e| e.is_p0)
913 .unwrap_or(false);
914 self.policy_record_close(&unit.id, should_done, is_p0);
915
916 // ── claim release vs. hold (park-exclusion) ───────────────────────────
917 match &outcome {
918 CloseOutcome::Closed => {
919 // Success: release the claim so the node is no longer live-claimed.
920 let session_key = self
921 .active_claims
922 .remove(&unit.id)
923 .map(|e| e.session_key)
924 .unwrap_or_else(|| unit.session_key.clone());
925 let claim_key = format!("node:{}", unit.id);
926 let claim_holder = format!("target-session:{session_key}");
927
928 let release_result = retry_etxtbsy(|| {
929 abi_cmd(&self.abi_bin)
930 .args(["claim", "release", &claim_key, "--holder", &claim_holder])
931 .env(
932 "FNO_CLAIMS_ROOT",
933 std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()),
934 )
935 .output()
936 });
937
938 match release_result {
939 Ok(o) if !o.status.success() => {
940 eprintln!(
941 "loop-megawalk: WARNING: claim release {} failed (exit {}): {}",
942 claim_key,
943 o.status,
944 String::from_utf8_lossy(&o.stderr).trim()
945 );
946 }
947 Err(e) => {
948 eprintln!(
949 "loop-megawalk: WARNING: claim release {} spawn failed: {e}",
950 claim_key
951 );
952 }
953 Ok(_) => {}
954 }
955 }
956 CloseOutcome::Parked(_) | CloseOutcome::Refused(_) => {
957 // Park-exclusion: HOLD the claim so the live-claims filter keeps
958 // skipping this node. The active_claims entry is NOT removed.
959 //
960 // Also refresh the TTL via a same-holder re-acquire. The worker's
961 // idempotent re-acquire during init-target-state.sh rewrites
962 // acquired_at with the worker's pid; after the worker exits the
963 // TTL window is measured from the worker's re-acquire time.
964 // Re-acquiring here refreshes acquired_at to NOW so the claim
965 // stays live through the next walk iteration.
966 let session_key = self
967 .active_claims
968 .get(&unit.id)
969 .map(|e| e.session_key.clone())
970 .unwrap_or_else(|| unit.session_key.clone());
971 let claim_key = format!("node:{}", unit.id);
972 let claim_holder = format!("target-session:{session_key}");
973
974 let refresh_result = retry_etxtbsy(|| {
975 abi_cmd(&self.abi_bin)
976 .args([
977 "claim",
978 "acquire",
979 &claim_key,
980 "--holder",
981 &claim_holder,
982 "--ttl",
983 "2h",
984 "--reason",
985 "megawalk park-exclusion hold",
986 ])
987 .env(
988 "FNO_CLAIMS_ROOT",
989 std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()),
990 )
991 .output()
992 });
993
994 match refresh_result {
995 Ok(o) if !o.status.success() => {
996 eprintln!(
997 "loop-megawalk: WARNING: claim refresh (park-hold) {} failed (exit {}): {}",
998 claim_key,
999 o.status,
1000 String::from_utf8_lossy(&o.stderr).trim()
1001 );
1002 }
1003 Err(e) => {
1004 eprintln!(
1005 "loop-megawalk: WARNING: claim refresh (park-hold) {} spawn failed: {e}",
1006 claim_key
1007 );
1008 }
1009 Ok(_) => {}
1010 }
1011 }
1012 }
1013
1014 // Increment the closed-units counter (used by max_units cap).
1015 //
1016 // Intentional: Parked and Refused outcomes count toward --max-units just
1017 // like Closed outcomes. The semantics of --max-units N are "process N
1018 // units, whatever the outcome" (once-mode). A parked unit consumed one
1019 // dispatch slot; counting it avoids an unbounded walk when every unit parks.
1020 self.units_closed += 1;
1021
1022 Ok(outcome)
1023 }
1024}
1025
1026// ── MegawalkDispatcher ────────────────────────────────────────────────────────
1027
1028/// A Dispatcher that wraps ShelloutDispatcher with per-unit env injection.
1029///
1030/// For each Dispatcher::run call, builds env = static env +
1031/// `CONTINUE_PROMPT="/target no-merge <unit.id>"` (or `/target <unit.id>`
1032/// when allow_merge) + `TARGET_SESSION_ID=<unit.session_key>`.
1033///
1034/// This injects TARGET_SESSION_ID into the worker session so
1035/// init-target-state.sh uses it verbatim (Task 4: the override path in
1036/// init-target-state.sh). The claim re-acquire in init-target-state.sh then
1037/// has `holder = target-session:<session_key>` matching the walker's claim,
1038/// making it idempotent (see module doc).
1039pub struct MegawalkDispatcher {
1040 driver_lib: PathBuf,
1041 static_env: Vec<(String, String)>,
1042 cwd: PathBuf,
1043 /// Reserved for future use (e.g. fno claim refresh per dispatch).
1044 _abi_bin: String,
1045 allow_merge: bool,
1046}
1047
1048impl MegawalkDispatcher {
1049 pub fn new(
1050 driver_lib: PathBuf,
1051 static_env: Vec<(String, String)>,
1052 cwd: PathBuf,
1053 abi_bin: String,
1054 allow_merge: bool,
1055 ) -> Self {
1056 Self {
1057 driver_lib,
1058 static_env,
1059 // Root dispatched target workers at canonical main: megawalk is
1060 // single-repo target-class, so a walk launched from a linked
1061 // worktree must not start each worker in that worktree (the shared
1062 // .fno/ session-state collision, ab-77b691dc). canonical_repo_root
1063 // is a no-op when already canonical and falls back to the given cwd
1064 // when resolution is ambiguous (git missing / bare).
1065 cwd: crate::paths::canonical_repo_root(&cwd).unwrap_or(cwd),
1066 _abi_bin: abi_bin,
1067 allow_merge,
1068 }
1069 }
1070}
1071
1072impl Dispatcher for MegawalkDispatcher {
1073 fn run(&self, unit: &Unit, ctx: &DispatchCtx) -> Result<Box<dyn Session>, LoopError> {
1074 let continue_prompt = if self.allow_merge {
1075 format!("/target {}", unit.id)
1076 } else {
1077 format!("/target no-merge {}", unit.id)
1078 };
1079
1080 // Build merged env: static + per-unit overrides.
1081 let mut env = self.static_env.clone();
1082 // Override CONTINUE_PROMPT with the per-unit prompt.
1083 // Remove any existing CONTINUE_PROMPT from the static list to avoid
1084 // duplicates (the last value wins in most shells, but explicit removal
1085 // is cleaner).
1086 env.retain(|(k, _)| k != "CONTINUE_PROMPT" && k != "TARGET_SESSION_ID");
1087 env.push(("CONTINUE_PROMPT".to_string(), continue_prompt));
1088 env.push(("TARGET_SESSION_ID".to_string(), unit.session_key.clone()));
1089
1090 // Inject driver-specific extra env (e.g. TARGET_MISSION_* for fleet nodes).
1091 // These come after the static env so they take precedence over any
1092 // identically-named static values (last write wins in most shells).
1093 env.extend(unit.extra_env.iter().cloned());
1094
1095 // Construct a ShelloutDispatcher with the merged env per-unit.
1096 // ShelloutDispatcher is cheap to construct (no I/O at construction time).
1097 let dispatcher = ShelloutDispatcher::new(self.driver_lib.clone(), env, self.cwd.clone());
1098 dispatcher.run(unit, ctx)
1099 }
1100}
1101
1102// ── walk-as-unit termination emission (group 3, ab-9fd662c6) ──────────────────
1103
1104/// Emit a `termination` event for the WALK itself, keyed by `session_key`.
1105///
1106/// A parent loop (megatron) that dispatched this walk as a unit awaits a
1107/// `termination` event matching the unit's session_key - the same contract a
1108/// target session's loop-check satisfies one altitude down. Reuses the
1109/// existing `termination` event kind (loopcheck's), so no new kind and no
1110/// 4-place lockstep edit; `Journal::append` mirrors to the global journal,
1111/// which is how the parent (running in a different cwd) finds it.
1112///
1113/// The reason string is the serde serialization of the walk-level
1114/// TerminationReason (e.g. "NoWork"), the same spelling
1115/// `parse_termination_reason` round-trips.
1116pub fn emit_walk_termination(
1117 journal: &Journal,
1118 session_key: &str,
1119 reason: &TerminationReason,
1120 iterations_used: u64,
1121 units_closed: usize,
1122) -> Result<(), crate::loop_runtime::LoopError> {
1123 // Loud on a non-string serialization: a Debug-format fallback would emit
1124 // a spelling parse_termination_reason cannot match, turning a future
1125 // enum-shape change into a silent find_termination miss at the parent
1126 // (sigma-review). All current variants are unit -> always strings.
1127 let reason_str = match serde_json::to_value(reason) {
1128 Ok(serde_json::Value::String(s)) => s,
1129 other => {
1130 return Err(crate::loop_runtime::LoopError::Journal(format!(
1131 "walk termination reason did not serialize to a string \
1132 (got {other:?}); refusing to journal an unparseable reason"
1133 )));
1134 }
1135 };
1136 journal.append(
1137 "termination",
1138 serde_json::json!({
1139 "session_id": session_key,
1140 "reason": reason_str,
1141 "message": format!(
1142 "megawalk walk terminated: {reason_str} ({iterations_used} iterations, {units_closed} units closed)"
1143 ),
1144 }),
1145 )
1146}
1147
1148// ── verb glue: pub fn run() ────────────────────────────────────────────────────
1149
1150/// Entry point for `fno-agents loop run --driver megawalk ...`.
1151///
1152/// Called from loop_target.rs when --driver megawalk is specified.
1153///
1154/// Exit codes:
1155/// - 0: NoWork | DonePRGreen | DoneAdvisory (walk completed or backlog empty)
1156/// - 1: Budget | NoProgress | Aborted (walk hit ceiling or failed)
1157/// - 2: usage / configuration error
1158/// - 77: driver binary missing from PATH (preflight failure)
1159/// - 130: Interrupted (SIGINT)
1160#[allow(clippy::too_many_arguments)]
1161pub fn run(
1162 // Parsed flags forwarded from loop_target.rs run_loop_verb_inner.
1163 dispatcher_name: &str,
1164 max_iterations: Option<u64>,
1165 max_turns: u64,
1166 budget_usd: f64,
1167 model: Option<&str>,
1168 prompt_file: Option<&str>,
1169 cli_alias: Option<&str>,
1170 driver_lib_dir: Option<PathBuf>,
1171 cwd: PathBuf,
1172 project: Option<String>,
1173 all: bool,
1174 allow_merge: bool,
1175 parallel_cap: Option<u64>,
1176 max_units: Option<u64>,
1177 mission: Option<String>,
1178 termination_key: Option<String>,
1179) -> i32 {
1180 match run_inner(
1181 dispatcher_name,
1182 max_iterations,
1183 max_turns,
1184 budget_usd,
1185 model,
1186 prompt_file,
1187 cli_alias,
1188 driver_lib_dir,
1189 cwd,
1190 project,
1191 all,
1192 allow_merge,
1193 parallel_cap,
1194 max_units,
1195 mission,
1196 termination_key,
1197 ) {
1198 Ok(code) => code,
1199 Err(e) => {
1200 eprintln!("fno-agents loop megawalk: {e}");
1201 2
1202 }
1203 }
1204}
1205
1206#[allow(clippy::too_many_arguments)]
1207fn run_inner(
1208 dispatcher_name: &str,
1209 max_iterations: Option<u64>,
1210 max_turns: u64,
1211 budget_usd: f64,
1212 model: Option<&str>,
1213 prompt_file: Option<&str>,
1214 cli_alias: Option<&str>,
1215 driver_lib_dir: Option<PathBuf>,
1216 cwd: PathBuf,
1217 project: Option<String>,
1218 all: bool,
1219 allow_merge: bool,
1220 parallel_cap: Option<u64>,
1221 max_units: Option<u64>,
1222 mission: Option<String>,
1223 termination_key: Option<String>,
1224) -> Result<i32, Box<dyn std::error::Error>> {
1225 use crate::loop_dispatch::{driver_default_max, preflight, resolve_driver_binary};
1226 use crate::loop_runtime::{
1227 run_loop, GlobalJournalPath, Journal, LoopBudget, ProjectJournalPath,
1228 };
1229 use crate::loop_target::{exit_code_for_reason, install_sigint_handler, SIGINT_RECEIVED};
1230 use std::sync::atomic::Ordering;
1231
1232 // ── resolve driver-lib-dir ────────────────────────────────────────────────
1233 let lib_dir = match driver_lib_dir {
1234 Some(d) => d,
1235 None => {
1236 if let Ok(env_dir) = std::env::var("FNO_DRIVER_LIB_DIR") {
1237 PathBuf::from(env_dir)
1238 } else {
1239 let candidate = cwd.join("scripts").join("lib");
1240 if candidate.is_dir() {
1241 candidate
1242 } else {
1243 eprintln!(
1244 "fno-agents loop megawalk: cannot resolve driver lib directory. \
1245 Pass --driver-lib-dir <path> or set FNO_DRIVER_LIB_DIR env."
1246 );
1247 return Ok(2);
1248 }
1249 }
1250 }
1251 };
1252
1253 // ── preflight: driver whitelist, lib file, binary ─────────────────────────
1254 let lib_path = match preflight(dispatcher_name, &lib_dir, cli_alias) {
1255 Ok(p) => p,
1256 Err(crate::loop_runtime::LoopError::Dispatch(msg)) => {
1257 eprintln!("fno-agents loop megawalk: {msg}");
1258 return Ok(77);
1259 }
1260 Err(e) => {
1261 eprintln!("fno-agents loop megawalk: {e}");
1262 return Ok(2);
1263 }
1264 };
1265
1266 // ── resolve max_iterations ────────────────────────────────────────────────
1267 let max_iters = match max_iterations {
1268 Some(n) => n,
1269 None => match driver_default_max(&lib_path) {
1270 Ok(n) => n,
1271 Err(e) => {
1272 eprintln!(
1273 "fno-agents loop megawalk: could not query driver_default_max: {e}; \
1274 pass --max-iterations explicitly"
1275 );
1276 return Ok(2);
1277 }
1278 },
1279 };
1280
1281 // ── acquire walker singleton claim ────────────────────────────────────────
1282 // Prevents two concurrent megawalk processes from racing on the same backlog.
1283 // Key the singleton on the CANONICAL repo root, matching the canonical cwd
1284 // each dispatched worker is rooted at (MegawalkDispatcher). Otherwise two
1285 // walkers launched from two linked worktrees of the same repo would take
1286 // different `walker:<worktree>` keys yet dispatch workers into the same
1287 // canonical .fno/ state, recreating the collision this change removes
1288 // (codex P2). No-op when already canonical; falls back to cwd when ambiguous.
1289 let walker_root = crate::paths::canonical_repo_root(&cwd).unwrap_or_else(|| cwd.clone());
1290 let walker_key = format!("walker:{}", walker_root.display());
1291 let walker_holder = format!("megawalk-loop:{}", std::process::id());
1292 let abi_bin = std::env::var("FNO_BIN").unwrap_or_else(|_| "fno".to_string());
1293
1294 let walker_claim_result = abi_cmd(&abi_bin)
1295 .args([
1296 "claim",
1297 "acquire",
1298 &walker_key,
1299 "--holder",
1300 &walker_holder,
1301 "--ttl",
1302 "24h",
1303 "--reason",
1304 "megawalk walker singleton",
1305 ])
1306 .output();
1307
1308 match walker_claim_result {
1309 Ok(o) if !o.status.success() => {
1310 let stderr = String::from_utf8_lossy(&o.stderr).trim().to_string();
1311 eprintln!(
1312 "fno-agents loop megawalk: walker singleton already running: {stderr}; \
1313 another megawalk is active for this project (holder in claim file)"
1314 );
1315 return Ok(1);
1316 }
1317 Err(e) => {
1318 // If fno is not available, warn and continue (best-effort singleton).
1319 eprintln!(
1320 "fno-agents loop megawalk: WARNING: walker claim acquire failed: {e} (continuing)"
1321 );
1322 }
1323 Ok(_) => {}
1324 }
1325
1326 // ── build static env ──────────────────────────────────────────────────────
1327 let abilities_dir = cwd.join(".fno");
1328 let output_file = abilities_dir.join("target-last-output.txt");
1329 let history_file = abilities_dir.join("target-history.txt");
1330 let signal_file = abilities_dir.join("target-promise.signal");
1331
1332 let mut env: Vec<(String, String)> = vec![
1333 (
1334 "OUTPUT_FILE".to_string(),
1335 output_file.to_str().unwrap_or("").to_string(),
1336 ),
1337 (
1338 "HISTORY_FILE".to_string(),
1339 history_file.to_str().unwrap_or("").to_string(),
1340 ),
1341 (
1342 "SIGNAL_FILE".to_string(),
1343 signal_file.to_str().unwrap_or("").to_string(),
1344 ),
1345 ("MAX_TURNS".to_string(), max_turns.to_string()),
1346 ("BUDGET_USD".to_string(), format!("{budget_usd}")),
1347 // CONTINUE_PROMPT is set per-unit by MegawalkDispatcher.
1348 ("CONTINUE_PROMPT".to_string(), String::new()),
1349 ];
1350
1351 if let Some(m) = model {
1352 env.push(("MODEL_FLAG".to_string(), format!("--model {m}")));
1353 } else {
1354 env.push(("MODEL_FLAG".to_string(), String::new()));
1355 }
1356
1357 if let Some(pf) = prompt_file {
1358 env.push(("PROMPT_FILE".to_string(), pf.to_string()));
1359 }
1360
1361 if let Some(cli) = cli_alias {
1362 env.push(("CLI".to_string(), cli.to_string()));
1363 }
1364
1365 env.push((
1366 "FNO_CWD".to_string(),
1367 cwd.to_str().unwrap_or(".").to_string(),
1368 ));
1369
1370 // ── SIGINT handler ────────────────────────────────────────────────────────
1371 install_sigint_handler();
1372
1373 // ── build journal ─────────────────────────────────────────────────────────
1374 let project_events = abilities_dir.join("events.jsonl");
1375 let home_dir = std::env::var("HOME")
1376 .map(PathBuf::from)
1377 .unwrap_or_else(|_| PathBuf::from("/tmp"));
1378 let global_events = home_dir.join(".fno").join("events.jsonl");
1379 let journal = Journal::new(
1380 ProjectJournalPath(project_events),
1381 GlobalJournalPath(global_events),
1382 );
1383
1384 // ── print header ──────────────────────────────────────────────────────────
1385 let binary_name = resolve_driver_binary(dispatcher_name, cli_alias);
1386 let scope = if all {
1387 "all projects".to_string()
1388 } else if let Some(ref p) = project {
1389 format!("project={p}")
1390 } else {
1391 "auto-detected project".to_string()
1392 };
1393 println!("fno-agents loop megawalk");
1394 println!(" driver: megawalk");
1395 println!(" dispatcher: {dispatcher_name} (binary: {binary_name})");
1396 println!(" scope: {scope}");
1397 println!(" iterations: {max_iters} max");
1398 println!(" budget: ${budget_usd} USD");
1399
1400 // ── resume narration (AC4-UI) ─────────────────────────────────────────────
1401 // Shell `fno claim list --prefix node: --include-stale --json` and print
1402 // one header line when stale node claims exist. A stale claim means a prior
1403 // walk was interrupted mid-unit; the walker will re-acquire on contact and
1404 // recover the work. Best-effort only: if the command fails for any reason,
1405 // skip silently (claim narration is informational, not a gate).
1406 {
1407 let claim_out = abi_cmd(&abi_bin)
1408 .args([
1409 "claim",
1410 "list",
1411 "--prefix",
1412 "node:",
1413 "--include-stale",
1414 "--json",
1415 ])
1416 .output();
1417 if let Ok(o) = claim_out {
1418 if o.status.success() {
1419 let stdout = String::from_utf8_lossy(&o.stdout);
1420 // JSON output is a list of claim objects with a "status" field.
1421 // Count entries where status == "stale".
1422 if let Ok(arr) = serde_json::from_str::<serde_json::Value>(stdout.trim()) {
1423 let stale_count = arr
1424 .as_array()
1425 .map(|a| {
1426 a.iter()
1427 .filter(|v| v["status"].as_str() == Some("stale"))
1428 .count()
1429 })
1430 .unwrap_or(0);
1431 if stale_count > 0 {
1432 println!(
1433 "resume: {stale_count} stale node claim(s) from a prior walk \
1434 will be recovered on contact"
1435 );
1436 }
1437 }
1438 }
1439 }
1440 }
1441
1442 // ── build queue and dispatcher ────────────────────────────────────────────
1443 let mut queue = MegawalkQueue::new_with_max_units(abi_bin.clone(), project, all, max_units)
1444 .with_mission(mission.clone());
1445 let dispatcher =
1446 MegawalkDispatcher::new(lib_path, env, cwd.clone(), abi_bin.clone(), allow_merge);
1447
1448 // ── parallel-cap notice ───────────────────────────────────────────────────
1449 // Group-2 ships the conservative sequential default (Claude's Discretion 3).
1450 // run_loop is single-threaded; when cap > 1, print one honest line so the
1451 // flag is accepted-but-explicit, never a silent drop.
1452 if let Some(cap) = parallel_cap {
1453 if cap > 1 {
1454 println!(
1455 "megawalk: --parallel-cap {cap} accepted; execution is SEQUENTIAL \
1456 (collision-conservative default; group-2 serializes regardless of cap)"
1457 );
1458 }
1459 }
1460
1461 // ── max-units notice ──────────────────────────────────────────────────────
1462 if let Some(n) = max_units {
1463 println!("megawalk: --max-units {n} (walk stops after {n} unit(s) closed)");
1464 }
1465
1466 // ── walker-claim release helper ───────────────────────────────────────────
1467 // Called on every early-return path after claim acquisition so the
1468 // documented contract ("releases on all exit paths") is met. TTL/PID
1469 // makes leaked claims recoverable, but explicit release is cleaner and
1470 // makes tests deterministic (Gemini HIGH finding).
1471 let release_walker_claim = || {
1472 let _ = abi_cmd(&abi_bin)
1473 .args(["claim", "release", &walker_key, "--holder", &walker_holder])
1474 .output();
1475 };
1476
1477 // ── build budget ──────────────────────────────────────────────────────────
1478 let budget = match LoopBudget::new(max_iters) {
1479 Ok(b) => b,
1480 Err(e) => {
1481 eprintln!("fno-agents loop megawalk: {e}");
1482 release_walker_claim();
1483 return Ok(2);
1484 }
1485 };
1486
1487 // ── cancel closure ────────────────────────────────────────────────────────
1488 let cancel_file = cwd.join(".fno").join(".target-cancelled");
1489 let cancel = move || SIGINT_RECEIVED.load(Ordering::SeqCst) || cancel_file.exists();
1490
1491 // ── run the loop ──────────────────────────────────────────────────────────
1492 // Per-unit dispatch cap (plan Failure Mode: a session that dies without a
1493 // TerminationReason event is synthesized as node_failed and must count
1494 // toward the consecutive-failure pause). Re-dispatch is the NORMAL
1495 // continuation mechanism for multi-session work, so the cap is generous:
1496 // 15 sessions x MAX_TURNS turns is ample for an L-sized node, while a
1497 // crash-looping driver parks after 15 fast failures (close(NoProgress) ->
1498 // streak) instead of burning the whole walk budget on one unit.
1499 const PER_UNIT_MAX_DISPATCHES: u64 = 15;
1500 let outcome = match run_loop(
1501 &mut queue,
1502 &dispatcher,
1503 &budget,
1504 &journal,
1505 &cancel,
1506 Some(PER_UNIT_MAX_DISPATCHES),
1507 ) {
1508 Ok(o) => o,
1509 Err(e) => {
1510 eprintln!("fno-agents loop megawalk: fatal loop error: {e}");
1511 release_walker_claim();
1512 return Ok(2);
1513 }
1514 };
1515
1516 // ── release walker singleton claim ────────────────────────────────────────
1517 release_walker_claim();
1518
1519 // ── walk-as-unit termination event (group 3) ──────────────────────────────
1520 // When a parent loop (megatron) dispatched this walk with a session key,
1521 // journal the walk's own termination so the parent's find_termination
1522 // observes it (via the global mirror when cwds differ). Fatal on project-
1523 // journal failure, consistent with every other journal.append here.
1524 if let Some(ref key) = termination_key {
1525 if let Err(e) = emit_walk_termination(
1526 &journal,
1527 key,
1528 &outcome.reason,
1529 outcome.iterations_used,
1530 outcome.units.len(),
1531 ) {
1532 eprintln!("fno-agents loop megawalk: failed to journal walk termination: {e}");
1533 return Ok(2);
1534 }
1535 }
1536
1537 // ── report outcome ────────────────────────────────────────────────────────
1538 let exit_code = exit_code_for_reason(&outcome.reason);
1539 println!(
1540 "megawalk: {:?} ({} iterations used, {} units closed)",
1541 outcome.reason,
1542 outcome.iterations_used,
1543 outcome.units.len()
1544 );
1545 for unit_result in &outcome.units {
1546 println!(
1547 " unit {}: {:?} ({:?})",
1548 unit_result.unit_id, unit_result.evidence.reason, unit_result.close
1549 );
1550 }
1551
1552 Ok(exit_code)
1553}
1554
1555#[cfg(test)]
1556mod fresh_tests {
1557 //! ab-77b691dc: a megawalk worker is rooted at canonical main, so a walk
1558 //! launched from a linked worktree does not start each target worker in that
1559 //! worktree (the shared .fno/ session-state collision).
1560 use super::*;
1561
1562 fn git(dir: &std::path::Path, args: &[&str]) -> bool {
1563 std::process::Command::new("git")
1564 .arg("-C")
1565 .arg(dir)
1566 .args(args)
1567 .output()
1568 .map(|o| o.status.success())
1569 .unwrap_or(false)
1570 }
1571
1572 #[test]
1573 fn dispatcher_roots_worker_cwd_at_canonical_from_worktree() {
1574 // Skip when git is unavailable (mirrors the Python skipif(no git)).
1575 if std::process::Command::new("git")
1576 .arg("--version")
1577 .output()
1578 .is_err()
1579 {
1580 return;
1581 }
1582 let tmp = tempfile::tempdir().unwrap();
1583 let main = tmp.path().join("main");
1584 std::fs::create_dir(&main).unwrap();
1585 assert!(git(&main, &["init", "-q"]));
1586 assert!(git(&main, &["config", "user.email", "t@t"]));
1587 assert!(git(&main, &["config", "user.name", "t"]));
1588 assert!(git(&main, &["commit", "-q", "--allow-empty", "-m", "init"]));
1589 let linked = tmp.path().join("wt");
1590 assert!(git(
1591 &main,
1592 &[
1593 "worktree",
1594 "add",
1595 "-q",
1596 linked.to_str().unwrap(),
1597 "-b",
1598 "feat"
1599 ]
1600 ));
1601
1602 let d = MegawalkDispatcher::new(
1603 std::path::PathBuf::from("/driver/lib.sh"),
1604 vec![],
1605 linked.clone(),
1606 "fno".to_string(),
1607 false,
1608 );
1609 let want = std::fs::canonicalize(&main).unwrap();
1610 assert_eq!(
1611 d.cwd, want,
1612 "megawalk worker cwd must be rooted at canonical main, not the worktree"
1613 );
1614 }
1615
1616 #[test]
1617 fn dispatcher_keeps_cwd_when_not_a_worktree() {
1618 // A non-git cwd -> canonical resolution returns None -> keep the given
1619 // cwd (the safe-side fallback; also the already-canonical no-op case).
1620 let tmp = tempfile::tempdir().unwrap();
1621 let d = MegawalkDispatcher::new(
1622 std::path::PathBuf::from("/driver/lib.sh"),
1623 vec![],
1624 tmp.path().to_path_buf(),
1625 "fno".to_string(),
1626 false,
1627 );
1628 assert_eq!(d.cwd, tmp.path());
1629 }
1630
1631 #[test]
1632 fn retry_etxtbsy_passes_success_through_without_retry() {
1633 let mut calls = 0u32;
1634 let r: std::io::Result<u8> = retry_etxtbsy(|| {
1635 calls += 1;
1636 Ok(7)
1637 });
1638 assert_eq!(r.unwrap(), 7);
1639 assert_eq!(calls, 1, "a successful spawn must not retry");
1640 }
1641
1642 #[test]
1643 fn retry_etxtbsy_retries_then_succeeds() {
1644 // Simulate ETXTBSY clearing after a couple of attempts.
1645 let mut calls = 0u32;
1646 let r: std::io::Result<u8> = retry_etxtbsy(|| {
1647 calls += 1;
1648 if calls < 3 {
1649 Err(std::io::Error::from_raw_os_error(libc::ETXTBSY))
1650 } else {
1651 Ok(42)
1652 }
1653 });
1654 assert_eq!(r.unwrap(), 42);
1655 assert_eq!(calls, 3, "must retry past transient ETXTBSY");
1656 }
1657
1658 #[test]
1659 fn retry_etxtbsy_does_not_swallow_other_errors() {
1660 // A non-ETXTBSY error returns immediately, no retry.
1661 let mut calls = 0u32;
1662 let r: std::io::Result<u8> = retry_etxtbsy(|| {
1663 calls += 1;
1664 Err(std::io::Error::from_raw_os_error(libc::ENOENT))
1665 });
1666 assert_eq!(r.unwrap_err().raw_os_error(), Some(libc::ENOENT));
1667 assert_eq!(calls, 1, "a non-ETXTBSY error must not retry");
1668 }
1669
1670 #[test]
1671 fn retry_etxtbsy_gives_up_after_max_retries() {
1672 // Persistent ETXTBSY surfaces after the bounded retry budget (1 initial
1673 // + 5 retries = 6 calls) rather than spinning forever.
1674 let mut calls = 0u32;
1675 let r: std::io::Result<u8> = retry_etxtbsy(|| {
1676 calls += 1;
1677 Err(std::io::Error::from_raw_os_error(libc::ETXTBSY))
1678 });
1679 assert_eq!(r.unwrap_err().raw_os_error(), Some(libc::ETXTBSY));
1680 assert_eq!(calls, 6, "1 initial attempt + MAX_RETRIES(5)");
1681 }
1682}