fno_agents/loop_runtime.rs
1//! Unified loop runtime primitive for target, megawalk, and megatron drivers.
2//!
3//! This module is the generic "walk a queue of units, dispatch sessions,
4//! handle termination" engine. Drivers differ only in their Queue and
5//! Dispatcher implementations. The runtime itself has no opinion about what
6//! a "unit" is (backlog node, fleet project, ...) or how sessions are launched.
7//!
8//! ## Why journal write failure is fatal (contrast with loopcheck.rs)
9//!
10//! `loopcheck.rs` runs as a stop hook: it is a read-only observer that must
11//! never block the session from exiting. Journal writes there are best-effort
12//! (logged to stderr, never propagated as errors) because a failed write
13//! cannot undo a decision the runtime already made.
14//!
15//! Here the journal is the observability record of an *active walk*. If the
16//! runtime cannot record that it dispatched a session, an operator watching
17//! the log sees nothing and cannot tell whether work is happening. An
18//! unobservable walk that continues spending compute (and potentially money)
19//! is worse than stopping loudly. The invariant: "The system must handle
20//! journal write failure by stopping dispatch loudly; an unobservable walk
21//! must not continue spending."
22//!
23//! The global mirror (`~/.fno/events.jsonl`) is best-effort: a write
24//! failure there is logged to stderr but never fatal, because the project
25//! journal is the authoritative record.
26//!
27//! ## Source field
28//!
29//! Events written by this runtime carry `source: "loop"`, distinct from the
30//! `source: "hook"` that loopcheck uses. The two streams have different
31//! semantics: hook events are stop-hook decision records; loop events are
32//! walk-level orchestration records. Consumers that aggregate both streams
33//! can use the source field to distinguish them.
34//!
35//! ## Module naming
36//!
37//! The module name starts with `loop` so that the LOC-ratchet glob
38//! `crates/fno-agents/src/loop*` counts this file's LOC toward the ratchet
39//! budget deliberately (alongside loopcheck.rs).
40
41use crate::loopcheck::TerminationReason;
42use chrono::Utc;
43use serde_json::{json, Value};
44use std::fs;
45use std::io::{BufRead, BufReader, Write};
46use std::path::{Path, PathBuf};
47
48// ── newtype wrappers for Journal paths (F9) ───────────────────────────────────
49
50/// Newtype for the project-authoritative journal path. Writes are FATAL on failure.
51/// Using a distinct type prevents silent positional swap of the two same-type args.
52pub struct ProjectJournalPath(pub PathBuf);
53
54/// Newtype for the global mirror journal path (`~/.fno/events.jsonl`).
55/// Writes are best-effort only.
56pub struct GlobalJournalPath(pub PathBuf);
57
58// ── public error type ─────────────────────────────────────────────────────────
59
60/// Errors returned by the loop runtime.
61#[derive(Debug, thiserror::Error)]
62pub enum LoopError {
63 /// I/O failure (file open, read, write).
64 #[error("I/O error: {0}")]
65 Io(#[from] std::io::Error),
66
67 /// Journal write to the project events file failed (fatal per spec).
68 #[error("journal write failure (project): {0}")]
69 Journal(String),
70
71 /// Queue operation failed.
72 #[error("queue error: {0}")]
73 Queue(String),
74
75 /// Dispatcher operation failed.
76 #[error("dispatch error: {0}")]
77 Dispatch(String),
78
79 /// Configuration error (e.g., invalid budget).
80 #[error("configuration error: {0}")]
81 Config(String),
82}
83
84// ── public types ──────────────────────────────────────────────────────────────
85
86/// A single unit of work to be dispatched. Drivers map their domain objects
87/// (backlog nodes, fleet projects, ...) to this common shape.
88pub struct Unit {
89 /// Stable identifier, e.g. `ab-XXXXXXXX` for a backlog node.
90 pub id: String,
91 /// Human-readable title for log output.
92 pub title: String,
93 /// Correlation key matched against termination events' `data.session_id`.
94 /// For target sessions this is the session identifier written into the
95 /// target-state.md manifest; for other drivers it is whatever key the
96 /// dispatcher embeds in its events.
97 pub session_key: String,
98 /// Optional plan path for context (not used by the runtime itself).
99 pub plan_path: Option<String>,
100 /// Driver-specific extra env vars to inject into the child process.
101 /// The runtime passes these through to the Dispatcher without inspecting
102 /// them; TargetQueue leaves this empty.
103 pub extra_env: Vec<(String, String)>,
104}
105
106/// Evidence of termination extracted from the project journal.
107pub struct Evidence {
108 /// The parsed TerminationReason.
109 pub reason: TerminationReason,
110 /// Human-readable message from the event's `data.message` field.
111 pub message: String,
112}
113
114/// Outcome of queue.close() for a single unit.
115#[derive(Debug, PartialEq)]
116pub enum CloseOutcome {
117 /// Unit was closed successfully.
118 Closed,
119 /// Queue refused to close the unit (e.g. it was claimed by another walker).
120 Refused(String),
121 /// Unit was parked for later (e.g. dependents not yet resolved).
122 Parked(String),
123 /// Unit built successfully (PR up, reviewed) but its PR is not yet merged
124 /// (x-aba7: graph done = merged). Success-shaped: the claim is RELEASED and
125 /// no failure is recorded; the node stays `in_review` and is closed at the
126 /// actual merge by `fno backlog reconcile` / merge-triggered advance.
127 AwaitingMerge,
128}
129
130/// Runtime-varying context passed to each Dispatcher::run call. Static
131/// configuration (project root, env vars, etc.) lives in the Dispatcher impl.
132pub struct DispatchCtx {
133 /// 1-based iteration counter across all units in this walk.
134 pub iteration: u64,
135}
136
137// ── traits ────────────────────────────────────────────────────────────────────
138
139/// Source of work units. Each call to `next` either returns the next unit to
140/// dispatch or `None` to signal that the walk is complete.
141///
142/// ## Why `&mut self` (F8 rationale)
143///
144/// A Queue can carry real cursor state. Using `&self` would force pointless
145/// `Mutex`-wrapping for state that is inherently sequential in the
146/// single-threaded walk loop. No threading requirement exists at this seam:
147/// `run_loop` is always called from a single thread. `&mut self` is the natural
148/// fit and avoids unnecessary interior-mutability noise.
149pub trait Queue {
150 /// Return the next unit, or `None` if the queue is empty.
151 fn next(&mut self) -> Result<Option<Unit>, LoopError>;
152
153 /// Mark a unit as closed (done/parked/refused) given the termination
154 /// evidence extracted from the journal.
155 fn close(&mut self, unit: &Unit, evidence: &Evidence) -> Result<CloseOutcome, LoopError>;
156}
157
158/// A live session handle returned by a Dispatcher.
159pub trait Session {
160 /// Block until the session exits and return its exit code.
161 fn wait(&mut self) -> Result<i32, LoopError>;
162
163 /// Tail of the session's captured driver output (the `OUTPUT_FILE` the bash
164 /// driver redirects claude stdout+stderr into), if available. The walk reads
165 /// it to classify a non-termination exit: claude's bg-guard refusal
166 /// ("running as a background agent (bg)") must terminate the unit rather than
167 /// be re-dispatched into an infinite respawn loop (x-4504, AC1-ERR). Default
168 /// `None` -> a Session that captures no output is treated as an ordinary
169 /// crash and re-dispatched exactly as before.
170 fn output_tail(&self) -> Option<String> {
171 None
172 }
173}
174
175/// Launches sessions for units. Stateless with respect to the walk loop;
176/// any per-dispatch state is internal to the impl.
177pub trait Dispatcher {
178 /// Launch a session for the given unit in the given walk context.
179 fn run(&self, unit: &Unit, ctx: &DispatchCtx) -> Result<Box<dyn Session>, LoopError>;
180}
181
182// ── budget ────────────────────────────────────────────────────────────────────
183
184/// Walk-level iteration budget. Prevents unbounded loops when sessions never
185/// produce termination events.
186pub struct LoopBudget {
187 max_iterations: u64,
188}
189
190impl LoopBudget {
191 /// Create a budget. Rejects `max_iterations = 0` because a budget of zero
192 /// would immediately terminate every walk at the pre-dispatch check,
193 /// which is never a useful configuration.
194 pub fn new(max_iterations: u64) -> Result<Self, LoopError> {
195 if max_iterations == 0 {
196 return Err(LoopError::Config("max_iterations must be > 0".to_string()));
197 }
198 Ok(Self { max_iterations })
199 }
200}
201
202// ── journal ───────────────────────────────────────────────────────────────────
203
204/// Append-only event log with a project-authoritative path and a best-effort
205/// global mirror.
206pub struct Journal {
207 /// Project-scoped events file. Writes here are FATAL on failure.
208 project_path: PathBuf,
209 /// Global mirror (`~/.fno/events.jsonl`). Writes here are best-effort.
210 global_path: PathBuf,
211}
212
213impl Journal {
214 /// Create a Journal. Paths are injected via newtypes (F9) to prevent silent
215 /// positional swap of the two same-type `PathBuf` arguments.
216 /// The runtime never hard-codes `~/.fno/`. The driver CLI wires the real
217 /// paths (task 1.2). Tests use tempdir paths.
218 pub fn new(project_path: ProjectJournalPath, global_path: GlobalJournalPath) -> Self {
219 Self {
220 project_path: project_path.0,
221 global_path: global_path.0,
222 }
223 }
224
225 /// Convenience constructor accepting plain `PathBuf`s directly. Intended for
226 /// tests where importing the newtype wrappers would add noise. Production
227 /// callers should prefer `Journal::new` (which enforces distinct types).
228 pub fn new_raw(project_path: PathBuf, global_path: PathBuf) -> Self {
229 Self {
230 project_path,
231 global_path,
232 }
233 }
234
235 /// Append one event line to the project journal (fatal on failure) and
236 /// mirror it to the global file (best-effort).
237 ///
238 /// The envelope shape is:
239 /// `{"ts":"YYYY-MM-DDTHH:MM:SSZ","type":"<kind>","source":"loop","data":{...}}`
240 ///
241 /// Method name is `append` (NOT `emit`/`emit_fields`) so the
242 /// daemon-stream parity scanner in lib.rs (which greps for `.emit(`)
243 /// does not require these kinds to be registered in KNOWN_EVENT_KINDS.
244 /// These are loop-stream events, not daemon-stream events.
245 pub fn append(&self, event_type: &str, data: Value) -> Result<(), LoopError> {
246 let ts = Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
247 let env = json!({
248 "ts": ts,
249 "type": event_type,
250 "source": "loop",
251 "data": data,
252 });
253 let mut line = serde_json::to_string(&env)
254 .map_err(|e| LoopError::Journal(format!("serialize {event_type}: {e}")))?;
255 line.push('\n');
256
257 // Write to project file - FATAL on failure.
258 self.append_to_file(&self.project_path, &line, true)?;
259
260 // Mirror to global file - best-effort (warn, never fatal).
261 if self.project_path != self.global_path {
262 if let Err(e) = self.append_to_file(&self.global_path, &line, false) {
263 eprintln!("loop-runtime: global mirror write failed (non-fatal): {e}");
264 }
265 }
266
267 Ok(())
268 }
269
270 /// Open `path` in append+create mode and write `line`. If `fatal` is true,
271 /// returns `Err(LoopError::Journal)` on failure; otherwise returns `Ok`.
272 fn append_to_file(&self, path: &Path, line: &str, fatal: bool) -> Result<(), LoopError> {
273 // Ensure parent directory exists.
274 if let Some(parent) = path.parent() {
275 if let Err(e) = fs::create_dir_all(parent) {
276 let msg = format!("create_dir_all {}: {e}", parent.display());
277 if fatal {
278 return Err(LoopError::Journal(msg));
279 } else {
280 return Err(LoopError::Io(e));
281 }
282 }
283 }
284
285 match fs::OpenOptions::new().create(true).append(true).open(path) {
286 Ok(mut f) => {
287 if let Err(e) = f.write_all(line.as_bytes()) {
288 let msg = format!("write to {}: {e}", path.display());
289 if fatal {
290 return Err(LoopError::Journal(msg));
291 } else {
292 return Err(LoopError::Io(e));
293 }
294 }
295 Ok(())
296 }
297 Err(e) => {
298 let msg = format!("open {}: {e}", path.display());
299 if fatal {
300 Err(LoopError::Journal(msg))
301 } else {
302 Err(LoopError::Io(e))
303 }
304 }
305 }
306 }
307
308 /// Scan journals for the LAST termination event matching `session_key`.
309 ///
310 /// ## Search order (ab-7303e5d7: cross-cwd delivery via global mirror)
311 ///
312 /// 1. Scan the project journal first (authoritative for single-cwd target walks).
313 /// 2. When no match is found AND the global path differs from the project path,
314 /// scan the global journal (`~/.fno/events.jsonl`).
315 ///
316 /// Rationale: worker sessions dispatched by the megawalk walker run `/target`
317 /// in their OWN conductor worktrees, so their termination events land in the
318 /// WORKTREE's events.jsonl. loopcheck's `emit_to_both` also mirrors them to
319 /// `~/.fno/events.jsonl` (the global path). The walker's project journal
320 /// lives at the walker's cwd; only the global mirror is shared across cwds.
321 ///
322 /// Returns `None` if not found in either journal or on read errors (fail
323 /// tolerant: unreadable journal = no pre-existing termination).
324 ///
325 /// Uses `BufReader` + `.lines()` to stream files rather than loading them
326 /// entirely; the journal rotates at 8 MB but can grow to that before
327 /// rotation, so streaming avoids a large allocation on hot paths.
328 ///
329 /// Unknown reason strings are treated as no-match (warn to stderr).
330 /// Unreadable lines (I/O errors from `.lines()`) are skipped silently.
331 pub fn find_termination(&self, session_key: &str) -> Result<Option<Evidence>, LoopError> {
332 // Scan project journal first.
333 if let Some(ev) = Self::scan_journal_with_rotation(&self.project_path, session_key) {
334 return Ok(Some(ev));
335 }
336
337 // Fall back to global journal when it differs from the project journal
338 // and no match was found in the project journal. This handles the
339 // megawalk case where worker termination events land in a different
340 // worktree's journal but are mirrored to the global file.
341 if self.project_path != self.global_path {
342 if let Some(ev) = Self::scan_journal_with_rotation(&self.global_path, session_key) {
343 return Ok(Some(ev));
344 }
345 }
346
347 Ok(None)
348 }
349
350 /// Strict termination lookup for accounting decisions.
351 ///
352 /// Unlike `find_termination`, an unreadable existing journal or matching
353 /// corrupt termination is an error, not evidence of absence. Missing files
354 /// remain a clean no-match. The retained `.1` generation is searched after
355 /// the active file so completed sessions survive rotation.
356 pub fn find_termination_strict(
357 &self,
358 session_key: &str,
359 ) -> Result<Option<Evidence>, LoopError> {
360 let mut first_error: Option<LoopError> = None;
361 let mut paths = vec![self.project_path.clone()];
362 if self.global_path != self.project_path {
363 paths.push(self.global_path.clone());
364 }
365 for path in paths {
366 for candidate in [path.clone(), rotated_journal_path(&path)] {
367 match Self::scan_journal_strict(&candidate, session_key) {
368 Ok(Some(ev)) => return Ok(Some(ev)),
369 Ok(None) => {}
370 Err(err) => {
371 if first_error.is_none() {
372 first_error = Some(err);
373 }
374 }
375 }
376 }
377 }
378 match first_error {
379 Some(err) => Err(err),
380 None => Ok(None),
381 }
382 }
383
384 fn scan_journal_with_rotation(path: &Path, session_key: &str) -> Option<Evidence> {
385 Self::scan_journal(path, session_key)
386 .or_else(|| Self::scan_journal(&rotated_journal_path(path), session_key))
387 }
388
389 fn scan_journal_strict(path: &Path, session_key: &str) -> Result<Option<Evidence>, LoopError> {
390 let file = match fs::File::open(path) {
391 Ok(file) => file,
392 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
393 Err(err) => {
394 return Err(LoopError::Journal(format!(
395 "could not read journal {}: {err}",
396 path.display()
397 )))
398 }
399 };
400 let mut last_match = None;
401 for line_result in BufReader::new(file).lines() {
402 let raw = line_result.map_err(|err| {
403 LoopError::Journal(format!("read journal {}: {err}", path.display()))
404 })?;
405 let line = raw.trim();
406 if line.is_empty() {
407 continue;
408 }
409 let value: Value = match serde_json::from_str(line) {
410 Ok(value) => value,
411 Err(_) => continue,
412 };
413 if value["type"].as_str() != Some("termination")
414 || value["data"]["session_id"].as_str() != Some(session_key)
415 {
416 continue;
417 }
418 let reason_raw = value["data"]["reason"].as_str().ok_or_else(|| {
419 LoopError::Journal(format!(
420 "termination event for {session_key} in {} has no reason",
421 path.display()
422 ))
423 })?;
424 let reason = parse_termination_reason(reason_raw).ok_or_else(|| {
425 LoopError::Journal(format!(
426 "termination event for {session_key} in {} has unknown reason {reason_raw}",
427 path.display()
428 ))
429 })?;
430 let message = value["data"]["message"].as_str().unwrap_or("").to_string();
431 last_match = Some(Evidence { reason, message });
432 }
433 Ok(last_match)
434 }
435
436 /// Scan a single journal file for the LAST termination event matching
437 /// `session_key`. Returns `None` on missing file, read errors, or no
438 /// matching event (all fail-tolerant).
439 fn scan_journal(path: &Path, session_key: &str) -> Option<Evidence> {
440 if !path.exists() {
441 return None;
442 }
443
444 let file = match fs::File::open(path) {
445 Ok(f) => f,
446 Err(e) => {
447 eprintln!(
448 "loop-runtime: could not read journal {}: {e}",
449 path.display()
450 );
451 return None;
452 }
453 };
454
455 let mut last_match: Option<Evidence> = None;
456
457 for line_result in BufReader::new(file).lines() {
458 // Unreadable line (I/O error mid-file) -> skip silently.
459 let raw = match line_result {
460 Ok(l) => l,
461 Err(_) => continue,
462 };
463 let line = raw.trim();
464 if line.is_empty() {
465 continue;
466 }
467 // Skip unparseable lines silently.
468 let v: Value = match serde_json::from_str(line) {
469 Ok(v) => v,
470 Err(_) => continue,
471 };
472
473 // Match type == "termination" && data.session_id == session_key.
474 if v["type"].as_str() != Some("termination") {
475 continue;
476 }
477 if v["data"]["session_id"].as_str() != Some(session_key) {
478 continue;
479 }
480
481 let reason_str = match v["data"]["reason"].as_str() {
482 Some(s) => s,
483 None => {
484 // F1: missing reason field is authoritative-record corruption; warn loudly.
485 eprintln!(
486 "loop-runtime: termination event for {session_key} missing 'reason' field, skipping"
487 );
488 continue;
489 }
490 };
491 let reason = match parse_termination_reason(reason_str) {
492 Some(r) => r,
493 None => {
494 // Unknown reason: warn and skip (fail tolerant).
495 eprintln!(
496 "loop-runtime: unknown TerminationReason '{reason_str}' in journal, skipping"
497 );
498 continue;
499 }
500 };
501 let message = v["data"]["message"].as_str().unwrap_or("").to_string();
502
503 last_match = Some(Evidence { reason, message });
504 }
505
506 last_match
507 }
508}
509
510fn rotated_journal_path(path: &Path) -> PathBuf {
511 let mut name = path.as_os_str().to_os_string();
512 name.push(".1");
513 PathBuf::from(name)
514}
515
516/// Parse a reason string into TerminationReason. Returns None for unknown values.
517///
518/// F7: uses serde round-trip instead of a hand-written match so that new
519/// variants added in group 2 are picked up automatically without a silent desync.
520fn parse_termination_reason(s: &str) -> Option<TerminationReason> {
521 serde_json::from_value(Value::String(s.to_string())).ok()
522}
523
524// ── outcome types ─────────────────────────────────────────────────────────────
525
526/// The result of closing a single unit (after its session produced a
527/// TerminationReason event).
528pub struct UnitResult {
529 /// The unit's identifier.
530 pub unit_id: String,
531 /// Termination evidence that drove the close decision.
532 pub evidence: Evidence,
533 /// What the queue did when asked to close the unit.
534 pub close: CloseOutcome,
535}
536
537/// The result of a complete walk.
538pub struct LoopOutcome {
539 /// Why the walk stopped (walk-level reason, not per-unit reason).
540 pub reason: TerminationReason,
541 /// Total iterations consumed across all units.
542 pub iterations_used: u64,
543 /// Per-unit results for every unit that reached the close step.
544 pub units: Vec<UnitResult>,
545}
546
547// ── bg-guard refusal classifier ─────────────────────────────────────────────
548
549/// Claude's bg-guard refusal marker. When a `/target --resume` lands on a
550/// session claude still has registered as a live background agent, claude exits
551/// via `exit_with_message` (exit 1) after printing a message containing this
552/// phrase (e.g. "<sid> is currently running as a background agent (bg)"). The
553/// walk resumes by shelling `claude --resume`, so re-dispatching just re-hits
554/// the guard forever -- the x-4504 respawn loop. Match is case-insensitive.
555const BG_GUARD_MARKER: &str = "running as a background agent";
556
557/// True iff a non-termination session exit is claude's bg-guard refusal and
558/// therefore must be treated as terminal (parked) rather than re-dispatched.
559/// Gated on a non-zero exit AND the marker in the captured output: a bare
560/// non-zero exit WITHOUT the marker stays an ordinary crash-respawn, and a clean
561/// (exit 0) run that merely mentions the phrase is never suppressed.
562fn is_bg_guard_refusal(exit_code: i32, output_tail: Option<&str>) -> bool {
563 exit_code != 0
564 && output_tail
565 .map(|t| t.to_ascii_lowercase().contains(BG_GUARD_MARKER))
566 .unwrap_or(false)
567}
568
569// ── main loop ─────────────────────────────────────────────────────────────────
570
571/// Run a walk over the queue until the queue is empty, the budget is exhausted,
572/// or the cancel sentinel fires.
573///
574/// ## Algorithm
575///
576/// ```text
577/// loop:
578/// check cancel -> Interrupted
579/// unit = queue.next() -> None: NoWork
580/// resume guard: if journal has a termination event for unit.session_key,
581/// close the unit without dispatching, journal node_closed, and continue.
582/// inner dispatch loop:
583/// check budget -> Budget
584/// check cancel -> Interrupted
585/// iterations_used += 1
586/// per_unit_cap check: if unit_dispatches >= cap, synthesize NoProgress park
587/// emit loop_unit_dispatched
588/// run session, wait
589/// if journal has termination event: close unit, journal node_closed, break
590/// if exit is claude's bg-guard refusal: close unit (NoProgress) + node_closed,
591/// break -- do NOT re-dispatch (x-4504, AC1-ERR)
592/// else: emit node_failed, continue inner loop (re-dispatch)
593/// ```
594///
595/// ## Journal invariant
596///
597/// Every `journal.append` call for project events is fatal on failure.
598/// An unobservable walk must not continue spending.
599///
600/// ## per_unit_max_dispatches
601///
602/// When `Some(N)`, a unit that accumulates N dispatches without a termination
603/// event is synthesized a `NoProgress` Evidence and parked via `queue.close()`.
604/// The walk continues to the next unit. `None` means no per-unit cap (the
605/// walk-level budget is the only ceiling, as in the original degenerate policy).
606pub fn run_loop(
607 queue: &mut dyn Queue,
608 dispatcher: &dyn Dispatcher,
609 budget: &LoopBudget,
610 journal: &Journal,
611 cancel: &dyn Fn() -> bool,
612 per_unit_max_dispatches: Option<u64>,
613) -> Result<LoopOutcome, LoopError> {
614 let mut iterations_used: u64 = 0;
615 let mut units: Vec<UnitResult> = Vec::new();
616
617 loop {
618 // ── cancel check (outer loop top) ─────────────────────────────────
619 if cancel() {
620 journal.append(
621 "loop_terminated",
622 json!({
623 "reason": "Interrupted",
624 "iterations_used": iterations_used,
625 "units_closed": units.len(),
626 }),
627 )?;
628 return Ok(LoopOutcome {
629 reason: TerminationReason::Interrupted,
630 iterations_used,
631 units,
632 });
633 }
634
635 // ── dequeue next unit ─────────────────────────────────────────────
636 // queue.next() may return:
637 // Ok(None) -> backlog empty -> NoWork
638 // Ok(Some) -> dispatch this unit
639 // Err(other) -> hard failure (a real LoopError::Queue falls here)
640 let unit = match queue.next() {
641 Ok(None) => {
642 journal.append(
643 "loop_terminated",
644 json!({
645 "reason": "NoWork",
646 "iterations_used": iterations_used,
647 "units_closed": units.len(),
648 }),
649 )?;
650 return Ok(LoopOutcome {
651 reason: TerminationReason::NoWork,
652 iterations_used,
653 units,
654 });
655 }
656 Ok(Some(u)) => u,
657 Err(e) => return Err(e),
658 };
659
660 // ── resume guard (AC1-FR): check for pre-existing termination ─────
661 // If a prior session for this unit already terminated (e.g. the walk
662 // restarted mid-flight), close the unit without dispatching so work
663 // is not duplicated.
664 if let Some(evidence) = journal.find_termination(&unit.session_key)? {
665 let close = queue.close(&unit, &evidence)?;
666 // AC2-UI: journal node_closed for every close path.
667 journal_node_closed(journal, &unit, &evidence, &close, iterations_used)?;
668 units.push(UnitResult {
669 unit_id: unit.id.clone(),
670 evidence,
671 close,
672 });
673 // Do NOT increment iterations_used: no dispatch happened.
674 continue;
675 }
676
677 // ── inner dispatch loop ───────────────────────────────────────────
678 // Re-dispatch until a TerminationReason event appears, the per-unit
679 // cap is hit, or the walk-level budget is exhausted.
680 let mut unit_dispatches: u64 = 0;
681 loop {
682 // Budget check (inner loop top, before dispatch).
683 if iterations_used >= budget.max_iterations {
684 journal.append(
685 "loop_terminated",
686 json!({
687 "reason": "Budget",
688 "iterations_used": iterations_used,
689 "units_closed": units.len(),
690 "axis": "iterations",
691 }),
692 )?;
693 return Ok(LoopOutcome {
694 reason: TerminationReason::Budget,
695 iterations_used,
696 units,
697 });
698 }
699
700 // Cancel check (inner loop, before dispatch).
701 if cancel() {
702 journal.append(
703 "loop_terminated",
704 json!({
705 "reason": "Interrupted",
706 "iterations_used": iterations_used,
707 "units_closed": units.len(),
708 }),
709 )?;
710 return Ok(LoopOutcome {
711 reason: TerminationReason::Interrupted,
712 iterations_used,
713 units,
714 });
715 }
716
717 // Per-unit dispatch cap: if this unit has been dispatched N times
718 // without a termination event, synthesize a NoProgress park and
719 // continue to the next unit.
720 if let Some(cap) = per_unit_max_dispatches {
721 if unit_dispatches >= cap {
722 let evidence = Evidence {
723 reason: TerminationReason::NoProgress,
724 message: format!(
725 "no termination event after {cap} dispatch(es); unit parked"
726 ),
727 };
728 let close = queue.close(&unit, &evidence)?;
729 journal_node_closed(journal, &unit, &evidence, &close, iterations_used)?;
730 units.push(UnitResult {
731 unit_id: unit.id.clone(),
732 evidence,
733 close,
734 });
735 break; // Break inner loop -> continue outer loop (next unit).
736 }
737 }
738
739 iterations_used += 1;
740 unit_dispatches += 1;
741
742 // Emit loop_unit_dispatched before running the session.
743 journal.append(
744 "loop_unit_dispatched",
745 json!({
746 "unit_id": unit.id,
747 "session_id": unit.session_key,
748 "iteration": iterations_used,
749 "title": unit.title,
750 }),
751 )?;
752
753 // Launch and wait for the session.
754 let mut session = dispatcher
755 .run(
756 &unit,
757 &DispatchCtx {
758 iteration: iterations_used,
759 },
760 )
761 .map_err(|e| LoopError::Dispatch(e.to_string()))?;
762 let exit_code = session.wait()?;
763
764 // Check whether the session produced a termination event.
765 if let Some(evidence) = journal.find_termination(&unit.session_key)? {
766 let close = queue.close(&unit, &evidence)?;
767 // AC2-UI: journal node_closed for every close path.
768 journal_node_closed(journal, &unit, &evidence, &close, iterations_used)?;
769 units.push(UnitResult {
770 unit_id: unit.id.clone(),
771 evidence,
772 close,
773 });
774 break; // Break inner loop -> continue outer loop (next unit).
775 }
776
777 // x-4504 / AC1-ERR: claude's bg-guard refusal is terminal, not a
778 // crash to re-dispatch. When `/target --resume` lands on a session
779 // claude still holds as a live background agent, claude refuses with
780 // `exit_with_message` ("running as a background agent (bg)"). The
781 // next dispatch re-runs `claude --resume` and re-hits the guard, so
782 // re-dispatching is an infinite respawn loop. Park the unit instead
783 // (a later native attach / detach frees the slot for a fresh walk).
784 // Mirror the per-unit-cap park: close with NoProgress evidence + a
785 // node_closed event whose detail identifies the bg-guard cause (a
786 // per-unit park, not a queue-level pause).
787 if is_bg_guard_refusal(exit_code, session.output_tail().as_deref()) {
788 let evidence = Evidence {
789 reason: TerminationReason::NoProgress,
790 message: "claude bg-guard refusal (session running as a background agent); re-dispatch halted".to_string(),
791 };
792 let close = queue.close(&unit, &evidence)?;
793 journal_node_closed(journal, &unit, &evidence, &close, iterations_used)?;
794 units.push(UnitResult {
795 unit_id: unit.id.clone(),
796 evidence,
797 close,
798 });
799 break; // Break inner loop -> continue outer loop (next unit).
800 }
801
802 // No termination event: emit node_failed (watchdog synthesis) and
803 // re-dispatch in the next inner iteration.
804 journal.append(
805 "node_failed",
806 json!({
807 "unit_id": unit.id,
808 "session_id": unit.session_key,
809 "iteration": iterations_used,
810 "exit_code": exit_code,
811 }),
812 )?;
813 }
814 }
815}
816
817/// Journal a `node_closed` loop event after every queue.close() call.
818///
819/// Fields: unit_id, session_id, reason (evidence reason string), close
820/// ("closed"|"parked"|"refused"), detail (Parked/Refused string, "" for Closed),
821/// iterations_used (walk iteration count at close time).
822///
823/// The TUI and progress-line consumers read this event to track per-unit
824/// close outcomes. It is emitted on EVERY close path: resume guard, normal
825/// termination, per-unit cap park, and (in megawalk) consecutive-failure park.
826fn journal_node_closed(
827 journal: &Journal,
828 unit: &Unit,
829 evidence: &Evidence,
830 close: &CloseOutcome,
831 iterations_used: u64,
832) -> Result<(), LoopError> {
833 let (close_str, detail) = match close {
834 CloseOutcome::Closed => ("closed", String::new()),
835 CloseOutcome::Parked(s) => ("parked", s.clone()),
836 CloseOutcome::Refused(s) => ("refused", s.clone()),
837 CloseOutcome::AwaitingMerge => (
838 "awaiting-merge",
839 "PR not merged; node stays in_review, reconcile/advance close it at merge".to_string(),
840 ),
841 };
842 let reason_str = format!("{:?}", evidence.reason);
843 journal.append(
844 "node_closed",
845 json!({
846 "unit_id": unit.id,
847 "session_id": unit.session_key,
848 "reason": reason_str,
849 "close": close_str,
850 "detail": detail,
851 "iterations_used": iterations_used,
852 }),
853 )
854}
855
856#[cfg(test)]
857mod bg_guard_tests {
858 use super::is_bg_guard_refusal;
859
860 #[test]
861 fn refusal_marker_with_nonzero_exit_is_terminal() {
862 let out = "abc123 is currently running as a background agent (bg). \
863 Use 'claude agents' to view it, or add --fork-session.";
864 assert!(is_bg_guard_refusal(1, Some(out)));
865 // Case-insensitive.
866 assert!(is_bg_guard_refusal(
867 1,
868 Some("RUNNING AS A BACKGROUND AGENT")
869 ));
870 }
871
872 #[test]
873 fn bare_nonzero_exit_without_marker_is_not_terminal() {
874 // An ordinary crash must still re-dispatch (not be suppressed).
875 assert!(!is_bg_guard_refusal(1, Some("panic: index out of bounds")));
876 assert!(!is_bg_guard_refusal(1, None));
877 assert!(!is_bg_guard_refusal(137, Some("killed"))); // SIGKILL
878 }
879
880 #[test]
881 fn clean_exit_is_never_a_refusal_even_with_marker() {
882 // A successful run that merely mentions the phrase must not be parked.
883 assert!(!is_bg_guard_refusal(
884 0,
885 Some("running as a background agent")
886 ));
887 }
888}