1use crate::events::EventEmitter;
15use crate::paths::{self, AgentsHome};
16use crate::protocol::{
17 read_request, write_request, write_response, ErrorCode, Namespace, Request, Response,
18};
19use crate::state::{self, RegistryEntry};
20use crate::AgentStatus;
21use serde_json::{json, Map, Value};
22use std::os::unix::process::CommandExt; use std::path::PathBuf;
24use std::sync::Arc;
25use std::time::{Duration, Instant};
26use tokio::net::{UnixListener, UnixStream};
27
28#[derive(Debug, Clone, Copy, PartialEq, Eq)]
31pub enum DaemonState {
32 ColdStart,
33 Recovering,
34 Serving,
35 IdlePendingExit,
36 ShuttingDown,
37 Exited,
38}
39
40impl DaemonState {
41 pub fn as_str(&self) -> &'static str {
42 match self {
43 DaemonState::ColdStart => "cold_start",
44 DaemonState::Recovering => "recovering",
45 DaemonState::Serving => "serving",
46 DaemonState::IdlePendingExit => "idle_pending_exit",
47 DaemonState::ShuttingDown => "shutting_down",
48 DaemonState::Exited => "exited",
49 }
50 }
51}
52
53#[derive(Debug, Clone)]
55pub struct DaemonOptions {
56 pub idle_exit: Duration,
57 pub worker_bin: PathBuf,
61 pub reconcile_on_start: bool,
66 pub dead_row_grace: Duration,
70 pub notify_on_blocked: bool,
73 pub notify_on_done: bool,
76}
77
78impl Default for DaemonOptions {
79 fn default() -> Self {
80 DaemonOptions {
81 idle_exit: Duration::from_secs(1800),
82 worker_bin: resolve_worker_bin(),
83 reconcile_on_start: true,
84 dead_row_grace: Duration::from_secs(crate::agents_config::DEFAULT_DEAD_ROW_GRACE_SECS),
85 notify_on_blocked: true,
86 notify_on_done: false,
87 }
88 }
89}
90
91fn resolve_worker_bin() -> PathBuf {
92 if let Some(v) = std::env::var_os("FNO_AGENTS_WORKER_BIN") {
93 return PathBuf::from(v);
94 }
95 std::env::current_exe()
97 .ok()
98 .and_then(|p| p.parent().map(|d| d.join("fno-agents-worker")))
99 .unwrap_or_else(|| PathBuf::from("fno-agents-worker"))
100}
101
102#[derive(Debug, thiserror::Error)]
103pub enum DaemonError {
104 #[error("io: {0}")]
105 Io(#[from] std::io::Error),
106 #[error("another daemon is already serving on {0}")]
107 AlreadyRunning(PathBuf),
108 #[error("socket permission invariant failed: {0}")]
109 Permission(String),
110 #[error("filesystem does not support advisory locking at {0}: {1}")]
111 FlockUnsupported(PathBuf, String),
112 #[error("state: {0}")]
113 State(#[from] state::StateError),
114}
115
116#[derive(Debug, Clone, PartialEq, Eq)]
121pub enum InconsistencyReason {
122 MissingStateJson,
125 UnreadableStateJson,
127}
128
129impl InconsistencyReason {
130 pub fn as_str(&self) -> &'static str {
131 match self {
132 InconsistencyReason::MissingStateJson => "missing_state_json",
133 InconsistencyReason::UnreadableStateJson => "unreadable_state_json",
134 }
135 }
136}
137
138#[derive(Debug, Default, PartialEq)]
140pub struct RecoveryReport {
141 pub inconsistent: Vec<(String, InconsistencyReason)>,
145 pub archived_orphans: Vec<String>,
146 pub reaped_pids: Vec<u32>,
147 pub recovered_drives: Vec<String>,
148}
149
150pub fn recover(home: &AgentsHome, emitter: &EventEmitter) -> RecoveryReport {
160 let mut report = RecoveryReport::default();
161 let registry = state::load_registry(&home.registry_json()).unwrap_or_default();
162
163 let registered: std::collections::BTreeSet<String> = registry
164 .entries
165 .iter()
166 .map(|e| e.short_id.clone())
167 .collect();
168
169 for entry in ®istry.entries {
171 let is_claude_shellout = entry.harness_name() == "claude"
182 && entry.host_mode_or_default() != crate::state::HOST_MODE_INTERACTIVE;
183 if entry.short_id.is_empty() || is_claude_shellout {
184 continue;
185 }
186 let state_path = home.state_json(&entry.short_id);
187 match state::load_state(&state_path) {
188 Ok(Some(mut st)) => {
189 let taken = st.pty.as_mut().and_then(|p| p.take_active_drive());
191 if let Some(drive) = taken {
192 let mut fields = Map::new();
193 if let Some(sid) = &drive.session_id {
194 fields.insert("session_id".into(), Value::String(sid.clone()));
195 }
196 fields.insert("reason".into(), Value::String("daemon_restart".into()));
197 let _ = emitter.emit_fields("drive_crashed", fields);
200 let _ = state::write_state_atomic(&state_path, &st);
201 report.recovered_drives.push(entry.short_id.clone());
202 }
203 }
204 Ok(None) => {
205 let reason = InconsistencyReason::MissingStateJson;
208 let _ = emitter.emit_fields(
209 "agent_inconsistent",
210 json_obj(&[
211 ("short_id", Value::String(entry.short_id.clone())),
212 ("reason", Value::String(reason.as_str().into())),
213 ]),
214 );
215 report.inconsistent.push((entry.short_id.clone(), reason));
216 }
217 Err(_) => {
218 let reason = InconsistencyReason::UnreadableStateJson;
222 let _ = emitter.emit_fields(
223 "agent_inconsistent",
224 json_obj(&[
225 ("short_id", Value::String(entry.short_id.clone())),
226 ("reason", Value::String(reason.as_str().into())),
227 ]),
228 );
229 report.inconsistent.push((entry.short_id.clone(), reason));
230 }
231 }
232 }
233
234 if let Ok(read) = std::fs::read_dir(home.root()) {
236 for entry in read.flatten() {
237 if !entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
238 continue;
239 }
240 let name = match entry.file_name().into_string() {
241 Ok(n) if !n.starts_with('.') => n,
242 _ => continue,
243 };
244 if registered.contains(&name) {
245 continue;
246 }
247 if home.state_json(&name).exists() {
249 let ts = now_compact();
250 let dest = home.orphan_archive_dest(&name, &ts);
251 let _ = std::fs::create_dir_all(home.orphaned_dir());
252 if std::fs::rename(home.agent_dir(&name), &dest).is_ok() {
253 let _ = emitter.emit_fields(
254 "agent_orphan_state_archived",
255 json_obj(&[
256 ("short_id", Value::String(name.clone())),
257 (
258 "archived_to",
259 Value::String(dest.to_string_lossy().into_owned()),
260 ),
261 ]),
262 );
263 report.archived_orphans.push(name);
264 }
265 }
266 }
267 }
268
269 let live_workers = home.scan_worker_sockets();
276 let mut to_reap: Vec<(String, u32)> = Vec::new();
277 for entry in ®istry.entries {
278 if live_workers.contains(&entry.short_id) {
279 continue; }
281 if let Some(pid) = entry.pid {
282 if !pid_is_ours(pid, entry.pid_start_time) {
283 to_reap.push((entry.short_id.clone(), pid));
284 }
285 }
286 }
287 if !to_reap.is_empty() {
288 let reaped: std::collections::BTreeSet<String> =
289 to_reap.iter().map(|(s, _)| s.clone()).collect();
290 for e in ®istry.entries {
293 if reaped.contains(&e.short_id) {
294 emit_inside_leg_completion(emitter, e);
295 }
296 }
297 if let Err(e) = state::update_registry(&home.registry_json(), |r| {
300 for e in r.entries.iter_mut() {
301 if reaped.contains(&e.short_id) {
302 e.status = AgentStatus::Exited;
303 e.inside_leg = None;
307 e.screen_state = None;
308 }
309 }
310 }) {
311 let _ = emitter.emit(
312 "daemon_recovery_error",
313 &json!({"op": "reap_orphans", "error": e.to_string()}),
314 );
315 }
316 for (short_id, pid) in to_reap {
317 let _ = emitter.emit_fields(
318 "agent_orphan_reaped",
319 json_obj(&[
320 ("short_id", Value::String(short_id)),
321 ("pid", Value::Number(pid.into())),
322 ]),
323 );
324 report.reaped_pids.push(pid);
325 }
326 }
327
328 report
329}
330
331#[cfg(target_os = "linux")]
337pub fn process_start_time(pid: u32) -> Option<u64> {
338 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
343 let after = stat.rsplit_once(')')?.1;
344 after.split_whitespace().nth(19)?.parse::<u64>().ok()
345}
346
347#[cfg(target_os = "macos")]
351pub fn process_start_time(pid: u32) -> Option<u64> {
352 use std::mem;
353 let mut info: libc::proc_bsdinfo = unsafe { mem::zeroed() };
354 let size = mem::size_of::<libc::proc_bsdinfo>() as libc::c_int;
355 let written = unsafe {
359 libc::proc_pidinfo(
360 pid as libc::c_int,
361 libc::PROC_PIDTBSDINFO,
362 0,
363 &mut info as *mut _ as *mut libc::c_void,
364 size,
365 )
366 };
367 if written != size {
368 return None;
369 }
370 Some(info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec)
371}
372
373#[cfg(not(any(target_os = "linux", target_os = "macos")))]
374pub fn process_start_time(_pid: u32) -> Option<u64> {
375 None
376}
377
378#[derive(Debug, Default, PartialEq)]
384pub struct GcSummary {
385 pub reaped: Vec<String>,
386 pub kept_dirty: Vec<(String, String)>,
387}
388
389fn worktree_clean_probe(cwd: &str) -> Option<bool> {
394 let out = std::process::Command::new("git")
395 .current_dir(cwd)
396 .args(["status", "--porcelain"])
397 .output()
398 .ok()?;
399 if !out.status.success() {
400 return None;
401 }
402 Some(out.stdout.iter().all(u8::is_ascii_whitespace))
403}
404
405fn now_epoch_secs() -> i64 {
408 std::time::SystemTime::now()
409 .duration_since(std::time::UNIX_EPOCH)
410 .map(|d| d.as_secs() as i64)
411 .unwrap_or(0)
412}
413
414fn dispatch_node_id(name: &str) -> Option<String> {
420 let mut parts = name.split('-');
421 match parts.next()? {
422 "target" | "reconcile" => {}
423 _ => return None,
424 }
425 let prefix = parts.next()?;
426 let hex = parts.next()?;
427 if prefix.is_empty()
428 || !prefix
429 .chars()
430 .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit())
431 || hex.is_empty()
432 || !hex.chars().all(|c| c.is_ascii_hexdigit())
433 {
434 return None;
435 }
436 Some(format!("{prefix}-{hex}"))
437}
438
439fn global_events_path(home: &AgentsHome) -> PathBuf {
440 home.root()
441 .parent()
442 .unwrap_or_else(|| home.root())
443 .join("events.jsonl")
444}
445
446#[derive(Debug, PartialEq, Eq)]
447enum DispatchTermination {
448 Found(String),
449 Absent(Option<String>),
450 Unknown(String),
451}
452
453fn dispatch_target_session_id(
454 entry: &RegistryEntry,
455 node_id: &str,
456) -> Result<Option<String>, String> {
457 let manifest = PathBuf::from(&entry.cwd).join(".fno/target-state.md");
458 let content = match std::fs::read_to_string(&manifest) {
459 Ok(content) => content,
460 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
461 Err(err) => return Err(format!("read {}: {err}", manifest.display())),
462 };
463 let parsed = crate::loop_target::parse_target_manifest(&content)
464 .ok_or_else(|| format!("parse target session from {}", manifest.display()))?;
465 if parsed.input != node_id {
466 return Err(format!(
467 "target manifest {} belongs to input {}, not registry node {node_id}",
468 manifest.display(),
469 parsed.input
470 ));
471 }
472 if let (Some(row_session), Some(manifest_session)) = (
473 entry.harness_session_id.as_deref(),
474 parsed.harness_session_id.as_deref(),
475 ) {
476 if row_session != manifest_session {
477 return Err(format!(
478 "target manifest {} harness session {manifest_session} does not match registry {row_session}",
479 manifest.display()
480 ));
481 }
482 }
483 Ok(Some(parsed.session_id))
484}
485
486fn dispatch_termination(
487 home: &AgentsHome,
488 entry: &RegistryEntry,
489 node_id: &str,
490) -> DispatchTermination {
491 let session_id = match dispatch_target_session_id(entry, node_id) {
492 Ok(session_id) => session_id,
493 Err(err) => return DispatchTermination::Unknown(err),
494 };
495 let Some(session_id) = session_id else {
496 return DispatchTermination::Absent(None);
499 };
500 let journal = crate::loop_runtime::Journal::new(
501 crate::loop_runtime::ProjectJournalPath(
502 PathBuf::from(&entry.cwd).join(".fno/events.jsonl"),
503 ),
504 crate::loop_runtime::GlobalJournalPath(global_events_path(home)),
505 );
506 match journal.find_termination_strict(&session_id) {
507 Ok(Some(_)) => DispatchTermination::Found(session_id),
508 Ok(None) => DispatchTermination::Absent(Some(session_id)),
509 Err(err) => DispatchTermination::Unknown(err.to_string()),
510 }
511}
512
513fn record_dead_dispatch(
514 home: &AgentsHome,
515 entry: &RegistryEntry,
516 node_id: &str,
517 target_session_id: Option<&str>,
518) -> Result<(), String> {
519 EventEmitter::new(global_events_path(home), "daemon")
523 .emit(
524 "node_failed",
525 &json!({
526 "unit_id": node_id,
527 "session_id": target_session_id.unwrap_or(&entry.short_id),
528 "iteration": 0,
529 "exit_code": 1,
530 "short_id": entry.short_id,
531 "reason": "agent-row-reaped-no-termination",
532 }),
533 )
534 .map_err(|err| err.to_string())
535}
536
537fn restore_unaccounted_row(home: &AgentsHome, entry: &RegistryEntry) -> Result<(), String> {
538 let mut restored = false;
539 state::update_registry(&home.registry_json(), |registry| {
540 if !registry.entries.iter().any(|row| row.name == entry.name) {
541 registry.entries.push(entry.clone());
542 restored = true;
543 }
544 })
545 .map_err(|err| err.to_string())?;
546 if restored {
547 Ok(())
548 } else {
549 Err(format!(
550 "could not restore {}: a replacement row now owns that name",
551 entry.name
552 ))
553 }
554}
555
556pub fn gc_sweep(home: &AgentsHome, emitter: &EventEmitter, grace: Duration) -> GcSummary {
569 let mut summary = GcSummary::default();
570 let registry = state::load_registry(&home.registry_json()).unwrap_or_default();
571 if registry.entries.is_empty() {
572 return summary; }
574 let live_workers = home.scan_worker_sockets();
575 let now = now_epoch_secs();
576 let grace_secs = grace.as_secs() as i64;
577
578 let mut to_reap: std::collections::BTreeMap<String, String> = std::collections::BTreeMap::new();
586 let mut to_stamp: std::collections::BTreeMap<String, String> =
587 std::collections::BTreeMap::new();
588 let mut to_clear: std::collections::BTreeMap<String, String> =
589 std::collections::BTreeMap::new();
590
591 for e in ®istry.entries {
592 let is_live = live_workers.contains(&e.short_id)
593 || e.pid
594 .map(|p| pid_is_ours(p, e.pid_start_time))
595 .unwrap_or(false);
596 let pid_confirmed_dead = e
597 .pid
598 .map(|p| !pid_is_ours(p, e.pid_start_time))
599 .unwrap_or(false);
600 let is_ask = e.is_one_shot_ask();
601 let exited_at = e
602 .exited_at
603 .as_deref()
604 .and_then(state::rfc3339_like_to_secs)
605 .map(|s| s as i64);
606
607 let terminal_or_dead = matches!(e.status, AgentStatus::Exited | AgentStatus::PermanentDead)
611 || pid_confirmed_dead;
612 let past_grace = matches!(exited_at, Some(t) if now.saturating_sub(t) > grace_secs);
613 let needs_probe = !is_live && terminal_or_dead && past_grace && !is_ask;
614 let worktree_clean = if needs_probe {
615 worktree_clean_probe(&e.cwd)
616 } else {
617 None
618 };
619
620 let row = crate::gc::GcRow {
621 status: e.status,
622 is_live,
623 pid_confirmed_dead,
624 is_ask,
625 exited_at,
626 worktree_clean,
627 };
628 let id = if e.short_id.is_empty() {
629 e.name.clone()
630 } else {
631 e.short_id.clone()
632 };
633 match crate::gc::gc_action(&row, now, grace_secs) {
634 crate::gc::GcAction::Reap => {
635 to_reap.insert(e.name.clone(), e.created_at.clone());
636 }
637 crate::gc::GcAction::StampExit => {
638 to_stamp.insert(e.name.clone(), e.created_at.clone());
639 }
640 crate::gc::GcAction::Keep => {
641 if is_live && e.exited_at.is_some() {
642 to_clear.insert(e.name.clone(), e.created_at.clone());
645 } else if needs_probe && matches!(worktree_clean, Some(false) | None) {
646 summary.kept_dirty.push((id, e.cwd.clone()));
648 }
649 }
650 }
651 }
652
653 if to_reap.is_empty() && to_stamp.is_empty() && to_clear.is_empty() {
654 return summary;
655 }
656
657 let now_stamp = now_rfc3339_like();
658 let mut reaped_names: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
661 let write = state::update_registry(&home.registry_json(), |r| {
662 for e in r.entries.iter_mut() {
668 if to_stamp.get(&e.name) == Some(&e.created_at) {
669 e.exited_at = Some(now_stamp.clone());
670 }
671 if to_clear.get(&e.name) == Some(&e.created_at) {
672 e.exited_at = None;
673 }
674 }
675 r.entries.retain(|e| {
676 if to_reap.get(&e.name) == Some(&e.created_at) {
677 reaped_names.insert(e.name.clone());
678 false
679 } else {
680 true
681 }
682 });
683 });
684 match write {
685 Ok(()) => {
686 for e in ®istry.entries {
690 if reaped_names.contains(&e.name) {
691 let node_id = dispatch_node_id(&e.name);
692 let mut target_session_id = None;
693 let mut termination_event = false;
694 let mut accounted = true;
695 if let Some(node_id) = node_id.as_deref() {
696 match dispatch_termination(home, e, node_id) {
697 DispatchTermination::Found(session_id) => {
698 target_session_id = Some(session_id);
699 termination_event = true;
700 }
701 DispatchTermination::Absent(session_id) => {
702 target_session_id = session_id;
703 if let Err(err) = record_dead_dispatch(
704 home,
705 e,
706 node_id,
707 target_session_id.as_deref(),
708 ) {
709 accounted = false;
710 let restore = restore_unaccounted_row(home, e);
711 let _ = emitter.emit(
712 "daemon_recovery_error",
713 &json!({
714 "op": "record_dead_dispatch",
715 "short_id": e.short_id,
716 "error": err,
717 "restore_error": restore.err(),
718 }),
719 );
720 }
721 }
722 DispatchTermination::Unknown(err) => {
723 accounted = false;
724 let restore = restore_unaccounted_row(home, e);
725 let _ = emitter.emit(
726 "daemon_recovery_error",
727 &json!({
728 "op": "observe_dead_dispatch_termination",
729 "short_id": e.short_id,
730 "error": err,
731 "restore_error": restore.err(),
732 }),
733 );
734 }
735 }
736 }
737 if !accounted {
738 continue;
739 }
740 let _ = emitter.emit_fields(
741 "agent_row_reaped",
742 json_obj(&[
743 ("short_id", Value::String(e.short_id.clone())),
744 ("name", Value::String(e.name.clone())),
745 (
746 "node_id",
747 node_id.clone().map_or(Value::Null, Value::String),
748 ),
749 (
750 "session_id",
751 target_session_id.map_or(Value::Null, Value::String),
752 ),
753 ("termination_event", Value::Bool(termination_event)),
754 ]),
755 );
756 summary.reaped.push(if e.short_id.is_empty() {
757 e.name.clone()
758 } else {
759 e.short_id.clone()
760 });
761 }
762 }
763 }
764 Err(err) => {
765 let _ = emitter.emit(
766 "daemon_recovery_error",
767 &json!({"op": "gc_sweep", "error": err.to_string()}),
768 );
769 summary.reaped.clear();
771 }
772 }
773 summary
774}
775
776async fn terminal_stop_sweep(home: &AgentsHome, emitter: &EventEmitter) {
788 let home_read = home.clone();
793 let loaded = tokio::task::spawn_blocking(move || {
794 let markers = crate::terminal_stop::read_markers(&home_read);
795 if markers.is_empty() {
796 return (markers, None);
797 }
798 let roster = crate::claude_roster::ClaudeRoster::load_default();
799 (markers, Some(roster))
800 })
801 .await;
802 let (markers, roster) = match loaded {
803 Ok(v) => v,
804 Err(e) => {
805 eprintln!("daemon: terminal-stop sweep: read task failed: {e}");
806 return;
807 }
808 };
809 if markers.is_empty() {
810 return;
811 }
812 let roster = match roster {
819 Some(Ok(r)) => r,
820 Some(Err(e)) => {
821 eprintln!("daemon: terminal-stop sweep: roster load failed: {e} (retry next tick)");
822 return;
823 }
824 None => return,
825 };
826 for marker in markers {
827 let short = roster.find(&marker.uuid).map(|w| w.short_id().to_string());
828 match crate::terminal_stop::stop_decision(short) {
829 crate::terminal_stop::StopAction::Stop(short) => {
830 let stop = tokio::process::Command::new("claude")
837 .arg("stop")
838 .arg(&short)
839 .kill_on_drop(true)
840 .output();
841 let stopped = tokio::time::timeout(Duration::from_secs(15), stop).await;
842 match stopped {
843 Err(_) => eprintln!("daemon: claude stop {short} timed out (retry next tick)"),
844 Ok(Ok(o)) if o.status.success() => {
845 let _ = emitter.emit(
846 "bg_worker_terminal_stopped",
847 &json!({
848 "short_id": short,
849 "session_id": marker.uuid,
850 "reason": marker.reason,
851 }),
852 );
853 crate::terminal_stop::remove_marker(home, &marker.uuid);
854 }
855 Ok(Ok(o)) => eprintln!(
857 "daemon: claude stop {short} failed: {}",
858 String::from_utf8_lossy(&o.stderr).trim()
859 ),
860 Ok(Err(e)) => eprintln!("daemon: could not exec `claude stop`: {e}"),
861 }
862 }
863 crate::terminal_stop::StopAction::RemoveStale => {
866 crate::terminal_stop::remove_marker(home, &marker.uuid);
867 }
868 }
869 }
870}
871
872pub fn pid_is_ours(pid: u32, recorded: Option<u64>) -> bool {
879 if pid <= 1 || pid > i32::MAX as u32 {
893 return false;
894 }
895 if unsafe { libc::kill(pid as libc::pid_t, 0) } != 0 {
903 return false;
904 }
905 match (recorded, process_start_time(pid)) {
906 (Some(rec), Some(now)) => rec == now,
907 _ => true,
909 }
910}
911
912pub async fn bind_supervisor_socket(home: &AgentsHome) -> Result<UnixListener, DaemonError> {
925 home.ensure_root()?;
926 flock_self_test(home)?;
927
928 let sock = home.supervisor_sock();
929 if sock.exists() {
930 if UnixStream::connect(&sock).await.is_ok() {
932 return Err(DaemonError::AlreadyRunning(sock));
933 }
934 let _ = std::fs::remove_file(&sock);
936 }
937
938 let listener = match UnixListener::bind(&sock) {
939 Ok(l) => l,
940 Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {
941 return Err(DaemonError::AlreadyRunning(sock));
943 }
944 Err(e) => return Err(e.into()),
945 };
946
947 paths::set_file_mode_0600(&sock)?;
948
949 #[cfg(unix)]
951 {
952 if !paths::is_dir_mode_0700(home.root()) {
953 return Err(DaemonError::Permission(format!(
954 "{} is not mode 0700",
955 home.root().display()
956 )));
957 }
958 if !paths::is_file_mode_0600(&sock) {
959 return Err(DaemonError::Permission(format!(
960 "{} is not mode 0600",
961 sock.display()
962 )));
963 }
964 }
965
966 Ok(listener)
967}
968
969fn flock_self_test(home: &AgentsHome) -> Result<(), DaemonError> {
973 let probe = home.root().join(".flock-probe");
974 let file = std::fs::OpenOptions::new()
975 .create(true)
976 .read(true)
977 .write(true)
978 .truncate(false)
979 .open(&probe)
980 .map_err(|e| DaemonError::FlockUnsupported(probe.clone(), e.to_string()))?;
981 let lock_res = file.lock();
984 if lock_res.is_ok() {
985 let _ = file.unlock();
986 }
987 let _ = std::fs::remove_file(&probe);
988 lock_res.map_err(|e| DaemonError::FlockUnsupported(probe.clone(), e.to_string()))?;
989 Ok(())
990}
991
992pub async fn run(home: AgentsHome, opts: DaemonOptions) -> Result<(), DaemonError> {
1001 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
1002
1003 let listener = match bind_supervisor_socket(&home).await {
1005 Ok(l) => l,
1006 Err(DaemonError::AlreadyRunning(_)) => {
1007 return Ok(());
1009 }
1010 Err(e) => return Err(e),
1011 };
1012
1013 emit_state(&emitter, DaemonState::Recovering);
1015 let report = recover(&home, &emitter);
1016
1017 if opts.reconcile_on_start {
1028 let swept: Result<ReconcileSweepResult, String> =
1033 if std::env::var("FNO_AGENTS_FAIL_STARTUP_RECONCILE").is_ok() {
1034 Err(
1035 "forced startup-reconcile failure (FNO_AGENTS_FAIL_STARTUP_RECONCILE)"
1036 .to_string(),
1037 )
1038 } else {
1039 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
1040 run_reconcile_sweep(&home, &emitter)
1041 }))
1042 .unwrap_or_else(|_| {
1043 Err(
1044 "startup reconcile sweep panicked; serving last-recorded status"
1045 .to_string(),
1046 )
1047 })
1048 };
1049 match swept {
1050 Ok(result) => {
1051 let _ = emitter.emit(
1052 "startup_reconcile_done",
1053 &json!({
1054 "updated": result.outcome.updated.len(),
1055 "deferred": result.outcome.deferred,
1056 }),
1057 );
1058 }
1059 Err(msg) => {
1060 let _ = emitter.emit("startup_reconcile_failed", &json!({"error": msg}));
1061 }
1062 }
1063 }
1064
1065 let started_at = Instant::now();
1068 let exe_fingerprint = crate::drift::ExeFingerprint::current();
1073 if exe_fingerprint.is_none() {
1074 let _ = emitter.emit("daemon_exe_fingerprint_unavailable", &json!({}));
1077 }
1078 let pid_start_time = process_start_time(std::process::id());
1079 let _ = emitter.emit(
1080 "daemon_started",
1081 &json!({
1082 "pid": std::process::id(),
1083 "version": env!("CARGO_PKG_VERSION"),
1084 "recovered_drives": report.recovered_drives.len(),
1085 }),
1086 );
1087 emit_state(&emitter, DaemonState::Serving);
1088
1089 let ctx = Arc::new(Ctx {
1091 home,
1092 emitter,
1093 opts,
1094 started_at,
1095 exe_fingerprint,
1096 pid_start_time,
1097 pending_inside_leg: std::sync::Mutex::new(std::collections::HashMap::new()),
1098 });
1099
1100 let ab_live = Arc::new(std::sync::atomic::AtomicBool::new(false));
1108 let ab_shutdown = Arc::new(std::sync::atomic::AtomicBool::new(false));
1109 let ab_handle = {
1110 let fno_bin = std::env::var("FNO_BIN").unwrap_or_else(|_| "fno".to_string());
1111 let ab_emitter = EventEmitter::new(ctx.home.events_jsonl(), "active-backlog");
1112 let live = Arc::clone(&ab_live);
1113 let shutdown = Arc::clone(&ab_shutdown);
1114 tokio::spawn(crate::active_backlog::run_supervisor(
1115 fno_bin, ab_emitter, live, shutdown,
1116 ))
1117 };
1118
1119 let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
1121 let mut idle_check = tokio::time::interval(Duration::from_secs(5));
1122 idle_check.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
1123 let mut last_activity = Instant::now();
1124 let scrape_in_flight = Arc::new(std::sync::atomic::AtomicBool::new(false));
1127 let terminal_stop_in_flight = Arc::new(std::sync::atomic::AtomicBool::new(false));
1131
1132 loop {
1133 tokio::select! {
1134 accepted = listener.accept() => {
1135 if let Ok((stream, _)) = accepted {
1136 last_activity = Instant::now();
1137 let ctx = Arc::clone(&ctx);
1142 tokio::spawn(async move {
1143 serve_connection(ctx, stream).await;
1144 });
1145 }
1146 }
1147 _ = sigterm.recv() => {
1148 emit_state(&ctx.emitter, DaemonState::ShuttingDown);
1149 let _ = ctx.emitter.emit("daemon_shutting_down", &json!({"reason": "sigterm"}));
1150 break;
1151 }
1152 _ = idle_check.tick() => {
1153 reap_zombies();
1156 if !scrape_in_flight.swap(true, std::sync::atomic::Ordering::SeqCst) {
1160 let flag = Arc::clone(&scrape_in_flight);
1161 let home = ctx.home.clone();
1162 let emitter = EventEmitter::new(ctx.home.events_jsonl(), "daemon");
1163 let notify_on_blocked = ctx.opts.notify_on_blocked;
1164 tokio::task::spawn_blocking(move || {
1165 crate::scrape::scrape_sweep(&home, &emitter, notify_on_blocked);
1166 flag.store(false, std::sync::atomic::Ordering::SeqCst);
1167 });
1168 }
1169 let _ = gc_sweep(&ctx.home, &ctx.emitter, ctx.opts.dead_row_grace);
1175 if !terminal_stop_in_flight.swap(true, std::sync::atomic::Ordering::SeqCst) {
1182 let flag = Arc::clone(&terminal_stop_in_flight);
1183 let home = ctx.home.clone();
1184 let emitter = EventEmitter::new(ctx.home.events_jsonl(), "daemon");
1185 tokio::spawn(async move {
1186 terminal_stop_sweep(&home, &emitter).await;
1187 flag.store(false, std::sync::atomic::Ordering::SeqCst);
1188 });
1189 }
1190 let empty = state::load_registry(&ctx.home.registry_json())
1191 .map(|r| r.entries.is_empty())
1192 .unwrap_or(true);
1193 let ab_active = ab_live.load(std::sync::atomic::Ordering::SeqCst);
1197 if empty && !ab_active && last_activity.elapsed() >= ctx.opts.idle_exit {
1198 emit_state(&ctx.emitter, DaemonState::IdlePendingExit);
1199 let _ = ctx.emitter.emit("daemon_idle_pending_exit", &json!({}));
1200 emit_state(&ctx.emitter, DaemonState::ShuttingDown);
1201 let _ = ctx.emitter.emit(
1202 "daemon_shutting_down",
1203 &json!({"reason": "idle"}),
1204 );
1205 break;
1206 }
1207 }
1208 }
1209 }
1210
1211 ab_shutdown.store(true, std::sync::atomic::Ordering::SeqCst);
1217 ab_handle.abort();
1218
1219 let _ = std::fs::remove_file(ctx.home.supervisor_sock());
1220 emit_state(&ctx.emitter, DaemonState::Exited);
1221 let _ = ctx.emitter.emit("daemon_exited", &json!({"clean": true}));
1222 Ok(())
1223}
1224
1225struct Ctx {
1227 home: AgentsHome,
1228 emitter: EventEmitter,
1229 opts: DaemonOptions,
1230 started_at: Instant,
1231 exe_fingerprint: Option<crate::drift::ExeFingerprint>,
1235 pid_start_time: Option<u64>,
1239 pending_inside_leg: std::sync::Mutex<std::collections::HashMap<String, state::InsideLegReport>>,
1247}
1248
1249const PENDING_INSIDE_LEG_CAP: usize = 64;
1254
1255fn emit_state(emitter: &EventEmitter, state: DaemonState) {
1256 let _ = emitter.emit("daemon_state", &json!({"state": state.as_str()}));
1257}
1258
1259const CONN_READ_TIMEOUT: Duration = Duration::from_secs(30);
1262
1263async fn serve_connection(ctx: Arc<Ctx>, mut stream: UnixStream) {
1264 let req = match tokio::time::timeout(CONN_READ_TIMEOUT, read_request(&mut stream)).await {
1269 Err(_elapsed) => return, Ok(Ok(r)) => r,
1271 Ok(Err(crate::protocol::ProtocolError::UnexpectedEof)) => return, Ok(Err(e)) => {
1273 let resp = Response::err(0, ErrorCode::MalformedFrame, format!("{e}"));
1276 let _ = write_response(&mut stream, &resp).await;
1277 return;
1278 }
1279 };
1280 if req.method == "agent.logs" {
1284 crate::logs::handle_logs(&ctx.home, &req, stream).await;
1285 return;
1286 }
1287 let resp = dispatch(&ctx, &req).await;
1288 let _ = write_response(&mut stream, &resp).await;
1289}
1290
1291async fn run_blocking<F>(ctx: &Arc<Ctx>, req: &Request, f: F) -> Response
1294where
1295 F: FnOnce(&Ctx, &Request) -> Response + Send + 'static,
1296{
1297 let ctx = Arc::clone(ctx);
1298 let req = req.clone();
1299 let id = req.id;
1300 match tokio::task::spawn_blocking(move || f(&ctx, &req)).await {
1301 Ok(resp) => resp,
1302 Err(_) => Response::err(id, ErrorCode::Internal, "handler task panicked"),
1303 }
1304}
1305
1306async fn load_registry_offloaded(path: PathBuf) -> state::Registry {
1312 tokio::task::spawn_blocking(move || state::load_registry(&path))
1313 .await
1314 .ok()
1315 .and_then(|r| r.ok())
1316 .unwrap_or_default()
1317}
1318
1319async fn update_registry_offloaded<F, T>(path: PathBuf, f: F) -> Result<T, state::StateError>
1324where
1325 F: FnOnce(&mut state::Registry) -> T + Send + 'static,
1326 T: Send + 'static,
1327{
1328 match tokio::task::spawn_blocking(move || state::update_registry(&path, f)).await {
1329 Ok(result) => result,
1330 Err(e) => Err(state::StateError::Io(std::io::Error::other(format!(
1331 "update_registry task panicked: {e}"
1332 )))),
1333 }
1334}
1335
1336async fn dispatch(ctx: &Arc<Ctx>, req: &Request) -> Response {
1337 match Namespace::of(&req.method) {
1338 Namespace::Agent => dispatch_agent(ctx, req).await,
1339 Namespace::Channel => dispatch_channel(ctx, req).await,
1340 Namespace::Unknown => Response::err(
1341 req.id,
1342 ErrorCode::UnknownMethod,
1343 format!("unknown namespace for method `{}`", req.method),
1344 ),
1345 }
1346}
1347
1348async fn dispatch_agent(ctx: &Arc<Ctx>, req: &Request) -> Response {
1349 match Namespace::verb(&req.method) {
1352 Some("spawn") => handle_spawn(ctx, req).await,
1353 Some("ask") => handle_ask(ctx, req).await,
1354 Some("switchboard") | Some("switchboard_v2") => handle_switchboard(ctx, req).await,
1355 Some("stop") => handle_stop(ctx, req).await,
1356 Some("rm") => handle_rm(ctx, req).await,
1357 Some("list") => run_blocking(ctx, req, handle_list).await,
1358 Some("status") => handle_status(ctx, req).await,
1361 Some("reconcile") => run_blocking(ctx, req, handle_reconcile).await,
1362 Some("report") => run_blocking(ctx, req, handle_report).await,
1365 _ => Response::err(
1366 req.id,
1367 ErrorCode::UnknownMethod,
1368 format!("unknown agent verb in `{}`", req.method),
1369 ),
1370 }
1371}
1372
1373fn valid_agent_name(name: &str) -> bool {
1375 !name.is_empty()
1376 && name.len() <= 64
1377 && name
1378 .chars()
1379 .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')
1380}
1381
1382fn derive_short_id(name: &str, registry: &state::Registry) -> String {
1384 let base: String = name
1385 .chars()
1386 .filter(|c| c.is_ascii_alphanumeric())
1387 .take(8)
1388 .collect();
1389 let base = if base.is_empty() {
1390 "agent".into()
1391 } else {
1392 base
1393 };
1394 if registry.entries.iter().all(|e| e.short_id != base) {
1395 return base;
1396 }
1397 for n in 1..10_000 {
1398 let cand = format!("{base}{n}");
1399 if registry.entries.iter().all(|e| e.short_id != cand) {
1400 return cand;
1401 }
1402 }
1403 format!("{base}-{}", now_compact())
1404}
1405
1406pub fn entry_holds_session(e: &RegistryEntry, uuid: &str) -> bool {
1410 e.codex_session_id.as_deref() == Some(uuid)
1411 || e.gemini_session_id.as_deref() == Some(uuid)
1412 || e.session_id.as_deref() == Some(uuid)
1413 || e.claude_session_uuid.as_deref() == Some(uuid)
1417}
1418
1419fn is_non_terminal(s: AgentStatus) -> bool {
1422 !matches!(s, AgentStatus::Exited | AgentStatus::PermanentDead)
1423}
1424
1425async fn handle_spawn(ctx: &Ctx, req: &Request) -> Response {
1426 let p = &req.params;
1427 let name = match p.get("name").and_then(|v| v.as_str()) {
1428 Some(n) if valid_agent_name(n) => n.to_string(),
1429 Some(_) => {
1430 return Response::err(
1431 req.id,
1432 ErrorCode::InvalidParams,
1433 "name must be 1-64 chars of [A-Za-z0-9_-]",
1434 )
1435 }
1436 None => return Response::err(req.id, ErrorCode::InvalidParams, "missing `name`"),
1437 };
1438 let provider = p
1439 .get("provider")
1440 .and_then(|v| v.as_str())
1441 .unwrap_or("codex")
1442 .to_string();
1443 let cwd = match p.get("cwd").and_then(|v| v.as_str()) {
1448 Some(c) => PathBuf::from(c),
1449 None => {
1450 let fallback = std::env::temp_dir();
1451 let _ = ctx.emitter.emit(
1452 "agent_spawn_cwd_fallback",
1453 &json!({"name": name, "fallback": fallback.to_string_lossy()}),
1454 );
1455 fallback
1456 }
1457 };
1458 let host_mode = p
1466 .get("host_mode")
1467 .and_then(|v| v.as_str())
1468 .unwrap_or(crate::state::HOST_MODE_EXEC);
1469 let resume_id = p
1470 .get("resume_id")
1471 .and_then(|v| v.as_str())
1472 .map(|s| s.to_string());
1473 if host_mode == crate::state::HOST_MODE_INTERACTIVE && provider == "claude" {
1474 let claude_mode = p
1475 .get("mode")
1476 .and_then(|v| v.as_str())
1477 .unwrap_or(crate::state::CLAUDE_MODE_STREAM_JSON);
1478 if claude_mode != crate::state::CLAUDE_MODE_INTERACTIVE {
1479 let explicit_argv = p.get("argv").and_then(|v| v.as_array()).map(|a| {
1480 a.iter()
1481 .filter_map(|v| v.as_str().map(String::from))
1482 .collect::<Vec<String>>()
1483 });
1484 return spawn_claude_stream_lane(
1485 ctx,
1486 req,
1487 &name,
1488 &cwd,
1489 resume_id.as_deref(),
1490 explicit_argv,
1491 )
1492 .await;
1493 }
1494 }
1495 let _ = ctx.emitter.emit(
1496 "agent_spawn_failed",
1497 &json!({"name": name, "reason": "daemon_pty_hosting_retired", "provider": provider}),
1498 );
1499 Response::err(
1500 req.id,
1501 ErrorCode::InvalidParams,
1502 "daemon PTY hosting was retired at G4 (x-f54c): spawn a mux-hosted agent pane with \
1503 `fno agents spawn --substrate pane`, or use `--substrate bg|headless`. The daemon \
1504 serves only claude stream-json adoption (host_mode=interactive, mode=stream_json).",
1505 )
1506}
1507
1508fn stream_claim_holder(short_id: &str) -> String {
1517 format!("stream:{short_id}")
1518}
1519
1520fn is_live_writer(status: AgentStatus) -> bool {
1529 matches!(
1530 status,
1531 AgentStatus::Live
1532 | AgentStatus::Ready
1533 | AgentStatus::Idle
1534 | AgentStatus::Busy
1535 | AgentStatus::Spawning
1536 | AgentStatus::Restarting
1537 )
1538}
1539
1540fn claude_stream_worker_args(
1546 short_id: &str,
1547 home: &std::path::Path,
1548 cwd: &std::path::Path,
1549 uuid: &str,
1550 holder: &str,
1551 child_argv: &[String],
1552) -> Vec<String> {
1553 let mut args = vec![
1554 "--stream".into(),
1555 "--short-id".into(),
1556 short_id.into(),
1557 "--home".into(),
1558 home.to_string_lossy().into_owned(),
1559 "--cwd".into(),
1560 cwd.to_string_lossy().into_owned(),
1561 "--session-uuid".into(),
1562 uuid.into(),
1563 "--holder".into(),
1564 holder.into(),
1565 "--".into(),
1566 ];
1567 args.extend(child_argv.iter().cloned());
1568 args
1569}
1570
1571fn build_claude_stream_entry(
1577 name: &str,
1578 short_id: &str,
1579 cwd: &std::path::Path,
1580 uuid: &str,
1581 pid: u32,
1582 pid_start_time: Option<u64>,
1583 log_path: PathBuf,
1584) -> RegistryEntry {
1585 let cwd_s = cwd.to_string_lossy().into_owned();
1586 RegistryEntry {
1587 name: name.into(),
1588 short_id: short_id.into(),
1589 legacy_provider: String::new(),
1590 harness: Some("claude".into()),
1591 harness_session_id: Some(uuid.into()),
1592 cwd: cwd_s.clone(),
1593 project_root: cwd_s,
1594 session_id: None,
1595 legacy_claude_short_id: None,
1596 claude_session_uuid: Some(uuid.into()),
1597 messaging_socket_path: None,
1598 codex_session_id: None,
1599 gemini_session_id: None,
1600 mcp_channel_id: None,
1601 cc_session_id: None,
1602 host_mode: Some(crate::state::HOST_MODE_INTERACTIVE.into()),
1603 status: AgentStatus::Live,
1604 last_message_at: Some(now_rfc3339_like()),
1605 created_at: now_rfc3339_like(),
1606 pid: Some(pid),
1607 pid_start_time,
1608 log_path: Some(log_path.to_string_lossy().into_owned()),
1609 last_reconciled_at: None,
1610 inside_leg: None,
1611 exited_at: None,
1612 mux: None,
1613 screen_state: None,
1614 crown_level: None,
1615 crown_scope: None,
1616 crown_grantor: None,
1617 }
1618}
1619
1620#[derive(Debug)]
1622enum ClaimOutcome {
1623 Acquired,
1625 HeldByOther(String),
1627 Unavailable(String),
1632}
1633
1634fn acquire_session_claim(uuid: &str, holder: &str) -> ClaimOutcome {
1645 match crate::claims::acquire(
1646 &format!("session:{uuid}"),
1647 holder,
1648 crate::claims::AcquireOpts::default(),
1649 ) {
1650 crate::claims::AcquireOutcome::Acquired(_) => ClaimOutcome::Acquired,
1651 crate::claims::AcquireOutcome::HeldByOther { holder, .. } => {
1652 ClaimOutcome::HeldByOther(holder)
1653 }
1654 crate::claims::AcquireOutcome::Error(e) => ClaimOutcome::Unavailable(e),
1655 }
1656}
1657
1658struct DaemonClaimGuard {
1667 session_uuid: String,
1668 holder: String,
1669 armed: bool,
1670}
1671
1672impl DaemonClaimGuard {
1673 fn disarm(mut self) {
1676 self.armed = false;
1677 }
1678}
1679
1680impl Drop for DaemonClaimGuard {
1681 fn drop(&mut self) {
1682 if !self.armed {
1683 return;
1684 }
1685 let _ = crate::claims::release(
1691 &format!("session:{}", self.session_uuid),
1692 &self.holder,
1693 None,
1694 None,
1695 );
1696 }
1697}
1698
1699async fn stream_worker_reports_child_alive(sock: &std::path::Path) -> bool {
1707 let probe = async {
1708 let mut conn = UnixStream::connect(sock).await.ok()?;
1709 write_request(&mut conn, &Request::new(1, "stream.status", json!({})))
1710 .await
1711 .ok()?;
1712 let resp = crate::protocol::read_response(&mut conn).await.ok()?;
1713 Some(
1714 resp.result()
1715 .and_then(|r| r.get("child_alive"))
1716 .and_then(Value::as_bool)
1717 .unwrap_or(false),
1718 )
1719 };
1720 matches!(
1721 tokio::time::timeout(Duration::from_secs(STREAM_PROBE_TIMEOUT_S), probe).await,
1722 Ok(Some(true))
1723 )
1724}
1725
1726async fn spawn_claude_stream_lane(
1735 ctx: &Ctx,
1736 req: &Request,
1737 name: &str,
1738 cwd: &std::path::Path,
1739 resume_id: Option<&str>,
1740 explicit_argv: Option<Vec<String>>,
1741) -> Response {
1742 let uuid = match resume_id {
1745 Some(u) if !u.trim().is_empty() => u,
1746 _ => {
1747 let _ = ctx.emitter.emit(
1748 "agent_spawn_failed",
1749 &json!({"name": name, "reason": "claude_host_needs_from"}),
1750 );
1751 return Response::err(
1752 req.id,
1753 ErrorCode::InvalidParams,
1754 "claude has no fresh interactive host; adopt an idle session: `fno agents promote <name> --from <session-uuid> --provider claude`",
1755 );
1756 }
1757 };
1758
1759 let registry = load_registry_offloaded(ctx.home.registry_json()).await;
1762 if let Some(existing) = registry.find(name) {
1763 return Response::err(
1764 req.id,
1765 ErrorCode::AgentExists,
1766 format!(
1767 "agent {name} already exists (short_id={}); use `fno agents rm` first",
1768 existing.short_id
1769 ),
1770 );
1771 }
1772 if let Some(h) = registry.entries.iter().rev().find(|e| {
1776 e.harness_name() == "claude"
1777 && e.claude_session_uuid.as_deref() == Some(uuid)
1778 && is_live_writer(e.status)
1779 }) {
1780 return Response::err(
1781 req.id,
1782 ErrorCode::InvalidParams,
1783 format!(
1784 "session '{uuid}' is already hosted by live stream thread '{}'; one writer per session",
1785 h.name
1786 ),
1787 );
1788 }
1789 let short_id = derive_short_id(name, ®istry);
1790 let holder = stream_claim_holder(&short_id);
1791
1792 let uuid_owned = uuid.to_string();
1797 let holder_for_acq = holder.clone();
1798 let claim_outcome =
1799 tokio::task::spawn_blocking(move || acquire_session_claim(&uuid_owned, &holder_for_acq))
1800 .await
1801 .unwrap_or_else(|e| ClaimOutcome::Unavailable(format!("claim task panicked: {e}")));
1802 let claim_guard = match claim_outcome {
1805 ClaimOutcome::Acquired => DaemonClaimGuard {
1806 session_uuid: uuid.to_string(),
1807 holder: holder.clone(),
1808 armed: true,
1809 },
1810 ClaimOutcome::HeldByOther(who) => {
1811 let _ = ctx.emitter.emit(
1812 "agent_spawn_failed",
1813 &json!({"name": name, "reason": "session_claimed", "detail": who}),
1814 );
1815 return Response::err(
1816 req.id,
1817 ErrorCode::InvalidParams,
1818 format!(
1819 "session '{uuid}' is held by another writer ({who}); refusing to double-adopt"
1820 ),
1821 );
1822 }
1823 ClaimOutcome::Unavailable(why) => {
1824 let _ = ctx.emitter.emit(
1825 "agent_stream_claim_unavailable",
1826 &json!({"name": name, "session_uuid": uuid, "detail": why}),
1827 );
1828 DaemonClaimGuard {
1831 session_uuid: uuid.to_string(),
1832 holder: holder.clone(),
1833 armed: false,
1834 }
1835 }
1836 };
1837
1838 let child_argv =
1843 explicit_argv.unwrap_or_else(|| crate::provider::claude_stream_json_resume_argv(uuid));
1844 let worker_args =
1845 claude_stream_worker_args(&short_id, ctx.home.root(), cwd, uuid, &holder, &child_argv);
1846 let mut cmd = std::process::Command::new(&ctx.opts.worker_bin);
1847 cmd.args(&worker_args);
1848 cmd.process_group(0);
1849 let child = match cmd.spawn() {
1850 Ok(c) => c,
1851 Err(e) => {
1852 let _ = ctx.emitter.emit(
1854 "agent_spawn_failed",
1855 &json!({"name": name, "reason": "binary_not_found", "detail": e.to_string()}),
1856 );
1857 return Response::err(
1858 req.id,
1859 ErrorCode::SpawnFailed,
1860 format!("could not launch stream worker: {e}"),
1861 );
1862 }
1863 };
1864 let worker_pid = child.id();
1865 let worker_pid_start_time = process_start_time(worker_pid);
1866 drop(child);
1867
1868 let sock = ctx.home.worker_sock(&short_id);
1870 let start = Instant::now();
1871 while !sock.exists() && start.elapsed() < Duration::from_secs(10) {
1872 tokio::time::sleep(Duration::from_millis(25)).await;
1873 }
1874 if !sock.exists() {
1875 let _ = ctx.emitter.emit(
1877 "agent_create_no_session",
1878 &json!({"name": name, "short_id": short_id, "lane": "stream"}),
1879 );
1880 return Response::err(
1881 req.id,
1882 ErrorCode::SpawnFailed,
1883 "stream worker did not come up within 10s",
1884 );
1885 }
1886
1887 if !is_live_stream_thread(&sock).await {
1892 best_effort_worker_shutdown(&sock).await;
1893 let _ = ctx.emitter.emit(
1894 "agent_create_no_session",
1895 &json!({"name": name, "short_id": short_id, "reason": "not_a_stream_thread"}),
1896 );
1897 return Response::err(
1898 req.id,
1899 ErrorCode::SpawnFailed,
1900 "stream worker came up but does not serve the stream protocol",
1901 );
1902 }
1903
1904 if !stream_worker_reports_child_alive(&sock).await {
1911 best_effort_worker_shutdown(&sock).await;
1912 let _ = ctx.emitter.emit(
1913 "agent_create_no_session",
1914 &json!({"name": name, "short_id": short_id, "reason": "resume_child_exited"}),
1915 );
1916 return Response::err(
1917 req.id,
1918 ErrorCode::SpawnFailed,
1919 "claude --resume child exited before adoption (bad/expired session id, auth failure, or dead cwd)",
1920 );
1921 }
1922
1923 let entry = build_claude_stream_entry(
1929 name,
1930 &short_id,
1931 cwd,
1932 uuid,
1933 worker_pid,
1934 worker_pid_start_time,
1935 ctx.home.timeline_jsonl(&short_id),
1936 );
1937 let uuid_for_lock = uuid.to_string();
1938 let insert = update_registry_offloaded(ctx.home.registry_json(), move |r| {
1939 if r.entries.iter().any(|e| e.name == entry.name) {
1940 return false;
1941 }
1942 if r.entries.iter().any(|e| {
1943 e.harness_name() == "claude"
1944 && e.claude_session_uuid.as_deref() == Some(&uuid_for_lock)
1945 && is_live_writer(e.status)
1946 }) {
1947 return false;
1948 }
1949 r.entries.push(entry);
1950 true
1951 })
1952 .await;
1953 match insert {
1954 Ok(true) => flush_buffered_inside_leg(ctx, uuid, name),
1957 Ok(false) => {
1958 best_effort_worker_shutdown(&sock).await;
1959 let _ = ctx.emitter.emit(
1960 "agent_spawn_failed",
1961 &json!({"name": name, "short_id": short_id, "reason": "session_taken_concurrent"}),
1962 );
1963 return Response::err(
1964 req.id,
1965 ErrorCode::AgentExists,
1966 format!("session '{uuid}' was adopted by a concurrent call; this one refused"),
1967 );
1968 }
1969 Err(e) => {
1970 best_effort_worker_shutdown(&sock).await;
1971 let _ = ctx.emitter.emit(
1972 "agent_spawn_failed",
1973 &json!({"name": name, "short_id": short_id, "reason": "registry_write_failed"}),
1974 );
1975 return Response::err(req.id, ErrorCode::Internal, format!("registry write: {e}"));
1976 }
1977 }
1978 claim_guard.disarm();
1981 let _ = ctx.emitter.emit(
1982 "agent_spawned",
1983 &json!({"name": name, "provider": "claude", "short_id": short_id, "lane": "stream", "session_uuid": uuid}),
1984 );
1985
1986 Response::ok(
1987 req.id,
1988 json!({"short_id": short_id, "provider": "claude", "status": "live", "lane": "stream"}),
1989 )
1990}
1991
1992fn provider_readiness_detector(provider: &str) -> Box<dyn crate::readiness::ReadinessDetector> {
2000 use crate::provider::ProviderWithPty as _;
2001 match provider {
2002 "codex" => crate::provider::CodexProvider.readiness_detector(),
2003 "gemini" => crate::provider::GeminiProvider.readiness_detector(),
2004 "agy" => crate::provider::AgyProvider.readiness_detector(),
2005 "opencode" => crate::provider::OpencodeProvider.readiness_detector(),
2006 "claude" => crate::provider::ClaudeInteractiveProvider.readiness_detector(),
2010 other => Box::new(crate::readiness::NoSignalDetector {
2014 provider: other.to_string(),
2015 }),
2016 }
2017}
2018
2019#[derive(Debug, PartialEq, Eq)]
2042enum PollError {
2043 Timeout { secs: u64 },
2045 WorkerUnresponsive { secs: u64 },
2047}
2048
2049impl std::fmt::Display for PollError {
2050 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2051 match self {
2052 PollError::Timeout { secs } => {
2053 write!(f, "ask timed out after {secs}s before reply settled")
2054 }
2055 PollError::WorkerUnresponsive { secs } => write!(
2056 f,
2057 "ask timed out after {secs}s before reply settled (worker snapshot read did not return)"
2058 ),
2059 }
2060 }
2061}
2062
2063async fn poll_until_ready<F, Fut>(
2064 fetcher: F,
2065 detector: Box<dyn crate::readiness::ReadinessDetector>,
2066 poll_interval: Duration,
2067 timeout: Duration,
2068) -> Result<String, PollError>
2069where
2070 F: Fn() -> Fut,
2071 Fut: std::future::Future<Output = Option<String>>,
2072{
2073 let deadline = tokio::time::Instant::now() + timeout;
2074 loop {
2075 let now = tokio::time::Instant::now();
2076 if now >= deadline {
2077 return Err(PollError::Timeout {
2078 secs: timeout.as_secs(),
2079 });
2080 }
2081 let remaining = deadline.saturating_duration_since(now);
2088 let fetched = match tokio::time::timeout(remaining, fetcher()).await {
2089 Ok(opt) => opt,
2090 Err(_) => {
2091 return Err(PollError::WorkerUnresponsive {
2092 secs: timeout.as_secs(),
2093 })
2094 }
2095 };
2096 if let Some(text) = fetched {
2097 let mut grid = crate::screen::TerminalGrid::with_default_size();
2099 grid.feed(text.as_bytes());
2100 let owned = grid.snapshot();
2101 let view = owned.view();
2102 match detector.is_ready(&view) {
2103 Ok(true) => return Ok(owned.text),
2104 Ok(false) | Err(_) => {} }
2106 }
2107 tokio::time::sleep(poll_interval).await;
2108 }
2109}
2110
2111async fn handle_ask(ctx: &Ctx, req: &Request) -> Response {
2112 let name = match req.params.get("name").and_then(|v| v.as_str()) {
2113 Some(n) => n.to_string(),
2114 None => return Response::err(req.id, ErrorCode::InvalidParams, "missing `name`"),
2115 };
2116 let message = req
2117 .params
2118 .get("message")
2119 .and_then(|v| v.as_str())
2120 .unwrap_or("")
2121 .to_string();
2122
2123 let provider_param = req
2124 .params
2125 .get("provider")
2126 .and_then(|v| v.as_str())
2127 .map(String::from);
2128 let cwd_param = req
2129 .params
2130 .get("cwd")
2131 .and_then(|v| v.as_str())
2132 .map(PathBuf::from);
2133 let from_name_param = req
2134 .params
2135 .get("from_name")
2136 .and_then(|v| v.as_str())
2137 .map(String::from);
2138 let yolo_param = req
2139 .params
2140 .get("yolo")
2141 .and_then(|v| v.as_bool())
2142 .unwrap_or(false);
2143
2144 let registry = load_registry_offloaded(ctx.home.registry_json()).await;
2145 let entry = match registry.find(&name) {
2146 Some(e) => e.clone(),
2147 None => {
2148 let provider = match provider_param {
2151 Some(p) => p,
2152 None => {
2153 return Response::err(
2154 req.id,
2155 ErrorCode::InvalidParams,
2156 format!(
2157 "agent '{name}' not found; pass --provider to create it on first contact"
2158 ),
2159 )
2160 }
2161 };
2162 let spawn_cwd = match cwd_param {
2167 Some(c) => c,
2168 None => {
2169 let fallback = std::env::temp_dir();
2170 let _ = ctx.emitter.emit(
2171 "agent_spawn_cwd_fallback",
2172 &json!({
2173 "name": name,
2174 "fallback": fallback.to_string_lossy(),
2175 "via": "ask_first_contact",
2176 }),
2177 );
2178 fallback
2179 }
2180 };
2181 let mut spawn_params = serde_json::Map::new();
2183 spawn_params.insert("name".into(), serde_json::Value::String(name.clone()));
2184 spawn_params.insert("provider".into(), serde_json::Value::String(provider));
2185 spawn_params.insert(
2186 "cwd".into(),
2187 serde_json::Value::String(spawn_cwd.to_str().unwrap_or(".").to_string()),
2188 );
2189 spawn_params.insert("message".into(), serde_json::Value::String(message.clone()));
2190 if let Some(ref fn_val) = from_name_param {
2191 spawn_params.insert(
2192 "from_name".into(),
2193 serde_json::Value::String(fn_val.clone()),
2194 );
2195 }
2196 if yolo_param {
2197 spawn_params.insert("yolo".into(), serde_json::Value::Bool(true));
2198 }
2199 let spawn_req = Request::new(
2200 req.id,
2201 "agent.spawn",
2202 serde_json::Value::Object(spawn_params),
2203 );
2204 let spawn_resp = handle_spawn(ctx, &spawn_req).await;
2205 return match spawn_resp.payload {
2206 crate::protocol::ResponsePayload::Ok(ref result) => {
2207 let short_id = result
2208 .get("short_id")
2209 .and_then(|v| v.as_str())
2210 .unwrap_or("")
2211 .to_string();
2212 Response::ok(req.id, json!({"created": true, "short_id": short_id}))
2213 }
2214 crate::protocol::ResponsePayload::Err(_) => spawn_resp,
2215 };
2216 }
2217 };
2218 if entry.status == AgentStatus::Orphaned {
2219 return Response::err(
2220 req.id,
2221 ErrorCode::InvalidStatus,
2222 format!("agent {name} is orphaned; use `fno agents reconcile` or `rm`"),
2223 );
2224 }
2225
2226 let sock = ctx.home.worker_sock(&entry.short_id);
2227 let mut conn = match UnixStream::connect(&sock).await {
2228 Ok(c) => c,
2229 Err(_) => {
2230 return Response::err(
2231 req.id,
2232 ErrorCode::InvalidStatus,
2233 format!("worker for {name} is not reachable"),
2234 )
2235 }
2236 };
2237
2238 let mut payload = message.clone();
2242 if !payload.ends_with('\n') {
2243 payload.push('\n');
2244 }
2245 if write_request(
2246 &mut conn,
2247 &Request::new(1, "worker.write", json!({"data": payload})),
2248 )
2249 .await
2250 .is_err()
2251 {
2252 return Response::err(req.id, ErrorCode::Internal, "worker write failed");
2253 }
2254 match crate::protocol::read_response(&mut conn).await {
2258 Ok(ack) if ack.is_err() => {
2259 let msg = ack
2260 .error()
2261 .map(|e| e.message.clone())
2262 .unwrap_or_else(|| "worker rejected the write".into());
2263 return Response::err(req.id, ErrorCode::Internal, msg);
2264 }
2265 Ok(_) => {}
2266 Err(_) => {
2267 return Response::err(req.id, ErrorCode::Internal, "no write-ack from worker");
2268 }
2269 }
2270
2271 let timeout_secs = req
2275 .params
2276 .get("timeout")
2277 .and_then(|v| v.as_u64())
2278 .unwrap_or(600);
2279 let detector = provider_readiness_detector(entry.harness_name());
2280 let sock_path = sock.clone();
2281 let fetcher = move || {
2282 let p = sock_path.clone();
2283 async move { read_worker_snapshot(&p).await }
2284 };
2285 let reply = match poll_until_ready(
2286 fetcher,
2287 detector,
2288 Duration::from_millis(200),
2289 Duration::from_secs(timeout_secs),
2290 )
2291 .await
2292 {
2293 Ok(text) => text,
2294 Err(e) => {
2295 return Response::err(req.id, ErrorCode::Internal, e.to_string());
2296 }
2297 };
2298
2299 let ask_name = name.clone();
2300 let _ = update_registry_offloaded(ctx.home.registry_json(), move |r| {
2301 if let Some(e) = r.find_mut(&ask_name) {
2302 e.last_message_at = Some(now_rfc3339_like());
2303 }
2304 })
2305 .await;
2306 let _ = ctx
2307 .emitter
2308 .emit("agent_ask_done", &json!({"name": name, "backend": "pty"}));
2309
2310 Response::ok(req.id, json!({"reply": reply, "backend": "pty"}))
2311}
2312
2313const MAX_INJECT_BODY_BYTES: usize = 16 * 1024 * 1024;
2317
2318const SWITCHBOARD_TURN_TIMEOUT_MS: u64 = 120_000;
2333const SWITCHBOARD_POLL_MS: u64 = 50;
2335const STREAM_PROBE_TIMEOUT_S: u64 = 2;
2338const SWITCHBOARD_MIRROR_TIMEOUT_S: u64 = 5;
2340const SWITCHBOARD_DRIVE_GRACE_S: u64 = 5;
2344
2345struct SwitchboardTurn {
2347 reply: String,
2349 is_error: bool,
2351 saw_receipt: bool,
2354}
2355
2356async fn is_live_stream_thread(sock: &std::path::Path) -> bool {
2363 let probe = async {
2366 let mut conn = UnixStream::connect(sock).await.ok()?;
2367 write_request(&mut conn, &Request::new(1, "stream.ping", json!({})))
2368 .await
2369 .ok()?;
2370 let resp = crate::protocol::read_response(&mut conn).await.ok()?;
2371 Some(!resp.is_err())
2372 };
2373 matches!(
2374 tokio::time::timeout(Duration::from_secs(STREAM_PROBE_TIMEOUT_S), probe).await,
2375 Ok(Some(true))
2376 )
2377}
2378
2379async fn drive_stream_turn(
2385 worker_sock: &std::path::Path,
2386 text: &str,
2387 deadline: Duration,
2388) -> Result<SwitchboardTurn, String> {
2389 let mut conn = UnixStream::connect(worker_sock)
2390 .await
2391 .map_err(|e| format!("target not live (worker unreachable): {e}"))?;
2392
2393 write_request(
2400 &mut conn,
2401 &Request::new(0, "stream.read_frames", json!({ "cursor": u64::MAX })),
2402 )
2403 .await
2404 .map_err(|e| format!("cursor probe send failed: {e}"))?;
2405 let probe = crate::protocol::read_response(&mut conn)
2406 .await
2407 .map_err(|e| format!("cursor probe recv failed: {e}"))?;
2408 let mut cursor = probe
2409 .result()
2410 .and_then(|r| r.get("next"))
2411 .and_then(|v| v.as_u64())
2412 .ok_or_else(|| "cursor probe returned no result".to_string())?;
2413
2414 write_request(
2416 &mut conn,
2417 &Request::new(1, "stream.write_turn", json!({ "text": text })),
2418 )
2419 .await
2420 .map_err(|e| format!("write_turn send failed: {e}"))?;
2421 match crate::protocol::read_response(&mut conn).await {
2422 Ok(ack) if ack.is_err() => {
2423 return Err(format!(
2424 "write_turn rejected: {}",
2425 ack.error().map(|e| e.message.as_str()).unwrap_or("?")
2426 ))
2427 }
2428 Ok(_) => {}
2429 Err(e) => return Err(format!("no write_turn ack: {e}")),
2430 }
2431
2432 let start = Instant::now();
2434 let mut reply = String::new();
2435 let mut saw_receipt = false;
2436 let mut req_id = 100u64;
2437 loop {
2438 if start.elapsed() > deadline {
2439 return Err("turn timed out before result".into());
2440 }
2441 write_request(
2442 &mut conn,
2443 &Request::new(req_id, "stream.read_frames", json!({ "cursor": cursor })),
2444 )
2445 .await
2446 .map_err(|e| format!("read_frames send failed: {e}"))?;
2447 req_id += 1;
2448 let resp = crate::protocol::read_response(&mut conn)
2449 .await
2450 .map_err(|e| format!("read_frames recv failed: {e}"))?;
2451 let res = resp
2452 .result()
2453 .ok_or_else(|| "read_frames returned no result".to_string())?;
2454 if let Some(next) = res.get("next").and_then(|v| v.as_u64()) {
2455 cursor = next;
2456 }
2457 let child_alive = res
2458 .get("child_alive")
2459 .and_then(|v| v.as_bool())
2460 .unwrap_or(true);
2461 if let Some(frames) = res.get("frames").and_then(|v| v.as_array()) {
2462 for fr in frames {
2463 match fr.get("kind").and_then(|k| k.as_str()) {
2464 Some("user_echo") => saw_receipt = true,
2465 Some("assistant") => {
2466 if let Some(t) = fr.get("text").and_then(|t| t.as_str()) {
2467 reply.push_str(t);
2468 }
2469 }
2470 Some("result") => {
2471 let is_error = fr
2472 .get("is_error")
2473 .and_then(|e| e.as_bool())
2474 .unwrap_or(false);
2475 if reply.is_empty() {
2478 if let Some(r) = fr.get("result").and_then(|r| r.as_str()) {
2479 reply.push_str(r);
2480 }
2481 }
2482 return Ok(SwitchboardTurn {
2483 reply,
2484 is_error,
2485 saw_receipt,
2486 });
2487 }
2488 _ => {}
2490 }
2491 }
2492 }
2493 if !child_alive {
2494 return Err("target child exited before result (orphaned)".into());
2495 }
2496 tokio::time::sleep(Duration::from_millis(SWITCHBOARD_POLL_MS)).await;
2497 }
2498}
2499
2500async fn mirror_into(worker_sock: &std::path::Path, text: &str) -> Result<(), String> {
2505 let inner = async {
2506 let mut conn = UnixStream::connect(worker_sock)
2507 .await
2508 .map_err(|e| format!("mirror target unreachable: {e}"))?;
2509 write_request(
2510 &mut conn,
2511 &Request::new(1, "stream.write_turn", json!({ "text": text })),
2512 )
2513 .await
2514 .map_err(|e| format!("mirror write failed: {e}"))?;
2515 match crate::protocol::read_response(&mut conn).await {
2516 Ok(ack) if ack.is_err() => Err(format!(
2517 "mirror rejected: {}",
2518 ack.error().map(|e| e.message.as_str()).unwrap_or("?")
2519 )),
2520 Ok(_) => Ok(()),
2521 Err(e) => Err(format!("no mirror ack: {e}")),
2522 }
2523 };
2524 match tokio::time::timeout(Duration::from_secs(SWITCHBOARD_MIRROR_TIMEOUT_S), inner).await {
2526 Ok(r) => r,
2527 Err(_) => Err("mirror timed out".into()),
2528 }
2529}
2530
2531async fn stamp_orphaned(
2535 home: &AgentsHome,
2536 name: String,
2537 identity: Value,
2538) -> Result<bool, state::StateError> {
2539 update_registry_offloaded(home.registry_json(), move |registry| {
2540 let Some(entry) = registry.find_mut(&name) else {
2541 return false;
2542 };
2543 if !switchboard_identity_matches(entry, &identity) {
2544 return false;
2545 }
2546 if entry.status == AgentStatus::Live {
2551 entry.status = AgentStatus::Orphaned;
2552 }
2553 true
2554 })
2555 .await
2556}
2557
2558fn switchboard_identity_matches(entry: &RegistryEntry, identity: &Value) -> bool {
2575 let Some(expected) = identity.as_object() else {
2576 return false;
2577 };
2578 let Some(harness) = expected.get("harness").and_then(Value::as_str) else {
2579 return false;
2580 };
2581 let Some(short_id) = expected.get("short_id").and_then(Value::as_str) else {
2582 return false;
2583 };
2584 let Some(created_at) = expected.get("created_at").and_then(Value::as_str) else {
2585 return false;
2586 };
2587 let session_id = match expected.get("session_id") {
2588 Some(Value::Null) => None,
2589 Some(Value::String(value)) => Some(value.as_str()),
2590 _ => return false,
2591 };
2592 entry.harness_name() == harness
2593 && entry.harness_session_id.as_deref() == session_id
2594 && entry.short_id == short_id
2595 && entry.created_at == created_at
2596}
2597
2598async fn handle_switchboard(ctx: &Ctx, req: &Request) -> Response {
2599 let to = match req.params.get("to").and_then(|v| v.as_str()) {
2600 Some(s) => s.to_string(),
2601 None => return Response::err(req.id, ErrorCode::InvalidParams, "missing `to`"),
2602 };
2603 let from = req
2604 .params
2605 .get("from")
2606 .and_then(|v| v.as_str())
2607 .unwrap_or("unknown")
2608 .to_string();
2609 let body = match req.params.get("body").and_then(|v| v.as_str()) {
2610 Some(b) => b.to_string(),
2611 None => return Response::err(req.id, ErrorCode::InvalidParams, "missing `body`"),
2612 };
2613 let mirror = req
2614 .params
2615 .get("mirror")
2616 .and_then(|v| v.as_bool())
2617 .unwrap_or(true);
2618 let recipient_identity = match req.params.get("recipient_identity") {
2619 Some(value) if value.is_object() => value,
2620 _ => {
2621 return Response::err(
2622 req.id,
2623 ErrorCode::InvalidParams,
2624 "missing `recipient_identity`",
2625 )
2626 }
2627 };
2628 let from_identity = req.params.get("from_identity");
2629 if mirror && !from_identity.is_some_and(Value::is_object) {
2630 return Response::err(
2631 req.id,
2632 ErrorCode::InvalidParams,
2633 "missing `from_identity` for mirrored switchboard turn",
2634 );
2635 }
2636 let timeout_ms = req
2637 .params
2638 .get("timeout_ms")
2639 .and_then(|v| v.as_u64())
2640 .unwrap_or(SWITCHBOARD_TURN_TIMEOUT_MS);
2641
2642 if body.len() > MAX_INJECT_BODY_BYTES {
2643 return Response::err(
2644 req.id,
2645 ErrorCode::InvalidParams,
2646 format!(
2647 "body too large: {} bytes > {MAX_INJECT_BODY_BYTES}",
2648 body.len()
2649 ),
2650 );
2651 }
2652
2653 let registry = load_registry_offloaded(ctx.home.registry_json()).await;
2654 let to_entry = match registry.find(&to) {
2655 Some(e) => e.clone(),
2656 None => {
2657 return Response::err(
2658 req.id,
2659 ErrorCode::AgentNotFound,
2660 format!("agent '{to}' not found"),
2661 )
2662 }
2663 };
2664 if !switchboard_identity_matches(&to_entry, recipient_identity) {
2665 return Response::ok(
2666 req.id,
2667 json!({"delivered": false, "reason": "recipient-identity-changed"}),
2668 );
2669 }
2670
2671 let to_sock = ctx.home.worker_sock(&to_entry.short_id);
2674 if to_entry.harness_name() != "claude" || !is_live_stream_thread(&to_sock).await {
2675 return Response::ok(
2676 req.id,
2677 json!({"delivered": false, "reason": "not-a-live-stream-thread"}),
2678 );
2679 }
2680
2681 let drive_deadline = Duration::from_millis(timeout_ms);
2686 let outer = drive_deadline + Duration::from_secs(SWITCHBOARD_DRIVE_GRACE_S);
2687 let drive_result =
2688 match tokio::time::timeout(outer, drive_stream_turn(&to_sock, &body, drive_deadline)).await
2689 {
2690 Ok(inner) => inner,
2691 Err(_) => Err("drive hung past the turn budget (timed out)".to_string()),
2692 };
2693 let outcome = match drive_result {
2694 Ok(o) => o,
2695 Err(reason) => {
2696 match stamp_orphaned(&ctx.home, to.clone(), recipient_identity.clone()).await {
2700 Ok(true) => {}
2701 Ok(false) => {
2702 let _ = ctx.emitter.emit(
2703 "agent_deliver_status_write_failed",
2704 &json!({
2705 "name": to,
2706 "from_name": from,
2707 "provider": "claude",
2708 "transport": "switchboard",
2709 "reason": "recipient-identity-changed",
2710 }),
2711 );
2712 }
2713 Err(error) => {
2714 let _ = ctx.emitter.emit(
2715 "agent_deliver_status_write_failed",
2716 &json!({
2717 "name": to,
2718 "from_name": from,
2719 "provider": "claude",
2720 "transport": "switchboard",
2721 "reason": "registry-write-failed",
2722 "error": error.to_string(),
2723 }),
2724 );
2725 }
2726 }
2727 let _ = ctx.emitter.emit(
2728 "agent_deliver_demoted",
2729 &json!({
2730 "name": to,
2731 "from_name": from,
2732 "provider": "claude",
2733 "transport": "switchboard",
2734 "reason": reason,
2735 }),
2736 );
2737 return Response::ok(req.id, json!({"delivered": false, "reason": reason}));
2738 }
2739 };
2740
2741 let mut mirrored = false;
2746 if mirror && from != to {
2747 let fresh = load_registry_offloaded(ctx.home.registry_json()).await;
2751 if let Some(from_entry) = fresh.find(&from).filter(|entry| {
2752 from_identity.is_some_and(|identity| switchboard_identity_matches(entry, identity))
2753 }) {
2754 let from_sock = ctx.home.worker_sock(&from_entry.short_id);
2755 if from_entry.harness_name() == "claude" && is_live_stream_thread(&from_sock).await {
2756 match mirror_into(&from_sock, &outcome.reply).await {
2757 Ok(()) => mirrored = true,
2758 Err(e) => {
2759 let _ = ctx.emitter.emit(
2763 "agent_deliver_demoted",
2764 &json!({
2765 "name": from,
2766 "from_name": to,
2767 "provider": "claude",
2768 "transport": "switchboard-mirror",
2769 "reason": e,
2770 }),
2771 );
2772 }
2773 }
2774 }
2775 }
2776 }
2777
2778 let _ = ctx.emitter.emit(
2779 "agent_deliver_injected",
2780 &json!({
2781 "name": to,
2782 "from_name": from,
2783 "provider": "claude",
2784 "transport": "switchboard",
2785 "mirrored": mirrored,
2786 "is_error": outcome.is_error,
2787 }),
2788 );
2789
2790 Response::ok(
2791 req.id,
2792 json!({
2793 "delivered": true,
2794 "identity_verified": true,
2795 "transport": "switchboard",
2796 "reply": outcome.reply,
2797 "is_error": outcome.is_error,
2798 "mirrored": mirrored,
2799 "receipt": outcome.saw_receipt,
2800 }),
2801 )
2802}
2803
2804async fn read_worker_snapshot(sock: &std::path::Path) -> Option<String> {
2805 let mut conn = UnixStream::connect(sock).await.ok()?;
2806 write_request(&mut conn, &Request::new(2, "worker.snapshot", json!({})))
2807 .await
2808 .ok()?;
2809 let resp = crate::protocol::read_response(&mut conn).await.ok()?;
2810 resp.result()
2811 .and_then(|r| r.get("text").and_then(|t| t.as_str()).map(String::from))
2812}
2813
2814fn reap_zombies() {
2818 loop {
2819 let mut status: libc::c_int = 0;
2820 let pid = unsafe { libc::waitpid(-1, &mut status, libc::WNOHANG) };
2823 if pid <= 0 {
2824 break;
2825 }
2826 }
2827}
2828
2829fn rendered_status_from_truth(truth: Option<&str>) -> &'static str {
2830 match truth {
2831 Some("working" | "watching" | "your-move") => "live",
2832 Some("done" | "stalled") => "orphaned",
2833 _ => "unknown",
2834 }
2835}
2836
2837fn registry_truth_handle(entry: &RegistryEntry) -> String {
2838 if let Some(session_id) = entry.harness_session_id.as_deref() {
2839 return session_id.to_string();
2840 }
2841 if !entry.short_id.is_empty() {
2842 entry.short_id.clone()
2843 } else {
2844 entry.name.clone()
2845 }
2846}
2847
2848fn handle_list(ctx: &Ctx, req: &Request) -> Response {
2849 handle_list_with_truth(ctx, req, crate::claude_ask::family1_truth_state)
2850}
2851
2852fn handle_list_with_truth<F>(ctx: &Ctx, req: &Request, truth_fn: F) -> Response
2853where
2854 F: Fn(&str) -> Option<String>,
2855{
2856 let all = req
2857 .params
2858 .get("all")
2859 .and_then(|v| v.as_bool())
2860 .unwrap_or(false);
2861
2862 let filter_cwd = req
2865 .params
2866 .get("cwd")
2867 .and_then(|v| v.as_str())
2868 .map(String::from);
2869 let filter_provider = req
2870 .params
2871 .get("provider")
2872 .and_then(|v| v.as_str())
2873 .map(String::from);
2874 let filter_status = req
2875 .params
2876 .get("status")
2877 .and_then(|v| v.as_str())
2878 .map(String::from);
2879 let cwd_project = req
2880 .params
2881 .get("project_root")
2882 .and_then(|v| v.as_str())
2883 .map(String::from);
2884
2885 if let Some(ref st) = filter_status {
2890 if st != "live" && st != "orphaned" && st != "unknown" {
2891 return Response::err(
2892 req.id,
2893 ErrorCode::InvalidStatus,
2894 format!("invalid --status '{st}' (expected: live | orphaned | unknown)"),
2895 );
2896 }
2897 }
2898 let norm_path = |p: &str| -> String {
2904 std::fs::canonicalize(p)
2905 .ok()
2906 .and_then(|pb| pb.to_str().map(String::from))
2907 .unwrap_or_else(|| p.to_string())
2908 };
2909 let filter_cwd_norm = filter_cwd.as_deref().map(&norm_path);
2910
2911 let registry = state::load_registry(&ctx.home.registry_json()).unwrap_or_default();
2912 let classified: Vec<_> = registry
2913 .entries
2914 .iter()
2915 .filter(|e| {
2916 if !all {
2917 if let Some(ref p) = cwd_project {
2918 if &e.project_root != p {
2919 return false;
2920 }
2921 }
2922 }
2923 if let Some(ref cwd) = filter_cwd_norm {
2924 if &norm_path(&e.cwd) != cwd {
2925 return false;
2926 }
2927 }
2928 if let Some(ref prov) = filter_provider {
2929 if e.harness_name() != prov.as_str() {
2930 return false;
2931 }
2932 }
2933 true
2934 })
2935 .map(|e| {
2936 let truth_handle = registry_truth_handle(e);
2937 let truth = truth_fn(&truth_handle);
2938 (e, rendered_status_from_truth(truth.as_deref()))
2939 })
2940 .collect();
2941 let entries: Vec<Value> = classified
2942 .into_iter()
2943 .filter(|(_e, rendered_status)| {
2944 if let Some(ref st) = filter_status {
2945 if rendered_status != &st.as_str() {
2946 return false;
2947 }
2948 }
2949 true
2950 })
2951 .map(|(e, rendered_status)| {
2952 let resume_id: Option<String> = match e.harness_name() {
2973 "claude" => e
2974 .transport_short()
2975 .map(str::to_string)
2976 .or_else(|| e.session_id.clone()),
2977 "codex" => e.codex_session_id.clone().or_else(|| e.session_id.clone()),
2978 "gemini" => e.gemini_session_id.clone().or_else(|| e.session_id.clone()),
2979 "opencode" => e
2984 .harness_session_id
2985 .clone()
2986 .filter(|s| !s.is_empty())
2987 .or_else(|| e.session_id.clone()),
2988 _ => e.session_id.clone(),
2989 };
2990 let session_id: Value = resume_id.map(Value::String).unwrap_or(Value::Null);
2991 let short_id: Value = e
2992 .transport_short()
2993 .map(|s| Value::String(s.to_string()))
2994 .unwrap_or(Value::Null);
2995 let log_path: Value = e
2996 .log_path
2997 .as_deref()
2998 .map(|s| Value::String(s.to_string()))
2999 .unwrap_or(Value::Null);
3000 let crown: Value = match e.crown_level {
3005 Some(level) => Value::String(format!(
3006 "L{level} {}",
3007 e.crown_scope
3008 .as_deref()
3009 .filter(|s| !s.is_empty())
3010 .unwrap_or("?")
3011 )),
3012 None => Value::Null,
3013 };
3014 json!({
3015 "name": e.name,
3016 "harness": e.harness_name(),
3020 "provider": e.harness_name(),
3021 "harness_session_id": e.harness_session_id,
3022 "short_id": short_id,
3023 "session_id": session_id,
3024 "cwd": e.cwd,
3025 "created_at": e.created_at,
3026 "last_message_at": e.last_message_at,
3027 "status": rendered_status,
3028 "live_status": null,
3029 "pid": e.pid,
3037 "last_reconciled_at": e.last_reconciled_at,
3038 "log_path": log_path,
3039 "mux": e.mux,
3044 "crown": crown,
3047 "crown_level": e.crown_level,
3048 "crown_scope": e.crown_scope,
3049 "crown_grantor": e.crown_grantor,
3050 "project_root": e.project_root,
3055 })
3056 })
3057 .collect();
3058 let filters_applied = json!({
3064 "cwd": filter_cwd_norm,
3065 "provider": filter_provider,
3066 "status": filter_status,
3067 });
3068 Response::ok(
3069 req.id,
3070 json!({"agents": entries, "filters_applied": filters_applied}),
3071 )
3072}
3073
3074async fn handle_status(ctx: &Ctx, req: &Request) -> Response {
3092 let reg_path = ctx.home.registry_json();
3095 let registry = tokio::task::spawn_blocking(move || state::load_registry(®_path))
3096 .await
3097 .ok()
3098 .and_then(|r| r.ok())
3099 .unwrap_or_default();
3100 let mut by_status: Map<String, Value> = Map::new();
3101 let mut restarting: u64 = 0;
3102 let mut channels_registered: u64 = 0;
3103 for e in ®istry.entries {
3104 let key = format!("{:?}", e.status).to_lowercase();
3105 let n = by_status.get(&key).and_then(|v| v.as_u64()).unwrap_or(0) + 1;
3106 by_status.insert(key, Value::Number(n.into()));
3107 if e.status == AgentStatus::Restarting {
3108 restarting += 1;
3109 }
3110 if e.mcp_channel_id.is_some() {
3111 channels_registered += 1;
3112 }
3113 }
3114 Response::ok(
3115 req.id,
3116 json!({
3117 "schema_version": 1,
3118 "daemon": {
3119 "state": DaemonState::Serving.as_str(),
3120 "pid": std::process::id(),
3121 "uptime_secs": ctx.started_at.elapsed().as_secs(),
3122 "version": env!("CARGO_PKG_VERSION"),
3123 "exe_path": ctx
3126 .exe_fingerprint
3127 .as_ref()
3128 .map(|f| f.path.to_string_lossy().into_owned()),
3129 "exe_mtime": ctx.exe_fingerprint.as_ref().map(|f| f.mtime_nanos),
3130 "exe_size": ctx.exe_fingerprint.as_ref().map(|f| f.size),
3131 "pid_start_time": ctx.pid_start_time,
3134 },
3135 "agents": {
3136 "total": registry.entries.len(),
3137 "by_status": by_status,
3138 },
3139 "restarts": {
3140 "queue_depth": restarting,
3145 "consecutive_failures_max_seen": 0,
3146 },
3147 "channels": { "registered": channels_registered },
3148 }),
3149 )
3150}
3151
3152async fn entry_for_lifecycle(
3156 registry: &state::Registry,
3157 token: &str,
3158 registry_path: &std::path::Path,
3159) -> Result<Option<RegistryEntry>, String> {
3160 let Value::Array(rows) = serde_json::to_value(®istry.entries)
3161 .map_err(|exc| format!("could not inspect registry identities: {exc}"))?
3162 else {
3163 return Err("could not inspect registry identities".to_string());
3164 };
3165 let worker_token = token.to_string();
3166 let path = registry_path.to_path_buf();
3167 let resolved = tokio::task::spawn_blocking(move || {
3168 crate::client_verbs::resolve_entry_with_heal(&rows, &worker_token, &path)
3169 })
3170 .await
3171 .map_err(|exc| format!("identity resolution task failed: {exc}"))?;
3172 match resolved {
3173 Ok(entry) => {
3174 let mut entry: RegistryEntry = serde_json::from_value(entry)
3175 .map_err(|exc| format!("resolved identity row is unreadable: {exc}"))?;
3176 entry.backfill_harness_aliases();
3177 if let Some(legacy) = entry.backfill_short_id() {
3178 return Err(format!(
3179 "resolved identity row {:?} has conflicting transport ids (legacy={legacy:?})",
3180 entry.name
3181 ));
3182 }
3183 Ok(Some(entry))
3184 }
3185 Err(crate::client_verbs::ResolveError::NotFound(_)) => Ok(None),
3186 Err(err) => Err(err.message()),
3187 }
3188}
3189
3190async fn handle_stop(ctx: &Ctx, req: &Request) -> Response {
3191 let requested_name = match req.params.get("name").and_then(|v| v.as_str()) {
3192 Some(n) => n.to_string(),
3193 None => return Response::err(req.id, ErrorCode::InvalidParams, "missing `name`"),
3194 };
3195 let registry = load_registry_offloaded(ctx.home.registry_json()).await;
3196 let entry =
3197 match entry_for_lifecycle(®istry, &requested_name, &ctx.home.registry_json()).await {
3198 Ok(Some(entry)) => entry,
3199 Ok(None) => {
3200 return Response::err(
3201 req.id,
3202 ErrorCode::AgentNotFound,
3203 format!("agent {requested_name} not found"),
3204 )
3205 }
3206 Err(message) => return Response::err(req.id, ErrorCode::InvalidParams, message),
3207 };
3208 let name = entry.name.clone();
3209 if entry.status == AgentStatus::Exited {
3210 return Response::ok(
3213 req.id,
3214 json!({"already_exited": true, "short_id": entry.short_id}),
3215 );
3216 }
3217 if entry.harness_name() == "claude" {
3220 return stop_claude(ctx, req, &name, &entry).await;
3221 }
3222 if entry.short_id.is_empty() {
3231 let _ = ctx.emitter.emit(
3232 "agent_stopped",
3233 &json!({"name": name, "provider": entry.harness_name(), "claude_exit": Value::Null}),
3234 );
3235 return Response::ok(
3236 req.id,
3237 json!({"stopped": true, "provider": entry.harness_name(), "no_op": true}),
3238 );
3239 }
3240 if !stop_worker_confirmed(ctx, &entry).await {
3245 return Response::err(
3246 req.id,
3247 ErrorCode::Internal,
3248 format!("agent {name}: worker did not confirm shutdown; it may still be running"),
3249 );
3250 }
3251 let stop_name = name.clone();
3256 if let Err(e) = update_registry_offloaded(ctx.home.registry_json(), move |r| {
3257 if let Some(e) = r.find_mut(&stop_name) {
3258 e.status = AgentStatus::Exited;
3259 }
3260 })
3261 .await
3262 {
3263 let _ = ctx.emitter.emit(
3264 "agent_stop_error",
3265 &json!({"name": name, "error": e.to_string()}),
3266 );
3267 return Response::err(
3268 req.id,
3269 ErrorCode::Internal,
3270 format!("agent {name}: worker stopped but registry write failed: {e}"),
3271 );
3272 }
3273 let _ = ctx.emitter.emit("agent_stopped", &json!({"name": name}));
3274 Response::ok(req.id, json!({"stopped": true, "short_id": entry.short_id}))
3275}
3276
3277async fn best_effort_worker_shutdown(sock: &std::path::Path) {
3281 if let Ok(mut conn) = UnixStream::connect(sock).await {
3282 let _ = write_request(&mut conn, &Request::new(1, "worker.shutdown", json!({}))).await;
3283 let _ = crate::protocol::read_response(&mut conn).await;
3284 }
3285}
3286
3287async fn stop_worker_confirmed(ctx: &Ctx, entry: &RegistryEntry) -> bool {
3293 let sock = ctx.home.worker_sock(&entry.short_id);
3294 if let Ok(mut conn) = UnixStream::connect(&sock).await {
3296 let _ = write_request(&mut conn, &Request::new(1, "worker.shutdown", json!({}))).await;
3297 let _ = crate::protocol::read_response(&mut conn).await;
3298 }
3299 let mut down = worker_down_within(&sock, Duration::from_secs(5)).await;
3304 if !down {
3314 if let Some(pid) = entry.pid {
3315 if pid_is_ours(pid, entry.pid_start_time) {
3316 unsafe {
3317 libc::kill(pid as libc::pid_t, libc::SIGTERM);
3318 }
3319 down = worker_down_within(&sock, Duration::from_secs(5)).await;
3320 if !down && pid_is_ours(pid, entry.pid_start_time) {
3321 unsafe {
3322 libc::kill(pid as libc::pid_t, libc::SIGKILL);
3323 }
3324 down = worker_down_within(&sock, Duration::from_secs(2)).await;
3325 }
3326 }
3327 }
3328 }
3329 if down {
3334 let _ = std::fs::remove_file(&sock);
3335 }
3336 down
3337}
3338
3339async fn worker_socket_reachable(sock: &std::path::Path) -> bool {
3343 UnixStream::connect(sock).await.is_ok()
3344}
3345
3346async fn worker_down_within(sock: &std::path::Path, budget: Duration) -> bool {
3351 let start = Instant::now();
3352 loop {
3353 if !worker_socket_reachable(sock).await {
3354 return true;
3355 }
3356 if start.elapsed() >= budget {
3357 return false;
3358 }
3359 tokio::time::sleep(Duration::from_millis(50)).await;
3360 }
3361}
3362
3363fn pid_confirmed_dead(pid: u32) -> bool {
3374 if pid <= 1 || pid > i32::MAX as u32 {
3375 return true;
3377 }
3378 if unsafe { libc::kill(pid as libc::pid_t, 0) } == 0 {
3380 return false; }
3382 std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH)
3383}
3384
3385fn pid_recycled(pid: u32, recorded_start: Option<u64>) -> bool {
3395 let Some(recorded) = recorded_start else {
3396 return false; };
3398 if pid <= 1 || pid > i32::MAX as u32 {
3399 return false;
3400 }
3401 if unsafe { libc::kill(pid as libc::pid_t, 0) } != 0 {
3403 return false; }
3405 match process_start_time(pid) {
3406 Some(now) => now != recorded,
3407 None => false, }
3409}
3410
3411async fn pid_gone_within(pid: u32, recorded_start: Option<u64>, budget: Duration) -> bool {
3419 let start = Instant::now();
3420 loop {
3421 if pid_confirmed_dead(pid) || pid_recycled(pid, recorded_start) {
3422 return true;
3423 }
3424 if start.elapsed() >= budget {
3425 return false;
3426 }
3427 tokio::time::sleep(Duration::from_millis(50)).await;
3428 }
3429}
3430
3431async fn stop_claude_pid_confirmed(entry: &RegistryEntry) -> bool {
3445 let Some(pid) = entry.pid else {
3446 return false;
3447 };
3448 if entry.pid_start_time.is_none() {
3454 return false;
3455 }
3456 if !pid_is_ours(pid, entry.pid_start_time) {
3457 return false;
3458 }
3459 unsafe {
3461 libc::kill(pid as libc::pid_t, libc::SIGTERM);
3462 }
3463 if pid_gone_within(pid, entry.pid_start_time, Duration::from_secs(5)).await {
3464 return true;
3465 }
3466 if pid_is_ours(pid, entry.pid_start_time) {
3467 unsafe {
3470 libc::kill(pid as libc::pid_t, libc::SIGKILL);
3471 }
3472 }
3473 pid_gone_within(pid, entry.pid_start_time, Duration::from_secs(2)).await
3474}
3475
3476async fn stop_claude(ctx: &Ctx, req: &Request, name: &str, entry: &RegistryEntry) -> Response {
3477 let short = match entry
3478 .transport_short()
3479 .or(entry.session_id.as_deref())
3480 .filter(|s| !s.is_empty())
3481 {
3482 Some(s) => s.to_string(),
3483 None => {
3484 if stop_claude_pid_confirmed(entry).await {
3487 let claude_name = name.to_string();
3488 if let Err(e) = update_registry_offloaded(ctx.home.registry_json(), move |r| {
3489 if let Some(e) = r.find_mut(&claude_name) {
3490 e.status = AgentStatus::Exited;
3491 }
3492 })
3493 .await
3494 {
3495 return Response::err(
3496 req.id,
3497 ErrorCode::Internal,
3498 format!("claude {name} stopped but registry write failed: {e}"),
3499 );
3500 }
3501 let _ = ctx.emitter.emit(
3502 "agent_stopped",
3503 &json!({"name": name, "backend": "claude", "stopped_by": "pid"}),
3504 );
3505 return Response::ok(
3506 req.id,
3507 json!({"stopped": true, "backend": "claude", "pid": entry.pid}),
3508 );
3509 }
3510 return Response::err(
3511 req.id,
3512 ErrorCode::InvalidStatus,
3513 format!(
3514 "agent {name} is claude but has no short id and no live process \
3515 to stop; note that `rm` clears the row but does not stop a session"
3516 ),
3517 );
3518 }
3519 };
3520 match tokio::process::Command::new("claude")
3521 .arg("stop")
3522 .arg(&short)
3523 .output()
3524 .await
3525 {
3526 Ok(o) if o.status.success() => {
3527 let claude_name = name.to_string();
3530 if let Err(e) = update_registry_offloaded(ctx.home.registry_json(), move |r| {
3531 if let Some(e) = r.find_mut(&claude_name) {
3532 e.status = AgentStatus::Exited;
3533 }
3534 })
3535 .await
3536 {
3537 return Response::err(
3538 req.id,
3539 ErrorCode::Internal,
3540 format!("claude {name} stopped but registry write failed: {e}"),
3541 );
3542 }
3543 let _ = ctx
3544 .emitter
3545 .emit("agent_stopped", &json!({"name": name, "backend": "claude"}));
3546 Response::ok(
3552 req.id,
3553 json!({"stopped": true, "backend": "claude", "short_id": short}),
3554 )
3555 }
3556 Ok(o) => Response::err(
3557 req.id,
3558 ErrorCode::Internal,
3559 format!(
3560 "claude stop {short} failed: {}",
3561 String::from_utf8_lossy(&o.stderr).trim()
3562 ),
3563 ),
3564 Err(e) => Response::err(
3565 req.id,
3566 ErrorCode::Internal,
3567 format!("could not exec `claude stop`: {e}"),
3568 ),
3569 }
3570}
3571
3572async fn handle_rm(ctx: &Ctx, req: &Request) -> Response {
3573 let requested_name = match req.params.get("name").and_then(|v| v.as_str()) {
3574 Some(n) => n.to_string(),
3575 None => return Response::err(req.id, ErrorCode::InvalidParams, "missing `name`"),
3576 };
3577 let force = req
3578 .params
3579 .get("force")
3580 .and_then(|v| v.as_bool())
3581 .unwrap_or(false);
3582 let registry = load_registry_offloaded(ctx.home.registry_json()).await;
3583 let entry =
3584 match entry_for_lifecycle(®istry, &requested_name, &ctx.home.registry_json()).await {
3585 Ok(Some(entry)) => entry,
3586 Ok(None) => {
3587 return Response::err(
3588 req.id,
3589 ErrorCode::AgentNotFound,
3590 format!("agent {requested_name} not found"),
3591 )
3592 }
3593 Err(message) => return Response::err(req.id, ErrorCode::InvalidParams, message),
3594 };
3595 let name = entry.name.clone();
3596 if entry.status == AgentStatus::Live && !force {
3597 return Response::err(
3598 req.id,
3599 ErrorCode::Busy,
3600 format!("agent {name} is still live; use `stop` first or pass --force"),
3601 );
3602 }
3603 if entry.status == AgentStatus::Live && force && !stop_worker_confirmed(ctx, &entry).await {
3606 return Response::err(
3607 req.id,
3608 ErrorCode::Internal,
3609 format!("agent {name}: could not stop the worker before force-remove; refusing to orphan a live PTY"),
3610 );
3611 }
3612 let was_orphaned = entry.status == AgentStatus::Orphaned;
3615 let rm_name = name.clone();
3620 if let Err(e) = update_registry_offloaded(ctx.home.registry_json(), move |r| {
3621 r.entries.retain(|e| e.name != rm_name);
3622 })
3623 .await
3624 {
3625 return Response::err(
3626 req.id,
3627 ErrorCode::Internal,
3628 format!("agent {name}: removal did not persist: {e}"),
3629 );
3630 }
3631 let _ = ctx.emitter.emit(
3632 "agent_removed",
3633 &json!({"name": name, "was_orphaned": was_orphaned}),
3634 );
3635 Response::ok(
3636 req.id,
3637 json!({"removed": true, "was_orphaned": was_orphaned}),
3638 )
3639}
3640
3641const RECONCILE_PROBE_TIMEOUT: Duration = Duration::from_millis(250);
3643const RECONCILE_SWEEP_BUDGET: Duration = Duration::from_secs(5);
3646
3647struct ReconcileChange {
3651 name: String,
3652 new_status: Option<AgentStatus>,
3653}
3654
3655#[derive(Default, PartialEq, Debug)]
3657struct ReconcileOutcome {
3658 updated: Vec<String>,
3659 orphans: Vec<String>,
3660 recovered: Vec<String>,
3661 inconsistent: Vec<(String, String)>,
3664 deferred: usize,
3666}
3667
3668fn plan_reconcile<P, D, L>(
3683 entries: &[RegistryEntry],
3684 mut probe: P,
3685 mut budget_exhausted: D,
3686 mut pid_live: L,
3687) -> (Vec<ReconcileChange>, ReconcileOutcome)
3688where
3689 P: FnMut(&RegistryEntry) -> Result<bool, crate::provider::ReachabilityProbeError>,
3690 D: FnMut() -> bool,
3691 L: FnMut(&RegistryEntry) -> bool,
3692{
3693 let mut changes = Vec::new();
3694 let mut out = ReconcileOutcome::default();
3695 for (i, entry) in entries.iter().enumerate() {
3696 if budget_exhausted() {
3697 out.deferred = entries.len() - i;
3698 break;
3699 }
3700 if entry.is_one_shot_ask() {
3709 let new_status = if is_non_terminal(entry.status) {
3710 out.updated.push(entry.name.clone());
3711 Some(AgentStatus::Exited)
3712 } else {
3713 None
3714 };
3715 changes.push(ReconcileChange {
3716 name: entry.name.clone(),
3717 new_status,
3718 });
3719 continue;
3720 }
3721 let new_status = match probe(entry) {
3722 Ok(true) => {
3723 if entry.status == AgentStatus::Orphaned && pid_live(entry) {
3731 out.recovered.push(entry.name.clone());
3732 out.updated.push(entry.name.clone());
3733 Some(AgentStatus::Live)
3734 } else {
3735 None
3736 }
3737 }
3738 Ok(false) if entry.is_interactive() => {
3739 if pid_live(entry) {
3745 None
3746 } else {
3747 out.updated.push(entry.name.clone());
3748 Some(AgentStatus::Exited)
3749 }
3750 }
3751 Ok(false) if entry.mux.is_some() => {
3752 if entry.pid.is_some() && pid_live(entry) {
3762 None
3763 } else if entry.pid.is_some() {
3764 out.updated.push(entry.name.clone());
3765 Some(AgentStatus::Exited)
3766 } else {
3767 let live_ish = matches!(
3768 entry.status,
3769 AgentStatus::Live
3770 | AgentStatus::Ready
3771 | AgentStatus::Idle
3772 | AgentStatus::Busy
3773 | AgentStatus::Spawning
3774 );
3775 if live_ish {
3776 out.orphans.push(entry.name.clone());
3777 out.updated.push(entry.name.clone());
3778 Some(AgentStatus::Orphaned)
3779 } else {
3780 None
3781 }
3782 }
3783 }
3784 Ok(false) => {
3785 let live_ish = matches!(
3792 entry.status,
3793 AgentStatus::Live
3794 | AgentStatus::Ready
3795 | AgentStatus::Idle
3796 | AgentStatus::Busy
3797 | AgentStatus::Spawning
3798 );
3799 if live_ish {
3800 out.orphans.push(entry.name.clone());
3801 out.updated.push(entry.name.clone());
3802 Some(AgentStatus::Orphaned)
3803 } else {
3804 None
3805 }
3806 }
3807 Err(e) => {
3808 out.inconsistent
3809 .push((entry.name.clone(), e.reason.clone()));
3810 None
3811 }
3812 };
3813 changes.push(ReconcileChange {
3814 name: entry.name.clone(),
3815 new_status,
3816 });
3817 }
3818 (changes, out)
3819}
3820
3821fn apply_reconcile_change(e: &mut RegistryEntry, new_status: Option<AgentStatus>, now: &str) {
3832 e.last_reconciled_at = Some(now.to_string());
3833 if let Some(s) = new_status {
3834 e.status = s;
3835 if matches!(s, AgentStatus::Exited) {
3836 e.pid = None;
3837 e.pid_start_time = None;
3838 e.inside_leg = None;
3844 e.screen_state = None;
3845 }
3846 }
3847}
3848
3849fn emit_inside_leg_completion(emitter: &EventEmitter, e: &RegistryEntry) {
3855 if let Some(rep) = &e.inside_leg {
3856 let _ = emitter.emit(
3857 "inside_leg_completed",
3858 &json!({
3859 "name": e.name,
3860 "session_id": e.session_id,
3861 "final_state": inside_leg_state_str(rep.state),
3862 "seq": rep.seq,
3863 }),
3864 );
3865 }
3866}
3867
3868fn inside_leg_state_str(state: state::InsideLegState) -> &'static str {
3872 match state {
3873 state::InsideLegState::Working => "working",
3874 state::InsideLegState::Blocked => "blocked",
3875 state::InsideLegState::Done => "done",
3876 }
3877}
3878
3879fn to_agent_entry(e: &RegistryEntry) -> crate::provider::AgentEntry {
3882 let session_id = match e.harness_name() {
3883 "codex" => e.codex_session_id.clone().or_else(|| e.session_id.clone()),
3884 "gemini" => e.gemini_session_id.clone().or_else(|| e.session_id.clone()),
3885 "claude" => e
3886 .transport_short()
3887 .map(str::to_string)
3888 .or_else(|| e.session_id.clone()),
3889 "opencode" => e
3894 .harness_session_id
3895 .clone()
3896 .or_else(|| e.session_id.clone()),
3897 _ => e.session_id.clone(),
3898 };
3899 crate::provider::AgentEntry {
3900 name: e.name.clone(),
3901 provider: e.harness_name().to_string(),
3902 session_id,
3903 cwd: PathBuf::from(&e.cwd),
3904 }
3905}
3906
3907struct ReconcileSweepResult {
3911 registry: crate::state::Registry,
3913 entries: Vec<RegistryEntry>,
3915 outcome: ReconcileOutcome,
3916}
3917
3918fn run_reconcile_sweep(
3928 home: &AgentsHome,
3929 emitter: &EventEmitter,
3930) -> Result<ReconcileSweepResult, String> {
3931 use crate::provider::ReachabilityProbeError;
3932 let registry = state::load_registry(&home.registry_json()).unwrap_or_default();
3933
3934 let mut entries = registry.entries.clone();
3937 entries.sort_by(|a, b| a.last_reconciled_at.cmp(&b.last_reconciled_at));
3938
3939 let start = Instant::now();
3940 let probe = |e: &RegistryEntry| -> Result<bool, ReachabilityProbeError> {
3941 if std::os::unix::net::UnixStream::connect(home.worker_sock(&e.short_id)).is_ok() {
3946 return Ok(true);
3947 }
3948 match crate::provider::for_name(e.harness_name()) {
3950 Some(p) => p.reachability(&to_agent_entry(e), RECONCILE_PROBE_TIMEOUT),
3951 None => Err(ReachabilityProbeError::new(
3952 e.harness_name(),
3953 "unknown provider; cannot probe reachability",
3954 )),
3955 }
3956 };
3957 let pid_live = |e: &RegistryEntry| -> bool {
3962 e.pid.map_or(true, |pid| pid_is_ours(pid, e.pid_start_time))
3963 };
3964 let (changes, outcome) = plan_reconcile(
3965 &entries,
3966 probe,
3967 || start.elapsed() >= RECONCILE_SWEEP_BUDGET,
3968 pid_live,
3969 );
3970
3971 for ch in &changes {
3976 if matches!(ch.new_status, Some(AgentStatus::Exited)) {
3977 if let Some(e) = registry.entries.iter().find(|e| e.name == ch.name) {
3978 emit_inside_leg_completion(emitter, e);
3979 }
3980 }
3981 }
3982
3983 let now = now_rfc3339_like();
3986 if let Err(err) = state::update_registry(&home.registry_json(), |r| {
3991 for ch in &changes {
3992 if let Some(e) = r.find_mut(&ch.name) {
3993 apply_reconcile_change(e, ch.new_status, &now);
3994 }
3995 }
3996 }) {
3997 let _ = emitter.emit("reconcile_error", &json!({"error": err.to_string()}));
3998 return Err(format!(
3999 "reconcile computed {} change(s) but the registry write failed: {err}",
4000 changes.len()
4001 ));
4002 }
4003
4004 for (name, reason) in &outcome.inconsistent {
4005 let _ = emitter.emit(
4006 "agent_inconsistent",
4007 &json!({"name": name, "reason": reason}),
4008 );
4009 }
4010 if outcome.deferred > 0 {
4011 let _ = emitter.emit(
4012 "reconcile_deferred",
4013 &json!({"remaining_count": outcome.deferred}),
4014 );
4015 }
4016 let _ = emitter.emit(
4017 "reconcile_done",
4018 &json!({
4019 "updated": outcome.updated.len(),
4020 "orphans": outcome.orphans.len(),
4021 "recovered": outcome.recovered.len(),
4022 }),
4023 );
4024 Ok(ReconcileSweepResult {
4025 registry,
4026 entries,
4027 outcome,
4028 })
4029}
4030
4031fn handle_reconcile(ctx: &Ctx, req: &Request) -> Response {
4032 let ReconcileSweepResult {
4033 registry,
4034 entries,
4035 outcome,
4036 } = match run_reconcile_sweep(&ctx.home, &ctx.emitter) {
4037 Ok(r) => r,
4038 Err(msg) => return Response::err(req.id, ErrorCode::Internal, msg),
4039 };
4040 let scanned = entries.len();
4059 let probed = entries.len() - outcome.deferred;
4065 let skipped_py: Vec<Value> = entries
4066 .iter()
4067 .skip(probed)
4068 .map(|e| json!({"name": e.name, "provider": e.harness_name()}))
4069 .collect();
4070 let orphaned_py: Vec<Value> = outcome
4071 .orphans
4072 .iter()
4073 .map(|n| {
4074 let prov = registry
4075 .entries
4076 .iter()
4077 .find(|e| &e.name == n)
4078 .map(|e| e.harness_name())
4079 .unwrap_or("unknown");
4080 json!({"name": n, "provider": prov})
4081 })
4082 .collect();
4083 let recovered_py: Vec<Value> = outcome
4084 .recovered
4085 .iter()
4086 .map(|n| {
4087 let prov = registry
4088 .entries
4089 .iter()
4090 .find(|e| &e.name == n)
4091 .map(|e| e.harness_name())
4092 .unwrap_or("unknown");
4093 json!({"name": n, "provider": prov})
4094 })
4095 .collect();
4096 let errors_py: Vec<Value> = outcome
4097 .inconsistent
4098 .iter()
4099 .map(|(n, reason)| json!({"name": n, "reason": reason}))
4100 .collect();
4101 Response::ok(
4102 req.id,
4103 json!({
4104 "scanned": scanned,
4106 "orphaned": orphaned_py,
4107 "recovered": recovered_py,
4108 "skipped": skipped_py,
4109 "errors": errors_py,
4110 "updated": outcome.updated,
4112 "orphans": outcome.orphans,
4113 "inconsistent": outcome.inconsistent.iter().map(|(n, _)| n.clone()).collect::<Vec<_>>(),
4114 "deferred": outcome.deferred,
4115 }),
4116 )
4117}
4118
4119enum BufferOutcome {
4134 Buffered,
4136 StaleSeq { last: u64 },
4138 Full,
4140}
4141
4142fn buffer_pending_report(
4147 map: &mut std::collections::HashMap<String, state::InsideLegReport>,
4148 session_id: &str,
4149 report: state::InsideLegReport,
4150) -> BufferOutcome {
4151 if let Some(prev) = map.get(session_id) {
4152 if report.seq <= prev.seq {
4153 return BufferOutcome::StaleSeq { last: prev.seq };
4154 }
4155 map.insert(session_id.to_string(), report);
4156 return BufferOutcome::Buffered;
4157 }
4158 if map.len() >= PENDING_INSIDE_LEG_CAP {
4159 return BufferOutcome::Full;
4160 }
4161 map.insert(session_id.to_string(), report);
4162 BufferOutcome::Buffered
4163}
4164
4165fn flush_buffered_inside_leg(ctx: &Ctx, session_uuid: &str, name: &str) {
4178 let rep = match ctx.pending_inside_leg.lock() {
4179 Ok(mut buf) => buf.remove(session_uuid),
4180 Err(_) => None,
4181 };
4182 let Some(rep) = rep else {
4183 return;
4184 };
4185 let (seq, state_str) = (rep.seq, inside_leg_state_str(rep.state));
4186 let (rep_state, rep_reason) = (rep.state, rep.reason.clone());
4190 let mut notify: Option<(String, String, bool)> = None;
4191 let _ = state::update_registry(&ctx.home.registry_json(), |r| {
4194 if let Some(e) = r
4195 .entries
4196 .iter_mut()
4197 .find(|e| entry_holds_session(e, session_uuid))
4198 {
4199 let newer = e.inside_leg.as_ref().is_none_or(|cur| rep.seq > cur.seq);
4200 if newer {
4201 let prev_state = e.inside_leg.as_ref().map(|r| r.state);
4202 let body = rep_reason.clone().unwrap_or_else(|| state_str.to_string());
4203 if state::enters(prev_state, rep_state, state::InsideLegState::Blocked) {
4204 notify = Some((name.to_string(), body, false));
4205 } else if state::enters(prev_state, rep_state, state::InsideLegState::Done) {
4206 notify = Some((name.to_string(), body, true));
4207 }
4208 e.inside_leg = Some(rep);
4209 e.screen_state = None;
4211 }
4212 }
4213 });
4214 if let Some((title, body, is_done)) = notify {
4215 let want = if is_done {
4216 ctx.opts.notify_on_done
4217 } else {
4218 ctx.opts.notify_on_blocked
4219 };
4220 if want {
4221 notify_transition(title, body);
4222 }
4223 }
4224 let _ = ctx.emitter.emit(
4225 "inside_leg_buffer_flushed",
4226 &json!({"name": name, "session_id": session_uuid, "state": state_str, "seq": seq}),
4227 );
4228}
4229
4230pub(crate) fn notify_transition(title: String, body: String) {
4239 let fno = std::env::var_os("FNO_BIN").unwrap_or_else(|| std::ffi::OsString::from("fno"));
4242 std::thread::spawn(move || {
4245 match std::process::Command::new(&fno)
4246 .args(["notify", &title, &body])
4247 .stdin(std::process::Stdio::null())
4248 .stdout(std::process::Stdio::null())
4249 .stderr(std::process::Stdio::null())
4250 .spawn()
4251 {
4252 Ok(mut child) => {
4253 let _ = child.wait();
4254 }
4255 Err(e) => eprintln!(
4256 "fno-agents-daemon: badge notify skipped ({} notify): {e}",
4257 fno.to_string_lossy()
4258 ),
4259 }
4260 });
4261}
4262
4263enum UuidBackfill {
4266 None,
4267 One(usize),
4268 Ambiguous,
4269}
4270
4271fn find_uuid_backfill_row(entries: &[RegistryEntry], full_uuid: &str) -> UuidBackfill {
4280 let mut found = None;
4281 for (i, e) in entries.iter().enumerate() {
4282 if e.harness_name() != "claude" || e.claude_session_uuid.is_some() {
4285 continue;
4286 }
4287 let Some(short) = e.transport_short() else {
4288 continue;
4289 };
4290 if short.is_empty()
4293 || !full_uuid
4294 .strip_prefix(short)
4295 .is_some_and(|rest| rest.starts_with('-'))
4296 {
4297 continue;
4298 }
4299 if found.is_some() {
4300 return UuidBackfill::Ambiguous;
4301 }
4302 found = Some(i);
4303 }
4304 found.map_or(UuidBackfill::None, UuidBackfill::One)
4305}
4306
4307fn handle_report(ctx: &Ctx, req: &Request) -> Response {
4308 let session_id = match req.params.get("session_id").and_then(|v| v.as_str()) {
4309 Some(s) if !s.is_empty() => s.to_string(),
4310 _ => return Response::err(req.id, ErrorCode::InvalidParams, "missing `session_id`"),
4311 };
4312 let seq = match req.params.get("seq").and_then(|v| v.as_u64()) {
4313 Some(n) => n,
4314 None => {
4315 return Response::err(
4316 req.id,
4317 ErrorCode::InvalidParams,
4318 "missing or non-integer `seq`",
4319 )
4320 }
4321 };
4322 let state_label = match req.params.get("state").and_then(|v| v.as_str()) {
4325 Some(s @ ("working" | "blocked" | "done")) => s.to_string(),
4326 _ => {
4327 return Response::err(
4328 req.id,
4329 ErrorCode::InvalidParams,
4330 "`state` must be working|blocked|done",
4331 )
4332 }
4333 };
4334 let state = match state_label.as_str() {
4335 "working" => state::InsideLegState::Working,
4336 "blocked" => state::InsideLegState::Blocked,
4337 _ => state::InsideLegState::Done,
4338 };
4339 let reason = req
4340 .params
4341 .get("reason")
4342 .and_then(|v| v.as_str())
4343 .map(String::from);
4344 let ttl_ms = req.params.get("ttl_ms").and_then(|v| v.as_u64());
4345
4346 let report = state::InsideLegReport {
4349 state,
4350 seq,
4351 reason,
4352 received_at: now_rfc3339_like(),
4353 ttl_ms,
4354 };
4355 let report_for_store = report.clone();
4356
4357 enum Outcome {
4360 Stored,
4361 StaleSeq { last: u64 },
4362 Unknown,
4363 }
4364 let mut outcome = Outcome::Unknown;
4365 let mut notify: Option<(String, String, bool)> = None;
4369 if let Err(e) = state::update_registry(&ctx.home.registry_json(), |r| {
4370 let idx = match r
4375 .entries
4376 .iter()
4377 .position(|e| entry_holds_session(e, &session_id))
4378 {
4379 Some(i) => Some(i),
4380 None => match find_uuid_backfill_row(&r.entries, &session_id) {
4381 UuidBackfill::One(i) => {
4382 r.entries[i].claude_session_uuid = Some(session_id.clone());
4383 Some(i)
4384 }
4385 UuidBackfill::None | UuidBackfill::Ambiguous => None,
4386 },
4387 };
4388 let Some(idx) = idx else {
4389 outcome = Outcome::Unknown;
4390 return;
4391 };
4392 let entry = &mut r.entries[idx];
4393 if let Some(prev) = &entry.inside_leg {
4394 if seq <= prev.seq {
4395 outcome = Outcome::StaleSeq { last: prev.seq };
4396 return;
4397 }
4398 }
4399 let prev_state = entry.inside_leg.as_ref().map(|r| r.state);
4400 if state::enters(prev_state, state, state::InsideLegState::Blocked) {
4401 let body = report_for_store
4402 .reason
4403 .clone()
4404 .unwrap_or_else(|| state_label.clone());
4405 notify = Some((entry.name.clone(), body, false));
4406 } else if state::enters(prev_state, state, state::InsideLegState::Done) {
4407 let body = report_for_store
4408 .reason
4409 .clone()
4410 .unwrap_or_else(|| state_label.clone());
4411 notify = Some((entry.name.clone(), body, true));
4412 }
4413 entry.inside_leg = Some(report_for_store);
4414 entry.screen_state = None;
4417 outcome = Outcome::Stored;
4418 }) {
4419 return Response::err(
4420 req.id,
4421 ErrorCode::Internal,
4422 format!("registry write failed during inside-leg report: {e}"),
4423 );
4424 }
4425
4426 match outcome {
4427 Outcome::Stored => {
4428 let _ = ctx.emitter.emit(
4429 "inside_leg_report",
4430 &json!({"session_id": session_id, "seq": seq, "state": state_label}),
4431 );
4432 if let Some((title, body, is_done)) = notify {
4433 let want = if is_done {
4434 ctx.opts.notify_on_done
4435 } else {
4436 ctx.opts.notify_on_blocked
4437 };
4438 if want {
4439 notify_transition(title, body);
4440 }
4441 }
4442 Response::ok(req.id, json!({"stored": true, "seq": seq}))
4443 }
4444 Outcome::StaleSeq { last } => {
4445 let _ = ctx.emitter.emit(
4446 "inside_leg_report_dropped",
4447 &json!({"session_id": session_id, "seq": seq, "last_seq": last, "reason": "stale_seq"}),
4448 );
4449 Response::ok(
4450 req.id,
4451 json!({"stored": false, "dropped": "stale_seq", "last_seq": last}),
4452 )
4453 }
4454 Outcome::Unknown => {
4461 let buffered = ctx
4462 .pending_inside_leg
4463 .lock()
4464 .map(|mut buf| buffer_pending_report(&mut buf, &session_id, report))
4465 .ok();
4466 match buffered {
4467 Some(BufferOutcome::Buffered) => {
4468 let _ = ctx.emitter.emit(
4469 "inside_leg_report_buffered",
4470 &json!({"session_id": session_id, "seq": seq, "state": state_label}),
4471 );
4472 Response::ok(
4473 req.id,
4474 json!({"stored": false, "buffered": true, "seq": seq}),
4475 )
4476 }
4477 Some(BufferOutcome::StaleSeq { last }) => {
4478 let _ = ctx.emitter.emit(
4479 "inside_leg_report_dropped",
4480 &json!({"session_id": session_id, "seq": seq, "last_seq": last, "reason": "stale_seq"}),
4481 );
4482 Response::ok(
4483 req.id,
4484 json!({"stored": false, "dropped": "stale_seq", "last_seq": last}),
4485 )
4486 }
4487 Some(BufferOutcome::Full) => {
4488 let _ = ctx.emitter.emit(
4489 "inside_leg_report_dropped",
4490 &json!({"session_id": session_id, "seq": seq, "reason": "buffer_full"}),
4491 );
4492 Response::ok(req.id, json!({"stored": false, "dropped": "buffer_full"}))
4493 }
4494 None => {
4497 let _ = ctx.emitter.emit(
4498 "inside_leg_report_dropped",
4499 &json!({"session_id": session_id, "seq": seq, "reason": "unknown_session"}),
4500 );
4501 Response::ok(
4502 req.id,
4503 json!({"stored": false, "dropped": "unknown_session"}),
4504 )
4505 }
4506 }
4507 }
4508 }
4509}
4510
4511async fn dispatch_channel(ctx: &Arc<Ctx>, req: &Request) -> Response {
4516 match Namespace::verb(&req.method) {
4518 Some("register_channel") => run_blocking(ctx, req, handle_register_channel).await,
4519 Some("unregister_channel") => run_blocking(ctx, req, handle_unregister_channel).await,
4520 Some("push_to_channel") => run_blocking(ctx, req, handle_push_to_channel).await,
4521 _ => Response::err(
4522 req.id,
4523 ErrorCode::UnknownMethod,
4524 format!("unknown channel verb in `{}`", req.method),
4525 ),
4526 }
4527}
4528
4529fn handle_register_channel(ctx: &Ctx, req: &Request) -> Response {
4530 let cc_session_id = match req.params.get("cc_session_id").and_then(|v| v.as_str()) {
4531 Some(s) => s.to_string(),
4532 None => return Response::err(req.id, ErrorCode::InvalidParams, "missing `cc_session_id`"),
4533 };
4534 let name = req
4536 .params
4537 .get("name")
4538 .and_then(|v| v.as_str())
4539 .map(String::from);
4540 let channel_id = uuid_v4();
4541 let mut matched = false;
4542 if let Err(e) = state::update_registry(&ctx.home.registry_json(), |r| {
4547 let target = match &name {
4548 Some(n) => r.find_mut(n),
4549 None => r
4550 .entries
4551 .iter_mut()
4552 .find(|e| e.cc_session_id.as_deref() == Some(&cc_session_id)),
4553 };
4554 if let Some(e) = target {
4555 e.cc_session_id = Some(cc_session_id.clone());
4556 e.mcp_channel_id = Some(channel_id.clone());
4557 matched = true;
4558 }
4559 }) {
4560 return Response::err(
4561 req.id,
4562 ErrorCode::Internal,
4563 format!("registry write failed during channel registration: {e}"),
4564 );
4565 }
4566 if !matched {
4567 return Response::err(
4568 req.id,
4569 ErrorCode::ChannelUnknown,
4570 "no agent matched cc_session_id/name for registration",
4571 );
4572 }
4573 let _ = ctx
4574 .emitter
4575 .emit("channel_registered", &json!({"mcp_channel_id": channel_id}));
4576 Response::ok(req.id, json!({"mcp_channel_id": channel_id}))
4577}
4578
4579fn handle_unregister_channel(ctx: &Ctx, req: &Request) -> Response {
4580 let channel_id = match req.params.get("mcp_channel_id").and_then(|v| v.as_str()) {
4581 Some(s) => s.to_string(),
4582 None => return Response::err(req.id, ErrorCode::InvalidParams, "missing `mcp_channel_id`"),
4583 };
4584 let mut cleared = false;
4585 let _ = state::update_registry(&ctx.home.registry_json(), |r| {
4586 for e in r.entries.iter_mut() {
4587 if e.mcp_channel_id.as_deref() == Some(&channel_id) {
4588 e.mcp_channel_id = None;
4589 cleared = true;
4590 }
4591 }
4592 });
4593 if !cleared {
4594 return Response::err(req.id, ErrorCode::ChannelUnknown, "unknown channel id");
4595 }
4596 Response::ok(req.id, json!({"unregistered": true}))
4597}
4598
4599fn handle_push_to_channel(ctx: &Ctx, req: &Request) -> Response {
4600 let channel_id = match req.params.get("mcp_channel_id").and_then(|v| v.as_str()) {
4601 Some(s) => s.to_string(),
4602 None => return Response::err(req.id, ErrorCode::InvalidParams, "missing `mcp_channel_id`"),
4603 };
4604 let envelope = match req.params.get("envelope") {
4607 None => None,
4608 Some(v @ Value::Object(_)) => Some(v.clone()),
4609 Some(_) => {
4610 return Response::err(
4611 req.id,
4612 ErrorCode::InvalidParams,
4613 "`envelope` must be a JSON object",
4614 )
4615 }
4616 };
4617 let registry = state::load_registry(&ctx.home.registry_json()).unwrap_or_default();
4618 let found = registry
4619 .entries
4620 .iter()
4621 .any(|e| e.mcp_channel_id.as_deref() == Some(&channel_id));
4622 if !found {
4623 return Response::err(
4624 req.id,
4625 ErrorCode::ChannelUnknown,
4626 "channel id not registered (channel server should re-register)",
4627 );
4628 }
4629 let envelope = match envelope {
4630 Some(e) => e,
4631 None => {
4632 return Response::ok(req.id, json!({"routed": true}));
4634 }
4635 };
4636 match deliver_envelope(&channel_id, &envelope) {
4641 Ok(()) => Response::ok(req.id, json!({"routed": true, "delivered": true})),
4642 Err(reason) => Response::ok(
4643 req.id,
4644 json!({"routed": true, "delivered": false, "reason": reason}),
4645 ),
4646 }
4647}
4648
4649fn deliver_envelope(channel_id: &str, envelope: &Value) -> Result<(), String> {
4653 use std::io::Write;
4654 use std::process::Stdio;
4655 let mut child = crate::loop_dispatch::fno_cmd("fno")
4656 .args(["mcp", "send", "--session-id", channel_id])
4657 .stdin(Stdio::piped())
4658 .stdout(Stdio::piped())
4659 .stderr(Stdio::piped())
4660 .spawn()
4661 .map_err(|e| format!("spawn `fno mcp send` failed: {e}"))?;
4662 {
4664 let mut stdin = child
4665 .stdin
4666 .take()
4667 .ok_or_else(|| "child stdin unavailable".to_string())?;
4668 let bytes = serde_json::to_vec(envelope).map_err(|e| format!("serialize envelope: {e}"))?;
4669 stdin
4670 .write_all(&bytes)
4671 .map_err(|e| format!("write envelope to `fno mcp send`: {e}"))?;
4672 }
4673 let out = child
4674 .wait_with_output()
4675 .map_err(|e| format!("wait for `fno mcp send`: {e}"))?;
4676 if out.status.success() {
4677 return Ok(());
4678 }
4679 let stderr = String::from_utf8_lossy(&out.stderr);
4680 let tail = stderr.trim().rsplit('\n').next().unwrap_or("").trim();
4681 Err(if tail.is_empty() {
4682 format!("`fno mcp send` exited {}", out.status)
4683 } else {
4684 tail.to_string()
4685 })
4686}
4687
4688fn json_obj(pairs: &[(&str, Value)]) -> Map<String, Value> {
4693 let mut m = Map::new();
4694 for (k, v) in pairs {
4695 m.insert((*k).to_string(), v.clone());
4696 }
4697 m
4698}
4699
4700fn now_compact() -> String {
4702 let secs = std::time::SystemTime::now()
4703 .duration_since(std::time::UNIX_EPOCH)
4704 .unwrap_or_default()
4705 .as_secs();
4706 let (y, mo, d, h, mi, s) = civil(secs);
4707 format!("{y:04}{mo:02}{d:02}T{h:02}{mi:02}{s:02}Z")
4708}
4709
4710pub(crate) fn now_rfc3339_like() -> String {
4712 let secs = std::time::SystemTime::now()
4713 .duration_since(std::time::UNIX_EPOCH)
4714 .unwrap_or_default()
4715 .as_secs();
4716 let (y, mo, d, h, mi, s) = civil(secs);
4717 format!("{y:04}-{mo:02}-{d:02}T{h:02}:{mi:02}:{s:02}Z")
4718}
4719
4720fn civil(secs: u64) -> (i64, u32, u32, u32, u32, u32) {
4721 let days = (secs / 86_400) as i64;
4722 let rem = secs % 86_400;
4723 let (hh, mm, ss) = (
4724 (rem / 3600) as u32,
4725 ((rem % 3600) / 60) as u32,
4726 (rem % 60) as u32,
4727 );
4728 let z = days + 719_468;
4729 let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
4730 let doe = z - era * 146_097;
4731 let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
4732 let y = yoe + era * 400;
4733 let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
4734 let mp = (5 * doy + 2) / 153;
4735 let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
4736 let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
4737 (if m <= 2 { y + 1 } else { y }, m, d, hh, mm, ss)
4738}
4739
4740fn uuid_v4() -> String {
4743 let mut b = [0u8; 16];
4744 fill_random(&mut b);
4745 b[6] = (b[6] & 0x0f) | 0x40; b[8] = (b[8] & 0x3f) | 0x80; format!(
4748 "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}",
4749 b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10], b[11], b[12], b[13],
4750 b[14], b[15]
4751 )
4752}
4753
4754fn fill_random(buf: &mut [u8]) {
4755 if let Ok(mut f) = std::fs::File::open("/dev/urandom") {
4759 use std::io::Read;
4760 if f.read_exact(buf).is_ok() {
4761 return;
4762 }
4763 }
4764 let seed = std::time::SystemTime::now()
4765 .duration_since(std::time::UNIX_EPOCH)
4766 .unwrap_or_default()
4767 .as_nanos() as u64
4768 ^ (std::process::id() as u64).rotate_left(17);
4769 let mut x = seed | 1;
4770 for byte in buf.iter_mut() {
4771 x ^= x << 13;
4773 x ^= x >> 7;
4774 x ^= x << 17;
4775 *byte = (x & 0xff) as u8;
4776 }
4777}
4778
4779#[cfg(test)]
4780mod tests {
4781 use super::*;
4782
4783 fn canonical_name_in(registry: &state::Registry, token: &str) -> String {
4785 let Ok(Value::Array(rows)) = serde_json::to_value(®istry.entries) else {
4786 return token.to_string();
4787 };
4788 match crate::client_verbs::find_agent_entry(&rows, token) {
4789 Ok(entry) => entry
4790 .get("name")
4791 .and_then(Value::as_str)
4792 .unwrap_or(token)
4793 .to_string(),
4794 Err(_) => token.to_string(),
4795 }
4796 }
4797 use crate::state::{AgentState, DriveWindow, PtyState};
4798
4799 fn tmp_home(tag: &str) -> AgentsHome {
4800 let mut p = std::env::temp_dir();
4801 p.push(format!(
4802 "fno-agents-daemon-{}-{}-{}",
4803 tag,
4804 std::process::id(),
4805 std::time::SystemTime::now()
4806 .duration_since(std::time::UNIX_EPOCH)
4807 .unwrap()
4808 .as_nanos()
4809 ));
4810 let home = AgentsHome::at(&p);
4811 home.ensure_root().unwrap();
4812 home
4813 }
4814
4815 fn read_events(home: &AgentsHome) -> Vec<Value> {
4816 std::fs::read_to_string(home.events_jsonl())
4817 .unwrap_or_default()
4818 .lines()
4819 .filter_map(|l| serde_json::from_str::<Value>(l).ok())
4820 .collect()
4821 }
4822
4823 fn ask_row(name: &str, exited_at: Option<&str>) -> RegistryEntry {
4826 RegistryEntry {
4827 name: name.into(),
4828 short_id: String::new(),
4829 legacy_provider: "claude".into(),
4830 harness: None,
4831 harness_session_id: None,
4832 cwd: "/tmp".into(),
4833 project_root: String::new(),
4834 session_id: None,
4835 legacy_claude_short_id: None,
4836 claude_session_uuid: None,
4837 messaging_socket_path: None,
4838 codex_session_id: None,
4839 gemini_session_id: None,
4840 mcp_channel_id: None,
4841 cc_session_id: None,
4842 host_mode: None,
4843 status: AgentStatus::Exited,
4844 last_message_at: None,
4845 created_at: "2020-01-01T00:00:00Z".into(),
4846 pid: None,
4847 pid_start_time: None,
4848 log_path: None,
4849 last_reconciled_at: None,
4850 inside_leg: None,
4851 exited_at: exited_at.map(str::to_string),
4852 mux: None,
4853 screen_state: None,
4854 crown_level: None,
4855 crown_scope: None,
4856 crown_grantor: None,
4857 }
4858 }
4859
4860 #[test]
4861 fn gc_sweep_reaps_stamped_stamps_unstamped_keeps_live() {
4862 let home = tmp_home("gc-sweep");
4863 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
4864
4865 state::update_registry(&home.registry_json(), |r| {
4866 r.entries
4869 .push(ask_row("ask-old", Some("2020-01-01T00:00:00Z")));
4870 r.entries.push(ask_row("ask-new", None));
4872 let mut live = ask_row("live", None);
4875 live.name = "live".into();
4876 live.short_id = "wkL".into();
4877 live.status = AgentStatus::Live;
4878 live.pid = Some(std::process::id());
4879 r.entries.push(live);
4880 })
4881 .unwrap();
4882
4883 let summary = gc_sweep(&home, &emitter, Duration::from_secs(3600));
4884
4885 assert_eq!(summary.reaped, vec!["ask-old".to_string()]);
4886
4887 let reg = state::load_registry(&home.registry_json()).unwrap();
4888 let names: Vec<&str> = reg.entries.iter().map(|e| e.name.as_str()).collect();
4889 assert!(!names.contains(&"ask-old"), "ask-old should be reaped");
4890 assert!(
4891 names.contains(&"ask-new"),
4892 "ask-new should be kept (in grace)"
4893 );
4894 assert!(names.contains(&"live"), "live row must never be reaped");
4895
4896 let new = reg.entries.iter().find(|e| e.name == "ask-new").unwrap();
4898 assert!(
4899 new.exited_at.is_some(),
4900 "ask-new should be stamped this pass"
4901 );
4902 let live = reg.entries.iter().find(|e| e.name == "live").unwrap();
4903 assert!(live.exited_at.is_none());
4904
4905 let events = read_events(&home);
4907 let reaped: Vec<&Value> = events
4908 .iter()
4909 .filter(|e| e.get("type").and_then(Value::as_str) == Some("agent_row_reaped"))
4910 .collect();
4911 assert_eq!(reaped.len(), 1);
4912 assert_eq!(
4913 reaped[0]
4914 .get("data")
4915 .and_then(|d| d.get("name"))
4916 .and_then(Value::as_str),
4917 Some("ask-old")
4918 );
4919 }
4920
4921 #[test]
4922 fn gc_sweep_empty_registry_is_noop() {
4923 let home = tmp_home("gc-empty");
4924 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
4925 let summary = gc_sweep(&home, &emitter, Duration::from_secs(3600));
4926 assert!(summary.reaped.is_empty());
4927 assert!(summary.kept_dirty.is_empty());
4928 }
4929
4930 #[test]
4931 fn gc_sweep_turns_unterminated_node_reap_into_durable_failure() {
4932 let sandbox = tmp_home("gc-dead-dispatch");
4933 let home = AgentsHome::at(sandbox.root().join("agents"));
4934 home.ensure_root().unwrap();
4935 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
4936 let dead_repo = home.root().join("dead-repo");
4937 let done_repo = home.root().join("done-repo");
4938 for repo in [&dead_repo, &done_repo] {
4939 std::fs::create_dir_all(repo.join(".fno")).unwrap();
4940 assert!(std::process::Command::new("git")
4941 .args(["init", "-q"])
4942 .current_dir(repo)
4943 .status()
4944 .unwrap()
4945 .success());
4946 }
4947
4948 let dead_session = "target-run-dead";
4949 let done_session = "target-run-done";
4950 std::fs::write(
4951 dead_repo.join(".fno/target-state.md"),
4952 format!("---\nfno_id: {dead_session}\ninput: x-a35a\nplan_path: \"\"\n---\n"),
4953 )
4954 .unwrap();
4955 std::fs::write(
4956 done_repo.join(".fno/target-state.md"),
4957 format!("---\nfno_id: {done_session}\ninput: x-b44e\nplan_path: \"\"\n---\n"),
4958 )
4959 .unwrap();
4960 state::update_registry(&home.registry_json(), |r| {
4961 let mut dead = bg_claude_row("target-x-a35a-route-atomicity", "dead0001");
4962 dead.status = AgentStatus::Exited;
4963 dead.cwd = dead_repo.to_string_lossy().into_owned();
4964 dead.exited_at = Some("2020-01-01T00:00:00Z".into());
4965 dead.harness_session_id = Some("harness-dead-uuid".into());
4966 r.entries.push(dead);
4967
4968 let mut done = bg_claude_row("target-x-b44e-finished", "done0002");
4969 done.status = AgentStatus::Exited;
4970 done.cwd = done_repo.to_string_lossy().into_owned();
4971 done.exited_at = Some("2020-01-01T00:00:00Z".into());
4972 done.harness_session_id = Some("harness-done-uuid".into());
4973 r.entries.push(done);
4974 })
4975 .unwrap();
4976
4977 let global_events = home.root().parent().unwrap().join("events.jsonl");
4978 std::fs::write(
4979 done_repo.join(".fno/events.jsonl.1"),
4980 format!(
4981 "{{\"ts\":\"2026-07-24T00:00:00Z\",\"type\":\"termination\",\"source\":\"loop\",\"data\":{{\"session_id\":\"{done_session}\",\"reason\":\"DonePRGreen\",\"message\":\"done\"}}}}\n"
4982 ),
4983 )
4984 .unwrap();
4985
4986 let summary = gc_sweep(&home, &emitter, Duration::from_secs(0));
4987 assert_eq!(summary.reaped.len(), 2);
4988
4989 let reaps = read_events(&home);
4990 let dead_reap = reaps
4991 .iter()
4992 .find(|e| e["data"]["short_id"] == "dead0001")
4993 .expect("dead dispatch reap event");
4994 assert_eq!(dead_reap["data"]["node_id"], "x-a35a");
4995 assert_eq!(dead_reap["data"]["termination_event"], false);
4996 let done_reap = reaps
4997 .iter()
4998 .find(|e| e["data"]["short_id"] == "done0002")
4999 .expect("completed dispatch reap event");
5000 assert_eq!(done_reap["data"]["node_id"], "x-b44e");
5001 assert_eq!(done_reap["data"]["termination_event"], true);
5002
5003 let global = std::fs::read_to_string(&global_events).unwrap();
5004 let failures: Vec<Value> = global
5005 .lines()
5006 .filter_map(|line| serde_json::from_str(line).ok())
5007 .filter(|e: &Value| e["type"] == "node_failed")
5008 .collect();
5009 assert_eq!(failures.len(), 1);
5010 assert_eq!(failures[0]["data"]["unit_id"], "x-a35a");
5011 assert_eq!(failures[0]["data"]["session_id"], dead_session);
5012 assert_eq!(
5013 failures[0]["data"]["reason"],
5014 "agent-row-reaped-no-termination"
5015 );
5016 }
5017
5018 #[test]
5019 fn gc_sweep_restores_row_when_termination_evidence_is_unknown() {
5020 let sandbox = tmp_home("gc-unknown-termination");
5021 let home = AgentsHome::at(sandbox.root().join("agents"));
5022 home.ensure_root().unwrap();
5023 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
5024 let repo = home.root().join("repo");
5025 std::fs::create_dir_all(repo.join(".fno")).unwrap();
5026 assert!(std::process::Command::new("git")
5027 .args(["init", "-q"])
5028 .current_dir(&repo)
5029 .status()
5030 .unwrap()
5031 .success());
5032 std::fs::write(
5033 repo.join(".fno/target-state.md"),
5034 "---\nfno_id: reused-run\ninput: x-other\nplan_path: \"\"\n---\n",
5035 )
5036 .unwrap();
5037 state::update_registry(&home.registry_json(), |registry| {
5038 let mut row = bg_claude_row("target-x-a35a-route-atomicity", "dead0001");
5039 row.status = AgentStatus::Exited;
5040 row.cwd = repo.to_string_lossy().into_owned();
5041 row.exited_at = Some("2020-01-01T00:00:00Z".into());
5042 registry.entries.push(row);
5043 })
5044 .unwrap();
5045
5046 let summary = gc_sweep(&home, &emitter, Duration::from_secs(0));
5047
5048 assert!(summary.reaped.is_empty());
5049 let registry = state::load_registry(&home.registry_json()).unwrap();
5050 assert!(registry
5051 .entries
5052 .iter()
5053 .any(|row| row.name == "target-x-a35a-route-atomicity"));
5054 let events = read_events(&home);
5055 assert!(events.iter().any(|event| {
5056 event["type"] == "daemon_recovery_error"
5057 && event["data"]["op"] == "observe_dead_dispatch_termination"
5058 }));
5059 }
5060
5061 #[test]
5062 fn gc_sweep_restores_row_when_dead_dispatch_receipt_cannot_persist() {
5063 let sandbox = tmp_home("gc-dead-dispatch-write-failure");
5064 let home = AgentsHome::at(sandbox.root().join("agents"));
5065 home.ensure_root().unwrap();
5066 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
5067 let repo = home.root().join("repo");
5068 std::fs::create_dir_all(&repo).unwrap();
5069 assert!(std::process::Command::new("git")
5070 .args(["init", "-q"])
5071 .current_dir(&repo)
5072 .status()
5073 .unwrap()
5074 .success());
5075 state::update_registry(&home.registry_json(), |registry| {
5076 let mut row = bg_claude_row("target-x-a35a-route-atomicity", "dead0001");
5077 row.status = AgentStatus::Exited;
5078 row.cwd = repo.to_string_lossy().into_owned();
5079 row.exited_at = Some("2020-01-01T00:00:00Z".into());
5080 registry.entries.push(row);
5081 })
5082 .unwrap();
5083 std::fs::create_dir_all(global_events_path(&home)).unwrap();
5084
5085 let summary = gc_sweep(&home, &emitter, Duration::from_secs(0));
5086
5087 assert!(summary.reaped.is_empty());
5088 let registry = state::load_registry(&home.registry_json()).unwrap();
5089 assert!(registry
5090 .entries
5091 .iter()
5092 .any(|row| row.name == "target-x-a35a-route-atomicity"));
5093 let events = read_events(&home);
5094 assert!(events.iter().any(|event| {
5095 event["type"] == "daemon_recovery_error"
5096 && event["data"]["op"] == "record_dead_dispatch"
5097 }));
5098 }
5099
5100 #[test]
5101 fn recovery_emits_drive_crashed_before_clearing_window() {
5102 let home = tmp_home("recover-drive");
5103 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
5104
5105 state::update_registry(&home.registry_json(), |r| {
5107 r.entries.push(RegistryEntry {
5108 name: "worker-A".into(),
5109 short_id: "wkA".into(),
5110 legacy_provider: "codex".into(),
5111 harness: None,
5112 harness_session_id: None,
5113 cwd: "/tmp".into(),
5114 project_root: "/tmp".into(),
5115 session_id: None,
5116 legacy_claude_short_id: None,
5117 claude_session_uuid: None,
5118 messaging_socket_path: None,
5119 codex_session_id: None,
5120 gemini_session_id: None,
5121 mcp_channel_id: None,
5122 cc_session_id: None,
5123 host_mode: None,
5124 status: AgentStatus::Live,
5125 last_message_at: None,
5126 created_at: "2026-05-24T00:00:00Z".into(),
5127 pid: Some(std::process::id()), pid_start_time: None,
5129 log_path: None,
5130 last_reconciled_at: None,
5131 inside_leg: None,
5132 exited_at: None,
5133 mux: None,
5134 screen_state: None,
5135 crown_level: None,
5136 crown_scope: None,
5137 crown_grantor: None,
5138 });
5139 })
5140 .unwrap();
5141 let mut st = AgentState::new_pty("wkA");
5142 st.status = AgentStatus::Live;
5143 st.pty = Some(PtyState {
5144 active: true,
5145 drive: Some(DriveWindow {
5146 session_id: Some("drive-xyz".into()),
5147 mode: Some("interactive".into()),
5148 last_heartbeat_at_monotonic_ns: Some(123),
5149 }),
5150 });
5151 state::write_state_atomic(&home.state_json("wkA"), &st).unwrap();
5152
5153 let report = recover(&home, &emitter);
5154 assert_eq!(report.recovered_drives, vec!["wkA".to_string()]);
5155
5156 let events = read_events(&home);
5158 let crashed = events
5159 .iter()
5160 .find(|e| e["type"] == "drive_crashed")
5161 .expect("drive_crashed emitted");
5162 assert_eq!(crashed["data"]["session_id"], "drive-xyz");
5163 assert_eq!(crashed["data"]["reason"], "daemon_restart");
5164
5165 let after = state::load_state(&home.state_json("wkA")).unwrap().unwrap();
5167 let pty = after.pty.unwrap();
5168 assert!(pty.drive.is_none());
5169 std::fs::remove_dir_all(home.root()).ok();
5170 }
5171
5172 #[test]
5173 fn recovery_marks_missing_state_inconsistent() {
5174 let home = tmp_home("recover-missing");
5175 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
5176 state::update_registry(&home.registry_json(), |r| {
5177 r.entries.push(RegistryEntry {
5178 name: "ghost".into(),
5179 short_id: "ghost".into(),
5180 legacy_provider: "codex".into(),
5181 harness: None,
5182 harness_session_id: None,
5183 cwd: "/tmp".into(),
5184 project_root: "/tmp".into(),
5185 session_id: None,
5186 legacy_claude_short_id: None,
5187 claude_session_uuid: None,
5188 messaging_socket_path: None,
5189 codex_session_id: None,
5190 gemini_session_id: None,
5191 mcp_channel_id: None,
5192 cc_session_id: None,
5193 host_mode: None,
5194 status: AgentStatus::Live,
5195 last_message_at: None,
5196 created_at: "2026-05-24T00:00:00Z".into(),
5197 pid: None,
5198 pid_start_time: None,
5199 log_path: None,
5200 last_reconciled_at: None,
5201 inside_leg: None,
5202 exited_at: None,
5203 mux: None,
5204 screen_state: None,
5205 crown_level: None,
5206 crown_scope: None,
5207 crown_grantor: None,
5208 });
5209 })
5210 .unwrap();
5211 let report = recover(&home, &emitter);
5213 assert_eq!(
5214 report.inconsistent,
5215 vec![("ghost".to_string(), InconsistencyReason::MissingStateJson)]
5216 );
5217 let events = read_events(&home);
5218 assert!(events
5219 .iter()
5220 .any(|e| e["type"] == "agent_inconsistent"
5221 && e["data"]["reason"] == "missing_state_json"));
5222 std::fs::remove_dir_all(home.root()).ok();
5223 }
5224
5225 #[test]
5226 fn recovery_skips_claude_shellout_rows_no_spurious_inconsistent() {
5227 let home = tmp_home("recover-claude-shellout");
5233 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
5234 state::update_registry(&home.registry_json(), |r| {
5235 let mut bg = bg_claude_row("bg-ask", "7c5dcf5d");
5237 bg.host_mode = None;
5238 r.entries.push(bg);
5239 let mut adopted = bg_claude_row("cc-adopt", "deadbeef");
5241 adopted.host_mode = Some(crate::state::HOST_MODE_ATTACHED.into());
5242 adopted.pid = Some(4242);
5243 r.entries.push(adopted);
5244 })
5245 .unwrap();
5246 let report = recover(&home, &emitter);
5248 assert!(
5249 report.inconsistent.is_empty(),
5250 "claude shellout/adopted rows must not be flagged inconsistent: {:?}",
5251 report.inconsistent
5252 );
5253 let events = read_events(&home);
5254 assert!(
5255 !events.iter().any(|e| e["type"] == "agent_inconsistent"),
5256 "no agent_inconsistent event for claude shellout rows"
5257 );
5258 std::fs::remove_dir_all(home.root()).ok();
5259 }
5260
5261 #[test]
5262 fn canonical_name_in_resolves_all_three_address_forms() {
5263 let full = "aabbccdd-1111-2222-3333-444455556666";
5268 let mut row = rentry("billing", AgentStatus::Live, None);
5269 row.short_id = "a1b2c3d4".into();
5270 row.harness_session_id = Some(full.into());
5271 let reg = crate::state::Registry {
5272 schema_version: crate::state::REGISTRY_SCHEMA_VERSION,
5273 entries: vec![row],
5274 };
5275 assert_eq!(canonical_name_in(®, "billing"), "billing"); assert_eq!(canonical_name_in(®, "a1b2c3d4"), "billing"); assert_eq!(canonical_name_in(®, full), "billing"); assert_eq!(
5279 canonical_name_in(®, "AABBCCDD-1111-2222-3333-444455556666"),
5280 "billing"
5281 ); assert_eq!(canonical_name_in(®, "nope"), "nope");
5284 }
5285
5286 #[tokio::test]
5287 async fn lifecycle_name_resolution_never_falls_back_on_ambiguity() {
5288 let mut named = rentry("deadbeef", AgentStatus::Live, None);
5289 named.short_id = "transport-a".into();
5290 named.harness_session_id = Some("aaaaaaaa-1111-2222-3333-444455556666".into());
5291 let mut short = rentry("other", AgentStatus::Live, None);
5292 short.short_id = "deadbeef".into();
5293 short.harness_session_id = Some("bbbbbbbb-1111-2222-3333-000000000002".into());
5294 let reg = crate::state::Registry {
5295 schema_version: crate::state::REGISTRY_SCHEMA_VERSION,
5296 entries: vec![named, short],
5297 };
5298
5299 let error = entry_for_lifecycle(
5300 ®,
5301 "deadbeef",
5302 std::path::Path::new("/nonexistent/registry.json"),
5303 )
5304 .await
5305 .expect_err("ambiguous token must not fall back to the matching row name");
5306
5307 assert!(error.contains("ambiguous across 2 agents"));
5308 }
5309
5310 #[test]
5311 fn recovery_reaps_dead_pid() {
5312 let home = tmp_home("recover-reap");
5313 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
5314 state::update_registry(&home.registry_json(), |r| {
5315 r.entries.push(RegistryEntry {
5316 name: "dead".into(),
5317 short_id: "dead".into(),
5318 legacy_provider: "codex".into(),
5319 harness: None,
5320 harness_session_id: None,
5321 cwd: "/tmp".into(),
5322 project_root: "/tmp".into(),
5323 session_id: None,
5324 legacy_claude_short_id: None,
5325 claude_session_uuid: None,
5326 messaging_socket_path: None,
5327 codex_session_id: None,
5328 gemini_session_id: None,
5329 mcp_channel_id: None,
5330 cc_session_id: None,
5331 host_mode: None,
5332 status: AgentStatus::Live,
5333 last_message_at: None,
5334 created_at: "2026-05-24T00:00:00Z".into(),
5335 pid: Some(0x7fff_fff0),
5337 pid_start_time: None,
5338 log_path: None,
5339 last_reconciled_at: None,
5340 inside_leg: None,
5341 exited_at: None,
5342 mux: None,
5343 screen_state: None,
5344 crown_level: None,
5345 crown_scope: None,
5346 crown_grantor: None,
5347 });
5348 })
5349 .unwrap();
5350 let mut st = AgentState::new_pty("dead");
5352 st.status = AgentStatus::Live;
5353 state::write_state_atomic(&home.state_json("dead"), &st).unwrap();
5354
5355 let report = recover(&home, &emitter);
5356 assert_eq!(report.reaped_pids, vec![0x7fff_fff0]);
5357 let reg = state::load_registry(&home.registry_json()).unwrap();
5358 assert_eq!(reg.find("dead").unwrap().status, AgentStatus::Exited);
5359 std::fs::remove_dir_all(home.root()).ok();
5360 }
5361
5362 #[test]
5363 fn recovery_marks_dead_interactive_exited_and_preserves_host_mode() {
5364 let home = tmp_home("recover-interactive");
5369 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
5370 state::update_registry(&home.registry_json(), |r| {
5371 let mut e = rentry("hosted", AgentStatus::Live, None);
5372 e.host_mode = Some(crate::state::HOST_MODE_INTERACTIVE.to_string());
5373 e.pid = Some(0x7fff_fff0); r.entries.push(e);
5375 })
5376 .unwrap();
5377 let mut st = AgentState::new_pty("hosted");
5378 st.status = AgentStatus::Live;
5379 state::write_state_atomic(&home.state_json("hosted"), &st).unwrap();
5380
5381 let _ = recover(&home, &emitter);
5382 let reg = state::load_registry(&home.registry_json()).unwrap();
5383 let row = reg.find("hosted").unwrap();
5384 assert_eq!(
5385 row.status,
5386 AgentStatus::Exited,
5387 "a dead interactive worker is exited, never orphaned"
5388 );
5389 assert_eq!(
5390 row.host_mode_or_default(),
5391 crate::state::HOST_MODE_INTERACTIVE,
5392 "host_mode must survive recovery"
5393 );
5394 std::fs::remove_dir_all(home.root()).ok();
5395 }
5396
5397 #[test]
5398 fn pid_is_ours_distinguishes_recycled_pid() {
5399 let me = std::process::id();
5402 let Some(st) = process_start_time(me) else {
5403 return; };
5405 assert!(pid_is_ours(me, Some(st)), "correct start time -> ours");
5406 assert!(
5407 !pid_is_ours(me, Some(st.wrapping_add(1))),
5408 "alive but mismatched start time -> recycled, not ours"
5409 );
5410 assert!(
5411 !pid_is_ours(0x7fff_fff0, Some(st)),
5412 "dead pid is never ours"
5413 );
5414 assert!(
5415 pid_is_ours(me, None),
5416 "no recorded start time -> fall back to bare liveness (legacy)"
5417 );
5418 }
5419
5420 #[tokio::test]
5421 async fn stop_claude_pid_kills_a_real_child_and_spares_a_recycled_pid() {
5422 let mut entry = ask_row("orphan", None);
5426
5427 assert!(
5429 !stop_claude_pid_confirmed(&entry).await,
5430 "no pid -> nothing to stop"
5431 );
5432
5433 let out = std::process::Command::new("sh")
5440 .arg("-c")
5441 .arg("sleep 60 >/dev/null 2>&1 & echo $!")
5445 .output()
5446 .expect("spawn detached sleeper");
5447 let pid: u32 = String::from_utf8_lossy(&out.stdout)
5448 .trim()
5449 .parse()
5450 .expect("sleeper pid");
5451 let start = process_start_time(pid);
5452
5453 let ps_says_alive = |pid: u32| {
5458 std::process::Command::new("ps")
5459 .args(["-p", &pid.to_string()])
5460 .output()
5461 .map(|o| {
5462 String::from_utf8_lossy(&o.stdout)
5463 .lines()
5464 .filter(|l| l.split_whitespace().next() == Some(&pid.to_string()))
5465 .count()
5466 > 0
5467 })
5468 .unwrap_or(false)
5469 };
5470
5471 entry.pid = Some(pid);
5473 entry.pid_start_time = None;
5474 assert!(
5475 !stop_claude_pid_confirmed(&entry).await,
5476 "no start token -> refuse"
5477 );
5478 assert!(ps_says_alive(pid), "a refused row must not be signalled");
5479
5480 if let Some(st) = start {
5482 entry.pid_start_time = Some(st.wrapping_add(1));
5483 assert!(
5484 !stop_claude_pid_confirmed(&entry).await,
5485 "recycled pid -> refuse"
5486 );
5487 assert!(
5488 ps_says_alive(pid),
5489 "an unrelated process must not be signalled"
5490 );
5491 }
5492
5493 entry.pid_start_time = start;
5495 if start.is_some() {
5496 assert!(
5497 stop_claude_pid_confirmed(&entry).await,
5498 "owned live pid -> stopped"
5499 );
5500 assert!(!ps_says_alive(pid), "process is gone");
5501 } else {
5502 unsafe {
5505 libc::kill(pid as libc::pid_t, libc::SIGKILL);
5506 }
5507 }
5508 }
5509
5510 #[test]
5511 fn pid_is_ours_rejects_an_out_of_range_pid() {
5512 assert!(!pid_is_ours(u32::MAX, None), "u32::MAX must never be ours");
5517 assert!(
5518 !pid_is_ours(i32::MAX as u32 + 1, Some(123)),
5519 "anything past i32::MAX wraps negative"
5520 );
5521 assert!(
5522 pid_confirmed_dead(u32::MAX),
5523 "out-of-range is never running"
5524 );
5525 }
5526
5527 #[test]
5528 fn recycle_and_death_each_demand_positive_evidence() {
5529 let me = std::process::id();
5533 let Some(st) = process_start_time(me) else {
5534 return; };
5536
5537 assert!(!pid_confirmed_dead(me), "a live pid is not dead");
5539 assert!(
5540 !pid_recycled(me, Some(st)),
5541 "matching token is not a recycle"
5542 );
5543
5544 assert!(
5546 pid_recycled(me, Some(st.wrapping_add(1))),
5547 "reachable + differing token is a recycle"
5548 );
5549
5550 assert!(!pid_recycled(me, None), "no token -> no recycle verdict");
5552
5553 let dead = 0x7fff_fff0u32;
5556 assert!(pid_confirmed_dead(dead), "unused high pid reads as dead");
5557 assert!(
5558 !pid_recycled(dead, Some(st)),
5559 "dead is not a recycle finding"
5560 );
5561 }
5562
5563 #[test]
5564 fn recovery_reaps_recycled_pid() {
5565 let home = tmp_home("recover-recycled");
5570 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
5571 let me = std::process::id();
5572 if process_start_time(me).is_none() {
5573 std::fs::remove_dir_all(home.root()).ok();
5574 return; }
5576 state::update_registry(&home.registry_json(), |r| {
5577 r.entries.push(RegistryEntry {
5578 name: "recycled".into(),
5579 short_id: "recycled".into(),
5580 legacy_provider: "codex".into(),
5581 harness: None,
5582 harness_session_id: None,
5583 cwd: "/tmp".into(),
5584 project_root: "/tmp".into(),
5585 session_id: None,
5586 legacy_claude_short_id: None,
5587 claude_session_uuid: None,
5588 messaging_socket_path: None,
5589 codex_session_id: None,
5590 gemini_session_id: None,
5591 mcp_channel_id: None,
5592 cc_session_id: None,
5593 host_mode: None,
5594 status: AgentStatus::Live,
5595 last_message_at: None,
5596 created_at: "2026-05-24T00:00:00Z".into(),
5597 pid: Some(me),
5598 pid_start_time: Some(1),
5600 log_path: None,
5601 last_reconciled_at: None,
5602 inside_leg: None,
5603 exited_at: None,
5604 mux: None,
5605 screen_state: None,
5606 crown_level: None,
5607 crown_scope: None,
5608 crown_grantor: None,
5609 });
5610 })
5611 .unwrap();
5612 let mut st = AgentState::new_pty("recycled");
5613 st.status = AgentStatus::Live;
5614 state::write_state_atomic(&home.state_json("recycled"), &st).unwrap();
5615
5616 let report = recover(&home, &emitter);
5617 assert_eq!(report.reaped_pids, vec![me]);
5618 let reg = state::load_registry(&home.registry_json()).unwrap();
5619 assert_eq!(reg.find("recycled").unwrap().status, AgentStatus::Exited);
5620 std::fs::remove_dir_all(home.root()).ok();
5621 }
5622
5623 #[test]
5624 fn recovery_archives_orphan_state_dir() {
5625 let home = tmp_home("recover-orphan");
5626 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
5627 let mut st = AgentState::new_pty("loner");
5629 st.status = AgentStatus::Live;
5630 state::write_state_atomic(&home.state_json("loner"), &st).unwrap();
5631
5632 let report = recover(&home, &emitter);
5633 assert_eq!(report.archived_orphans, vec!["loner".to_string()]);
5634 assert!(!home.agent_dir("loner").exists(), "orphan dir moved aside");
5635 assert!(home.orphaned_dir().exists());
5636 std::fs::remove_dir_all(home.root()).ok();
5637 }
5638
5639 #[test]
5640 fn agent_name_validation() {
5641 assert!(valid_agent_name("worker-A_1"));
5642 assert!(!valid_agent_name(""));
5643 assert!(!valid_agent_name(&"x".repeat(65)));
5644 assert!(!valid_agent_name("has space"));
5645 assert!(!valid_agent_name("inject;rm"));
5646 }
5647
5648 #[test]
5649 fn uuid_v4_shape_and_uniqueness() {
5650 let a = uuid_v4();
5651 let b = uuid_v4();
5652 assert_ne!(a, b);
5653 assert_eq!(a.len(), 36);
5654 let parts: Vec<&str> = a.split('-').collect();
5655 assert_eq!(
5656 parts.iter().map(|p| p.len()).collect::<Vec<_>>(),
5657 vec![8, 4, 4, 4, 12]
5658 );
5659 assert_eq!(&a[14..15], "4");
5661 assert!(matches!(&a[19..20], "8" | "9" | "a" | "b"));
5662 }
5663
5664 #[test]
5665 fn short_id_derivation_dedups() {
5666 let mut reg = state::Registry::default();
5667 assert_eq!(derive_short_id("worker-A", ®), "workerA");
5668 reg.entries.push(RegistryEntry {
5669 name: "x".into(),
5670 short_id: "workerA".into(),
5671 legacy_provider: "codex".into(),
5672 harness: None,
5673 harness_session_id: None,
5674 cwd: "/".into(),
5675 project_root: "/".into(),
5676 session_id: None,
5677 legacy_claude_short_id: None,
5678 claude_session_uuid: None,
5679 messaging_socket_path: None,
5680 codex_session_id: None,
5681 gemini_session_id: None,
5682 mcp_channel_id: None,
5683 cc_session_id: None,
5684 host_mode: None,
5685 status: AgentStatus::Live,
5686 last_message_at: None,
5687 created_at: "t".into(),
5688 pid: None,
5689 pid_start_time: None,
5690 log_path: None,
5691 last_reconciled_at: None,
5692 inside_leg: None,
5693 exited_at: None,
5694 mux: None,
5695 screen_state: None,
5696 crown_level: None,
5697 crown_scope: None,
5698 crown_grantor: None,
5699 });
5700 assert_eq!(derive_short_id("worker-A", ®), "workerA1");
5701 }
5702
5703 fn rentry(name: &str, status: AgentStatus, last_reconciled: Option<&str>) -> RegistryEntry {
5706 RegistryEntry {
5707 name: name.into(),
5708 short_id: name.into(),
5709 legacy_provider: "codex".into(),
5710 harness: None,
5711 harness_session_id: None,
5712 cwd: "/tmp".into(),
5713 project_root: "/tmp".into(),
5714 session_id: Some("sid".into()),
5715 legacy_claude_short_id: None,
5716 claude_session_uuid: None,
5717 messaging_socket_path: None,
5718 codex_session_id: None,
5719 gemini_session_id: None,
5720 mcp_channel_id: None,
5721 host_mode: None,
5722 cc_session_id: None,
5723 status,
5724 last_message_at: None,
5725 created_at: "t".into(),
5726 pid: None,
5727 pid_start_time: None,
5728 log_path: None,
5729 last_reconciled_at: last_reconciled.map(String::from),
5730 inside_leg: None,
5731 exited_at: None,
5732 mux: None,
5733 screen_state: None,
5734 crown_level: None,
5735 crown_scope: None,
5736 crown_grantor: None,
5737 }
5738 }
5739
5740 fn probe_err() -> crate::provider::ReachabilityProbeError {
5741 crate::provider::ReachabilityProbeError::new("codex", "store unavailable")
5742 }
5743
5744 fn bg_claude_row(name: &str, short_id: &str) -> RegistryEntry {
5748 let mut e = rentry(name, AgentStatus::Live, None);
5749 e.legacy_provider = "claude".into();
5750 e.short_id = short_id.into();
5751 e.claude_session_uuid = None;
5752 e
5753 }
5754
5755 #[test]
5756 fn find_uuid_backfill_row_matches_null_uuid_by_short_prefix() {
5757 let rows = vec![bg_claude_row("w", "3228ccad")];
5759 assert!(matches!(
5760 find_uuid_backfill_row(&rows, "3228ccad-c078-4b53-a8c9-7199b831eae4"),
5761 UuidBackfill::One(0)
5762 ));
5763 }
5764
5765 #[test]
5766 fn find_uuid_backfill_row_refuses_ambiguous_short_collision() {
5767 let rows = vec![
5769 bg_claude_row("w1", "3228ccad"),
5770 bg_claude_row("w2", "3228ccad"),
5771 ];
5772 assert!(matches!(
5773 find_uuid_backfill_row(&rows, "3228ccad-c078-4b53-a8c9-7199b831eae4"),
5774 UuidBackfill::Ambiguous
5775 ));
5776 }
5777
5778 #[test]
5779 fn find_uuid_backfill_row_skips_rows_that_already_have_a_uuid() {
5780 let mut row = bg_claude_row("w", "3228ccad");
5783 row.claude_session_uuid = Some("3228ccad-c078-4b53-a8c9-7199b831eae4".into());
5784 assert!(matches!(
5785 find_uuid_backfill_row(&[row], "3228ccad-c078-4b53-a8c9-7199b831eae4"),
5786 UuidBackfill::None
5787 ));
5788 }
5789
5790 #[test]
5791 fn find_uuid_backfill_row_skips_non_claude_rows() {
5792 let mut row = bg_claude_row("w", "3228ccad");
5795 row.legacy_provider = "codex".into();
5796 assert!(matches!(
5797 find_uuid_backfill_row(&[row], "3228ccad-c078-4b53-a8c9-7199b831eae4"),
5798 UuidBackfill::None
5799 ));
5800 }
5801
5802 #[test]
5803 fn find_uuid_backfill_row_requires_group_boundary() {
5804 let rows = vec![bg_claude_row("w", "3228ccad")];
5807 assert!(matches!(
5808 find_uuid_backfill_row(&rows, "3228ccadd-c078-4b53-a8c9-7199b831eae4"),
5809 UuidBackfill::None
5810 ));
5811 }
5812
5813 #[test]
5814 fn concurrent_spawn_name_reservation_inserts_once() {
5815 let home = tmp_home("spawn-reserve");
5822 let path = home.registry_json();
5823 let reserve = |entry: RegistryEntry| -> bool {
5824 state::update_registry(&path, move |r| {
5825 if r.entries.iter().any(|e| e.name == entry.name) {
5826 return false;
5827 }
5828 r.entries.push(entry);
5829 true
5830 })
5831 .unwrap()
5832 };
5833 assert!(
5834 reserve(rentry("dup", AgentStatus::Live, None)),
5835 "first wins"
5836 );
5837 assert!(
5838 !reserve(rentry("dup", AgentStatus::Live, None)),
5839 "second loses the race -> no insert"
5840 );
5841 let reg = state::load_registry(&path).unwrap();
5842 assert_eq!(
5843 reg.entries.iter().filter(|e| e.name == "dup").count(),
5844 1,
5845 "exactly one row for the contended name"
5846 );
5847 std::fs::remove_dir_all(home.root()).ok();
5848 }
5849
5850 #[test]
5851 fn reconcile_flips_unreachable_live_to_orphaned_and_recovers_orphaned() {
5852 let entries = vec![
5853 rentry("live-but-gone", AgentStatus::Live, None),
5854 rentry("back-from-dead", AgentStatus::Orphaned, None),
5855 ];
5856 let (changes, out) = plan_reconcile(
5857 &entries,
5858 |e| match e.name.as_str() {
5859 "live-but-gone" => Ok(false), _ => Ok(true), },
5862 || false,
5863 |_| true,
5864 );
5865 assert_eq!(out.orphans, vec!["live-but-gone".to_string()]);
5866 assert_eq!(out.recovered, vec!["back-from-dead".to_string()]);
5867 assert_eq!(out.updated.len(), 2);
5868 assert_eq!(
5870 changes[0].new_status,
5871 Some(AgentStatus::Orphaned),
5872 "unreachable live agent should orphan"
5873 );
5874 assert_eq!(changes[1].new_status, Some(AgentStatus::Live));
5875 }
5876
5877 #[test]
5878 fn reconcile_does_not_orphan_a_live_interactive_host_on_store_miss() {
5879 let mut interactive = rentry("hosted-tui", AgentStatus::Live, None);
5885 interactive.host_mode = Some(crate::state::HOST_MODE_INTERACTIVE.to_string());
5886 let exec = rentry("one-shot", AgentStatus::Live, None);
5887 let entries = vec![interactive, exec];
5888 let (changes, out) = plan_reconcile(&entries, |_| Ok(false), || false, |_| true);
5889 assert_eq!(
5890 changes[0].new_status, None,
5891 "a live interactive host must not be orphaned on a session-store miss"
5892 );
5893 assert_eq!(
5894 changes[1].new_status,
5895 Some(AgentStatus::Orphaned),
5896 "an exec sibling with the same probe result is still orphaned"
5897 );
5898 assert_eq!(out.orphans, vec!["one-shot".to_string()]);
5899 }
5900
5901 #[test]
5902 fn reconcile_reaps_a_dead_interactive_host_to_exited() {
5903 let mut dead = rentry("dead-tui", AgentStatus::Live, None);
5908 dead.host_mode = Some(crate::state::HOST_MODE_INTERACTIVE.to_string());
5909 let mut live = rentry("live-tui", AgentStatus::Live, None);
5910 live.host_mode = Some(crate::state::HOST_MODE_INTERACTIVE.to_string());
5911 let entries = vec![dead, live];
5912 let (changes, out) = plan_reconcile(
5913 &entries,
5914 |_| Ok(false), || false,
5916 |e| e.name == "live-tui", );
5918 assert_eq!(
5919 changes[0].new_status,
5920 Some(AgentStatus::Exited),
5921 "a dead interactive host is reaped to Exited during reconcile"
5922 );
5923 assert_eq!(
5924 changes[1].new_status, None,
5925 "a live interactive host is left untouched"
5926 );
5927 assert!(out.orphans.is_empty());
5929 assert_eq!(out.updated, vec!["dead-tui".to_string()]);
5930 }
5931
5932 #[test]
5933 fn reconcile_mux_pane_liveness_follows_the_pid_not_the_store() {
5934 let mk = |name: &str, pid: Option<u32>| {
5941 let mut e = rentry(name, AgentStatus::Live, None);
5942 e.mux = Some(crate::state::MuxRef {
5943 session: "main".into(),
5944 pane_id: 7,
5945 });
5946 e.pid = pid;
5947 e
5948 };
5949 let entries = vec![
5950 mk("live-pane", Some(4242)), mk("dead-pane", Some(4243)), mk("pidless-pane", None), ];
5954 let (changes, out) = plan_reconcile(
5955 &entries,
5956 |_| Ok(false), || false,
5958 |e| e.name == "live-pane", );
5960 assert_eq!(
5961 changes[0].new_status, None,
5962 "a live-pid mux pane is preserved"
5963 );
5964 assert_eq!(
5965 changes[1].new_status,
5966 Some(AgentStatus::Exited),
5967 "a dead-pid mux pane is reaped to Exited"
5968 );
5969 assert_eq!(
5970 changes[2].new_status,
5971 Some(AgentStatus::Orphaned),
5972 "a pid-less mux pane defers to store liveness (orphan), not immortal"
5973 );
5974 assert_eq!(out.orphans, vec!["pidless-pane".to_string()]);
5975 }
5976
5977 #[test]
5978 fn reconcile_store_hit_does_not_resurrect_a_pid_dead_row() {
5979 let entries = vec![
5984 rentry("dead-orphan", AgentStatus::Orphaned, None),
5985 rentry("live-orphan", AgentStatus::Orphaned, None),
5986 ];
5987 let (changes, out) = plan_reconcile(
5988 &entries,
5989 |_| Ok(true), || false,
5991 |e| e.name == "live-orphan",
5992 );
5993 assert_eq!(
5994 changes[0].new_status, None,
5995 "a store hit must not recover a row whose pid is dead"
5996 );
5997 assert_eq!(
5998 changes[1].new_status,
5999 Some(AgentStatus::Live),
6000 "a store hit on a live pid still recovers"
6001 );
6002 assert_eq!(out.recovered, vec!["live-orphan".to_string()]);
6003 }
6004
6005 #[test]
6006 fn reconcile_pidless_orphan_still_recovers_on_store_hit() {
6007 let entries = vec![rentry("pidless", AgentStatus::Orphaned, None)];
6010 let (changes, out) = plan_reconcile(&entries, |_| Ok(true), || false, |_| true);
6011 assert_eq!(changes[0].new_status, Some(AgentStatus::Live));
6012 assert_eq!(out.recovered, vec!["pidless".to_string()]);
6013 }
6014
6015 #[test]
6016 fn to_agent_entry_projects_the_opencode_session_id() {
6017 let mut e = rentry("oc", AgentStatus::Live, None);
6021 e.legacy_provider = "opencode".into();
6022 e.harness = Some("opencode".into());
6023 e.harness_session_id = Some("ses_09679f284ffeJv7NdBAoLQLnLZ".into());
6024 e.session_id = None;
6025 assert_eq!(
6026 to_agent_entry(&e).session_id.as_deref(),
6027 Some("ses_09679f284ffeJv7NdBAoLQLnLZ")
6028 );
6029 }
6030
6031 #[test]
6032 fn reconcile_inconclusive_preserves_status() {
6033 let entries = vec![rentry("flaky", AgentStatus::Live, None)];
6034 let (changes, out) = plan_reconcile(&entries, |_| Err(probe_err()), || false, |_| true);
6035 assert_eq!(changes[0].new_status, None, "must NOT flip on inconclusive");
6036 assert!(out.orphans.is_empty());
6037 assert_eq!(out.inconsistent.len(), 1);
6038 assert_eq!(out.inconsistent[0].0, "flaky");
6039 }
6040
6041 #[test]
6042 fn reconcile_leaves_terminal_states_untouched() {
6043 let entries = vec![
6046 rentry("done", AgentStatus::Exited, None),
6047 rentry("dead", AgentStatus::PermanentDead, None),
6048 ];
6049 let (changes, out) = plan_reconcile(&entries, |_| Ok(false), || false, |_| true);
6050 assert!(changes.iter().all(|c| c.new_status.is_none()));
6051 assert!(out.orphans.is_empty() && out.updated.is_empty());
6052 }
6053
6054 fn ask_entry(name: &str, status: AgentStatus) -> RegistryEntry {
6057 let mut e = rentry(name, status, None);
6058 e.short_id = String::new();
6059 e.pid = None;
6060 e.codex_session_id = Some("resume-uuid".into());
6061 e.session_id = None;
6062 e
6063 }
6064
6065 #[test]
6066 fn reconcile_one_shot_ask_settles_to_exited_even_when_reachable() {
6067 let entries = vec![ask_entry("codex-ask", AgentStatus::Live)];
6073 let (changes, out) = plan_reconcile(
6074 &entries,
6075 |_| Ok(true), || false,
6077 |_| true,
6078 );
6079 assert_eq!(
6080 changes[0].new_status,
6081 Some(AgentStatus::Exited),
6082 "a finished ask settles to exited even when its session file is reachable"
6083 );
6084 assert_eq!(out.updated, vec!["codex-ask".to_string()]);
6085 assert!(out.orphans.is_empty(), "an ask is exited, never orphaned");
6086 assert_eq!(entries[0].codex_session_id.as_deref(), Some("resume-uuid"));
6089 }
6090
6091 #[test]
6092 fn reconcile_one_shot_ask_already_terminal_is_untouched() {
6093 let entries = vec![ask_entry("done-ask", AgentStatus::Exited)];
6095 let (changes, out) = plan_reconcile(&entries, |_| Ok(true), || false, |_| true);
6096 assert_eq!(changes[0].new_status, None);
6097 assert!(out.updated.is_empty());
6098 }
6099
6100 #[test]
6101 fn apply_reconcile_change_clears_pid_only_on_exited() {
6102 let mut to_exited = rentry("x", AgentStatus::Live, None);
6105 to_exited.pid = Some(4242);
6106 to_exited.pid_start_time = Some(99);
6107 to_exited.inside_leg = Some(state::InsideLegReport {
6108 state: state::InsideLegState::Working,
6109 seq: 3,
6110 reason: None,
6111 received_at: "2026-06-27T00:00:00Z".into(),
6112 ttl_ms: None,
6113 });
6114 apply_reconcile_change(&mut to_exited, Some(AgentStatus::Exited), "T1");
6115 assert_eq!(to_exited.status, AgentStatus::Exited);
6116 assert_eq!(to_exited.pid, None, "exited row must drop its pid");
6117 assert_eq!(to_exited.pid_start_time, None);
6118 assert_eq!(
6119 to_exited.inside_leg, None,
6120 "exited row must clear the inside-leg authority (E3.3 / AC-X2-4)"
6121 );
6122 assert_eq!(to_exited.last_reconciled_at.as_deref(), Some("T1"));
6123
6124 let mut to_orphaned = rentry("y", AgentStatus::Live, None);
6125 to_orphaned.pid = Some(4242);
6126 to_orphaned.inside_leg = Some(state::InsideLegReport {
6127 state: state::InsideLegState::Working,
6128 seq: 1,
6129 reason: None,
6130 received_at: "2026-06-27T00:00:00Z".into(),
6131 ttl_ms: None,
6132 });
6133 apply_reconcile_change(&mut to_orphaned, Some(AgentStatus::Orphaned), "T2");
6134 assert_eq!(to_orphaned.status, AgentStatus::Orphaned);
6135 assert_eq!(
6136 to_orphaned.pid,
6137 Some(4242),
6138 "non-exited transition keeps pid"
6139 );
6140 assert!(
6141 to_orphaned.inside_leg.is_some(),
6142 "a non-exit transition keeps the inside-leg report (only exit tears it down)"
6143 );
6144
6145 let mut no_change = rentry("z", AgentStatus::Live, Some("OLD"));
6147 no_change.pid = Some(4242);
6148 apply_reconcile_change(&mut no_change, None, "T3");
6149 assert_eq!(no_change.status, AgentStatus::Live);
6150 assert_eq!(no_change.pid, Some(4242));
6151 assert_eq!(no_change.last_reconciled_at.as_deref(), Some("T3"));
6152 }
6153
6154 #[test]
6155 fn emit_inside_leg_completion_publishes_only_for_report_bearing_rows() {
6156 let home = tmp_home("inside-leg-completion");
6160 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
6161
6162 let mut with_report = rentry("pane", AgentStatus::Live, None);
6163 with_report.session_id = Some("sess-uuid".into());
6164 with_report.inside_leg = Some(state::InsideLegReport {
6165 state: state::InsideLegState::Working,
6166 seq: 9,
6167 reason: Some("running tests".into()),
6168 received_at: "2026-06-27T00:00:00Z".into(),
6169 ttl_ms: Some(5000),
6170 });
6171 emit_inside_leg_completion(&emitter, &with_report);
6172 emit_inside_leg_completion(&emitter, &rentry("plain", AgentStatus::Live, None));
6173
6174 let log = std::fs::read_to_string(home.events_jsonl()).unwrap_or_default();
6175 let events: Vec<serde_json::Value> = log
6176 .lines()
6177 .filter_map(|l| serde_json::from_str(l).ok())
6178 .filter(|v: &serde_json::Value| v["type"] == "inside_leg_completed")
6179 .collect();
6180 assert_eq!(
6181 events.len(),
6182 1,
6183 "exactly one completion, only for the report-bearing row"
6184 );
6185 let ev = &events[0];
6186 assert_eq!(ev["data"]["name"], "pane");
6187 assert_eq!(ev["data"]["session_id"], "sess-uuid");
6188 assert_eq!(ev["data"]["final_state"], "working");
6189 assert_eq!(ev["data"]["seq"], 9);
6190
6191 std::fs::remove_dir_all(home.root()).ok();
6192 }
6193
6194 #[test]
6195 fn buffer_pending_report_highest_seq_wins_and_is_bounded() {
6196 use std::collections::HashMap;
6197 let rep = |seq| state::InsideLegReport {
6198 state: state::InsideLegState::Working,
6199 seq,
6200 reason: None,
6201 received_at: "2026-06-27T00:00:00Z".into(),
6202 ttl_ms: None,
6203 };
6204 let mut map: HashMap<String, state::InsideLegReport> = HashMap::new();
6205
6206 assert!(matches!(
6208 buffer_pending_report(&mut map, "s1", rep(2)),
6209 BufferOutcome::Buffered
6210 ));
6211 assert_eq!(map["s1"].seq, 2);
6212
6213 assert!(matches!(
6215 buffer_pending_report(&mut map, "s1", rep(1)),
6216 BufferOutcome::StaleSeq { last: 2 }
6217 ));
6218 assert_eq!(
6219 map["s1"].seq, 2,
6220 "stale early push must not regress the buffer"
6221 );
6222
6223 assert!(matches!(
6225 buffer_pending_report(&mut map, "s1", rep(5)),
6226 BufferOutcome::Buffered
6227 ));
6228 assert_eq!(map["s1"].seq, 5);
6229
6230 for i in 0..PENDING_INSIDE_LEG_CAP {
6233 buffer_pending_report(&mut map, &format!("fill{i}"), rep(1));
6234 }
6235 assert!(map.len() >= PENDING_INSIDE_LEG_CAP);
6236 assert!(matches!(
6237 buffer_pending_report(&mut map, "brand-new", rep(1)),
6238 BufferOutcome::Full
6239 ));
6240 assert!(!map.contains_key("brand-new"));
6241 assert!(
6242 matches!(
6243 buffer_pending_report(&mut map, "s1", rep(9)),
6244 BufferOutcome::Buffered
6245 ),
6246 "an already-buffered session advances even at cap (no new key)"
6247 );
6248 }
6249
6250 #[test]
6251 fn flush_buffered_inside_leg_drains_onto_row_under_seq_gate() {
6252 let home = tmp_home("inside-leg-flush");
6257 let ctx = test_ctx_with_events(home.clone(), PathBuf::from("fno-agents-worker"));
6258 let report = |seq| state::InsideLegReport {
6259 state: state::InsideLegState::Working,
6260 seq,
6261 reason: None,
6262 received_at: "2026-06-27T00:00:00Z".into(),
6263 ttl_ms: Some(5000),
6264 };
6265
6266 let mut row = rentry("pane", AgentStatus::Live, None);
6268 row.legacy_provider = "claude".into();
6269 row.claude_session_uuid = Some("uuid-x".into());
6270 state::update_registry(&home.registry_json(), |r| r.entries.push(row)).unwrap();
6271 ctx.pending_inside_leg
6272 .lock()
6273 .unwrap()
6274 .insert("uuid-x".into(), report(4));
6275
6276 flush_buffered_inside_leg(&ctx, "uuid-x", "pane");
6277
6278 assert!(!ctx
6280 .pending_inside_leg
6281 .lock()
6282 .unwrap()
6283 .contains_key("uuid-x"));
6284 let reg = state::load_registry(&home.registry_json()).unwrap();
6285 assert_eq!(reg.entries[0].inside_leg.as_ref().map(|r| r.seq), Some(4));
6286 let events = read_events(&home);
6287 assert!(events
6288 .iter()
6289 .any(|e| e["type"] == "inside_leg_buffer_flushed"
6290 && e["data"]["name"] == "pane"
6291 && e["data"]["session_id"] == "uuid-x"
6292 && e["data"]["seq"] == 4));
6293
6294 state::update_registry(&home.registry_json(), |r| {
6297 r.entries[0].inside_leg = Some(report(10));
6298 })
6299 .unwrap();
6300 ctx.pending_inside_leg
6301 .lock()
6302 .unwrap()
6303 .insert("uuid-x".into(), report(7));
6304 flush_buffered_inside_leg(&ctx, "uuid-x", "pane");
6305 let reg = state::load_registry(&home.registry_json()).unwrap();
6306 assert_eq!(
6307 reg.entries[0].inside_leg.as_ref().map(|r| r.seq),
6308 Some(10),
6309 "a stale buffered report must not regress a newer row state"
6310 );
6311
6312 std::fs::remove_dir_all(home.root()).ok();
6313 }
6314
6315 #[test]
6316 fn reconcile_defers_remaining_when_budget_exhausted() {
6317 let entries = vec![
6318 rentry("a", AgentStatus::Live, None),
6319 rentry("b", AgentStatus::Live, None),
6320 rentry("c", AgentStatus::Live, None),
6321 ];
6322 let mut probes = 0;
6324 let (changes, out) = plan_reconcile(
6325 &entries,
6326 |_| {
6327 probes += 1;
6328 Ok(true)
6329 },
6330 {
6331 let mut checked = 0;
6332 move || {
6333 let exhausted = checked >= 1;
6334 checked += 1;
6335 exhausted
6336 }
6337 },
6338 |_| true,
6339 );
6340 assert_eq!(out.deferred, 2, "two trailing entries should defer");
6341 assert_eq!(changes.len(), 1, "only one entry probed before budget");
6342 }
6343
6344 #[test]
6345 fn run_reconcile_sweep_empty_registry_is_noop() {
6346 let home = tmp_home("sweep-empty");
6350 let emitter = EventEmitter::new(home.events_jsonl(), "daemon");
6351 let result = run_reconcile_sweep(&home, &emitter).expect("empty sweep ok");
6352 assert!(result.entries.is_empty());
6353 assert_eq!(result.outcome, ReconcileOutcome::default());
6354 std::fs::remove_dir_all(home.root()).ok();
6355 }
6356
6357 struct PromptDetector;
6363 impl crate::readiness::ReadinessDetector for PromptDetector {
6364 fn provider_name(&self) -> &str {
6365 "test-cli"
6366 }
6367 fn is_ready(
6368 &self,
6369 screen: &crate::readiness::ScreenView,
6370 ) -> Result<bool, crate::readiness::ReadinessError> {
6371 Ok(screen.visible_text.trim_end().ends_with('\u{276f}'))
6372 }
6373 }
6374
6375 struct NeverReadyDetector;
6377 impl crate::readiness::ReadinessDetector for NeverReadyDetector {
6378 fn provider_name(&self) -> &str {
6379 "never"
6380 }
6381 fn is_ready(
6382 &self,
6383 _screen: &crate::readiness::ScreenView,
6384 ) -> Result<bool, crate::readiness::ReadinessError> {
6385 Ok(false)
6386 }
6387 }
6388
6389 #[tokio::test(flavor = "current_thread")]
6393 async fn poll_until_ready_returns_settled_reply_on_ready_prompt() {
6394 let snapshots: &[&str] = &["loading...", "still loading...", "done \u{276f}"];
6396 let idx = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
6397 let idx2 = idx.clone();
6398 let fetcher = move || {
6399 let i = idx2.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6400 let text = snapshots[i.min(snapshots.len() - 1)].to_string();
6401 std::future::ready(Some(text))
6402 };
6403 let result = poll_until_ready(
6404 fetcher,
6405 Box::new(PromptDetector),
6406 Duration::from_millis(1),
6407 Duration::from_secs(5),
6408 )
6409 .await;
6410 assert!(result.is_ok(), "expected Ok, got {result:?}");
6411 let reply = result.unwrap();
6412 assert_eq!(
6413 reply, "done \u{276f}",
6414 "reply must be the settled snapshot text, got {reply:?}"
6415 );
6416 }
6417
6418 #[tokio::test(flavor = "current_thread")]
6422 async fn poll_until_ready_returns_error_on_timeout() {
6423 let fetcher = || std::future::ready(Some("still thinking...".to_string()));
6424 let result = poll_until_ready(
6425 fetcher,
6426 Box::new(NeverReadyDetector),
6427 Duration::from_millis(10),
6428 Duration::from_millis(40), )
6430 .await;
6431 assert!(
6432 result.is_err(),
6433 "expected Err on timeout, got Ok({:?})",
6434 result.ok()
6435 );
6436 }
6437
6438 #[tokio::test(flavor = "current_thread")]
6441 async fn poll_until_ready_empty_settled_screen_returns_empty_string() {
6442 let fetcher = || std::future::ready(Some("\u{276f}".to_string()));
6444 let result = poll_until_ready(
6445 fetcher,
6446 Box::new(PromptDetector),
6447 Duration::from_millis(1),
6448 Duration::from_secs(5),
6449 )
6450 .await;
6451 assert!(result.is_ok(), "expected Ok, got {result:?}");
6452 let reply = result.unwrap();
6455 assert!(!reply.contains("fabricated"), "must not fabricate content");
6456 }
6457
6458 #[test]
6465 fn entry_holds_session_matches_claude_session_uuid() {
6466 let row = build_claude_stream_entry(
6467 "peer",
6468 "ab12cd34",
6469 std::path::Path::new("/work"),
6470 "sess-uuid-9",
6471 4242,
6472 None,
6473 PathBuf::from("/tmp/log.jsonl"),
6474 );
6475 assert!(
6476 entry_holds_session(&row, "sess-uuid-9"),
6477 "a claude row must be matched by its claude_session_uuid"
6478 );
6479 assert!(!entry_holds_session(&row, "other-uuid"));
6480 }
6481
6482 fn test_ctx(home: AgentsHome, worker_bin: PathBuf) -> Ctx {
6483 Ctx {
6484 home,
6485 emitter: EventEmitter::new(std::path::PathBuf::from("/dev/null"), "daemon"),
6486 opts: DaemonOptions {
6487 idle_exit: Duration::from_secs(1800),
6488 worker_bin,
6489 reconcile_on_start: true,
6490 dead_row_grace: Duration::from_secs(3600),
6491 notify_on_blocked: false,
6493 notify_on_done: false,
6494 },
6495 started_at: std::time::Instant::now(),
6496 exe_fingerprint: crate::drift::ExeFingerprint::current(),
6497 pid_start_time: process_start_time(std::process::id()),
6498 pending_inside_leg: std::sync::Mutex::new(std::collections::HashMap::new()),
6499 }
6500 }
6501
6502 fn test_ctx_with_events(home: AgentsHome, worker_bin: PathBuf) -> Ctx {
6505 let events_path = home.events_jsonl();
6506 Ctx {
6507 home,
6508 emitter: EventEmitter::new(events_path, "daemon"),
6509 opts: DaemonOptions {
6510 idle_exit: Duration::from_secs(1800),
6511 worker_bin,
6512 reconcile_on_start: true,
6513 dead_row_grace: Duration::from_secs(3600),
6514 notify_on_blocked: false,
6516 notify_on_done: false,
6517 },
6518 started_at: std::time::Instant::now(),
6519 exe_fingerprint: crate::drift::ExeFingerprint::current(),
6520 pid_start_time: process_start_time(std::process::id()),
6521 pending_inside_leg: std::sync::Mutex::new(std::collections::HashMap::new()),
6522 }
6523 }
6524
6525 const FAKE_STREAM_EMITTER: &str = r#"
6532printf '%s\n' '{"type":"system","subtype":"init","session_id":"s1"}'
6533while IFS= read -r line; do
6534 printf '%s\n' '{"type":"user","message":{"role":"user"}}'
6535 printf '%s\n' '{"type":"stream_event","event":{"type":"content_block_delta","delta":{"type":"text_delta","text":"par"}}}'
6536 printf '%s\n' '{"type":"assistant","message":{"content":[{"type":"text","text":"reply-text"}]}}'
6537 printf '%s\n' '{"type":"result","subtype":"success","is_error":false,"result":"reply-text"}'
6538done
6539"#;
6540
6541 fn short_home(tag: &str) -> AgentsHome {
6546 use std::sync::atomic::{AtomicU32, Ordering};
6547 static C: AtomicU32 = AtomicU32::new(0);
6548 let n = C.fetch_add(1, Ordering::Relaxed);
6549 let p = PathBuf::from(format!("/tmp/fnosb{tag}{}_{n}", std::process::id()));
6550 let home = AgentsHome::at(&p);
6551 home.ensure_root().unwrap();
6552 home
6553 }
6554
6555 fn seed_stream_row(home: &AgentsHome, name: &str, short_id: &str) {
6557 state::update_registry(&home.registry_json(), |r| {
6558 r.entries.push(RegistryEntry {
6559 name: name.into(),
6560 short_id: short_id.into(),
6561 legacy_provider: "claude".into(),
6562 harness: None,
6563 harness_session_id: None,
6564 cwd: "/tmp".into(),
6565 project_root: "/tmp".into(),
6566 session_id: None,
6567 legacy_claude_short_id: None,
6568 claude_session_uuid: Some(format!("uuid-{short_id}")),
6569 messaging_socket_path: None,
6570 codex_session_id: None,
6571 gemini_session_id: None,
6572 mcp_channel_id: None,
6573 cc_session_id: None,
6574 host_mode: None,
6575 status: AgentStatus::Live,
6576 last_message_at: None,
6577 created_at: "2026-06-09T00:00:00Z".into(),
6578 pid: None,
6579 pid_start_time: None,
6580 log_path: None,
6581 last_reconciled_at: None,
6582 inside_leg: None,
6583 exited_at: None,
6584 mux: None,
6585 screen_state: None,
6586 crown_level: None,
6587 crown_scope: None,
6588 crown_grantor: None,
6589 });
6590 })
6591 .unwrap();
6592 }
6593
6594 #[test]
6607 fn list_row_key_set_matches_shared_contract() {
6608 const CONTRACT: &str = include_str!(concat!(
6609 env!("CARGO_MANIFEST_DIR"),
6610 "/../../schemas/agents-list-row.json"
6611 ));
6612 let contract: Value = serde_json::from_str(CONTRACT).expect("contract is valid JSON");
6613 let mut expected: std::collections::BTreeSet<String> = contract["required"]
6614 .as_array()
6615 .expect("required is an array")
6616 .iter()
6617 .map(|k| k.as_str().unwrap().to_string())
6618 .collect();
6619 expected.extend(
6620 contract["rust_only"]["keys"]
6621 .as_array()
6622 .expect("rust_only.keys is an array")
6623 .iter()
6624 .map(|k| k.as_str().unwrap().to_string()),
6625 );
6626
6627 let home = short_home("listcontract");
6628 seed_stream_row(&home, "worker-contract", "abc12345");
6629 state::update_registry(&home.registry_json(), |r| {
6630 let e = &mut r.entries[0];
6631 e.short_id = String::new();
6636 e.harness = Some("claude".into());
6637 e.harness_session_id = Some("e6f78b98-e594-47ed-ad81-84f8a78b8bb7".into());
6638 e.claude_session_uuid = Some("e6f78b98-e594-47ed-ad81-84f8a78b8bb7".into());
6639 e.mux = Some(crate::state::MuxRef {
6640 session: "main".into(),
6641 pane_id: 10,
6642 });
6643 e.crown_level = Some(1);
6644 e.crown_scope = Some("epic-x".into());
6645 e.crown_grantor = Some("king".into());
6646 })
6647 .unwrap();
6648 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
6649 let req = Request::new(1, "agent.list", json!({}));
6650
6651 let response = handle_list_with_truth(&ctx, &req, |_handle| Some("working".into()));
6652 let result = response.result().unwrap();
6653 let row = &result["agents"][0];
6654
6655 let actual: std::collections::BTreeSet<String> =
6656 row.as_object().unwrap().keys().cloned().collect();
6657 assert_eq!(actual, expected, "list row key set drifted from contract");
6658
6659 assert_eq!(row["harness"], "claude");
6663 assert_eq!(row["provider"], "claude", "legacy alias still emitted");
6664 assert_eq!(
6665 row["harness_session_id"],
6666 "e6f78b98-e594-47ed-ad81-84f8a78b8bb7"
6667 );
6668 assert!(row["session_id"].is_null());
6672 assert_eq!(row["mux"]["session"], "main");
6673 assert_eq!(row["mux"]["pane_id"], 10);
6674 assert_eq!(
6675 row["crown"], "L1 epic-x",
6676 "same formatter as Python crown_label"
6677 );
6678 assert_eq!(row["crown_level"], 1);
6682 assert_eq!(row["crown_scope"], "epic-x");
6683 assert_eq!(row["crown_grantor"], "king");
6684
6685 std::fs::remove_dir_all(home.root()).ok();
6686 }
6687
6688 #[test]
6694 fn list_row_resolves_opencode_session_id_from_harness_session_id() {
6695 let home = short_home("listopencode");
6696 seed_stream_row(&home, "worker-opencode", "abc12345");
6697 state::update_registry(&home.registry_json(), |r| {
6698 let e = &mut r.entries[0];
6699 e.harness = Some("opencode".into());
6700 e.harness_session_id = Some("oc-sess-9f2".into());
6701 e.session_id = None;
6702 })
6703 .unwrap();
6704 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
6705 let req = Request::new(1, "agent.list", json!({}));
6706
6707 let response = handle_list_with_truth(&ctx, &req, |_handle| Some("working".into()));
6708 let result = response.result().unwrap();
6709 let row = &result["agents"][0];
6710
6711 assert_eq!(row["harness"], "opencode");
6712 assert_eq!(row["session_id"], "oc-sess-9f2");
6713
6714 std::fs::remove_dir_all(home.root()).ok();
6715 }
6716
6717 #[test]
6721 fn list_row_crown_label_falls_back_on_an_empty_scope() {
6722 let home = short_home("listcrownempty");
6723 seed_stream_row(&home, "worker-crown", "abc12345");
6724 state::update_registry(&home.registry_json(), |r| {
6725 let e = &mut r.entries[0];
6726 e.crown_level = Some(1);
6727 e.crown_scope = Some(String::new());
6728 })
6729 .unwrap();
6730 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
6731 let req = Request::new(1, "agent.list", json!({}));
6732
6733 let response = handle_list_with_truth(&ctx, &req, |_handle| Some("working".into()));
6734 let result = response.result().unwrap();
6735
6736 assert_eq!(result["agents"][0]["crown"], "L1 ?");
6737
6738 std::fs::remove_dir_all(home.root()).ok();
6739 }
6740
6741 #[test]
6744 fn list_row_emits_absent_optional_fields_as_null() {
6745 let home = short_home("listnulls");
6746 seed_stream_row(&home, "worker-bare", "abc12345");
6747 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
6748 let req = Request::new(1, "agent.list", json!({}));
6749
6750 let response = handle_list_with_truth(&ctx, &req, |_handle| Some("working".into()));
6751 let result = response.result().unwrap();
6752 let row = &result["agents"][0];
6753
6754 let obj = row.as_object().unwrap();
6758 for key in ["mux", "crown", "crown_level"] {
6759 assert!(obj.contains_key(key), "row omits key: {key}");
6760 assert!(obj[key].is_null(), "key {key} should be null on a bare row");
6761 }
6762
6763 std::fs::remove_dir_all(home.root()).ok();
6764 }
6765
6766 fn stream_identity(short_id: &str) -> Value {
6767 json!({
6768 "harness": "claude",
6769 "session_id": format!("uuid-{short_id}"),
6770 "short_id": short_id,
6771 "created_at": "2026-06-09T00:00:00Z",
6772 })
6773 }
6774
6775 fn switchboard_params(
6776 to: &str,
6777 to_short: &str,
6778 from: &str,
6779 from_short: Option<&str>,
6780 body: &str,
6781 ) -> Value {
6782 let mut params = json!({
6783 "to": to,
6784 "from": from,
6785 "body": body,
6786 "mirror": from_short.is_some(),
6787 "recipient_identity": stream_identity(to_short),
6788 });
6789 if let Some(short_id) = from_short {
6790 params["from_identity"] = stream_identity(short_id);
6791 }
6792 params
6793 }
6794
6795 #[test]
6796 fn list_renders_family1_truth_instead_of_stored_registry_status() {
6797 let home = short_home("listtruth");
6798 seed_stream_row(&home, "worker-list", "abc12345");
6799 state::update_registry(&home.registry_json(), |registry| {
6800 registry.entries[0].status = AgentStatus::Orphaned;
6801 })
6802 .unwrap();
6803 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
6804 let req = Request::new(1, "agent.list", json!({"status": "live"}));
6805
6806 let response = handle_list_with_truth(&ctx, &req, |_handle| Some("working".into()));
6807 let result = response.result().unwrap();
6808 let agents = result["agents"].as_array().unwrap();
6809 assert_eq!(agents.len(), 1);
6810 assert_eq!(agents[0]["status"], "live");
6811
6812 std::fs::remove_dir_all(home.root()).ok();
6813 }
6814
6815 #[test]
6816 fn list_queries_family1_by_session_identity_not_custom_name() {
6817 let home = short_home("listidentity");
6818 seed_stream_row(&home, "custom-worker-name", "abc12345");
6819 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
6820 let req = Request::new(1, "agent.list", json!({"status": "live"}));
6821 let seen = std::cell::RefCell::new(Vec::new());
6822
6823 let response = handle_list_with_truth(&ctx, &req, |handle| {
6824 seen.borrow_mut().push(handle.to_string());
6825 Some("working".into())
6826 });
6827
6828 assert!(response.result().is_some());
6829 assert_eq!(seen.into_inner(), vec!["uuid-abc12345"]);
6830 std::fs::remove_dir_all(home.root()).ok();
6831 }
6832
6833 #[test]
6834 fn list_queries_pidless_row_by_bare_canonical_handle() {
6835 let home = short_home("listpidless");
6836 seed_stream_row(&home, "custom-worker-name", "unused");
6837 state::update_registry(&home.registry_json(), |registry| {
6838 registry.entries[0].short_id.clear();
6839 registry.entries[0].harness = Some("codex".into());
6840 registry.entries[0].harness_session_id =
6841 Some("019f8ff2-1111-2222-3333-444444444444".into());
6842 })
6843 .unwrap();
6844 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
6845 let req = Request::new(1, "agent.list", json!({"status": "live"}));
6846 let seen = std::cell::RefCell::new(Vec::new());
6847
6848 let response = handle_list_with_truth(&ctx, &req, |handle| {
6849 seen.borrow_mut().push(handle.to_string());
6850 Some("working".into())
6851 });
6852
6853 assert!(response.result().is_some());
6854 assert_eq!(
6855 seen.into_inner(),
6856 vec!["019f8ff2-1111-2222-3333-444444444444"]
6857 );
6858 std::fs::remove_dir_all(home.root()).ok();
6859 }
6860
6861 #[test]
6862 fn list_queries_non_claude_row_by_transcript_identity() {
6863 let home = short_home("listnonclaude");
6864 seed_stream_row(&home, "custom-worker-name", "transport");
6865 state::update_registry(&home.registry_json(), |registry| {
6866 registry.entries[0].harness = Some("codex".into());
6867 registry.entries[0].harness_session_id =
6868 Some("019f8ff2-1111-2222-3333-444444444444".into());
6869 })
6870 .unwrap();
6871 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
6872 let req = Request::new(1, "agent.list", json!({"status": "live"}));
6873 let seen = std::cell::RefCell::new(Vec::new());
6874
6875 let response = handle_list_with_truth(&ctx, &req, |handle| {
6876 seen.borrow_mut().push(handle.to_string());
6877 Some("working".into())
6878 });
6879
6880 assert!(response.result().is_some());
6881 assert_eq!(
6882 seen.into_inner(),
6883 vec!["019f8ff2-1111-2222-3333-444444444444"]
6884 );
6885 std::fs::remove_dir_all(home.root()).ok();
6886 }
6887
6888 #[test]
6889 fn list_applies_cheap_filters_before_family1_subprocesses() {
6890 let home = short_home("listprefilter");
6891 seed_stream_row(&home, "claude-worker", "aaaaaaaa");
6892 seed_stream_row(&home, "codex-worker", "bbbbbbbb");
6893 state::update_registry(&home.registry_json(), |registry| {
6894 registry.entries[1].harness = Some("codex".into());
6895 registry.entries[1].harness_session_id =
6896 Some("bbbbbbbb-1111-2222-3333-444444444444".into());
6897 })
6898 .unwrap();
6899 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
6900 let req = Request::new(1, "agent.list", json!({"provider": "codex"}));
6901 let seen = std::cell::RefCell::new(Vec::new());
6902
6903 let response = handle_list_with_truth(&ctx, &req, |handle| {
6904 seen.borrow_mut().push(handle.to_string());
6905 Some("working".into())
6906 });
6907
6908 assert!(response.result().is_some());
6909 assert_eq!(
6910 seen.into_inner(),
6911 vec!["bbbbbbbb-1111-2222-3333-444444444444"]
6912 );
6913 std::fs::remove_dir_all(home.root()).ok();
6914 }
6915
6916 async fn start_stream_worker(home: &AgentsHome, short_id: &str, script: &str) -> PathBuf {
6919 let cfg = crate::stream_worker::StreamWorkerConfig::new(
6920 short_id,
6921 home.root().to_path_buf(),
6922 std::env::temp_dir(),
6923 vec!["bash".into(), "-c".into(), script.into()],
6924 );
6925 let short_id_dbg = short_id.to_string();
6926 std::thread::spawn(move || {
6927 let rt = tokio::runtime::Builder::new_current_thread()
6928 .enable_all()
6929 .build()
6930 .unwrap();
6931 rt.block_on(async {
6932 if let Err(e) = crate::stream_worker::run(cfg).await {
6933 eprintln!("STREAM WORKER RUN ERROR ({short_id_dbg}): {e}");
6934 }
6935 });
6936 });
6937 let sock = home.worker_sock(short_id);
6938 let bind_start = std::time::Instant::now();
6940 while !sock.exists() && bind_start.elapsed() < Duration::from_secs(20) {
6941 tokio::time::sleep(Duration::from_millis(50)).await;
6942 }
6943 assert!(
6944 sock.exists(),
6945 "stream worker socket never appeared for {short_id}"
6946 );
6947 let live_start = std::time::Instant::now();
6958 let mut live = false;
6959 while live_start.elapsed() < Duration::from_secs(30) {
6960 if is_live_stream_thread(&sock).await {
6961 live = true;
6962 break;
6963 }
6964 tokio::time::sleep(Duration::from_millis(50)).await;
6965 }
6966 assert!(
6967 live,
6968 "stream worker socket appeared but never answered a ping for {short_id}"
6969 );
6970 sock
6971 }
6972
6973 fn built_worker_bin() -> Option<PathBuf> {
6978 let exe = std::env::current_exe().ok()?;
6979 let dir = exe.parent()?.parent()?; let cand = dir.join("fno-agents-worker");
6981 cand.exists().then_some(cand)
6982 }
6983
6984 #[test]
6987 fn stream_claim_holder_is_short_id_scoped() {
6988 assert_eq!(stream_claim_holder("sw7"), "stream:sw7");
6989 }
6990
6991 #[test]
6995 fn provider_readiness_detector_handles_claude() {
6996 let d = provider_readiness_detector("claude");
6997 assert_eq!(d.provider_name(), "claude");
6998 assert_eq!(
7004 provider_readiness_detector("aider").provider_name(),
7005 "aider"
7006 );
7007 }
7008
7009 #[test]
7010 fn is_live_writer_excludes_orphaned_and_terminal() {
7011 for s in [
7013 AgentStatus::Live,
7014 AgentStatus::Ready,
7015 AgentStatus::Idle,
7016 AgentStatus::Busy,
7017 AgentStatus::Spawning,
7018 AgentStatus::Restarting,
7019 ] {
7020 assert!(is_live_writer(s), "{s:?} should count as a live writer");
7021 }
7022 for s in [
7024 AgentStatus::Orphaned,
7025 AgentStatus::Failed,
7026 AgentStatus::Exited,
7027 AgentStatus::PermanentDead,
7028 ] {
7029 assert!(!is_live_writer(s), "{s:?} must NOT block re-adoption");
7030 }
7031 }
7032
7033 #[test]
7034 fn acquire_session_claim_maps_native_outcomes() {
7035 let td = tempfile::tempdir().unwrap();
7036 let _guard = crate::claims::test_env_lock()
7037 .lock()
7038 .unwrap_or_else(|e| e.into_inner());
7039 std::env::set_var("FNO_CLAIMS_ROOT", td.path());
7040 assert!(matches!(
7042 acquire_session_claim("U-1", "stream:sw1"),
7043 ClaimOutcome::Acquired
7044 ));
7045 assert!(matches!(
7047 acquire_session_claim("U-1", "stream:sw1"),
7048 ClaimOutcome::Acquired
7049 ));
7050 match acquire_session_claim("U-1", "stream:other") {
7053 ClaimOutcome::HeldByOther(who) => assert_eq!(who, "stream:sw1"),
7054 other => panic!("expected HeldByOther, got {other:?}"),
7055 }
7056 std::env::remove_var("FNO_CLAIMS_ROOT");
7057 }
7058
7059 #[test]
7060 fn claude_stream_worker_args_carry_stream_flags_and_child_argv() {
7061 let child = crate::provider::claude_stream_json_resume_argv("U-9");
7062 let args = claude_stream_worker_args(
7063 "sw9",
7064 std::path::Path::new("/home/agents"),
7065 std::path::Path::new("/work"),
7066 "U-9",
7067 "stream:sw9",
7068 &child,
7069 );
7070 assert!(args.contains(&"--stream".to_string()));
7073 assert_eq!(
7074 args.iter()
7075 .position(|a| a == "--session-uuid")
7076 .map(|i| &args[i + 1]),
7077 Some(&"U-9".to_string())
7078 );
7079 assert_eq!(
7080 args.iter()
7081 .position(|a| a == "--holder")
7082 .map(|i| &args[i + 1]),
7083 Some(&"stream:sw9".to_string())
7084 );
7085 let sep = args
7086 .iter()
7087 .position(|a| a == "--")
7088 .expect("missing -- separator");
7089 assert_eq!(&args[sep + 1..], child.as_slice());
7090 assert_eq!(child[0], "claude");
7091 assert!(child.contains(&"--resume".to_string()) && child.contains(&"U-9".to_string()));
7092 }
7093
7094 #[test]
7095 fn build_claude_stream_entry_marks_interactive_claude_with_full_uuid() {
7096 let e = build_claude_stream_entry(
7097 "adopted",
7098 "sw3",
7099 std::path::Path::new("/proj"),
7100 "FULL-UUID-3",
7101 4242,
7102 Some(99),
7103 PathBuf::from("/proj/.fno/agents/sw3/timeline.jsonl"),
7104 );
7105 assert_eq!(e.harness_name(), "claude");
7106 assert_eq!(
7107 e.host_mode.as_deref(),
7108 Some(crate::state::HOST_MODE_INTERACTIVE)
7109 );
7110 assert!(
7111 e.is_interactive(),
7112 "stream thread must read as interactive for reconcile"
7113 );
7114 assert_eq!(e.claude_session_uuid.as_deref(), Some("FULL-UUID-3"));
7115 assert_eq!(e.status, AgentStatus::Live);
7116 assert_eq!(e.pid, Some(4242));
7117 assert_eq!(e.short_id, "sw3");
7120 }
7121
7122 #[tokio::test(flavor = "current_thread")]
7127 async fn host_claude_without_from_rejected_with_adopt_pointer() {
7128 let home = short_home("clnofrom");
7129 let ctx = test_ctx(home.clone(), PathBuf::from("/nonexistent-worker"));
7130 let req = Request::new(
7131 1,
7132 "agent.spawn",
7133 json!({"name": "cl", "provider": "claude", "host_mode": "interactive"}),
7134 );
7135 let resp = handle_spawn(&ctx, &req).await;
7136 match &resp.payload {
7137 crate::protocol::ResponsePayload::Err(e) => {
7138 assert_eq!(e.code, ErrorCode::InvalidParams);
7139 assert!(
7140 e.message.contains("promote") && e.message.contains("--from"),
7141 "claude host without --from must point at the adopt verb; got: {}",
7142 e.message
7143 );
7144 }
7145 _ => panic!("expected error for claude host without --from"),
7146 }
7147 std::fs::remove_dir_all(home.root()).ok();
7148 }
7149
7150 #[tokio::test(flavor = "current_thread")]
7154 async fn promote_claude_duplicate_session_refused() {
7155 let home = short_home("cldup");
7156 seed_stream_row(&home, "first", "swDup"); let ctx = test_ctx(home.clone(), PathBuf::from("/nonexistent-worker"));
7158 let req = Request::new(
7159 1,
7160 "agent.spawn",
7161 json!({
7162 "name": "second", "provider": "claude", "host_mode": "interactive",
7163 "resume_id": "uuid-swDup"
7164 }),
7165 );
7166 let resp = handle_spawn(&ctx, &req).await;
7167 match &resp.payload {
7168 crate::protocol::ResponsePayload::Err(e) => {
7169 assert_eq!(e.code, ErrorCode::InvalidParams);
7170 assert!(
7171 e.message.contains("already hosted") && e.message.contains("first"),
7172 "duplicate adopt must name the existing host; got: {}",
7173 e.message
7174 );
7175 }
7176 _ => panic!("expected single-writer refusal for duplicate adopt"),
7177 }
7178 std::fs::remove_dir_all(home.root()).ok();
7179 }
7180
7181 #[tokio::test(flavor = "current_thread")]
7187 async fn promote_claude_spawns_live_stream_thread() {
7188 let Some(worker_bin) = built_worker_bin() else {
7189 eprintln!("skip promote_claude_spawns_live_stream_thread: worker bin not built");
7190 return;
7191 };
7192 let _guard = crate::claims::test_env_lock()
7193 .lock()
7194 .unwrap_or_else(|e| e.into_inner());
7195 let home = short_home("cle2e");
7196 std::env::set_var("FNO_CLAIMS_ROOT", home.root());
7203 let ctx = test_ctx(home.clone(), worker_bin);
7204 let req = Request::new(
7205 1,
7206 "agent.spawn",
7207 json!({
7208 "name": "cl", "provider": "claude", "host_mode": "interactive",
7209 "resume_id": "uuid-e2e", "cwd": "/tmp",
7210 "argv": ["bash", "-c", FAKE_STREAM_EMITTER]
7212 }),
7213 );
7214 let resp = handle_spawn(&ctx, &req).await;
7215 let res = resp.result().expect("claude adopt errored");
7216 assert_eq!(res["provider"], "claude");
7217 assert_eq!(res["status"], "live");
7218 assert_eq!(res["lane"], "stream");
7219
7220 let reg = load_registry_offloaded(home.registry_json()).await;
7221 let row = reg.find("cl").expect("adopted row missing");
7222 assert_eq!(row.harness_name(), "claude");
7223 assert_eq!(row.host_mode.as_deref(), Some("interactive"));
7224 assert_eq!(row.claude_session_uuid.as_deref(), Some("uuid-e2e"));
7225 assert_eq!(row.status, AgentStatus::Live);
7226
7227 let sock = home.worker_sock(&row.short_id);
7228 assert!(
7229 is_live_stream_thread(&sock).await,
7230 "adopted thread must serve the stream protocol"
7231 );
7232
7233 best_effort_worker_shutdown(&sock).await;
7237 std::fs::remove_dir_all(home.root()).ok();
7238 std::env::remove_var("FNO_CLAIMS_ROOT");
7239 }
7240
7241 #[tokio::test(flavor = "current_thread")]
7246 async fn promote_claude_dead_on_arrival_resume_rejected() {
7247 let Some(worker_bin) = built_worker_bin() else {
7248 eprintln!("skip promote_claude_dead_on_arrival_resume_rejected: worker bin not built");
7249 return;
7250 };
7251 let _guard = crate::claims::test_env_lock()
7252 .lock()
7253 .unwrap_or_else(|e| e.into_inner());
7254 let home = short_home("cldoa");
7255 std::env::set_var("FNO_CLAIMS_ROOT", home.root());
7256 let ctx = test_ctx(home.clone(), worker_bin);
7257 let req = Request::new(
7258 1,
7259 "agent.spawn",
7260 json!({
7261 "name": "cl", "provider": "claude", "host_mode": "interactive",
7262 "resume_id": "uuid-doa", "cwd": "/tmp",
7263 "argv": ["bash", "-c", "exit 1"]
7265 }),
7266 );
7267 let resp = handle_spawn(&ctx, &req).await;
7268 assert!(
7269 resp.is_err(),
7270 "DOA resume child must be rejected, not registered"
7271 );
7272 assert_eq!(resp.error().unwrap().code, ErrorCode::SpawnFailed);
7273 let reg = load_registry_offloaded(home.registry_json()).await;
7274 assert!(
7275 reg.find("cl").is_none(),
7276 "no row may be registered for a DOA adopt"
7277 );
7278 std::fs::remove_dir_all(home.root()).ok();
7279 std::env::remove_var("FNO_CLAIMS_ROOT");
7280 }
7281
7282 #[tokio::test(flavor = "current_thread")]
7285 async fn switchboard_drives_b_and_mirrors_into_a() {
7286 let home = short_home("hp");
7287 seed_stream_row(&home, "A", "swA");
7288 seed_stream_row(&home, "B", "swB");
7289 let _a = start_stream_worker(&home, "swA", FAKE_STREAM_EMITTER).await;
7290 let _b = start_stream_worker(&home, "swB", FAKE_STREAM_EMITTER).await;
7291 let ctx = test_ctx_with_events(home.clone(), PathBuf::from("/nonexistent-worker"));
7292
7293 let req = Request::new(
7294 1,
7295 "agent.switchboard",
7296 switchboard_params("B", "swB", "A", Some("swA"), "hello"),
7297 );
7298 let resp = handle_switchboard(&ctx, &req).await;
7299 let res = resp.result().expect("switchboard errored");
7300 assert_eq!(res["delivered"], true, "not delivered: {res:?}");
7301 assert_eq!(res["reply"], "reply-text");
7302 assert_eq!(res["is_error"], false);
7303 assert_eq!(res["receipt"], true, "user-echo receipt not observed");
7304 assert_eq!(res["mirrored"], true, "B's reply was not mirrored into A");
7305 assert_eq!(res["identity_verified"], true);
7306
7307 let events = read_events(&home);
7309 assert!(
7310 events.iter().any(|e| e["type"] == "agent_deliver_injected"
7311 && e["data"]["transport"] == "switchboard"
7312 && e["data"]["mirrored"] == true),
7313 "switchboard injected event missing: {events:?}"
7314 );
7315 std::fs::remove_dir_all(home.root()).ok();
7316 }
7317
7318 #[tokio::test(flavor = "current_thread")]
7323 async fn switchboard_second_turn_returns_fresh_reply() {
7324 const COUNTING_EMITTER: &str = r#"
7325printf '%s\n' '{"type":"system","subtype":"init","session_id":"s1"}'
7326n=0
7327while IFS= read -r line; do
7328 n=$((n+1))
7329 printf '%s\n' '{"type":"user","message":{"role":"user"}}'
7330 printf '%s\n' "{\"type\":\"assistant\",\"message\":{\"content\":[{\"type\":\"text\",\"text\":\"reply-$n\"}]}}"
7331 printf '%s\n' "{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"result\":\"reply-$n\"}"
7332done
7333"#;
7334 let home = short_home("fresh");
7335 seed_stream_row(&home, "B", "swB");
7336 let _b = start_stream_worker(&home, "swB", COUNTING_EMITTER).await;
7337 let ctx = test_ctx(home.clone(), PathBuf::from("/nonexistent-worker"));
7338
7339 let r1 = handle_switchboard(
7340 &ctx,
7341 &Request::new(
7342 1,
7343 "agent.switchboard",
7344 switchboard_params("B", "swB", "ghost", None, "first"),
7345 ),
7346 )
7347 .await;
7348 assert_eq!(r1.result().expect("hop1")["reply"], "reply-1");
7349
7350 let r2 = handle_switchboard(
7351 &ctx,
7352 &Request::new(
7353 2,
7354 "agent.switchboard",
7355 switchboard_params("B", "swB", "ghost", None, "second"),
7356 ),
7357 )
7358 .await;
7359 assert_eq!(
7360 r2.result().expect("hop2")["reply"],
7361 "reply-2",
7362 "second drive returned a STALE reply (cursor not advanced past the prior turn)"
7363 );
7364 std::fs::remove_dir_all(home.root()).ok();
7365 }
7366
7367 #[tokio::test(flavor = "current_thread")]
7370 async fn switchboard_demotes_when_b_not_a_live_stream_thread() {
7371 let home = short_home("demote");
7372 seed_stream_row(&home, "B", "swB"); let ctx = test_ctx(home.clone(), PathBuf::from("/nonexistent-worker"));
7374
7375 let req = Request::new(
7376 1,
7377 "agent.switchboard",
7378 switchboard_params("B", "swB", "A", None, "hi"),
7379 );
7380 let resp = handle_switchboard(&ctx, &req).await;
7381 let res = resp
7382 .result()
7383 .expect("should be Ok-demote, not an RPC error");
7384 assert_eq!(res["delivered"], false);
7385 assert_eq!(res["reason"], "not-a-live-stream-thread");
7386 std::fs::remove_dir_all(home.root()).ok();
7387 }
7388
7389 #[tokio::test(flavor = "current_thread")]
7392 async fn switchboard_one_way_when_peer_absent() {
7393 let home = short_home("oneway");
7394 seed_stream_row(&home, "B", "swB");
7395 let _b = start_stream_worker(&home, "swB", FAKE_STREAM_EMITTER).await;
7396 let ctx = test_ctx(home.clone(), PathBuf::from("/nonexistent-worker"));
7397
7398 let req = Request::new(
7399 1,
7400 "agent.switchboard",
7401 switchboard_params("B", "swB", "ghost", None, "hi"),
7402 );
7403 let resp = handle_switchboard(&ctx, &req).await;
7404 let res = resp.result().expect("switchboard errored");
7405 assert_eq!(res["delivered"], true);
7406 assert_eq!(res["reply"], "reply-text");
7407 assert_eq!(res["mirrored"], false, "no peer to mirror into");
7408 std::fs::remove_dir_all(home.root()).ok();
7409 }
7410
7411 #[tokio::test(flavor = "current_thread")]
7413 async fn switchboard_unknown_target_is_not_found() {
7414 let home = short_home("404");
7415 let ctx = test_ctx(home.clone(), PathBuf::from("/nonexistent-worker"));
7416 let req = Request::new(
7417 1,
7418 "agent.switchboard",
7419 switchboard_params("nope", "swNope", "A", None, "hi"),
7420 );
7421 let resp = handle_switchboard(&ctx, &req).await;
7422 if let crate::protocol::ResponsePayload::Err(ref e) = resp.payload {
7423 assert_eq!(e.code, ErrorCode::AgentNotFound);
7424 } else {
7425 panic!("expected AgentNotFound, got {resp:?}");
7426 }
7427 std::fs::remove_dir_all(home.root()).ok();
7428 }
7429
7430 #[tokio::test(flavor = "current_thread")]
7431 async fn switchboard_refuses_replaced_recipient_identity() {
7432 let home = short_home("replaced");
7433 seed_stream_row(&home, "victim", "swB");
7434 let ctx = test_ctx(home.clone(), PathBuf::from("/nonexistent-worker"));
7435 let mut params = switchboard_params("victim", "swB", "ghost", None, "secret");
7436 params["recipient_identity"]["session_id"] = json!("uuid-swA");
7437
7438 let response =
7439 handle_switchboard(&ctx, &Request::new(1, "agent.switchboard", params)).await;
7440 let result = response.result().expect("identity mismatch is a demotion");
7441 assert_eq!(result["delivered"], false);
7442 assert_eq!(result["reason"], "recipient-identity-changed");
7443 std::fs::remove_dir_all(home.root()).ok();
7444 }
7445
7446 #[tokio::test(flavor = "current_thread")]
7447 async fn switchboard_failed_drive_does_not_orphan_restamped_recipient() {
7448 let home = short_home("failedrestamp");
7449 seed_stream_row(&home, "B", "swB");
7450 let turn_started = home.root().join("turn-started");
7451 let restamp_done = home.root().join("restamp-done");
7452 let script = format!(
7453 r#"
7454printf '%s\n' '{{"type":"system","subtype":"init","session_id":"s1"}}'
7455while IFS= read -r line; do
7456 touch '{}'
7457 while [ ! -f '{}' ]; do sleep 0.01; done
7458 exit 1
7459done
7460"#,
7461 turn_started.display(),
7462 restamp_done.display()
7463 );
7464 let _b = start_stream_worker(&home, "swB", &script).await;
7465 let ctx = test_ctx_with_events(home.clone(), PathBuf::from("/nonexistent-worker"));
7466
7467 let registry_path = home.registry_json();
7468 let restamp_signal = turn_started.clone();
7469 let restamp_complete = restamp_done.clone();
7470 let restamp = tokio::spawn(async move {
7471 let start = Instant::now();
7472 while !restamp_signal.exists() && start.elapsed() < Duration::from_secs(5) {
7473 tokio::time::sleep(Duration::from_millis(5)).await;
7474 }
7475 assert!(restamp_signal.exists(), "drive never reached the worker");
7476 state::update_registry(®istry_path, |registry| {
7477 let row = registry.find_mut("B").expect("recipient row missing");
7478 row.short_id = "swC".into();
7479 row.harness_session_id = Some("uuid-replacement".into());
7480 row.claude_session_uuid = Some("uuid-replacement".into());
7481 row.created_at = "2026-06-09T00:00:01Z".into();
7482 row.status = AgentStatus::Live;
7483 })
7484 .unwrap();
7485 std::fs::write(restamp_complete, b"done\n").unwrap();
7486 });
7487
7488 let response = handle_switchboard(
7489 &ctx,
7490 &Request::new(
7491 1,
7492 "agent.switchboard",
7493 switchboard_params("B", "swB", "ghost", None, "fail after receipt"),
7494 ),
7495 )
7496 .await;
7497 restamp.await.unwrap();
7498 let result = response.result().expect("failed drive is a demotion");
7499 assert_eq!(result["delivered"], false);
7500
7501 let registry = state::load_registry(&home.registry_json()).unwrap();
7502 let replacement = registry.find("B").expect("replacement row missing");
7503 assert_eq!(replacement.status, AgentStatus::Live);
7504 assert_eq!(
7505 replacement.harness_session_id.as_deref(),
7506 Some("uuid-replacement")
7507 );
7508 let events = read_events(&home);
7509 assert!(
7510 events.iter().any(|event| {
7511 event["type"] == "agent_deliver_status_write_failed"
7512 && event["data"]["name"] == "B"
7513 && event["data"]["reason"] == "recipient-identity-changed"
7514 }),
7515 "identity-CAS failure event missing: {events:?}"
7516 );
7517 std::fs::remove_dir_all(home.root()).ok();
7518 }
7519
7520 #[tokio::test(flavor = "current_thread")]
7521 async fn switchboard_requires_recipient_identity() {
7522 let home = short_home("identity-required");
7523 let ctx = test_ctx(home.clone(), PathBuf::from("/nonexistent-worker"));
7524 let response = handle_switchboard(
7525 &ctx,
7526 &Request::new(
7527 1,
7528 "agent.switchboard",
7529 json!({"to": "victim", "from": "ghost", "body": "secret", "mirror": false}),
7530 ),
7531 )
7532 .await;
7533 assert_eq!(
7534 response.error().expect("missing identity must fail").code,
7535 ErrorCode::InvalidParams
7536 );
7537 std::fs::remove_dir_all(home.root()).ok();
7538 }
7539
7540 #[tokio::test(flavor = "current_thread")]
7541 async fn switchboard_v2_routes_to_identity_guard() {
7542 let home = short_home("v2route");
7543 let ctx = Arc::new(test_ctx(home.clone(), PathBuf::from("/nonexistent-worker")));
7544 let response = dispatch_agent(
7545 &ctx,
7546 &Request::new(
7547 1,
7548 "agent.switchboard_v2",
7549 json!({"to": "victim", "from": "ghost", "body": "secret"}),
7550 ),
7551 )
7552 .await;
7553 let error = response.error().expect("missing identity must fail");
7554 assert_eq!(error.code, ErrorCode::InvalidParams);
7555 assert!(error.message.contains("recipient_identity"));
7556 std::fs::remove_dir_all(home.root()).ok();
7557 }
7558
7559 #[tokio::test(flavor = "current_thread")]
7563 async fn handle_spawn_codex_pty_hosting_retired_returns_pointer() {
7564 let home = tmp_home("spawn-provider-argv");
7565 let ctx = test_ctx(
7566 home.clone(),
7567 PathBuf::from("/nonexistent/fno-agents-worker"),
7568 );
7569 let req = Request::new(
7570 1,
7571 "agent.spawn",
7572 json!({"name": "test-agent", "provider": "codex"}),
7573 );
7574 let resp = handle_spawn(&ctx, &req).await;
7575 match &resp.payload {
7576 crate::protocol::ResponsePayload::Err(e) => {
7577 assert_eq!(e.code, ErrorCode::InvalidParams);
7578 assert!(
7579 e.message.contains("retired at G4"),
7580 "codex spawn must point at the mux; got: {}",
7581 e.message
7582 );
7583 }
7584 _ => panic!("expected the G4 retirement error for a codex PTY spawn"),
7585 }
7586 std::fs::remove_dir_all(home.root()).ok();
7587 }
7588
7589 #[tokio::test(flavor = "current_thread")]
7591 async fn handle_spawn_unknown_provider_no_argv_returns_invalid_params() {
7592 let home = tmp_home("spawn-unknown-provider");
7593 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
7594 let req = Request::new(
7595 1,
7596 "agent.spawn",
7597 json!({"name": "test-agent", "provider": "nonexistent-provider"}),
7598 );
7599 let resp = handle_spawn(&ctx, &req).await;
7600 match &resp.payload {
7601 crate::protocol::ResponsePayload::Err(e) => {
7602 assert_eq!(
7603 e.code,
7604 ErrorCode::InvalidParams,
7605 "unknown provider without argv must return InvalidParams"
7606 );
7607 }
7608 _ => panic!("expected error response for unknown provider"),
7609 }
7610 std::fs::remove_dir_all(home.root()).ok();
7611 }
7612
7613 #[tokio::test(flavor = "current_thread")]
7619 async fn handle_ask_first_contact_with_provider_routes_into_spawn() {
7620 let home = tmp_home("ask-first-contact");
7621 let ctx = test_ctx(
7622 home.clone(),
7623 PathBuf::from("/nonexistent/fno-agents-worker"),
7624 );
7625 let req = Request::new(
7627 1,
7628 "agent.ask",
7629 json!({"name": "new-agent", "message": "hello", "provider": "codex"}),
7630 );
7631 let resp = handle_ask(&ctx, &req).await;
7632 match &resp.payload {
7633 crate::protocol::ResponsePayload::Err(e) => {
7634 assert_ne!(
7635 e.code,
7636 ErrorCode::AgentNotFound,
7637 "first-contact ask with --provider must route into the spawn branch, not short-circuit AgentNotFound; got: {}",
7638 e.message
7639 );
7640 assert!(
7642 e.message.contains("retired at G4"),
7643 "first-contact codex spawn must surface the G4 mux pointer; got: {}",
7644 e.message
7645 );
7646 }
7647 crate::protocol::ResponsePayload::Ok(v) => {
7648 panic!("post-G4 a codex first-contact spawn must fail, got Ok: {v}")
7649 }
7650 }
7651 std::fs::remove_dir_all(home.root()).ok();
7652 }
7653
7654 #[tokio::test(flavor = "current_thread")]
7657 async fn handle_ask_first_contact_without_provider_returns_invalid_params() {
7658 let home = tmp_home("ask-no-provider");
7659 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
7660 let req = Request::new(
7662 1,
7663 "agent.ask",
7664 json!({"name": "ghost-agent", "message": "hello"}),
7665 );
7666 let resp = handle_ask(&ctx, &req).await;
7667 match &resp.payload {
7668 crate::protocol::ResponsePayload::Err(e) => {
7669 assert_eq!(
7670 e.code,
7671 ErrorCode::InvalidParams,
7672 "first-contact ask without --provider must return InvalidParams; got: {}",
7673 e.message
7674 );
7675 assert!(
7676 e.message.contains("provider"),
7677 "error message must mention 'provider', got: {}",
7678 e.message
7679 );
7680 }
7681 _ => panic!("expected error for first-contact ask without provider"),
7682 }
7683 std::fs::remove_dir_all(home.root()).ok();
7684 }
7685
7686 #[test]
7694 fn handle_report_stores_on_matching_row() {
7695 let home = tmp_home("report-store");
7696 seed_stream_row(&home, "worker-A", "repA"); let ctx = test_ctx_with_events(home.clone(), PathBuf::from("fno-agents-worker"));
7698 let req = Request::new(
7699 1,
7700 "agent.report",
7701 json!({"session_id": "uuid-repA", "seq": 3, "state": "working", "reason": "running tests"}),
7702 );
7703 let resp = handle_report(&ctx, &req);
7704 assert!(!resp.is_err(), "report must return Ok: {resp:?}");
7705 assert_eq!(resp.result().unwrap()["stored"], true);
7706
7707 let reg = state::load_registry(&home.registry_json()).unwrap();
7708 let rep = reg.entries[0]
7709 .inside_leg
7710 .as_ref()
7711 .expect("inside_leg stored");
7712 assert_eq!(rep.state, state::InsideLegState::Working);
7713 assert_eq!(rep.seq, 3);
7714 assert_eq!(rep.reason.as_deref(), Some("running tests"));
7715 assert!(!rep.received_at.is_empty(), "daemon stamps received_at");
7716
7717 let events = read_events(&home);
7718 assert!(
7719 events.iter().any(|e| e["type"] == "inside_leg_report"),
7720 "inside_leg_report not emitted: {events:?}"
7721 );
7722 std::fs::remove_dir_all(home.root()).ok();
7723 }
7724
7725 #[test]
7730 fn handle_report_capability_flip_clears_screen_state() {
7731 let home = tmp_home("report-flip-clears-scrape");
7732 seed_stream_row(&home, "worker-A", "repF");
7733 state::update_registry(&home.registry_json(), |r| {
7734 r.entries[0].screen_state = Some(state::ScreenStateReport {
7735 state: "idle".into(),
7736 rule: "idle_prompt".into(),
7737 seq: 4,
7738 at: "2026-07-02T00:00:00Z".into(),
7739 ttl_ms: Some(120_000),
7740 answerable: None,
7741 });
7742 })
7743 .unwrap();
7744 let ctx = test_ctx_with_events(home.clone(), PathBuf::from("fno-agents-worker"));
7745 let resp = handle_report(
7746 &ctx,
7747 &Request::new(
7748 1,
7749 "agent.report",
7750 json!({"session_id": "uuid-repF", "seq": 1, "state": "working"}),
7751 ),
7752 );
7753 assert_eq!(resp.result().unwrap()["stored"], true);
7754 let reg = state::load_registry(&home.registry_json()).unwrap();
7755 assert!(reg.entries[0].inside_leg.is_some());
7756 assert_eq!(
7757 reg.entries[0].screen_state, None,
7758 "capability flip must clear the scrape verdict"
7759 );
7760 std::fs::remove_dir_all(home.root()).ok();
7761 }
7762
7763 #[test]
7766 fn handle_report_drops_stale_seq() {
7767 let home = tmp_home("report-stale");
7768 seed_stream_row(&home, "worker-A", "repB");
7769 let ctx = test_ctx_with_events(home.clone(), PathBuf::from("fno-agents-worker"));
7770 let _ = handle_report(
7772 &ctx,
7773 &Request::new(
7774 1,
7775 "agent.report",
7776 json!({"session_id": "uuid-repB", "seq": 2, "state": "working"}),
7777 ),
7778 );
7779 let resp = handle_report(
7780 &ctx,
7781 &Request::new(
7782 2,
7783 "agent.report",
7784 json!({"session_id": "uuid-repB", "seq": 1, "state": "done"}),
7785 ),
7786 );
7787 assert!(!resp.is_err());
7788 assert_eq!(resp.result().unwrap()["stored"], false);
7789 assert_eq!(resp.result().unwrap()["dropped"], "stale_seq");
7790
7791 let reg = state::load_registry(&home.registry_json()).unwrap();
7793 let rep = reg.entries[0].inside_leg.as_ref().unwrap();
7794 assert_eq!(rep.seq, 2);
7795 assert_eq!(rep.state, state::InsideLegState::Working);
7796
7797 let events = read_events(&home);
7798 assert!(events.iter().any(
7799 |e| e["type"] == "inside_leg_report_dropped" && e["data"]["reason"] == "stale_seq"
7800 ));
7801 std::fs::remove_dir_all(home.root()).ok();
7802 }
7803
7804 #[test]
7808 fn handle_report_buffers_early_push_for_unknown_session() {
7809 let home = tmp_home("report-unknown");
7810 let ctx = test_ctx_with_events(home.clone(), PathBuf::from("fno-agents-worker"));
7811 let resp = handle_report(
7812 &ctx,
7813 &Request::new(
7814 1,
7815 "agent.report",
7816 json!({"session_id": "uuid-nope", "seq": 1, "state": "working"}),
7817 ),
7818 );
7819 assert!(!resp.is_err());
7820 assert_eq!(resp.result().unwrap()["stored"], false);
7821 assert_eq!(
7822 resp.result().unwrap()["buffered"],
7823 true,
7824 "an early push is held, not dropped (E3.3)"
7825 );
7826
7827 let reg = state::load_registry(&home.registry_json()).unwrap();
7828 assert!(reg.entries.is_empty(), "no phantom row created");
7829 assert_eq!(
7831 ctx.pending_inside_leg
7832 .lock()
7833 .unwrap()
7834 .get("uuid-nope")
7835 .map(|r| r.seq),
7836 Some(1)
7837 );
7838
7839 let events = read_events(&home);
7840 assert!(events
7841 .iter()
7842 .any(|e| e["type"] == "inside_leg_report_buffered"
7843 && e["data"]["session_id"] == "uuid-nope"));
7844 std::fs::remove_dir_all(home.root()).ok();
7845 }
7846
7847 #[test]
7849 fn handle_report_rejects_bad_params() {
7850 let home = tmp_home("report-bad");
7851 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
7852 for params in [
7853 json!({"seq": 1, "state": "working"}), json!({"session_id": "x", "state": "working"}), json!({"session_id": "x", "seq": 1}), json!({"session_id": "x", "seq": 1, "state": "idle"}), ] {
7858 let resp = handle_report(&ctx, &Request::new(1, "agent.report", params.clone()));
7859 assert!(resp.is_err(), "expected InvalidParams for {params}");
7860 }
7861 std::fs::remove_dir_all(home.root()).ok();
7862 }
7863
7864 #[test]
7867 fn push_to_channel_rejects_non_object_envelope() {
7868 let home = tmp_home("push-badenv");
7869 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
7870 let resp = handle_push_to_channel(
7871 &ctx,
7872 &Request::new(
7873 1,
7874 "channel.push_to_channel",
7875 json!({"mcp_channel_id": "c1", "envelope": "not-an-object"}),
7876 ),
7877 );
7878 assert!(resp.is_err(), "non-object envelope must be InvalidParams");
7879 std::fs::remove_dir_all(home.root()).ok();
7880 }
7881
7882 #[test]
7885 fn push_to_channel_unknown_channel_errors() {
7886 let home = tmp_home("push-unknown");
7887 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
7888 let resp = handle_push_to_channel(
7889 &ctx,
7890 &Request::new(
7891 1,
7892 "channel.push_to_channel",
7893 json!({"mcp_channel_id": "nope", "envelope": {"a": 1}}),
7894 ),
7895 );
7896 assert!(resp.is_err(), "unknown channel must error");
7897 std::fs::remove_dir_all(home.root()).ok();
7898 }
7899
7900 #[test]
7903 fn push_to_channel_no_envelope_is_confirm_only() {
7904 let home = tmp_home("push-confirm");
7905 seed_stream_row(&home, "worker-c", "chA");
7906 state::update_registry(&home.registry_json(), |r| {
7907 r.entries[0].mcp_channel_id = Some("c1".into());
7908 })
7909 .unwrap();
7910 let ctx = test_ctx(home.clone(), PathBuf::from("fno-agents-worker"));
7911 let resp = handle_push_to_channel(
7912 &ctx,
7913 &Request::new(
7914 1,
7915 "channel.push_to_channel",
7916 json!({"mcp_channel_id": "c1"}),
7917 ),
7918 );
7919 let result = resp.result().unwrap();
7920 assert_eq!(result["routed"], true);
7921 assert!(
7922 result.get("delivered").is_none(),
7923 "confirm-only must not claim delivery"
7924 );
7925 std::fs::remove_dir_all(home.root()).ok();
7926 }
7927}