use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
use serde::Deserialize;
use serde_json::json;
use crate::claims::{self, ClaimState};
use crate::events::EventEmitter;
use crate::loop_dispatch::{fno_cmd, retry_etxtbsy};
use crate::loop_runtime::{
CloseOutcome, Evidence, GlobalJournalPath, Journal, ProjectJournalPath, UnitResult,
};
use crate::loopcheck::TerminationReason;
#[derive(Debug, Default)]
pub struct CircuitBreaker {
failure_limit: u32,
failures: HashMap<String, u32>,
}
impl CircuitBreaker {
pub fn new(failure_limit: u32) -> Self {
Self {
failure_limit: failure_limit.max(1),
failures: HashMap::new(),
}
}
pub fn record_failure(&mut self, node: &str) -> bool {
let n = self.failures.entry(node.to_string()).or_insert(0);
*n += 1;
*n >= self.failure_limit
}
pub fn record_success(&mut self, node: &str) {
self.failures.remove(node);
}
pub fn reset(&mut self, node: &str) {
self.failures.remove(node);
}
pub fn consecutive_failures(&self, node: &str) -> u32 {
self.failures.get(node).copied().unwrap_or(0)
}
}
#[derive(Debug, Clone)]
pub struct DrainConfig {
pub cwd: PathBuf,
pub fno_bin: String,
pub mission: String,
pub failure_limit: u32,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DrainOutcome {
Dispatched { node: String },
Parked { node: String, failures: u32 },
NoWork,
Skipped { reason: String },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MissionDispatch {
Continue,
Retire,
}
fn defer_node(fno_bin: &str, cwd: &Path, node: &str, reason: &str) -> bool {
match retry_etxtbsy(|| {
fno_cmd(fno_bin)
.current_dir(cwd)
.args(["backlog", "defer", node, "--reason", reason])
.output()
}) {
Ok(out) if out.status.success() => true,
Ok(out) => {
eprintln!(
"active-backlog: defer of {node} failed (exit {:?}): {}",
out.status.code(),
String::from_utf8_lossy(&out.stderr).trim()
);
false
}
Err(e) => {
eprintln!("active-backlog: defer of {node} could not run: {e}");
false
}
}
}
fn node_has_pr_ref(cfg: &DrainConfig, node_id: &str) -> bool {
let Ok(out) = retry_etxtbsy(|| {
fno_cmd(&cfg.fno_bin)
.args(["backlog", "get", node_id])
.current_dir(&cfg.cwd)
.output()
}) else {
return true;
};
if !out.status.success() {
return true;
}
let Ok(v) = serde_json::from_slice::<serde_json::Value>(&out.stdout) else {
return true;
};
if v.get("pr_number").and_then(|n| n.as_u64()).is_some() {
return true;
}
if v.get("pr_url")
.and_then(|u| u.as_str())
.is_some_and(|u| !u.trim().is_empty())
{
return true;
}
v.get("additional_prs")
.and_then(|a| a.as_array())
.is_some_and(|a| !a.is_empty())
}
const PR_STAMP_GRACE_TICKS: u32 = 3;
fn map_outcome(
cfg: &DrainConfig,
breaker: &mut CircuitBreaker,
journal: &Journal,
reason: &TerminationReason,
last_unit: Option<&crate::loop_runtime::UnitResult>,
) -> DrainOutcome {
let Some(last) = last_unit else {
return match reason {
TerminationReason::NoWork => DrainOutcome::NoWork,
other => {
let _ = journal.append(
"active_backlog_skip",
json!({"reason": "no-close", "termination": format!("{other:?}")}),
);
DrainOutcome::Skipped {
reason: format!("{other:?}"),
}
}
};
};
let node = last.unit_id.clone();
if matches!(last.evidence.reason, TerminationReason::DoneBatched) {
breaker.record_success(&node);
let _ = journal.append(
"active_backlog_dispatched",
json!({"node_id": node, "termination": "DoneBatched", "batched": true}),
);
return DrainOutcome::Dispatched { node };
}
if matches!(last.evidence.reason, TerminationReason::DoneAwaitingMerge) {
breaker.record_success(&node);
let _ = journal.append(
"active_backlog_dispatched",
json!({"node_id": node, "termination": "DoneAwaitingMerge", "awaiting_merge": true}),
);
return DrainOutcome::Dispatched { node };
}
match &last.close {
CloseOutcome::Closed => {
breaker.record_success(&node);
let _ = journal.append(
"active_backlog_dispatched",
json!({"node_id": node, "termination": format!("{:?}", last.evidence.reason)}),
);
DrainOutcome::Dispatched { node }
}
CloseOutcome::AwaitingMerge => {
breaker.record_success(&node);
let _ = journal.append(
"active_backlog_dispatched",
json!({"node_id": node, "awaiting_merge": true, "close": "awaiting-merge"}),
);
DrainOutcome::Dispatched { node }
}
CloseOutcome::Parked(detail) | CloseOutcome::Refused(detail) => {
let tripped = breaker.record_failure(&node);
if tripped {
let reason_str = format!(
"auto-failure: {} consecutive failed drains",
cfg.failure_limit
);
let deferred = defer_node(&cfg.fno_bin, &cfg.cwd, &node, &reason_str);
breaker.reset(&node);
let _ = journal.append(
"active_backlog_parked",
json!({"node_id": node, "consecutive_failures": cfg.failure_limit, "detail": detail, "deferred": deferred}),
);
DrainOutcome::Parked {
node,
failures: cfg.failure_limit,
}
} else {
let _ = journal.append(
"active_backlog_skip",
json!({
"reason": "node-not-closed",
"node_id": node,
"close": detail,
"consecutive_failures": breaker.consecutive_failures(&node),
}),
);
DrainOutcome::Skipped {
reason: format!("node {node} not closed: {detail}"),
}
}
}
}
}
#[derive(Debug, Clone)]
pub struct PendingDispatch {
node_id: String,
session_id: Option<String>,
ticks: u32,
stamp_waits: u32,
}
const BOOT_GRACE_TICKS: u32 = 3;
fn is_done_reason(r: &TerminationReason) -> bool {
matches!(
r,
TerminationReason::DonePRGreen
| TerminationReason::DoneAdvisory
| TerminationReason::DoneDelivery
)
}
fn reconcile_pending(
cfg: &DrainConfig,
breaker: &mut CircuitBreaker,
pending: &mut Vec<PendingDispatch>,
journal: &Journal,
) {
pending.retain_mut(|p| {
p.ticks += 1;
let (state, rec) = claims::status(&format!("node:{}", p.node_id), None);
if let Some(sid) = rec
.as_ref()
.and_then(|r| r.holder.strip_prefix("target-session:"))
{
p.session_id = Some(sid.to_string());
}
let worker_live = matches!(state, ClaimState::Live | ClaimState::Suspect);
if let Some(sid) = p.session_id.clone() {
match journal.find_termination(&sid) {
Ok(Some(ev)) => {
if matches!(ev.reason, TerminationReason::DonePRGreen)
&& !node_has_pr_ref(cfg, &p.node_id)
&& p.stamp_waits < PR_STAMP_GRACE_TICKS
{
p.stamp_waits += 1;
return true;
}
resolve_dispatch(cfg, breaker, journal, &p.node_id, ev);
return false;
}
Ok(None) if !worker_live => {
resolve_crash(cfg, breaker, journal, &p.node_id);
return false;
}
_ => {} }
} else if !worker_live && p.ticks >= BOOT_GRACE_TICKS {
resolve_crash(cfg, breaker, journal, &p.node_id);
return false;
}
true
});
}
fn resolve_dispatch(
cfg: &DrainConfig,
breaker: &mut CircuitBreaker,
journal: &Journal,
node_id: &str,
ev: Evidence,
) {
let close = if matches!(ev.reason, TerminationReason::DonePRGreen)
&& !node_has_pr_ref(cfg, node_id)
{
CloseOutcome::Parked(
"DonePRGreen terminal with no PR ref on the node (zero-artifact dispatch)".to_string(),
)
} else if is_done_reason(&ev.reason) {
match retry_etxtbsy(|| {
fno_cmd(&cfg.fno_bin)
.args(["backlog", "done", node_id])
.current_dir(&cfg.cwd)
.output()
}) {
Ok(o) if o.status.success() => CloseOutcome::Closed,
Ok(o) if o.status.code() == Some(5) => CloseOutcome::AwaitingMerge,
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).trim().to_string();
CloseOutcome::Parked(if stderr.is_empty() {
format!("fno backlog done {node_id} failed (exit {})", o.status)
} else {
stderr
})
}
Err(e) => CloseOutcome::Parked(format!("fno backlog done {node_id} spawn failed: {e}")),
}
} else {
CloseOutcome::Parked(format!("session terminated: {:?}", ev.reason))
};
let reason = ev.reason.clone();
let ur = UnitResult {
unit_id: node_id.to_string(),
evidence: ev,
close,
};
map_outcome(cfg, breaker, journal, &reason, Some(&ur));
}
fn resolve_crash(
cfg: &DrainConfig,
breaker: &mut CircuitBreaker,
journal: &Journal,
node_id: &str,
) {
let message = "worker exited with no termination event (fire-and-forget crash floor)";
let ur = UnitResult {
unit_id: node_id.to_string(),
evidence: Evidence {
reason: TerminationReason::NoProgress,
message: message.to_string(),
},
close: CloseOutcome::Parked(message.to_string()),
};
map_outcome(
cfg,
breaker,
journal,
&TerminationReason::NoProgress,
Some(&ur),
);
}
#[derive(Debug, Default, Deserialize)]
struct AdvanceEpicReceipt {
#[serde(default)]
deactivated: bool,
#[serde(default)]
all_done: bool,
#[serde(default)]
dispatched: Vec<String>,
}
fn dispatch_mission(
cfg: &DrainConfig,
pending: &mut Vec<PendingDispatch>,
journal: &Journal,
) -> MissionDispatch {
let out = match retry_etxtbsy(|| {
fno_cmd(&cfg.fno_bin)
.args([
"backlog",
"advance",
"--epic",
&cfg.mission,
"--continuation",
"--json",
])
.current_dir(&cfg.cwd)
.output()
}) {
Ok(o) if o.status.success() => o,
Ok(o) => {
let detail = String::from_utf8_lossy(&o.stderr).trim().to_string();
let _ = journal.append(
"active_backlog_skip",
json!({"reason": "advance-epic-failed", "mission": cfg.mission, "detail": detail}),
);
return MissionDispatch::Continue;
}
Err(e) => {
let _ = journal.append(
"active_backlog_skip",
json!({"reason": "advance-epic-failed", "mission": cfg.mission, "detail": format!("{e}")}),
);
return MissionDispatch::Continue;
}
};
let receipt: AdvanceEpicReceipt = match serde_json::from_slice(&out.stdout) {
Ok(r) => r,
Err(e) => {
let _ = journal.append(
"active_backlog_skip",
json!({"reason": "advance-epic-unparseable", "mission": cfg.mission, "detail": format!("{e}")}),
);
return MissionDispatch::Continue;
}
};
if receipt.deactivated || receipt.all_done {
return MissionDispatch::Retire;
}
let mut new_ids = Vec::new();
for node_id in &receipt.dispatched {
if pending.iter().any(|p| p.node_id == *node_id) {
continue;
}
pending.push(PendingDispatch {
node_id: node_id.clone(),
session_id: None,
ticks: 0,
stamp_waits: 0,
});
new_ids.push(node_id.clone());
}
if !new_ids.is_empty() {
let _ = journal.append(
"active_backlog_dispatched",
json!({"mission": cfg.mission, "dispatched": new_ids, "fire_and_forget": true}),
);
}
MissionDispatch::Continue
}
pub fn mission_drain_tick(
cfg: &DrainConfig,
breaker: &mut CircuitBreaker,
pending: &mut Vec<PendingDispatch>,
journal: &Journal,
) -> MissionDispatch {
reconcile_pending(cfg, breaker, pending, journal);
dispatch_mission(cfg, pending, journal)
}
#[derive(Debug, Clone, Deserialize)]
pub struct ResolvedTarget {
pub project: String,
pub cwd: String,
pub interval_seconds: u64,
pub failure_limit: u32,
#[serde(default)]
pub mission: Option<String>,
}
pub fn resolve_targets(fno_bin: &str) -> Vec<ResolvedTarget> {
match fno_cmd(fno_bin)
.args(["config", "active-backlog", "--json"])
.output()
{
Ok(o) if o.status.success() => serde_json::from_slice(&o.stdout).unwrap_or_default(),
_ => Vec::new(),
}
}
#[derive(Debug, Clone, serde::Deserialize)]
struct FanoutTarget {
pub project: String,
pub cwd: String,
pub interval_seconds: u64,
}
fn resolve_fanout_targets(fno_bin: &str) -> Vec<FanoutTarget> {
match fno_cmd(fno_bin)
.args(["config", "status-sinks", "--json"])
.output()
{
Ok(o) if o.status.success() => serde_json::from_slice(&o.stdout).unwrap_or_default(),
_ => Vec::new(),
}
}
const TICK_CHILD_CAP: Duration = Duration::from_secs(300);
async fn output_with_cap(mut cmd: tokio::process::Command, cap: Duration) -> bool {
cmd.kill_on_drop(true);
match tokio::time::timeout(cap, cmd.output()).await {
Ok(Ok(_)) => false,
Ok(Err(e)) => {
eprintln!("fanout tick failed to execute: {e}");
false
}
Err(_) => true,
}
}
async fn per_project_fanout_loop(target: FanoutTarget, fno_bin: String, shutdown: Arc<AtomicBool>) {
let project = target.project.clone();
loop {
if shutdown.load(Ordering::SeqCst) {
break;
}
let interval = match resolve_fanout_targets(&fno_bin)
.into_iter()
.find(|t| t.project == project)
{
Some(t) => Duration::from_secs(t.interval_seconds.max(1)),
None => break, };
let mut cmd = tokio::process::Command::new(&fno_bin);
cmd.args(["status-fanout", "tick"]).current_dir(&target.cwd);
if output_with_cap(cmd, TICK_CHILD_CAP).await {
eprintln!(
"fanout tick for {project} exceeded {TICK_CHILD_CAP:?}; killed, retrying next tick"
);
}
sleep_interruptible(interval, &shutdown).await;
}
}
fn journal_for(cwd: &Path) -> Journal {
let project_events = cwd.join(".fno").join("events.jsonl");
let home = std::env::var("HOME")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("/tmp"));
let global_events = home.join(".fno").join("events.jsonl");
Journal::new(
ProjectJournalPath(project_events),
GlobalJournalPath(global_events),
)
}
fn drain_config_for(target: &ResolvedTarget, fno_bin: &str) -> Option<DrainConfig> {
let mission = target.mission.clone()?;
Some(DrainConfig {
cwd: PathBuf::from(&target.cwd),
fno_bin: fno_bin.to_string(),
mission,
failure_limit: target.failure_limit,
})
}
fn nudge_sentinel_path() -> PathBuf {
let home = std::env::var("HOME")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("/tmp"));
home.join(".fno").join(".active-backlog-nudge")
}
async fn nudge_mtime() -> Option<std::time::SystemTime> {
tokio::task::spawn_blocking(|| {
std::fs::metadata(nudge_sentinel_path())
.and_then(|m| m.modified())
.ok()
})
.await
.ok()
.flatten()
}
async fn wait_for_wake(
total: Duration,
shutdown: &Arc<AtomicBool>,
last: &mut Option<std::time::SystemTime>,
) {
let step = Duration::from_millis(500);
let mut elapsed = Duration::ZERO;
while elapsed < total {
if shutdown.load(Ordering::SeqCst) {
return;
}
let current = nudge_mtime().await;
if current != *last {
*last = current;
return; }
let chunk = step.min(total - elapsed);
tokio::time::sleep(chunk).await;
elapsed += chunk;
}
}
pub async fn run_supervisor(
fno_bin: String,
emitter: EventEmitter,
live: Arc<AtomicBool>,
shutdown: Arc<AtomicBool>,
) {
let mut tasks: HashMap<String, tokio::task::JoinHandle<()>> = HashMap::new();
let mut fanout_tasks: HashMap<String, tokio::task::JoinHandle<()>> = HashMap::new();
let recheck = Duration::from_secs(60);
loop {
if shutdown.load(Ordering::SeqCst) {
break;
}
tasks.retain(|_, h| !h.is_finished());
fanout_tasks.retain(|_, h| !h.is_finished());
let targets = resolve_targets(&fno_bin);
let fanout_targets = resolve_fanout_targets(&fno_bin);
live.store(
!targets.is_empty() || !fanout_targets.is_empty(),
Ordering::SeqCst,
);
for target in targets {
let Some(mission) = target.mission.clone() else {
continue;
};
if let std::collections::hash_map::Entry::Vacant(slot) = tasks.entry(mission) {
slot.insert(tokio::spawn(mission_drain_loop(
target,
fno_bin.clone(),
emitter.clone(),
Arc::clone(&shutdown),
)));
}
}
for ft in fanout_targets {
if let std::collections::hash_map::Entry::Vacant(slot) =
fanout_tasks.entry(ft.project.clone())
{
slot.insert(tokio::spawn(per_project_fanout_loop(
ft,
fno_bin.clone(),
Arc::clone(&shutdown),
)));
}
}
sleep_interruptible(recheck, &shutdown).await;
}
for (_, h) in tasks {
h.abort();
}
for (_, h) in fanout_tasks {
h.abort();
}
live.store(false, Ordering::SeqCst);
}
async fn sleep_interruptible(total: Duration, shutdown: &Arc<AtomicBool>) {
let step = Duration::from_millis(500);
let mut elapsed = Duration::ZERO;
while elapsed < total {
if shutdown.load(Ordering::SeqCst) {
return;
}
let chunk = step.min(total - elapsed);
tokio::time::sleep(chunk).await;
elapsed += chunk;
}
}
async fn mission_drain_loop(
target: ResolvedTarget,
fno_bin: String,
emitter: EventEmitter,
shutdown: Arc<AtomicBool>,
) {
let mission = target.mission.clone().unwrap_or_default();
let mut breaker = CircuitBreaker::new(target.failure_limit);
let mut pending: Vec<PendingDispatch> = Vec::new();
let mut last_nudge = nudge_mtime().await;
let mut backoff = Duration::from_secs(1);
loop {
if shutdown.load(Ordering::SeqCst) {
break;
}
let current = resolve_targets(&fno_bin)
.into_iter()
.find(|t| t.mission.as_deref() == Some(mission.as_str()));
let Some(t) = current else {
break;
};
let interval = Duration::from_secs(t.interval_seconds.max(1));
let Some(cfg) = drain_config_for(&t, &fno_bin) else {
sleep_interruptible(interval, &shutdown).await;
continue;
};
let journal = journal_for(&cfg.cwd);
let taken_b = std::mem::take(&mut breaker);
let taken_p = std::mem::take(&mut pending);
let handle = tokio::task::spawn_blocking(move || {
let mut b = taken_b;
let mut p = taken_p;
let outcome = mission_drain_tick(&cfg, &mut b, &mut p, &journal);
(outcome, b, p)
});
match handle.await {
Ok((outcome, b, p)) => {
breaker = b;
pending = p;
backoff = Duration::from_secs(1);
if outcome == MissionDispatch::Retire {
let _ = emitter.emit(
"active_backlog_mission_retired",
&json!({"mission": mission}),
);
break;
}
}
Err(join_err) => {
let _ = emitter.emit(
"active_backlog_task_crashed",
&json!({"mission": mission, "error": join_err.to_string()}),
);
breaker = CircuitBreaker::new(t.failure_limit);
pending = Vec::new();
sleep_interruptible(backoff, &shutdown).await;
backoff = (backoff * 2).min(Duration::from_secs(60));
continue;
}
}
wait_for_wake(interval, &shutdown, &mut last_nudge).await;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn status_fanout_targets_parse_from_json() {
let json = br#"[{"project":"fno","cwd":"/repo/fno","interval_seconds":5}]"#;
let targets: Vec<FanoutTarget> = serde_json::from_slice(json).unwrap();
assert_eq!(targets.len(), 1);
assert_eq!(targets[0].project, "fno");
assert_eq!(targets[0].cwd, "/repo/fno");
assert_eq!(targets[0].interval_seconds, 5);
}
#[test]
fn status_fanout_targets_empty_on_garbage() {
let targets: Vec<FanoutTarget> = serde_json::from_slice(b"not json").unwrap_or_default();
assert!(targets.is_empty());
}
#[tokio::test]
async fn tick_child_killed_at_cap() {
let mut cmd = tokio::process::Command::new("sleep");
cmd.arg("60");
let start = std::time::Instant::now();
let timed_out = output_with_cap(cmd, Duration::from_millis(150)).await;
assert!(timed_out, "a hung child must report timed-out");
assert!(
start.elapsed() < Duration::from_secs(5),
"must return near the cap, not wait on the 60s child"
);
}
#[tokio::test]
async fn tick_child_within_cap_reports_ok() {
let cmd = tokio::process::Command::new("true");
let timed_out = output_with_cap(cmd, Duration::from_secs(30)).await;
assert!(!timed_out, "a fast child must not be reported as timed-out");
}
#[test]
fn advance_epic_receipt_parses_dispatched_and_liveness() {
let r: AdvanceEpicReceipt = serde_json::from_slice(
br#"{"epic_id":"x-e","error":null,"activated":true,"deactivated":false,
"all_done":false,"dispatched":["x-a","x-b"],"children":[]}"#,
)
.unwrap();
assert_eq!(r.dispatched, vec!["x-a", "x-b"]);
assert!(!r.deactivated);
assert!(!r.all_done);
}
#[test]
fn advance_epic_receipt_defaults_on_partial_json() {
let r: AdvanceEpicReceipt = serde_json::from_slice(br#"{"epic_id":"x-e"}"#).unwrap();
assert!(r.dispatched.is_empty());
assert!(!r.deactivated && !r.all_done);
}
#[test]
fn is_done_reason_includes_generic_delivery() {
assert!(is_done_reason(&TerminationReason::DonePRGreen));
assert!(is_done_reason(&TerminationReason::DoneAdvisory));
assert!(is_done_reason(&TerminationReason::DoneDelivery));
assert!(!is_done_reason(&TerminationReason::DoneBatched));
assert!(!is_done_reason(&TerminationReason::DoneAwaitingMerge));
assert!(!is_done_reason(&TerminationReason::NoProgress));
}
use std::os::unix::fs::PermissionsExt;
fn env_guard() -> std::sync::MutexGuard<'static, ()> {
crate::claims::test_env_lock()
.lock()
.unwrap_or_else(|e| e.into_inner())
}
fn stub_fno(dir: &std::path::Path, record: &std::path::Path) -> String {
std::fs::create_dir_all(dir).unwrap();
let p = dir.join("fno");
std::fs::write(
&p,
format!(
"#!/usr/bin/env bash\necho \"$@\" >> \"{}\"\nexit 0\n",
record.display()
),
)
.unwrap();
std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap();
p.display().to_string()
}
fn stub_fno_defer_fails(dir: &std::path::Path, record: &std::path::Path) -> String {
std::fs::create_dir_all(dir).unwrap();
let p = dir.join("fno");
std::fs::write(
&p,
format!(
"#!/usr/bin/env bash\n\
echo \"$@\" >> \"{}\"\n\
if [ \"$2\" = \"defer\" ]; then echo 'node not found' >&2; exit 1; fi\n\
exit 0\n",
record.display()
),
)
.unwrap();
std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap();
p.display().to_string()
}
fn stub_fno_get(dir: &std::path::Path, record: &std::path::Path, node_json: &str) -> String {
std::fs::create_dir_all(dir).unwrap();
let p = dir.join("fno");
std::fs::write(
&p,
format!(
"#!/usr/bin/env bash\n\
if [ \"$2\" = \"get\" ]; then printf '%s' '{}'; exit 0; fi\n\
echo \"$@\" >> \"{}\"\nexit 0\n",
node_json,
record.display()
),
)
.unwrap();
std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap();
p.display().to_string()
}
fn test_cfg(tmp: &std::path::Path, fno_bin: String, failure_limit: u32) -> DrainConfig {
DrainConfig {
cwd: tmp.to_path_buf(),
fno_bin,
mission: "x-epic".to_string(),
failure_limit,
}
}
fn test_journal(tmp: &std::path::Path) -> (Journal, PathBuf) {
let project = tmp.join(".fno").join("events.jsonl");
let global = tmp.join("global-events.jsonl");
std::fs::create_dir_all(project.parent().unwrap()).unwrap();
(Journal::new_raw(project.clone(), global), project)
}
fn journal_lines(p: &std::path::Path) -> Vec<String> {
std::fs::read_to_string(p)
.unwrap_or_default()
.lines()
.map(str::to_string)
.collect()
}
#[test]
fn resolve_dispatch_done_records_success_and_marks_done() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno(&tmp.path().join("bin"), &record);
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, project_journal) = test_journal(tmp.path());
let mut breaker = CircuitBreaker::new(3);
breaker.record_failure("x-suc0001");
resolve_dispatch(
&cfg,
&mut breaker,
&journal,
"x-suc0001",
Evidence {
reason: TerminationReason::DonePRGreen,
message: "done".to_string(),
},
);
assert_eq!(
breaker.consecutive_failures("x-suc0001"),
0,
"success resets the streak"
);
let calls = std::fs::read_to_string(&record).unwrap_or_default();
assert!(calls.contains("backlog done x-suc0001"), "calls: {calls}");
assert!(journal_lines(&project_journal)
.iter()
.any(|l| l.contains("active_backlog_dispatched") && l.contains("x-suc0001")));
}
#[test]
fn resolve_dispatch_done_pr_green_without_pr_ref_is_a_failure() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno_get(
&tmp.path().join("bin"),
&record,
r#"{"id":"x-dead0001","status":"in_review"}"#,
);
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, project_journal) = test_journal(tmp.path());
let mut breaker = CircuitBreaker::new(3);
resolve_dispatch(
&cfg,
&mut breaker,
&journal,
"x-dead0001",
Evidence {
reason: TerminationReason::DonePRGreen,
message: "promised".to_string(),
},
);
assert_eq!(
breaker.consecutive_failures("x-dead0001"),
1,
"a zero-artifact DonePRGreen counts toward the streak"
);
let calls = std::fs::read_to_string(&record).unwrap_or_default();
assert!(
!calls.contains("backlog done"),
"must not close a node whose terminal lied: {calls}"
);
assert!(journal_lines(&project_journal)
.iter()
.any(|l| l.contains("active_backlog_skip") && l.contains("x-dead0001")));
}
#[test]
fn resolve_dispatch_done_pr_green_with_pr_ref_is_success() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno_get(
&tmp.path().join("bin"),
&record,
r#"{"id":"x-live0001","pr_number":477}"#,
);
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, _pj) = test_journal(tmp.path());
let mut breaker = CircuitBreaker::new(3);
resolve_dispatch(
&cfg,
&mut breaker,
&journal,
"x-live0001",
Evidence {
reason: TerminationReason::DonePRGreen,
message: String::new(),
},
);
assert_eq!(breaker.consecutive_failures("x-live0001"), 0);
let calls = std::fs::read_to_string(&record).unwrap_or_default();
assert!(calls.contains("backlog done x-live0001"), "calls: {calls}");
}
#[test]
fn zero_artifact_check_fails_open_on_unreadable_node() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno(&tmp.path().join("bin"), &record);
let cfg = test_cfg(tmp.path(), fno, 3);
assert!(
node_has_pr_ref(&cfg, "x-unknown1"),
"unreadable node must fail open"
);
}
#[test]
fn pr_ref_read_unions_additional_prs() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno_get(
&tmp.path().join("bin"),
&record,
r#"{"id":"x-addl0001","additional_prs":[{"number":12}]}"#,
);
let cfg = test_cfg(tmp.path(), fno, 3);
assert!(node_has_pr_ref(&cfg, "x-addl0001"));
}
#[test]
fn empty_pr_url_is_not_a_ref() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno_get(
&tmp.path().join("bin"),
&record,
r#"{"id":"x-empt0001","pr_url":" "}"#,
);
let cfg = test_cfg(tmp.path(), fno, 3);
assert!(!node_has_pr_ref(&cfg, "x-empt0001"));
}
#[test]
fn resolve_dispatch_advisory_without_pr_ref_is_still_success() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno_get(
&tmp.path().join("bin"),
&record,
r#"{"id":"x-doc00001","status":"in_review"}"#,
);
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, _pj) = test_journal(tmp.path());
let mut breaker = CircuitBreaker::new(3);
resolve_dispatch(
&cfg,
&mut breaker,
&journal,
"x-doc00001",
Evidence {
reason: TerminationReason::DoneAdvisory,
message: String::new(),
},
);
assert_eq!(breaker.consecutive_failures("x-doc00001"), 0);
let calls = std::fs::read_to_string(&record).unwrap_or_default();
assert!(calls.contains("backlog done x-doc00001"), "calls: {calls}");
}
#[test]
fn resolve_dispatch_awaiting_merge_is_success_without_done() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno(&tmp.path().join("bin"), &record);
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, _pj) = test_journal(tmp.path());
let mut breaker = CircuitBreaker::new(3);
breaker.record_failure("x-awm0001");
resolve_dispatch(
&cfg,
&mut breaker,
&journal,
"x-awm0001",
Evidence {
reason: TerminationReason::DoneAwaitingMerge,
message: String::new(),
},
);
assert_eq!(breaker.consecutive_failures("x-awm0001"), 0);
let calls = std::fs::read_to_string(&record).unwrap_or_default();
assert!(
!calls.contains("backlog done"),
"awaiting-merge must not mark done: {calls}"
);
}
#[test]
fn resolve_dispatch_failed_done_records_failure_not_false_success() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let bin = tmp.path().join("bin");
std::fs::create_dir_all(&bin).unwrap();
let fno = bin.join("fno");
std::fs::write(
&fno,
"#!/usr/bin/env bash\nif [[ \"$1\" == backlog && \"$2\" == done ]]; then echo 'node has open blockers' >&2; exit 1; fi\nexit 0\n",
)
.unwrap();
std::fs::set_permissions(&fno, std::fs::Permissions::from_mode(0o755)).unwrap();
let cfg = test_cfg(tmp.path(), fno.display().to_string(), 3);
let (journal, _pj) = test_journal(tmp.path());
let mut breaker = CircuitBreaker::new(3);
resolve_dispatch(
&cfg,
&mut breaker,
&journal,
"x-donefail",
Evidence {
reason: TerminationReason::DonePRGreen,
message: "done".to_string(),
},
);
assert_eq!(
breaker.consecutive_failures("x-donefail"),
1,
"a failed `backlog done` must count as a failure, not a false success"
);
}
#[test]
fn resolve_dispatch_done_exit5_is_awaiting_merge_success() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let bin = tmp.path().join("bin");
std::fs::create_dir_all(&bin).unwrap();
let fno = bin.join("fno");
std::fs::write(
&fno,
"#!/usr/bin/env bash\nif [[ \"$1\" == backlog && \"$2\" == done ]]; then echo 'awaiting merge: PR OPEN' >&2; exit 5; fi\nexit 0\n",
)
.unwrap();
std::fs::set_permissions(&fno, std::fs::Permissions::from_mode(0o755)).unwrap();
let cfg = test_cfg(tmp.path(), fno.display().to_string(), 3);
let (journal, project_journal) = test_journal(tmp.path());
let mut breaker = CircuitBreaker::new(3);
breaker.record_failure("x-awm5001");
resolve_dispatch(
&cfg,
&mut breaker,
&journal,
"x-awm5001",
Evidence {
reason: TerminationReason::DonePRGreen,
message: "done".to_string(),
},
);
assert_eq!(
breaker.consecutive_failures("x-awm5001"),
0,
"done exit 5 (awaiting merge) is a success, never a failure"
);
assert!(journal_lines(&project_journal)
.iter()
.any(|l| l.contains("active_backlog_dispatched") && l.contains("awaiting_merge")));
}
#[test]
fn resolve_crash_at_limit_defers_and_parks() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno(&tmp.path().join("bin"), &record);
let cfg = test_cfg(tmp.path(), fno, 2);
let (journal, project_journal) = test_journal(tmp.path());
let mut breaker = CircuitBreaker::new(2);
resolve_crash(&cfg, &mut breaker, &journal, "x-cra0001"); assert_eq!(breaker.consecutive_failures("x-cra0001"), 1);
resolve_crash(&cfg, &mut breaker, &journal, "x-cra0001");
assert_eq!(breaker.consecutive_failures("x-cra0001"), 0);
let calls = std::fs::read_to_string(&record).unwrap_or_default();
assert!(calls.contains("backlog defer x-cra0001"), "calls: {calls}");
let parked = journal_lines(&project_journal)
.into_iter()
.find(|l| l.contains("active_backlog_parked") && l.contains("x-cra0001"))
.expect("parked event");
assert!(parked.contains("\"deferred\":true"), "parked: {parked}");
}
#[test]
fn park_records_a_defer_that_did_not_land() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno_defer_fails(&tmp.path().join("bin"), &record);
let cfg = test_cfg(tmp.path(), fno, 2);
let (journal, project_journal) = test_journal(tmp.path());
let mut breaker = CircuitBreaker::new(2);
resolve_crash(&cfg, &mut breaker, &journal, "x-cra0002");
resolve_crash(&cfg, &mut breaker, &journal, "x-cra0002");
let calls = std::fs::read_to_string(&record).unwrap_or_default();
assert!(calls.contains("backlog defer x-cra0002"), "calls: {calls}");
let parked = journal_lines(&project_journal)
.into_iter()
.find(|l| l.contains("active_backlog_parked") && l.contains("x-cra0002"))
.expect("parked event still emitted on a failed defer");
assert!(
parked.contains("\"deferred\":false"),
"a defer that exited non-zero must be recorded as not landed: {parked}"
);
}
#[test]
fn reconcile_boot_grace_then_crash_floor() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno(&tmp.path().join("bin"), &record);
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, _pj) = test_journal(tmp.path());
let mut breaker = CircuitBreaker::new(3);
let mut pending = vec![PendingDispatch {
node_id: "x-bootgrace-never-real".to_string(),
session_id: None,
ticks: 0,
stamp_waits: 0,
}];
for _ in 1..BOOT_GRACE_TICKS {
reconcile_pending(&cfg, &mut breaker, &mut pending, &journal);
assert_eq!(
pending.len(),
1,
"must keep the dispatch during the boot window"
);
assert_eq!(breaker.consecutive_failures("x-bootgrace-never-real"), 0);
}
reconcile_pending(&cfg, &mut breaker, &mut pending, &journal);
assert!(
pending.is_empty(),
"the never-booted dispatch is retired as a crash"
);
assert_eq!(breaker.consecutive_failures("x-bootgrace-never-real"), 1);
}
#[test]
fn refless_done_pr_green_waits_for_the_stamp_before_parking() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let record = tmp.path().join("fno-calls.txt");
let fno = stub_fno_get(
&tmp.path().join("bin"),
&record,
r#"{"id":"x-grace-never-real"}"#,
);
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, project_journal) = test_journal(tmp.path());
std::fs::write(
&project_journal,
"{\"type\":\"termination\",\"data\":{\"session_id\":\"sid-grace\",\"reason\":\"DonePRGreen\"}}\n",
)
.unwrap();
let mut breaker = CircuitBreaker::new(3);
let mut pending = vec![PendingDispatch {
node_id: "x-grace-never-real".to_string(),
session_id: Some("sid-grace".to_string()),
ticks: 0,
stamp_waits: 0,
}];
for _ in 0..PR_STAMP_GRACE_TICKS {
reconcile_pending(&cfg, &mut breaker, &mut pending, &journal);
assert_eq!(pending.len(), 1, "held while the stamp may still land");
assert_eq!(breaker.consecutive_failures("x-grace-never-real"), 0);
}
reconcile_pending(&cfg, &mut breaker, &mut pending, &journal);
assert!(
pending.is_empty(),
"grace exhausted: the dispatch is retired"
);
assert_eq!(
breaker.consecutive_failures("x-grace-never-real"),
1,
"a still-ref-less DonePRGreen counts toward the streak"
);
}
#[test]
fn resolved_target_parses_mission_target() {
let t: ResolvedTarget = serde_json::from_str(
r#"{"project":"fno","cwd":"/x","interval_seconds":60,"failure_limit":3,"mission":"x-epic"}"#,
)
.unwrap();
assert_eq!(t.mission.as_deref(), Some("x-epic"));
let no_mission: ResolvedTarget = serde_json::from_str(
r#"{"project":"p","cwd":"/x","interval_seconds":60,"failure_limit":3}"#,
)
.unwrap();
assert_eq!(no_mission.mission, None);
}
#[test]
fn breaker_trips_at_limit() {
let mut b = CircuitBreaker::new(3);
assert!(!b.record_failure("n1"));
assert_eq!(b.consecutive_failures("n1"), 1);
assert!(!b.record_failure("n1"));
assert_eq!(b.consecutive_failures("n1"), 2);
assert!(b.record_failure("n1"));
assert_eq!(b.consecutive_failures("n1"), 3);
}
#[test]
fn breaker_success_resets_streak() {
let mut b = CircuitBreaker::new(2);
b.record_failure("n1");
assert_eq!(b.consecutive_failures("n1"), 1);
b.record_success("n1");
assert_eq!(b.consecutive_failures("n1"), 0);
assert!(!b.record_failure("n1"));
assert!(b.record_failure("n1"));
}
#[test]
fn breaker_reset_gives_fresh_attempts() {
let mut b = CircuitBreaker::new(2);
assert!(!b.record_failure("n1"));
assert!(b.record_failure("n1")); b.reset("n1"); assert_eq!(b.consecutive_failures("n1"), 0);
assert!(!b.record_failure("n1")); assert!(b.record_failure("n1")); }
#[test]
fn breaker_tracks_nodes_independently() {
let mut b = CircuitBreaker::new(2);
b.record_failure("a");
b.record_failure("b");
assert_eq!(b.consecutive_failures("a"), 1);
assert_eq!(b.consecutive_failures("b"), 1);
assert!(b.record_failure("a")); assert_eq!(b.consecutive_failures("b"), 1); }
#[test]
fn zero_limit_is_clamped_to_one() {
let mut b = CircuitBreaker::new(0);
assert!(b.record_failure("n1"));
}
fn stub_fno_advance(dir: &std::path::Path, receipt_json: &str) -> String {
std::fs::create_dir_all(dir).unwrap();
let p = dir.join("fno");
std::fs::write(
&p,
format!(
"#!/usr/bin/env bash\nif [[ \"$1\" == backlog && \"$2\" == advance ]]; then \
cat <<'JSON'\n{receipt_json}\nJSON\nfi\nexit 0\n"
),
)
.unwrap();
std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap();
p.display().to_string()
}
#[test]
fn dispatch_mission_records_dispatched_and_continues() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let fno = stub_fno_advance(
&tmp.path().join("bin"),
r#"{"epic_id":"x-epic","deactivated":false,"all_done":false,"dispatched":["x-a","x-b"]}"#,
);
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, project_journal) = test_journal(tmp.path());
let mut pending = Vec::new();
let outcome = dispatch_mission(&cfg, &mut pending, &journal);
assert_eq!(outcome, MissionDispatch::Continue);
assert_eq!(
pending
.iter()
.map(|p| p.node_id.clone())
.collect::<Vec<_>>(),
vec!["x-a", "x-b"]
);
assert!(journal_lines(&project_journal)
.iter()
.any(|l| l.contains("active_backlog_dispatched") && l.contains("x-a")));
}
#[test]
fn dispatch_mission_retires_on_deactivated() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let fno = stub_fno_advance(
&tmp.path().join("bin"),
r#"{"epic_id":"x-epic","deactivated":true,"all_done":false,"dispatched":[]}"#,
);
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, _pj) = test_journal(tmp.path());
let mut pending = Vec::new();
assert_eq!(
dispatch_mission(&cfg, &mut pending, &journal),
MissionDispatch::Retire
);
}
#[test]
fn dispatch_mission_retires_on_all_done() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let fno = stub_fno_advance(
&tmp.path().join("bin"),
r#"{"epic_id":"x-epic","deactivated":false,"all_done":true,"dispatched":[]}"#,
);
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, _pj) = test_journal(tmp.path());
let mut pending = Vec::new();
assert_eq!(
dispatch_mission(&cfg, &mut pending, &journal),
MissionDispatch::Retire
);
}
#[test]
fn dispatch_mission_dedups_already_pending() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let fno = stub_fno_advance(
&tmp.path().join("bin"),
r#"{"epic_id":"x-epic","dispatched":["x-a"]}"#,
);
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, _pj) = test_journal(tmp.path());
let mut pending = vec![PendingDispatch {
node_id: "x-a".to_string(),
session_id: None,
ticks: 2,
stamp_waits: 0,
}];
dispatch_mission(&cfg, &mut pending, &journal);
assert_eq!(pending.len(), 1, "x-a already pending must not be re-added");
}
#[test]
fn dispatch_mission_unparseable_receipt_continues() {
let _env = env_guard();
let tmp = tempfile::TempDir::new().unwrap();
let fno = stub_fno_advance(&tmp.path().join("bin"), "wedged python traceback");
let cfg = test_cfg(tmp.path(), fno, 3);
let (journal, project_journal) = test_journal(tmp.path());
let mut pending = Vec::new();
assert_eq!(
dispatch_mission(&cfg, &mut pending, &journal),
MissionDispatch::Continue
);
assert!(pending.is_empty());
assert!(journal_lines(&project_journal)
.iter()
.any(|l| l.contains("advance-epic-unparseable")));
}
}