use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Mutex, MutexGuard};
use std::time::Duration;
use anyhow::{Context, Result, bail};
use jiff::Timestamp;
use serde::{Deserialize, Serialize};
use tokio::sync::Notify;
use crate::ask::{self, Questions};
use crate::clean;
use crate::conduct::Conductor;
use crate::config::{Config, MergeMode};
use crate::graph::Runner;
use crate::land;
use crate::queue::{Queue, Task, TaskStatus};
use crate::run::{QuotaLoss, RunState, RunStatus};
pub const SCHEMA: u32 = 1;
pub const HEARTBEAT: Duration = Duration::from_secs(5);
pub const STALE_SECS: i64 = 30;
pub const POLL: Duration = Duration::from_secs(5);
pub const STALE_CLAIM: Duration = Duration::from_secs(6 * 60 * 60);
pub const STALLED_RUNNING: Duration = Duration::from_secs(30 * 60);
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct Current {
pub task: String,
pub run: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Status {
pub schema: u32,
pub pid: u32,
pub started_at: Timestamp,
pub updated_at: Timestamp,
pub idle: bool,
pub current: Vec<Current>,
pub completed: usize,
pub polls: u64,
}
impl Status {
#[must_use]
pub fn new() -> Self {
let now = Timestamp::now();
Self {
schema: SCHEMA,
pid: std::process::id(),
started_at: now,
updated_at: now,
idle: true,
current: Vec::new(),
completed: 0,
polls: 0,
}
}
}
impl Default for Status {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct Opts {
pub repo: PathBuf,
pub config: Option<PathBuf>,
pub poll: Duration,
pub max_attempts: usize,
pub once: bool,
pub merge: Option<String>,
pub worktrees_root: Option<PathBuf>,
}
impl Default for Opts {
fn default() -> Self {
Self {
repo: PathBuf::from("."),
config: None,
poll: POLL,
max_attempts: 2,
once: false,
merge: None,
worktrees_root: None,
}
}
}
fn max_concurrent(n: usize) -> usize {
n.max(1)
}
#[must_use]
pub fn status_path() -> PathBuf {
crate::run::home().join("daemon.json")
}
pub fn write_status(status: &Status) -> Result<()> {
write_status_to(&status_path(), status)
}
pub fn write_status_to(path: &Path, status: &Status) -> Result<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
}
let body = serde_json::to_string_pretty(status).context("serialize daemon status")?;
let tmp = path.with_extension("json.tmp");
std::fs::write(&tmp, &body).with_context(|| format!("write {}", tmp.display()))?;
std::fs::rename(&tmp, path).with_context(|| format!("replace {}", path.display()))?;
Ok(())
}
pub fn clear_status() {
clear_status_at(&status_path());
}
fn clear_status_at(path: &Path) {
let _ = std::fs::remove_file(path);
}
#[derive(Debug, Clone, Default)]
pub struct Stop {
stopped: Arc<AtomicBool>,
busy: Arc<std::sync::atomic::AtomicUsize>,
wake: Arc<Notify>,
pause: crate::graph::Pause,
}
impl Stop {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn stop(&self) {
self.stopped.store(true, Ordering::SeqCst);
self.wake.notify_one();
}
#[must_use]
pub fn stopped(&self) -> bool {
self.stopped.load(Ordering::SeqCst)
}
#[must_use]
pub fn finishing(&self) -> bool {
self.stopped() && self.busy_now()
}
pub fn park(&self) {
self.pause.park();
self.stop();
}
#[must_use]
pub fn parking(&self) -> bool {
self.pause.parked()
}
#[must_use]
pub fn pause(&self) -> crate::graph::Pause {
self.pause.clone()
}
#[must_use]
pub fn busy_now(&self) -> bool {
self.busy.load(Ordering::SeqCst) > 0
}
fn enter(&self) {
self.busy.fetch_add(1, Ordering::SeqCst);
}
fn exit(&self) {
self.busy.fetch_sub(1, Ordering::SeqCst);
}
async fn idle(&self, poll: Duration) {
tokio::select! {
() = tokio::time::sleep(poll) => {}
() = self.wake.notified() => {}
}
}
}
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(default)]
pub struct Reading {
pub schema: u32,
pub pid: Option<u32>,
pub started_at: Option<Timestamp>,
pub updated_at: Option<Timestamp>,
pub idle: bool,
#[serde(deserialize_with = "de_current")]
pub current: Vec<Current>,
pub completed: u64,
pub polls: u64,
}
fn de_current<'de, D>(deserializer: D) -> std::result::Result<Vec<Current>, D::Error>
where
D: serde::Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum Shape {
Many(Vec<Current>),
One(Current),
}
Ok(
Option::<Shape>::deserialize(deserializer)?.map_or_else(Vec::new, |shape| match shape {
Shape::Many(v) => v,
Shape::One(c) => vec![c],
}),
)
}
impl Reading {
#[must_use]
pub fn age_secs(&self, now: Timestamp) -> Option<i64> {
self.updated_at
.map(|at| (now.as_second() - at.as_second()).max(0))
}
#[must_use]
pub fn running(&self, now: Timestamp) -> bool {
self.age_secs(now).is_some_and(|secs| secs <= STALE_SECS)
}
}
#[must_use]
pub fn read_status(home: &Path) -> Option<Reading> {
let body = std::fs::read_to_string(home.join("daemon.json")).ok()?;
serde_json::from_str(&body).ok()
}
#[must_use]
pub fn current_work(home: &Path, now: Timestamp) -> Vec<Current> {
read_status(home)
.filter(|reading| reading.running(now))
.map(|reading| reading.current)
.unwrap_or_default()
}
#[must_use]
pub fn is_working_on(home: &Path, run: &str, now: Timestamp) -> bool {
current_work(home, now).iter().any(|c| c.run == run)
}
#[must_use]
pub fn is_working_on_short(home: &Path, short: &str, now: Timestamp) -> bool {
current_work(home, now)
.iter()
.any(|c| crate::run::short_of(&c.run) == short)
}
#[must_use]
pub fn is_working_on_task(home: &Path, task: &str, now: Timestamp) -> bool {
current_work(home, now).iter().any(|c| c.task == task)
}
pub fn sweep_stale_claims(queue: &Queue, older_than: Duration) -> Vec<String> {
sweep_stale_claims_with(queue, older_than, crate::proc::pid_alive)
}
fn sweep_stale_claims_with<F>(queue: &Queue, older_than: Duration, pid_alive: F) -> Vec<String>
where
F: Fn(u32) -> bool,
{
let this_process = std::process::id();
let mut swept: Vec<String> = std::fs::read_dir(queue.root())
.into_iter()
.flatten()
.flatten()
.map(|e| e.path())
.filter(|p| p.extension().is_some_and(|x| x == "lock"))
.filter(|p| {
match std::fs::read_to_string(p)
.ok()
.and_then(|body| body.trim().parse::<u32>().ok())
{
Some(pid) if pid == this_process => false,
Some(pid) => !pid_alive(pid),
None => p
.metadata()
.and_then(|m| m.modified())
.and_then(|t| t.elapsed().map_err(std::io::Error::other))
.is_ok_and(|age| age >= older_than),
}
})
.filter(|p| std::fs::remove_file(p).is_ok())
.filter_map(|p| {
p.file_stem()
.and_then(|s| s.to_str())
.map(std::borrow::ToOwned::to_owned)
})
.collect();
swept.sort_unstable();
swept
}
fn is_stalled(task: &Task, home: &Path, now: Timestamp) -> bool {
task.status == TaskStatus::Running
&& (now.as_second() - task.updated_at.as_second()) >= STALLED_RUNNING.as_secs() as i64
&& !is_working_on_task(home, &task.id, now)
}
fn stalled_tasks(queue: &Queue, home: &Path, now: Timestamp) -> Vec<Task> {
queue
.list()
.into_iter()
.filter(|t| is_stalled(t, home, now))
.collect()
}
fn queued_tasks(queue: &Queue) -> Vec<Task> {
queue
.list()
.into_iter()
.filter(|t| t.status == TaskStatus::Queued)
.collect()
}
fn finished_tasks(queue: &Queue) -> Vec<Task> {
queue
.list()
.into_iter()
.filter(|t| matches!(t.status, TaskStatus::Failed | TaskStatus::Held))
.collect()
}
fn resolve_blockers(queue: &Queue, questions: &Questions) {
for listed in queue.list() {
if listed.status != TaskStatus::Blocked || listed.blocked_by.is_empty() {
continue;
}
let Ok(_claim) = queue.claim(&listed.id) else {
continue;
};
let Ok(mut task) = queue.get(&listed.id) else {
continue;
};
if task.status != TaskStatus::Blocked {
continue;
}
let mut changed = false;
for id in task.blocked_by.clone() {
if let Ok(dep) = queue.get(&id) {
if dep.status == TaskStatus::Done {
task.unblock(&id);
changed = true;
}
continue;
}
if let Ok(q) = questions.get(&id)
&& q.status == ask::QuestionStatus::Answered
{
let answer = match &q.answer {
Some(ask::Answer::Choice(c) | ask::Answer::Text(c)) => c.clone(),
None => String::new(),
};
task.record_answer(q.summary.clone(), answer);
task.unblock(&id);
changed = true;
}
}
if changed {
record(queue, &mut task);
}
}
}
fn reconcile_task_questions(queue: &Queue, questions: &Questions) {
let tasks = queue.list();
let by_id: std::collections::BTreeMap<&str, &Task> =
tasks.iter().map(|t| (t.id.as_str(), t)).collect();
let referenced: std::collections::BTreeSet<&str> = tasks
.iter()
.flat_map(|task| task.blocked_by.iter().map(String::as_str))
.collect();
for mut question in questions.list() {
if !question.status.open() || question.node != crate::conduct::NODE {
continue;
}
if referenced.contains(question.id.as_str()) {
continue;
}
let Some(task) = by_id.get(question.run.as_str()) else {
continue;
};
question.abandon(format!(
"task {} no longer waits for this answer",
task.short()
));
if let Err(e) = questions.put(&mut question) {
tracing::warn!(
"could not retire question {} for task {}: {e:#}",
question.short(),
task.short()
);
}
}
}
#[derive(Debug, Clone, Copy)]
pub struct Verdict {
pub status: RunStatus,
pub left_pr: bool,
pub quota_hit: bool,
pub parked: bool,
pub no_viable_candidates: bool,
}
pub fn settle(task: &mut Task, verdict: Verdict, detail: &str, max_attempts: usize) {
if verdict.parked {
task.stall(detail);
return;
}
match verdict.status {
RunStatus::Merged | RunStatus::Ready => task.succeed(),
RunStatus::Stalled if verdict.quota_hit => task.stall(detail),
RunStatus::Failed if verdict.quota_hit && verdict.no_viable_candidates => {
task.stall(detail)
}
RunStatus::Stalled | RunStatus::Failed => task.fail(detail, max_attempts),
RunStatus::Blocked if verdict.left_pr => task.handed_off(detail),
RunStatus::Blocked => task.fail(detail, max_attempts),
other => task.fail(
format!(
"the graph stopped at `{}` without reaching a terminal status: {detail}",
label(other)
),
max_attempts,
),
}
}
fn settle_and_diagnose(
task: &mut Task,
verdict: Verdict,
detail: &str,
max_attempts: usize,
state: &RunState,
) {
settle(task, verdict, detail, max_attempts);
if task.status == TaskStatus::Held {
task.diagnostic = diagnostic(state);
}
}
fn reclaim(task: &mut Task, last_run: Option<RunState>, max_attempts: usize) {
match last_run {
Some(state) => {
let verdict = Verdict {
status: state.status,
left_pr: state.pr.is_some(),
quota_hit: !state.quota.is_empty(),
parked: state.parked,
no_viable_candidates: state.viable().is_empty(),
};
let detail = format!(
"recovered a `running` task whose daemon never recorded the outcome: {}",
describe(&state)
);
settle_and_diagnose(task, verdict, &detail, max_attempts, &state);
}
None => {
let why = "task was `running` with no live daemon and no readable \
run to recover; held for a human to check what happened";
task.last_error = Some(why.to_owned());
task.hold_machine(Some(why.to_owned()));
}
}
}
fn reclaim_orphaned_running(queue: &Queue, max_attempts: usize) -> Vec<String> {
let mut reclaimed = Vec::new();
for listed in queue.list() {
if listed.status != TaskStatus::Running {
continue;
}
let Ok(_claim) = queue.claim(&listed.id) else {
continue;
};
let Ok(mut task) = queue.get(&listed.id) else {
continue;
};
if task.status != TaskStatus::Running {
continue;
}
let last_run = task.runs.last().and_then(|id| RunState::load(id).ok());
if let Some(state) = &last_run
&& let Err(e) = ask::Questions::open().settle_run(&state.id, state.status)
{
tracing::warn!("abandon questions for {}: {e:#}", state.id);
}
reclaim(&mut task, last_run, max_attempts);
record(queue, &mut task);
reclaimed.push(task.id.clone());
}
reclaimed
}
fn reclaim_abandoned_runs(home: &Path, now: Timestamp) -> Vec<String> {
let mut abandoned = Vec::new();
for entry in std::fs::read_dir(home.join("runs"))
.into_iter()
.flatten()
.flatten()
{
let id = entry.file_name().to_string_lossy().into_owned();
if !crate::run::is_run_id(&id) {
continue;
}
let Ok(body) = std::fs::read_to_string(entry.path().join("run.json")) else {
continue;
};
let Ok(mut state) = serde_json::from_str::<RunState>(&body) else {
continue;
};
if state.status.done() || !state.active_all_overrun(now) || is_working_on(home, &id, now) {
continue;
}
state.abandon("daemon");
if let Err(e) = state.save_under(home) {
tracing::warn!("could not persist abandoned run {id}: {e:#}");
continue;
}
if let Err(e) = Questions::at(home.join("questions")).settle_run(&id, state.status) {
tracing::warn!("abandon questions for {id}: {e:#}");
}
abandoned.push(id);
}
abandoned
}
pub async fn serve(opts: Opts) -> Result<()> {
serve_until(opts, Stop::new()).await
}
pub async fn serve_until(opts: Opts, stop: Stop) -> Result<()> {
let signal = {
let stop = stop.clone();
tokio::spawn(async move {
if tokio::signal::ctrl_c().await.is_ok() {
stop.stop();
tracing::info!("shutdown requested; a run in flight will be finished first");
}
})
};
let worktrees_root = opts
.worktrees_root
.clone()
.unwrap_or_else(crate::run::default_worktree_root);
let outcome = drive(
&opts,
&Queue::open(),
&status_path(),
&crate::run::home(),
&worktrees_root,
&stop,
)
.await;
signal.abort();
outcome
}
async fn drive(
opts: &Opts,
queue: &Queue,
status_file: &Path,
home: &Path,
worktrees_root: &Path,
stop: &Stop,
) -> Result<()> {
let status = Arc::new(Mutex::new(Status::new()));
write_status_to(status_file, &lock(&status)).context("publish the daemon status file")?;
let beat = tokio::spawn(heartbeat(Arc::clone(&status), status_file.to_path_buf()));
let concurrency = max_concurrent(
prepare(&opts.repo, opts)
.map(|c| c.daemon.max_concurrent_runs)
.unwrap_or(1),
);
tracing::info!(
"magi serve: queue {} (poll {}s, {} attempts per task, {} run(s) at once)",
queue.root().display(),
opts.poll.as_secs(),
opts.max_attempts,
concurrency
);
janitor(&opts.repo, opts, home, worktrees_root).await;
let outcome = poll(
opts,
queue,
&status,
home,
worktrees_root,
stop,
concurrency,
)
.await;
beat.abort();
clear_status_at(status_file);
outcome
}
async fn heartbeat(status: Arc<Mutex<Status>>, path: PathBuf) {
loop {
tokio::time::sleep(HEARTBEAT).await;
let snapshot = {
let mut guard = lock(&status);
guard.updated_at = Timestamp::now();
guard.clone()
};
if let Err(e) = write_status_to(&path, &snapshot) {
tracing::warn!("could not refresh the daemon status file: {e:#}");
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum LandResume {
NotLanding,
StillWaiting,
Ready,
}
fn land_resume_state(task: &Task) -> LandResume {
let Some(run_id) = task.runs.last() else {
return LandResume::NotLanding;
};
let Ok(state) = RunState::load(run_id) else {
return LandResume::NotLanding;
};
if state.status != RunStatus::Landing || !state.parked {
return LandResume::NotLanding;
}
let store = ask::Questions::open();
let waiting = store
.list()
.into_iter()
.filter(|q| &q.run == run_id && q.node == land::APPROVAL_NODE)
.max_by(|a, b| a.id.cmp(&b.id));
let Some(mut q) = waiting else {
return LandResume::Ready;
};
if !q.status.open() {
return LandResume::Ready;
}
let timeout = Duration::from_secs(state.config.graph.answer_timeout);
let elapsed = Timestamp::now().as_second() - q.asked_at.as_second();
if elapsed >= 0 && elapsed as u64 >= timeout.as_secs() {
q.abandon(format!(
"no answer within {}s of asking",
timeout.as_secs().max(1)
));
if store.put(&mut q).is_ok() {
return LandResume::Ready;
}
}
LandResume::StillWaiting
}
const RECHECK_WHILE_BUSY: Duration = Duration::from_millis(200);
struct InFlightGuard<'a> {
status: &'a Arc<Mutex<Status>>,
stop: &'a Stop,
task_id: &'a str,
}
impl Drop for InFlightGuard<'_> {
fn drop(&mut self) {
lock(self.status).current.retain(|c| c.task != self.task_id);
self.stop.exit();
}
}
async fn poll(
opts: &Opts,
queue: &Queue,
status: &Arc<Mutex<Status>>,
home: &Path,
worktrees_root: &Path,
stop: &Stop,
max_concurrent: usize,
) -> Result<()> {
let mut attempted: Vec<String> = Vec::new();
let sem = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
let quota_cooldown_until: Arc<Mutex<Option<Timestamp>>> = Arc::new(Mutex::new(None));
let mut inflight: tokio::task::JoinSet<()> = tokio::task::JoinSet::new();
let mut conductor = Conductor::new();
while !stop.stopped() {
lock(status).polls += 1;
while let Some(result) = inflight.try_join_next() {
if let Err(e) = result {
tracing::error!("a spawned attempt did not finish cleanly: {e}");
}
}
let swept = sweep_stale_claims(queue, STALE_CLAIM);
if !swept.is_empty() {
tracing::warn!(
"swept {} stale claim(s) left behind by an earlier daemon: {}",
swept.len(),
swept.join(", ")
);
}
let now = Timestamp::now();
let stalled = stalled_tasks(queue, home, now);
let stalled_ids: std::collections::BTreeSet<_> =
stalled.iter().map(|task| task.id.clone()).collect();
let reclaimed = reclaim_orphaned_running(queue, opts.max_attempts);
if !reclaimed.is_empty() {
tracing::warn!(
"reclaimed {} task(s) left `running` by a daemon that never \
recorded the outcome: {}",
reclaimed.len(),
reclaimed.join(", ")
);
}
let abandoned_runs = reclaim_abandoned_runs(home, now);
if !abandoned_runs.is_empty() {
tracing::warn!(
"failed {} run(s) left behind by a killed process, past every \
active seat's own timeout: {}",
abandoned_runs.len(),
abandoned_runs.join(", ")
);
}
let questions = Questions::at(home.join("questions"));
resolve_blockers(queue, &questions);
reconcile_task_questions(queue, &questions);
let finished: Vec<Task> = finished_tasks(queue)
.into_iter()
.filter(|task| !stalled_ids.contains(&task.id))
.collect();
let queued = queued_tasks(queue);
if !(queued.is_empty() && stalled.is_empty() && finished.is_empty())
&& conductor.worth_a_look(queue, &stalled, &finished)
{
match prepare(&opts.repo, opts) {
Ok(cfg) => {
conductor
.maybe_run(
&cfg,
&opts.repo,
queue,
&questions,
home,
&queued,
&stalled,
&finished,
opts.max_attempts,
)
.await;
}
Err(e) => tracing::warn!("conductor: no config: {e:#}"),
}
}
let candidates: Vec<Task> = runnable(queue)
.into_iter()
.filter(|t| !opts.once || !attempted.contains(&t.id))
.collect();
let cooling_down =
lock("a_cooldown_until).is_some_and(|until| Timestamp::now() < until);
let mut started_any = false;
for candidate in candidates {
if stop.stopped() {
break;
}
let resume = land_resume_state(&candidate);
if resume == LandResume::StillWaiting {
continue;
}
let priority = resume == LandResume::Ready;
if !priority && cooling_down {
continue;
}
let permit = if priority {
None
} else {
match Arc::clone(&sem).try_acquire_owned() {
Ok(p) => Some(p),
Err(_) => continue,
}
};
let Ok(claim) = queue.claim(&candidate.id) else {
tracing::info!("task {} is claimed elsewhere; skipping", candidate.short());
continue;
};
let mut task = match queue.get(&candidate.id) {
Ok(t) if t.status.runnable() => t,
Ok(_) => continue,
Err(e) => {
tracing::warn!("could not re-read task {}: {e:#}", candidate.short());
continue;
}
};
let task_id = task.id.clone();
attempted.push(task_id.clone());
lock(status).idle = false;
stop.enter();
started_any = true;
let opts = opts.clone();
let queue = queue.clone();
let status = Arc::clone(status);
let stop = stop.clone();
let quota_cooldown_until = Arc::clone("a_cooldown_until);
inflight.spawn(async move {
let _claim = claim;
let _permit = permit;
let _inflight = InFlightGuard {
status: &status,
stop: &stop,
task_id: &task_id,
};
let quota = attempt(&opts, &queue, &status, &stop, &mut task).await;
lock(&status).completed += 1;
if !quota.is_empty() {
let hint = quota.iter().find_map(|q| q.reset.as_deref());
let reset_at = hint.and_then(|h| parse_reset_hint(h, Timestamp::now()));
let wait = quota_wait(
reset_at,
Timestamp::now(),
QUOTA_WAIT_FALLBACK,
QUOTA_WAIT_CAP,
);
let secs = i64::try_from(wait.as_secs()).unwrap_or(i64::MAX);
let until = Timestamp::now()
.checked_add(jiff::SignedDuration::from_secs(secs))
.unwrap_or(Timestamp::MAX);
*lock("a_cooldown_until) = Some(until);
match hint {
Some(h) => tracing::warn!(
"quota hit; waiting {}s before taking another ordinary task \
(CLI reported reset: {h})",
wait.as_secs()
),
None => tracing::warn!(
"quota hit; waiting {}s before taking another ordinary task \
(no reset hint reported)",
wait.as_secs()
),
}
}
});
}
if started_any {
continue;
}
if stop.busy_now() {
stop.idle(RECHECK_WHILE_BUSY.min(opts.poll)).await;
continue;
}
lock(status).idle = true;
if opts.once {
janitor(&opts.repo, opts, home, worktrees_root).await;
break;
}
stop.idle(opts.poll).await;
if stop.stopped() {
continue;
}
janitor(&opts.repo, opts, home, worktrees_root).await;
}
while let Some(result) = inflight.join_next().await {
if let Err(e) = result {
tracing::error!("a spawned attempt did not finish cleanly: {e}");
}
}
Ok(())
}
async fn attempt(
opts: &Opts,
queue: &Queue,
status: &Arc<Mutex<Status>>,
stop: &Stop,
task: &mut Task,
) -> Vec<QuotaLoss> {
let repo = repo_for(task, &opts.repo);
tracing::info!(
"task {} — {} (repo {})",
task.short(),
task.title,
repo.display()
);
let mut config = match prepare(&repo, opts) {
Ok(c) => c,
Err(e) => {
task.attempts += 1;
task.fail(format!("config: {e:#}"), opts.max_attempts);
record(queue, task);
return Vec::new();
}
};
apply_solo(&mut config, task);
if let Some(reason) = disk_gate(&repo, &config) {
task.last_error = Some(reason.clone());
task.hold_machine(Some(reason.clone()));
record(queue, task);
tracing::warn!("holding {} for want of disk space: {reason}", task.short());
return Vec::new();
}
let unfinished = (!task.fresh_start)
.then(|| unfinished_run(&task.runs, task.short()))
.flatten();
let review_branch = task.review_branch.take();
let branch_exists = match &review_branch {
Some(branch) => crate::git::branch_exists(&repo, branch)
.await
.unwrap_or(false),
None => false,
};
let starter = choose_starter(
review_branch.as_deref(),
branch_exists,
unfinished.as_deref(),
);
let started = match &starter {
Starter::Review(branch) => {
tracing::info!(
"task {} reopens `{branch}` as a review-only pass",
task.short()
);
Runner::review(&repo, branch, config).await
}
Starter::Resume(id) => {
tracing::info!("resuming run {id} rather than competing again");
Runner::resume(id).map(|mut r| {
if let Some(instruction) =
prepare_instruction(&starter, Some(&r.state.instruction), task)
{
r.state.instruction = instruction;
}
r
})
}
Starter::Start => {
if let Some(branch) = &review_branch {
tracing::warn!(
"conductor chose review for task {} but branch `{branch}` no longer \
exists; requeuing as a fresh competition instead",
task.short()
);
}
let instruction = prepare_instruction(&starter, None, task)
.unwrap_or_else(|| task.instruction.clone());
Runner::start(&repo, instruction, config).await
}
};
let mut runner = match started {
Ok(r) => r,
Err(e) => {
task.attempts += 1;
task.fail(format!("could not start the run: {e:#}"), opts.max_attempts);
record(queue, task);
return Vec::new();
}
};
runner.on_pause(stop.pause());
let run = runner.state.id.clone();
task.start(run.clone());
record(queue, task);
lock(status).current.push(Current {
task: task.id.clone(),
run,
});
let detail = match runner.execute().await {
Ok(()) => describe(&runner.state),
Err(e) => format!("{e:#}"),
};
let verdict = Verdict {
status: runner.state.status,
left_pr: runner.state.pr.is_some(),
quota_hit: !runner.state.quota.is_empty(),
parked: runner.state.parked,
no_viable_candidates: runner.state.viable().is_empty(),
};
settle_and_diagnose(task, verdict, &detail, opts.max_attempts, &runner.state);
record(queue, task);
tracing::info!(
"task {} is {} after run {} ({})",
task.short(),
task.status.as_str(),
runner.state.short(),
label(runner.state.status)
);
runner.state.quota
}
fn apply_solo(config: &mut Config, task: &Task) {
if task.solo {
config.graph.candidates = 1;
}
}
fn prepare(repo: &Path, opts: &Opts) -> Result<Config> {
let (mut config, _layers) = Config::discover(repo, opts.config.as_deref())?;
if let Some(mode) = &opts.merge {
config.merge.mode = merge_mode(mode)?;
}
Ok(config)
}
async fn janitor(repo: &Path, opts: &Opts, home: &Path, worktrees_root: &Path) {
let cfg = match prepare(repo, opts) {
Ok(cfg) => cfg,
Err(e) => {
tracing::warn!("housekeep: no config: {e:#}");
return;
}
};
let worktrees_root = cfg.graph.worktree_root.as_deref().unwrap_or(worktrees_root);
let out = clean::housekeep(&cfg, home, worktrees_root, repo, Timestamp::now()).await;
if out.folded > 0 || out.unreadable > 0 || out.orphaned_worktrees > 0 {
let mut extra = Vec::new();
if out.unreadable > 0 {
extra.push(format!("{} unreadable", out.unreadable));
}
if out.orphaned_worktrees > 0 {
extra.push(format!("{} orphaned worktree(s)", out.orphaned_worktrees));
}
let detail = if extra.is_empty() {
String::new()
} else {
format!(" ({})", extra.join(", "))
};
tracing::info!("housekeep: folded {} run(s){detail}", out.folded);
}
if out.cache_files > 0 {
tracing::info!(
"housekeep: pruned {} file(s) ({} bytes) from the shared cache",
out.cache_files,
out.cache_freed
);
}
if out.questions_abandoned > 0 {
tracing::info!(
"housekeep: abandoned {} question(s) left open by a finished run",
out.questions_abandoned
);
}
}
fn disk_gate(repo: &Path, config: &Config) -> Option<String> {
let min = config.disk.min_free_bytes;
if min == 0 {
return None;
}
match crate::disk::free_bytes(repo) {
Ok(free) => crate::disk::gate(free, min),
Err(e) => Some(format!(
"could not measure free space on {} ({e}); the disk gate refuses \
to let a run start blind",
repo.display()
)),
}
}
const QUOTA_WAIT_FALLBACK: Duration = Duration::from_secs(5 * 60);
const QUOTA_WAIT_CAP: Duration = Duration::from_secs(30 * 60);
fn quota_wait(
reset_at: Option<Timestamp>,
now: Timestamp,
fallback: Duration,
cap: Duration,
) -> Duration {
match reset_at {
Some(at) if at > now => {
let secs = u64::try_from(at.as_second() - now.as_second()).unwrap_or(0);
Duration::from_secs(secs).min(cap)
}
_ => fallback,
}
}
fn parse_reset_hint(text: &str, now: Timestamp) -> Option<Timestamp> {
let open = text.find('(')?;
let close = text.rfind(')')?;
if close <= open {
return None;
}
let zone = text[open + 1..close].trim();
let clock = text[..open].trim().to_lowercase();
let (digits, pm) = clock
.strip_suffix("am")
.map(|d| (d, false))
.or_else(|| clock.strip_suffix("pm").map(|d| (d, true)))?;
let (h, m) = digits.trim().split_once(':')?;
let mut hour: i8 = h.trim().parse().ok()?;
let minute: i8 = m.trim().parse().ok()?;
if !(1..=12).contains(&hour) || !(0..=59).contains(&minute) {
return None;
}
if pm && hour != 12 {
hour += 12;
} else if !pm && hour == 12 {
hour = 0;
}
let tz = jiff::tz::TimeZone::get(zone).ok()?;
let candidate = now
.to_zoned(tz)
.with()
.hour(hour)
.minute(minute)
.second(0)
.millisecond(0)
.microsecond(0)
.nanosecond(0)
.build()
.ok()?;
let mut at = candidate.timestamp();
if at <= now {
at += jiff::SignedDuration::from_hours(24);
}
Some(at)
}
fn exhausted_review_budget(state: &RunState) -> bool {
state.status == RunStatus::Blocked && state.reviews.len() >= state.config.graph.review_rounds
}
fn unfinished_run(runs: &[String], short: &str) -> Option<String> {
unfinished_run_with(runs, short, RunState::load)
}
fn unfinished_run_with<F>(runs: &[String], short: &str, load: F) -> Option<String>
where
F: FnOnce(&str) -> Result<RunState>,
{
let id = runs.last()?;
match load(id) {
Ok(s) if s.status.resumable() && !exhausted_review_budget(&s) => Some(id.clone()),
Ok(_) => None,
Err(e) => {
tracing::warn!("could not read run {id} for task {short}: {e:#}");
None
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum Starter {
Review(String),
Resume(String),
Start,
}
fn choose_starter(
review_branch: Option<&str>,
branch_exists: bool,
unfinished: Option<&str>,
) -> Starter {
match review_branch {
Some(branch) if branch_exists => Starter::Review(branch.to_owned()),
Some(_) => Starter::Start,
None => match unfinished {
Some(id) => Starter::Resume(id.to_owned()),
None => Starter::Start,
},
}
}
fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
return fallback.to_path_buf();
}
task.repo.clone()
}
const ANSWERS_HEADER: &str = "\n\n# Operator answers\n\n";
fn answers_block(task: &Task, count: usize) -> String {
let mut s = ANSWERS_HEADER.to_owned();
for a in &task.answers[..count] {
s.push_str(&format!("- {}: {}\n", a.question, a.answer));
}
s
}
fn append_answers(base: &str, task: &Task) -> String {
if task.answers.is_empty() {
return base.to_owned();
}
let mut s = base.to_owned();
s.push_str(&answers_block(task, task.answers.len()));
s
}
fn strip_answers_block<'a>(instruction: &'a str, task: &Task) -> &'a str {
for count in (1..=task.answers.len()).rev() {
let block = answers_block(task, count);
if let Some(base) = instruction.strip_suffix(&block) {
return base;
}
}
instruction
}
fn instruction_for(task: &Task) -> String {
append_answers(&task.instruction, task)
}
fn resumed_instruction(old_instruction: &str, task: &Task) -> String {
append_answers(strip_answers_block(old_instruction, task), task)
}
fn prepare_instruction(
starter: &Starter,
old_instruction: Option<&str>,
task: &Task,
) -> Option<String> {
match starter {
Starter::Start => Some(instruction_for(task)),
Starter::Resume(_) => Some(resumed_instruction(
old_instruction.expect("a resumed run always has a prior instruction"),
task,
)),
Starter::Review(_) => None,
}
}
fn record(queue: &Queue, task: &mut Task) {
if let Err(e) = queue.put(task) {
tracing::error!("could not record task {}: {e:#}", task.short());
}
}
fn runnable(queue: &Queue) -> Vec<Task> {
let mut tasks: Vec<Task> = queue
.list()
.into_iter()
.filter(|t| t.status.runnable())
.collect();
tasks.sort_unstable_by(|a, b| b.priority.cmp(&a.priority).then(a.id.cmp(&b.id)));
tasks
}
fn describe(state: &RunState) -> String {
let mut detail = if state.status == RunStatus::Stalled {
let mut seats: Vec<&str> = state.quota.iter().map(|q| q.seat.as_str()).collect();
seats.sort_unstable();
seats.dedup();
if seats.is_empty() {
"the judging panel lost its quorum".to_owned()
} else {
format!(
"the judging panel lost its quorum; quota took out {}",
seats.join(", ")
)
}
} else {
format!("run ended {}", label(state.status))
};
if let Some(last) = state.events.last() {
detail.push_str(&format!(" ({}: {})", last.node, last.message));
}
detail.push_str(&format!(" [run {}]", state.id));
detail
}
const DIAGNOSTIC_MAX: usize = 4_000;
const DIAGNOSTIC_OUTPUT_TAIL: usize = 800;
fn diagnostic(state: &RunState) -> Option<String> {
let mut parts: Vec<String> = Vec::new();
for o in state.gate.iter().filter(|o| !o.ok()) {
parts.push(format!(
"gate `{}` failed ({:?}):\n{}",
o.command,
o.code,
crate::run::tail(&o.output_tail, DIAGNOSTIC_OUTPUT_TAIL)
));
}
if let Some(last) = state
.events
.iter()
.rev()
.find(|e| e.node == "land" && e.message.contains("fixer produced no commit"))
{
parts.push(last.message.clone());
}
if state.viable().is_empty() {
for c in &state.candidates {
if !c.summary.trim().is_empty() {
parts.push(format!("candidate {}: {}", c.label, c.summary.trim()));
} else if let Some(why) = &c.failed {
parts.push(format!("candidate {}: {why}", c.label));
}
}
}
if parts.is_empty() {
return None;
}
Some(crate::run::tail(
&parts.join("\n\n"),
DIAGNOSTIC_MAX.saturating_sub(100),
))
}
fn label(status: RunStatus) -> &'static str {
status.as_str()
}
fn merge_mode(mode: &str) -> Result<MergeMode> {
match mode {
"none" => Ok(MergeMode::None),
"local" => Ok(MergeMode::Local),
"pr" => Ok(MergeMode::Pr),
other => bail!("unknown merge mode `{other}`; expected none, local or pr"),
}
}
fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::queue::{Source, TaskStatus};
use crate::run::{Candidate, CommandOutcome};
use pretty_assertions::assert_eq;
fn task() -> Task {
Task::new(
"add retries".to_owned(),
"add retries".to_owned(),
PathBuf::from("/repo"),
Source::Human,
)
}
#[test]
fn every_run_status_settles_the_task_it_came_from() {
let table = [
(RunStatus::Merged, TaskStatus::Done, 1),
(RunStatus::Ready, TaskStatus::Done, 1),
(RunStatus::Stalled, TaskStatus::Failed, 0),
(RunStatus::Blocked, TaskStatus::Failed, 1),
(RunStatus::Failed, TaskStatus::Failed, 1),
(RunStatus::Prep, TaskStatus::Failed, 1),
(RunStatus::Implementing, TaskStatus::Failed, 1),
(RunStatus::Judging, TaskStatus::Failed, 1),
(RunStatus::Deliberating, TaskStatus::Failed, 1),
(RunStatus::Voting, TaskStatus::Failed, 1),
(RunStatus::Reviewing, TaskStatus::Failed, 1),
(RunStatus::Gating, TaskStatus::Failed, 1),
];
for (run, want, attempts) in table {
let mut t = task();
t.start("20260902-000000-aaaa".to_owned());
settle(
&mut t,
Verdict {
status: run,
left_pr: false,
parked: false,
quota_hit: matches!(run, RunStatus::Stalled),
no_viable_candidates: false,
},
"why",
2,
);
assert_eq!(t.status, want, "task status after {}", label(run));
assert_eq!(t.attempts, attempts, "attempts after {}", label(run));
}
}
#[test]
fn a_quota_stall_costs_the_task_no_attempt_but_a_block_does() {
let mut stalled = task();
stalled.start("20260902-000000-aaaa".to_owned());
settle(
&mut stalled,
Verdict {
status: RunStatus::Stalled,
left_pr: false,
parked: false,
quota_hit: true,
no_viable_candidates: false,
},
"quota",
1,
);
assert_eq!(stalled.attempts, 0);
assert!(
stalled.status.runnable(),
"a machine problem must leave the task in line"
);
let mut blocked = task();
blocked.start("20260902-000000-aaaa".to_owned());
settle(
&mut blocked,
Verdict {
status: RunStatus::Blocked,
left_pr: false,
parked: false,
quota_hit: false,
no_viable_candidates: false,
},
"findings open",
1,
);
assert_eq!(blocked.attempts, 1);
assert_eq!(
blocked.status,
TaskStatus::Held,
"the last attempt hands the task to a human"
);
}
#[test]
fn a_run_that_opened_a_pull_request_is_never_re_competed() {
let mut delivered = task();
delivered.start("20260903-080619-01c2".to_owned());
settle(
&mut delivered,
Verdict {
status: RunStatus::Blocked,
left_pr: true,
parked: false,
quota_hit: false,
no_viable_candidates: false,
},
"no check status",
4,
);
assert_eq!(
delivered.status,
TaskStatus::Held,
"a pull request waiting on CI or a person is not a retryable failure"
);
assert!(
!delivered.status.runnable(),
"the loop must not pick this task up again"
);
assert_eq!(
delivered.last_error.as_deref(),
Some("no check status"),
"the operator needs to be told what the gate was waiting for"
);
let mut empty_handed = task();
empty_handed.start("20260903-080619-01c2".to_owned());
settle(
&mut empty_handed,
Verdict {
status: RunStatus::Blocked,
left_pr: false,
parked: false,
quota_hit: false,
no_viable_candidates: false,
},
"findings open",
4,
);
assert_eq!(empty_handed.status, TaskStatus::Failed);
assert!(empty_handed.status.runnable());
}
#[test]
fn parking_costs_the_task_no_attempt_and_leaves_it_in_line() {
let mut parked = task();
parked.start("20260903-183634-2d98".to_owned());
settle(
&mut parked,
Verdict {
status: RunStatus::Implementing,
left_pr: false,
quota_hit: false,
parked: true,
no_viable_candidates: false,
},
"parked after `implementing`",
2,
);
assert_eq!(parked.attempts, 0, "a park is refunded");
assert!(
parked.status.runnable(),
"and the task stays in line so the next loop resumes its run"
);
assert_eq!(
parked.last_error.as_deref(),
Some("parked after `implementing`"),
"the card says where it stopped"
);
let mut broken = task();
broken.start("20260903-183634-2d98".to_owned());
settle(
&mut broken,
Verdict {
status: RunStatus::Implementing,
left_pr: false,
quota_hit: false,
parked: false,
no_viable_candidates: false,
},
"returned mid-flight",
2,
);
assert_eq!(broken.attempts, 1);
}
#[test]
fn only_a_rate_limit_buys_the_task_its_attempt_back() {
let mut flaky = task();
flaky.start("20260903-123023-e633".to_owned());
settle(
&mut flaky,
Verdict {
status: RunStatus::Stalled,
left_pr: false,
parked: false,
quota_hit: false,
no_viable_candidates: false,
},
"verdict rests on 1 of 3 judges",
2,
);
assert_eq!(
flaky.attempts, 1,
"flakiness spends an attempt, so `max_attempts` still bounds it"
);
assert!(flaky.status.runnable(), "and it is still worth retrying");
let mut limited = task();
limited.start("20260903-123023-e633".to_owned());
settle(
&mut limited,
Verdict {
status: RunStatus::Stalled,
left_pr: false,
parked: false,
quota_hit: true,
no_viable_candidates: false,
},
"judge-2, judge-3 out of quota",
2,
);
assert_eq!(limited.attempts, 0, "a quota window is refunded");
assert!(limited.status.runnable());
let mut worn = task();
for _ in 0..2 {
worn.release();
}
worn.start("20260903-123023-e633".to_owned());
worn.attempts = 2;
settle(
&mut worn,
Verdict {
status: RunStatus::Stalled,
left_pr: false,
parked: false,
quota_hit: false,
no_viable_candidates: false,
},
"no quorum again",
2,
);
assert_eq!(worn.status, TaskStatus::Held);
assert!(!worn.status.runnable());
}
#[test]
fn a_quota_wipeout_that_leaves_nothing_to_judge_also_costs_no_attempt() {
let mut wiped_out = task();
wiped_out.start("20260907-025000-a1b2".to_owned());
settle(
&mut wiped_out,
Verdict {
status: RunStatus::Failed,
left_pr: false,
parked: false,
quota_hit: true,
no_viable_candidates: true,
},
"no candidate produced a change; nothing to judge",
2,
);
assert_eq!(wiped_out.attempts, 0, "a total quota wipeout is refunded");
assert!(
wiped_out.status.runnable(),
"a machine problem must leave the task in line"
);
let mut partial_progress = task();
partial_progress.start("20260907-025500-c3d4".to_owned());
settle(
&mut partial_progress,
Verdict {
status: RunStatus::Failed,
left_pr: false,
parked: false,
quota_hit: true,
no_viable_candidates: false,
},
"gate failed on the winning candidate",
2,
);
assert_eq!(
partial_progress.attempts, 1,
"a candidate that actually produced a change spends the attempt \
even though some other seat hit its quota"
);
assert!(partial_progress.status.runnable());
}
#[test]
fn reclaim_refunds_a_recovered_quota_wipeout_the_same_way_a_live_settle_does() {
let mut t = task();
t.start("20260907-025000-a1b2".to_owned());
let mut state = run_state(RunStatus::Failed);
state.quota.push(QuotaLoss {
seat: "cand-a".to_owned(),
node: "implement".to_owned(),
at: Timestamp::now(),
reset: None,
});
assert!(
state.viable().is_empty(),
"no candidate was added, so nothing is viable"
);
reclaim(&mut t, Some(state), 2);
assert_eq!(t.attempts, 0, "a recovered quota wipeout is refunded");
assert!(t.status.runnable());
}
#[test]
fn a_held_task_is_never_offered_to_the_loop() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().to_path_buf());
for (n, priority) in [(1, 0), (2, 5), (3, 5)] {
let mut t = task();
t.id = format!("2026090{n}-000000-000{n}");
t.priority = priority;
queue.put(&mut t).unwrap();
}
let mut held = task();
held.id = "20260909-000000-9999".to_owned();
held.priority = 99;
held.hold_machine(None);
queue.put(&mut held).unwrap();
let order: Vec<String> = runnable(&queue).into_iter().map(|t| t.id).collect();
assert_eq!(order.len(), 3);
assert!(!order.contains(&held.id));
assert_eq!(
order.first().cloned(),
queue.next_runnable().map(|t| t.id),
"the loop's first candidate is exactly what the queue offers"
);
assert_eq!(
order,
vec![
"20260902-000000-0002".to_owned(),
"20260903-000000-0003".to_owned(),
"20260901-000000-0001".to_owned(),
],
"priority first, then oldest, so nothing starves"
);
}
#[test]
fn sweep_removes_an_old_unparseable_lock_and_keeps_a_live_one() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().to_path_buf());
let mut old = task();
old.id = "20260101-000000-old0".to_owned();
queue.put(&mut old).unwrap();
let mut fresh = task();
fresh.id = "20260101-000000-new0".to_owned();
queue.put(&mut fresh).unwrap();
std::fs::write(dir.path().join(format!("{}.lock", old.id)), "not a pid").unwrap();
std::thread::sleep(Duration::from_millis(60));
let live = queue.claim(&fresh.id).unwrap();
let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
assert_eq!(swept, vec![old.id.clone()]);
assert!(
queue.claim(&old.id).is_ok(),
"an unparseable lock older than the threshold is swept"
);
assert!(
queue.claim(&fresh.id).is_err(),
"a live pid protects its lock regardless of age"
);
drop(live);
}
#[test]
fn an_old_lock_whose_pid_is_still_alive_is_never_swept_by_age_alone() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().to_path_buf());
let mut t = task();
t.id = "20260101-000000-live".to_owned();
queue.put(&mut t).unwrap();
let claim = queue.claim(&t.id).unwrap();
std::thread::sleep(Duration::from_millis(60));
let swept = sweep_stale_claims(&queue, Duration::from_millis(50));
assert!(
swept.is_empty(),
"a lock naming a live pid must never be swept by age, no matter how old: {swept:?}"
);
assert!(
queue.claim(&t.id).is_err(),
"the lock still protects its task"
);
drop(claim);
}
fn injected_dead_pid() -> u32 {
std::process::id().checked_add(1).unwrap_or(1)
}
#[test]
fn a_lock_naming_a_dead_pid_is_swept_at_once_regardless_of_age() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().to_path_buf());
let mut t = task();
t.id = "20260101-000000-dead".to_owned();
queue.put(&mut t).unwrap();
let dead_pid = injected_dead_pid();
std::fs::write(
dir.path().join(format!("{}.lock", t.id)),
dead_pid.to_string(),
)
.unwrap();
let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
pid != dead_pid
});
assert_eq!(
swept,
vec![t.id.clone()],
"a dead owner is reclaimed immediately, not after STALE_CLAIM"
);
assert!(queue.claim(&t.id).is_ok(), "the task is claimable again");
}
#[test]
fn sweeping_on_every_poll_catches_a_lock_that_appears_after_the_first_sweep() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().to_path_buf());
let mut t = task();
t.id = "20260101-000000-late".to_owned();
queue.put(&mut t).unwrap();
let dead_pid = injected_dead_pid();
assert!(
sweep_stale_claims(&queue, Duration::from_secs(6 * 60 * 60)).is_empty(),
"nothing has claimed the task yet"
);
std::fs::write(
dir.path().join(format!("{}.lock", t.id)),
dead_pid.to_string(),
)
.unwrap();
let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
pid != dead_pid
});
assert_eq!(swept, vec![t.id.clone()]);
}
#[test]
fn a_running_task_behind_a_dead_daemons_lock_recovers_once_swept_and_keeps_its_history() {
crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().to_path_buf());
let mut t = task();
t.id = "20260101-000000-crsh".to_owned();
t.status = TaskStatus::Running;
t.attempts = 1;
t.runs.push("20260904-000000-4043".to_owned());
queue.put(&mut t).unwrap();
let dead_pid = injected_dead_pid();
std::fs::write(
dir.path().join(format!("{}.lock", t.id)),
dead_pid.to_string(),
)
.unwrap();
assert!(reclaim_orphaned_running(&queue, 2).is_empty());
assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
let swept = sweep_stale_claims_with(&queue, Duration::from_secs(6 * 60 * 60), |pid| {
pid != dead_pid
});
assert_eq!(swept, vec![t.id.clone()]);
let reclaimed = reclaim_orphaned_running(&queue, 2);
assert_eq!(reclaimed, vec![t.id.clone()]);
let after = queue.get(&t.id).unwrap();
assert_eq!(
after.status,
TaskStatus::Held,
"no run.json to recover from, so a human is asked"
);
assert_eq!(
after.runs,
vec!["20260904-000000-4043".to_owned()],
"the crashed run's id is kept as evidence, not discarded"
);
}
#[test]
fn a_lock_is_kept_when_the_process_query_is_unavailable() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().to_path_buf());
let mut t = task();
t.id = "20260101-000000-unknown".to_owned();
queue.put(&mut t).unwrap();
let dead_pid = injected_dead_pid();
std::fs::write(
dir.path().join(format!("{}.lock", t.id)),
dead_pid.to_string(),
)
.unwrap();
let swept = sweep_stale_claims_with(&queue, Duration::ZERO, |_| true);
assert!(swept.is_empty(), "an unknown pid must keep its lock");
assert!(queue.claim(&t.id).is_err(), "the lock remains protective");
}
fn run_state(status: RunStatus) -> RunState {
let mut state = RunState::new(
PathBuf::from("/repo"),
"main".to_owned(),
"abc1234def".to_owned(),
"add retries".to_owned(),
Config::default(),
);
state.status = status;
state
}
fn candidate(label: char, summary: &str, empty: bool, failed: Option<&str>) -> Candidate {
Candidate {
index: 0,
label,
agent: "claude".to_owned(),
branch: format!("magi/x/{label}"),
worktree: PathBuf::from("/repo"),
summary: summary.to_owned(),
stat: String::new(),
files: 0,
commits: usize::from(!empty),
empty,
failed: failed.map(str::to_owned),
duration_ms: 0,
folded: false,
}
}
#[test]
fn diagnostic_names_the_failing_gate_checks_and_their_output() {
let mut state = run_state(RunStatus::Blocked);
state.gate = vec![
CommandOutcome {
command: "cargo make check".to_owned(),
code: Some(0),
output_tail: "ok".to_owned(),
duration_ms: 0,
},
CommandOutcome {
command: "cargo test".to_owned(),
code: Some(101),
output_tail: "thread 'x' panicked: assertion failed".to_owned(),
duration_ms: 0,
},
];
let d = diagnostic(&state).expect("a failing gate must produce a diagnostic");
assert!(d.contains("cargo test"), "{d}");
assert!(
!d.contains("cargo make check"),
"a passing check is not a diagnostic: {d}"
);
assert!(d.contains("assertion failed"), "{d}");
}
#[test]
fn diagnostic_names_the_checks_the_fixer_gave_up_in_front_of() {
let mut state = run_state(RunStatus::Blocked);
state.event(
"land",
"stopped: the fixer produced no commit while 2 check(s) were failing \
(build, lint); stopping instead of looping on an unchanged tree",
);
let d = diagnostic(&state).expect("a stalled land loop must produce a diagnostic");
assert!(d.contains("build"), "{d}");
assert!(d.contains("lint"), "{d}");
assert!(d.contains("fixer produced no commit"), "{d}");
}
#[test]
fn diagnostic_carries_a_candidates_own_final_word_when_none_was_viable() {
let mut state = run_state(RunStatus::Failed);
state.candidates = vec![candidate(
'A',
"opened pull request #42, merged it, tagged v1.2.3 and published the release",
true,
None,
)];
let d = diagnostic(&state).expect("an empty candidate with a summary must be surfaced");
assert!(d.contains("candidate A"), "{d}");
assert!(d.contains("tagged v1.2.3"), "{d}");
}
#[test]
fn diagnostic_falls_back_to_a_candidates_failure_reason_when_it_has_no_summary() {
let mut state = run_state(RunStatus::Failed);
state.candidates = vec![candidate('A', "", true, Some("agent timed out"))];
let d = diagnostic(&state).expect("a candidate's own failure reason must be surfaced");
assert!(d.contains("candidate A"), "{d}");
assert!(d.contains("agent timed out"), "{d}");
}
#[test]
fn diagnostic_is_none_when_nothing_recognisable_explains_the_hold() {
let mut state = run_state(RunStatus::Failed);
state.candidates = vec![candidate('A', "did the work", false, None)];
assert!(diagnostic(&state).is_none());
}
#[test]
fn diagnostic_is_bounded_however_much_a_run_printed() {
let mut state = run_state(RunStatus::Blocked);
state.gate = vec![
CommandOutcome {
command: "cargo test".to_owned(),
code: Some(101),
output_tail: "x".repeat(50_000),
duration_ms: 0,
},
CommandOutcome {
command: "cargo clippy".to_owned(),
code: Some(1),
output_tail: "y".repeat(50_000),
duration_ms: 0,
},
];
state.candidates = vec![
candidate('A', &"z".repeat(50_000), true, None),
candidate('B', &"w".repeat(50_000), true, None),
];
let d = diagnostic(&state).expect("plenty here to diagnose");
assert!(
d.len() <= DIAGNOSTIC_MAX,
"diagnostic grew to {} bytes, unbounded",
d.len()
);
}
#[test]
fn settle_and_diagnose_attaches_a_diagnostic_only_once_the_task_is_held() {
let mut state = run_state(RunStatus::Blocked);
state.gate = vec![CommandOutcome {
command: "cargo test".to_owned(),
code: Some(101),
output_tail: "assertion failed".to_owned(),
duration_ms: 0,
}];
let verdict = Verdict {
status: RunStatus::Blocked,
left_pr: false,
quota_hit: false,
parked: false,
no_viable_candidates: false,
};
let mut t = task();
t.start("run-1".to_owned());
settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
assert_eq!(t.status, TaskStatus::Failed);
assert!(t.diagnostic.is_none());
t.start("run-2".to_owned());
settle_and_diagnose(&mut t, verdict, "gate failed", 2, &state);
assert_eq!(t.status, TaskStatus::Held);
let d = t.diagnostic.expect("a held task must carry its diagnostic");
assert!(d.contains("cargo test"), "{d}");
}
fn approval_question(run: &str) -> ask::Question {
ask::Question::new(
run.to_owned(),
land::APPROVAL_NODE.to_owned(),
"land".to_owned(),
"merge?".to_owned(),
String::new(),
vec!["merge".to_owned(), "hold".to_owned()],
)
}
#[test]
fn land_resume_state_leaves_a_fresh_open_question_waiting() {
crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
let mut state = run_state(RunStatus::Landing);
state.id = "20260101-000000-fre1".to_owned();
state.parked = true;
state.save().unwrap();
ask::Questions::open()
.put(&mut approval_question(&state.id))
.unwrap();
let mut t = task();
t.runs.push(state.id.clone());
assert_eq!(
land_resume_state(&t),
LandResume::StillWaiting,
"nobody has answered and the timeout has not passed"
);
}
#[test]
fn land_resume_state_abandons_a_question_that_outlived_answer_timeout() {
crate::run::set_home(std::env::temp_dir().join("magi-daemon-test-home"));
let mut state = run_state(RunStatus::Landing);
state.id = "20260101-000000-exp1".to_owned();
state.parked = true;
state.config.graph.answer_timeout = 60;
state.save().unwrap();
let store = ask::Questions::open();
let mut q = approval_question(&state.id);
q.asked_at = Timestamp::now() - jiff::SignedDuration::from_secs(120);
store.put(&mut q).unwrap();
let mut t = task();
t.runs.push(state.id.clone());
assert_eq!(
land_resume_state(&t),
LandResume::Ready,
"an expired question must not be waited on forever"
);
let after = store.get(&q.id).unwrap();
assert!(
!after.status.open(),
"the question is abandoned, not silently ignored"
);
assert!(
after.resolution().is_none(),
"an abandoned question is not read as a decision"
);
}
#[test]
fn reclaim_settles_a_running_task_against_its_last_run() {
let mut t = task();
t.start("20260904-000000-4043".to_owned());
reclaim(&mut t, Some(run_state(RunStatus::Ready)), 2);
assert_eq!(
t.status,
TaskStatus::Done,
"a run that actually finished must not stay `running` forever"
);
}
#[test]
fn reclaim_reuses_the_same_retry_policy_as_a_live_settle() {
let mut t = task();
t.start("20260904-000000-4043".to_owned());
reclaim(&mut t, Some(run_state(RunStatus::Blocked)), 2);
assert_eq!(t.status, TaskStatus::Failed);
assert!(t.status.runnable());
}
#[test]
fn reclaim_holds_a_running_task_whose_run_cannot_be_found() {
let mut t = task();
t.start("20260904-000000-4043".to_owned());
reclaim(&mut t, None, 2);
assert_eq!(t.status, TaskStatus::Held);
assert!(
t.last_error
.as_deref()
.is_some_and(|e| e.contains("running")),
"the operator needs to know why this task was held"
);
}
#[test]
fn orphaned_running_tasks_are_reclaimed_but_live_ones_are_left_alone() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().to_path_buf());
let mut orphaned = task();
orphaned.id = "20260904-000000-orph".to_owned();
orphaned.status = TaskStatus::Running;
orphaned.attempts = 1;
queue.put(&mut orphaned).unwrap();
let mut alive = task();
alive.id = "20260904-000000-live".to_owned();
alive.status = TaskStatus::Running;
alive.attempts = 1;
queue.put(&mut alive).unwrap();
let _held_by_a_live_daemon = queue.claim(&alive.id).unwrap();
let mut queued = task();
queued.id = "20260904-000000-wait".to_owned();
queue.put(&mut queued).unwrap();
let reclaimed = reclaim_orphaned_running(&queue, 2);
assert_eq!(reclaimed, vec![orphaned.id.clone()]);
assert_eq!(
queue.get(&orphaned.id).unwrap().status,
TaskStatus::Held,
"nothing was driving it and there was no run to recover"
);
assert_eq!(
queue.get(&alive.id).unwrap().status,
TaskStatus::Running,
"a live claim must protect the task it belongs to"
);
assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
}
fn read_run_under(home: &Path, id: &str) -> RunState {
let body = std::fs::read_to_string(home.join("runs").join(id).join("run.json")).unwrap();
serde_json::from_str(&body).unwrap()
}
#[test]
fn reclaim_abandoned_runs_fails_a_run_whose_active_seats_are_all_provably_dead() {
let dir = tempfile::tempdir().unwrap();
let home = dir.path().to_path_buf();
let now = Timestamp::now();
let overrun_seat = || crate::run::ActiveSeat {
node: "implement".to_owned(),
started_at: now - jiff::SignedDuration::new(21_000, 0),
timeout_secs: 3_600,
attempt: 0,
};
let mut dead = run_state(RunStatus::Implementing);
dead.id = "20260101-000000-dead".to_owned();
dead.active.insert("impl-A".to_owned(), overrun_seat());
dead.save_under(&home).unwrap();
let mut alive = run_state(RunStatus::Implementing);
alive.id = "20260101-000000-aliv".to_owned();
alive.active.insert("impl-A".to_owned(), overrun_seat());
alive.save_under(&home).unwrap();
let mut status = Status::new();
status.current = vec![Current {
task: "20260101-000000-task".to_owned(),
run: alive.id.clone(),
}];
write_status_to(&home.join("daemon.json"), &status).unwrap();
let questions = Questions::at(home.join("questions"));
let mut q = ask::Question::new(
dead.id.clone(),
"implement".to_owned(),
"impl-A".to_owned(),
"Which storage backend?".to_owned(),
String::new(),
vec!["SQLite".to_owned(), "Redis".to_owned()],
);
questions.put(&mut q).unwrap();
let abandoned = reclaim_abandoned_runs(&home, now);
assert_eq!(abandoned, vec![dead.id.clone()]);
let reloaded = read_run_under(&home, &dead.id);
assert_eq!(reloaded.status, RunStatus::Failed);
assert!(reloaded.active.is_empty());
assert!(
!questions.get(&q.id).unwrap().status.open(),
"the failed run's own open question must be settled in the same pass"
);
let still_alive = read_run_under(&home, &alive.id);
assert_eq!(
still_alive.status,
RunStatus::Implementing,
"a live daemon's claim protects it"
);
assert!(!still_alive.active.is_empty());
}
#[test]
fn an_already_claimed_task_is_skipped_rather_than_failed() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().to_path_buf());
let mut only = task();
queue.put(&mut only).unwrap();
let _elsewhere = queue.claim(&only.id).unwrap();
let candidates = runnable(&queue);
assert_eq!(candidates.len(), 1, "the task is still runnable");
assert!(
queue.claim(&candidates[0].id).is_err(),
"the loop cannot take a claim somebody else holds"
);
let after = queue.get(&only.id).unwrap();
assert_eq!(after.status, TaskStatus::Queued);
assert_eq!(
after.attempts, 0,
"losing the race is not an attempt at the task"
);
assert_eq!(after.last_error, None);
}
#[test]
fn the_status_file_round_trips_and_its_heartbeat_advances() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("daemon.json");
let mut status = Status::new();
status.idle = false;
status.completed = 7;
status.current = vec![Current {
task: "20260902-000000-t111".to_owned(),
run: "20260902-000001-r111".to_owned(),
}];
write_status_to(&path, &status).unwrap();
let first: Status = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
assert_eq!(first.schema, SCHEMA);
assert_eq!(first.pid, std::process::id());
assert!(!first.idle);
assert_eq!(first.completed, 7);
assert_eq!(first.current, status.current);
assert!(
!path.with_extension("json.tmp").exists(),
"the temp file is renamed, not left behind"
);
std::thread::sleep(Duration::from_millis(5));
status.updated_at = Timestamp::now();
status.polls = 3;
write_status_to(&path, &status).unwrap();
let second: Status =
serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
assert!(
second.updated_at > first.updated_at,
"a reader can only detect staleness if the heartbeat moves"
);
assert_eq!(
second.started_at, first.started_at,
"the start time is not a heartbeat"
);
assert_eq!(second.polls, 3);
}
#[test]
fn reading_counts_as_running_only_while_its_heartbeat_is_fresh() {
let dir = tempfile::tempdir().unwrap();
assert!(read_status(dir.path()).is_none(), "no file, no daemon");
let mut status = Status::new();
status.updated_at = Timestamp::now() - jiff::SignedDuration::from_secs(60);
write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
let stale = read_status(dir.path()).unwrap();
assert!(
!stale.running(Timestamp::now()),
"a minute without a heartbeat is a dead daemon, not a busy one"
);
assert!(stale.age_secs(Timestamp::now()).is_some_and(|s| s >= 55));
status.updated_at = Timestamp::now();
write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
let fresh = read_status(dir.path()).unwrap();
assert!(fresh.running(Timestamp::now()));
}
#[test]
fn only_a_live_daemon_on_this_very_run_counts_as_working_on_it() {
let dir = tempfile::tempdir().unwrap();
let now = Timestamp::now();
let mine = "20260903-080619-01c2";
assert!(
!is_working_on(dir.path(), mine, now),
"no status file means nobody is working on anything"
);
let mut status = Status::new();
status.current = vec![Current {
task: "20260903-080340-0167".to_owned(),
run: mine.to_owned(),
}];
status.updated_at = now;
write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
assert!(is_working_on(dir.path(), mine, now));
assert!(
!is_working_on(dir.path(), "20260903-105039-3cbf", now),
"a daemon busy with one run is not working on another"
);
status.updated_at = now - jiff::SignedDuration::from_secs(600);
write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
assert!(
!is_working_on(dir.path(), mine, now),
"a stale heartbeat is a dead daemon, so its run is a leftover"
);
}
#[test]
fn is_working_on_short_matches_by_the_worktree_bays_own_name() {
let dir = tempfile::tempdir().unwrap();
let now = Timestamp::now();
assert!(
!is_working_on_short(dir.path(), "01c2", now),
"no status file means nobody is working on anything"
);
let mut status = Status::new();
status.current = vec![Current {
task: "20260903-080340-0167".to_owned(),
run: "20260903-080619-01c2".to_owned(),
}];
status.updated_at = now;
write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
assert!(
is_working_on_short(dir.path(), "01c2", now),
"the run's short id is the last block of its full id"
);
assert!(
!is_working_on_short(dir.path(), "3cbf", now),
"a daemon busy with one worktree bay is not working on another"
);
}
#[test]
fn a_newer_status_file_still_yields_a_reading() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(
dir.path().join("daemon.json"),
serde_json::json!({
"schema": 2,
"updated_at": Timestamp::now().to_string(),
"idle": true,
"surprise": { "nested": [1, 2, 3] },
})
.to_string(),
)
.unwrap();
let reading = read_status(dir.path()).expect("a forward-compatible read");
assert!(reading.running(Timestamp::now()));
assert!(reading.idle);
assert!(reading.current.is_empty());
}
#[test]
fn an_older_daemons_single_object_current_still_reads_as_a_one_item_list() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(
dir.path().join("daemon.json"),
serde_json::json!({
"schema": 1,
"pid": 4242,
"updated_at": Timestamp::now().to_string(),
"idle": false,
"current": {"task": "20260902-140501-aaaa", "run": "20260902-140502-bbbb"},
"completed": 3,
"polls": 9,
})
.to_string(),
)
.unwrap();
let reading = read_status(dir.path()).expect("an older shape must still parse");
assert!(reading.running(Timestamp::now()));
assert_eq!(
reading.current,
vec![Current {
task: "20260902-140501-aaaa".to_owned(),
run: "20260902-140502-bbbb".to_owned(),
}]
);
}
#[test]
fn an_absent_or_null_current_reads_as_idle_not_a_parse_failure() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(
dir.path().join("daemon.json"),
serde_json::json!({
"schema": 1,
"updated_at": Timestamp::now().to_string(),
"idle": true,
"current": null,
})
.to_string(),
)
.unwrap();
let with_null = read_status(dir.path()).expect("null must still parse");
assert!(with_null.current.is_empty());
std::fs::write(
dir.path().join("daemon.json"),
serde_json::json!({
"schema": 1,
"updated_at": Timestamp::now().to_string(),
"idle": true,
})
.to_string(),
)
.unwrap();
let absent = read_status(dir.path()).expect("a missing field must still parse");
assert!(absent.current.is_empty());
}
#[test]
fn a_task_without_a_repository_runs_in_the_daemons_default() {
let fallback = Path::new("/default");
let mut blank = task();
blank.repo = PathBuf::new();
assert_eq!(repo_for(&blank, fallback), PathBuf::from("/default"));
let mut dot = task();
dot.repo = PathBuf::from(".");
assert_eq!(repo_for(&dot, fallback), PathBuf::from("/default"));
assert_eq!(
repo_for(&task(), fallback),
PathBuf::from("/repo"),
"a task that names a repository keeps it"
);
}
#[test]
fn a_solo_task_runs_with_one_candidate_and_a_plain_task_keeps_the_configs() {
let mut solo_cfg = Config::default();
solo_cfg.graph.candidates = 3;
let mut solo_task = task();
solo_task.solo = true;
apply_solo(&mut solo_cfg, &solo_task);
assert_eq!(solo_cfg.graph.candidates, 1);
let mut plain_cfg = Config::default();
plain_cfg.graph.candidates = 3;
let plain_task = task();
assert!(!plain_task.solo);
apply_solo(&mut plain_cfg, &plain_task);
assert_eq!(
plain_cfg.graph.candidates, 3,
"a task that did not ask to run alone keeps the config's candidates"
);
}
#[test]
fn merge_overrides_are_parsed_or_refused() {
assert_eq!(merge_mode("none").unwrap(), MergeMode::None);
assert_eq!(merge_mode("local").unwrap(), MergeMode::Local);
assert_eq!(merge_mode("pr").unwrap(), MergeMode::Pr);
assert!(merge_mode("squash").is_err());
}
#[test]
fn quota_wait_uses_a_future_reset_time_capped_and_falls_back_otherwise() {
let now = Timestamp::now();
let fallback = Duration::from_secs(300);
let cap = Duration::from_secs(1800);
assert_eq!(quota_wait(None, now, fallback, cap), fallback);
let soon = now + jiff::SignedDuration::from_secs(600);
assert_eq!(
quota_wait(Some(soon), now, fallback, cap),
Duration::from_secs(600)
);
let past = now - jiff::SignedDuration::from_secs(60);
assert_eq!(quota_wait(Some(past), now, fallback, cap), fallback);
let far = now + jiff::SignedDuration::from_secs(3 * 3600);
assert_eq!(quota_wait(Some(far), now, fallback, cap), cap);
}
#[test]
fn parse_reset_hint_reads_the_claude_cli_shape_and_rolls_a_past_clock_to_tomorrow() {
let now = "2026-09-07T02:50:00Z".parse::<Timestamp>().unwrap();
let at = parse_reset_hint("4:50am (UTC)", now).expect("a recognised shape parses");
assert_eq!(at.to_string(), "2026-09-07T04:50:00Z");
let already_past =
parse_reset_hint("1:00am (UTC)", now).expect("a recognised shape parses");
assert_eq!(already_past.to_string(), "2026-09-08T01:00:00Z");
assert!(
parse_reset_hint("session limit reached", now).is_none(),
"free text with no recognised shape is not guessed at"
);
assert!(
parse_reset_hint("4:50am (Nowhere/Fake)", now).is_none(),
"an unresolvable zone name is not guessed at either"
);
}
fn idle_loop(dir: &Path) -> (Opts, Queue, PathBuf, PathBuf, PathBuf) {
let config = dir.join("magi.toml");
std::fs::write(
&config,
"[disk]\nmin_free_bytes = 0\nauto_fold = false\ncache_limit_bytes = 0\n",
)
.unwrap();
let opts = Opts {
poll: Duration::from_secs(30),
config: Some(config),
repo: dir.join("repo"),
..Opts::default()
};
let home = dir.join("home");
let worktrees = dir.join("wt");
(
opts,
Queue::at(dir.join("queue")),
home.join("daemon.json"),
home,
worktrees,
)
}
#[test]
fn a_stop_is_idempotent_and_once_set_stays_set() {
let stop = Stop::new();
assert!(!stop.stopped());
stop.stop();
assert!(stop.stopped());
stop.stop();
assert!(stop.stopped(), "a second stop is not a toggle");
let shared = stop.clone();
assert!(
shared.stopped(),
"a clone is the same stop; that is how the loop and its caller share one"
);
}
#[test]
fn only_a_stop_with_a_run_in_flight_reads_as_finishing() {
let stop = Stop::new();
stop.enter();
assert!(
!stop.finishing(),
"a busy loop nobody has asked to stop is just running"
);
stop.stop();
assert!(
stop.finishing(),
"a stop asked for mid-run has not landed until the run is settled"
);
stop.exit();
assert!(
!stop.finishing(),
"once the run is settled the stop has landed and there is nothing to finish"
);
}
#[test]
fn finishing_stays_true_until_the_last_of_several_runs_exits() {
let stop = Stop::new();
stop.enter();
stop.enter();
stop.stop();
assert!(stop.finishing(), "two runs still in flight");
stop.exit();
assert!(
stop.finishing(),
"one run finished, but a sibling is still working"
);
stop.exit();
assert!(
!stop.finishing(),
"the last run out is what actually lands the stop"
);
}
#[tokio::test]
async fn a_loop_already_asked_to_stop_returns_without_waiting_out_a_poll() {
let dir = tempfile::tempdir().unwrap();
let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
let stop = Stop::new();
stop.stop();
let began = std::time::Instant::now();
tokio::time::timeout(
Duration::from_secs(2),
drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
)
.await
.expect("a stopped loop must return, not sit out its poll interval")
.expect("the loop's own setup and teardown must not fail");
assert!(
began.elapsed() < opts.poll,
"returned only after {:?}, which is a poll interval, not a stop",
began.elapsed()
);
}
#[tokio::test]
async fn a_stop_while_idle_wakes_the_wait_instead_of_sleeping_it_out() {
let dir = tempfile::tempdir().unwrap();
let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
let stop = Stop::new();
let asker = {
let stop = stop.clone();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(20)).await;
stop.stop();
})
};
let began = std::time::Instant::now();
tokio::time::timeout(
Duration::from_secs(2),
drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
)
.await
.expect("a stop asked for while idle must wake the wait")
.expect("the loop's own setup and teardown must not fail");
asker.await.unwrap();
assert!(
began.elapsed() < opts.poll,
"returned only after {:?}, so the stop waited on the sleep",
began.elapsed()
);
}
#[tokio::test]
async fn a_stopped_loop_leaves_no_status_file_claiming_it_is_running() {
let dir = tempfile::tempdir().unwrap();
let (opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
let stop = Stop::new();
stop.stop();
tokio::time::timeout(
Duration::from_secs(2),
drive(&opts, &queue, &status_file, &home, &worktrees, &stop),
)
.await
.expect("a stopped loop must return")
.expect("the loop's own setup and teardown must not fail");
assert!(
home.is_dir(),
"the loop did publish a status file, so its removal is the teardown and not an absence"
);
assert!(
!status_file.exists(),
"a stopped loop clears its status file"
);
assert!(
read_status(&home).is_none(),
"a reader must see no daemon at all, not a heartbeat that merely stopped"
);
}
#[tokio::test]
async fn once_runs_startup_housekeeping_before_an_empty_queue_exits() {
let dir = tempfile::tempdir().unwrap();
let (mut opts, queue, status_file, home, worktrees) = idle_loop(dir.path());
opts.once = true;
let mut settled = RunState::new(
dir.path().join("repo"),
"main".to_owned(),
"abc1234".to_owned(),
"fixture".to_owned(),
Config::default(),
);
settled.status = RunStatus::Ready;
let run_dir = home.join("runs").join(&settled.id);
std::fs::create_dir_all(&run_dir).unwrap();
std::fs::write(
run_dir.join("run.json"),
serde_json::to_string_pretty(&settled).unwrap(),
)
.unwrap();
let questions = Questions::at(home.join("questions"));
let mut question = ask::Question::new(
settled.id.clone(),
"review".to_owned(),
"reviewer-1".to_owned(),
"Continue?".to_owned(),
String::new(),
Vec::new(),
);
questions.put(&mut question).unwrap();
drive(&opts, &queue, &status_file, &home, &worktrees, &Stop::new())
.await
.unwrap();
assert_eq!(
questions.get(&question.id).unwrap().status,
ask::QuestionStatus::Abandoned,
"an empty --once drain still performs startup question cleanup"
);
}
#[test]
fn task_question_reconciliation_keeps_references_and_retires_manual_releases() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().join("queue"));
let questions = Questions::at(dir.path().join("questions"));
let mut task = task();
queue.put(&mut task).unwrap();
let mut task_question = ask::Question::new(
task.id.clone(),
crate::conduct::NODE.to_owned(),
"conduct".to_owned(),
"Which backend?".to_owned(),
String::new(),
Vec::new(),
);
questions.put(&mut task_question).unwrap();
task.block(vec![task_question.id.clone()], None);
queue.put(&mut task).unwrap();
let mut run_question = ask::Question::new(
"20260101-000000-run1".to_owned(),
"review".to_owned(),
"reviewer-1".to_owned(),
"Run question".to_owned(),
String::new(),
Vec::new(),
);
questions.put(&mut run_question).unwrap();
let mut coincidental = ask::Question::new(
task.id.clone(),
"review".to_owned(),
"reviewer-1".to_owned(),
"Unrelated review question".to_owned(),
String::new(),
Vec::new(),
);
questions.put(&mut coincidental).unwrap();
reconcile_task_questions(&queue, &questions);
assert!(questions.get(&task_question.id).unwrap().status.open());
assert!(questions.get(&run_question.id).unwrap().status.open());
assert!(questions.get(&coincidental.id).unwrap().status.open());
task.release();
queue.put(&mut task).unwrap();
reconcile_task_questions(&queue, &questions);
assert_eq!(
questions.get(&task_question.id).unwrap().status,
ask::QuestionStatus::Abandoned
);
assert!(
questions.get(&run_question.id).unwrap().status.open(),
"run questions remain the run janitor's responsibility"
);
assert!(
questions.get(&coincidental.id).unwrap().status.open(),
"a non-conductor question must not be abandoned just because its \
run id coincides with a task id"
);
}
#[test]
fn a_freshly_started_running_task_is_never_stalled() {
let dir = tempfile::tempdir().unwrap();
let mut t = task();
t.start("run-1".to_owned());
assert!(!is_stalled(&t, dir.path(), Timestamp::now()));
}
#[test]
fn a_long_running_task_with_no_live_daemon_is_stalled() {
let dir = tempfile::tempdir().unwrap();
let mut t = task();
t.start("run-1".to_owned());
t.updated_at = Timestamp::now()
- jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
assert!(is_stalled(&t, dir.path(), Timestamp::now()));
assert_eq!(
stalled_tasks(
&Queue::at(dir.path().join("q")),
dir.path(),
Timestamp::now()
)
.len(),
0,
"the task was never written to this queue"
);
}
#[test]
fn a_long_running_task_a_live_daemon_still_names_is_not_stalled() {
let dir = tempfile::tempdir().unwrap();
let mut t = task();
t.id = "20260903-080340-0167".to_owned();
t.start("20260903-080619-01c2".to_owned());
t.updated_at = Timestamp::now()
- jiff::SignedDuration::from_secs(STALLED_RUNNING.as_secs() as i64 + 60);
let mut status = Status::new();
status.current = vec![Current {
task: t.id.clone(),
run: "20260903-080619-01c2".to_owned(),
}];
write_status_to(&dir.path().join("daemon.json"), &status).unwrap();
assert!(
!is_stalled(&t, dir.path(), Timestamp::now()),
"a live daemon's own heartbeat rules out stalled, however long the task has run"
);
}
fn backdate_task(queue: &Queue, id: &str, seconds_ago: i64) {
let path = queue.path_of(id);
let body = std::fs::read_to_string(&path).unwrap();
let mut v: serde_json::Value = serde_json::from_str(&body).unwrap();
let old = Timestamp::now() - jiff::SignedDuration::from_secs(seconds_ago);
v["updated_at"] = serde_json::Value::String(old.to_string());
std::fs::write(&path, serde_json::to_string_pretty(&v).unwrap()).unwrap();
}
#[test]
fn stalled_tasks_still_reaches_a_task_reclaim_could_not_claim_yet() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().join("queue"));
let home = dir.path().join("home");
let mut t = task();
t.id = "20260101-000001-lock".to_owned();
t.start("run-1".to_owned());
queue.put(&mut t).unwrap();
backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
std::fs::write(
dir.path().join("queue").join(format!("{}.lock", t.id)),
"not a pid",
)
.unwrap();
let now = Timestamp::now();
assert!(
reclaim_orphaned_running(&queue, 2).is_empty(),
"the unparseable lock is still well within STALE_CLAIM, so the claim fails \
and reclaim must leave the task alone"
);
assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Running);
let stalled = stalled_tasks(&queue, &home, now);
assert_eq!(
stalled.len(),
1,
"reclaim's inability to claim it yet must not hide it from the conductor"
);
assert_eq!(stalled[0].id, t.id);
}
#[test]
fn ordinary_dead_daemon_task_is_shown_stalled_before_reclaim_and_can_be_requeued() {
let dir = tempfile::tempdir().unwrap();
crate::run::set_home(dir.path().join("run-home"));
let queue = Queue::at(dir.path().join("queue"));
let home = dir.path().join("home");
let questions = Questions::at(dir.path().join("questions"));
let mut t = task();
t.id = "20260101-000003-dead".to_owned();
t.start("missing-run".to_owned());
queue.put(&mut t).unwrap();
backdate_task(&queue, &t.id, STALLED_RUNNING.as_secs() as i64 + 60);
let stalled = stalled_tasks(&queue, &home, Timestamp::now());
assert_eq!(
stalled.iter().map(|task| &task.id).collect::<Vec<_>>(),
[&t.id]
);
assert_eq!(reclaim_orphaned_running(&queue, 2), [t.id.clone()]);
assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
crate::conduct::apply(
&queue,
&questions,
&crate::conduct::Verdict {
decisions: vec![crate::conduct::Decision {
id: t.id.clone(),
recovery: Some(crate::conduct::Recovery::Requeue),
..crate::conduct::Decision::default()
}],
},
)
.unwrap();
assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
}
#[test]
fn stalled_tasks_reports_exactly_the_tasks_is_stalled_agrees_on() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().join("queue"));
let home = dir.path().join("home");
let mut fresh = task();
fresh.id = "20260101-000001-aaaa".to_owned();
fresh.start("run-1".to_owned());
queue.put(&mut fresh).unwrap();
let mut old = task();
old.id = "20260101-000002-bbbb".to_owned();
old.start("run-2".to_owned());
queue.put(&mut old).unwrap();
backdate_task(&queue, &old.id, STALLED_RUNNING.as_secs() as i64 + 60);
let stalled = stalled_tasks(&queue, &home, Timestamp::now());
assert_eq!(stalled.len(), 1);
assert_eq!(stalled[0].id, old.id);
}
#[test]
fn queued_and_finished_task_views_partition_by_status() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().join("queue"));
let mut queued = task();
queued.id = "20260101-000001-aaaa".to_owned();
queue.put(&mut queued).unwrap();
let mut failed = task();
failed.id = "20260101-000002-bbbb".to_owned();
failed.start("run-1".to_owned());
failed.fail("gate red", 5);
queue.put(&mut failed).unwrap();
let mut held = task();
held.id = "20260101-000003-cccc".to_owned();
held.hold_machine(None);
queue.put(&mut held).unwrap();
let mut running = task();
running.id = "20260101-000004-dddd".to_owned();
running.start("run-2".to_owned());
queue.put(&mut running).unwrap();
let queued_ids: Vec<String> = queued_tasks(&queue).into_iter().map(|t| t.id).collect();
assert_eq!(queued_ids, [queued.id.clone()]);
let mut finished_ids: Vec<String> =
finished_tasks(&queue).into_iter().map(|t| t.id).collect();
finished_ids.sort_unstable();
let mut want = vec![failed.id.clone(), held.id.clone()];
want.sort_unstable();
assert_eq!(finished_ids, want);
}
#[test]
fn resolve_blockers_clears_a_done_dependency_and_keeps_an_unresolved_one() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().join("queue"));
let questions = ask::Questions::at(dir.path().join("questions"));
let mut dep = task();
dep.id = "20260101-000001-dep0".to_owned();
dep.succeed();
queue.put(&mut dep).unwrap();
let mut still_going = task();
still_going.id = "20260101-000002-dep1".to_owned();
queue.put(&mut still_going).unwrap();
let mut blocked = task();
blocked.id = "20260101-000003-main".to_owned();
blocked.block(
vec![dep.id.clone(), still_going.id.clone()],
Some("waits on both".to_owned()),
);
queue.put(&mut blocked).unwrap();
resolve_blockers(&queue, &questions);
let after = queue.get(&blocked.id).unwrap();
assert_eq!(
after.status,
TaskStatus::Blocked,
"one dependency is still outstanding"
);
assert_eq!(after.blocked_by, [still_going.id.clone()]);
}
#[test]
fn resolve_blockers_carries_an_answers_content_onto_the_task_and_unblocks_it() {
let dir = tempfile::tempdir().unwrap();
let queue = Queue::at(dir.path().join("queue"));
let questions = ask::Questions::at(dir.path().join("questions"));
let mut q = crate::ask::Question::new(
"20260101-000001-main".to_owned(),
crate::conduct::NODE.to_owned(),
"conduct".to_owned(),
"Which backend?".to_owned(),
String::new(),
Vec::new(),
);
questions.put(&mut q).unwrap();
q.answer(crate::ask::Answer::Text("SQLite".to_owned()))
.unwrap();
questions.put(&mut q).unwrap();
let mut blocked = task();
blocked.id = "20260101-000001-main".to_owned();
blocked.block(vec![q.id.clone()], Some("which backend?".to_owned()));
queue.put(&mut blocked).unwrap();
resolve_blockers(&queue, &questions);
let after = queue.get(&blocked.id).unwrap();
assert_eq!(
after.status,
TaskStatus::Queued,
"the only blocker resolved"
);
assert_eq!(after.answers.len(), 1);
assert_eq!(after.answers[0].question, "Which backend?");
assert_eq!(after.answers[0].answer, "SQLite");
let instruction = instruction_for(&after);
assert!(instruction.contains("Which backend?"));
assert!(instruction.contains("SQLite"));
}
#[test]
fn instruction_for_is_unchanged_without_any_answers() {
let t = task();
assert_eq!(instruction_for(&t), t.instruction);
}
#[test]
fn resumed_instruction_is_unchanged_without_any_answers() {
let t = task();
assert_eq!(resumed_instruction(&t.instruction, &t), t.instruction);
}
#[test]
fn resumed_instruction_carries_a_new_answer_onto_the_old_run() {
let mut t = task();
t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
let old = t.instruction.clone();
let refreshed = resumed_instruction(&old, &t);
assert!(refreshed.starts_with(&old), "the original text is kept");
assert!(refreshed.contains("Which backend?"));
assert!(refreshed.contains("SQLite"));
}
#[test]
fn resumed_instruction_keeps_an_original_answers_heading() {
let mut t = task();
t.instruction = "Context\n\n# Operator answers\n\nThis is part of the task.".to_owned();
t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
let refreshed = resumed_instruction(&t.instruction, &t);
assert!(
refreshed.starts_with(&t.instruction),
"an answers heading in the original instruction is not the appended block"
);
assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 2);
assert!(refreshed.contains("Which backend?"));
assert!(refreshed.contains("SQLite"));
let repeated = resumed_instruction(&refreshed, &t);
assert_eq!(
repeated, refreshed,
"only the final appended block is refreshed"
);
}
#[test]
fn resumed_instruction_does_not_duplicate_across_repeated_resumes() {
let mut t = task();
t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
let once = resumed_instruction(&t.instruction, &t);
let twice = resumed_instruction(&once, &t);
assert_eq!(once, twice);
assert_eq!(once.matches("Which backend?").count(), 1);
t.record_answer("Which cache?".to_owned(), "Redis".to_owned());
let refreshed = resumed_instruction(&once, &t);
assert_eq!(refreshed.matches(ANSWERS_HEADER).count(), 1);
assert!(refreshed.contains("Which backend?"));
assert!(refreshed.contains("Which cache?"));
}
#[test]
fn prepare_instruction_covers_all_three_starters() {
let mut t = task();
t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
assert_eq!(
prepare_instruction(&Starter::Start, None, &t),
Some(instruction_for(&t))
);
let old = t.instruction.clone();
assert_eq!(
prepare_instruction(&Starter::Resume("some-run".to_owned()), Some(&old), &t),
Some(resumed_instruction(&old, &t))
);
assert_eq!(
prepare_instruction(&Starter::Review("magi/eba2/A".to_owned()), Some(&old), &t),
None
);
}
#[test]
fn choose_starter_prefers_review_over_resume_when_the_branch_survived() {
assert_eq!(
choose_starter(Some("magi/eba2/A"), true, Some("some-run")),
Starter::Review("magi/eba2/A".to_owned())
);
}
#[test]
fn choose_starter_falls_back_to_start_when_the_review_branch_is_gone() {
assert_eq!(
choose_starter(Some("magi/eba2/A"), false, Some("some-run")),
Starter::Start,
"a vanished review branch must not fall back to resuming the old run either"
);
}
#[test]
fn choose_starter_resumes_or_starts_when_there_is_no_review_choice_at_all() {
assert_eq!(
choose_starter(None, false, Some("some-run")),
Starter::Resume("some-run".to_owned())
);
assert_eq!(choose_starter(None, false, None), Starter::Start);
}
#[test]
fn an_explicit_release_forces_a_fresh_competition_even_with_a_resumable_run() {
let mut released = task();
released.start("stalled-run".to_owned());
released.requeue();
let unfinished = (!released.fresh_start)
.then(|| Some("stalled-run".to_owned()))
.flatten();
assert_eq!(
choose_starter(None, false, unfinished.as_deref()),
Starter::Start,
"release keeps run history but must not resume it"
);
assert_eq!(released.runs, ["stalled-run"]);
}
#[test]
fn an_ordinary_release_keeps_a_resumable_run_available() {
let mut released = task();
released.start("stalled-run".to_owned());
released.release();
let unfinished = (!released.fresh_start)
.then(|| Some("stalled-run".to_owned()))
.flatten();
assert_eq!(
choose_starter(None, false, unfinished.as_deref()),
Starter::Resume("stalled-run".to_owned()),
"manual release must preserve the normal resume path"
);
}
#[test]
fn a_blocked_run_that_spent_every_review_round_has_exhausted_its_budget() {
let mut state = run_state(RunStatus::Blocked);
state.config.graph.review_rounds = 3;
state.reviews = vec![review_round(1), review_round(2), review_round(3)];
assert!(exhausted_review_budget(&state));
state.reviews.pop();
assert!(!exhausted_review_budget(&state));
let mut stalled = run_state(RunStatus::Stalled);
stalled.config.graph.review_rounds = 1;
stalled.reviews = vec![review_round(1)];
assert!(!exhausted_review_budget(&stalled));
}
fn review_round(round: usize) -> crate::run::ReviewRound {
crate::run::ReviewRound {
round,
head: "deadbeef".to_owned(),
verified_head: None,
reviews: Vec::new(),
e2e: Vec::new(),
verify_retried: false,
e2e_deferred: false,
e2e_defer_reason: None,
fix: None,
blocking: 0,
answered: 1,
expected: 1,
clean: false,
progressed: true,
vote_split: false,
reconsideration: Vec::new(),
verdict: None,
}
}
#[test]
fn unfinished_run_skips_a_round_exhausted_blocked_run_so_requeue_means_a_fresh_competition() {
let mut exhausted = RunState::new(
PathBuf::from("/repo"),
"main".to_owned(),
"abc1234def".to_owned(),
"add retries".to_owned(),
Config::default(),
);
exhausted.status = RunStatus::Blocked;
exhausted.config.graph.review_rounds = 1;
exhausted.reviews = vec![review_round(1)];
assert_eq!(
unfinished_run_with(&[exhausted.id.clone()], "t", |_| Ok(exhausted.clone())),
None,
"an exhausted `Blocked` run must not be offered as resumable"
);
let mut has_budget_left = RunState::new(
PathBuf::from("/repo"),
"main".to_owned(),
"abc1234def".to_owned(),
"add retries".to_owned(),
Config::default(),
);
has_budget_left.status = RunStatus::Blocked;
has_budget_left.config.graph.review_rounds = 3;
has_budget_left.reviews = vec![review_round(1)];
assert_eq!(
unfinished_run_with(&[has_budget_left.id.clone()], "t", |_| {
Ok(has_budget_left.clone())
}),
Some(has_budget_left.id.clone())
);
}
#[test]
fn unfinished_run_never_falls_back_to_an_older_resumable_run() {
let mut older_stalled = RunState::new(
PathBuf::from("/repo"),
"main".to_owned(),
"abc1234def".to_owned(),
"add retries".to_owned(),
Config::default(),
);
older_stalled.status = RunStatus::Stalled;
let mut newest_exhausted = RunState::new(
PathBuf::from("/repo"),
"main".to_owned(),
"abc1234def".to_owned(),
"add retries".to_owned(),
Config::default(),
);
newest_exhausted.status = RunStatus::Blocked;
newest_exhausted.config.graph.review_rounds = 1;
newest_exhausted.reviews = vec![review_round(1)];
assert_eq!(
unfinished_run_with(
&[older_stalled.id.clone(), newest_exhausted.id.clone()],
"t",
|_| Ok(newest_exhausted.clone())
),
None,
"the newest run is exhausted, so nothing here is worth resuming - \
least of all the older, already-superseded run"
);
}
#[test]
fn unfinished_run_warns_and_skips_a_run_it_cannot_read() {
assert_eq!(
unfinished_run_with(&["20260101-000000-gone".to_owned()], "t", |_| {
Err(anyhow::anyhow!("fixture is absent"))
}),
None
);
}
}