Skip to main content

magi/
conduct.rs

1//! The conductor: a single agent seat that arranges the queue.
2//!
3//! [`crate::queue`] orders runnable tasks by `priority` alone; nothing in it
4//! can express that one task should wait for another, or that a task whose
5//! last run stopped short deserves a second look before the loop blindly
6//! retries it. Once per polling cycle, [`Conductor::maybe_run`] shows one
7//! agent seat three things - the runnable tasks, the tasks stuck `running`
8//! with no live daemon behind them, and the `failed`/`held` tasks nobody has
9//! decided about yet - and asks it to decide `blocked_by` for the first and a
10//! recovery for the other two. Everything else about the loop - which
11//! unblocked task runs next, one at a time, in `priority` order - is
12//! unchanged; see `crate::daemon`.
13//!
14//! # What the conductor may not do
15//!
16//! [`Decision`] has no field for `priority`, for deleting a task, or for
17//! touching git, a worktree, or a branch directly. [`Recovery::Review`] only
18//! ever reopens a branch `crate::daemon` itself resolved from the task's own
19//! run record ([`surviving_branch`]) - never a name the model wrote - through
20//! `crate::graph::Runner::review`, which reviews and verifies but never
21//! rewrites history.
22//!
23//! # Non-blocking by construction
24//!
25//! [`crate::ask::ask_and_wait`] is never called from here, and the prompt
26//! tells the model the same: that CLI command blocks until a human answers,
27//! and calling it from inside the conductor's own invocation would park the
28//! whole polling loop behind one task's question. Instead a decision that
29//! wants the operator's judgement carries a `question` field, and [`apply`]
30//! files it with [`Question::new`] and [`Questions::put`] and moves on in the
31//! same call. Something still has to read the owner's reply: the question
32//! carries a [`Deputy`] brief, and `magi serve` starts a short-lived seat of
33//! its own for it ([`crate::deputy`]) - never this loop.
34//!
35//! # Fails soft, always
36//!
37//! [`Conductor::maybe_run`] never returns an error: an unusable roster, a
38//! timed-out invocation, or a reply [`verdict::extract_json`] cannot parse are
39//! all logged and treated as "this cycle changes nothing." `crate::daemon`'s
40//! loop always falls through to its own `Queue::next_runnable` regardless of
41//! what happened here.
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::path::{Path, PathBuf};
45use std::time::Duration;
46
47use anyhow::{Context as _, Result};
48use serde::Deserialize;
49
50use crate::agent::{self, Invocation, SeatState};
51use crate::ask::{Deputy, Question, Questions};
52use crate::config::Config;
53use crate::prompt;
54use crate::queue::{Queue, Task, TaskStatus};
55use crate::run::RunState;
56use crate::verdict;
57
58/// Seat name for the conductor's own CLI-side conversation, scoped away from
59/// every other seat magi ever opens - the same rule every other seat follows.
60const SEAT: &str = "conduct";
61
62/// Node name reported to the invoked agent (`MAGI_NODE`) and recorded on any
63/// question it files, so an operator reading the questions list can tell a
64/// conductor's question from one a run's own agent asked.
65pub const NODE: &str = "conduct";
66
67/// Wall-clock limit for one conductor turn. The conductor reads a queue
68/// listing and replies with json; it does not implement anything or run a
69/// build, so this is short - the same order of magnitude as
70/// `crate::chat`'s own single-turn, no-write invocations.
71const TURN_TIMEOUT: Duration = Duration::from_secs(300);
72
73/// What the conductor may choose for a `running`-but-stalled or a
74/// `failed`/`held` task.
75#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
76#[serde(rename_all = "lowercase")]
77pub enum Recovery {
78    /// Put it back in line, attempts reset - the same effect as
79    /// `magi task release`.
80    Requeue,
81    /// Leave it for a human, unchanged otherwise - the same effect as
82    /// `magi task hold`.
83    Hold,
84    /// Reopen the task's surviving branch as a review-only pass
85    /// (`crate::graph::Runner::review`) instead of competing from scratch.
86    /// Only takes effect when [`surviving_branch`] can actually name one;
87    /// otherwise `crate::daemon` falls back to [`Recovery::Requeue`].
88    Review,
89    /// Close the task outright, as [`Task::succeed`] - the same effect as
90    /// `magi task done`. For a `failed`/`held` task only, never `running`:
91    /// this is for a task whose own goal is already known to be met outside
92    /// the loop entirely (the branch was merged and the worktree cleaned up
93    /// by hand, say) and competing it again would only spend attempts on
94    /// work that has nothing left to do, not for a task that merely stopped
95    /// mid-competition and might still need to run.
96    Done,
97}
98
99/// The conductor's decision for one task. Deliberately has no `priority`
100/// field: see this module's doc.
101#[derive(Debug, Clone, Default, Deserialize)]
102pub struct Decision {
103    /// Task id, expected to be copied verbatim from what it was shown.
104    pub id: String,
105    /// Ids this task should be blocked on. Meaningful only for a runnable
106    /// (`queued`) task; ignored otherwise.
107    #[serde(default)]
108    pub blocked_by: Vec<String>,
109    /// One line explaining the block or the recovery.
110    #[serde(default)]
111    pub reason: Option<String>,
112    /// Recovery for a stalled or finished task. For a runnable (`queued`)
113    /// one, only [`Recovery::Hold`] has any effect - and only when
114    /// `blocked_by` is empty, since a `queued` task with something to wait on
115    /// is handled by that field instead - holding a task the operator has
116    /// already said should not compete again without filing another
117    /// `question` that only restates the same answer. `Requeue`, `Review`,
118    /// and `Done` have no meaning for a task that is already in line.
119    #[serde(default)]
120    pub recovery: Option<Recovery>,
121    /// A question for the operator. When present, [`apply`] files it (unless
122    /// one is already open for this task) and blocks the task on its id
123    /// instead of acting on `blocked_by` or `recovery`.
124    #[serde(default)]
125    pub question: Option<String>,
126    /// Fixed answers for `question`, if it has any. Empty means free text.
127    #[serde(default)]
128    pub choices: Vec<String>,
129    /// Task or question ids carrying independently verifiable new blocking evidence.
130    #[serde(default)]
131    pub evidence: Vec<String>,
132}
133
134/// The conductor's whole reply for one cycle.
135///
136/// `decisions` is deliberately **not** `#[serde(default)]`, unlike every
137/// other field in this module. [`verdict::extract_json`] disambiguates
138/// between several balanced `{...}` spans in one reply by trying the type the
139/// caller wants against each of them, last first, and keeping the first that
140/// fits - which only works when a span that is not really the answer can
141/// fail to fit. A `Verdict` with no required field at all would make every
142/// span fit, including a `{}` left by stray trailing prose, and the reply's
143/// real `decisions` - earlier in the text - would never be reached. Requiring
144/// the key costs nothing: the prompt already asks for it on every reply, `[]`
145/// included.
146#[derive(Debug, Clone, Default, Deserialize)]
147pub struct Verdict {
148    /// One entry per task the conductor chose to say something about. A task
149    /// left out of this list is left exactly as it was.
150    pub decisions: Vec<Decision>,
151}
152
153/// A view of a task built for [`prompt::conduct`], shared by the runnable and
154/// stalled sections.
155fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
156    prompt::ConductTask {
157        id: t.id.clone(),
158        title: t.title.clone(),
159        instruction: t.instruction.clone(),
160        repo: t.repo.display().to_string(),
161        priority: t.priority,
162        status: t.status.as_str().to_owned(),
163        attempts: t.attempts,
164        max_attempts: t.effective_attempt_limit(max_attempts),
165        last_error: t.last_error.clone(),
166        hold_reason: t.hold_reason.clone(),
167        hold_source: t.hold_source.map(|source| source.label().to_owned()),
168        blocked_by: t.blocked_by.clone(),
169        answers: t
170            .answers
171            .iter()
172            .map(|a| prompt::ConductAnswer {
173                question: a.question.clone(),
174                answer: a.answer.clone(),
175            })
176            .collect(),
177        operator_resume: t.resumption.as_ref().map(|r| format!(
178            "Explicit operator resumption generation {} via {} at {}. Approved instruction: {}. \
179             The first {} historical answers and prior hold ({:?}) are superseded as waiting premises. \
180             To block again cite evidence ids from a later owned question, changed dependency, or new failed run; \
181             wording alone is insufficient.",
182            r.generation, r.source, r.at, r.instruction, r.answers_through, r.cleared_reason
183        )).or_else(|| t.resume_override.as_ref().map(|o| {
184            format!(
185                "the operator explicitly answered \"resume\" at {}; do not hold this \
186                 task again for the same reason unless there is new information",
187                o.at
188            )
189        })),
190    }
191}
192
193/// Did the operator answer a `magi ask` choice with a `resume` action that
194/// this task has not yet acted on? Then `Recovery::Requeue` must not turn the
195/// task into a fresh competition either.
196fn pinned_resume(task: &Task) -> bool {
197    task.resume_override
198        .as_ref()
199        .is_some_and(|o| o.pinned_run.is_some())
200}
201
202/// Verify references independently of the model's reason text. Old evidence
203/// cannot become new merely by being described differently.
204fn new_blocking_evidence(
205    queue: &Queue,
206    questions: &Questions,
207    task: &Task,
208    d: &Decision,
209) -> Vec<String> {
210    let Some(r) = &task.resumption else {
211        return Vec::new();
212    };
213    d.evidence
214        .iter()
215        .filter_map(|id| {
216            let version = if let Ok(q) = questions.get(id) {
217                (q.asked_at > r.at
218                    && (q.run == task.id || task.runs.contains(&q.run))
219                    && q.node != NODE
220                    && q.status != crate::ask::QuestionStatus::Abandoned)
221                    .then(|| {
222                        format!(
223                            "question:{id}:{}:{:?}:{}",
224                            q.asked_at,
225                            q.status,
226                            q.resolution().unwrap_or_default()
227                        )
228                    })
229            } else if d.blocked_by.contains(id) {
230                queue
231                    .get(id)
232                    .ok()
233                    .filter(|dep| {
234                        dep.id != task.id && dep.updated_at > r.at && dep.status != TaskStatus::Done
235                    })
236                    .map(|dep| {
237                        format!(
238                            "dependency:{id}:{:?}:{:?}:{:?}",
239                            dep.status, dep.hold_reason, dep.last_error
240                        )
241                    })
242            } else if task.runs.iter().any(|run| run == id) {
243                RunState::load(id)
244                    .ok()
245                    .filter(|run| {
246                        // A resumed run keeps its id. Only a pass started after
247                        // approval (or a newly created legacy run) is new evidence.
248                        run.updated_at > r.at
249                            && run
250                                .passes
251                                .last()
252                                .map_or(run.created_at > r.at, |p| p.started_at > r.at)
253                            && matches!(
254                                run.status,
255                                crate::run::RunStatus::Failed
256                                    | crate::run::RunStatus::Blocked
257                                    | crate::run::RunStatus::Stalled
258                            )
259                    })
260                    .map(|run| {
261                        format!(
262                            "run:{id}:{:?}:{}:{}:{:?}",
263                            run.status,
264                            run.reviews.len(),
265                            task.attempts,
266                            run.passes.last().map(|p| (p.index, p.started_at))
267                        )
268                    })
269            } else {
270                None
271            };
272            version.filter(|v| !r.blocking_evidence.contains(v))
273        })
274        .collect()
275}
276
277/// Durable approval is enforced by verified evidence at the mutation boundary.
278/// A pinned run stays protected; legacy transient overrides keep their original
279/// bounded contradiction policy until an explicit new approval replaces it.
280fn may_hold(task: &mut Task, reason: &str) -> bool {
281    if pinned_resume(task) {
282        return false;
283    }
284    if task.resumption.is_some() {
285        return true; // The mutation boundary already verified new evidence.
286    }
287    let Some(o) = task.resume_override.as_mut() else {
288        return true;
289    };
290    if o.forced || o.pinned_run.is_some() {
291        tracing::warn!(
292            "conductor tried to hold task {} after the operator forced a resume: {reason}",
293            task.id
294        );
295        return false;
296    }
297    if o.conductor_rehold.is_some() {
298        return false;
299    }
300    o.conductor_rehold = Some(reason.to_owned());
301    true
302}
303
304/// Severity as a lowercase word, matching how `crate::verdict::Severity` is
305/// spelled everywhere else an operator or a model reads it.
306fn severity_str(s: crate::verdict::Severity) -> &'static str {
307    match s {
308        crate::verdict::Severity::Nit => "nit",
309        crate::verdict::Severity::Minor => "minor",
310        crate::verdict::Severity::Major => "major",
311        crate::verdict::Severity::Blocker => "blocker",
312    }
313}
314
315/// The branch a task's last run left behind, if the tally ever ran on it -
316/// what [`Recovery::Review`] reopens. Derived from `crate::run::RunState`
317/// alone, never from anything the conductor wrote, so a hallucinated branch
318/// name can never reach `crate::graph::Runner::review`.
319fn surviving_branch(task: &Task) -> Option<String> {
320    let last = task.runs.last()?;
321    let state = RunState::load(last).ok()?;
322    if !crate::daemon::compatible_run_instruction(&state, task) {
323        return None;
324    }
325    state.winner().map(|c| c.branch.clone())
326}
327
328/// The reason to record when the conductor holds a `failed`/`held` task via
329/// [`Recovery::Hold`]. Always `Some`, never a bare `d.reason.clone()`.
330///
331/// [`Task::hold_machine`] only overwrites [`Task::hold_reason`] when given
332/// one, precisely so a stalled task's existing diagnosis survives a
333/// decision that has nothing new to add. That is the right default when the
334/// task was not already `held` - but a task that *is* already `held`, and
335/// is being held again here, is a different case: its current
336/// `hold_reason` may still read as a cause `crate::triage` knows how to
337/// re-check on its own (a disk-pressure message, say - see
338/// `triage::machine_cause_resolved`), and if the model gives no reason,
339/// leaving that text untouched would make this decision - the conductor
340/// choosing, informed by the operator's own answer, to keep the task held
341/// anyway - indistinguishable from the original, never-reconsidered hold.
342/// `crate::triage` would then read the stale text the next time its one
343/// recognised cause looks resolved and release the task straight through
344/// the decision this call was recording. Prepending (not appending) the new
345/// note keeps the old text as context without leaving the string starting
346/// with whatever pattern `crate::triage` matched before.
347///
348/// Two bounds keep the string from growing, which it once did without limit
349/// (every re-hold nested the whole previous reason one level deeper, and every
350/// change relit the task's notice): a note equal to the current reason - or to
351/// its outermost note - adds nothing, so the result is `None` and the caller
352/// writes nothing; and only ONE `(previously: ...)` level is ever kept, the
353/// prior being cut back to its outermost note before it is embedded.
354fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> Option<String> {
355    let note = match &d.reason {
356        Some(reason) => reason.clone(),
357        None => match task.answers.last() {
358            Some(a) => format!(
359                "conduct held this again with no new reason given; last operator \
360                 answer on record: {}",
361                a.answer
362            ),
363            None => "conduct held this again with no reason given".to_owned(),
364        },
365    };
366    match task.hold_reason.as_deref() {
367        Some(prior) if !prior.is_empty() => {
368            let outer = outermost_hold_note(prior);
369            if note.trim() == prior.trim() || note.trim() == outer {
370                None
371            } else {
372                Some(format!("{note}\n\n(previously: {outer})"))
373            }
374        }
375        _ => Some(note),
376    }
377}
378
379/// Separator [`reaffirmed_hold_reason`] puts between a note and the reason it
380/// replaced.
381const PREVIOUSLY: &str = "\n\n(previously: ";
382
383/// A hold reason without any `(previously: ...)` tail: the text before the
384/// first separator, trimmed.
385fn outermost_hold_note(reason: &str) -> &str {
386    reason.split(PREVIOUSLY).next().unwrap_or(reason).trim()
387}
388
389/// The conductor's stated reason for a hold, or a stand-in when it gave none.
390fn hold_note(d: &Decision) -> String {
391    d.reason
392        .clone()
393        .unwrap_or_else(|| "(no reason given)".to_owned())
394}
395
396/// Everything the conductor is shown about a `failed`/`held` task's last run.
397async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
398    let Some(run_id) = task.runs.last().cloned() else {
399        return prompt::ConductOutcome {
400            run_id: "(none)".to_owned(),
401            unreadable: Some("this task has not produced a run yet".to_owned()),
402            run_status: None,
403            open_findings: Vec::new(),
404            rounds_used: 0,
405            rounds_max: 0,
406            rounds: Vec::new(),
407            branch: None,
408            branch_head: None,
409            references: None,
410            empty_candidate: false,
411        };
412    };
413    let state = match RunState::load(&run_id) {
414        Ok(s) => s,
415        Err(e) => {
416            // The exact failure this feature exists to stop hiding: a schema
417            // mismatch (or any other unreadable state) must never be treated
418            // as "nothing to recover" - it is surfaced here, verbatim, rather
419            // than swallowed into a quiet re-competition.
420            tracing::warn!(
421                "conductor: could not read run {run_id} for task {}: {e:#}",
422                task.short()
423            );
424            return prompt::ConductOutcome {
425                run_id,
426                unreadable: Some(format!("{e:#}")),
427                run_status: None,
428                open_findings: Vec::new(),
429                rounds_used: 0,
430                rounds_max: 0,
431                rounds: Vec::new(),
432                branch: None,
433                branch_head: None,
434                references: None,
435                empty_candidate: false,
436            };
437        }
438    };
439
440    let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
441        id: f.id.clone(),
442        title: f.title.clone(),
443        severity: severity_str(f.severity).to_owned(),
444    };
445    let open_findings = state
446        .open_findings()
447        .into_iter()
448        .map(finding_view)
449        .collect();
450    let rounds = state
451        .reviews
452        .iter()
453        .map(|r| prompt::ConductRound {
454            round: r.round,
455            findings: r
456                .reviews
457                .iter()
458                .flat_map(|rec| rec.findings.iter())
459                .map(finding_view)
460                .collect(),
461            addressed: r
462                .fix
463                .as_ref()
464                .map(|fx| fx.addressed.clone())
465                .unwrap_or_default(),
466            rejected: r
467                .fix
468                .as_ref()
469                .map(|fx| {
470                    fx.rejected
471                        .iter()
472                        .map(|rej| prompt::ConductRejection {
473                            id: rej.id.clone(),
474                            why: rej.why.clone(),
475                        })
476                        .collect()
477                })
478                .unwrap_or_default(),
479        })
480        .collect();
481    let branch = state.winner().map(|c| c.branch.clone());
482    let branch_head = match &branch {
483        Some(b) => crate::git::rev_parse(repo, b)
484            .await
485            .ok()
486            .map(|h| h.chars().take(8).collect()),
487        None => None,
488    };
489
490    prompt::ConductOutcome {
491        run_id,
492        unreadable: None,
493        run_status: Some(state.status.as_str().to_owned()),
494        open_findings,
495        rounds_used: state.reviews.len(),
496        rounds_max: state.config.graph.review_rounds,
497        rounds,
498        branch,
499        branch_head,
500        references: crate::refs::describe(&state.seeds),
501        empty_candidate: state.merge.as_ref().is_some_and(|m| m.empty),
502    }
503}
504
505/// Whether the task's latest run's winner branch already has an open pull
506/// request into the run's own base - a fact the repository can answer, so the
507/// operator is never asked it. `None` when there is no run or winner to check.
508async fn open_pr_fact(task: &Task, repo: &Path) -> Option<String> {
509    let id = task.runs.last()?;
510    let state = match RunState::load(id) {
511        Ok(s) => s,
512        Err(e) => return Some(format!("could not check open pull requests: {e:#}")),
513    };
514    let branch = state.winner()?.branch.clone();
515    let base = state.base_branch.clone();
516    match crate::land::find_open_pr(repo, &branch, &base).await {
517        Ok(crate::land::OpenPr::None) => None,
518        Ok(crate::land::OpenPr::One { url, .. }) => Some(format!(
519            "pull request {url} is already open for {branch} into {base}"
520        )),
521        Ok(crate::land::OpenPr::Many(urls)) => Some(format!(
522            "several pull requests are already open for {branch} into {base}: {}",
523            urls.join(" ")
524        )),
525        Err(e) => Some(format!("could not check open pull requests: {e:#}")),
526    }
527}
528
529/// Append what the repository says about the branches and commits a task
530/// names to every question the conductor is about to file, so the operator is
531/// never asked a fact git can answer ("is it already on main?"). Done here,
532/// in code, rather than trusted to the model: a question the model wrote
533/// without checking still reaches the operator with the answer attached.
534/// When it cannot be checked the question says so instead of guessing.
535async fn attach_facts(cfg: &Config, repo: &Path, queue: &Queue, verdict: &mut Verdict) {
536    for d in verdict
537        .decisions
538        .iter_mut()
539        .filter(|d| d.question.is_some())
540    {
541        let Ok(task) = queue.get(&d.id) else {
542            continue;
543        };
544        let text = format!("{}\n{}", task.title, task.instruction);
545        let pr_fact = open_pr_fact(&task, repo_for(&task, repo).as_path()).await;
546        if crate::refs::scan(&text).is_empty() {
547            if let (Some(fact), Some(q)) = (pr_fact, d.question.as_mut()) {
548                q.push_str(&format!(
549                    "\n\nChecked against the repository (magi did this, not the model):\n{fact}"
550                ));
551            }
552            continue;
553        }
554        let repo = repo_for(&task, repo);
555        let remote = &cfg.merge.remote;
556        let base = crate::git::merge_base_branch(&repo, remote, cfg.merge.base.as_deref())
557            .await
558            .ok();
559        let facts = match base {
560            Some(base) => {
561                let base_name = base.clone();
562                let tracking = format!("{remote}/{base}");
563                // Best effort: a stale tracking ref would report work that
564                // has since landed as still missing from the base.
565                let refreshed = crate::git::fetch(&repo, remote, &base)
566                    .await
567                    .is_ok_and(|o| o.ok());
568                // Only the remote's tip counts: a local branch of the same
569                // name is whatever the checkout happens to hold.
570                let against = tracking;
571                match crate::git::rev_parse(&repo, &against).await {
572                    Ok(tip) => crate::refs::describe(
573                        &crate::refs::resolve(&repo, &tip, remote, &text).await,
574                    )
575                    .map(|facts| {
576                        if refreshed {
577                            facts
578                        } else {
579                            format!(
580                                "{facts}\n(could not fetch {remote}/{base_name}: this is \
581                                 against the last fetched `{against}`, which may be behind \
582                                 the remote)"
583                            )
584                        }
585                    }),
586                    Err(_) => Some(format!(
587                        "could NOT be checked against the repository: {against} is not \
588                         available here, so whether the work is in the base is unknown"
589                    )),
590                }
591            }
592            None => Some("could not check the repository: no base branch known".to_owned()),
593        };
594        let facts = match (facts, pr_fact) {
595            (Some(f), Some(p)) => Some(format!("{f}\n{p}")),
596            (f, p) => f.or(p),
597        };
598        if let (Some(facts), Some(q)) = (facts, d.question.as_mut()) {
599            q.push_str(&format!(
600                "\n\nChecked against the repository (magi did this, not the model):\n{facts}"
601            ));
602        }
603    }
604}
605
606/// A `failed`/`held` task together with how its last run ended.
607async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
608    prompt::ConductFinished {
609        task: view(t, max_attempts),
610        outcome: outcome_for(t, &repo_for(t, repo)).await,
611    }
612}
613
614/// The repository containing a task's branch. A task filed without a
615/// repository uses the daemon's repository, exactly as its later attempt does.
616fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
617    if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
618        fallback.to_path_buf()
619    } else {
620        task.repo.clone()
621    }
622}
623
624/// Apply one decision to the queue and the question store.
625///
626/// Takes the task's own claim before touching it: a model call spans a whole
627/// agent turn, and the queue can have moved on by the time its answer comes
628/// back. A claim that cannot be taken means something else owns this task
629/// right now - most often a live daemon mid-competition on it - so the
630/// conductor's now-stale view of it is dropped rather than raced against; see
631/// `crate::queue::Queue::claim`'s own doc on why a claim is proof, not a
632/// guess.
633///
634/// A decision is matched against the task's *current* status, re-read under
635/// the claim, not against whichever section of the prompt it came from: a
636/// `blocked_by` only takes effect on a `queued` task, and `recovery` only on
637/// one `running` (stalled) or `failed`/`held`, so a decision that no longer
638/// matches what the task actually is - it moved on between the read that
639/// built the prompt and this write - changes nothing.
640fn apply_one(
641    queue: &Queue,
642    questions: &Questions,
643    d: &Decision,
644    snapshot: Option<&Task>,
645) -> Result<()> {
646    let _claim = queue
647        .claim(&d.id)
648        .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
649    let mut task = queue.get(&d.id).context("no such task")?;
650
651    // Compare the actual input, not a generation echoed by the model. Even
652    // question creation must be discarded after an intervening operator edit.
653    if snapshot
654        .is_some_and(|old| serde_json::to_value(old).ok() != serde_json::to_value(&task).ok())
655    {
656        return Ok(());
657    }
658    let blocking =
659        d.question.is_some() || !d.blocked_by.is_empty() || d.recovery == Some(Recovery::Hold);
660    let evidence = new_blocking_evidence(queue, questions, &task, d);
661    if blocking && task.resumption.is_some() && evidence.is_empty() {
662        crate::notices::raise(
663            crate::notices::Notice::warn(
664                &format!("conductor-resumption:{}", task.id),
665                format!(
666                    "Conductor concern after explicit resumption (no verified new evidence): {}; question: {:?}; dependencies: {:?}; evidence: {:?}",
667                    hold_note(d), d.question, d.blocked_by, d.evidence
668                ),
669            )
670            .about([task.id.clone()]),
671        );
672        return Ok(());
673    }
674
675    if blocking {
676        if let Some(r) = task.resumption.as_mut() {
677            r.blocking_evidence.extend(evidence);
678        }
679    }
680
681    // A conductor answer is never operator authorization.  In particular,
682    // do this before questions and blocking too: either would reclassify a
683    // manual hold and let a later deterministic resolver queue it.
684    if task.operator_held() {
685        return Ok(());
686    }
687
688    // A claimed task may have deferred the pre-scheduling triage sweep.
689    // Leave an open or answered-unapplied question to that sweep rather
690    // than move the task out from under it.
691    if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
692        return Ok(());
693    }
694
695    // The operator's `resume` action is waiting for its turn: a question or a
696    // dependency would park the task before the daemon can run it.
697    let pinned = pinned_resume(&task);
698    if pinned && (d.question.is_some() || !d.blocked_by.is_empty()) {
699        return Ok(());
700    }
701
702    if let Some(text) = &d.question {
703        if task.status == TaskStatus::Done {
704            return Ok(());
705        }
706        // `Question::run` is a task id for conductor questions, but ordinary
707        // graph questions use it as a run id. A coincidental equality must
708        // not block this task on an answer meant for another node.
709        let question_id = match questions
710            .list()
711            .into_iter()
712            .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
713        {
714            Some(existing) => existing.id,
715            // No cap on settled answers: a question the operator can answer
716            // is never refused. The task goes `Blocked` and at most one
717            // conductor question is open at a time (reused above), so each
718            // further question costs one human answer. Run attempts are
719            // bounded by `max_attempts` and the queue's attempt accounting.
720            // A model rewording the same question can still recur once per
721            // answer; that cost falls on the operator, by choice.
722            None => {
723                let mut q = Question::new(
724                    task.id.clone(),
725                    NODE.to_owned(),
726                    SEAT.to_owned(),
727                    text.clone(),
728                    d.reason.clone().unwrap_or_default(),
729                    d.choices.clone(),
730                );
731                // The wait is handed to a deputy (`crate::deputy`), so a
732                // free-text reply reaches an agent that knows why this was
733                // asked. The conductor still never blocks: this only records
734                // what the deputy will be told, and `serve` starts it.
735                q.deputy = Some(Deputy::new(crate::deputy::brief(
736                    &task.id,
737                    d.reason.as_deref().unwrap_or_default(),
738                    &q.choices,
739                    &q.actions,
740                )));
741                if task.repo.is_dir() && task.repo != Path::new(".") {
742                    q.cwd = Some(task.repo.to_string_lossy().into_owned());
743                }
744                questions.put(&mut q)?;
745                q.id
746            }
747        };
748        task.block(vec![question_id], d.reason.clone());
749        return queue.put(&mut task);
750    }
751
752    match task.status {
753        TaskStatus::Queued if !d.blocked_by.is_empty() => {
754            task.block(d.blocked_by.clone(), d.reason.clone());
755            queue.put(&mut task)?;
756        }
757        // See `Decision::recovery`'s doc: the only lever a runnable task has
758        // besides `blocked_by` is holding it outright, for a task whose
759        // answers already say it should not compete again.
760        TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
761            if may_hold(&mut task, &hold_note(d)) {
762                task.hold_machine(d.reason.clone());
763                queue.put(&mut task)?;
764            }
765        }
766        TaskStatus::Running => match d.recovery {
767            Some(Recovery::Requeue) if pinned_resume(&task) => {}
768            Some(Recovery::Requeue) => {
769                task.requeue();
770                queue.put(&mut task)?;
771            }
772            Some(Recovery::Hold) if may_hold(&mut task, &hold_note(d)) => {
773                task.hold_machine(d.reason.clone());
774                queue.put(&mut task)?;
775            }
776            // `Review` reopens a branch, which only makes sense once a run
777            // has actually stopped; a task still `running` has nothing to
778            // reopen yet.
779            _ => {}
780        },
781        TaskStatus::Failed | TaskStatus::Held => match d.recovery {
782            // The operator picked a specific run to continue; a fresh
783            // competition would throw that run away.
784            Some(Recovery::Requeue) if pinned_resume(&task) => {}
785            Some(Recovery::Requeue) => {
786                task.requeue();
787                queue.put(&mut task)?;
788            }
789            Some(Recovery::Hold) => {
790                // Decided before `may_hold`, which spends the one override: a
791                // note that adds nothing writes nothing (and so raises no
792                // notice and moves no queue revision).
793                if let Some(reason) = reaffirmed_hold_reason(&task, d) {
794                    if may_hold(&mut task, &hold_note(d)) {
795                        task.hold_machine(Some(reason));
796                        queue.put(&mut task)?;
797                    }
798                }
799            }
800            Some(Recovery::Review) => {
801                if let Some(branch) = surviving_branch(&task) {
802                    task.request_review(branch);
803                    queue.put(&mut task)?;
804                }
805                // No survivable branch: a decision naming `review` here is
806                // simply not actionable, and is dropped rather than guessed
807                // at - `crate::daemon` applies the same "no branch, no
808                // review" rule again, from its own read, right before it
809                // would actually start the run.
810            }
811            Some(Recovery::Done) => {
812                task.succeed();
813                // Same reason `magi task done` and the web UI's equivalent
814                // do this: the conductor closing a task by hand is just as
815                // much a finished story as the loop's own settle path.
816                crate::daemon::supersede_prior_runs(&task, &crate::run::home());
817                queue.put(&mut task)?;
818            }
819            None => {}
820        },
821        // `queued` with nothing to block on, `done`, or already `blocked`:
822        // nothing for this decision to do.
823        _ => {}
824    }
825    Ok(())
826}
827
828/// Apply every decision in `verdict`. A single bad decision - a task id that
829/// no longer exists, one already claimed elsewhere - is logged and skipped
830/// rather than losing every other decision in the same reply.
831pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
832    for d in &verdict.decisions {
833        if let Err(e) = apply_one(queue, questions, d, None) {
834            tracing::warn!("conductor decision for task {}: {e:#}", d.id);
835        }
836    }
837    Ok(())
838}
839
840/// Where the conductor's seat is kept between turns, so the daemon waiter can
841/// resume it when the owner answers a question it asked.
842///
843/// A graph seat's session lives in its run's `RunState`; the conductor has no
844/// run, and until this file its seat existed only in memory.
845pub fn seat_path(home: &Path) -> PathBuf {
846    home.join("conduct").join("seat.json")
847}
848
849/// The seat the conductor last used, if it has taken a turn.
850pub fn load_seat(home: &Path) -> Option<SeatState> {
851    serde_json::from_str(&std::fs::read_to_string(seat_path(home)).ok()?).ok()
852}
853
854fn busy_path(home: &Path) -> PathBuf {
855    home.join("conduct").join("busy")
856}
857
858/// Is a conductor turn in flight right now?
859///
860/// A marker file rather than shared memory because the waiter reads it and the
861/// conductor is not necessarily in its process. A marker older than one turn's
862/// timeout (plus slack) was left by a turn that died and counts for nothing.
863pub fn busy(home: &Path) -> bool {
864    std::fs::metadata(busy_path(home))
865        .and_then(|m| m.modified())
866        .ok()
867        .and_then(|t| t.elapsed().ok())
868        .is_some_and(|age| age < TURN_TIMEOUT + Duration::from_secs(30))
869}
870
871/// Removes the busy marker when a turn ends, however it ends.
872struct Busy(PathBuf);
873
874impl Busy {
875    fn mark(home: &Path) -> Self {
876        let path = busy_path(home);
877        if let Some(dir) = path.parent() {
878            let _ = std::fs::create_dir_all(dir);
879        }
880        let _ = std::fs::write(&path, std::process::id().to_string());
881        Self(path)
882    }
883}
884
885impl Drop for Busy {
886    fn drop(&mut self) {
887        let _ = std::fs::remove_file(&self.0);
888    }
889}
890
891/// The conductor's state across polling cycles: its own CLI-side conversation
892/// and the last (revision, stalled ∪ finished ids) pair it actually acted on.
893#[derive(Debug, Default)]
894pub struct Conductor {
895    seat: Option<SeatState>,
896    last_seen: Option<(u64, BTreeSet<String>)>,
897    /// Content fingerprint of each held task the model was last shown, see
898    /// [`input_fingerprint`]. A held task whose fingerprint still matches is
899    /// settled: the conductor's own rewrites of it do not count as news.
900    considered: BTreeMap<String, String>,
901}
902
903/// What the conductor's decision about a held task is a function of: the task
904/// without the fields the conductor itself rewrites (`updated_at`,
905/// `hold_reason`, the recorded re-hold). Independent of how the model words a
906/// repeated decision.
907fn input_fingerprint(task: &Task) -> String {
908    let mut v = serde_json::to_value(task).unwrap_or_default();
909    if let Some(o) = v.as_object_mut() {
910        o.remove("updated_at");
911        o.remove("hold_reason");
912        if let Some(ro) = o.get_mut("resume_override").and_then(|r| r.as_object_mut()) {
913            ro.remove("conductor_rehold");
914        }
915    }
916    v.to_string()
917}
918
919impl Conductor {
920    /// A conductor that has never run.
921    #[must_use]
922    pub fn new() -> Self {
923        Self::default()
924    }
925
926    /// Held tasks among `stalled` / `finished` whose input is unchanged since
927    /// the model last saw them. Their files are left out of the revision, so
928    /// the conductor re-holding one (however it words it) is not news, while
929    /// an owner's answer changes the fingerprint and brings the file back in.
930    fn settled(
931        considered: &BTreeMap<String, String>,
932        stalled: &[Task],
933        finished: &[Task],
934    ) -> BTreeSet<String> {
935        stalled
936            .iter()
937            .chain(finished)
938            .filter(|t| t.status == TaskStatus::Held)
939            .filter(|t| considered.get(&t.id) == Some(&input_fingerprint(t)))
940            .map(|t| t.id.clone())
941            .collect()
942    }
943
944    fn snapshot_with(
945        considered: &BTreeMap<String, String>,
946        queue: &Queue,
947        stalled: &[Task],
948        finished: &[Task],
949    ) -> (u64, BTreeSet<String>) {
950        let ids = stalled
951            .iter()
952            .chain(finished)
953            .map(|t| t.id.clone())
954            .collect();
955        let skip = Self::settled(considered, stalled, finished);
956        (queue.revision_excluding(&skip), ids)
957    }
958
959    fn snapshot(
960        &self,
961        queue: &Queue,
962        stalled: &[Task],
963        finished: &[Task],
964    ) -> (u64, BTreeSet<String>) {
965        Self::snapshot_with(&self.considered, queue, stalled, finished)
966    }
967
968    /// Whether calling the conductor could possibly do anything different
969    /// from last time: [`Queue::revision`] moved, or the set of stalled and
970    /// finished task ids changed.
971    ///
972    /// Deliberately **not** "stalled or finished is non-empty" - a task
973    /// sitting stalled or finished with nobody changing anything about it
974    /// must not be re-shown to the model every single poll forever; only a
975    /// change in *which* tasks are stalled or finished, or a queue write
976    /// changing something about a runnable one, is worth another look.
977    ///
978    /// Cheap and config-free on purpose, so `crate::daemon`'s poll loop can
979    /// skip `Config::discover`'s synchronous I/O entirely on a cycle where
980    /// this says no - which [`Conductor::maybe_run`] would otherwise only
981    /// discover after paying for that load. Both ask the identical question,
982    /// from the same [`Conductor::last_seen`], so they can never disagree
983    /// about whether there is anything to look at.
984    #[must_use]
985    pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
986        self.last_seen.as_ref() != Some(&self.snapshot(queue, stalled, finished))
987    }
988
989    /// Call the conductor once, unless nothing has changed since the last
990    /// time it was worth calling - see [`Conductor::worth_a_look`], the exact
991    /// same test. Never fatal - see this module's doc.
992    #[allow(clippy::too_many_arguments)]
993    pub async fn maybe_run(
994        &mut self,
995        cfg: &Config,
996        repo: &Path,
997        queue: &Queue,
998        questions: &Questions,
999        home: &Path,
1000        queued: &[Task],
1001        stalled: &[Task],
1002        finished: &[Task],
1003        max_attempts: usize,
1004    ) {
1005        if self.last_seen.as_ref() == Some(&self.snapshot(queue, stalled, finished)) {
1006            return;
1007        }
1008        // What the model is about to be shown becomes "considered" before the
1009        // snapshot is taken, so its own re-hold writes land in the excluded set.
1010        self.considered = stalled
1011            .iter()
1012            .chain(finished)
1013            .filter(|t| t.status == TaskStatus::Held)
1014            .map(|t| (t.id.clone(), input_fingerprint(t)))
1015            .collect();
1016        self.last_seen = Some(self.snapshot(queue, stalled, finished));
1017        if let Err(e) = self
1018            .run_once(
1019                cfg,
1020                repo,
1021                queue,
1022                questions,
1023                home,
1024                queued,
1025                stalled,
1026                finished,
1027                max_attempts,
1028            )
1029            .await
1030        {
1031            // The conductor's own write moves the revision, so one more cycle
1032            // follows; a repeat decision writes nothing (see
1033            // `reaffirmed_hold_reason`), which is what ends the loop. The
1034            // revision is deliberately never re-read here: it cannot tell our
1035            // write from an owner's concurrent update.
1036            tracing::warn!("conductor: {e:#}");
1037        }
1038    }
1039
1040    #[allow(clippy::too_many_arguments)]
1041    async fn run_once(
1042        &mut self,
1043        cfg: &Config,
1044        repo: &Path,
1045        queue: &Queue,
1046        questions: &Questions,
1047        home: &Path,
1048        queued: &[Task],
1049        stalled: &[Task],
1050        finished: &[Task],
1051        max_attempts: usize,
1052    ) -> Result<()> {
1053        if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
1054            return Ok(());
1055        }
1056
1057        let primary = cfg
1058            .resolve_roles()
1059            .context("resolving the conductor seat")?
1060            .conductor;
1061        // A string keeps today's resolution exactly (no CLI preflight here;
1062        // an unavailable CLI fails at invocation). Only a multi-id array
1063        // builds a chain, whose members are each tried at most once.
1064        let chain = match cfg.roles.conductor.as_ref() {
1065            Some(c) if c.ids().len() > 1 => {
1066                agent::pick_chain(&cfg.agents, Some(c), &agent::installed, "conductor")?
1067            }
1068            _ => vec![primary],
1069        };
1070
1071        let runnable_views: Vec<prompt::ConductTask> =
1072            queued.iter().map(|t| view(t, max_attempts)).collect();
1073        let stalled_views: Vec<prompt::ConductTask> =
1074            stalled.iter().map(|t| view(t, max_attempts)).collect();
1075        let mut finished_views = Vec::with_capacity(finished.len());
1076        for t in finished {
1077            finished_views.push(finished_view(t, repo, max_attempts).await);
1078        }
1079
1080        let body = prompt::with_overlay(
1081            prompt::conduct(
1082                &runnable_views,
1083                &stalled_views,
1084                &finished_views,
1085                &cfg.graph.language,
1086            ),
1087            cfg.prompts.overlay(NODE),
1088        );
1089
1090        let artifacts = home.join("conduct").join("artifacts");
1091        // Bound to a local: `Invocation` only borrows the cache path, and the
1092        // `Option<PathBuf>` `cache_dir()` returns has to outlive that borrow.
1093        let cache_dir = cfg.cache_dir();
1094
1095        // One marker for the whole chain: a fallback is still this turn.
1096        let busy = Busy::mark(home);
1097        let mut result: Result<Verdict> = Err(anyhow::anyhow!("no conductor agent ran"));
1098        for spec in &chain {
1099            // The prompt carries the whole queue state every cycle, so a
1100            // fallback agent needs no history - but its seat is its own.
1101            let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
1102            if needs_new_seat {
1103                self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
1104            }
1105            let seat = self.seat.as_mut().expect("just ensured a seat exists");
1106            // The first agent keeps the plain stem; a fallback's own artifacts
1107            // must not overwrite it.
1108            let stem = if std::ptr::eq(spec, &chain[0]) {
1109                format!("turn-{}", seat.turns + 1)
1110            } else {
1111                format!("turn-{}-{}", seat.turns + 1, spec.id)
1112            };
1113            let inv = Invocation {
1114                cwd: repo,
1115                prompt: &body,
1116                timeout: TURN_TIMEOUT,
1117                // The conductor never edits anything - it only decides what
1118                // blocks a task and what to do about one stuck or finished.
1119                allow_write: false,
1120                unsandboxed: false,
1121                sessions: cfg.graph.sessions,
1122                artifacts: &artifacts,
1123                stem: &stem,
1124                run: NODE,
1125                node: NODE,
1126                cache_dir: cache_dir.as_deref(),
1127                attachments: &[],
1128                writable: &[],
1129            };
1130            let out = agent::invoke(spec, seat, &inv).await;
1131            // Kept after every turn, failed or not: the CLI-side conversation is
1132            // what a resumed question needs, and it exists once the turn ran.
1133            if let Ok(body) = serde_json::to_string(&*seat) {
1134                let path = seat_path(home);
1135                if let Some(dir) = path.parent() {
1136                    let _ = std::fs::create_dir_all(dir);
1137                }
1138                let _ = std::fs::write(path, body);
1139            }
1140            let advance = agent::chain_advances(&out);
1141            result = match out {
1142                Err(e) => Err(e.context("invoking the conductor")),
1143                Ok(out) if advance => Err(anyhow::anyhow!(
1144                    "no usable reply (exit {:?}, timed out {})",
1145                    out.exit_code,
1146                    out.timed_out
1147                )),
1148                Ok(out) => verdict::extract_json(&out.text)
1149                    .context("the conductor's reply could not be parsed"),
1150            };
1151            if result.is_ok() {
1152                break;
1153            }
1154            if chain.len() > 1 {
1155                tracing::warn!("conductor: `{}` failed, trying the next agent", spec.id);
1156            }
1157        }
1158        drop(busy);
1159        let mut verdict = result?;
1160        attach_facts(cfg, repo, queue, &mut verdict).await;
1161        for d in &verdict.decisions {
1162            let snapshot = queued
1163                .iter()
1164                .chain(stalled)
1165                .chain(finished)
1166                .find(|t| t.id == d.id);
1167            if let Some(snapshot) = snapshot {
1168                if let Err(e) = apply_one(queue, questions, d, Some(snapshot)) {
1169                    tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1170                }
1171            }
1172        }
1173        Ok(())
1174    }
1175}
1176
1177#[cfg(test)]
1178mod tests {
1179    use std::collections::BTreeMap;
1180
1181    use tempfile::tempdir;
1182
1183    use super::*;
1184    use crate::ask::{Answer, QuestionStatus};
1185    use crate::config::{AgentKind, AgentSpec, Graph};
1186    use crate::queue::Source;
1187
1188    fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
1189        let path = dir.join("mock-conduct-agent.sh");
1190        std::fs::write(&path, script).expect("write mock");
1191        AgentSpec {
1192            id: "mock".to_owned(),
1193            kind: AgentKind::Command,
1194            model: None,
1195            command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
1196            extra_args: Vec::new(),
1197            env,
1198            prompt_delivery: None,
1199        }
1200    }
1201
1202    fn config(spec: AgentSpec) -> Config {
1203        Config {
1204            agents: vec![spec],
1205            graph: Graph {
1206                language: "en".to_owned(),
1207                ..Graph::default()
1208            },
1209            ..Config::default()
1210        }
1211    }
1212
1213    fn task(title: &str) -> Task {
1214        Task::new(
1215            title.to_owned(),
1216            format!("do {title}"),
1217            std::path::PathBuf::from("."),
1218            Source::Human,
1219        )
1220    }
1221
1222    const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
1223    const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
1224
1225    fn env(reply: &str) -> BTreeMap<String, String> {
1226        BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
1227    }
1228
1229    const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
1230
1231    /// A throwaway repo with one commit on `main` and a second branch ahead
1232    /// of it, so `outcome_for`'s own `git::rev_parse` call has a real head to
1233    /// resolve.
1234    fn init_repo_with_branch(dir: &Path, branch: &str) {
1235        use crate::proc::Quiet as _;
1236        let run = |args: &[&str]| {
1237            let out = std::process::Command::new("git")
1238                .args(args)
1239                .current_dir(dir)
1240                .quiet()
1241                .output()
1242                .expect("spawn git");
1243            assert!(
1244                out.status.success(),
1245                "git {args:?} failed: {}",
1246                String::from_utf8_lossy(&out.stderr)
1247            );
1248        };
1249        run(&["init", "-b", "main"]);
1250        run(&["config", "user.name", "magi test"]);
1251        run(&["config", "user.email", "magi@example.com"]);
1252        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
1253        run(&["add", "-A"]);
1254        run(&["commit", "-m", "init"]);
1255        run(&["checkout", "-b", branch]);
1256        std::fs::write(dir.join("change.txt"), "x\n").unwrap();
1257        run(&["add", "-A"]);
1258        run(&["commit", "-m", "candidate work"]);
1259    }
1260
1261    fn review_round_with_finding(
1262        round: usize,
1263        finding_id: &str,
1264        title: &str,
1265        addressed: &[&str],
1266        rejected: &[(&str, &str)],
1267    ) -> crate::run::ReviewRound {
1268        crate::run::ReviewRound {
1269            round,
1270            head: "deadbeef".to_owned(),
1271            verified_head: None,
1272            verified_at: None,
1273            reviews: vec![crate::run::ReviewRecord {
1274                attempts: 0,
1275                reviewer: 1,
1276                agent: "mock".to_owned(),
1277                summary: String::new(),
1278                findings: vec![crate::verdict::Finding {
1279                    id: finding_id.to_owned(),
1280                    severity: crate::verdict::Severity::Major,
1281                    file: None,
1282                    line: None,
1283                    title: title.to_owned(),
1284                    detail: String::new(),
1285                }],
1286                vote: None,
1287                failed: None,
1288                duration_ms: 0,
1289            }],
1290            e2e: Vec::new(),
1291            verify_retried: false,
1292            e2e_deferred: false,
1293            e2e_defer_reason: None,
1294            fix: Some(crate::run::FixRecord {
1295                agent: "mock".to_owned(),
1296                addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
1297                rejected: rejected
1298                    .iter()
1299                    .map(|(id, why)| crate::verdict::Rejection {
1300                        id: (*id).to_owned(),
1301                        why: (*why).to_owned(),
1302                    })
1303                    .collect(),
1304                notes: String::new(),
1305                committed: false,
1306                failed: None,
1307                duration_ms: 0,
1308                continuation: None,
1309            }),
1310            blocking: 1,
1311            answered: 1,
1312            expected: 1,
1313            clean: false,
1314            progressed: true,
1315            vote_split: false,
1316            reconsideration: Vec::new(),
1317            verdict: None,
1318        }
1319    }
1320
1321    #[test]
1322    fn explicit_resumption_supersedes_history_and_rejects_stale_decisions() {
1323        crate::run::pin_test_home();
1324        let dir = tempdir().unwrap();
1325        let queue = Queue::at(dir.path().join("queue"));
1326        let questions = Questions::at(dir.path().join("questions"));
1327        let mut t = task("recover");
1328        t.answers.push(crate::queue::AnsweredQuestion {
1329            question: "Inspect remote first?".into(),
1330            answer: "Inspect, then await resumption instructions".into(),
1331        });
1332        t.hold_machine(Some("waiting for remote inspection".into()));
1333        queue.put(&mut t).unwrap();
1334        let snapshot = queue.get(&t.id).unwrap();
1335        t.edit(
1336            "recover".into(),
1337            "Reuse the measured PR head; fix Clippy; review".into(),
1338        )
1339        .unwrap();
1340        t.release_operator("cli", None);
1341        queue.put(&mut t).unwrap();
1342        let approval = t.resumption.clone();
1343        let d = Decision {
1344            id: t.id.clone(),
1345            recovery: Some(Recovery::Hold),
1346            reason: Some("remote inspection still required".into()),
1347            ..Decision::default()
1348        };
1349        apply_one(&queue, &questions, &d, Some(&snapshot)).unwrap();
1350        let stale_question = Decision {
1351            question: Some("Inspect remote?".into()),
1352            ..d.clone()
1353        };
1354        apply_one(&queue, &questions, &stale_question, Some(&snapshot)).unwrap();
1355        assert!(questions.list().is_empty());
1356        for reason in [
1357            "waiting for remote inspection",
1358            "inspection has not been confirmed",
1359        ] {
1360            let repeated = Decision {
1361                reason: Some(reason.into()),
1362                ..d.clone()
1363            };
1364            apply(
1365                &queue,
1366                &questions,
1367                &Verdict {
1368                    decisions: vec![repeated],
1369                },
1370            )
1371            .unwrap();
1372            assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1373        }
1374        let reloaded = Queue::at(dir.path().join("queue")).get(&t.id).unwrap();
1375        assert_eq!(reloaded.resumption, approval);
1376        assert_eq!(reloaded.answers, t.answers);
1377        let v = view(&reloaded, 3);
1378        assert!(v.operator_resume.unwrap().contains("superseded"));
1379        let mut dep = task("new prerequisite");
1380        dep.hold_machine(Some("new authentication failure".into()));
1381        queue.put(&mut dep).unwrap();
1382        let new_blocker = Decision {
1383            blocked_by: vec![dep.id.clone()],
1384            evidence: vec![dep.id.clone()],
1385            reason: Some("new prerequisite failed".into()),
1386            ..d.clone()
1387        };
1388        apply(
1389            &queue,
1390            &questions,
1391            &Verdict {
1392                decisions: vec![new_blocker.clone()],
1393            },
1394        )
1395        .unwrap();
1396        let mut after = queue.get(&t.id).unwrap();
1397        assert_eq!(after.status, TaskStatus::Blocked);
1398        // Automatic release does not grant fresh operator authority or make
1399        // the same unchanged evidence reusable.
1400        after.release();
1401        queue.put(&mut after).unwrap();
1402        apply(
1403            &queue,
1404            &questions,
1405            &Verdict {
1406                decisions: vec![new_blocker],
1407            },
1408        )
1409        .unwrap();
1410        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1411        after.start("new-run".into());
1412        assert_eq!(after.resumption.as_ref().unwrap().generation, 1);
1413        after.hold_manual(Some("operator says wait".into()));
1414        queue.put(&mut after).unwrap();
1415        apply(&queue, &questions, &Verdict { decisions: vec![d] }).unwrap();
1416        assert!(queue.get(&t.id).unwrap().operator_held());
1417    }
1418
1419    #[test]
1420    fn resumed_run_evidence_uses_approval_time_not_run_list_offsets() {
1421        crate::run::pin_test_home();
1422        let dir = tempdir().unwrap();
1423        let queue = Queue::at(dir.path().join("queue"));
1424        let questions = Questions::at(dir.path().join("questions"));
1425        let mut t = task("resume evidence");
1426        let mut run = RunState::new(
1427            t.repo.clone(),
1428            "main".into(),
1429            "base".into(),
1430            "work".into(),
1431            crate::config::Config::default(),
1432        );
1433        run.begin_pass();
1434        run.status = crate::run::RunStatus::Failed;
1435        run.save().unwrap();
1436        t.start(run.id.clone());
1437        // Model legacy duplicate history, which the next start will deduplicate.
1438        t.runs.push(run.id.clone());
1439        t.release_operator("cli", None);
1440        let approval = t.resumption.as_ref().unwrap().at;
1441        let decision = Decision {
1442            evidence: vec![run.id.clone()],
1443            ..Decision::default()
1444        };
1445        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1446        t.start(run.id.clone());
1447        assert_eq!(t.runs.len(), 1);
1448        run.begin_pass();
1449        run.save().unwrap();
1450        assert!(run.passes.last().unwrap().started_at > approval);
1451        let evidence = new_blocking_evidence(&queue, &questions, &t, &decision);
1452        assert_eq!(evidence.len(), 1);
1453        t.resumption.as_mut().unwrap().blocking_evidence = evidence;
1454        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1455        run.begin_pass();
1456        run.save().unwrap();
1457        assert_eq!(
1458            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1459            1
1460        );
1461
1462        let mut q = Question::new(
1463            run.id.clone(),
1464            "review".into(),
1465            "check".into(),
1466            "New failure".into(),
1467            String::new(),
1468            Vec::new(),
1469        );
1470        questions.put(&mut q).unwrap();
1471        let decision = Decision {
1472            evidence: vec![q.id.clone()],
1473            ..Decision::default()
1474        };
1475        assert_eq!(
1476            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1477            1
1478        );
1479        q.asked_at = approval;
1480        questions.put(&mut q).unwrap();
1481        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1482        q.asked_at = jiff::Timestamp::now();
1483        q.run = "unrelated".into();
1484        questions.put(&mut q).unwrap();
1485        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1486
1487        let mut fresh = RunState::new(
1488            t.repo.clone(),
1489            "main".into(),
1490            "base".into(),
1491            "fresh work".into(),
1492            crate::config::Config::default(),
1493        );
1494        fresh.status = crate::run::RunStatus::Failed;
1495        fresh.save().unwrap();
1496        t.start(fresh.id.clone());
1497        let decision = Decision {
1498            evidence: vec![fresh.id.clone()],
1499            ..Decision::default()
1500        };
1501        assert_eq!(
1502            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1503            1
1504        );
1505    }
1506
1507    #[test]
1508    fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
1509        crate::run::pin_test_home();
1510        let dir = tempdir().unwrap();
1511        let default_repo = dir.path().join("default");
1512        let task_repo = dir.path().join("task");
1513        std::fs::create_dir_all(&default_repo).unwrap();
1514        std::fs::create_dir_all(&task_repo).unwrap();
1515        init_repo_with_branch(&default_repo, "other-branch");
1516        init_repo_with_branch(&task_repo, "magi/f00d/A");
1517
1518        let mut config = Config::default();
1519        config.graph.review_rounds = 6;
1520        let mut state = crate::run::RunState::new(
1521            task_repo.clone(),
1522            "main".to_owned(),
1523            "deadbeef".to_owned(),
1524            "task".to_owned(),
1525            config,
1526        );
1527        state.status = crate::run::RunStatus::Blocked;
1528        state.candidates.push(crate::run::Candidate {
1529            index: 0,
1530            label: 'A',
1531            agent: "mock".to_owned(),
1532            branch: "magi/f00d/A".to_owned(),
1533            worktree: task_repo.clone(),
1534            summary: String::new(),
1535            stat: String::new(),
1536            files: 1,
1537            commits: 1,
1538            empty: false,
1539            failed: None,
1540            verified_noop: None,
1541            duration_ms: 0,
1542            folded: false,
1543        });
1544        state.tally = Some(crate::run::Tally {
1545            first_choice: std::collections::BTreeMap::new(),
1546            borda: std::collections::BTreeMap::new(),
1547            winner: 'A',
1548            rankings: 0,
1549            unanimous_initial: false,
1550            deliberated: false,
1551            changed_votes: 0,
1552            unanimous_final: false,
1553            tie_break: None,
1554            judges: 0,
1555            present: 0,
1556            quorum: 0,
1557            met_quorum: true,
1558            uncontested: Some("solo".to_owned()),
1559        });
1560        state.reviews = vec![
1561            review_round_with_finding(
1562                1,
1563                "R1-1-2",
1564                "answer content is dropped",
1565                &[],
1566                &[("R1-1-2", "the id leaving blocked_by is enough")],
1567            ),
1568            review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
1569        ];
1570        state.save().unwrap();
1571
1572        let mut t = task("outcome test");
1573        t.repo = task_repo;
1574        t.runs.push(state.id.clone());
1575
1576        let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
1577        let outcome = finished.outcome;
1578
1579        assert!(outcome.unreadable.is_none());
1580        assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
1581        assert_eq!(outcome.rounds_used, 2);
1582        assert_eq!(outcome.rounds_max, 6);
1583        assert_eq!(outcome.rounds.len(), 2);
1584        assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
1585        assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
1586        assert!(outcome.rounds[1].addressed.is_empty());
1587        assert!(outcome.rounds[1].rejected.is_empty());
1588        assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
1589        assert!(
1590            outcome.branch_head.is_some(),
1591            "a real branch must resolve a head commit: {outcome:?}"
1592        );
1593    }
1594
1595    /// A tiny single-threaded block-on, so an `async fn` can be exercised
1596    /// from a plain `#[test]` without pulling `tokio::test`'s multi-thread
1597    /// runtime into a test that does no other async work.
1598    fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
1599        tokio::runtime::Builder::new_current_thread()
1600            .enable_all()
1601            .build()
1602            .unwrap()
1603            .block_on(f)
1604    }
1605
1606    #[test]
1607    fn view_resolves_task_attempt_limit_before_daemon_default() {
1608        let mut t = task("budget");
1609        assert_eq!(view(&t, 7).max_attempts, 7);
1610        t.change_attempt_limit(std::num::NonZeroUsize::new(3))
1611            .unwrap();
1612        assert_eq!(view(&t, 7).max_attempts, 3);
1613        t.change_attempt_limit(None).unwrap();
1614        assert_eq!(view(&t, 7).max_attempts, 7);
1615    }
1616
1617    #[test]
1618    fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
1619        let mut t = task("answered");
1620        t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
1621        let v = view(&t, 2);
1622        assert_eq!(v.answers.len(), 1);
1623        assert_eq!(v.answers[0].question, "Which backend?");
1624        assert_eq!(v.answers[0].answer, "SQLite");
1625    }
1626
1627    #[test]
1628    fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
1629        let dir = tempdir().unwrap();
1630        let queue = Queue::at(dir.path().join("queue"));
1631        let questions = Questions::at(dir.path().join("questions"));
1632        let mut a = task("a");
1633        a.priority = 9;
1634        queue.put(&mut a).unwrap();
1635
1636        let verdict = Verdict {
1637            decisions: vec![Decision {
1638                id: a.id.clone(),
1639                blocked_by: vec!["20260101-000000-dead".to_owned()],
1640                reason: Some("waits on the other task".to_owned()),
1641                recovery: None,
1642                question: None,
1643                choices: Vec::new(),
1644                evidence: Vec::new(),
1645            }],
1646        };
1647        apply(&queue, &questions, &verdict).unwrap();
1648
1649        let back = queue.get(&a.id).unwrap();
1650        assert_eq!(back.status, TaskStatus::Blocked);
1651        assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
1652        assert_eq!(
1653            back.priority, 9,
1654            "the conductor's reply cannot carry priority"
1655        );
1656    }
1657
1658    #[test]
1659    fn a_question_decision_files_one_and_blocks_on_its_id() {
1660        let dir = tempdir().unwrap();
1661        let queue = Queue::at(dir.path().join("queue"));
1662        let questions = Questions::at(dir.path().join("questions"));
1663        let mut t = task("ambiguous");
1664        queue.put(&mut t).unwrap();
1665
1666        let verdict = Verdict {
1667            decisions: vec![Decision {
1668                id: t.id.clone(),
1669                blocked_by: Vec::new(),
1670                reason: Some("which backend?".to_owned()),
1671                recovery: None,
1672                question: Some("Which storage backend?".to_owned()),
1673                choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
1674                evidence: Vec::new(),
1675            }],
1676        };
1677        apply(&queue, &questions, &verdict).unwrap();
1678
1679        let back = queue.get(&t.id).unwrap();
1680        assert_eq!(back.status, TaskStatus::Blocked);
1681        assert_eq!(back.blocked_by.len(), 1);
1682        let q = questions.get(&back.blocked_by[0]).unwrap();
1683        assert_eq!(q.summary, "Which storage backend?");
1684        assert_eq!(q.node, NODE);
1685        assert!(q.status.open());
1686    }
1687
1688    #[test]
1689    fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
1690        let dir = tempdir().unwrap();
1691        let queue = Queue::at(dir.path().join("queue"));
1692        let questions = Questions::at(dir.path().join("questions"));
1693        let mut t = task("asked once");
1694        queue.put(&mut t).unwrap();
1695
1696        let decision = Decision {
1697            id: t.id.clone(),
1698            reason: Some("still deciding".to_owned()),
1699            question: Some("Which backend?".to_owned()),
1700            ..Decision::default()
1701        };
1702        apply(
1703            &queue,
1704            &questions,
1705            &Verdict {
1706                decisions: vec![decision.clone()],
1707            },
1708        )
1709        .unwrap();
1710        assert_eq!(questions.list().len(), 1);
1711        let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
1712
1713        // An operator releasing the blocked task by hand, without answering,
1714        // puts it back at `Queued` while the question stays open - exactly
1715        // the case the guard in `apply_one` exists for: a later cycle
1716        // proposing the very same question must reuse it, not file a second.
1717        let mut released = queue.get(&t.id).unwrap();
1718        released.release();
1719        queue.put(&mut released).unwrap();
1720
1721        apply(
1722            &queue,
1723            &questions,
1724            &Verdict {
1725                decisions: vec![decision],
1726            },
1727        )
1728        .unwrap();
1729        assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
1730        let after = queue.get(&t.id).unwrap();
1731        assert_eq!(
1732            after.blocked_by,
1733            [first_question_id],
1734            "the existing open question is reused, not replaced"
1735        );
1736    }
1737
1738    #[test]
1739    fn a_same_id_question_from_another_node_is_not_reused() {
1740        let dir = tempdir().unwrap();
1741        let queue = Queue::at(dir.path().join("queue"));
1742        let questions = Questions::at(dir.path().join("questions"));
1743        let mut t = task("must ask the conductor");
1744        queue.put(&mut t).unwrap();
1745
1746        let mut unrelated = Question::new(
1747            t.id.clone(),
1748            "review".to_owned(),
1749            "reviewer-1".to_owned(),
1750            "An unrelated review question".to_owned(),
1751            String::new(),
1752            Vec::new(),
1753        );
1754        questions.put(&mut unrelated).unwrap();
1755
1756        apply(
1757            &queue,
1758            &questions,
1759            &Verdict {
1760                decisions: vec![Decision {
1761                    id: t.id.clone(),
1762                    question: Some("Which backend?".to_owned()),
1763                    ..Decision::default()
1764                }],
1765            },
1766        )
1767        .unwrap();
1768
1769        let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
1770        assert_eq!(blocked_by.len(), 1);
1771        assert_ne!(blocked_by[0], unrelated.id);
1772        assert!(questions.get(&unrelated.id).unwrap().status.open());
1773        assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
1774    }
1775
1776    #[test]
1777    fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
1778        let dir = tempdir().unwrap();
1779        let queue = Queue::at(dir.path().join("queue"));
1780        let questions = Questions::at(dir.path().join("questions"));
1781        let mut t = task("waits on an answer");
1782        queue.put(&mut t).unwrap();
1783
1784        apply(
1785            &queue,
1786            &questions,
1787            &Verdict {
1788                decisions: vec![Decision {
1789                    id: t.id.clone(),
1790                    blocked_by: Vec::new(),
1791                    reason: None,
1792                    recovery: None,
1793                    question: Some("Which backend?".to_owned()),
1794                    choices: Vec::new(),
1795                    evidence: Vec::new(),
1796                }],
1797            },
1798        )
1799        .unwrap();
1800        let blocked = queue.get(&t.id).unwrap();
1801        let question_id = blocked.blocked_by[0].clone();
1802
1803        let mut q = questions.get(&question_id).unwrap();
1804        q.answer(Answer::Text("SQLite".to_owned())).unwrap();
1805        questions.put(&mut q).unwrap();
1806        assert_eq!(q.status, QuestionStatus::Answered);
1807
1808        // `crate::daemon::resolve_blockers` is the deterministic resolver
1809        // that actually does this on the real queue; here it is enough to
1810        // prove the pure steps it is built from behave together.
1811        let mut task_after = queue.get(&t.id).unwrap();
1812        task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
1813        task_after.unblock(&question_id);
1814        assert_eq!(task_after.status, TaskStatus::Queued);
1815        assert_eq!(task_after.answers[0].answer, "SQLite");
1816    }
1817
1818    #[test]
1819    fn a_stalled_task_can_be_requeued_or_held() {
1820        let dir = tempdir().unwrap();
1821        let queue = Queue::at(dir.path().join("queue"));
1822        let questions = Questions::at(dir.path().join("questions"));
1823
1824        let mut requeue_me = task("stuck a");
1825        requeue_me.start("run-1".to_owned());
1826        queue.put(&mut requeue_me).unwrap();
1827
1828        let mut hold_me = task("stuck b");
1829        hold_me.start("run-2".to_owned());
1830        queue.put(&mut hold_me).unwrap();
1831
1832        apply(
1833            &queue,
1834            &questions,
1835            &Verdict {
1836                decisions: vec![
1837                    Decision {
1838                        id: requeue_me.id.clone(),
1839                        recovery: Some(Recovery::Requeue),
1840                        ..Decision::default()
1841                    },
1842                    Decision {
1843                        id: hold_me.id.clone(),
1844                        recovery: Some(Recovery::Hold),
1845                        reason: Some("looks broken".to_owned()),
1846                        ..Decision::default()
1847                    },
1848                ],
1849            },
1850        )
1851        .unwrap();
1852
1853        let requeued = queue.get(&requeue_me.id).unwrap();
1854        assert_eq!(requeued.status, TaskStatus::Queued);
1855        assert_eq!(requeued.attempts, 0);
1856
1857        let held = queue.get(&hold_me.id).unwrap();
1858        assert_eq!(held.status, TaskStatus::Held);
1859        assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
1860    }
1861
1862    #[test]
1863    fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
1864        // Reproduces the reported bug: a task held after burning its
1865        // attempts, once the conductor asks a follow-up question about it,
1866        // must come back `held` - not `queued` - once the question is
1867        // answered, whatever the answer said.
1868        let dir = tempdir().unwrap();
1869        let queue = Queue::at(dir.path().join("queue"));
1870        let questions = Questions::at(dir.path().join("questions"));
1871        let mut t = task("held out of attempts");
1872        t.hold_machine(Some("out of attempts".to_owned()));
1873        queue.put(&mut t).unwrap();
1874
1875        apply(
1876            &queue,
1877            &questions,
1878            &Verdict {
1879                decisions: vec![Decision {
1880                    id: t.id.clone(),
1881                    reason: Some("what should happen to this one?".to_owned()),
1882                    question: Some("Hold it, or try again?".to_owned()),
1883                    ..Decision::default()
1884                }],
1885            },
1886        )
1887        .unwrap();
1888        let blocked = queue.get(&t.id).unwrap();
1889        assert_eq!(blocked.status, TaskStatus::Blocked);
1890        let question_id = blocked.blocked_by[0].clone();
1891
1892        let mut q = questions.get(&question_id).unwrap();
1893        q.answer(Answer::Text("leave it held".to_owned())).unwrap();
1894        questions.put(&mut q).unwrap();
1895
1896        // What `crate::daemon::resolve_blockers` does on the real queue.
1897        let mut after = queue.get(&t.id).unwrap();
1898        after.record_answer(q.summary.clone(), "leave it held".to_owned());
1899        after.unblock(&question_id);
1900        assert_eq!(
1901            after.status,
1902            TaskStatus::Held,
1903            "must not fall back to queued"
1904        );
1905        assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
1906    }
1907
1908    #[test]
1909    fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
1910        // A different path to the same bug round 1 fixed: a task
1911        // machine-held for disk pressure, blocked on a conductor question,
1912        // answered "keep it held", and restored to `held`. If the
1913        // conductor's next decision reconfirms `recovery: hold` with no new
1914        // `reason` - allowed, `Decision::reason` is optional - `hold_machine`
1915        // must not silently leave the stale disk-pressure text in place, or
1916        // `crate::triage::run_once` reads it as an unexamined, auto-resolvable
1917        // hold and releases the task straight through the operator's answer
1918        // the instant disk space looks fine again.
1919        let dir = tempdir().unwrap();
1920        let queue = Queue::at(dir.path().join("queue"));
1921        let questions = Questions::at(dir.path().join("questions"));
1922        let mut t = task("disk pressure, then reconsidered");
1923        t.hold_machine(Some(
1924            "not enough free space to start a run: 10 bytes free, 100 required by \
1925             `[disk] min_free_bytes`"
1926                .to_owned(),
1927        ));
1928        t.record_answer(
1929            "How should this be handled?".to_owned(),
1930            "keep it held, a human will look at it later".to_owned(),
1931        );
1932        queue.put(&mut t).unwrap();
1933
1934        apply(
1935            &queue,
1936            &questions,
1937            &Verdict {
1938                decisions: vec![Decision {
1939                    id: t.id.clone(),
1940                    recovery: Some(Recovery::Hold),
1941                    ..Decision::default()
1942                }],
1943            },
1944        )
1945        .unwrap();
1946
1947        let after = queue.get(&t.id).unwrap();
1948        assert_eq!(after.status, TaskStatus::Held);
1949        assert!(
1950            !after
1951                .hold_reason
1952                .as_deref()
1953                .unwrap_or_default()
1954                .starts_with("not enough free space"),
1955            "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
1956            after.hold_reason
1957        );
1958
1959        // The disk gate would report space is fine now - `crate::triage`
1960        // must not read the old text and release the task through it.
1961        let cfg_dir = tempdir().unwrap();
1962        let config = cfg_dir.path().join("magi.toml");
1963        std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
1964        let report =
1965            crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
1966        assert!(report.resumed.is_empty(), "must not be auto-released");
1967        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
1968    }
1969
1970    #[test]
1971    fn explicit_resumption_rehold_requires_new_evidence_and_another_answer() {
1972        crate::run::pin_test_home();
1973        let dir = tempdir().unwrap();
1974        let queue = Queue::at(dir.path().join("queue"));
1975        let questions = Questions::at(dir.path().join("questions"));
1976        let config = dir.path().join("magi.toml");
1977        std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
1978        let now = jiff::Timestamp::now();
1979        let triage = || crate::triage::run_once(&queue, &questions, Some(&config), now);
1980        let open = || {
1981            questions
1982                .list()
1983                .into_iter()
1984                .filter(|q| q.node == "triage" && q.status.open())
1985                .collect::<Vec<_>>()
1986        };
1987        let hold = Decision {
1988            recovery: Some(Recovery::Hold),
1989            reason: Some("waiting on manual worktree cleanup".to_owned()),
1990            ..Decision::default()
1991        };
1992
1993        let mut t = task("looping hold");
1994        t.hold_machine(Some("waiting on manual worktree cleanup".to_owned()));
1995        queue.put(&mut t).unwrap();
1996        let hold = Decision {
1997            id: t.id.clone(),
1998            ..hold
1999        };
2000
2001        // 1. triage asks the ordinary machine question; the operator resumes.
2002        assert_eq!(triage().asked.len(), 1);
2003        let first = open().remove(0);
2004        let mut q = questions.get(&first.id).unwrap();
2005        let resume = q.choices[0].clone();
2006        q.answer(Answer::Choice(resume)).unwrap();
2007        questions.put(&mut q).unwrap();
2008        assert_eq!(triage().answered.len(), 1);
2009        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2010
2011        // Rewording the old cause cannot undo the first explicit answer.
2012        apply(
2013            &queue,
2014            &questions,
2015            &Verdict {
2016                decisions: vec![hold.clone()],
2017            },
2018        )
2019        .unwrap();
2020        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2021        assert!(triage().asked.is_empty());
2022
2023        // A new independently filed check can justify a later hold.
2024        let mut evidence = Question::new(
2025            t.id.clone(),
2026            "preflight".into(),
2027            "check".into(),
2028            "New permission failure".into(),
2029            String::new(),
2030            Vec::new(),
2031        );
2032        questions.put(&mut evidence).unwrap();
2033        let hold = Decision {
2034            evidence: vec![evidence.id.clone()],
2035            reason: Some("new permission check failed".into()),
2036            ..hold
2037        };
2038        apply(
2039            &queue,
2040            &questions,
2041            &Verdict {
2042                decisions: vec![hold.clone()],
2043            },
2044        )
2045        .unwrap();
2046        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2047
2048        // The new hold earns one question identifying the new evidence.
2049        assert_eq!(triage().asked.len(), 1);
2050        let second = open().remove(0);
2051        assert!(second.detail.contains("new permission check failed"));
2052        assert!(triage().asked.is_empty(), "no duplicate question");
2053        assert_eq!(open().len(), 1);
2054
2055        // Another explicit answer supersedes this later hold too.
2056        let mut q = questions.get(&second.id).unwrap();
2057        let force = q.choices[0].clone();
2058        q.answer(Answer::Choice(force)).unwrap();
2059        questions.put(&mut q).unwrap();
2060        assert_eq!(triage().answered.len(), 1);
2061        apply(
2062            &queue,
2063            &questions,
2064            &Verdict {
2065                decisions: vec![hold],
2066            },
2067        )
2068        .unwrap();
2069        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2070    }
2071
2072    #[test]
2073    fn a_runnable_task_can_be_held_directly_without_a_question() {
2074        let dir = tempdir().unwrap();
2075        let queue = Queue::at(dir.path().join("queue"));
2076        let questions = Questions::at(dir.path().join("questions"));
2077        let mut t = task("already answered, should stay put");
2078        queue.put(&mut t).unwrap();
2079
2080        apply(
2081            &queue,
2082            &questions,
2083            &Verdict {
2084                decisions: vec![Decision {
2085                    id: t.id.clone(),
2086                    recovery: Some(Recovery::Hold),
2087                    reason: Some("operator already said keep this held".to_owned()),
2088                    ..Decision::default()
2089                }],
2090            },
2091        )
2092        .unwrap();
2093
2094        let after = queue.get(&t.id).unwrap();
2095        assert_eq!(after.status, TaskStatus::Held);
2096        assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
2097    }
2098
2099    #[test]
2100    fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
2101        crate::run::pin_test_home();
2102        // Reproduces the other half of the reported bug (task 6081): once
2103        // the operator's answer says the work already happened outside the
2104        // loop - PR merged, worktree cleaned up - the conductor needs an
2105        // actual terminal state to put the task in, not just a hold it will
2106        // keep being re-asked about.
2107        let dir = tempdir().unwrap();
2108        let queue = Queue::at(dir.path().join("queue"));
2109        let questions = Questions::at(dir.path().join("questions"));
2110        let mut t = task("already merged by hand");
2111        t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
2112        t.record_answer(
2113            "Handle this one?".to_owned(),
2114            "already merged and cleaned up, close it".to_owned(),
2115        );
2116        queue.put(&mut t).unwrap();
2117
2118        apply(
2119            &queue,
2120            &questions,
2121            &Verdict {
2122                decisions: vec![Decision {
2123                    id: t.id.clone(),
2124                    recovery: Some(Recovery::Done),
2125                    reason: Some("operator confirmed this already landed".to_owned()),
2126                    ..Decision::default()
2127                }],
2128            },
2129        )
2130        .unwrap();
2131
2132        let after = queue.get(&t.id).unwrap();
2133        assert_eq!(after.status, TaskStatus::Done);
2134        assert!(after.hold_reason.is_none());
2135        assert_eq!(after.answers.len(), 1, "the record of why is kept");
2136    }
2137
2138    #[test]
2139    fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
2140        let dir = tempdir().unwrap();
2141        let queue = Queue::at(dir.path().join("queue"));
2142        let questions = Questions::at(dir.path().join("questions"));
2143
2144        let mut queued = task("never ran yet");
2145        queue.put(&mut queued).unwrap();
2146
2147        let mut running = task("mid-run");
2148        running.start("run-1".to_owned());
2149        queue.put(&mut running).unwrap();
2150
2151        for id in [queued.id.clone(), running.id.clone()] {
2152            apply(
2153                &queue,
2154                &questions,
2155                &Verdict {
2156                    decisions: vec![Decision {
2157                        id,
2158                        recovery: Some(Recovery::Done),
2159                        ..Decision::default()
2160                    }],
2161                },
2162            )
2163            .unwrap();
2164        }
2165
2166        assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
2167        assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
2168    }
2169
2170    #[test]
2171    fn a_question_after_two_settled_answers_is_still_filed_and_blocks() {
2172        // A genuine question must stay answerable however many were settled
2173        // before it: refusing it left `magi answer --list` empty.
2174        let dir = tempdir().unwrap();
2175        let queue = Queue::at(dir.path().join("queue"));
2176        let questions = Questions::at(dir.path().join("questions"));
2177        let mut t = task("asked about repeatedly");
2178        t.hold_machine(Some("out of attempts".to_owned()));
2179        t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
2180        t.record_answer(
2181            "Handle this one? (2)".to_owned(),
2182            "still not yet".to_owned(),
2183        );
2184        queue.put(&mut t).unwrap();
2185        assert_eq!(questions.list().len(), 0);
2186
2187        let verdict = Verdict {
2188            decisions: vec![Decision {
2189                id: t.id.clone(),
2190                question: Some("Branch conflicts with origin/main, how do we proceed?".to_owned()),
2191                ..Decision::default()
2192            }],
2193        };
2194        apply(&queue, &questions, &verdict).unwrap();
2195
2196        let filed = questions.list();
2197        assert_eq!(filed.len(), 1, "the question was filed");
2198        assert_eq!(
2199            filed[0].summary,
2200            "Branch conflicts with origin/main, how do we proceed?"
2201        );
2202        let deputy = filed[0]
2203            .deputy
2204            .as_ref()
2205            .expect("the wait is handed to a deputy");
2206        assert!(deputy.brief.contains(&t.id), "{}", deputy.brief);
2207        assert_eq!(
2208            deputy.starts, 0,
2209            "filing never starts anything: the loop does not wait"
2210        );
2211        assert!(filed[0].status.open());
2212        assert_eq!(filed[0].node, NODE);
2213        let after = queue.get(&t.id).unwrap();
2214        assert_eq!(after.status, TaskStatus::Blocked);
2215        assert_eq!(after.blocked_by, vec![filed[0].id.clone()]);
2216        assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
2217        assert!(
2218            !after
2219                .hold_reason
2220                .clone()
2221                .unwrap_or_default()
2222                .contains("conduct tried to ask"),
2223            "no hold was applied"
2224        );
2225
2226        apply(&queue, &questions, &verdict).unwrap();
2227        let again = questions.list();
2228        assert_eq!(again.len(), 1, "the open question is reused");
2229        assert_eq!(again[0].id, filed[0].id);
2230    }
2231
2232    #[test]
2233    fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
2234        let dir = tempdir().unwrap();
2235        let queue = Queue::at(dir.path().join("queue"));
2236        let questions = Questions::at(dir.path().join("questions"));
2237        let mut t = task("asked about once already");
2238        t.hold_machine(Some("out of attempts".to_owned()));
2239        t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
2240        queue.put(&mut t).unwrap();
2241
2242        apply(
2243            &queue,
2244            &questions,
2245            &Verdict {
2246                decisions: vec![Decision {
2247                    id: t.id.clone(),
2248                    question: Some("Still not sure - now what?".to_owned()),
2249                    ..Decision::default()
2250                }],
2251            },
2252        )
2253        .unwrap();
2254
2255        assert_eq!(questions.list().len(), 1, "the second question was filed");
2256        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
2257    }
2258
2259    #[test]
2260    fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
2261        // `crate::triage` never calls `Task::block`, and only ever looks at
2262        // tasks still `held` - so a conductor question that moved this task
2263        // to `blocked` would orphan triage's own open question. The guard in
2264        // `apply_one` must leave the task alone until triage's question
2265        // settles.
2266        let dir = tempdir().unwrap();
2267        let queue = Queue::at(dir.path().join("queue"));
2268        let questions = Questions::at(dir.path().join("questions"));
2269        let mut t = task("held, triage already asking about it");
2270        t.hold_machine(Some("cause unclear".to_owned()));
2271        queue.put(&mut t).unwrap();
2272
2273        let mut triage_q = Question::new(
2274            t.id.clone(),
2275            crate::triage::NODE.to_owned(),
2276            "triage".to_owned(),
2277            "Still needed?".to_owned(),
2278            String::new(),
2279            vec![
2280                "resume".to_owned(),
2281                "not yet".to_owned(),
2282                "discard".to_owned(),
2283            ],
2284        );
2285        questions.put(&mut triage_q).unwrap();
2286
2287        for decision in [
2288            Decision {
2289                id: t.id.clone(),
2290                question: Some("what now?".to_owned()),
2291                ..Decision::default()
2292            },
2293            Decision {
2294                id: t.id.clone(),
2295                recovery: Some(Recovery::Requeue),
2296                ..Decision::default()
2297            },
2298        ] {
2299            apply(
2300                &queue,
2301                &questions,
2302                &Verdict {
2303                    decisions: vec![decision],
2304                },
2305            )
2306            .unwrap();
2307        }
2308
2309        let after = queue.get(&t.id).unwrap();
2310        assert_eq!(
2311            after.status,
2312            TaskStatus::Held,
2313            "triage still owns this hold"
2314        );
2315        assert!(after.blocked_by.is_empty());
2316        assert_eq!(
2317            questions.list().len(),
2318            1,
2319            "no second, conductor-owned question was filed"
2320        );
2321    }
2322
2323    #[test]
2324    fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
2325        // The race `crate::triage::pending_for` exists to close: the
2326        // operator has already answered the triage question (it is no
2327        // longer `open`), but `crate::triage::run_once` - which only runs on
2328        // a fully idle daemon tick, far less often than the conductor polls
2329        // - has not had a turn to apply it yet. `apply_one` must not treat
2330        // "not open" as "settled" here, or it would block the task out from
2331        // under an answer triage has not read back yet.
2332        let dir = tempdir().unwrap();
2333        let queue = Queue::at(dir.path().join("queue"));
2334        let questions = Questions::at(dir.path().join("questions"));
2335        let mut t = task("held, triage question answered but not yet applied");
2336        t.hold_machine(Some("cause unclear".to_owned()));
2337        queue.put(&mut t).unwrap();
2338
2339        let mut triage_q = Question::new(
2340            t.id.clone(),
2341            crate::triage::NODE.to_owned(),
2342            "triage".to_owned(),
2343            "Still needed?".to_owned(),
2344            String::new(),
2345            vec![
2346                "resume".to_owned(),
2347                "not yet".to_owned(),
2348                "discard".to_owned(),
2349            ],
2350        );
2351        questions.put(&mut triage_q).unwrap();
2352        triage_q
2353            .answer(Answer::Choice("not yet".to_owned()))
2354            .unwrap();
2355        questions.put(&mut triage_q).unwrap();
2356        assert!(!triage_q.status.open());
2357
2358        apply(
2359            &queue,
2360            &questions,
2361            &Verdict {
2362                decisions: vec![Decision {
2363                    id: t.id.clone(),
2364                    question: Some("what now?".to_owned()),
2365                    ..Decision::default()
2366                }],
2367            },
2368        )
2369        .unwrap();
2370
2371        let after = queue.get(&t.id).unwrap();
2372        assert_eq!(
2373            after.status,
2374            TaskStatus::Held,
2375            "triage's own answer is not yet applied - conduct must wait"
2376        );
2377        assert_eq!(
2378            questions.list().len(),
2379            1,
2380            "no conductor question was filed over the pending triage answer"
2381        );
2382    }
2383
2384    #[test]
2385    fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
2386        let dir = tempdir().unwrap();
2387        let queue = Queue::at(dir.path().join("queue"));
2388        let questions = Questions::at(dir.path().join("questions"));
2389        let mut held = task("manual recovery");
2390        held.priority = 300;
2391        held.runs.push("run20260912-224242-daf5".to_owned());
2392        held.hold_manual(Some(
2393            "active manual recovery run20260912-224242-daf5".to_owned(),
2394        ));
2395        queue.put(&mut held).unwrap();
2396
2397        // Every field a conductor may use to alter lifecycle state is ignored:
2398        // requeue/review would dispatch duplicate work, hold could overwrite
2399        // evidence, and a question would turn the hold into `blocked`.
2400        for decision in [
2401            Decision {
2402                id: held.id.clone(),
2403                recovery: Some(Recovery::Requeue),
2404                ..Decision::default()
2405            },
2406            Decision {
2407                id: held.id.clone(),
2408                recovery: Some(Recovery::Hold),
2409                reason: Some("stale replacement reason".to_owned()),
2410                ..Decision::default()
2411            },
2412            Decision {
2413                id: held.id.clone(),
2414                recovery: Some(Recovery::Review),
2415                ..Decision::default()
2416            },
2417            Decision {
2418                id: held.id.clone(),
2419                blocked_by: vec!["other-task".to_owned()],
2420                question: Some("retry now?".to_owned()),
2421                ..Decision::default()
2422            },
2423        ] {
2424            apply(
2425                &queue,
2426                &questions,
2427                &Verdict {
2428                    decisions: vec![decision],
2429                },
2430            )
2431            .unwrap();
2432        }
2433
2434        let after = queue.get(&held.id).unwrap();
2435        assert_eq!(after.status, TaskStatus::Held);
2436        assert!(after.operator_held());
2437        assert_eq!(after.priority, 300);
2438        assert_eq!(after.runs, ["run20260912-224242-daf5"]);
2439        assert_eq!(
2440            after.hold_reason.as_deref(),
2441            Some("active manual recovery run20260912-224242-daf5")
2442        );
2443        assert!(after.blocked_by.is_empty());
2444        assert!(questions.list().is_empty());
2445        assert!(
2446            queue.next_runnable().is_none(),
2447            "must not dispatch a duplicate"
2448        );
2449    }
2450
2451    #[test]
2452    fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
2453        let dir = tempdir().unwrap();
2454        let queue = Queue::at(dir.path().join("queue"));
2455        let questions = Questions::at(dir.path().join("questions"));
2456
2457        let mut automatic = task("disk gate");
2458        automatic.hold_machine(Some("disk full".to_owned()));
2459        queue.put(&mut automatic).unwrap();
2460        let requeue = || Verdict {
2461            decisions: vec![Decision {
2462                id: automatic.id.clone(),
2463                recovery: Some(Recovery::Requeue),
2464                ..Decision::default()
2465            }],
2466        };
2467        apply(&queue, &questions, &requeue()).unwrap();
2468        assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
2469
2470        let mut manual = task("operator gate");
2471        manual.hold_manual(Some("wait for operator".to_owned()));
2472        queue.put(&mut manual).unwrap();
2473        apply(
2474            &queue,
2475            &questions,
2476            &Verdict {
2477                decisions: vec![Decision {
2478                    id: manual.id.clone(),
2479                    recovery: Some(Recovery::Requeue),
2480                    ..Decision::default()
2481                }],
2482            },
2483        )
2484        .unwrap();
2485        assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
2486
2487        // This mirrors the CLI and web release routes: only an explicit
2488        // operator action clears the manual boundary.
2489        let mut released = queue.get(&manual.id).unwrap();
2490        released.release();
2491        queue.put(&mut released).unwrap();
2492        assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
2493    }
2494
2495    #[test]
2496    fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
2497        let dir = tempdir().unwrap();
2498        let queue = Queue::at(dir.path().join("queue"));
2499        let questions = Questions::at(dir.path().join("questions"));
2500        let mut legacy = task("old explicit hold");
2501        legacy.status = TaskStatus::Held;
2502        legacy.hold_reason = Some("manual recovery already active".to_owned());
2503        legacy.hold_source = None;
2504        legacy.blocked_by = vec!["dependency".to_owned()];
2505        queue.put(&mut legacy).unwrap();
2506
2507        apply(
2508            &queue,
2509            &questions,
2510            &Verdict {
2511                decisions: vec![Decision {
2512                    id: legacy.id.clone(),
2513                    recovery: Some(Recovery::Requeue),
2514                    ..Decision::default()
2515                }],
2516            },
2517        )
2518        .unwrap();
2519
2520        let after = queue.get(&legacy.id).unwrap();
2521        assert_eq!(after.status, TaskStatus::Held);
2522        assert_eq!(after.hold_source, None);
2523        assert_eq!(after.hold_reason, legacy.hold_reason);
2524        assert_eq!(after.blocked_by, legacy.blocked_by);
2525    }
2526
2527    #[test]
2528    fn review_recovery_is_a_no_op_without_a_survivable_branch() {
2529        // `surviving_branch` reaches `RunState::load`, which reaches the
2530        // process-global `run::home()` - a `OnceLock`, so this only wins the
2531        // race the first time it runs in the binary; every other test still
2532        // reaches the same directory whichever call won, and this test's own
2533        // run id never collides with another test's.
2534        crate::run::pin_test_home();
2535        let dir = tempdir().unwrap();
2536        let queue = Queue::at(dir.path().join("queue"));
2537        let questions = Questions::at(dir.path().join("questions"));
2538        let mut t = task("blocked with no readable run");
2539        t.start("20260101-000000-dead".to_owned()); // no such run on disk
2540        t.fail("blocked", 5);
2541        queue.put(&mut t).unwrap();
2542
2543        apply(
2544            &queue,
2545            &questions,
2546            &Verdict {
2547                decisions: vec![Decision {
2548                    id: t.id.clone(),
2549                    recovery: Some(Recovery::Review),
2550                    ..Decision::default()
2551                }],
2552            },
2553        )
2554        .unwrap();
2555
2556        let after = queue.get(&t.id).unwrap();
2557        assert_eq!(
2558            after.status,
2559            TaskStatus::Failed,
2560            "with nothing to reopen, the decision is dropped rather than guessed at"
2561        );
2562        assert!(after.review_branch.is_none());
2563    }
2564
2565    #[test]
2566    fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
2567        // `Hold` is the one exception - see `Decision::recovery`'s doc and
2568        // `a_runnable_task_can_be_held_directly_without_a_question` - but a
2569        // task already in line has nothing for `requeue` or `review` to do.
2570        let dir = tempdir().unwrap();
2571        let queue = Queue::at(dir.path().join("queue"));
2572        let questions = Questions::at(dir.path().join("questions"));
2573
2574        for recovery in [Recovery::Requeue, Recovery::Review] {
2575            let mut t = task("ordinary");
2576            queue.put(&mut t).unwrap();
2577
2578            apply(
2579                &queue,
2580                &questions,
2581                &Verdict {
2582                    decisions: vec![Decision {
2583                        id: t.id.clone(),
2584                        recovery: Some(recovery),
2585                        ..Decision::default()
2586                    }],
2587                },
2588            )
2589            .unwrap();
2590
2591            assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2592        }
2593    }
2594
2595    #[tokio::test]
2596    async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
2597        let dir = tempdir().unwrap();
2598        let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
2599        let queue = Queue::at(dir.path().join("queue"));
2600        let questions = Questions::at(dir.path().join("questions"));
2601        let mut t = task("normal");
2602        queue.put(&mut t).unwrap();
2603
2604        let mut conductor = Conductor::new();
2605        conductor
2606            .maybe_run(
2607                &cfg,
2608                dir.path(),
2609                &queue,
2610                &questions,
2611                dir.path(),
2612                &[t.clone()],
2613                &[],
2614                &[],
2615                2,
2616            )
2617            .await;
2618
2619        assert_eq!(
2620            queue.get(&t.id).unwrap().status,
2621            TaskStatus::Queued,
2622            "a failed invocation must change nothing"
2623        );
2624        assert!(
2625            queue.next_runnable().is_some(),
2626            "the loop must still be able to take the next task"
2627        );
2628    }
2629
2630    #[tokio::test]
2631    async fn a_reply_with_no_json_leaves_the_queue_untouched() {
2632        let dir = tempdir().unwrap();
2633        let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
2634        let queue = Queue::at(dir.path().join("queue"));
2635        let questions = Questions::at(dir.path().join("questions"));
2636        let mut t = task("normal");
2637        queue.put(&mut t).unwrap();
2638
2639        let mut conductor = Conductor::new();
2640        conductor
2641            .maybe_run(
2642                &cfg,
2643                dir.path(),
2644                &queue,
2645                &questions,
2646                dir.path(),
2647                &[t.clone()],
2648                &[],
2649                &[],
2650                2,
2651            )
2652            .await;
2653
2654        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2655    }
2656
2657    #[tokio::test]
2658    async fn a_conductor_chain_falls_to_the_next_agent_once_each() {
2659        let dir = tempdir().unwrap();
2660        let mut t = task("chained");
2661        let reply = format!(
2662            "{{\"decisions\":[{{\"id\":\"{}\",\"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}",
2663            t.id
2664        );
2665        let counted = |id: &str, tail: &str| {
2666            let calls = dir.path().join(format!("{id}.calls"));
2667            let path = dir.path().join(format!("{id}.sh"));
2668            std::fs::write(
2669                &path,
2670                format!(
2671                    "#!/bin/sh\ncat >/dev/null\necho x >> '{}'\n{tail}\n",
2672                    calls.display()
2673                ),
2674            )
2675            .unwrap();
2676            AgentSpec {
2677                id: id.to_owned(),
2678                kind: AgentKind::Command,
2679                model: None,
2680                command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
2681                extra_args: Vec::new(),
2682                env: BTreeMap::new(),
2683                prompt_delivery: None,
2684            }
2685        };
2686        let n_calls = |id: &str| {
2687            std::fs::read_to_string(dir.path().join(format!("{id}.calls")))
2688                .map_or(0, |s| s.lines().count())
2689        };
2690        let mut cfg = config(counted("a", "exit 3"));
2691        cfg.agents = vec![
2692            counted("a", "exit 3"),
2693            counted("b", &format!("printf '%s' '{reply}'")),
2694        ];
2695        cfg.roles.conductor = Some(crate::config::AgentChoice::Chain(vec![
2696            "a".into(),
2697            "b".into(),
2698            "a".into(),
2699        ]));
2700        let queue = Queue::at(dir.path().join("queue"));
2701        let questions = Questions::at(dir.path().join("questions"));
2702        queue.put(&mut t).unwrap();
2703
2704        let mut conductor = Conductor::new();
2705        conductor
2706            .maybe_run(
2707                &cfg,
2708                dir.path(),
2709                &queue,
2710                &questions,
2711                dir.path(),
2712                &[t.clone()],
2713                &[],
2714                &[],
2715                2,
2716            )
2717            .await;
2718
2719        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
2720        assert_eq!(n_calls("a"), 1, "each id is tried once");
2721        assert_eq!(n_calls("b"), 1);
2722        let saved = std::fs::read_to_string(seat_path(dir.path())).unwrap();
2723        assert!(
2724            saved.contains("\"b\""),
2725            "the seat on disk is the one that ran: {saved}"
2726        );
2727    }
2728
2729    #[tokio::test]
2730    async fn json_survives_code_fences_and_a_preamble() {
2731        let dir = tempdir().unwrap();
2732        let mut t = task("fenced");
2733        let reply = format!(
2734            "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
2735             \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
2736            t.id
2737        );
2738        let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2739        let queue = Queue::at(dir.path().join("queue"));
2740        let questions = Questions::at(dir.path().join("questions"));
2741        queue.put(&mut t).unwrap();
2742
2743        let mut conductor = Conductor::new();
2744        conductor
2745            .maybe_run(
2746                &cfg,
2747                dir.path(),
2748                &queue,
2749                &questions,
2750                dir.path(),
2751                &[t.clone()],
2752                &[],
2753                &[],
2754                2,
2755            )
2756            .await;
2757
2758        let back = queue.get(&t.id).unwrap();
2759        assert_eq!(back.status, TaskStatus::Blocked);
2760        assert_eq!(back.blocked_by, ["x"]);
2761    }
2762
2763    #[tokio::test]
2764    async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
2765        // Each real invocation writes its own artifact stem, `turn-<n>`, so
2766        // whether a second one happened is read off the artifacts directory.
2767        let dir = tempdir().unwrap();
2768        let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
2769        let queue = Queue::at(dir.path().join("queue"));
2770        let questions = Questions::at(dir.path().join("questions"));
2771        let mut t = task("stable");
2772        queue.put(&mut t).unwrap();
2773        let artifacts = dir.path().join("conduct").join("artifacts");
2774        let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
2775
2776        let mut conductor = Conductor::new();
2777        conductor
2778            .maybe_run(
2779                &cfg,
2780                dir.path(),
2781                &queue,
2782                &questions,
2783                dir.path(),
2784                &[t.clone()],
2785                &[],
2786                &[],
2787                2,
2788            )
2789            .await;
2790        assert!(turn(1).is_file(), "the first cycle must call the conductor");
2791
2792        conductor
2793            .maybe_run(
2794                &cfg,
2795                dir.path(),
2796                &queue,
2797                &questions,
2798                dir.path(),
2799                &[t.clone()],
2800                &[],
2801                &[],
2802                2,
2803            )
2804            .await;
2805        assert!(
2806            !turn(2).is_file(),
2807            "an unchanged revision and an unchanged stalled/finished set must not call the \
2808             conductor twice"
2809        );
2810
2811        // Once the queue actually changes, the next `maybe_run` calls again.
2812        t.priority = 1;
2813        queue.put(&mut t).unwrap();
2814        conductor
2815            .maybe_run(
2816                &cfg,
2817                dir.path(),
2818                &queue,
2819                &questions,
2820                dir.path(),
2821                &[t.clone()],
2822                &[],
2823                &[],
2824                2,
2825            )
2826            .await;
2827        assert!(turn(2).is_file(), "a moved revision calls it again");
2828    }
2829
2830    #[tokio::test]
2831    async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
2832        // The queue's own revision has not moved - nothing wrote to it - but
2833        // a task now looks stalled, purely because time passed. Calling
2834        // again here, and never again once this exact set has been shown
2835        // once, is the whole point of keying `worth_a_look` on the id set
2836        // rather than on "is it non-empty".
2837        let dir = tempdir().unwrap();
2838        let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
2839        let queue = Queue::at(dir.path().join("queue"));
2840        let questions = Questions::at(dir.path().join("questions"));
2841        let mut t = task("quiet");
2842        queue.put(&mut t).unwrap();
2843        let artifacts = dir.path().join("conduct").join("artifacts");
2844        let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
2845
2846        let mut conductor = Conductor::new();
2847        conductor
2848            .maybe_run(
2849                &cfg,
2850                dir.path(),
2851                &queue,
2852                &questions,
2853                dir.path(),
2854                &[t.clone()],
2855                &[],
2856                &[],
2857                2,
2858            )
2859            .await;
2860        assert!(turn(1).is_file());
2861
2862        conductor
2863            .maybe_run(
2864                &cfg,
2865                dir.path(),
2866                &queue,
2867                &questions,
2868                dir.path(),
2869                &[],
2870                &[t.clone()],
2871                &[],
2872                2,
2873            )
2874            .await;
2875        assert!(
2876            turn(2).is_file(),
2877            "a task turning stalled must call the conductor again"
2878        );
2879
2880        // But once shown at this exact revision, showing the *same* stalled
2881        // set again must not call a third time.
2882        conductor
2883            .maybe_run(
2884                &cfg,
2885                dir.path(),
2886                &queue,
2887                &questions,
2888                dir.path(),
2889                &[],
2890                &[t.clone()],
2891                &[],
2892                2,
2893            )
2894            .await;
2895        assert!(
2896            !turn(3).is_file(),
2897            "the same stalled task lingering must not call the conductor every cycle"
2898        );
2899    }
2900
2901    fn rehold(queue: &Queue, questions: &Questions, id: &str, note: &str) {
2902        apply(
2903            queue,
2904            questions,
2905            &Verdict {
2906                decisions: vec![Decision {
2907                    id: id.to_owned(),
2908                    reason: Some(note.to_owned()),
2909                    recovery: Some(Recovery::Hold),
2910                    ..Decision::default()
2911                }],
2912            },
2913        )
2914        .unwrap();
2915    }
2916
2917    #[test]
2918    fn reholding_with_an_identical_note_changes_nothing() {
2919        let dir = tempdir().unwrap();
2920        let queue = Queue::at(dir.path().join("queue"));
2921        let questions = Questions::at(dir.path().join("questions"));
2922        let mut t = task("held");
2923        t.hold_machine(Some("waiting on a human".to_owned()));
2924        queue.put(&mut t).unwrap();
2925
2926        rehold(&queue, &questions, &t.id, "owner said keep it held");
2927        let first = queue.get(&t.id).unwrap();
2928        let rev = queue.revision();
2929        for _ in 0..5 {
2930            rehold(&queue, &questions, &t.id, "owner said keep it held");
2931        }
2932        let after = queue.get(&t.id).unwrap();
2933        assert_eq!(after.hold_reason, first.hold_reason);
2934        assert_eq!(after.updated_at, first.updated_at);
2935        assert_eq!(queue.revision(), rev, "nothing was written");
2936    }
2937
2938    #[test]
2939    fn reholding_with_different_notes_keeps_one_previously_level() {
2940        let dir = tempdir().unwrap();
2941        let queue = Queue::at(dir.path().join("queue"));
2942        let questions = Questions::at(dir.path().join("questions"));
2943        let mut t = task("held");
2944        t.hold_machine(Some("original".to_owned()));
2945        queue.put(&mut t).unwrap();
2946
2947        for i in 0..20 {
2948            rehold(&queue, &questions, &t.id, &format!("note {}", i % 2));
2949        }
2950        let reason = queue.get(&t.id).unwrap().hold_reason.unwrap();
2951        assert_eq!(reason.matches("(previously:").count(), 1, "{reason}");
2952        assert!(reason.starts_with("note "), "the new note leads: {reason}");
2953    }
2954
2955    #[test]
2956    fn an_already_nested_reason_is_matched_by_its_outermost_note() {
2957        let mut t = task("deep");
2958        let nested = "same\n\n(previously: same\n\n(previously: same))";
2959        t.hold_machine(Some(nested.to_owned()));
2960        let d = Decision {
2961            id: t.id.clone(),
2962            reason: Some("same".to_owned()),
2963            ..Decision::default()
2964        };
2965        assert_eq!(reaffirmed_hold_reason(&t, &d), None);
2966        let d2 = Decision {
2967            reason: Some("other".to_owned()),
2968            ..d
2969        };
2970        assert_eq!(
2971            reaffirmed_hold_reason(&t, &d2).as_deref(),
2972            Some("other\n\n(previously: same)")
2973        );
2974    }
2975
2976    #[tokio::test]
2977    async fn a_repeated_hold_decision_settles_instead_of_looping() {
2978        let dir = tempdir().unwrap();
2979        let queue = Queue::at(dir.path().join("queue"));
2980        let questions = Questions::at(dir.path().join("questions"));
2981        let reply = r#"{"decisions":[{"id":"ID","reason":"keep held","recovery":"hold"}]}"#;
2982        let mut t = task("held");
2983        t.hold_machine(Some("first".to_owned()));
2984        queue.put(&mut t).unwrap();
2985        let cfg = config(mock_agent(
2986            dir.path(),
2987            REPLY,
2988            env(&reply.replace("ID", &t.id)),
2989        ));
2990        let mut conductor = Conductor::new();
2991        // Cycle 1 writes the new note; cycle 2 repeats it and writes nothing.
2992        for _ in 0..2 {
2993            let held = queue.get(&t.id).unwrap();
2994            conductor
2995                .maybe_run(
2996                    &cfg,
2997                    dir.path(),
2998                    &queue,
2999                    &questions,
3000                    dir.path(),
3001                    &[],
3002                    &[],
3003                    &[held],
3004                    2,
3005                )
3006                .await;
3007        }
3008        let held = queue.get(&t.id).unwrap();
3009        assert!(
3010            held.hold_reason
3011                .as_deref()
3012                .unwrap()
3013                .starts_with("keep held")
3014        );
3015        assert!(
3016            !conductor.worth_a_look(&queue, &[], &[held]),
3017            "an unchanged held task must stop being reconsidered"
3018        );
3019    }
3020
3021    #[tokio::test]
3022    async fn alternating_wording_does_not_keep_a_held_task_in_the_loop() {
3023        let dir = tempdir().unwrap();
3024        let queue = Queue::at(dir.path().join("queue"));
3025        let questions = Questions::at(dir.path().join("questions"));
3026        let mut t = task("held");
3027        t.hold_machine(Some("first".to_owned()));
3028        queue.put(&mut t).unwrap();
3029        let reply = |note: &str| {
3030            format!(
3031                r#"{{"decisions":[{{"id":"{}","reason":"{note}","recovery":"hold"}}]}}"#,
3032                t.id
3033            )
3034        };
3035        let cfg_a = config(mock_agent(dir.path(), REPLY, env(&reply("keep held"))));
3036        let cfg_b = config(mock_agent(dir.path(), REPLY, env(&reply("leave held"))));
3037        let mut conductor = Conductor::new();
3038        let mut writes = 0;
3039        for i in 0..6 {
3040            let held = queue.get(&t.id).unwrap();
3041            let before = queue.revision();
3042            conductor
3043                .maybe_run(
3044                    if i % 2 == 0 { &cfg_a } else { &cfg_b },
3045                    dir.path(),
3046                    &queue,
3047                    &questions,
3048                    dir.path(),
3049                    &[],
3050                    &[],
3051                    &[held],
3052                    2,
3053                )
3054                .await;
3055            if queue.revision() != before {
3056                writes += 1;
3057            }
3058        }
3059        assert!(writes <= 2, "the loop must settle, saw {writes} writes");
3060        let mut held = queue.get(&t.id).unwrap();
3061        held.instruction.push_str(" (edited)");
3062        queue.put(&mut held).unwrap();
3063        let held = queue.get(&t.id).unwrap();
3064        assert!(conductor.worth_a_look(&queue, &[], &[held]));
3065    }
3066
3067    #[test]
3068    fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
3069        let dir = tempdir().unwrap();
3070        let queue = Queue::at(dir.path().join("queue"));
3071        let mut t = task("t");
3072        queue.put(&mut t).unwrap();
3073
3074        let mut conductor = Conductor::new();
3075        assert!(
3076            conductor.worth_a_look(&queue, &[], &[]),
3077            "a conductor that has never run has something to look at"
3078        );
3079
3080        conductor.last_seen = Some(conductor.snapshot(&queue, &[], &[]));
3081        assert!(
3082            !conductor.worth_a_look(&queue, &[], &[]),
3083            "nothing changed and nothing is stalled or finished"
3084        );
3085        assert!(
3086            conductor.worth_a_look(&queue, &[t.clone()], &[]),
3087            "a stalled task is worth a look even at the same revision"
3088        );
3089        assert!(
3090            conductor.worth_a_look(&queue, &[], &[t.clone()]),
3091            "a finished task is worth a look even at the same revision"
3092        );
3093    }
3094
3095    #[tokio::test]
3096    async fn the_conduct_path_never_calls_ask_and_wait() {
3097        // Structural: grepping this module and `daemon.rs` for
3098        // `ask_and_wait` is the actual assertion this module's own doc
3099        // promises; this test exists so the promise has a name in the test
3100        // output too. `apply_one`'s question path uses `Questions::put`
3101        // exclusively.
3102        let dir = tempdir().unwrap();
3103        let queue = Queue::at(dir.path().join("queue"));
3104        let questions = Questions::at(dir.path().join("questions"));
3105        let mut t = task("asks without blocking");
3106        queue.put(&mut t).unwrap();
3107
3108        apply(
3109            &queue,
3110            &questions,
3111            &Verdict {
3112                decisions: vec![Decision {
3113                    id: t.id.clone(),
3114                    question: Some("ok?".to_owned()),
3115                    ..Decision::default()
3116                }],
3117            },
3118        )
3119        .unwrap();
3120        // Reaching here at all (no hang) is the assertion.
3121        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3122    }
3123
3124    #[test]
3125    fn a_pinned_resume_is_not_held_or_requeued_by_the_conductor() {
3126        let mut t = Task::new("t".into(), "t".into(), PathBuf::new(), Source::Human);
3127        t.resume_override = Some(crate::queue::OperatorResume {
3128            question_id: "q".into(),
3129            at: jiff::Timestamp::now(),
3130            conductor_rehold: None,
3131            forced: false,
3132            pinned_run: Some("run-1".into()),
3133        });
3134        assert!(pinned_resume(&t));
3135        assert!(!may_hold(&mut t, "waiting for magi resume"));
3136        assert!(
3137            t.resume_override
3138                .as_ref()
3139                .unwrap()
3140                .conductor_rehold
3141                .is_none(),
3142            "a refused hold is not recorded as an override"
3143        );
3144        t.resume_override = None;
3145        assert!(!pinned_resume(&t));
3146        assert!(may_hold(&mut t, "no override, so a hold is allowed"));
3147    }
3148
3149    #[test]
3150    fn a_pinned_resume_is_not_blocked_by_a_conductor_question_or_dependency() {
3151        let dir = tempdir().unwrap();
3152        let queue = Queue::at(dir.path().join("queue"));
3153        let questions = Questions::at(dir.path().join("questions"));
3154        let mut t = task("resume me");
3155        t.resume_override = Some(crate::queue::OperatorResume {
3156            question_id: "q".into(),
3157            at: jiff::Timestamp::now(),
3158            conductor_rehold: None,
3159            forced: true,
3160            pinned_run: Some("run-1".into()),
3161        });
3162        queue.put(&mut t).unwrap();
3163
3164        apply(
3165            &queue,
3166            &questions,
3167            &Verdict {
3168                decisions: vec![
3169                    Decision {
3170                        id: t.id.clone(),
3171                        question: Some("really?".to_owned()),
3172                        ..Decision::default()
3173                    },
3174                    Decision {
3175                        id: t.id.clone(),
3176                        blocked_by: vec!["other".to_owned()],
3177                        ..Decision::default()
3178                    },
3179                ],
3180            },
3181        )
3182        .unwrap();
3183
3184        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3185        assert!(questions.list().is_empty());
3186    }
3187}