Skip to main content

magi/
conduct.rs

1//! The conductor: a single agent seat that arranges the queue.
2//!
3//! [`crate::queue`] orders runnable tasks by `priority` alone; nothing in it
4//! can express that one task should wait for another, or that a task whose
5//! last run stopped short deserves a second look before the loop blindly
6//! retries it. Once per polling cycle, [`Conductor::maybe_run`] shows one
7//! agent seat three things - the runnable tasks, the tasks stuck `running`
8//! with no live daemon behind them, and the `failed`/`held` tasks nobody has
9//! decided about yet - and asks it to decide `blocked_by` for the first and a
10//! recovery for the other two. Everything else about the loop - which
11//! unblocked task runs next, one at a time, in `priority` order - is
12//! unchanged; see `crate::daemon`.
13//!
14//! # What the conductor may not do
15//!
16//! [`Decision`] has no field for `priority`, for deleting a task, or for
17//! touching git, a worktree, or a branch directly. [`Recovery::Review`] only
18//! ever reopens a branch `crate::daemon` itself resolved from the task's own
19//! run record ([`surviving_branch`]) - never a name the model wrote - through
20//! `crate::graph::Runner::review`, which reviews and verifies but never
21//! rewrites history.
22//!
23//! # Non-blocking by construction
24//!
25//! [`crate::ask::ask_and_wait`] is never called from here, and the prompt
26//! tells the model the same: that CLI command blocks until a human answers,
27//! and calling it from inside the conductor's own invocation would park the
28//! whole polling loop behind one task's question. Instead a decision that
29//! wants the operator's judgement carries a `question` field, and [`apply`]
30//! files it with [`Question::new`] and [`Questions::put`] and moves on in the
31//! same call. Something still has to read the owner's reply: the question
32//! carries a [`Deputy`] brief, and `magi serve` starts a short-lived seat of
33//! its own for it ([`crate::deputy`]) - never this loop.
34//!
35//! # Fails soft, always
36//!
37//! [`Conductor::maybe_run`] never returns an error: an unusable roster, a
38//! timed-out invocation, or a reply [`verdict::extract_json`] cannot parse are
39//! all logged and treated as "this cycle changes nothing." `crate::daemon`'s
40//! loop always falls through to its own `Queue::next_runnable` regardless of
41//! what happened here.
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::path::{Path, PathBuf};
45use std::time::Duration;
46
47use anyhow::{Context as _, Result};
48use serde::Deserialize;
49
50use crate::agent::{self, Invocation, SeatState};
51use crate::ask::{Deputy, Question, Questions};
52use crate::config::Config;
53use crate::prompt;
54use crate::queue::{Queue, Task, TaskStatus};
55use crate::run::RunState;
56use crate::verdict;
57
58/// Seat name for the conductor's own CLI-side conversation, scoped away from
59/// every other seat magi ever opens - the same rule every other seat follows.
60const SEAT: &str = "conduct";
61
62/// Node name reported to the invoked agent (`MAGI_NODE`) and recorded on any
63/// question it files, so an operator reading the questions list can tell a
64/// conductor's question from one a run's own agent asked.
65pub const NODE: &str = "conduct";
66
67/// Wall-clock limit for one conductor turn. The conductor reads a queue
68/// listing and replies with json; it does not implement anything or run a
69/// build, so this is short - the same order of magnitude as
70/// `crate::chat`'s own single-turn, no-write invocations.
71const TURN_TIMEOUT: Duration = Duration::from_secs(300);
72
73/// What the conductor may choose for a `running`-but-stalled or a
74/// `failed`/`held` task.
75#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
76#[serde(rename_all = "lowercase")]
77pub enum Recovery {
78    /// Put it back in line, attempts reset - the same effect as
79    /// `magi task release`.
80    Requeue,
81    /// Leave it for a human, unchanged otherwise - the same effect as
82    /// `magi task hold`.
83    Hold,
84    /// Reopen the task's surviving branch as a review-only pass
85    /// (`crate::graph::Runner::review`) instead of competing from scratch.
86    /// Only takes effect when [`surviving_branch`] can actually name one;
87    /// otherwise `crate::daemon` falls back to [`Recovery::Requeue`].
88    Review,
89    /// Close the task outright, as [`Task::succeed`] - the same effect as
90    /// `magi task done`. For a `failed`/`held` task only, never `running`:
91    /// this is for a task whose own goal is already known to be met outside
92    /// the loop entirely (the branch was merged and the worktree cleaned up
93    /// by hand, say) and competing it again would only spend attempts on
94    /// work that has nothing left to do, not for a task that merely stopped
95    /// mid-competition and might still need to run.
96    Done,
97}
98
99/// The conductor's decision for one task. Deliberately has no `priority`
100/// field: see this module's doc.
101#[derive(Debug, Clone, Default, Deserialize)]
102pub struct Decision {
103    /// Task id, expected to be copied verbatim from what it was shown.
104    pub id: String,
105    /// Ids this task should be blocked on. Meaningful only for a runnable
106    /// (`queued`) task, or a machine-held task whose missing-only decision
107    /// is unconfirmed against the queue. Ignored otherwise.
108    #[serde(default)]
109    pub blocked_by: Vec<String>,
110    /// Dependencies claimed missing as the sole reason for this question or
111    /// hold. Must name references in `blocked_by`. Empty (including legacy
112    /// replies) means the question/hold may have an independent purpose and
113    /// must be preserved. Never set for decisions needing separate consent.
114    #[serde(default)]
115    pub missing_dependencies: Vec<String>,
116    /// One line explaining the block or the recovery.
117    #[serde(default)]
118    pub reason: Option<String>,
119    /// Recovery for a stalled or finished task. For a runnable (`queued`)
120    /// one, only [`Recovery::Hold`] has any effect. An independent hold takes
121    /// precedence over `blocked_by`, so dependency completion cannot release it.
122    /// Use a hold when the operator already said the task must not compete,
123    /// rather than asking a `question` that restates that answer. `Requeue`, `Review`,
124    /// and `Done` have no meaning for a task that is already in line.
125    #[serde(default)]
126    pub recovery: Option<Recovery>,
127    /// A question for the operator. When present, [`apply`] files it (unless
128    /// one is already open for this task) and blocks the task on its id
129    /// instead of acting on `blocked_by` or `recovery`. Only a missing-only
130    /// decision explicitly marked by [`Self::missing_dependencies`] may be
131    /// replaced with dependency waiting unless the queue confirms missing blockers.
132    #[serde(default)]
133    pub question: Option<String>,
134    /// Fixed answers for `question`, if it has any. Empty means free text.
135    #[serde(default)]
136    pub choices: Vec<String>,
137    /// Task or question ids carrying independently verifiable new blocking evidence.
138    #[serde(default)]
139    pub evidence: Vec<String>,
140}
141
142/// The conductor's whole reply for one cycle.
143///
144/// `decisions` is deliberately **not** `#[serde(default)]`, unlike every
145/// other field in this module. [`verdict::extract_json`] disambiguates
146/// between several balanced `{...}` spans in one reply by trying the type the
147/// caller wants against each of them, last first, and keeping the first that
148/// fits - which only works when a span that is not really the answer can
149/// fail to fit. A `Verdict` with no required field at all would make every
150/// span fit, including a `{}` left by stray trailing prose, and the reply's
151/// real `decisions` - earlier in the text - would never be reached. Requiring
152/// the key costs nothing: the prompt already asks for it on every reply, `[]`
153/// included.
154#[derive(Debug, Clone, Default, Deserialize)]
155pub struct Verdict {
156    /// One entry per task the conductor chose to say something about. A task
157    /// left out of this list is left exactly as it was.
158    pub decisions: Vec<Decision>,
159}
160
161/// A view of a task built for [`prompt::conduct`], shared by the runnable and
162/// stalled sections.
163fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
164    prompt::ConductTask {
165        id: t.id.clone(),
166        title: t.title.clone(),
167        instruction: t.instruction.clone(),
168        repo: t.repo.display().to_string(),
169        priority: t.priority,
170        status: t.status.as_str().to_owned(),
171        attempts: t.attempts,
172        max_attempts: t.effective_attempt_limit(max_attempts),
173        last_error: t.last_error.clone(),
174        hold_reason: t.hold_reason.clone(),
175        hold_source: t.hold_source.map(|source| source.label().to_owned()),
176        blocked_by: t.blocked_by.clone(),
177        answers: t
178            .answers
179            .iter()
180            .map(|a| prompt::ConductAnswer {
181                question: a.question.clone(),
182                answer: a.answer.clone(),
183            })
184            .collect(),
185        operator_resume: t.resumption.as_ref().map(|r| format!(
186            "Explicit operator resumption generation {} via {} at {}. Approved instruction: {}. \
187             The first {} historical answers and prior hold ({:?}) are superseded as waiting premises. \
188             To block again cite evidence ids from a later owned question, changed dependency, or new failed run; \
189             wording alone is insufficient.",
190            r.generation, r.source, r.at, r.instruction, r.answers_through, r.cleared_reason
191        )).or_else(|| t.resume_override.as_ref().map(|o| {
192            format!(
193                "the operator explicitly answered \"resume\" at {}; do not hold this \
194                 task again for the same reason unless there is new information",
195                o.at
196            )
197        })),
198    }
199}
200
201/// Did the operator answer a `magi ask` choice with a `resume` action that
202/// this task has not yet acted on? Then `Recovery::Requeue` must not turn the
203/// task into a fresh competition either.
204fn pinned_resume(task: &Task) -> bool {
205    task.resume_override
206        .as_ref()
207        .is_some_and(|o| o.pinned_run.is_some())
208}
209
210/// Verify references independently of the model's reason text. Old evidence
211/// cannot become new merely by being described differently.
212fn new_blocking_evidence(
213    queue: &Queue,
214    questions: &Questions,
215    task: &Task,
216    d: &Decision,
217) -> Vec<String> {
218    let Some(r) = &task.resumption else {
219        return Vec::new();
220    };
221    d.evidence
222        .iter()
223        .filter_map(|id| {
224            let version = if let Ok(q) = questions.get(id) {
225                (q.asked_at > r.at
226                    && (q.run == task.id || task.runs.contains(&q.run))
227                    && q.node != NODE
228                    && q.status != crate::ask::QuestionStatus::Abandoned)
229                    .then(|| {
230                        format!(
231                            "question:{id}:{}:{:?}:{}",
232                            q.asked_at,
233                            q.status,
234                            q.resolution().unwrap_or_default()
235                        )
236                    })
237            } else if d.blocked_by.contains(id) {
238                queue
239                    .get(id)
240                    .ok()
241                    .filter(|dep| {
242                        dep.id != task.id && dep.updated_at > r.at && dep.status != TaskStatus::Done
243                    })
244                    .map(|dep| {
245                        format!(
246                            "dependency:{id}:{:?}:{:?}:{:?}",
247                            dep.status, dep.hold_reason, dep.last_error
248                        )
249                    })
250            } else if task.runs.iter().any(|run| run == id) {
251                RunState::load(id)
252                    .ok()
253                    .filter(|run| {
254                        // A resumed run keeps its id. Only a pass started after
255                        // approval (or a newly created legacy run) is new evidence.
256                        run.updated_at > r.at
257                            && run
258                                .passes
259                                .last()
260                                .map_or(run.created_at > r.at, |p| p.started_at > r.at)
261                            && matches!(
262                                run.status,
263                                crate::run::RunStatus::Failed
264                                    | crate::run::RunStatus::Blocked
265                                    | crate::run::RunStatus::Stalled
266                            )
267                    })
268                    .map(|run| {
269                        format!(
270                            "run:{id}:{:?}:{}:{}:{:?}",
271                            run.status,
272                            run.reviews.len(),
273                            task.attempts,
274                            run.passes.last().map(|p| (p.index, p.started_at))
275                        )
276                    })
277            } else {
278                None
279            };
280            version.filter(|v| !r.blocking_evidence.contains(v))
281        })
282        .collect()
283}
284
285/// Durable approval is enforced by verified evidence at the mutation boundary.
286/// A pinned run stays protected; legacy transient overrides keep their original
287/// bounded contradiction policy until an explicit new approval replaces it.
288fn may_hold(task: &mut Task, reason: &str) -> bool {
289    if pinned_resume(task) {
290        return false;
291    }
292    if task.resumption.is_some() {
293        return true; // The mutation boundary already verified new evidence.
294    }
295    let Some(o) = task.resume_override.as_mut() else {
296        return true;
297    };
298    if o.forced || o.pinned_run.is_some() {
299        tracing::warn!(
300            "conductor tried to hold task {} after the operator forced a resume: {reason}",
301            task.id
302        );
303        return false;
304    }
305    if o.conductor_rehold.is_some() {
306        return false;
307    }
308    o.conductor_rehold = Some(reason.to_owned());
309    true
310}
311
312/// Severity as a lowercase word, matching how `crate::verdict::Severity` is
313/// spelled everywhere else an operator or a model reads it.
314fn severity_str(s: crate::verdict::Severity) -> &'static str {
315    match s {
316        crate::verdict::Severity::Nit => "nit",
317        crate::verdict::Severity::Minor => "minor",
318        crate::verdict::Severity::Major => "major",
319        crate::verdict::Severity::Blocker => "blocker",
320    }
321}
322
323/// The branch a task's last run left behind, if the tally ever ran on it -
324/// what [`Recovery::Review`] reopens. Derived from `crate::run::RunState`
325/// alone, never from anything the conductor wrote, so a hallucinated branch
326/// name can never reach `crate::graph::Runner::review`.
327fn surviving_branch(task: &Task) -> Option<String> {
328    let last = task.runs.last()?;
329    let state = RunState::load(last).ok()?;
330    if !crate::daemon::compatible_run_instruction(&state, task) {
331        return None;
332    }
333    state.winner().map(|c| c.branch.clone())
334}
335
336/// The reason to record when the conductor holds a `failed`/`held` task via
337/// [`Recovery::Hold`]. Always `Some`, never a bare `d.reason.clone()`.
338///
339/// [`Task::hold_machine`] only overwrites [`Task::hold_reason`] when given
340/// one, precisely so a stalled task's existing diagnosis survives a
341/// decision that has nothing new to add. That is the right default when the
342/// task was not already `held` - but a task that *is* already `held`, and
343/// is being held again here, is a different case: its current
344/// `hold_reason` may still read as a cause `crate::triage` knows how to
345/// re-check on its own (a disk-pressure message, say - see
346/// `triage::machine_cause_resolved`), and if the model gives no reason,
347/// leaving that text untouched would make this decision - the conductor
348/// choosing, informed by the operator's own answer, to keep the task held
349/// anyway - indistinguishable from the original, never-reconsidered hold.
350/// `crate::triage` would then read the stale text the next time its one
351/// recognised cause looks resolved and release the task straight through
352/// the decision this call was recording. Prepending (not appending) the new
353/// note keeps the old text as context without leaving the string starting
354/// with whatever pattern `crate::triage` matched before.
355///
356/// Two bounds keep the string from growing, which it once did without limit
357/// (every re-hold nested the whole previous reason one level deeper, and every
358/// change relit the task's notice): a note equal to the current reason - or to
359/// its outermost note - adds nothing, so the result is `None` and the caller
360/// writes nothing; and only ONE `(previously: ...)` level is ever kept, the
361/// prior being cut back to its outermost note before it is embedded.
362fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> Option<String> {
363    let note = match &d.reason {
364        Some(reason) => reason.clone(),
365        None => match task.answers.last() {
366            Some(a) => format!(
367                "conduct held this again with no new reason given; last operator \
368                 answer on record: {}",
369                a.answer
370            ),
371            None => "conduct held this again with no reason given".to_owned(),
372        },
373    };
374    match task.hold_reason.as_deref() {
375        Some(prior) if !prior.is_empty() => {
376            let outer = outermost_hold_note(prior);
377            if note.trim() == prior.trim() || note.trim() == outer {
378                None
379            } else {
380                Some(format!("{note}\n\n(previously: {outer})"))
381            }
382        }
383        _ => Some(note),
384    }
385}
386
387/// Separator [`reaffirmed_hold_reason`] puts between a note and the reason it
388/// replaced.
389const PREVIOUSLY: &str = "\n\n(previously: ";
390
391/// A hold reason without any `(previously: ...)` tail: the text before the
392/// first separator, trimmed.
393fn outermost_hold_note(reason: &str) -> &str {
394    reason.split(PREVIOUSLY).next().unwrap_or(reason).trim()
395}
396
397/// The conductor's stated reason for a hold, or a stand-in when it gave none.
398fn hold_note(d: &Decision) -> String {
399    d.reason
400        .clone()
401        .unwrap_or_else(|| "(no reason given)".to_owned())
402}
403
404/// Everything the conductor is shown about a `failed`/`held` task's last run.
405async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
406    let Some(run_id) = task.runs.last().cloned() else {
407        return prompt::ConductOutcome {
408            run_id: "(none)".to_owned(),
409            unreadable: Some("this task has not produced a run yet".to_owned()),
410            run_status: None,
411            open_findings: Vec::new(),
412            rounds_used: 0,
413            rounds_max: 0,
414            rounds: Vec::new(),
415            branch: None,
416            branch_head: None,
417            references: None,
418            empty_candidate: false,
419        };
420    };
421    let state = match RunState::load(&run_id) {
422        Ok(s) => s,
423        Err(e) => {
424            // The exact failure this feature exists to stop hiding: a schema
425            // mismatch (or any other unreadable state) must never be treated
426            // as "nothing to recover" - it is surfaced here, verbatim, rather
427            // than swallowed into a quiet re-competition.
428            tracing::warn!(
429                "conductor: could not read run {run_id} for task {}: {e:#}",
430                task.short()
431            );
432            return prompt::ConductOutcome {
433                run_id,
434                unreadable: Some(format!("{e:#}")),
435                run_status: None,
436                open_findings: Vec::new(),
437                rounds_used: 0,
438                rounds_max: 0,
439                rounds: Vec::new(),
440                branch: None,
441                branch_head: None,
442                references: None,
443                empty_candidate: false,
444            };
445        }
446    };
447
448    let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
449        id: f.id.clone(),
450        title: f.title.clone(),
451        severity: severity_str(f.severity).to_owned(),
452    };
453    let open_findings = state
454        .open_findings()
455        .into_iter()
456        .map(finding_view)
457        .collect();
458    let rounds = state
459        .reviews
460        .iter()
461        .map(|r| prompt::ConductRound {
462            round: r.round,
463            findings: r
464                .reviews
465                .iter()
466                .flat_map(|rec| rec.findings.iter())
467                .map(finding_view)
468                .collect(),
469            addressed: r
470                .fix
471                .as_ref()
472                .map(|fx| fx.addressed.clone())
473                .unwrap_or_default(),
474            rejected: r
475                .fix
476                .as_ref()
477                .map(|fx| {
478                    fx.rejected
479                        .iter()
480                        .map(|rej| prompt::ConductRejection {
481                            id: rej.id.clone(),
482                            why: rej.why.clone(),
483                        })
484                        .collect()
485                })
486                .unwrap_or_default(),
487        })
488        .collect();
489    let branch = state.winner().map(|c| c.branch.clone());
490    let branch_head = match &branch {
491        Some(b) => crate::git::rev_parse(repo, b)
492            .await
493            .ok()
494            .map(|h| h.chars().take(8).collect()),
495        None => None,
496    };
497
498    prompt::ConductOutcome {
499        run_id,
500        unreadable: None,
501        run_status: Some(state.status.as_str().to_owned()),
502        open_findings,
503        rounds_used: state.reviews.len(),
504        rounds_max: state.config.graph.review_rounds,
505        rounds,
506        branch,
507        branch_head,
508        references: crate::refs::describe(&state.seeds),
509        empty_candidate: state.merge.as_ref().is_some_and(|m| m.empty),
510    }
511}
512
513/// Whether the task's latest run's winner branch already has an open pull
514/// request into the run's own base - a fact the repository can answer, so the
515/// operator is never asked it. `None` when there is no run or winner to check.
516async fn open_pr_fact(task: &Task, repo: &Path) -> Option<String> {
517    let id = task.runs.last()?;
518    let state = match RunState::load(id) {
519        Ok(s) => s,
520        Err(e) => return Some(format!("could not check open pull requests: {e:#}")),
521    };
522    let branch = state.winner()?.branch.clone();
523    let base = state.base_branch.clone();
524    match crate::land::find_open_pr(repo, &branch, &base).await {
525        Ok(crate::land::OpenPr::None) => None,
526        Ok(crate::land::OpenPr::One { url, .. }) => Some(format!(
527            "pull request {url} is already open for {branch} into {base}"
528        )),
529        Ok(crate::land::OpenPr::Many(urls)) => Some(format!(
530            "several pull requests are already open for {branch} into {base}: {}",
531            urls.join(" ")
532        )),
533        Err(e) => Some(format!("could not check open pull requests: {e:#}")),
534    }
535}
536
537/// Append what the repository says about the branches and commits a task
538/// names to every question the conductor is about to file, so the operator is
539/// never asked a fact git can answer ("is it already on main?"). Done here,
540/// in code, rather than trusted to the model: a question the model wrote
541/// without checking still reaches the operator with the answer attached.
542/// When it cannot be checked the question says so instead of guessing.
543async fn attach_facts(cfg: &Config, repo: &Path, queue: &Queue, verdict: &mut Verdict) {
544    for d in verdict
545        .decisions
546        .iter_mut()
547        .filter(|d| d.question.is_some())
548    {
549        let Ok(task) = queue.get(&d.id) else {
550            continue;
551        };
552        let text = format!("{}\n{}", task.title, task.instruction);
553        let pr_fact = open_pr_fact(&task, repo_for(&task, repo).as_path()).await;
554        if crate::refs::scan(&text).is_empty() {
555            if let (Some(fact), Some(q)) = (pr_fact, d.question.as_mut()) {
556                q.push_str(&format!(
557                    "\n\nChecked against the repository (magi did this, not the model):\n{fact}"
558                ));
559            }
560            continue;
561        }
562        let repo = repo_for(&task, repo);
563        let remote = &cfg.merge.remote;
564        let base = crate::git::merge_base_branch(&repo, remote, cfg.merge.base.as_deref())
565            .await
566            .ok();
567        let facts = match base {
568            Some(base) => {
569                let base_name = base.clone();
570                let tracking = format!("{remote}/{base}");
571                // Best effort: a stale tracking ref would report work that
572                // has since landed as still missing from the base.
573                let refreshed = crate::git::fetch(&repo, remote, &base)
574                    .await
575                    .is_ok_and(|o| o.ok());
576                // Only the remote's tip counts: a local branch of the same
577                // name is whatever the checkout happens to hold.
578                let against = tracking;
579                match crate::git::rev_parse(&repo, &against).await {
580                    Ok(tip) => crate::refs::describe(
581                        &crate::refs::resolve(&repo, &tip, remote, &text).await,
582                    )
583                    .map(|facts| {
584                        if refreshed {
585                            facts
586                        } else {
587                            format!(
588                                "{facts}\n(could not fetch {remote}/{base_name}: this is \
589                                 against the last fetched `{against}`, which may be behind \
590                                 the remote)"
591                            )
592                        }
593                    }),
594                    Err(_) => Some(format!(
595                        "could NOT be checked against the repository: {against} is not \
596                         available here, so whether the work is in the base is unknown"
597                    )),
598                }
599            }
600            None => Some("could not check the repository: no base branch known".to_owned()),
601        };
602        let facts = match (facts, pr_fact) {
603            (Some(f), Some(p)) => Some(format!("{f}\n{p}")),
604            (f, p) => f.or(p),
605        };
606        if let (Some(facts), Some(q)) = (facts, d.question.as_mut()) {
607            q.push_str(&format!(
608                "\n\nChecked against the repository (magi did this, not the model):\n{facts}"
609            ));
610        }
611    }
612}
613
614/// A `failed`/`held` task together with how its last run ended.
615async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
616    prompt::ConductFinished {
617        task: view(t, max_attempts),
618        outcome: outcome_for(t, &repo_for(t, repo)).await,
619    }
620}
621
622/// The repository containing a task's branch. A task filed without a
623/// repository uses the daemon's repository, exactly as its later attempt does.
624fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
625    if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
626        fallback.to_path_buf()
627    } else {
628        task.repo.clone()
629    }
630}
631
632/// Apply one decision to the queue and the question store.
633///
634/// Takes the task's own claim before touching it: a model call spans a whole
635/// agent turn, and the queue can have moved on by the time its answer comes
636/// back. A claim that cannot be taken means something else owns this task
637/// right now - most often a live daemon mid-competition on it - so the
638/// conductor's now-stale view of it is dropped rather than raced against; see
639/// `crate::queue::Queue::claim`'s own doc on why a claim is proof, not a
640/// guess.
641///
642/// A decision is matched against the task's *current* status, re-read under
643/// the claim, not against whichever section of the prompt it came from: a
644/// `blocked_by` only takes effect on a `queued` task, and `recovery` only on
645/// one `running` (stalled) or `failed`/`held`, so a decision that no longer
646/// matches what the task actually is - it moved on between the read that
647/// built the prompt and this write - changes nothing.
648fn apply_one(
649    queue: &Queue,
650    questions: &Questions,
651    d: &Decision,
652    snapshot: Option<&Task>,
653) -> Result<()> {
654    let _claim = queue
655        .claim(&d.id)
656        .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
657    let mut task = queue.get(&d.id).context("no such task")?;
658
659    // Compare the actual input, not a generation echoed by the model. Even
660    // question creation must be discarded after an intervening operator edit.
661    if snapshot
662        .is_some_and(|old| serde_json::to_value(old).ok() != serde_json::to_value(&task).ok())
663    {
664        return Ok(());
665    }
666    let blocking =
667        d.question.is_some() || !d.blocked_by.is_empty() || d.recovery == Some(Recovery::Hold);
668    let evidence = new_blocking_evidence(queue, questions, &task, d);
669    if blocking && task.resumption.is_some() && evidence.is_empty() {
670        crate::notices::raise(
671            crate::notices::Notice::warn(
672                &format!("conductor-resumption:{}", task.id),
673                format!(
674                    "Conductor concern after explicit resumption (no verified new evidence): {}; question: {:?}; dependencies: {:?}; evidence: {:?}",
675                    hold_note(d), d.question, d.blocked_by, d.evidence
676                ),
677            )
678            .about([task.id.clone()]),
679        );
680        return Ok(());
681    }
682
683    if blocking {
684        if let Some(r) = task.resumption.as_mut() {
685            r.blocking_evidence.extend(evidence);
686        }
687    }
688
689    // A conductor answer is never operator authorization.  In particular,
690    // do this before questions and blocking too: either would reclassify a
691    // manual hold and let a later deterministic resolver queue it.
692    if task.operator_held() {
693        return Ok(());
694    }
695
696    // A claimed task may have deferred the pre-scheduling triage sweep.
697    // Leave an open or answered-unapplied question to that sweep rather
698    // than move the task out from under it.
699    if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
700        return Ok(());
701    }
702
703    // The operator's `resume` action is waiting for its turn: a question or a
704    // dependency would park the task before the daemon can run it.
705    let pinned = pinned_resume(&task);
706    if pinned && (d.question.is_some() || !d.blocked_by.is_empty()) {
707        return Ok(());
708    }
709
710    // A missing-only decision with unverifiable aliases is not evidence for
711    // a question or hold. Do not guess that two ambiguous/unknown references
712    // mean the same record, or treat failure to prove that as proof of absence.
713    if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
714        && !d.missing_dependencies.is_empty()
715        && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
716        && d.missing_dependencies
717            .iter()
718            .any(|id| !d.blocked_by.contains(id))
719        && d.missing_dependencies
720            .iter()
721            .chain(&d.blocked_by)
722            .any(|id| {
723                matches!(
724                    crate::queue::dependency(queue, questions, id),
725                    crate::queue::Dependency::Ambiguous | crate::queue::Dependency::Unknown
726                )
727            })
728    {
729        return Ok(());
730    }
731
732    // Only an explicit missing-only claim can justify replacing a question
733    // or hold. Existing prerequisites alone say nothing about independent
734    // consent, destructive-operation questions, or other reasons to stay held.
735    // Ambiguous or unreadable inventories cannot confirm absence either: keep
736    // the original reference waiting until it can be resolved authoritatively.
737    if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
738        && !d.missing_dependencies.is_empty()
739        && d.missing_dependencies.iter().all(|id| {
740            let claim = crate::queue::dependency(queue, questions, id);
741            claim != crate::queue::Dependency::Missing
742                && d.blocked_by.iter().any(|blocker| {
743                    blocker == id
744                        || (matches!(
745                            &claim,
746                            crate::queue::Dependency::Task(_)
747                                | crate::queue::Dependency::Question(_)
748                                | crate::queue::Dependency::Deleted(_)
749                        ) && crate::queue::dependency(queue, questions, blocker) == claim)
750                })
751        })
752        && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
753        && crate::queue::missing_blockers(queue, questions, &d.blocked_by).is_empty()
754    {
755        let mut ids = d.blocked_by.clone();
756        crate::queue::normalize_blockers(queue, questions, &mut ids);
757        let reason = format!("Waiting on dependencies: {}", ids.join(", "));
758        task.block(ids, Some(reason));
759        return queue.put(&mut task);
760    }
761
762    if let Some(text) = &d.question {
763        if task.status == TaskStatus::Done {
764            return Ok(());
765        }
766        // `Question::run` is a task id for conductor questions, but ordinary
767        // graph questions use it as a run id. A coincidental equality must
768        // not block this task on an answer meant for another node.
769        let question_id = match questions
770            .list()
771            .into_iter()
772            .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
773        {
774            Some(existing) => existing.id,
775            // No cap on settled answers: a question the operator can answer
776            // is never refused. The task goes `Blocked` and at most one
777            // conductor question is open at a time (reused above), so each
778            // further question costs one human answer. Run attempts are
779            // bounded by `max_attempts` and the queue's attempt accounting.
780            // A model rewording the same question can still recur once per
781            // answer; that cost falls on the operator, by choice.
782            None => {
783                let mut q = Question::new(
784                    task.id.clone(),
785                    NODE.to_owned(),
786                    SEAT.to_owned(),
787                    text.clone(),
788                    d.reason.clone().unwrap_or_default(),
789                    d.choices.clone(),
790                );
791                // The wait is handed to a deputy (`crate::deputy`), so a
792                // free-text reply reaches an agent that knows why this was
793                // asked. The conductor still never blocks: this only records
794                // what the deputy will be told, and `serve` starts it.
795                q.deputy = Some(Deputy::new(crate::deputy::brief(
796                    &task.id,
797                    d.reason.as_deref().unwrap_or_default(),
798                    &q.choices,
799                    &q.actions,
800                )));
801                if task.repo.is_dir() && task.repo != Path::new(".") {
802                    q.cwd = Some(task.repo.to_string_lossy().into_owned());
803                }
804                questions.put(&mut q)?;
805                q.id
806            }
807        };
808        task.block(vec![question_id], d.reason.clone());
809        return queue.put(&mut task);
810    }
811
812    match task.status {
813        // See `Decision::recovery`'s doc: the only lever a runnable task has
814        // besides `blocked_by` is holding it outright, for a task whose
815        // answers already say it should not compete again.
816        TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
817            if may_hold(&mut task, &hold_note(d)) {
818                task.hold_machine(d.reason.clone());
819                queue.put(&mut task)?;
820            }
821        }
822        TaskStatus::Queued if !d.blocked_by.is_empty() => {
823            let mut ids = d.blocked_by.clone();
824            crate::queue::normalize_blockers(queue, questions, &mut ids);
825            task.block(ids, d.reason.clone());
826            queue.put(&mut task)?;
827        }
828        TaskStatus::Running => match d.recovery {
829            Some(Recovery::Requeue) if pinned_resume(&task) => {}
830            Some(Recovery::Requeue) => {
831                task.requeue();
832                queue.put(&mut task)?;
833            }
834            Some(Recovery::Hold) if may_hold(&mut task, &hold_note(d)) => {
835                task.hold_machine(d.reason.clone());
836                queue.put(&mut task)?;
837            }
838            // `Review` reopens a branch, which only makes sense once a run
839            // has actually stopped; a task still `running` has nothing to
840            // reopen yet.
841            _ => {}
842        },
843        TaskStatus::Failed | TaskStatus::Held => match d.recovery {
844            // The operator picked a specific run to continue; a fresh
845            // competition would throw that run away.
846            Some(Recovery::Requeue) if pinned_resume(&task) => {}
847            Some(Recovery::Requeue) => {
848                task.requeue();
849                queue.put(&mut task)?;
850            }
851            Some(Recovery::Hold) => {
852                // Decided before `may_hold`, which spends the one override: a
853                // note that adds nothing writes nothing (and so raises no
854                // notice and moves no queue revision).
855                if let Some(reason) = reaffirmed_hold_reason(&task, d) {
856                    if may_hold(&mut task, &hold_note(d)) {
857                        task.hold_machine(Some(reason));
858                        queue.put(&mut task)?;
859                    }
860                }
861            }
862            Some(Recovery::Review) => {
863                if let Some(branch) = surviving_branch(&task) {
864                    task.request_review(branch);
865                    queue.put(&mut task)?;
866                }
867                // No survivable branch: a decision naming `review` here is
868                // simply not actionable, and is dropped rather than guessed
869                // at - `crate::daemon` applies the same "no branch, no
870                // review" rule again, from its own read, right before it
871                // would actually start the run.
872            }
873            Some(Recovery::Done) => {
874                task.succeed();
875                // Same reason `magi task done` and the web UI's equivalent
876                // do this: the conductor closing a task by hand is just as
877                // much a finished story as the loop's own settle path.
878                crate::daemon::supersede_prior_runs(&task, &crate::run::home());
879                queue.put(&mut task)?;
880            }
881            None => {}
882        },
883        // `queued` with nothing to block on, `done`, or already `blocked`:
884        // nothing for this decision to do.
885        _ => {}
886    }
887    Ok(())
888}
889
890/// Apply every decision in `verdict`. A single bad decision - a task id that
891/// no longer exists, one already claimed elsewhere - is logged and skipped
892/// rather than losing every other decision in the same reply.
893pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
894    for d in &verdict.decisions {
895        if let Err(e) = apply_one(queue, questions, d, None) {
896            tracing::warn!("conductor decision for task {}: {e:#}", d.id);
897        }
898    }
899    Ok(())
900}
901
902/// Where the conductor's seat is kept between turns, so the daemon waiter can
903/// resume it when the owner answers a question it asked.
904///
905/// A graph seat's session lives in its run's `RunState`; the conductor has no
906/// run, and until this file its seat existed only in memory.
907pub fn seat_path(home: &Path) -> PathBuf {
908    home.join("conduct").join("seat.json")
909}
910
911/// The seat the conductor last used, if it has taken a turn.
912pub fn load_seat(home: &Path) -> Option<SeatState> {
913    serde_json::from_str(&std::fs::read_to_string(seat_path(home)).ok()?).ok()
914}
915
916fn busy_path(home: &Path) -> PathBuf {
917    home.join("conduct").join("busy")
918}
919
920/// Is a conductor turn in flight right now?
921///
922/// A marker file rather than shared memory because the waiter reads it and the
923/// conductor is not necessarily in its process. A marker older than one turn's
924/// timeout (plus slack) was left by a turn that died and counts for nothing.
925pub fn busy(home: &Path) -> bool {
926    std::fs::metadata(busy_path(home))
927        .and_then(|m| m.modified())
928        .ok()
929        .and_then(|t| t.elapsed().ok())
930        .is_some_and(|age| age < TURN_TIMEOUT + Duration::from_secs(30))
931}
932
933/// Removes the busy marker when a turn ends, however it ends.
934struct Busy(PathBuf);
935
936impl Busy {
937    fn mark(home: &Path) -> Self {
938        let path = busy_path(home);
939        if let Some(dir) = path.parent() {
940            let _ = std::fs::create_dir_all(dir);
941        }
942        let _ = std::fs::write(&path, std::process::id().to_string());
943        Self(path)
944    }
945}
946
947impl Drop for Busy {
948    fn drop(&mut self) {
949        let _ = std::fs::remove_file(&self.0);
950    }
951}
952
953/// The conductor's state across polling cycles: its own CLI-side conversation
954/// and the last (revision, stalled ∪ finished ids) pair it actually acted on.
955#[derive(Debug, Default)]
956pub struct Conductor {
957    seat: Option<SeatState>,
958    last_seen: Option<(u64, BTreeSet<String>)>,
959    /// Content fingerprint of each held task the model was last shown, see
960    /// [`input_fingerprint`]. A held task whose fingerprint still matches is
961    /// settled: the conductor's own rewrites of it do not count as news.
962    considered: BTreeMap<String, String>,
963}
964
965/// What the conductor's decision about a held task is a function of: the task
966/// without the fields the conductor itself rewrites (`updated_at`,
967/// `hold_reason`, the recorded re-hold). Independent of how the model words a
968/// repeated decision.
969fn input_fingerprint(task: &Task) -> String {
970    let mut v = serde_json::to_value(task).unwrap_or_default();
971    if let Some(o) = v.as_object_mut() {
972        o.remove("updated_at");
973        o.remove("hold_reason");
974        if let Some(ro) = o.get_mut("resume_override").and_then(|r| r.as_object_mut()) {
975            ro.remove("conductor_rehold");
976        }
977    }
978    v.to_string()
979}
980
981impl Conductor {
982    /// A conductor that has never run.
983    #[must_use]
984    pub fn new() -> Self {
985        Self::default()
986    }
987
988    /// Held tasks among `stalled` / `finished` whose input is unchanged since
989    /// the model last saw them. Their files are left out of the revision, so
990    /// the conductor re-holding one (however it words it) is not news, while
991    /// an owner's answer changes the fingerprint and brings the file back in.
992    fn settled(
993        considered: &BTreeMap<String, String>,
994        stalled: &[Task],
995        finished: &[Task],
996    ) -> BTreeSet<String> {
997        stalled
998            .iter()
999            .chain(finished)
1000            .filter(|t| t.status == TaskStatus::Held)
1001            .filter(|t| considered.get(&t.id) == Some(&input_fingerprint(t)))
1002            .map(|t| t.id.clone())
1003            .collect()
1004    }
1005
1006    fn snapshot_with(
1007        considered: &BTreeMap<String, String>,
1008        queue: &Queue,
1009        stalled: &[Task],
1010        finished: &[Task],
1011    ) -> (u64, BTreeSet<String>) {
1012        let ids = stalled
1013            .iter()
1014            .chain(finished)
1015            .map(|t| t.id.clone())
1016            .collect();
1017        let skip = Self::settled(considered, stalled, finished);
1018        (queue.revision_excluding(&skip), ids)
1019    }
1020
1021    fn snapshot(
1022        &self,
1023        queue: &Queue,
1024        stalled: &[Task],
1025        finished: &[Task],
1026    ) -> (u64, BTreeSet<String>) {
1027        Self::snapshot_with(&self.considered, queue, stalled, finished)
1028    }
1029
1030    /// Whether calling the conductor could possibly do anything different
1031    /// from last time: [`Queue::revision`] moved, or the set of stalled and
1032    /// finished task ids changed.
1033    ///
1034    /// Deliberately **not** "stalled or finished is non-empty" - a task
1035    /// sitting stalled or finished with nobody changing anything about it
1036    /// must not be re-shown to the model every single poll forever; only a
1037    /// change in *which* tasks are stalled or finished, or a queue write
1038    /// changing something about a runnable one, is worth another look.
1039    ///
1040    /// Cheap and config-free on purpose, so `crate::daemon`'s poll loop can
1041    /// skip `Config::discover`'s synchronous I/O entirely on a cycle where
1042    /// this says no - which [`Conductor::maybe_run`] would otherwise only
1043    /// discover after paying for that load. Both ask the identical question,
1044    /// from the same [`Conductor::last_seen`], so they can never disagree
1045    /// about whether there is anything to look at.
1046    #[must_use]
1047    pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
1048        self.last_seen.as_ref() != Some(&self.snapshot(queue, stalled, finished))
1049    }
1050
1051    /// Call the conductor once, unless nothing has changed since the last
1052    /// time it was worth calling - see [`Conductor::worth_a_look`], the exact
1053    /// same test. Never fatal - see this module's doc.
1054    #[allow(clippy::too_many_arguments)]
1055    pub async fn maybe_run(
1056        &mut self,
1057        cfg: &Config,
1058        repo: &Path,
1059        queue: &Queue,
1060        questions: &Questions,
1061        home: &Path,
1062        queued: &[Task],
1063        stalled: &[Task],
1064        finished: &[Task],
1065        max_attempts: usize,
1066    ) {
1067        if self.last_seen.as_ref() == Some(&self.snapshot(queue, stalled, finished)) {
1068            return;
1069        }
1070        // What the model is about to be shown becomes "considered" before the
1071        // snapshot is taken, so its own re-hold writes land in the excluded set.
1072        self.considered = stalled
1073            .iter()
1074            .chain(finished)
1075            .filter(|t| t.status == TaskStatus::Held)
1076            .map(|t| (t.id.clone(), input_fingerprint(t)))
1077            .collect();
1078        self.last_seen = Some(self.snapshot(queue, stalled, finished));
1079        if let Err(e) = self
1080            .run_once(
1081                cfg,
1082                repo,
1083                queue,
1084                questions,
1085                home,
1086                queued,
1087                stalled,
1088                finished,
1089                max_attempts,
1090            )
1091            .await
1092        {
1093            // The conductor's own write moves the revision, so one more cycle
1094            // follows; a repeat decision writes nothing (see
1095            // `reaffirmed_hold_reason`), which is what ends the loop. The
1096            // revision is deliberately never re-read here: it cannot tell our
1097            // write from an owner's concurrent update.
1098            tracing::warn!("conductor: {e:#}");
1099        }
1100    }
1101
1102    #[allow(clippy::too_many_arguments)]
1103    async fn run_once(
1104        &mut self,
1105        cfg: &Config,
1106        repo: &Path,
1107        queue: &Queue,
1108        questions: &Questions,
1109        home: &Path,
1110        queued: &[Task],
1111        stalled: &[Task],
1112        finished: &[Task],
1113        max_attempts: usize,
1114    ) -> Result<()> {
1115        if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
1116            return Ok(());
1117        }
1118
1119        let primary = cfg
1120            .resolve_roles()
1121            .context("resolving the conductor seat")?
1122            .conductor;
1123        // A string keeps today's resolution exactly (no CLI preflight here;
1124        // an unavailable CLI fails at invocation). Only a multi-id array
1125        // builds a chain, whose members are each tried at most once.
1126        let chain = match cfg.roles.conductor.as_ref() {
1127            Some(c) if c.ids().len() > 1 => {
1128                agent::pick_chain(&cfg.agents, Some(c), &agent::installed, "conductor")?
1129            }
1130            _ => vec![primary],
1131        };
1132
1133        let runnable_views: Vec<prompt::ConductTask> =
1134            queued.iter().map(|t| view(t, max_attempts)).collect();
1135        let stalled_views: Vec<prompt::ConductTask> =
1136            stalled.iter().map(|t| view(t, max_attempts)).collect();
1137        let mut finished_views = Vec::with_capacity(finished.len());
1138        for t in finished {
1139            finished_views.push(finished_view(t, repo, max_attempts).await);
1140        }
1141
1142        let body = prompt::with_overlay(
1143            prompt::conduct(
1144                &runnable_views,
1145                &stalled_views,
1146                &finished_views,
1147                &cfg.graph.language,
1148            ),
1149            cfg.prompts.overlay(NODE),
1150        );
1151
1152        let body = format!(
1153            "{body}\n\n{}",
1154            crate::queue::dependency_facts(queue, questions)
1155        );
1156        let artifacts = home.join("conduct").join("artifacts");
1157        // Bound to a local: `Invocation` only borrows the cache path, and the
1158        // `Option<PathBuf>` `cache_dir()` returns has to outlive that borrow.
1159        let cache_dir = cfg.cache_dir();
1160
1161        // One marker for the whole chain: a fallback is still this turn.
1162        let busy = Busy::mark(home);
1163        let mut result: Result<Verdict> = Err(anyhow::anyhow!("no conductor agent ran"));
1164        for spec in &chain {
1165            // The prompt carries the whole queue state every cycle, so a
1166            // fallback agent needs no history - but its seat is its own.
1167            let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
1168            if needs_new_seat {
1169                self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
1170            }
1171            let seat = self.seat.as_mut().expect("just ensured a seat exists");
1172            // The first agent keeps the plain stem; a fallback's own artifacts
1173            // must not overwrite it.
1174            let stem = if std::ptr::eq(spec, &chain[0]) {
1175                format!("turn-{}", seat.turns + 1)
1176            } else {
1177                format!("turn-{}-{}", seat.turns + 1, spec.id)
1178            };
1179            let inv = Invocation {
1180                cwd: repo,
1181                prompt: &body,
1182                timeout: TURN_TIMEOUT,
1183                // The conductor never edits anything - it only decides what
1184                // blocks a task and what to do about one stuck or finished.
1185                allow_write: false,
1186                unsandboxed: false,
1187                sessions: cfg.graph.sessions,
1188                artifacts: &artifacts,
1189                stem: &stem,
1190                run: NODE,
1191                node: NODE,
1192                cache_dir: cache_dir.as_deref(),
1193                attachments: &[],
1194                writable: &[],
1195            };
1196            let out = agent::invoke(spec, seat, &inv).await;
1197            // Kept after every turn, failed or not: the CLI-side conversation is
1198            // what a resumed question needs, and it exists once the turn ran.
1199            if let Ok(body) = serde_json::to_string(&*seat) {
1200                let path = seat_path(home);
1201                if let Some(dir) = path.parent() {
1202                    let _ = std::fs::create_dir_all(dir);
1203                }
1204                let _ = std::fs::write(path, body);
1205            }
1206            let advance = agent::chain_advances(&out);
1207            result = match out {
1208                Err(e) => Err(e.context("invoking the conductor")),
1209                Ok(out) if advance => Err(anyhow::anyhow!(
1210                    "no usable reply (exit {:?}, timed out {})",
1211                    out.exit_code,
1212                    out.timed_out
1213                )),
1214                Ok(out) => verdict::extract_json(&out.text)
1215                    .context("the conductor's reply could not be parsed"),
1216            };
1217            if result.is_ok() {
1218                break;
1219            }
1220            if chain.len() > 1 {
1221                tracing::warn!("conductor: `{}` failed, trying the next agent", spec.id);
1222            }
1223        }
1224        drop(busy);
1225        let mut verdict = result?;
1226        attach_facts(cfg, repo, queue, &mut verdict).await;
1227        for d in &verdict.decisions {
1228            let snapshot = queued
1229                .iter()
1230                .chain(stalled)
1231                .chain(finished)
1232                .find(|t| t.id == d.id);
1233            if let Some(snapshot) = snapshot {
1234                if let Err(e) = apply_one(queue, questions, d, Some(snapshot)) {
1235                    tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1236                }
1237            }
1238        }
1239        Ok(())
1240    }
1241}
1242
1243#[cfg(test)]
1244mod tests {
1245    use std::collections::BTreeMap;
1246
1247    use tempfile::tempdir;
1248
1249    use super::*;
1250    use crate::ask::{Answer, QuestionStatus};
1251    use crate::config::{AgentKind, AgentSpec, Graph};
1252    use crate::queue::Source;
1253
1254    fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
1255        let path = dir.join("mock-conduct-agent.sh");
1256        std::fs::write(&path, script).expect("write mock");
1257        AgentSpec {
1258            id: "mock".to_owned(),
1259            kind: AgentKind::Command,
1260            model: None,
1261            command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
1262            extra_args: Vec::new(),
1263            env,
1264            prompt_delivery: None,
1265        }
1266    }
1267
1268    fn config(spec: AgentSpec) -> Config {
1269        Config {
1270            agents: vec![spec],
1271            graph: Graph {
1272                language: "en".to_owned(),
1273                ..Graph::default()
1274            },
1275            ..Config::default()
1276        }
1277    }
1278
1279    fn task(title: &str) -> Task {
1280        Task::new(
1281            title.to_owned(),
1282            format!("do {title}"),
1283            std::path::PathBuf::from("."),
1284            Source::Human,
1285        )
1286    }
1287
1288    const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
1289    const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
1290
1291    fn env(reply: &str) -> BTreeMap<String, String> {
1292        BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
1293    }
1294
1295    const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
1296
1297    /// A throwaway repo with one commit on `main` and a second branch ahead
1298    /// of it, so `outcome_for`'s own `git::rev_parse` call has a real head to
1299    /// resolve.
1300    fn init_repo_with_branch(dir: &Path, branch: &str) {
1301        use crate::proc::Quiet as _;
1302        let run = |args: &[&str]| {
1303            let out = std::process::Command::new("git")
1304                .args(args)
1305                .current_dir(dir)
1306                .quiet()
1307                .output()
1308                .expect("spawn git");
1309            assert!(
1310                out.status.success(),
1311                "git {args:?} failed: {}",
1312                String::from_utf8_lossy(&out.stderr)
1313            );
1314        };
1315        run(&["init", "-b", "main"]);
1316        run(&["config", "user.name", "magi test"]);
1317        run(&["config", "user.email", "magi@example.com"]);
1318        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
1319        run(&["add", "-A"]);
1320        run(&["commit", "-m", "init"]);
1321        run(&["checkout", "-b", branch]);
1322        std::fs::write(dir.join("change.txt"), "x\n").unwrap();
1323        run(&["add", "-A"]);
1324        run(&["commit", "-m", "candidate work"]);
1325    }
1326
1327    fn review_round_with_finding(
1328        round: usize,
1329        finding_id: &str,
1330        title: &str,
1331        addressed: &[&str],
1332        rejected: &[(&str, &str)],
1333    ) -> crate::run::ReviewRound {
1334        crate::run::ReviewRound {
1335            round,
1336            head: "deadbeef".to_owned(),
1337            verified_head: None,
1338            verified_at: None,
1339            reviews: vec![crate::run::ReviewRecord {
1340                attempts: 0,
1341                reviewer: 1,
1342                agent: "mock".to_owned(),
1343                summary: String::new(),
1344                findings: vec![crate::verdict::Finding {
1345                    id: finding_id.to_owned(),
1346                    severity: crate::verdict::Severity::Major,
1347                    file: None,
1348                    line: None,
1349                    title: title.to_owned(),
1350                    detail: String::new(),
1351                }],
1352                vote: None,
1353                failed: None,
1354                duration_ms: 0,
1355            }],
1356            e2e: Vec::new(),
1357            verify_retried: false,
1358            e2e_deferred: false,
1359            e2e_defer_reason: None,
1360            fix: Some(crate::run::FixRecord {
1361                agent: "mock".to_owned(),
1362                addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
1363                rejected: rejected
1364                    .iter()
1365                    .map(|(id, why)| crate::verdict::Rejection {
1366                        id: (*id).to_owned(),
1367                        why: (*why).to_owned(),
1368                    })
1369                    .collect(),
1370                notes: String::new(),
1371                committed: false,
1372                failed: None,
1373                duration_ms: 0,
1374                continuation: None,
1375            }),
1376            blocking: 1,
1377            answered: 1,
1378            expected: 1,
1379            clean: false,
1380            progressed: true,
1381            vote_split: false,
1382            reconsideration: Vec::new(),
1383            verdict: None,
1384        }
1385    }
1386
1387    #[test]
1388    fn explicit_resumption_supersedes_history_and_rejects_stale_decisions() {
1389        crate::run::pin_test_home();
1390        let dir = tempdir().unwrap();
1391        let queue = Queue::at(dir.path().join("queue"));
1392        let questions = Questions::at(dir.path().join("questions"));
1393        let mut t = task("recover");
1394        t.answers.push(crate::queue::AnsweredQuestion {
1395            question: "Inspect remote first?".into(),
1396            answer: "Inspect, then await resumption instructions".into(),
1397        });
1398        t.hold_machine(Some("waiting for remote inspection".into()));
1399        queue.put(&mut t).unwrap();
1400        let snapshot = queue.get(&t.id).unwrap();
1401        t.edit(
1402            "recover".into(),
1403            "Reuse the measured PR head; fix Clippy; review".into(),
1404        )
1405        .unwrap();
1406        t.release_operator("cli", None);
1407        queue.put(&mut t).unwrap();
1408        let approval = t.resumption.clone();
1409        let d = Decision {
1410            id: t.id.clone(),
1411            recovery: Some(Recovery::Hold),
1412            reason: Some("remote inspection still required".into()),
1413            ..Decision::default()
1414        };
1415        apply_one(&queue, &questions, &d, Some(&snapshot)).unwrap();
1416        let stale_question = Decision {
1417            question: Some("Inspect remote?".into()),
1418            ..d.clone()
1419        };
1420        apply_one(&queue, &questions, &stale_question, Some(&snapshot)).unwrap();
1421        assert!(questions.list().is_empty());
1422        for reason in [
1423            "waiting for remote inspection",
1424            "inspection has not been confirmed",
1425        ] {
1426            let repeated = Decision {
1427                reason: Some(reason.into()),
1428                ..d.clone()
1429            };
1430            apply(
1431                &queue,
1432                &questions,
1433                &Verdict {
1434                    decisions: vec![repeated],
1435                },
1436            )
1437            .unwrap();
1438            assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1439        }
1440        let reloaded = Queue::at(dir.path().join("queue")).get(&t.id).unwrap();
1441        assert_eq!(reloaded.resumption, approval);
1442        assert_eq!(reloaded.answers, t.answers);
1443        let v = view(&reloaded, 3);
1444        assert!(v.operator_resume.unwrap().contains("superseded"));
1445        let mut dep = task("new prerequisite");
1446        dep.hold_machine(Some("new authentication failure".into()));
1447        queue.put(&mut dep).unwrap();
1448        let new_blocker = Decision {
1449            blocked_by: vec![dep.id.clone()],
1450            evidence: vec![dep.id.clone()],
1451            // Exercise dependency waiting, without inheriting a separate hold.
1452            recovery: None,
1453            reason: Some("new prerequisite failed".into()),
1454            ..d.clone()
1455        };
1456        apply(
1457            &queue,
1458            &questions,
1459            &Verdict {
1460                decisions: vec![new_blocker.clone()],
1461            },
1462        )
1463        .unwrap();
1464        let mut after = queue.get(&t.id).unwrap();
1465        assert_eq!(after.status, TaskStatus::Blocked);
1466        // Automatic release does not grant fresh operator authority or make
1467        // the same unchanged evidence reusable.
1468        after.release();
1469        queue.put(&mut after).unwrap();
1470        apply(
1471            &queue,
1472            &questions,
1473            &Verdict {
1474                decisions: vec![new_blocker],
1475            },
1476        )
1477        .unwrap();
1478        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1479        after.start("new-run".into());
1480        assert_eq!(after.resumption.as_ref().unwrap().generation, 1);
1481        after.hold_manual(Some("operator says wait".into()));
1482        queue.put(&mut after).unwrap();
1483        apply(&queue, &questions, &Verdict { decisions: vec![d] }).unwrap();
1484        assert!(queue.get(&t.id).unwrap().operator_held());
1485    }
1486
1487    #[test]
1488    fn resumed_run_evidence_uses_approval_time_not_run_list_offsets() {
1489        crate::run::pin_test_home();
1490        let dir = tempdir().unwrap();
1491        let queue = Queue::at(dir.path().join("queue"));
1492        let questions = Questions::at(dir.path().join("questions"));
1493        let mut t = task("resume evidence");
1494        let mut run = RunState::new(
1495            t.repo.clone(),
1496            "main".into(),
1497            "base".into(),
1498            "work".into(),
1499            crate::config::Config::default(),
1500        );
1501        run.begin_pass();
1502        run.status = crate::run::RunStatus::Failed;
1503        run.save().unwrap();
1504        t.start(run.id.clone());
1505        // Model legacy duplicate history, which the next start will deduplicate.
1506        t.runs.push(run.id.clone());
1507        t.release_operator("cli", None);
1508        let approval = t.resumption.as_ref().unwrap().at;
1509        let decision = Decision {
1510            evidence: vec![run.id.clone()],
1511            ..Decision::default()
1512        };
1513        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1514        t.start(run.id.clone());
1515        assert_eq!(t.runs.len(), 1);
1516        run.begin_pass();
1517        run.save().unwrap();
1518        assert!(run.passes.last().unwrap().started_at > approval);
1519        let evidence = new_blocking_evidence(&queue, &questions, &t, &decision);
1520        assert_eq!(evidence.len(), 1);
1521        t.resumption.as_mut().unwrap().blocking_evidence = evidence;
1522        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1523        run.begin_pass();
1524        run.save().unwrap();
1525        assert_eq!(
1526            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1527            1
1528        );
1529
1530        let mut q = Question::new(
1531            run.id.clone(),
1532            "review".into(),
1533            "check".into(),
1534            "New failure".into(),
1535            String::new(),
1536            Vec::new(),
1537        );
1538        questions.put(&mut q).unwrap();
1539        let decision = Decision {
1540            evidence: vec![q.id.clone()],
1541            ..Decision::default()
1542        };
1543        assert_eq!(
1544            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1545            1
1546        );
1547        q.asked_at = approval;
1548        questions.put(&mut q).unwrap();
1549        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1550        q.asked_at = jiff::Timestamp::now();
1551        q.run = "unrelated".into();
1552        questions.put(&mut q).unwrap();
1553        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1554
1555        let mut fresh = RunState::new(
1556            t.repo.clone(),
1557            "main".into(),
1558            "base".into(),
1559            "fresh work".into(),
1560            crate::config::Config::default(),
1561        );
1562        fresh.status = crate::run::RunStatus::Failed;
1563        fresh.save().unwrap();
1564        t.start(fresh.id.clone());
1565        let decision = Decision {
1566            evidence: vec![fresh.id.clone()],
1567            ..Decision::default()
1568        };
1569        assert_eq!(
1570            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1571            1
1572        );
1573    }
1574
1575    #[test]
1576    fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
1577        crate::run::pin_test_home();
1578        let dir = tempdir().unwrap();
1579        let default_repo = dir.path().join("default");
1580        let task_repo = dir.path().join("task");
1581        std::fs::create_dir_all(&default_repo).unwrap();
1582        std::fs::create_dir_all(&task_repo).unwrap();
1583        init_repo_with_branch(&default_repo, "other-branch");
1584        init_repo_with_branch(&task_repo, "magi/f00d/A");
1585
1586        let mut config = Config::default();
1587        config.graph.review_rounds = 6;
1588        let mut state = crate::run::RunState::new(
1589            task_repo.clone(),
1590            "main".to_owned(),
1591            "deadbeef".to_owned(),
1592            "task".to_owned(),
1593            config,
1594        );
1595        state.status = crate::run::RunStatus::Blocked;
1596        state.candidates.push(crate::run::Candidate {
1597            index: 0,
1598            label: 'A',
1599            agent: "mock".to_owned(),
1600            branch: "magi/f00d/A".to_owned(),
1601            worktree: task_repo.clone(),
1602            summary: String::new(),
1603            stat: String::new(),
1604            files: 1,
1605            commits: 1,
1606            empty: false,
1607            failed: None,
1608            verified_noop: None,
1609            duration_ms: 0,
1610            folded: false,
1611        });
1612        state.tally = Some(crate::run::Tally {
1613            first_choice: std::collections::BTreeMap::new(),
1614            borda: std::collections::BTreeMap::new(),
1615            winner: 'A',
1616            rankings: 0,
1617            unanimous_initial: false,
1618            deliberated: false,
1619            changed_votes: 0,
1620            unanimous_final: false,
1621            tie_break: None,
1622            judges: 0,
1623            present: 0,
1624            quorum: 0,
1625            met_quorum: true,
1626            uncontested: Some("solo".to_owned()),
1627        });
1628        state.reviews = vec![
1629            review_round_with_finding(
1630                1,
1631                "R1-1-2",
1632                "answer content is dropped",
1633                &[],
1634                &[("R1-1-2", "the id leaving blocked_by is enough")],
1635            ),
1636            review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
1637        ];
1638        state.save().unwrap();
1639
1640        let mut t = task("outcome test");
1641        t.repo = task_repo;
1642        t.runs.push(state.id.clone());
1643
1644        let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
1645        let outcome = finished.outcome;
1646
1647        assert!(outcome.unreadable.is_none());
1648        assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
1649        assert_eq!(outcome.rounds_used, 2);
1650        assert_eq!(outcome.rounds_max, 6);
1651        assert_eq!(outcome.rounds.len(), 2);
1652        assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
1653        assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
1654        assert!(outcome.rounds[1].addressed.is_empty());
1655        assert!(outcome.rounds[1].rejected.is_empty());
1656        assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
1657        assert!(
1658            outcome.branch_head.is_some(),
1659            "a real branch must resolve a head commit: {outcome:?}"
1660        );
1661    }
1662
1663    /// A tiny single-threaded block-on, so an `async fn` can be exercised
1664    /// from a plain `#[test]` without pulling `tokio::test`'s multi-thread
1665    /// runtime into a test that does no other async work.
1666    fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
1667        tokio::runtime::Builder::new_current_thread()
1668            .enable_all()
1669            .build()
1670            .unwrap()
1671            .block_on(f)
1672    }
1673
1674    #[test]
1675    fn view_resolves_task_attempt_limit_before_daemon_default() {
1676        let mut t = task("budget");
1677        assert_eq!(view(&t, 7).max_attempts, 7);
1678        t.change_attempt_limit(std::num::NonZeroUsize::new(3))
1679            .unwrap();
1680        assert_eq!(view(&t, 7).max_attempts, 3);
1681        t.change_attempt_limit(None).unwrap();
1682        assert_eq!(view(&t, 7).max_attempts, 7);
1683    }
1684
1685    #[test]
1686    fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
1687        let mut t = task("answered");
1688        t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
1689        let v = view(&t, 2);
1690        assert_eq!(v.answers.len(), 1);
1691        assert_eq!(v.answers[0].question, "Which backend?");
1692        assert_eq!(v.answers[0].answer, "SQLite");
1693    }
1694
1695    #[tokio::test]
1696    async fn dependency_resolution_rechecks_questions_in_the_production_conductor_path() {
1697        for held in [false, true] {
1698            for reference in ["ca5b", "20261013-193611-ca5b"] {
1699                for question in [false, true] {
1700                    let dir = tempdir().unwrap();
1701                    let queue = Queue::at(dir.path().join("queue"));
1702                    let questions = Questions::at(dir.path().join("questions"));
1703                    let mut dep = task("prerequisite");
1704                    dep.id = "20261013-193611-ca5b".into();
1705                    dep.start("existing-run".into());
1706                    queue.put(&mut dep).unwrap();
1707                    let mut waiting = task("waiting");
1708                    if held {
1709                        waiting.hold_machine(Some("dependency could not be located".into()));
1710                    }
1711                    queue.put(&mut waiting).unwrap();
1712                    let missing_reference = if reference == "ca5b" {
1713                        dep.id.as_str()
1714                    } else {
1715                        "ca5b"
1716                    };
1717                    let reply = serde_json::json!({"decisions": [{
1718                        "id": waiting.id, "blocked_by": [reference],
1719                        "missing_dependencies": [missing_reference],
1720                        "question": question.then_some("ca5b is missing; where is its replacement?"),
1721                        "recovery": (!question).then_some("hold"),
1722                        "reason": "ca5b is missing"
1723                    }]}).to_string();
1724                    let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1725                    let (queued, finished) = if held {
1726                        (vec![], vec![waiting.clone()])
1727                    } else {
1728                        (vec![waiting.clone()], vec![])
1729                    };
1730                    Conductor::new()
1731                        .run_once(
1732                            &cfg,
1733                            dir.path(),
1734                            &queue,
1735                            &questions,
1736                            dir.path(),
1737                            &queued,
1738                            &[],
1739                            &finished,
1740                            2,
1741                        )
1742                        .await
1743                        .unwrap();
1744                    let after = queue.get(&waiting.id).unwrap();
1745                    assert_eq!(after.status, TaskStatus::Blocked);
1746                    assert_eq!(after.blocked_by, [dep.id]);
1747                    assert_eq!(after.blocked_from, Some(waiting.status));
1748                    assert!(!after.block_reason.unwrap().contains("missing"));
1749                    assert!(questions.list().is_empty());
1750                    assert_eq!(after.instruction, waiting.instruction);
1751                }
1752            }
1753        }
1754    }
1755
1756    #[tokio::test]
1757    async fn dependency_resolution_inconclusive_missing_only_decisions_keep_waiting() {
1758        for held in [false, true] {
1759            for unknown in [false, true] {
1760                for question in [false, true] {
1761                    let dir = tempdir().unwrap();
1762                    let queue = Queue::at(dir.path().join("queue"));
1763                    let questions = Questions::at(dir.path().join("questions"));
1764                    let mut dep = task("running prerequisite");
1765                    dep.id = "20261013-193611-ca5b".into();
1766                    dep.start("existing-run".into());
1767                    queue.put(&mut dep).unwrap();
1768                    let reference = if unknown {
1769                        // Even an exact, existing task id is Unknown when
1770                        // the question inventory cannot be scanned.
1771                        std::fs::write(questions.root(), "not a directory").unwrap();
1772                        dep.id.clone()
1773                    } else {
1774                        let mut other = task("another prerequisite");
1775                        other.id = "20261014-193611-ca5b".into();
1776                        queue.put(&mut other).unwrap();
1777                        "ca5b".to_owned()
1778                    };
1779                    assert_eq!(
1780                        crate::queue::dependency(&queue, &questions, &reference),
1781                        if unknown {
1782                            crate::queue::Dependency::Unknown
1783                        } else {
1784                            crate::queue::Dependency::Ambiguous
1785                        }
1786                    );
1787                    let mut waiting = task("waiting");
1788                    if held {
1789                        waiting.hold_machine(Some("dependency could not be located".into()));
1790                    }
1791                    queue.put(&mut waiting).unwrap();
1792                    let reply = serde_json::json!({"decisions": [{
1793                        "id": waiting.id, "blocked_by": [reference],
1794                        "missing_dependencies": [reference],
1795                        "question": question.then_some("ca5b is missing"),
1796                        "recovery": (!question).then_some("hold"),
1797                        "reason": "ca5b is missing"
1798                    }]})
1799                    .to_string();
1800                    let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1801                    let (queued, finished) = if held {
1802                        (vec![], vec![waiting.clone()])
1803                    } else {
1804                        (vec![waiting.clone()], vec![])
1805                    };
1806                    Conductor::new()
1807                        .run_once(
1808                            &cfg,
1809                            dir.path(),
1810                            &queue,
1811                            &questions,
1812                            dir.path(),
1813                            &queued,
1814                            &[],
1815                            &finished,
1816                            2,
1817                        )
1818                        .await
1819                        .unwrap();
1820                    let after = queue.get(&waiting.id).unwrap();
1821                    assert_eq!(after.status, TaskStatus::Blocked);
1822                    assert_eq!(after.blocked_by, [reference]);
1823                    assert_eq!(after.blocked_from, Some(waiting.status));
1824                    assert!(!after.block_reason.unwrap().contains("missing"));
1825                    assert_eq!(after.instruction, waiting.instruction);
1826                    if unknown {
1827                        assert!(questions.root().is_file());
1828                    } else {
1829                        assert!(questions.list().is_empty());
1830                    }
1831                }
1832            }
1833        }
1834    }
1835
1836    #[tokio::test]
1837    async fn dependency_resolution_unverifiable_alias_decisions_are_discarded() {
1838        for held in [false, true] {
1839            for unknown in [false, true] {
1840                for question in [false, true] {
1841                    for short_blocker in [false, true] {
1842                        let dir = tempdir().unwrap();
1843                        let queue = Queue::at(dir.path().join("queue"));
1844                        let questions = Questions::at(dir.path().join("questions"));
1845                        let mut dep = task("running prerequisite");
1846                        dep.id = "20261013-193611-ca5b".into();
1847                        dep.start("existing-run".into());
1848                        queue.put(&mut dep).unwrap();
1849                        if unknown {
1850                            std::fs::write(questions.root(), "not a directory").unwrap();
1851                        } else {
1852                            let mut other = task("another prerequisite");
1853                            other.id = "20261014-193611-ca5b".into();
1854                            queue.put(&mut other).unwrap();
1855                        }
1856                        let (blocker, missing) = if short_blocker {
1857                            ("ca5b", dep.id.as_str())
1858                        } else {
1859                            (dep.id.as_str(), "ca5b")
1860                        };
1861                        let mut waiting = task("waiting");
1862                        if held {
1863                            waiting.hold_machine(Some("dependency could not be located".into()));
1864                        }
1865                        queue.put(&mut waiting).unwrap();
1866                        let reply = serde_json::json!({"decisions": [{
1867                            "id": waiting.id, "blocked_by": [blocker],
1868                            "missing_dependencies": [missing],
1869                            "question": question.then_some("ca5b is missing"),
1870                            "recovery": (!question).then_some("hold"),
1871                            "reason": "ca5b is missing"
1872                        }]})
1873                        .to_string();
1874                        let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1875                        let (queued, finished) = if held {
1876                            (vec![], vec![waiting.clone()])
1877                        } else {
1878                            (vec![waiting.clone()], vec![])
1879                        };
1880                        Conductor::new()
1881                            .run_once(
1882                                &cfg,
1883                                dir.path(),
1884                                &queue,
1885                                &questions,
1886                                dir.path(),
1887                                &queued,
1888                                &[],
1889                                &finished,
1890                                2,
1891                            )
1892                            .await
1893                            .unwrap();
1894                        assert_eq!(
1895                            serde_json::to_value(queue.get(&waiting.id).unwrap()).unwrap(),
1896                            serde_json::to_value(&waiting).unwrap(),
1897                            "unverifiable aliases must leave the entire task unchanged"
1898                        );
1899                        if unknown {
1900                            assert!(questions.root().is_file());
1901                        } else {
1902                            assert!(questions.list().is_empty());
1903                        }
1904                    }
1905                }
1906            }
1907        }
1908    }
1909
1910    #[tokio::test]
1911    async fn dependency_resolution_genuine_missing_reference_still_asks() {
1912        let dir = tempdir().unwrap();
1913        let queue = Queue::at(dir.path().join("queue"));
1914        let questions = Questions::at(dir.path().join("questions"));
1915        let mut waiting = task("missing prerequisite");
1916        queue.put(&mut waiting).unwrap();
1917        let reply = serde_json::json!({"decisions": [{
1918            "id": waiting.id, "blocked_by": ["not-present"],
1919            "missing_dependencies": ["not-present"],
1920            "question": "Which prerequisite replaces the missing task?"
1921        }]})
1922        .to_string();
1923        let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1924        Conductor::new()
1925            .run_once(
1926                &cfg,
1927                dir.path(),
1928                &queue,
1929                &questions,
1930                dir.path(),
1931                &[waiting.clone()],
1932                &[],
1933                &[],
1934                2,
1935            )
1936            .await
1937            .unwrap();
1938        let after = queue.get(&waiting.id).unwrap();
1939        assert_eq!(after.status, TaskStatus::Blocked);
1940        assert_eq!(
1941            questions.get(&after.blocked_by[0]).unwrap().summary,
1942            "Which prerequisite replaces the missing task?"
1943        );
1944    }
1945
1946    #[tokio::test]
1947    async fn dependency_resolution_keeps_decisions_during_unrelated_task_progress() {
1948        let dir = tempdir().unwrap();
1949        let queue = Queue::at(dir.path().join("queue"));
1950        let questions = Questions::at(dir.path().join("questions"));
1951        let mut dep = task("prerequisite");
1952        dep.id = "20261013-193611-ca5b".into();
1953        dep.start("existing-run".into());
1954        queue.put(&mut dep).unwrap();
1955        let mut unrelated = task("unrelated work");
1956        unrelated.start("other-run".into());
1957        queue.put(&mut unrelated).unwrap();
1958        // Make the old and new revision distinguishable without relying on
1959        // the filesystem clock advancing while the mock process starts.
1960        std::fs::File::options()
1961            .write(true)
1962            .open(queue.path_of(&unrelated.id))
1963            .unwrap()
1964            .set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH))
1965            .unwrap();
1966        unrelated.succeed();
1967        let mut waiting = task("waiting");
1968        let mut asking = task("needs a product decision");
1969        let mut holding = task("must stay held");
1970        queue.put(&mut waiting).unwrap();
1971        queue.put(&mut asking).unwrap();
1972        queue.put(&mut holding).unwrap();
1973        let reply = serde_json::json!({"decisions": [
1974            {"id": waiting.id, "blocked_by": ["ca5b"], "reason": "wait for implementation"},
1975            {"id": asking.id, "question": "Which behavior do you want?"},
1976            {"id": holding.id, "recovery": "hold", "reason": "already implemented elsewhere"}
1977        ]})
1978        .to_string();
1979        let mut vars = env(&reply);
1980        vars.insert(
1981            "UNRELATED_PATH".into(),
1982            queue.path_of(&unrelated.id).to_string_lossy().into_owned(),
1983        );
1984        vars.insert(
1985            "UNRELATED_RECORD".into(),
1986            serde_json::to_string(&unrelated).unwrap(),
1987        );
1988        let script = r#"#!/bin/sh
1989cat >/dev/null
1990printf '%s' "$UNRELATED_RECORD" > "$UNRELATED_PATH"
1991printf '%s\n' "$MOCK_REPLY"
1992"#;
1993        let cfg = config(mock_agent(dir.path(), script, vars));
1994        Conductor::new()
1995            .run_once(
1996                &cfg,
1997                dir.path(),
1998                &queue,
1999                &questions,
2000                dir.path(),
2001                &[waiting.clone(), asking.clone(), holding.clone()],
2002                &[],
2003                &[],
2004                2,
2005            )
2006            .await
2007            .unwrap();
2008        assert_eq!(queue.get(&unrelated.id).unwrap().status, TaskStatus::Done);
2009        let after = queue.get(&waiting.id).unwrap();
2010        assert_eq!(after.status, TaskStatus::Blocked);
2011        assert_eq!(after.blocked_by, [dep.id]);
2012        let asked = queue.get(&asking.id).unwrap();
2013        assert_eq!(asked.status, TaskStatus::Blocked);
2014        assert_eq!(
2015            questions.get(&asked.blocked_by[0]).unwrap().summary,
2016            "Which behavior do you want?"
2017        );
2018        assert_eq!(queue.get(&holding.id).unwrap().status, TaskStatus::Held);
2019    }
2020
2021    #[test]
2022    fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
2023        let dir = tempdir().unwrap();
2024        let queue = Queue::at(dir.path().join("queue"));
2025        let questions = Questions::at(dir.path().join("questions"));
2026        let mut a = task("a");
2027        a.priority = 9;
2028        queue.put(&mut a).unwrap();
2029
2030        let verdict = Verdict {
2031            decisions: vec![Decision {
2032                id: a.id.clone(),
2033                blocked_by: vec!["20260101-000000-dead".to_owned()],
2034                reason: Some("waits on the other task".to_owned()),
2035                recovery: None,
2036                question: None,
2037                choices: Vec::new(),
2038                evidence: Vec::new(),
2039                missing_dependencies: Vec::new(),
2040            }],
2041        };
2042        apply(&queue, &questions, &verdict).unwrap();
2043
2044        let back = queue.get(&a.id).unwrap();
2045        assert_eq!(back.status, TaskStatus::Blocked);
2046        assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
2047        assert_eq!(
2048            back.priority, 9,
2049            "the conductor's reply cannot carry priority"
2050        );
2051    }
2052
2053    #[test]
2054    fn a_question_decision_files_one_and_blocks_on_its_id() {
2055        let dir = tempdir().unwrap();
2056        let queue = Queue::at(dir.path().join("queue"));
2057        let questions = Questions::at(dir.path().join("questions"));
2058        let mut t = task("ambiguous");
2059        queue.put(&mut t).unwrap();
2060
2061        let verdict = Verdict {
2062            decisions: vec![Decision {
2063                id: t.id.clone(),
2064                blocked_by: Vec::new(),
2065                reason: Some("which backend?".to_owned()),
2066                recovery: None,
2067                question: Some("Which storage backend?".to_owned()),
2068                choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
2069                evidence: Vec::new(),
2070                missing_dependencies: Vec::new(),
2071            }],
2072        };
2073        apply(&queue, &questions, &verdict).unwrap();
2074
2075        let back = queue.get(&t.id).unwrap();
2076        assert_eq!(back.status, TaskStatus::Blocked);
2077        assert_eq!(back.blocked_by.len(), 1);
2078        let q = questions.get(&back.blocked_by[0]).unwrap();
2079        assert_eq!(q.summary, "Which storage backend?");
2080        assert_eq!(q.node, NODE);
2081        assert!(q.status.open());
2082    }
2083
2084    #[test]
2085    fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
2086        let dir = tempdir().unwrap();
2087        let queue = Queue::at(dir.path().join("queue"));
2088        let questions = Questions::at(dir.path().join("questions"));
2089        let mut t = task("asked once");
2090        queue.put(&mut t).unwrap();
2091
2092        let decision = Decision {
2093            id: t.id.clone(),
2094            reason: Some("still deciding".to_owned()),
2095            question: Some("Which backend?".to_owned()),
2096            ..Decision::default()
2097        };
2098        apply(
2099            &queue,
2100            &questions,
2101            &Verdict {
2102                decisions: vec![decision.clone()],
2103            },
2104        )
2105        .unwrap();
2106        assert_eq!(questions.list().len(), 1);
2107        let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
2108
2109        // An operator releasing the blocked task by hand, without answering,
2110        // puts it back at `Queued` while the question stays open - exactly
2111        // the case the guard in `apply_one` exists for: a later cycle
2112        // proposing the very same question must reuse it, not file a second.
2113        let mut released = queue.get(&t.id).unwrap();
2114        released.release();
2115        queue.put(&mut released).unwrap();
2116
2117        apply(
2118            &queue,
2119            &questions,
2120            &Verdict {
2121                decisions: vec![decision],
2122            },
2123        )
2124        .unwrap();
2125        assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
2126        let after = queue.get(&t.id).unwrap();
2127        assert_eq!(
2128            after.blocked_by,
2129            [first_question_id],
2130            "the existing open question is reused, not replaced"
2131        );
2132    }
2133
2134    #[test]
2135    fn a_same_id_question_from_another_node_is_not_reused() {
2136        let dir = tempdir().unwrap();
2137        let queue = Queue::at(dir.path().join("queue"));
2138        let questions = Questions::at(dir.path().join("questions"));
2139        let mut t = task("must ask the conductor");
2140        queue.put(&mut t).unwrap();
2141
2142        let mut unrelated = Question::new(
2143            t.id.clone(),
2144            "review".to_owned(),
2145            "reviewer-1".to_owned(),
2146            "An unrelated review question".to_owned(),
2147            String::new(),
2148            Vec::new(),
2149        );
2150        questions.put(&mut unrelated).unwrap();
2151
2152        apply(
2153            &queue,
2154            &questions,
2155            &Verdict {
2156                decisions: vec![Decision {
2157                    id: t.id.clone(),
2158                    question: Some("Which backend?".to_owned()),
2159                    ..Decision::default()
2160                }],
2161            },
2162        )
2163        .unwrap();
2164
2165        let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
2166        assert_eq!(blocked_by.len(), 1);
2167        assert_ne!(blocked_by[0], unrelated.id);
2168        assert!(questions.get(&unrelated.id).unwrap().status.open());
2169        assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
2170    }
2171
2172    #[test]
2173    fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
2174        let dir = tempdir().unwrap();
2175        let queue = Queue::at(dir.path().join("queue"));
2176        let questions = Questions::at(dir.path().join("questions"));
2177        let mut t = task("waits on an answer");
2178        queue.put(&mut t).unwrap();
2179
2180        apply(
2181            &queue,
2182            &questions,
2183            &Verdict {
2184                decisions: vec![Decision {
2185                    id: t.id.clone(),
2186                    blocked_by: Vec::new(),
2187                    reason: None,
2188                    recovery: None,
2189                    question: Some("Which backend?".to_owned()),
2190                    choices: Vec::new(),
2191                    evidence: Vec::new(),
2192                    missing_dependencies: Vec::new(),
2193                }],
2194            },
2195        )
2196        .unwrap();
2197        let blocked = queue.get(&t.id).unwrap();
2198        let question_id = blocked.blocked_by[0].clone();
2199
2200        let mut q = questions.get(&question_id).unwrap();
2201        q.answer(Answer::Text("SQLite".to_owned())).unwrap();
2202        questions.put(&mut q).unwrap();
2203        assert_eq!(q.status, QuestionStatus::Answered);
2204
2205        // `crate::daemon::resolve_blockers` is the deterministic resolver
2206        // that actually does this on the real queue; here it is enough to
2207        // prove the pure steps it is built from behave together.
2208        let mut task_after = queue.get(&t.id).unwrap();
2209        task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
2210        task_after.unblock(&question_id);
2211        assert_eq!(task_after.status, TaskStatus::Queued);
2212        assert_eq!(task_after.answers[0].answer, "SQLite");
2213    }
2214
2215    #[test]
2216    fn a_stalled_task_can_be_requeued_or_held() {
2217        let dir = tempdir().unwrap();
2218        let queue = Queue::at(dir.path().join("queue"));
2219        let questions = Questions::at(dir.path().join("questions"));
2220
2221        let mut requeue_me = task("stuck a");
2222        requeue_me.start("run-1".to_owned());
2223        queue.put(&mut requeue_me).unwrap();
2224
2225        let mut hold_me = task("stuck b");
2226        hold_me.start("run-2".to_owned());
2227        queue.put(&mut hold_me).unwrap();
2228
2229        apply(
2230            &queue,
2231            &questions,
2232            &Verdict {
2233                decisions: vec![
2234                    Decision {
2235                        id: requeue_me.id.clone(),
2236                        recovery: Some(Recovery::Requeue),
2237                        ..Decision::default()
2238                    },
2239                    Decision {
2240                        id: hold_me.id.clone(),
2241                        recovery: Some(Recovery::Hold),
2242                        reason: Some("looks broken".to_owned()),
2243                        ..Decision::default()
2244                    },
2245                ],
2246            },
2247        )
2248        .unwrap();
2249
2250        let requeued = queue.get(&requeue_me.id).unwrap();
2251        assert_eq!(requeued.status, TaskStatus::Queued);
2252        assert_eq!(requeued.attempts, 0);
2253
2254        let held = queue.get(&hold_me.id).unwrap();
2255        assert_eq!(held.status, TaskStatus::Held);
2256        assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
2257    }
2258
2259    #[test]
2260    fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
2261        // Reproduces the reported bug: a task held after burning its
2262        // attempts, once the conductor asks a follow-up question about it,
2263        // must come back `held` - not `queued` - once the question is
2264        // answered, whatever the answer said.
2265        let dir = tempdir().unwrap();
2266        let queue = Queue::at(dir.path().join("queue"));
2267        let questions = Questions::at(dir.path().join("questions"));
2268        let mut t = task("held out of attempts");
2269        t.hold_machine(Some("out of attempts".to_owned()));
2270        queue.put(&mut t).unwrap();
2271
2272        apply(
2273            &queue,
2274            &questions,
2275            &Verdict {
2276                decisions: vec![Decision {
2277                    id: t.id.clone(),
2278                    reason: Some("what should happen to this one?".to_owned()),
2279                    question: Some("Hold it, or try again?".to_owned()),
2280                    ..Decision::default()
2281                }],
2282            },
2283        )
2284        .unwrap();
2285        let blocked = queue.get(&t.id).unwrap();
2286        assert_eq!(blocked.status, TaskStatus::Blocked);
2287        let question_id = blocked.blocked_by[0].clone();
2288
2289        let mut q = questions.get(&question_id).unwrap();
2290        q.answer(Answer::Text("leave it held".to_owned())).unwrap();
2291        questions.put(&mut q).unwrap();
2292
2293        // What `crate::daemon::resolve_blockers` does on the real queue.
2294        let mut after = queue.get(&t.id).unwrap();
2295        after.record_answer(q.summary.clone(), "leave it held".to_owned());
2296        after.unblock(&question_id);
2297        assert_eq!(
2298            after.status,
2299            TaskStatus::Held,
2300            "must not fall back to queued"
2301        );
2302        assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
2303    }
2304
2305    #[test]
2306    fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
2307        // A different path to the same bug round 1 fixed: a task
2308        // machine-held for disk pressure, blocked on a conductor question,
2309        // answered "keep it held", and restored to `held`. If the
2310        // conductor's next decision reconfirms `recovery: hold` with no new
2311        // `reason` - allowed, `Decision::reason` is optional - `hold_machine`
2312        // must not silently leave the stale disk-pressure text in place, or
2313        // `crate::triage::run_once` reads it as an unexamined, auto-resolvable
2314        // hold and releases the task straight through the operator's answer
2315        // the instant disk space looks fine again.
2316        let dir = tempdir().unwrap();
2317        let queue = Queue::at(dir.path().join("queue"));
2318        let questions = Questions::at(dir.path().join("questions"));
2319        let mut t = task("disk pressure, then reconsidered");
2320        t.hold_machine(Some(
2321            "not enough free space to start a run: 10 bytes free, 100 required by \
2322             `[disk] min_free_bytes`"
2323                .to_owned(),
2324        ));
2325        t.record_answer(
2326            "How should this be handled?".to_owned(),
2327            "keep it held, a human will look at it later".to_owned(),
2328        );
2329        queue.put(&mut t).unwrap();
2330
2331        apply(
2332            &queue,
2333            &questions,
2334            &Verdict {
2335                decisions: vec![Decision {
2336                    id: t.id.clone(),
2337                    recovery: Some(Recovery::Hold),
2338                    ..Decision::default()
2339                }],
2340            },
2341        )
2342        .unwrap();
2343
2344        let after = queue.get(&t.id).unwrap();
2345        assert_eq!(after.status, TaskStatus::Held);
2346        assert!(
2347            !after
2348                .hold_reason
2349                .as_deref()
2350                .unwrap_or_default()
2351                .starts_with("not enough free space"),
2352            "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
2353            after.hold_reason
2354        );
2355
2356        // The disk gate would report space is fine now - `crate::triage`
2357        // must not read the old text and release the task through it.
2358        let cfg_dir = tempdir().unwrap();
2359        let config = cfg_dir.path().join("magi.toml");
2360        std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2361        let report =
2362            crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
2363        assert!(report.resumed.is_empty(), "must not be auto-released");
2364        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2365    }
2366
2367    #[test]
2368    fn explicit_resumption_rehold_requires_new_evidence_and_another_answer() {
2369        crate::run::pin_test_home();
2370        let dir = tempdir().unwrap();
2371        let queue = Queue::at(dir.path().join("queue"));
2372        let questions = Questions::at(dir.path().join("questions"));
2373        let config = dir.path().join("magi.toml");
2374        std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2375        let now = jiff::Timestamp::now();
2376        let triage = || crate::triage::run_once(&queue, &questions, Some(&config), now);
2377        let open = || {
2378            questions
2379                .list()
2380                .into_iter()
2381                .filter(|q| q.node == "triage" && q.status.open())
2382                .collect::<Vec<_>>()
2383        };
2384        let hold = Decision {
2385            recovery: Some(Recovery::Hold),
2386            reason: Some("waiting on manual worktree cleanup".to_owned()),
2387            ..Decision::default()
2388        };
2389
2390        let mut t = task("looping hold");
2391        t.hold_machine(Some("waiting on manual worktree cleanup".to_owned()));
2392        queue.put(&mut t).unwrap();
2393        let hold = Decision {
2394            id: t.id.clone(),
2395            ..hold
2396        };
2397
2398        // 1. triage asks the ordinary machine question; the operator resumes.
2399        assert_eq!(triage().asked.len(), 1);
2400        let first = open().remove(0);
2401        let mut q = questions.get(&first.id).unwrap();
2402        let resume = q.choices[0].clone();
2403        q.answer(Answer::Choice(resume)).unwrap();
2404        questions.put(&mut q).unwrap();
2405        assert_eq!(triage().answered.len(), 1);
2406        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2407
2408        // Rewording the old cause cannot undo the first explicit answer.
2409        apply(
2410            &queue,
2411            &questions,
2412            &Verdict {
2413                decisions: vec![hold.clone()],
2414            },
2415        )
2416        .unwrap();
2417        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2418        assert!(triage().asked.is_empty());
2419
2420        // A new independently filed check can justify a later hold.
2421        let mut evidence = Question::new(
2422            t.id.clone(),
2423            "preflight".into(),
2424            "check".into(),
2425            "New permission failure".into(),
2426            String::new(),
2427            Vec::new(),
2428        );
2429        questions.put(&mut evidence).unwrap();
2430        let hold = Decision {
2431            evidence: vec![evidence.id.clone()],
2432            reason: Some("new permission check failed".into()),
2433            ..hold
2434        };
2435        apply(
2436            &queue,
2437            &questions,
2438            &Verdict {
2439                decisions: vec![hold.clone()],
2440            },
2441        )
2442        .unwrap();
2443        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2444
2445        // The new hold earns one question identifying the new evidence.
2446        assert_eq!(triage().asked.len(), 1);
2447        let second = open().remove(0);
2448        assert!(second.detail.contains("new permission check failed"));
2449        assert!(triage().asked.is_empty(), "no duplicate question");
2450        assert_eq!(open().len(), 1);
2451
2452        // Another explicit answer supersedes this later hold too.
2453        let mut q = questions.get(&second.id).unwrap();
2454        let force = q.choices[0].clone();
2455        q.answer(Answer::Choice(force)).unwrap();
2456        questions.put(&mut q).unwrap();
2457        assert_eq!(triage().answered.len(), 1);
2458        apply(
2459            &queue,
2460            &questions,
2461            &Verdict {
2462                decisions: vec![hold],
2463            },
2464        )
2465        .unwrap();
2466        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2467    }
2468
2469    #[test]
2470    fn a_runnable_task_can_be_held_directly_without_a_question() {
2471        let dir = tempdir().unwrap();
2472        let queue = Queue::at(dir.path().join("queue"));
2473        let questions = Questions::at(dir.path().join("questions"));
2474        let mut t = task("already answered, should stay put");
2475        queue.put(&mut t).unwrap();
2476
2477        apply(
2478            &queue,
2479            &questions,
2480            &Verdict {
2481                decisions: vec![Decision {
2482                    id: t.id.clone(),
2483                    recovery: Some(Recovery::Hold),
2484                    reason: Some("operator already said keep this held".to_owned()),
2485                    ..Decision::default()
2486                }],
2487            },
2488        )
2489        .unwrap();
2490
2491        let after = queue.get(&t.id).unwrap();
2492        assert_eq!(after.status, TaskStatus::Held);
2493        assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
2494    }
2495
2496    #[test]
2497    fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
2498        crate::run::pin_test_home();
2499        // Reproduces the other half of the reported bug (task 6081): once
2500        // the operator's answer says the work already happened outside the
2501        // loop - PR merged, worktree cleaned up - the conductor needs an
2502        // actual terminal state to put the task in, not just a hold it will
2503        // keep being re-asked about.
2504        let dir = tempdir().unwrap();
2505        let queue = Queue::at(dir.path().join("queue"));
2506        let questions = Questions::at(dir.path().join("questions"));
2507        let mut t = task("already merged by hand");
2508        t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
2509        t.record_answer(
2510            "Handle this one?".to_owned(),
2511            "already merged and cleaned up, close it".to_owned(),
2512        );
2513        queue.put(&mut t).unwrap();
2514
2515        apply(
2516            &queue,
2517            &questions,
2518            &Verdict {
2519                decisions: vec![Decision {
2520                    id: t.id.clone(),
2521                    recovery: Some(Recovery::Done),
2522                    reason: Some("operator confirmed this already landed".to_owned()),
2523                    ..Decision::default()
2524                }],
2525            },
2526        )
2527        .unwrap();
2528
2529        let after = queue.get(&t.id).unwrap();
2530        assert_eq!(after.status, TaskStatus::Done);
2531        assert!(after.hold_reason.is_none());
2532        assert_eq!(after.answers.len(), 1, "the record of why is kept");
2533    }
2534
2535    #[test]
2536    fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
2537        let dir = tempdir().unwrap();
2538        let queue = Queue::at(dir.path().join("queue"));
2539        let questions = Questions::at(dir.path().join("questions"));
2540
2541        let mut queued = task("never ran yet");
2542        queue.put(&mut queued).unwrap();
2543
2544        let mut running = task("mid-run");
2545        running.start("run-1".to_owned());
2546        queue.put(&mut running).unwrap();
2547
2548        for id in [queued.id.clone(), running.id.clone()] {
2549            apply(
2550                &queue,
2551                &questions,
2552                &Verdict {
2553                    decisions: vec![Decision {
2554                        id,
2555                        recovery: Some(Recovery::Done),
2556                        ..Decision::default()
2557                    }],
2558                },
2559            )
2560            .unwrap();
2561        }
2562
2563        assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
2564        assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
2565    }
2566
2567    #[test]
2568    fn a_question_after_two_settled_answers_is_still_filed_and_blocks() {
2569        // A genuine question must stay answerable however many were settled
2570        // before it: refusing it left `magi answer --list` empty.
2571        let dir = tempdir().unwrap();
2572        let queue = Queue::at(dir.path().join("queue"));
2573        let questions = Questions::at(dir.path().join("questions"));
2574        let mut t = task("asked about repeatedly");
2575        t.hold_machine(Some("out of attempts".to_owned()));
2576        t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
2577        t.record_answer(
2578            "Handle this one? (2)".to_owned(),
2579            "still not yet".to_owned(),
2580        );
2581        queue.put(&mut t).unwrap();
2582        assert_eq!(questions.list().len(), 0);
2583
2584        let verdict = Verdict {
2585            decisions: vec![Decision {
2586                id: t.id.clone(),
2587                question: Some("Branch conflicts with origin/main, how do we proceed?".to_owned()),
2588                ..Decision::default()
2589            }],
2590        };
2591        apply(&queue, &questions, &verdict).unwrap();
2592
2593        let filed = questions.list();
2594        assert_eq!(filed.len(), 1, "the question was filed");
2595        assert_eq!(
2596            filed[0].summary,
2597            "Branch conflicts with origin/main, how do we proceed?"
2598        );
2599        let deputy = filed[0]
2600            .deputy
2601            .as_ref()
2602            .expect("the wait is handed to a deputy");
2603        assert!(deputy.brief.contains(&t.id), "{}", deputy.brief);
2604        assert_eq!(
2605            deputy.starts, 0,
2606            "filing never starts anything: the loop does not wait"
2607        );
2608        assert!(filed[0].status.open());
2609        assert_eq!(filed[0].node, NODE);
2610        let after = queue.get(&t.id).unwrap();
2611        assert_eq!(after.status, TaskStatus::Blocked);
2612        assert_eq!(after.blocked_by, vec![filed[0].id.clone()]);
2613        assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
2614        assert!(
2615            !after
2616                .hold_reason
2617                .clone()
2618                .unwrap_or_default()
2619                .contains("conduct tried to ask"),
2620            "no hold was applied"
2621        );
2622
2623        apply(&queue, &questions, &verdict).unwrap();
2624        let again = questions.list();
2625        assert_eq!(again.len(), 1, "the open question is reused");
2626        assert_eq!(again[0].id, filed[0].id);
2627    }
2628
2629    #[test]
2630    fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
2631        let dir = tempdir().unwrap();
2632        let queue = Queue::at(dir.path().join("queue"));
2633        let questions = Questions::at(dir.path().join("questions"));
2634        let mut t = task("asked about once already");
2635        t.hold_machine(Some("out of attempts".to_owned()));
2636        t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
2637        queue.put(&mut t).unwrap();
2638
2639        apply(
2640            &queue,
2641            &questions,
2642            &Verdict {
2643                decisions: vec![Decision {
2644                    id: t.id.clone(),
2645                    question: Some("Still not sure - now what?".to_owned()),
2646                    ..Decision::default()
2647                }],
2648            },
2649        )
2650        .unwrap();
2651
2652        assert_eq!(questions.list().len(), 1, "the second question was filed");
2653        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
2654    }
2655
2656    #[test]
2657    fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
2658        // `crate::triage` never calls `Task::block`, and only ever looks at
2659        // tasks still `held` - so a conductor question that moved this task
2660        // to `blocked` would orphan triage's own open question. The guard in
2661        // `apply_one` must leave the task alone until triage's question
2662        // settles.
2663        let dir = tempdir().unwrap();
2664        let queue = Queue::at(dir.path().join("queue"));
2665        let questions = Questions::at(dir.path().join("questions"));
2666        let mut t = task("held, triage already asking about it");
2667        t.hold_machine(Some("cause unclear".to_owned()));
2668        queue.put(&mut t).unwrap();
2669
2670        let mut triage_q = Question::new(
2671            t.id.clone(),
2672            crate::triage::NODE.to_owned(),
2673            "triage".to_owned(),
2674            "Still needed?".to_owned(),
2675            String::new(),
2676            vec![
2677                "resume".to_owned(),
2678                "not yet".to_owned(),
2679                "discard".to_owned(),
2680            ],
2681        );
2682        questions.put(&mut triage_q).unwrap();
2683
2684        for decision in [
2685            Decision {
2686                id: t.id.clone(),
2687                question: Some("what now?".to_owned()),
2688                ..Decision::default()
2689            },
2690            Decision {
2691                id: t.id.clone(),
2692                recovery: Some(Recovery::Requeue),
2693                ..Decision::default()
2694            },
2695        ] {
2696            apply(
2697                &queue,
2698                &questions,
2699                &Verdict {
2700                    decisions: vec![decision],
2701                },
2702            )
2703            .unwrap();
2704        }
2705
2706        let after = queue.get(&t.id).unwrap();
2707        assert_eq!(
2708            after.status,
2709            TaskStatus::Held,
2710            "triage still owns this hold"
2711        );
2712        assert!(after.blocked_by.is_empty());
2713        assert_eq!(
2714            questions.list().len(),
2715            1,
2716            "no second, conductor-owned question was filed"
2717        );
2718    }
2719
2720    #[test]
2721    fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
2722        // The race `crate::triage::pending_for` exists to close: the
2723        // operator has already answered the triage question (it is no
2724        // longer `open`), but `crate::triage::run_once` - which only runs on
2725        // a fully idle daemon tick, far less often than the conductor polls
2726        // - has not had a turn to apply it yet. `apply_one` must not treat
2727        // "not open" as "settled" here, or it would block the task out from
2728        // under an answer triage has not read back yet.
2729        let dir = tempdir().unwrap();
2730        let queue = Queue::at(dir.path().join("queue"));
2731        let questions = Questions::at(dir.path().join("questions"));
2732        let mut t = task("held, triage question answered but not yet applied");
2733        t.hold_machine(Some("cause unclear".to_owned()));
2734        queue.put(&mut t).unwrap();
2735
2736        let mut triage_q = Question::new(
2737            t.id.clone(),
2738            crate::triage::NODE.to_owned(),
2739            "triage".to_owned(),
2740            "Still needed?".to_owned(),
2741            String::new(),
2742            vec![
2743                "resume".to_owned(),
2744                "not yet".to_owned(),
2745                "discard".to_owned(),
2746            ],
2747        );
2748        questions.put(&mut triage_q).unwrap();
2749        triage_q
2750            .answer(Answer::Choice("not yet".to_owned()))
2751            .unwrap();
2752        questions.put(&mut triage_q).unwrap();
2753        assert!(!triage_q.status.open());
2754
2755        apply(
2756            &queue,
2757            &questions,
2758            &Verdict {
2759                decisions: vec![Decision {
2760                    id: t.id.clone(),
2761                    question: Some("what now?".to_owned()),
2762                    ..Decision::default()
2763                }],
2764            },
2765        )
2766        .unwrap();
2767
2768        let after = queue.get(&t.id).unwrap();
2769        assert_eq!(
2770            after.status,
2771            TaskStatus::Held,
2772            "triage's own answer is not yet applied - conduct must wait"
2773        );
2774        assert_eq!(
2775            questions.list().len(),
2776            1,
2777            "no conductor question was filed over the pending triage answer"
2778        );
2779    }
2780
2781    #[test]
2782    fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
2783        let dir = tempdir().unwrap();
2784        let queue = Queue::at(dir.path().join("queue"));
2785        let questions = Questions::at(dir.path().join("questions"));
2786        let mut held = task("manual recovery");
2787        held.priority = 300;
2788        held.runs.push("run20260912-224242-daf5".to_owned());
2789        held.hold_manual(Some(
2790            "active manual recovery run20260912-224242-daf5".to_owned(),
2791        ));
2792        queue.put(&mut held).unwrap();
2793
2794        // Every field a conductor may use to alter lifecycle state is ignored:
2795        // requeue/review would dispatch duplicate work, hold could overwrite
2796        // evidence, and a question would turn the hold into `blocked`.
2797        for decision in [
2798            Decision {
2799                id: held.id.clone(),
2800                recovery: Some(Recovery::Requeue),
2801                ..Decision::default()
2802            },
2803            Decision {
2804                id: held.id.clone(),
2805                recovery: Some(Recovery::Hold),
2806                reason: Some("stale replacement reason".to_owned()),
2807                ..Decision::default()
2808            },
2809            Decision {
2810                id: held.id.clone(),
2811                recovery: Some(Recovery::Review),
2812                ..Decision::default()
2813            },
2814            Decision {
2815                id: held.id.clone(),
2816                blocked_by: vec!["other-task".to_owned()],
2817                question: Some("retry now?".to_owned()),
2818                ..Decision::default()
2819            },
2820        ] {
2821            apply(
2822                &queue,
2823                &questions,
2824                &Verdict {
2825                    decisions: vec![decision],
2826                },
2827            )
2828            .unwrap();
2829        }
2830
2831        let after = queue.get(&held.id).unwrap();
2832        assert_eq!(after.status, TaskStatus::Held);
2833        assert!(after.operator_held());
2834        assert_eq!(after.priority, 300);
2835        assert_eq!(after.runs, ["run20260912-224242-daf5"]);
2836        assert_eq!(
2837            after.hold_reason.as_deref(),
2838            Some("active manual recovery run20260912-224242-daf5")
2839        );
2840        assert!(after.blocked_by.is_empty());
2841        assert!(questions.list().is_empty());
2842        assert!(
2843            queue.next_runnable().is_none(),
2844            "must not dispatch a duplicate"
2845        );
2846    }
2847
2848    #[test]
2849    fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
2850        let dir = tempdir().unwrap();
2851        let queue = Queue::at(dir.path().join("queue"));
2852        let questions = Questions::at(dir.path().join("questions"));
2853
2854        let mut automatic = task("disk gate");
2855        automatic.hold_machine(Some("disk full".to_owned()));
2856        queue.put(&mut automatic).unwrap();
2857        let requeue = || Verdict {
2858            decisions: vec![Decision {
2859                id: automatic.id.clone(),
2860                recovery: Some(Recovery::Requeue),
2861                ..Decision::default()
2862            }],
2863        };
2864        apply(&queue, &questions, &requeue()).unwrap();
2865        assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
2866
2867        let mut manual = task("operator gate");
2868        manual.hold_manual(Some("wait for operator".to_owned()));
2869        queue.put(&mut manual).unwrap();
2870        apply(
2871            &queue,
2872            &questions,
2873            &Verdict {
2874                decisions: vec![Decision {
2875                    id: manual.id.clone(),
2876                    recovery: Some(Recovery::Requeue),
2877                    ..Decision::default()
2878                }],
2879            },
2880        )
2881        .unwrap();
2882        assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
2883
2884        // This mirrors the CLI and web release routes: only an explicit
2885        // operator action clears the manual boundary.
2886        let mut released = queue.get(&manual.id).unwrap();
2887        released.release();
2888        queue.put(&mut released).unwrap();
2889        assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
2890    }
2891
2892    #[test]
2893    fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
2894        let dir = tempdir().unwrap();
2895        let queue = Queue::at(dir.path().join("queue"));
2896        let questions = Questions::at(dir.path().join("questions"));
2897        let mut legacy = task("old explicit hold");
2898        legacy.status = TaskStatus::Held;
2899        legacy.hold_reason = Some("manual recovery already active".to_owned());
2900        legacy.hold_source = None;
2901        legacy.blocked_by = vec!["dependency".to_owned()];
2902        queue.put(&mut legacy).unwrap();
2903
2904        apply(
2905            &queue,
2906            &questions,
2907            &Verdict {
2908                decisions: vec![Decision {
2909                    id: legacy.id.clone(),
2910                    recovery: Some(Recovery::Requeue),
2911                    ..Decision::default()
2912                }],
2913            },
2914        )
2915        .unwrap();
2916
2917        let after = queue.get(&legacy.id).unwrap();
2918        assert_eq!(after.status, TaskStatus::Held);
2919        assert_eq!(after.hold_source, None);
2920        assert_eq!(after.hold_reason, legacy.hold_reason);
2921        assert_eq!(after.blocked_by, legacy.blocked_by);
2922    }
2923
2924    #[test]
2925    fn review_recovery_is_a_no_op_without_a_survivable_branch() {
2926        // `surviving_branch` reaches `RunState::load`, which reaches the
2927        // process-global `run::home()` - a `OnceLock`, so this only wins the
2928        // race the first time it runs in the binary; every other test still
2929        // reaches the same directory whichever call won, and this test's own
2930        // run id never collides with another test's.
2931        crate::run::pin_test_home();
2932        let dir = tempdir().unwrap();
2933        let queue = Queue::at(dir.path().join("queue"));
2934        let questions = Questions::at(dir.path().join("questions"));
2935        let mut t = task("blocked with no readable run");
2936        t.start("20260101-000000-dead".to_owned()); // no such run on disk
2937        t.fail("blocked", 5);
2938        queue.put(&mut t).unwrap();
2939
2940        apply(
2941            &queue,
2942            &questions,
2943            &Verdict {
2944                decisions: vec![Decision {
2945                    id: t.id.clone(),
2946                    recovery: Some(Recovery::Review),
2947                    ..Decision::default()
2948                }],
2949            },
2950        )
2951        .unwrap();
2952
2953        let after = queue.get(&t.id).unwrap();
2954        assert_eq!(
2955            after.status,
2956            TaskStatus::Failed,
2957            "with nothing to reopen, the decision is dropped rather than guessed at"
2958        );
2959        assert!(after.review_branch.is_none());
2960    }
2961
2962    #[test]
2963    fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
2964        // `Hold` is the one exception - see `Decision::recovery`'s doc and
2965        // `a_runnable_task_can_be_held_directly_without_a_question` - but a
2966        // task already in line has nothing for `requeue` or `review` to do.
2967        let dir = tempdir().unwrap();
2968        let queue = Queue::at(dir.path().join("queue"));
2969        let questions = Questions::at(dir.path().join("questions"));
2970
2971        for recovery in [Recovery::Requeue, Recovery::Review] {
2972            let mut t = task("ordinary");
2973            queue.put(&mut t).unwrap();
2974
2975            apply(
2976                &queue,
2977                &questions,
2978                &Verdict {
2979                    decisions: vec![Decision {
2980                        id: t.id.clone(),
2981                        recovery: Some(recovery),
2982                        ..Decision::default()
2983                    }],
2984                },
2985            )
2986            .unwrap();
2987
2988            assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2989        }
2990    }
2991
2992    #[tokio::test]
2993    async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
2994        let dir = tempdir().unwrap();
2995        let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
2996        let queue = Queue::at(dir.path().join("queue"));
2997        let questions = Questions::at(dir.path().join("questions"));
2998        let mut t = task("normal");
2999        queue.put(&mut t).unwrap();
3000
3001        let mut conductor = Conductor::new();
3002        conductor
3003            .maybe_run(
3004                &cfg,
3005                dir.path(),
3006                &queue,
3007                &questions,
3008                dir.path(),
3009                &[t.clone()],
3010                &[],
3011                &[],
3012                2,
3013            )
3014            .await;
3015
3016        assert_eq!(
3017            queue.get(&t.id).unwrap().status,
3018            TaskStatus::Queued,
3019            "a failed invocation must change nothing"
3020        );
3021        assert!(
3022            queue.next_runnable().is_some(),
3023            "the loop must still be able to take the next task"
3024        );
3025    }
3026
3027    #[tokio::test]
3028    async fn a_reply_with_no_json_leaves_the_queue_untouched() {
3029        let dir = tempdir().unwrap();
3030        let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
3031        let queue = Queue::at(dir.path().join("queue"));
3032        let questions = Questions::at(dir.path().join("questions"));
3033        let mut t = task("normal");
3034        queue.put(&mut t).unwrap();
3035
3036        let mut conductor = Conductor::new();
3037        conductor
3038            .maybe_run(
3039                &cfg,
3040                dir.path(),
3041                &queue,
3042                &questions,
3043                dir.path(),
3044                &[t.clone()],
3045                &[],
3046                &[],
3047                2,
3048            )
3049            .await;
3050
3051        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3052    }
3053
3054    #[tokio::test]
3055    async fn a_conductor_chain_falls_to_the_next_agent_once_each() {
3056        let dir = tempdir().unwrap();
3057        let mut t = task("chained");
3058        let reply = format!(
3059            "{{\"decisions\":[{{\"id\":\"{}\",\"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}",
3060            t.id
3061        );
3062        let counted = |id: &str, tail: &str| {
3063            let calls = dir.path().join(format!("{id}.calls"));
3064            let path = dir.path().join(format!("{id}.sh"));
3065            std::fs::write(
3066                &path,
3067                format!(
3068                    "#!/bin/sh\ncat >/dev/null\necho x >> '{}'\n{tail}\n",
3069                    calls.display()
3070                ),
3071            )
3072            .unwrap();
3073            AgentSpec {
3074                id: id.to_owned(),
3075                kind: AgentKind::Command,
3076                model: None,
3077                command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
3078                extra_args: Vec::new(),
3079                env: BTreeMap::new(),
3080                prompt_delivery: None,
3081            }
3082        };
3083        let n_calls = |id: &str| {
3084            std::fs::read_to_string(dir.path().join(format!("{id}.calls")))
3085                .map_or(0, |s| s.lines().count())
3086        };
3087        let mut cfg = config(counted("a", "exit 3"));
3088        cfg.agents = vec![
3089            counted("a", "exit 3"),
3090            counted("b", &format!("printf '%s' '{reply}'")),
3091        ];
3092        cfg.roles.conductor = Some(crate::config::AgentChoice::Chain(vec![
3093            "a".into(),
3094            "b".into(),
3095            "a".into(),
3096        ]));
3097        let queue = Queue::at(dir.path().join("queue"));
3098        let questions = Questions::at(dir.path().join("questions"));
3099        queue.put(&mut t).unwrap();
3100
3101        let mut conductor = Conductor::new();
3102        conductor
3103            .maybe_run(
3104                &cfg,
3105                dir.path(),
3106                &queue,
3107                &questions,
3108                dir.path(),
3109                &[t.clone()],
3110                &[],
3111                &[],
3112                2,
3113            )
3114            .await;
3115
3116        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3117        assert_eq!(n_calls("a"), 1, "each id is tried once");
3118        assert_eq!(n_calls("b"), 1);
3119        let saved = std::fs::read_to_string(seat_path(dir.path())).unwrap();
3120        assert!(
3121            saved.contains("\"b\""),
3122            "the seat on disk is the one that ran: {saved}"
3123        );
3124    }
3125
3126    #[tokio::test]
3127    async fn json_survives_code_fences_and_a_preamble() {
3128        let dir = tempdir().unwrap();
3129        let mut t = task("fenced");
3130        let reply = format!(
3131            "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
3132             \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
3133            t.id
3134        );
3135        let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
3136        let queue = Queue::at(dir.path().join("queue"));
3137        let questions = Questions::at(dir.path().join("questions"));
3138        queue.put(&mut t).unwrap();
3139
3140        let mut conductor = Conductor::new();
3141        conductor
3142            .maybe_run(
3143                &cfg,
3144                dir.path(),
3145                &queue,
3146                &questions,
3147                dir.path(),
3148                &[t.clone()],
3149                &[],
3150                &[],
3151                2,
3152            )
3153            .await;
3154
3155        let back = queue.get(&t.id).unwrap();
3156        assert_eq!(back.status, TaskStatus::Blocked);
3157        assert_eq!(back.blocked_by, ["x"]);
3158    }
3159
3160    #[tokio::test]
3161    async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
3162        // Each real invocation writes its own artifact stem, `turn-<n>`, so
3163        // whether a second one happened is read off the artifacts directory.
3164        let dir = tempdir().unwrap();
3165        let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3166        let queue = Queue::at(dir.path().join("queue"));
3167        let questions = Questions::at(dir.path().join("questions"));
3168        let mut t = task("stable");
3169        queue.put(&mut t).unwrap();
3170        let artifacts = dir.path().join("conduct").join("artifacts");
3171        let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3172
3173        let mut conductor = Conductor::new();
3174        conductor
3175            .maybe_run(
3176                &cfg,
3177                dir.path(),
3178                &queue,
3179                &questions,
3180                dir.path(),
3181                &[t.clone()],
3182                &[],
3183                &[],
3184                2,
3185            )
3186            .await;
3187        assert!(turn(1).is_file(), "the first cycle must call the conductor");
3188
3189        conductor
3190            .maybe_run(
3191                &cfg,
3192                dir.path(),
3193                &queue,
3194                &questions,
3195                dir.path(),
3196                &[t.clone()],
3197                &[],
3198                &[],
3199                2,
3200            )
3201            .await;
3202        assert!(
3203            !turn(2).is_file(),
3204            "an unchanged revision and an unchanged stalled/finished set must not call the \
3205             conductor twice"
3206        );
3207
3208        // Once the queue actually changes, the next `maybe_run` calls again.
3209        t.priority = 1;
3210        queue.put(&mut t).unwrap();
3211        conductor
3212            .maybe_run(
3213                &cfg,
3214                dir.path(),
3215                &queue,
3216                &questions,
3217                dir.path(),
3218                &[t.clone()],
3219                &[],
3220                &[],
3221                2,
3222            )
3223            .await;
3224        assert!(turn(2).is_file(), "a moved revision calls it again");
3225    }
3226
3227    #[tokio::test]
3228    async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
3229        // The queue's own revision has not moved - nothing wrote to it - but
3230        // a task now looks stalled, purely because time passed. Calling
3231        // again here, and never again once this exact set has been shown
3232        // once, is the whole point of keying `worth_a_look` on the id set
3233        // rather than on "is it non-empty".
3234        let dir = tempdir().unwrap();
3235        let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3236        let queue = Queue::at(dir.path().join("queue"));
3237        let questions = Questions::at(dir.path().join("questions"));
3238        let mut t = task("quiet");
3239        queue.put(&mut t).unwrap();
3240        let artifacts = dir.path().join("conduct").join("artifacts");
3241        let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3242
3243        let mut conductor = Conductor::new();
3244        conductor
3245            .maybe_run(
3246                &cfg,
3247                dir.path(),
3248                &queue,
3249                &questions,
3250                dir.path(),
3251                &[t.clone()],
3252                &[],
3253                &[],
3254                2,
3255            )
3256            .await;
3257        assert!(turn(1).is_file());
3258
3259        conductor
3260            .maybe_run(
3261                &cfg,
3262                dir.path(),
3263                &queue,
3264                &questions,
3265                dir.path(),
3266                &[],
3267                &[t.clone()],
3268                &[],
3269                2,
3270            )
3271            .await;
3272        assert!(
3273            turn(2).is_file(),
3274            "a task turning stalled must call the conductor again"
3275        );
3276
3277        // But once shown at this exact revision, showing the *same* stalled
3278        // set again must not call a third time.
3279        conductor
3280            .maybe_run(
3281                &cfg,
3282                dir.path(),
3283                &queue,
3284                &questions,
3285                dir.path(),
3286                &[],
3287                &[t.clone()],
3288                &[],
3289                2,
3290            )
3291            .await;
3292        assert!(
3293            !turn(3).is_file(),
3294            "the same stalled task lingering must not call the conductor every cycle"
3295        );
3296    }
3297
3298    fn rehold(queue: &Queue, questions: &Questions, id: &str, note: &str) {
3299        apply(
3300            queue,
3301            questions,
3302            &Verdict {
3303                decisions: vec![Decision {
3304                    id: id.to_owned(),
3305                    reason: Some(note.to_owned()),
3306                    recovery: Some(Recovery::Hold),
3307                    ..Decision::default()
3308                }],
3309            },
3310        )
3311        .unwrap();
3312    }
3313
3314    #[test]
3315    fn reholding_with_an_identical_note_changes_nothing() {
3316        let dir = tempdir().unwrap();
3317        let queue = Queue::at(dir.path().join("queue"));
3318        let questions = Questions::at(dir.path().join("questions"));
3319        let mut t = task("held");
3320        t.hold_machine(Some("waiting on a human".to_owned()));
3321        queue.put(&mut t).unwrap();
3322
3323        rehold(&queue, &questions, &t.id, "owner said keep it held");
3324        let first = queue.get(&t.id).unwrap();
3325        let rev = queue.revision();
3326        for _ in 0..5 {
3327            rehold(&queue, &questions, &t.id, "owner said keep it held");
3328        }
3329        let after = queue.get(&t.id).unwrap();
3330        assert_eq!(after.hold_reason, first.hold_reason);
3331        assert_eq!(after.updated_at, first.updated_at);
3332        assert_eq!(queue.revision(), rev, "nothing was written");
3333    }
3334
3335    #[test]
3336    fn reholding_with_different_notes_keeps_one_previously_level() {
3337        let dir = tempdir().unwrap();
3338        let queue = Queue::at(dir.path().join("queue"));
3339        let questions = Questions::at(dir.path().join("questions"));
3340        let mut t = task("held");
3341        t.hold_machine(Some("original".to_owned()));
3342        queue.put(&mut t).unwrap();
3343
3344        for i in 0..20 {
3345            rehold(&queue, &questions, &t.id, &format!("note {}", i % 2));
3346        }
3347        let reason = queue.get(&t.id).unwrap().hold_reason.unwrap();
3348        assert_eq!(reason.matches("(previously:").count(), 1, "{reason}");
3349        assert!(reason.starts_with("note "), "the new note leads: {reason}");
3350    }
3351
3352    #[test]
3353    fn an_already_nested_reason_is_matched_by_its_outermost_note() {
3354        let mut t = task("deep");
3355        let nested = "same\n\n(previously: same\n\n(previously: same))";
3356        t.hold_machine(Some(nested.to_owned()));
3357        let d = Decision {
3358            id: t.id.clone(),
3359            reason: Some("same".to_owned()),
3360            ..Decision::default()
3361        };
3362        assert_eq!(reaffirmed_hold_reason(&t, &d), None);
3363        let d2 = Decision {
3364            reason: Some("other".to_owned()),
3365            ..d
3366        };
3367        assert_eq!(
3368            reaffirmed_hold_reason(&t, &d2).as_deref(),
3369            Some("other\n\n(previously: same)")
3370        );
3371    }
3372
3373    #[tokio::test]
3374    async fn a_repeated_hold_decision_settles_instead_of_looping() {
3375        let dir = tempdir().unwrap();
3376        let queue = Queue::at(dir.path().join("queue"));
3377        let questions = Questions::at(dir.path().join("questions"));
3378        let reply = r#"{"decisions":[{"id":"ID","reason":"keep held","recovery":"hold"}]}"#;
3379        let mut t = task("held");
3380        t.hold_machine(Some("first".to_owned()));
3381        queue.put(&mut t).unwrap();
3382        let cfg = config(mock_agent(
3383            dir.path(),
3384            REPLY,
3385            env(&reply.replace("ID", &t.id)),
3386        ));
3387        let mut conductor = Conductor::new();
3388        // Cycle 1 writes the new note; cycle 2 repeats it and writes nothing.
3389        for _ in 0..2 {
3390            let held = queue.get(&t.id).unwrap();
3391            conductor
3392                .maybe_run(
3393                    &cfg,
3394                    dir.path(),
3395                    &queue,
3396                    &questions,
3397                    dir.path(),
3398                    &[],
3399                    &[],
3400                    &[held],
3401                    2,
3402                )
3403                .await;
3404        }
3405        let held = queue.get(&t.id).unwrap();
3406        assert!(
3407            held.hold_reason
3408                .as_deref()
3409                .unwrap()
3410                .starts_with("keep held")
3411        );
3412        assert!(
3413            !conductor.worth_a_look(&queue, &[], &[held]),
3414            "an unchanged held task must stop being reconsidered"
3415        );
3416    }
3417
3418    #[tokio::test]
3419    async fn alternating_wording_does_not_keep_a_held_task_in_the_loop() {
3420        let dir = tempdir().unwrap();
3421        let queue = Queue::at(dir.path().join("queue"));
3422        let questions = Questions::at(dir.path().join("questions"));
3423        let mut t = task("held");
3424        t.hold_machine(Some("first".to_owned()));
3425        queue.put(&mut t).unwrap();
3426        let reply = |note: &str| {
3427            format!(
3428                r#"{{"decisions":[{{"id":"{}","reason":"{note}","recovery":"hold"}}]}}"#,
3429                t.id
3430            )
3431        };
3432        let cfg_a = config(mock_agent(dir.path(), REPLY, env(&reply("keep held"))));
3433        let cfg_b = config(mock_agent(dir.path(), REPLY, env(&reply("leave held"))));
3434        let mut conductor = Conductor::new();
3435        let mut writes = 0;
3436        for i in 0..6 {
3437            let held = queue.get(&t.id).unwrap();
3438            let before = queue.revision();
3439            conductor
3440                .maybe_run(
3441                    if i % 2 == 0 { &cfg_a } else { &cfg_b },
3442                    dir.path(),
3443                    &queue,
3444                    &questions,
3445                    dir.path(),
3446                    &[],
3447                    &[],
3448                    &[held],
3449                    2,
3450                )
3451                .await;
3452            if queue.revision() != before {
3453                writes += 1;
3454            }
3455        }
3456        assert!(writes <= 2, "the loop must settle, saw {writes} writes");
3457        let mut held = queue.get(&t.id).unwrap();
3458        held.instruction.push_str(" (edited)");
3459        queue.put(&mut held).unwrap();
3460        let held = queue.get(&t.id).unwrap();
3461        assert!(conductor.worth_a_look(&queue, &[], &[held]));
3462    }
3463
3464    #[test]
3465    fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
3466        let dir = tempdir().unwrap();
3467        let queue = Queue::at(dir.path().join("queue"));
3468        let mut t = task("t");
3469        queue.put(&mut t).unwrap();
3470
3471        let mut conductor = Conductor::new();
3472        assert!(
3473            conductor.worth_a_look(&queue, &[], &[]),
3474            "a conductor that has never run has something to look at"
3475        );
3476
3477        conductor.last_seen = Some(conductor.snapshot(&queue, &[], &[]));
3478        assert!(
3479            !conductor.worth_a_look(&queue, &[], &[]),
3480            "nothing changed and nothing is stalled or finished"
3481        );
3482        assert!(
3483            conductor.worth_a_look(&queue, &[t.clone()], &[]),
3484            "a stalled task is worth a look even at the same revision"
3485        );
3486        assert!(
3487            conductor.worth_a_look(&queue, &[], &[t.clone()]),
3488            "a finished task is worth a look even at the same revision"
3489        );
3490    }
3491
3492    #[tokio::test]
3493    async fn the_conduct_path_never_calls_ask_and_wait() {
3494        // Structural: grepping this module and `daemon.rs` for
3495        // `ask_and_wait` is the actual assertion this module's own doc
3496        // promises; this test exists so the promise has a name in the test
3497        // output too. `apply_one`'s question path uses `Questions::put`
3498        // exclusively.
3499        let dir = tempdir().unwrap();
3500        let queue = Queue::at(dir.path().join("queue"));
3501        let questions = Questions::at(dir.path().join("questions"));
3502        let mut t = task("asks without blocking");
3503        queue.put(&mut t).unwrap();
3504
3505        apply(
3506            &queue,
3507            &questions,
3508            &Verdict {
3509                decisions: vec![Decision {
3510                    id: t.id.clone(),
3511                    question: Some("ok?".to_owned()),
3512                    ..Decision::default()
3513                }],
3514            },
3515        )
3516        .unwrap();
3517        // Reaching here at all (no hang) is the assertion.
3518        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3519    }
3520
3521    #[test]
3522    fn a_pinned_resume_is_not_held_or_requeued_by_the_conductor() {
3523        let mut t = Task::new("t".into(), "t".into(), PathBuf::new(), Source::Human);
3524        t.resume_override = Some(crate::queue::OperatorResume {
3525            question_id: "q".into(),
3526            at: jiff::Timestamp::now(),
3527            conductor_rehold: None,
3528            forced: false,
3529            pinned_run: Some("run-1".into()),
3530        });
3531        assert!(pinned_resume(&t));
3532        assert!(!may_hold(&mut t, "waiting for magi resume"));
3533        assert!(
3534            t.resume_override
3535                .as_ref()
3536                .unwrap()
3537                .conductor_rehold
3538                .is_none(),
3539            "a refused hold is not recorded as an override"
3540        );
3541        t.resume_override = None;
3542        assert!(!pinned_resume(&t));
3543        assert!(may_hold(&mut t, "no override, so a hold is allowed"));
3544    }
3545
3546    #[test]
3547    fn a_pinned_resume_is_not_blocked_by_a_conductor_question_or_dependency() {
3548        let dir = tempdir().unwrap();
3549        let queue = Queue::at(dir.path().join("queue"));
3550        let questions = Questions::at(dir.path().join("questions"));
3551        let mut t = task("resume me");
3552        t.resume_override = Some(crate::queue::OperatorResume {
3553            question_id: "q".into(),
3554            at: jiff::Timestamp::now(),
3555            conductor_rehold: None,
3556            forced: true,
3557            pinned_run: Some("run-1".into()),
3558        });
3559        queue.put(&mut t).unwrap();
3560
3561        apply(
3562            &queue,
3563            &questions,
3564            &Verdict {
3565                decisions: vec![
3566                    Decision {
3567                        id: t.id.clone(),
3568                        question: Some("really?".to_owned()),
3569                        ..Decision::default()
3570                    },
3571                    Decision {
3572                        id: t.id.clone(),
3573                        blocked_by: vec!["other".to_owned()],
3574                        ..Decision::default()
3575                    },
3576                ],
3577            },
3578        )
3579        .unwrap();
3580
3581        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3582        assert!(questions.list().is_empty());
3583    }
3584}