Skip to main content

magi/
conduct.rs

1//! The conductor: a single agent seat that arranges the queue.
2//!
3//! [`crate::queue`] orders runnable tasks by `priority` alone; nothing in it
4//! can express that one task should wait for another, or that a task whose
5//! last run stopped short deserves a second look before the loop blindly
6//! retries it. Once per polling cycle, [`Conductor::maybe_run`] shows one
7//! agent seat three things - the runnable tasks, the tasks stuck `running`
8//! with no live daemon behind them, and the `failed`/`held` tasks nobody has
9//! decided about yet - and asks it to decide `blocked_by` for the first and a
10//! recovery for the other two. Everything else about the loop - which
11//! unblocked task runs next, one at a time, in `priority` order - is
12//! unchanged; see `crate::daemon`.
13//!
14//! # What the conductor may not do
15//!
16//! [`Decision`] has no field for `priority`, for deleting a task, or for
17//! touching git, a worktree, or a branch directly. [`Recovery::Review`] only
18//! ever reopens a branch `crate::daemon` itself resolved from the task's own
19//! run record ([`surviving_branch`]) - never a name the model wrote - through
20//! `crate::graph::Runner::review`, which reviews and verifies but never
21//! rewrites history.
22//!
23//! # Non-blocking by construction
24//!
25//! [`crate::ask::ask_and_wait`] is never called from here, and the prompt
26//! tells the model the same: that CLI command blocks until a human answers,
27//! and calling it from inside the conductor's own invocation would park the
28//! whole polling loop behind one task's question. Instead a decision that
29//! wants the operator's judgement carries a `question` field, and [`apply`]
30//! files it with [`Question::new`] and [`Questions::put`] and moves on in the
31//! same call. Something still has to read the owner's reply: the question
32//! carries a [`Deputy`] brief, and `magi serve` starts a short-lived seat of
33//! its own for it ([`crate::deputy`]) - never this loop.
34//!
35//! # Fails soft, always
36//!
37//! [`Conductor::maybe_run`] never returns an error: an unusable roster, a
38//! timed-out invocation, or a reply [`verdict::extract_json`] cannot parse are
39//! all logged and treated as "this cycle changes nothing." `crate::daemon`'s
40//! loop always falls through to its own `Queue::next_runnable` regardless of
41//! what happened here.
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::path::{Path, PathBuf};
45use std::time::Duration;
46
47use anyhow::{Context as _, Result};
48use serde::{Deserialize, Serialize};
49
50use crate::agent::{self, Invocation, SeatState};
51use crate::ask::{Deputy, Question, Questions};
52use crate::config::Config;
53use crate::prompt;
54use crate::queue::{Queue, Task, TaskStatus};
55use crate::run::RunState;
56use crate::verdict;
57
58/// Seat name for the conductor's own CLI-side conversation, scoped away from
59/// every other seat magi ever opens - the same rule every other seat follows.
60const SEAT: &str = "conduct";
61
62/// Node name reported to the invoked agent (`MAGI_NODE`) and recorded on any
63/// question it files, so an operator reading the questions list can tell a
64/// conductor's question from one a run's own agent asked.
65pub const NODE: &str = "conduct";
66
67/// Wall-clock limit for one conductor turn. The conductor reads a queue
68/// listing and replies with json; it does not implement anything or run a
69/// build, so this is short - the same order of magnitude as
70/// `crate::chat`'s own single-turn, no-write invocations.
71const TURN_TIMEOUT: Duration = Duration::from_secs(300);
72
73/// What the conductor may choose for a `running`-but-stalled or a
74/// `failed`/`held` task.
75#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
76#[serde(rename_all = "lowercase")]
77pub enum Recovery {
78    /// Put it back in line, attempts reset - the same effect as
79    /// `magi task release`.
80    Requeue,
81    /// Leave it for a human, unchanged otherwise - the same effect as
82    /// `magi task hold`.
83    Hold,
84    /// Reopen the task's surviving branch as a review-only pass
85    /// (`crate::graph::Runner::review`) instead of competing from scratch.
86    /// Only takes effect when [`surviving_branch`] can actually name one;
87    /// otherwise `crate::daemon` falls back to [`Recovery::Requeue`].
88    Review,
89    /// Close the task outright, as [`Task::succeed`] - the same effect as
90    /// `magi task done`. For a `failed`/`held` task only, never `running`:
91    /// this is for a task whose own goal is already known to be met outside
92    /// the loop entirely (the branch was merged and the worktree cleaned up
93    /// by hand, say) and competing it again would only spend attempts on
94    /// work that has nothing left to do, not for a task that merely stopped
95    /// mid-competition and might still need to run.
96    Done,
97}
98
99/// The operation for which the operator's decision is requested. These are
100/// fixed protocol values, independent of the language of the displayed choices.
101#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
102#[serde(rename_all = "snake_case")]
103pub enum QuestionOperation {
104    /// Choose an existing implementation as the recovery source.
105    Recover,
106    /// Continue an existing run from its saved state.
107    Resume,
108    /// Authorize repairing existing work before continuing.
109    Repair,
110    /// Confirm that the task may be closed as complete.
111    Complete,
112    /// Request an operator inspection of the target.
113    Inspect,
114    /// Request a configuration change for the target.
115    Configure,
116}
117
118/// The concrete object of an operator decision. Sources resolve to commit ids;
119/// runs must belong to this task. A generated prose label is never an identity.
120#[derive(Debug, Clone, Deserialize)]
121#[serde(tag = "kind", rename_all = "snake_case")]
122pub enum QuestionTarget {
123    /// The task itself, for decisions without a more specific recorded object.
124    Task,
125    /// A run already owned by this task.
126    Run {
127        /// Full recorded run id.
128        id: String,
129    },
130    /// A concrete TOML setting. Different fields on the same task require
131    /// different decisions even when both operations are `configure`.
132    Setting {
133        /// Repository-relative TOML file containing the setting.
134        path: String,
135        /// Exact table/key components copied from that file.
136        keys: Vec<String>,
137    },
138    /// A recovery source which the repository can resolve to a commit.
139    Source {
140        /// Existing branch name or commit reference, never a display label.
141        reference: String,
142    },
143}
144
145/// The structured meaning of a question, separate from its displayed wording.
146#[derive(Debug, Clone, Deserialize)]
147pub struct QuestionDecision {
148    /// The concrete type of operator decision requested.
149    pub operation: QuestionOperation,
150    /// The recorded object the decision concerns.
151    pub target: QuestionTarget,
152}
153
154impl QuestionDecision {
155    fn identity(&self, task: &Task) -> Result<(QuestionOperation, String)> {
156        let target = match &self.target {
157            QuestionTarget::Task => format!("task:{}", task.id),
158            QuestionTarget::Run { id } => {
159                anyhow::ensure!(
160                    task.runs.contains(id),
161                    "question target is not this task's run"
162                );
163                format!("run:{id}")
164            }
165            QuestionTarget::Setting { path, keys } => {
166                use std::path::Component;
167                anyhow::ensure!(
168                    !path.is_empty()
169                        && Path::new(path)
170                            .components()
171                            .all(|c| matches!(c, Component::Normal(_)))
172                        && !keys.is_empty()
173                        && keys.iter().all(|key| !key.is_empty()),
174                    "question setting must name a repository file and concrete keys"
175                );
176                let root = task.repo.canonicalize()?;
177                let file = root.join(path).canonicalize()?;
178                let relative = file
179                    .strip_prefix(&root)
180                    .context("question setting is outside the repository")?;
181                let content = std::fs::read_to_string(&file)?;
182                let value: toml::Value = toml::from_str(&content)?;
183                let mut setting = &value;
184                for key in keys {
185                    setting = setting
186                        .get(key)
187                        .context("question setting key does not exist")?;
188                }
189                format!("setting:{}", serde_json::to_string(&(relative, keys))?)
190            }
191            QuestionTarget::Source { reference } => {
192                use crate::proc::Quiet as _;
193                let output = std::process::Command::new("git")
194                    .args([
195                        "rev-parse",
196                        "--verify",
197                        "--end-of-options",
198                        &format!("{reference}^{{commit}}"),
199                    ])
200                    .current_dir(&task.repo)
201                    .quiet()
202                    .output()
203                    .context("resolve question source")?;
204                anyhow::ensure!(
205                    output.status.success(),
206                    "question source cannot be resolved"
207                );
208                let commit = String::from_utf8(output.stdout)?.trim().to_owned();
209                anyhow::ensure!(
210                    !commit.is_empty() && commit.bytes().all(|c| c.is_ascii_hexdigit()),
211                    "invalid question source commit"
212                );
213                format!("source:{commit}")
214            }
215        };
216        Ok((self.operation, target))
217    }
218}
219
220/// The conductor's decision for one task. Deliberately has no `priority`
221/// field: see this module's doc.
222#[derive(Debug, Clone, Default, Deserialize)]
223pub struct Decision {
224    /// Task id, expected to be copied verbatim from what it was shown.
225    pub id: String,
226    /// Ids this task should be blocked on. Meaningful only for a runnable
227    /// (`queued`) task, or a machine-held task whose missing-only decision
228    /// is unconfirmed against the queue. Ignored otherwise.
229    #[serde(default)]
230    pub blocked_by: Vec<String>,
231    /// Dependencies claimed missing as the sole reason for this question or
232    /// hold. Must name references in `blocked_by`. Empty (including legacy
233    /// replies) means the question/hold may have an independent purpose and
234    /// must be preserved. Never set for decisions needing separate consent.
235    #[serde(default)]
236    pub missing_dependencies: Vec<String>,
237    /// One line explaining the block or the recovery.
238    #[serde(default)]
239    pub reason: Option<String>,
240    /// Recovery for a stalled or finished task. For a runnable (`queued`)
241    /// one, only [`Recovery::Hold`] has any effect. An independent hold takes
242    /// precedence over `blocked_by`, so dependency completion cannot release it.
243    /// Use a hold when the operator already said the task must not compete,
244    /// rather than asking a `question` that restates that answer. `Requeue`, `Review`,
245    /// and `Done` have no meaning for a task that is already in line.
246    #[serde(default)]
247    pub recovery: Option<Recovery>,
248    /// A question for the operator. When present, [`apply`] files it (unless
249    /// one is already open for this task) and blocks the task on its id
250    /// instead of acting on `blocked_by` or `recovery`. Only a missing-only
251    /// decision explicitly marked by [`Self::missing_dependencies`] may be
252    /// replaced with dependency waiting unless the queue confirms missing blockers.
253    #[serde(default)]
254    pub question: Option<String>,
255    /// Fixed answers for `question`, if it has any. Empty means free text.
256    #[serde(default)]
257    pub choices: Vec<String>,
258    /// Stable operation and concrete target of a question. Missing on legacy
259    /// decisions: those share the task's existing cause, regardless of prose.
260    #[serde(default)]
261    pub question_decision: Option<QuestionDecision>,
262    /// Task or question ids carrying independently verifiable new blocking evidence.
263    #[serde(default)]
264    pub evidence: Vec<String>,
265}
266
267/// The conductor's whole reply for one cycle.
268///
269/// `decisions` is deliberately **not** `#[serde(default)]`, unlike every
270/// other field in this module. [`verdict::extract_json`] disambiguates
271/// between several balanced `{...}` spans in one reply by trying the type the
272/// caller wants against each of them, last first, and keeping the first that
273/// fits - which only works when a span that is not really the answer can
274/// fail to fit. A `Verdict` with no required field at all would make every
275/// span fit, including a `{}` left by stray trailing prose, and the reply's
276/// real `decisions` - earlier in the text - would never be reached. Requiring
277/// the key costs nothing: the prompt already asks for it on every reply, `[]`
278/// included.
279#[derive(Debug, Clone, Default, Deserialize)]
280pub struct Verdict {
281    /// One entry per task the conductor chose to say something about. A task
282    /// left out of this list is left exactly as it was.
283    pub decisions: Vec<Decision>,
284}
285
286/// A view of a task built for [`prompt::conduct`], shared by the runnable and
287/// stalled sections.
288fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
289    prompt::ConductTask {
290        id: t.id.clone(),
291        title: t.title.clone(),
292        instruction: t.instruction.clone(),
293        repo: t.repo.display().to_string(),
294        priority: t.priority,
295        status: t.status.as_str().to_owned(),
296        attempts: t.attempts,
297        max_attempts: t.effective_attempt_limit(max_attempts),
298        last_error: t.last_error.clone(),
299        hold_reason: t.hold_reason.clone(),
300        hold_source: t.hold_source.map(|source| source.label().to_owned()),
301        blocked_by: t.blocked_by.clone(),
302        answers: t
303            .answers
304            .iter()
305            .map(|a| prompt::ConductAnswer {
306                question: a.question.clone(),
307                answer: a.answer.clone(),
308            })
309            .collect(),
310        operator_resume: t.resumption.as_ref().map(|r| format!(
311            "Explicit operator resumption generation {} via {} at {}. Approved instruction: {}. \
312             The first {} historical answers and prior hold ({:?}) are superseded as waiting premises. \
313             To block again cite evidence ids from a later owned question, changed dependency, or new failed run; \
314             wording alone is insufficient.",
315            r.generation, r.source, r.at, r.instruction, r.answers_through, r.cleared_reason
316        )).or_else(|| t.resume_override.as_ref().map(|o| {
317            format!(
318                "the operator explicitly answered \"resume\" at {}; do not hold this \
319                 task again for the same reason unless there is new information",
320                o.at
321            )
322        })),
323    }
324}
325
326/// Did the operator answer a `magi ask` choice with a `resume` action that
327/// this task has not yet acted on? Then `Recovery::Requeue` must not turn the
328/// task into a fresh competition either.
329fn pinned_resume(task: &Task) -> bool {
330    task.resume_override
331        .as_ref()
332        .is_some_and(|o| o.pinned_run.is_some())
333}
334
335/// Verify references independently of the model's reason text. Old evidence
336/// cannot become new merely by being described differently.
337fn new_blocking_evidence(
338    queue: &Queue,
339    questions: &Questions,
340    task: &Task,
341    d: &Decision,
342) -> Vec<String> {
343    let Some(r) = &task.resumption else {
344        return Vec::new();
345    };
346    d.evidence
347        .iter()
348        .filter_map(|id| {
349            let version = if let Ok(q) = questions.get(id) {
350                (q.asked_at > r.at
351                    && (q.run == task.id || task.runs.contains(&q.run))
352                    && q.node != NODE
353                    && q.status != crate::ask::QuestionStatus::Abandoned)
354                    .then(|| {
355                        format!(
356                            "question:{id}:{}:{:?}:{}",
357                            q.asked_at,
358                            q.status,
359                            q.resolution().unwrap_or_default()
360                        )
361                    })
362            } else if d.blocked_by.contains(id) {
363                queue
364                    .get(id)
365                    .ok()
366                    .filter(|dep| {
367                        dep.id != task.id
368                            && dep.status != TaskStatus::Done
369                            && (dep.created_at > r.at
370                                || dep.held_at.is_some_and(|at| at > r.at)
371                                || dep.operator_decided_at.is_some_and(|at| at > r.at)
372                                || dep.passes.iter().any(|p| p.settled_at > r.at)
373                                || (dep.status == TaskStatus::Blocked
374                                    && dep.blocked_at.is_some_and(|at| at > r.at)))
375                    })
376                    .map(|dep| {
377                        format!(
378                            "dependency:{id}:{:?}:{:?}:{:?}",
379                            dep.status,
380                            dep.chat_notice_key(),
381                            dep.hold_notice_cause
382                        )
383                    })
384            } else if task.runs.iter().any(|run| run == id) {
385                RunState::load(id)
386                    .ok()
387                    .filter(|run| {
388                        // A resumed run keeps its id. Only a pass started after
389                        // approval (or a newly created legacy run) is new evidence.
390                        run.updated_at > r.at
391                            && run
392                                .passes
393                                .last()
394                                .map_or(run.created_at > r.at, |p| p.started_at > r.at)
395                            && matches!(
396                                run.status,
397                                crate::run::RunStatus::Failed
398                                    | crate::run::RunStatus::Blocked
399                                    | crate::run::RunStatus::Stalled
400                            )
401                    })
402                    .map(|run| {
403                        format!(
404                            "run:{id}:{:?}:{:?}",
405                            run.status,
406                            run.passes.last().map(|p| p.index)
407                        )
408                    })
409            } else {
410                None
411            };
412            version.filter(|v| !r.blocking_evidence.contains(v))
413        })
414        .collect()
415}
416
417/// Durable approval is enforced by verified evidence at the mutation boundary.
418/// A pinned run stays protected; legacy transient overrides keep their original
419/// bounded contradiction policy until an explicit new approval replaces it.
420fn may_hold(task: &mut Task, reason: &str) -> bool {
421    if pinned_resume(task) {
422        return false;
423    }
424    if task.resumption.is_some() {
425        return true; // The mutation boundary already verified new evidence.
426    }
427    let Some(o) = task.resume_override.as_mut() else {
428        return true;
429    };
430    if o.forced || o.pinned_run.is_some() {
431        tracing::warn!(
432            "conductor tried to hold task {} after the operator forced a resume: {reason}",
433            task.id
434        );
435        return false;
436    }
437    if o.conductor_rehold.is_some() {
438        return false;
439    }
440    o.conductor_rehold = Some(reason.to_owned());
441    true
442}
443
444/// Severity as a lowercase word, matching how `crate::verdict::Severity` is
445/// spelled everywhere else an operator or a model reads it.
446fn severity_str(s: crate::verdict::Severity) -> &'static str {
447    match s {
448        crate::verdict::Severity::Nit => "nit",
449        crate::verdict::Severity::Minor => "minor",
450        crate::verdict::Severity::Major => "major",
451        crate::verdict::Severity::Blocker => "blocker",
452    }
453}
454
455/// The branch a task's last run left behind, if the tally ever ran on it -
456/// what [`Recovery::Review`] reopens. Derived from `crate::run::RunState`
457/// alone, never from anything the conductor wrote, so a hallucinated branch
458/// name can never reach `crate::graph::Runner::review`.
459fn surviving_branch(task: &Task) -> Option<String> {
460    let last = task.runs.last()?;
461    let state = RunState::load(last).ok()?;
462    if !crate::daemon::compatible_run_instruction(&state, task) {
463        return None;
464    }
465    state.winner().map(|c| c.branch.clone())
466}
467
468/// The reason to record when the conductor holds a `failed`/`held` task via
469/// [`Recovery::Hold`]. Always `Some`, never a bare `d.reason.clone()`.
470///
471/// [`Task::hold_machine`] only overwrites [`Task::hold_reason`] when given
472/// one, precisely so a stalled task's existing diagnosis survives a
473/// decision that has nothing new to add. That is the right default when the
474/// task was not already `held` - but a task that *is* already `held`, and
475/// is being held again here, is a different case: its current
476/// `hold_reason` may still read as a cause `crate::triage` knows how to
477/// re-check on its own (a disk-pressure message, say - see
478/// `triage::machine_cause_resolved`), and if the model gives no reason,
479/// leaving that text untouched would make this decision - the conductor
480/// choosing, informed by the operator's own answer, to keep the task held
481/// anyway - indistinguishable from the original, never-reconsidered hold.
482/// `crate::triage` would then read the stale text the next time its one
483/// recognised cause looks resolved and release the task straight through
484/// the decision this call was recording. Prepending (not appending) the new
485/// note keeps the old text as context without leaving the string starting
486/// with whatever pattern `crate::triage` matched before.
487///
488/// Two bounds keep the string from growing, which it once did without limit
489/// (every re-hold nested the whole previous reason one level deeper, and every
490/// change relit the task's notice): a note equal to the current reason - or to
491/// its outermost note - adds nothing, so the result is `None` and the caller
492/// writes nothing; and only ONE `(previously: ...)` level is ever kept, the
493/// prior being cut back to its outermost note before it is embedded.
494fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> Option<String> {
495    let note = match &d.reason {
496        Some(reason) => reason.clone(),
497        None => match task.answers.last() {
498            Some(a) => format!(
499                "conduct held this again with no new reason given; last operator \
500                 answer on record: {}",
501                a.answer
502            ),
503            None => "conduct held this again with no reason given".to_owned(),
504        },
505    };
506    match task.hold_reason.as_deref() {
507        Some(prior) if !prior.is_empty() => {
508            let outer = outermost_hold_note(prior);
509            if note.trim() == prior.trim() || note.trim() == outer {
510                None
511            } else {
512                Some(format!("{note}\n\n(previously: {outer})"))
513            }
514        }
515        _ => Some(note),
516    }
517}
518
519/// Separator [`reaffirmed_hold_reason`] puts between a note and the reason it
520/// replaced.
521const PREVIOUSLY: &str = "\n\n(previously: ";
522
523/// A hold reason without any `(previously: ...)` tail: the text before the
524/// first separator, trimmed.
525fn outermost_hold_note(reason: &str) -> &str {
526    reason.split(PREVIOUSLY).next().unwrap_or(reason).trim()
527}
528
529/// The conductor's stated reason for a hold, or a stand-in when it gave none.
530fn hold_note(d: &Decision) -> String {
531    d.reason
532        .clone()
533        .unwrap_or_else(|| "(no reason given)".to_owned())
534}
535
536/// Everything the conductor is shown about a `failed`/`held` task's last run.
537async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
538    let Some(run_id) = task.runs.last().cloned() else {
539        return prompt::ConductOutcome {
540            run_id: "(none)".to_owned(),
541            unreadable: Some("this task has not produced a run yet".to_owned()),
542            run_status: None,
543            open_findings: Vec::new(),
544            rounds_used: 0,
545            rounds_max: 0,
546            rounds: Vec::new(),
547            branch: None,
548            branch_head: None,
549            references: None,
550            empty_candidate: false,
551        };
552    };
553    let state = match RunState::load(&run_id) {
554        Ok(s) => s,
555        Err(e) => {
556            // The exact failure this feature exists to stop hiding: a schema
557            // mismatch (or any other unreadable state) must never be treated
558            // as "nothing to recover" - it is surfaced here, verbatim, rather
559            // than swallowed into a quiet re-competition.
560            tracing::warn!(
561                "conductor: could not read run {run_id} for task {}: {e:#}",
562                task.short()
563            );
564            return prompt::ConductOutcome {
565                run_id,
566                unreadable: Some(format!("{e:#}")),
567                run_status: None,
568                open_findings: Vec::new(),
569                rounds_used: 0,
570                rounds_max: 0,
571                rounds: Vec::new(),
572                branch: None,
573                branch_head: None,
574                references: None,
575                empty_candidate: false,
576            };
577        }
578    };
579
580    let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
581        id: f.id.clone(),
582        title: f.title.clone(),
583        severity: severity_str(f.severity).to_owned(),
584    };
585    let open_findings = state
586        .open_findings()
587        .into_iter()
588        .map(finding_view)
589        .collect();
590    let rounds = state
591        .reviews
592        .iter()
593        .map(|r| prompt::ConductRound {
594            round: r.round,
595            findings: r
596                .reviews
597                .iter()
598                .flat_map(|rec| rec.findings.iter())
599                .map(finding_view)
600                .collect(),
601            addressed: r
602                .fix
603                .as_ref()
604                .map(|fx| fx.addressed.clone())
605                .unwrap_or_default(),
606            rejected: r
607                .fix
608                .as_ref()
609                .map(|fx| {
610                    fx.rejected
611                        .iter()
612                        .map(|rej| prompt::ConductRejection {
613                            id: rej.id.clone(),
614                            why: rej.why.clone(),
615                        })
616                        .collect()
617                })
618                .unwrap_or_default(),
619        })
620        .collect();
621    let branch = state.winner().map(|c| c.branch.clone());
622    let branch_head = match &branch {
623        Some(b) => crate::git::rev_parse(repo, b)
624            .await
625            .ok()
626            .map(|h| h.chars().take(8).collect()),
627        None => None,
628    };
629
630    prompt::ConductOutcome {
631        run_id,
632        unreadable: None,
633        run_status: Some(state.status.as_str().to_owned()),
634        open_findings,
635        rounds_used: state.reviews.len(),
636        rounds_max: state.config.graph.review_rounds,
637        rounds,
638        branch,
639        branch_head,
640        references: crate::refs::describe(&state.seeds),
641        empty_candidate: state.merge.as_ref().is_some_and(|m| m.empty),
642    }
643}
644
645/// Whether the task's latest run's winner branch already has an open pull
646/// request into the run's own base - a fact the repository can answer, so the
647/// operator is never asked it. `None` when there is no run or winner to check.
648async fn open_pr_fact(task: &Task, repo: &Path) -> Option<String> {
649    let id = task.runs.last()?;
650    let state = match RunState::load(id) {
651        Ok(s) => s,
652        Err(e) => return Some(format!("could not check open pull requests: {e:#}")),
653    };
654    let branch = state.winner()?.branch.clone();
655    let base = state.base_branch.clone();
656    match crate::land::find_open_pr(repo, &branch, &base).await {
657        Ok(crate::land::OpenPr::None) => None,
658        Ok(crate::land::OpenPr::One { url, .. }) => Some(format!(
659            "pull request {url} is already open for {branch} into {base}"
660        )),
661        Ok(crate::land::OpenPr::Many(urls)) => Some(format!(
662            "several pull requests are already open for {branch} into {base}: {}",
663            urls.join(" ")
664        )),
665        Err(e) => Some(format!("could not check open pull requests: {e:#}")),
666    }
667}
668
669/// Append what the repository says about the branches and commits a task
670/// names to every question the conductor is about to file, so the operator is
671/// never asked a fact git can answer ("is it already on main?"). Done here,
672/// in code, rather than trusted to the model: a question the model wrote
673/// without checking still reaches the operator with the answer attached.
674/// When it cannot be checked the question says so instead of guessing.
675async fn attach_facts(cfg: &Config, repo: &Path, queue: &Queue, verdict: &mut Verdict) {
676    for d in verdict
677        .decisions
678        .iter_mut()
679        .filter(|d| d.question.is_some())
680    {
681        let Ok(task) = queue.get(&d.id) else {
682            continue;
683        };
684        let text = format!("{}\n{}", task.title, task.instruction);
685        let pr_fact = open_pr_fact(&task, repo_for(&task, repo).as_path()).await;
686        if crate::refs::scan(&text).is_empty() {
687            if let (Some(fact), Some(q)) = (pr_fact, d.question.as_mut()) {
688                q.push_str(&format!(
689                    "\n\nChecked against the repository (magi did this, not the model):\n{fact}"
690                ));
691            }
692            continue;
693        }
694        let repo = repo_for(&task, repo);
695        let remote = &cfg.merge.remote;
696        let base = crate::git::merge_base_branch(&repo, remote, cfg.merge.base.as_deref())
697            .await
698            .ok();
699        let facts = match base {
700            Some(base) => {
701                let base_name = base.clone();
702                let tracking = format!("{remote}/{base}");
703                // Best effort: a stale tracking ref would report work that
704                // has since landed as still missing from the base.
705                let refreshed = crate::git::fetch(&repo, remote, &base)
706                    .await
707                    .is_ok_and(|o| o.ok());
708                // Only the remote's tip counts: a local branch of the same
709                // name is whatever the checkout happens to hold.
710                let against = tracking;
711                match crate::git::rev_parse(&repo, &against).await {
712                    Ok(tip) => crate::refs::describe(
713                        &crate::refs::resolve(&repo, &tip, remote, &text).await,
714                    )
715                    .map(|facts| {
716                        if refreshed {
717                            facts
718                        } else {
719                            format!(
720                                "{facts}\n(could not fetch {remote}/{base_name}: this is \
721                                 against the last fetched `{against}`, which may be behind \
722                                 the remote)"
723                            )
724                        }
725                    }),
726                    Err(_) => Some(format!(
727                        "could NOT be checked against the repository: {against} is not \
728                         available here, so whether the work is in the base is unknown"
729                    )),
730                }
731            }
732            None => Some("could not check the repository: no base branch known".to_owned()),
733        };
734        let facts = match (facts, pr_fact) {
735            (Some(f), Some(p)) => Some(format!("{f}\n{p}")),
736            (f, p) => f.or(p),
737        };
738        if let (Some(facts), Some(q)) = (facts, d.question.as_mut()) {
739            q.push_str(&format!(
740                "\n\nChecked against the repository (magi did this, not the model):\n{facts}"
741            ));
742        }
743    }
744}
745
746/// A `failed`/`held` task together with how its last run ended.
747async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
748    prompt::ConductFinished {
749        task: view(t, max_attempts),
750        outcome: outcome_for(t, &repo_for(t, repo)).await,
751    }
752}
753
754/// The repository containing a task's branch. A task filed without a
755/// repository uses the daemon's repository, exactly as its later attempt does.
756fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
757    if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
758        fallback.to_path_buf()
759    } else {
760        task.repo.clone()
761    }
762}
763
764/// Apply one decision to the queue and the question store.
765///
766/// Takes the task's own claim before touching it: a model call spans a whole
767/// agent turn, and the queue can have moved on by the time its answer comes
768/// back. A claim that cannot be taken means something else owns this task
769/// right now - most often a live daemon mid-competition on it - so the
770/// conductor's now-stale view of it is dropped rather than raced against; see
771/// `crate::queue::Queue::claim`'s own doc on why a claim is proof, not a
772/// guess.
773///
774/// A decision is matched against the task's *current* status, re-read under
775/// the claim, not against whichever section of the prompt it came from: a
776/// `blocked_by` only takes effect on a `queued` task, and `recovery` only on
777/// one `running` (stalled) or `failed`/`held`, so a decision that no longer
778/// matches what the task actually is - it moved on between the read that
779/// built the prompt and this write - changes nothing.
780fn apply_one(
781    queue: &Queue,
782    questions: &Questions,
783    d: &Decision,
784    snapshot: Option<&Task>,
785) -> Result<()> {
786    let _claim = queue
787        .claim(&d.id)
788        .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
789    let mut task = queue.get(&d.id).context("no such task")?;
790
791    // Compare the actual input, not a generation echoed by the model. Even
792    // question creation must be discarded after an intervening operator edit.
793    if snapshot.is_some_and(|old| decision_input(old) != decision_input(&task)) {
794        return Ok(());
795    }
796    let blocking =
797        d.question.is_some() || !d.blocked_by.is_empty() || d.recovery == Some(Recovery::Hold);
798    let evidence = new_blocking_evidence(queue, questions, &task, d);
799    if blocking && task.resumption.is_some() && evidence.is_empty() {
800        crate::notices::raise(
801            crate::notices::Notice::warn(
802                &format!("conductor-resumption:{}", task.id),
803                format!(
804                    "Conductor concern after explicit resumption (no verified new evidence): {}; question: {:?}; dependencies: {:?}; evidence: {:?}",
805                    hold_note(d), d.question, d.blocked_by, d.evidence
806                ),
807            )
808            .about([task.id.clone()]),
809        );
810        return Ok(());
811    }
812
813    if blocking && !evidence.is_empty() {
814        let mut cause = evidence.clone();
815        cause.sort();
816        cause.dedup();
817        task.notice_cause(
818            &format!("conductor-evidence:{}", cause.join(",")),
819            None,
820            None,
821        );
822    }
823    if blocking {
824        if let Some(r) = task.resumption.as_mut() {
825            r.blocking_evidence.extend(evidence);
826        }
827    }
828
829    // A conductor answer is never operator authorization.  In particular,
830    // do this before questions and blocking too: either would reclassify a
831    // manual hold and let a later deterministic resolver queue it.
832    if task.operator_held() {
833        return Ok(());
834    }
835
836    // A claimed task may have deferred the pre-scheduling triage sweep.
837    // Leave an open or answered-unapplied question to that sweep rather
838    // than move the task out from under it.
839    if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
840        return Ok(());
841    }
842
843    // The operator's `resume` action is waiting for its turn: a question or a
844    // dependency would park the task before the daemon can run it.
845    let pinned = pinned_resume(&task);
846    if pinned && (d.question.is_some() || !d.blocked_by.is_empty()) {
847        return Ok(());
848    }
849
850    // A missing-only decision with unverifiable aliases is not evidence for
851    // a question or hold. Do not guess that two ambiguous/unknown references
852    // mean the same record, or treat failure to prove that as proof of absence.
853    if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
854        && !d.missing_dependencies.is_empty()
855        && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
856        && d.missing_dependencies
857            .iter()
858            .any(|id| !d.blocked_by.contains(id))
859        && d.missing_dependencies
860            .iter()
861            .chain(&d.blocked_by)
862            .any(|id| {
863                matches!(
864                    crate::queue::dependency(queue, questions, id),
865                    crate::queue::Dependency::Ambiguous | crate::queue::Dependency::Unknown
866                )
867            })
868    {
869        return Ok(());
870    }
871
872    // Only an explicit missing-only claim can justify replacing a question
873    // or hold. Existing prerequisites alone say nothing about independent
874    // consent, destructive-operation questions, or other reasons to stay held.
875    // Ambiguous or unreadable inventories cannot confirm absence either: keep
876    // the original reference waiting until it can be resolved authoritatively.
877    if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
878        && !d.missing_dependencies.is_empty()
879        && d.missing_dependencies.iter().all(|id| {
880            let claim = crate::queue::dependency(queue, questions, id);
881            claim != crate::queue::Dependency::Missing
882                && d.blocked_by.iter().any(|blocker| {
883                    blocker == id
884                        || (matches!(
885                            &claim,
886                            crate::queue::Dependency::Task(_)
887                                | crate::queue::Dependency::Question(_)
888                                | crate::queue::Dependency::Deleted(_)
889                        ) && crate::queue::dependency(queue, questions, blocker) == claim)
890                })
891        })
892        && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
893        && crate::queue::missing_blockers(queue, questions, &d.blocked_by).is_empty()
894    {
895        let mut ids = d.blocked_by.clone();
896        crate::queue::normalize_blockers(queue, questions, &mut ids);
897        let reason = format!("Waiting on dependencies: {}", ids.join(", "));
898        task.block(ids, Some(reason));
899        return queue.put(&mut task);
900    }
901
902    if let Some(text) = &d.question {
903        if task.status == TaskStatus::Done {
904            return Ok(());
905        }
906        let mut held = task.clone();
907        held.status = TaskStatus::Held;
908        let targets: BTreeSet<_> = d
909            .blocked_by
910            .iter()
911            .filter_map(|id| {
912                if let Ok(dep) = queue.get(id) {
913                    Some(format!("task:{id}:{:?}", dep.chat_notice_key()))
914                } else if questions.get(id).is_ok() {
915                    Some(format!("question:{id}"))
916                } else {
917                    None
918                }
919            })
920            .collect();
921        let decision = d
922            .question_decision
923            .as_ref()
924            .and_then(|q| match q.identity(&task) {
925                Ok(identity) => Some(identity),
926                Err(error) => {
927                    // Invalid optional metadata must not discard the question
928                    // or leave a runnable task without its decision barrier.
929                    tracing::warn!(task = %task.id, %error, "ignoring invalid question decision target");
930                    None
931                }
932            });
933        let context_value =
934            serde_json::json!([&task.instruction, held.chat_notice_key(), targets, decision]);
935        let context = serde_json::to_string(&context_value)?;
936        // An answer alone does not justify asking the same decision again.
937        // New producer evidence, an instruction edit or a different dependency
938        // target can open another question; rewording cannot.
939        if questions.list().iter().any(|q| {
940            q.node == NODE
941                && q.run == task.id
942                && q.status == crate::ask::QuestionStatus::Answered
943                && q.decision_context.as_ref().is_some_and(|stored| {
944                    if stored == &context {
945                        return true;
946                    }
947                    // Missing metadata cannot manufacture a new target. This
948                    // also baselines older contexts whose last item was labels.
949                    decision.is_none()
950                        && serde_json::from_str::<serde_json::Value>(stored)
951                            .ok()
952                            .and_then(|v| {
953                                v.as_array().map(|items| {
954                                    items.len() == 4
955                                        && items[..3]
956                                            == context_value.as_array().expect("context")[..3]
957                                })
958                            })
959                            == Some(true)
960                })
961        }) {
962            return Ok(());
963        }
964        // `Question::run` is a task id for conductor questions, but ordinary
965        // graph questions use it as a run id. A coincidental equality must
966        // not block this task on an answer meant for another node.
967        let question_id = match questions
968            .list()
969            .into_iter()
970            .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
971        {
972            Some(existing) => existing.id,
973            None => {
974                let mut q = Question::new(
975                    task.id.clone(),
976                    NODE.to_owned(),
977                    SEAT.to_owned(),
978                    text.clone(),
979                    d.reason.clone().unwrap_or_default(),
980                    d.choices.clone(),
981                );
982                q.decision_context = Some(context);
983                // The wait is handed to a deputy (`crate::deputy`), so a
984                // free-text reply reaches an agent that knows why this was
985                // asked. The conductor still never blocks: this only records
986                // what the deputy will be told, and `serve` starts it.
987                q.deputy = Some(Deputy::new(crate::deputy::brief(
988                    &task.id,
989                    d.reason.as_deref().unwrap_or_default(),
990                    &q.choices,
991                    &q.actions,
992                )));
993                if task.repo.is_dir() && task.repo != Path::new(".") {
994                    q.cwd = Some(task.repo.to_string_lossy().into_owned());
995                }
996                questions.put(&mut q)?;
997                q.id
998            }
999        };
1000        task.block(vec![question_id], d.reason.clone());
1001        return queue.put(&mut task);
1002    }
1003
1004    match task.status {
1005        // See `Decision::recovery`'s doc: the only lever a runnable task has
1006        // besides `blocked_by` is holding it outright, for a task whose
1007        // answers already say it should not compete again.
1008        TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
1009            if may_hold(&mut task, &hold_note(d)) {
1010                task.hold_machine(d.reason.clone());
1011                queue.put(&mut task)?;
1012            }
1013        }
1014        TaskStatus::Queued if !d.blocked_by.is_empty() => {
1015            let mut ids = d.blocked_by.clone();
1016            crate::queue::normalize_blockers(queue, questions, &mut ids);
1017            task.block(ids, d.reason.clone());
1018            queue.put(&mut task)?;
1019        }
1020        TaskStatus::Running => match d.recovery {
1021            Some(Recovery::Requeue) if pinned_resume(&task) => {}
1022            Some(Recovery::Requeue) => {
1023                task.requeue();
1024                queue.put(&mut task)?;
1025            }
1026            Some(Recovery::Hold) if may_hold(&mut task, &hold_note(d)) => {
1027                task.hold_machine(d.reason.clone());
1028                queue.put(&mut task)?;
1029            }
1030            // `Review` reopens a branch, which only makes sense once a run
1031            // has actually stopped; a task still `running` has nothing to
1032            // reopen yet.
1033            _ => {}
1034        },
1035        TaskStatus::Failed | TaskStatus::Held => match d.recovery {
1036            // The operator picked a specific run to continue; a fresh
1037            // competition would throw that run away.
1038            Some(Recovery::Requeue) if pinned_resume(&task) => {}
1039            Some(Recovery::Requeue) => {
1040                task.requeue();
1041                queue.put(&mut task)?;
1042            }
1043            Some(Recovery::Hold) => {
1044                // Decided before `may_hold`, which spends the one override: a
1045                // note that adds nothing writes nothing (and so raises no
1046                // notice and moves no queue revision).
1047                if let Some(reason) = reaffirmed_hold_reason(&task, d) {
1048                    if may_hold(&mut task, &hold_note(d)) {
1049                        task.hold_machine(Some(reason));
1050                        queue.put(&mut task)?;
1051                    }
1052                }
1053            }
1054            Some(Recovery::Review) => {
1055                if let Some(branch) = surviving_branch(&task) {
1056                    task.request_review(branch);
1057                    queue.put(&mut task)?;
1058                }
1059                // No survivable branch: a decision naming `review` here is
1060                // simply not actionable, and is dropped rather than guessed
1061                // at - `crate::daemon` applies the same "no branch, no
1062                // review" rule again, from its own read, right before it
1063                // would actually start the run.
1064            }
1065            Some(Recovery::Done) => {
1066                task.succeed();
1067                // Same reason `magi task done` and the web UI's equivalent
1068                // do this: the conductor closing a task by hand is just as
1069                // much a finished story as the loop's own settle path.
1070                crate::daemon::supersede_prior_runs(&task, &crate::run::home());
1071                queue.put(&mut task)?;
1072            }
1073            None => {}
1074        },
1075        // `queued` with nothing to block on, `done`, or already `blocked`:
1076        // nothing for this decision to do.
1077        _ => {}
1078    }
1079    Ok(())
1080}
1081
1082/// Apply every decision in `verdict`. A single bad decision - a task id that
1083/// no longer exists, one already claimed elsewhere - is logged and skipped
1084/// rather than losing every other decision in the same reply.
1085pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
1086    for d in &verdict.decisions {
1087        if let Err(e) = apply_one(queue, questions, d, None) {
1088            tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1089        }
1090    }
1091    Ok(())
1092}
1093
1094/// Where the conductor's seat is kept between turns, so the daemon waiter can
1095/// resume it when the owner answers a question it asked.
1096///
1097/// A graph seat's session lives in its run's `RunState`; the conductor has no
1098/// run, and until this file its seat existed only in memory.
1099pub fn seat_path(home: &Path) -> PathBuf {
1100    home.join("conduct").join("seat.json")
1101}
1102
1103/// The seat the conductor last used, if it has taken a turn.
1104pub fn load_seat(home: &Path) -> Option<SeatState> {
1105    serde_json::from_str(&std::fs::read_to_string(seat_path(home)).ok()?).ok()
1106}
1107
1108fn busy_path(home: &Path) -> PathBuf {
1109    home.join("conduct").join("busy")
1110}
1111
1112/// Is a conductor turn in flight right now?
1113///
1114/// A marker file rather than shared memory because the waiter reads it and the
1115/// conductor is not necessarily in its process. A marker older than one turn's
1116/// timeout (plus slack) was left by a turn that died and counts for nothing.
1117pub fn busy(home: &Path) -> bool {
1118    std::fs::metadata(busy_path(home))
1119        .and_then(|m| m.modified())
1120        .ok()
1121        .and_then(|t| t.elapsed().ok())
1122        .is_some_and(|age| age < TURN_TIMEOUT + Duration::from_secs(30))
1123}
1124
1125/// Removes the busy marker when a turn ends, however it ends.
1126struct Busy(PathBuf);
1127
1128impl Busy {
1129    fn mark(home: &Path) -> Self {
1130        let path = busy_path(home);
1131        if let Some(dir) = path.parent() {
1132            let _ = std::fs::create_dir_all(dir);
1133        }
1134        let _ = std::fs::write(&path, std::process::id().to_string());
1135        Self(path)
1136    }
1137}
1138
1139impl Drop for Busy {
1140    fn drop(&mut self) {
1141        let _ = std::fs::remove_file(&self.0);
1142    }
1143}
1144
1145/// The conductor's state across polling cycles: its own CLI-side conversation
1146/// and the last (revision, stalled ∪ finished ids) pair it actually acted on.
1147#[derive(Debug, Default)]
1148pub struct Conductor {
1149    seat: Option<SeatState>,
1150    last_seen: Option<(u64, BTreeSet<String>)>,
1151    /// Content fingerprint of each held task the model was last shown, see
1152    /// [`input_fingerprint`]. A held task whose fingerprint still matches is
1153    /// settled: the conductor's own rewrites of it do not count as news.
1154    considered: BTreeMap<String, String>,
1155}
1156
1157/// What the conductor's decision about a held task is a function of: the task
1158/// without the fields the conductor itself rewrites (`updated_at`,
1159/// `hold_reason`, the recorded re-hold). Independent of how the model words a
1160/// repeated decision.
1161fn decision_input(task: &Task) -> serde_json::Value {
1162    let mut v = serde_json::to_value(task).unwrap_or_default();
1163    if let Some(o) = v.as_object_mut() {
1164        o.remove("chat_notice");
1165        o.remove("chat_notices");
1166        // A delivery write changes only bookkeeping, not the decision input.
1167        o.remove("updated_at");
1168    }
1169    v
1170}
1171
1172fn input_fingerprint(task: &Task) -> String {
1173    let mut v = decision_input(task);
1174    if let Some(o) = v.as_object_mut() {
1175        o.remove("hold_reason");
1176        if let Some(ro) = o.get_mut("resume_override").and_then(|r| r.as_object_mut()) {
1177            ro.remove("conductor_rehold");
1178        }
1179    }
1180    v.to_string()
1181}
1182
1183impl Conductor {
1184    /// A conductor that has never run.
1185    #[must_use]
1186    pub fn new() -> Self {
1187        Self::default()
1188    }
1189
1190    /// Held tasks among `stalled` / `finished` whose input is unchanged since
1191    /// the model last saw them. Their files are left out of the revision, so
1192    /// the conductor re-holding one (however it words it) is not news, while
1193    /// an owner's answer changes the fingerprint and brings the file back in.
1194    fn settled(
1195        considered: &BTreeMap<String, String>,
1196        stalled: &[Task],
1197        finished: &[Task],
1198    ) -> BTreeSet<String> {
1199        stalled
1200            .iter()
1201            .chain(finished)
1202            .filter(|t| t.status == TaskStatus::Held)
1203            .filter(|t| considered.get(&t.id) == Some(&input_fingerprint(t)))
1204            .map(|t| t.id.clone())
1205            .collect()
1206    }
1207
1208    fn snapshot_with(
1209        considered: &BTreeMap<String, String>,
1210        queue: &Queue,
1211        stalled: &[Task],
1212        finished: &[Task],
1213    ) -> (u64, BTreeSet<String>) {
1214        let ids = stalled
1215            .iter()
1216            .chain(finished)
1217            .map(|t| t.id.clone())
1218            .collect();
1219        let skip = Self::settled(considered, stalled, finished);
1220        (queue.revision_excluding(&skip), ids)
1221    }
1222
1223    fn snapshot(
1224        &self,
1225        queue: &Queue,
1226        stalled: &[Task],
1227        finished: &[Task],
1228    ) -> (u64, BTreeSet<String>) {
1229        Self::snapshot_with(&self.considered, queue, stalled, finished)
1230    }
1231
1232    /// Whether calling the conductor could possibly do anything different
1233    /// from last time: [`Queue::revision`] moved, or the set of stalled and
1234    /// finished task ids changed.
1235    ///
1236    /// Deliberately **not** "stalled or finished is non-empty" - a task
1237    /// sitting stalled or finished with nobody changing anything about it
1238    /// must not be re-shown to the model every single poll forever; only a
1239    /// change in *which* tasks are stalled or finished, or a queue write
1240    /// changing something about a runnable one, is worth another look.
1241    ///
1242    /// Cheap and config-free on purpose, so `crate::daemon`'s poll loop can
1243    /// skip `Config::discover`'s synchronous I/O entirely on a cycle where
1244    /// this says no - which [`Conductor::maybe_run`] would otherwise only
1245    /// discover after paying for that load. Both ask the identical question,
1246    /// from the same [`Conductor::last_seen`], so they can never disagree
1247    /// about whether there is anything to look at.
1248    #[must_use]
1249    pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
1250        self.last_seen.as_ref() != Some(&self.snapshot(queue, stalled, finished))
1251    }
1252
1253    /// Call the conductor once, unless nothing has changed since the last
1254    /// time it was worth calling - see [`Conductor::worth_a_look`], the exact
1255    /// same test. Never fatal - see this module's doc.
1256    #[allow(clippy::too_many_arguments)]
1257    pub async fn maybe_run(
1258        &mut self,
1259        cfg: &Config,
1260        repo: &Path,
1261        queue: &Queue,
1262        questions: &Questions,
1263        home: &Path,
1264        queued: &[Task],
1265        stalled: &[Task],
1266        finished: &[Task],
1267        max_attempts: usize,
1268    ) {
1269        if self.last_seen.as_ref() == Some(&self.snapshot(queue, stalled, finished)) {
1270            return;
1271        }
1272        // What the model is about to be shown becomes "considered" before the
1273        // snapshot is taken, so its own re-hold writes land in the excluded set.
1274        self.considered = stalled
1275            .iter()
1276            .chain(finished)
1277            .filter(|t| t.status == TaskStatus::Held)
1278            .map(|t| (t.id.clone(), input_fingerprint(t)))
1279            .collect();
1280        self.last_seen = Some(self.snapshot(queue, stalled, finished));
1281        if let Err(e) = self
1282            .run_once(
1283                cfg,
1284                repo,
1285                queue,
1286                questions,
1287                home,
1288                queued,
1289                stalled,
1290                finished,
1291                max_attempts,
1292            )
1293            .await
1294        {
1295            // The conductor's own write moves the revision, so one more cycle
1296            // follows; a repeat decision writes nothing (see
1297            // `reaffirmed_hold_reason`), which is what ends the loop. The
1298            // revision is deliberately never re-read here: it cannot tell our
1299            // write from an owner's concurrent update.
1300            tracing::warn!("conductor: {e:#}");
1301        }
1302    }
1303
1304    #[allow(clippy::too_many_arguments)]
1305    async fn run_once(
1306        &mut self,
1307        cfg: &Config,
1308        repo: &Path,
1309        queue: &Queue,
1310        questions: &Questions,
1311        home: &Path,
1312        queued: &[Task],
1313        stalled: &[Task],
1314        finished: &[Task],
1315        max_attempts: usize,
1316    ) -> Result<()> {
1317        if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
1318            return Ok(());
1319        }
1320
1321        let primary = cfg
1322            .resolve_roles()
1323            .context("resolving the conductor seat")?
1324            .conductor;
1325        // A string keeps today's resolution exactly (no CLI preflight here;
1326        // an unavailable CLI fails at invocation). Only a multi-id array
1327        // builds a chain, whose members are each tried at most once.
1328        let chain = match cfg.roles.conductor.as_ref() {
1329            Some(c) if c.ids().len() > 1 => {
1330                agent::pick_chain(&cfg.agents, Some(c), &agent::installed, "conductor")?
1331            }
1332            _ => vec![primary],
1333        };
1334
1335        let runnable_views: Vec<prompt::ConductTask> =
1336            queued.iter().map(|t| view(t, max_attempts)).collect();
1337        let stalled_views: Vec<prompt::ConductTask> =
1338            stalled.iter().map(|t| view(t, max_attempts)).collect();
1339        let mut finished_views = Vec::with_capacity(finished.len());
1340        for t in finished {
1341            finished_views.push(finished_view(t, repo, max_attempts).await);
1342        }
1343
1344        let body = prompt::with_overlay(
1345            prompt::conduct(
1346                &runnable_views,
1347                &stalled_views,
1348                &finished_views,
1349                &cfg.graph.language,
1350            ),
1351            cfg.prompts.overlay(NODE),
1352        );
1353
1354        let body = format!(
1355            "{body}\n\n{}",
1356            crate::queue::dependency_facts(queue, questions)
1357        );
1358        let artifacts = home.join("conduct").join("artifacts");
1359        // Bound to a local: `Invocation` only borrows the cache path, and the
1360        // `Option<PathBuf>` `cache_dir()` returns has to outlive that borrow.
1361        let cache_dir = cfg.cache_dir();
1362
1363        // One marker for the whole chain: a fallback is still this turn.
1364        let busy = Busy::mark(home);
1365        let mut result: Result<Verdict> = Err(anyhow::anyhow!("no conductor agent ran"));
1366        for spec in &chain {
1367            // The prompt carries the whole queue state every cycle, so a
1368            // fallback agent needs no history - but its seat is its own.
1369            let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
1370            if needs_new_seat {
1371                self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
1372            }
1373            let seat = self.seat.as_mut().expect("just ensured a seat exists");
1374            // The first agent keeps the plain stem; a fallback's own artifacts
1375            // must not overwrite it.
1376            let stem = if std::ptr::eq(spec, &chain[0]) {
1377                format!("turn-{}", seat.turns + 1)
1378            } else {
1379                format!("turn-{}-{}", seat.turns + 1, spec.id)
1380            };
1381            let inv = Invocation {
1382                cwd: repo,
1383                prompt: &body,
1384                timeout: TURN_TIMEOUT,
1385                // The conductor never edits anything - it only decides what
1386                // blocks a task and what to do about one stuck or finished.
1387                allow_write: false,
1388                unsandboxed: false,
1389                sessions: cfg.graph.sessions,
1390                artifacts: &artifacts,
1391                stem: &stem,
1392                run: NODE,
1393                node: NODE,
1394                cache_dir: cache_dir.as_deref(),
1395                attachments: &[],
1396                writable: &[],
1397            };
1398            let out = agent::invoke(spec, seat, &inv).await;
1399            // Kept after every turn, failed or not: the CLI-side conversation is
1400            // what a resumed question needs, and it exists once the turn ran.
1401            if let Ok(body) = serde_json::to_string(&*seat) {
1402                let path = seat_path(home);
1403                if let Some(dir) = path.parent() {
1404                    let _ = std::fs::create_dir_all(dir);
1405                }
1406                let _ = std::fs::write(path, body);
1407            }
1408            let advance = agent::chain_advances(&out);
1409            result = match out {
1410                Err(e) => Err(e.context("invoking the conductor")),
1411                Ok(out) if advance => Err(anyhow::anyhow!(
1412                    "no usable reply (exit {:?}, timed out {})",
1413                    out.exit_code,
1414                    out.timed_out
1415                )),
1416                Ok(out) => verdict::extract_json(&out.text)
1417                    .context("the conductor's reply could not be parsed"),
1418            };
1419            if result.is_ok() {
1420                break;
1421            }
1422            if chain.len() > 1 {
1423                tracing::warn!("conductor: `{}` failed, trying the next agent", spec.id);
1424            }
1425        }
1426        drop(busy);
1427        let mut verdict = result?;
1428        attach_facts(cfg, repo, queue, &mut verdict).await;
1429        for d in &verdict.decisions {
1430            let snapshot = queued
1431                .iter()
1432                .chain(stalled)
1433                .chain(finished)
1434                .find(|t| t.id == d.id);
1435            if let Some(snapshot) = snapshot {
1436                if let Err(e) = apply_one(queue, questions, d, Some(snapshot)) {
1437                    tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1438                }
1439            }
1440        }
1441        Ok(())
1442    }
1443}
1444
1445#[cfg(test)]
1446mod tests {
1447    use std::collections::BTreeMap;
1448
1449    use tempfile::tempdir;
1450
1451    use super::*;
1452    use crate::ask::{Answer, QuestionStatus};
1453    use crate::config::{AgentKind, AgentSpec, Graph};
1454    use crate::queue::Source;
1455
1456    fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
1457        let path = dir.join("mock-conduct-agent.sh");
1458        std::fs::write(&path, script).expect("write mock");
1459        AgentSpec {
1460            id: "mock".to_owned(),
1461            kind: AgentKind::Command,
1462            model: None,
1463            command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
1464            extra_args: Vec::new(),
1465            env,
1466            prompt_delivery: None,
1467        }
1468    }
1469
1470    fn config(spec: AgentSpec) -> Config {
1471        Config {
1472            agents: vec![spec],
1473            graph: Graph {
1474                language: "en".to_owned(),
1475                ..Graph::default()
1476            },
1477            ..Config::default()
1478        }
1479    }
1480
1481    fn task(title: &str) -> Task {
1482        Task::new(
1483            title.to_owned(),
1484            format!("do {title}"),
1485            std::path::PathBuf::from("."),
1486            Source::Human,
1487        )
1488    }
1489
1490    #[test]
1491    fn delivery_bookkeeping_does_not_restart_conductor_deliberation() {
1492        let mut t = task("waiting");
1493        t.hold_machine(Some("waiting for recovery".into()));
1494        let before = input_fingerprint(&t);
1495        let snapshot = decision_input(&t);
1496        t.chat_notice = Some(crate::queue::ChatNotice {
1497            key: t.chat_notice_key().unwrap(),
1498            at: jiff::Timestamp::now(),
1499            outcome: crate::queue::ChatNoticeOutcome::Posted,
1500        });
1501        t.chat_notices.push(t.chat_notice_key().unwrap());
1502        t.updated_at = jiff::Timestamp::now();
1503        assert_eq!(before, input_fingerprint(&t));
1504        assert_eq!(snapshot, decision_input(&t));
1505        t.hold_reason = Some("recovery remains under consideration".into());
1506        assert_eq!(before, input_fingerprint(&t));
1507        t.record_answer("a different question".into(), "a new decision".into());
1508        assert_ne!(before, input_fingerprint(&t));
1509    }
1510
1511    const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
1512    const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
1513
1514    fn env(reply: &str) -> BTreeMap<String, String> {
1515        BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
1516    }
1517
1518    const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
1519
1520    /// A throwaway repo with one commit on `main` and a second branch ahead
1521    /// of it, so `outcome_for`'s own `git::rev_parse` call has a real head to
1522    /// resolve.
1523    fn init_repo_with_branch(dir: &Path, branch: &str) {
1524        use crate::proc::Quiet as _;
1525        let run = |args: &[&str]| {
1526            let out = std::process::Command::new("git")
1527                .args(args)
1528                .current_dir(dir)
1529                .quiet()
1530                .output()
1531                .expect("spawn git");
1532            assert!(
1533                out.status.success(),
1534                "git {args:?} failed: {}",
1535                String::from_utf8_lossy(&out.stderr)
1536            );
1537        };
1538        run(&["init", "-b", "main"]);
1539        run(&["config", "user.name", "magi test"]);
1540        run(&["config", "user.email", "magi@example.com"]);
1541        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
1542        run(&["add", "-A"]);
1543        run(&["commit", "-m", "init"]);
1544        run(&["checkout", "-b", branch]);
1545        std::fs::write(dir.join("change.txt"), "x\n").unwrap();
1546        run(&["add", "-A"]);
1547        run(&["commit", "-m", "candidate work"]);
1548    }
1549
1550    fn review_round_with_finding(
1551        round: usize,
1552        finding_id: &str,
1553        title: &str,
1554        addressed: &[&str],
1555        rejected: &[(&str, &str)],
1556    ) -> crate::run::ReviewRound {
1557        crate::run::ReviewRound {
1558            round,
1559            head: "deadbeef".to_owned(),
1560            verified_head: None,
1561            verified_at: None,
1562            reviews: vec![crate::run::ReviewRecord {
1563                attempts: 0,
1564                reviewer: 1,
1565                agent: "mock".to_owned(),
1566                summary: String::new(),
1567                findings: vec![crate::verdict::Finding {
1568                    id: finding_id.to_owned(),
1569                    severity: crate::verdict::Severity::Major,
1570                    file: None,
1571                    line: None,
1572                    title: title.to_owned(),
1573                    detail: String::new(),
1574                }],
1575                vote: None,
1576                failed: None,
1577                duration_ms: 0,
1578            }],
1579            e2e: Vec::new(),
1580            verify_retried: false,
1581            e2e_deferred: false,
1582            e2e_defer_reason: None,
1583            fix: Some(crate::run::FixRecord {
1584                agent: "mock".to_owned(),
1585                addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
1586                rejected: rejected
1587                    .iter()
1588                    .map(|(id, why)| crate::verdict::Rejection {
1589                        id: (*id).to_owned(),
1590                        why: (*why).to_owned(),
1591                    })
1592                    .collect(),
1593                notes: String::new(),
1594                committed: false,
1595                failed: None,
1596                duration_ms: 0,
1597                continuation: None,
1598            }),
1599            blocking: 1,
1600            answered: 1,
1601            expected: 1,
1602            clean: false,
1603            progressed: true,
1604            vote_split: false,
1605            reconsideration: Vec::new(),
1606            verdict: None,
1607        }
1608    }
1609
1610    #[test]
1611    fn explicit_resumption_supersedes_history_and_rejects_stale_decisions() {
1612        crate::run::pin_test_home();
1613        let dir = tempdir().unwrap();
1614        let queue = Queue::at(dir.path().join("queue"));
1615        let questions = Questions::at(dir.path().join("questions"));
1616        let mut t = task("recover");
1617        t.answers.push(crate::queue::AnsweredQuestion {
1618            question: "Inspect remote first?".into(),
1619            answer: "Inspect, then await resumption instructions".into(),
1620        });
1621        t.hold_machine(Some("waiting for remote inspection".into()));
1622        queue.put(&mut t).unwrap();
1623        let snapshot = queue.get(&t.id).unwrap();
1624        t.edit(
1625            "recover".into(),
1626            "Reuse the measured PR head; fix Clippy; review".into(),
1627        )
1628        .unwrap();
1629        t.release_operator("cli", None);
1630        queue.put(&mut t).unwrap();
1631        let approval = t.resumption.clone();
1632        let d = Decision {
1633            id: t.id.clone(),
1634            recovery: Some(Recovery::Hold),
1635            reason: Some("remote inspection still required".into()),
1636            ..Decision::default()
1637        };
1638        apply_one(&queue, &questions, &d, Some(&snapshot)).unwrap();
1639        let stale_question = Decision {
1640            question: Some("Inspect remote?".into()),
1641            ..d.clone()
1642        };
1643        apply_one(&queue, &questions, &stale_question, Some(&snapshot)).unwrap();
1644        assert!(questions.list().is_empty());
1645        for reason in [
1646            "waiting for remote inspection",
1647            "inspection has not been confirmed",
1648        ] {
1649            let repeated = Decision {
1650                reason: Some(reason.into()),
1651                ..d.clone()
1652            };
1653            apply(
1654                &queue,
1655                &questions,
1656                &Verdict {
1657                    decisions: vec![repeated],
1658                },
1659            )
1660            .unwrap();
1661            assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1662        }
1663        let reloaded = Queue::at(dir.path().join("queue")).get(&t.id).unwrap();
1664        assert_eq!(reloaded.resumption, approval);
1665        assert_eq!(reloaded.answers, t.answers);
1666        let v = view(&reloaded, 3);
1667        assert!(v.operator_resume.unwrap().contains("superseded"));
1668        let mut dep = task("new prerequisite");
1669        dep.hold_machine(Some("new authentication failure".into()));
1670        queue.put(&mut dep).unwrap();
1671        let new_blocker = Decision {
1672            blocked_by: vec![dep.id.clone()],
1673            evidence: vec![dep.id.clone()],
1674            // Exercise dependency waiting, without inheriting a separate hold.
1675            recovery: None,
1676            reason: Some("new prerequisite failed".into()),
1677            ..d.clone()
1678        };
1679        apply(
1680            &queue,
1681            &questions,
1682            &Verdict {
1683                decisions: vec![new_blocker.clone()],
1684            },
1685        )
1686        .unwrap();
1687        let mut after = queue.get(&t.id).unwrap();
1688        assert_eq!(after.status, TaskStatus::Blocked);
1689        // Automatic release does not grant fresh operator authority or make
1690        // the same unchanged evidence reusable.
1691        after.release();
1692        queue.put(&mut after).unwrap();
1693        apply(
1694            &queue,
1695            &questions,
1696            &Verdict {
1697                decisions: vec![new_blocker],
1698            },
1699        )
1700        .unwrap();
1701        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1702        after.start("new-run".into());
1703        assert_eq!(after.resumption.as_ref().unwrap().generation, 1);
1704        after.hold_manual(Some("operator says wait".into()));
1705        queue.put(&mut after).unwrap();
1706        apply(&queue, &questions, &Verdict { decisions: vec![d] }).unwrap();
1707        assert!(queue.get(&t.id).unwrap().operator_held());
1708    }
1709
1710    #[test]
1711    fn newly_blocked_dependency_is_material_evidence_after_approval() {
1712        let dir = tempdir().unwrap();
1713        let queue = Queue::at(dir.path().join("queue"));
1714        let questions = Questions::at(dir.path().join("questions"));
1715        let mut dep = task("old queued dependency");
1716        queue.put(&mut dep).unwrap();
1717        let mut t = task("approved task");
1718        t.release_operator("test", None);
1719        queue.put(&mut t).unwrap();
1720        let mut q = Question::new(
1721            dep.id.clone(),
1722            "check".into(),
1723            "check".into(),
1724            "New dependency approval".into(),
1725            String::new(),
1726            Vec::new(),
1727        );
1728        questions.put(&mut q).unwrap();
1729        dep.block(vec![q.id.clone()], Some("new decision needed".into()));
1730        let changed = dep.blocked_at;
1731        queue.put(&mut dep).unwrap();
1732        dep.block(vec![q.id], Some("same decision paraphrased".into()));
1733        assert_eq!(
1734            dep.blocked_at, changed,
1735            "prose is not new blocking evidence"
1736        );
1737        queue.put(&mut dep).unwrap();
1738        let d = Decision {
1739            id: t.id.clone(),
1740            blocked_by: vec![dep.id.clone()],
1741            evidence: vec![dep.id],
1742            ..Decision::default()
1743        };
1744        assert_eq!(new_blocking_evidence(&queue, &questions, &t, &d).len(), 1);
1745        apply_one(&queue, &questions, &d, None).unwrap();
1746        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
1747    }
1748
1749    #[test]
1750    fn resumed_run_evidence_uses_approval_time_not_run_list_offsets() {
1751        crate::run::pin_test_home();
1752        let dir = tempdir().unwrap();
1753        let queue = Queue::at(dir.path().join("queue"));
1754        let questions = Questions::at(dir.path().join("questions"));
1755        let mut t = task("resume evidence");
1756        let mut run = RunState::new(
1757            t.repo.clone(),
1758            "main".into(),
1759            "base".into(),
1760            "work".into(),
1761            crate::config::Config::default(),
1762        );
1763        run.begin_pass();
1764        run.status = crate::run::RunStatus::Failed;
1765        run.save().unwrap();
1766        t.start(run.id.clone());
1767        // Model legacy duplicate history, which the next start will deduplicate.
1768        t.runs.push(run.id.clone());
1769        t.release_operator("cli", None);
1770        let approval = t.resumption.as_ref().unwrap().at;
1771        let decision = Decision {
1772            evidence: vec![run.id.clone()],
1773            ..Decision::default()
1774        };
1775        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1776        t.start(run.id.clone());
1777        assert_eq!(t.runs.len(), 1);
1778        run.begin_pass();
1779        run.save().unwrap();
1780        assert!(run.passes.last().unwrap().started_at > approval);
1781        let evidence = new_blocking_evidence(&queue, &questions, &t, &decision);
1782        assert_eq!(evidence.len(), 1);
1783        t.resumption.as_mut().unwrap().blocking_evidence = evidence;
1784        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1785        run.begin_pass();
1786        run.save().unwrap();
1787        assert_eq!(
1788            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1789            1
1790        );
1791
1792        let mut q = Question::new(
1793            run.id.clone(),
1794            "review".into(),
1795            "check".into(),
1796            "New failure".into(),
1797            String::new(),
1798            Vec::new(),
1799        );
1800        questions.put(&mut q).unwrap();
1801        let decision = Decision {
1802            evidence: vec![q.id.clone()],
1803            ..Decision::default()
1804        };
1805        assert_eq!(
1806            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1807            1
1808        );
1809        q.asked_at = approval;
1810        questions.put(&mut q).unwrap();
1811        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1812        q.asked_at = jiff::Timestamp::now();
1813        q.run = "unrelated".into();
1814        questions.put(&mut q).unwrap();
1815        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1816
1817        let mut fresh = RunState::new(
1818            t.repo.clone(),
1819            "main".into(),
1820            "base".into(),
1821            "fresh work".into(),
1822            crate::config::Config::default(),
1823        );
1824        fresh.status = crate::run::RunStatus::Failed;
1825        fresh.save().unwrap();
1826        t.start(fresh.id.clone());
1827        let decision = Decision {
1828            evidence: vec![fresh.id.clone()],
1829            ..Decision::default()
1830        };
1831        assert_eq!(
1832            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1833            1
1834        );
1835    }
1836
1837    #[test]
1838    fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
1839        crate::run::pin_test_home();
1840        let dir = tempdir().unwrap();
1841        let default_repo = dir.path().join("default");
1842        let task_repo = dir.path().join("task");
1843        std::fs::create_dir_all(&default_repo).unwrap();
1844        std::fs::create_dir_all(&task_repo).unwrap();
1845        init_repo_with_branch(&default_repo, "other-branch");
1846        init_repo_with_branch(&task_repo, "magi/f00d/A");
1847
1848        let mut config = Config::default();
1849        config.graph.review_rounds = 6;
1850        let mut state = crate::run::RunState::new(
1851            task_repo.clone(),
1852            "main".to_owned(),
1853            "deadbeef".to_owned(),
1854            "task".to_owned(),
1855            config,
1856        );
1857        state.status = crate::run::RunStatus::Blocked;
1858        state.candidates.push(crate::run::Candidate {
1859            index: 0,
1860            label: 'A',
1861            agent: "mock".to_owned(),
1862            branch: "magi/f00d/A".to_owned(),
1863            worktree: task_repo.clone(),
1864            summary: String::new(),
1865            stat: String::new(),
1866            files: 1,
1867            commits: 1,
1868            empty: false,
1869            failed: None,
1870            verified_noop: None,
1871            duration_ms: 0,
1872            folded: false,
1873        });
1874        state.tally = Some(crate::run::Tally {
1875            first_choice: std::collections::BTreeMap::new(),
1876            borda: std::collections::BTreeMap::new(),
1877            winner: 'A',
1878            rankings: 0,
1879            unanimous_initial: false,
1880            deliberated: false,
1881            changed_votes: 0,
1882            unanimous_final: false,
1883            tie_break: None,
1884            judges: 0,
1885            present: 0,
1886            quorum: 0,
1887            met_quorum: true,
1888            uncontested: Some("solo".to_owned()),
1889        });
1890        state.reviews = vec![
1891            review_round_with_finding(
1892                1,
1893                "R1-1-2",
1894                "answer content is dropped",
1895                &[],
1896                &[("R1-1-2", "the id leaving blocked_by is enough")],
1897            ),
1898            review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
1899        ];
1900        state.save().unwrap();
1901
1902        let mut t = task("outcome test");
1903        t.repo = task_repo;
1904        t.runs.push(state.id.clone());
1905
1906        let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
1907        let outcome = finished.outcome;
1908
1909        assert!(outcome.unreadable.is_none());
1910        assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
1911        assert_eq!(outcome.rounds_used, 2);
1912        assert_eq!(outcome.rounds_max, 6);
1913        assert_eq!(outcome.rounds.len(), 2);
1914        assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
1915        assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
1916        assert!(outcome.rounds[1].addressed.is_empty());
1917        assert!(outcome.rounds[1].rejected.is_empty());
1918        assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
1919        assert!(
1920            outcome.branch_head.is_some(),
1921            "a real branch must resolve a head commit: {outcome:?}"
1922        );
1923    }
1924
1925    /// A tiny single-threaded block-on, so an `async fn` can be exercised
1926    /// from a plain `#[test]` without pulling `tokio::test`'s multi-thread
1927    /// runtime into a test that does no other async work.
1928    fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
1929        tokio::runtime::Builder::new_current_thread()
1930            .enable_all()
1931            .build()
1932            .unwrap()
1933            .block_on(f)
1934    }
1935
1936    #[test]
1937    fn view_resolves_task_attempt_limit_before_daemon_default() {
1938        let mut t = task("budget");
1939        assert_eq!(view(&t, 7).max_attempts, 7);
1940        t.change_attempt_limit(std::num::NonZeroUsize::new(3))
1941            .unwrap();
1942        assert_eq!(view(&t, 7).max_attempts, 3);
1943        t.change_attempt_limit(None).unwrap();
1944        assert_eq!(view(&t, 7).max_attempts, 7);
1945    }
1946
1947    #[test]
1948    fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
1949        let mut t = task("answered");
1950        t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
1951        let v = view(&t, 2);
1952        assert_eq!(v.answers.len(), 1);
1953        assert_eq!(v.answers[0].question, "Which backend?");
1954        assert_eq!(v.answers[0].answer, "SQLite");
1955    }
1956
1957    #[tokio::test]
1958    async fn dependency_resolution_rechecks_questions_in_the_production_conductor_path() {
1959        for held in [false, true] {
1960            for reference in ["ca5b", "20261013-193611-ca5b"] {
1961                for question in [false, true] {
1962                    let dir = tempdir().unwrap();
1963                    let queue = Queue::at(dir.path().join("queue"));
1964                    let questions = Questions::at(dir.path().join("questions"));
1965                    let mut dep = task("prerequisite");
1966                    dep.id = "20261013-193611-ca5b".into();
1967                    dep.start("existing-run".into());
1968                    queue.put(&mut dep).unwrap();
1969                    let mut waiting = task("waiting");
1970                    if held {
1971                        waiting.hold_machine(Some("dependency could not be located".into()));
1972                    }
1973                    queue.put(&mut waiting).unwrap();
1974                    let missing_reference = if reference == "ca5b" {
1975                        dep.id.as_str()
1976                    } else {
1977                        "ca5b"
1978                    };
1979                    let reply = serde_json::json!({"decisions": [{
1980                        "id": waiting.id, "blocked_by": [reference],
1981                        "missing_dependencies": [missing_reference],
1982                        "question": question.then_some("ca5b is missing; where is its replacement?"),
1983                        "recovery": (!question).then_some("hold"),
1984                        "reason": "ca5b is missing"
1985                    }]}).to_string();
1986                    let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1987                    let (queued, finished) = if held {
1988                        (vec![], vec![waiting.clone()])
1989                    } else {
1990                        (vec![waiting.clone()], vec![])
1991                    };
1992                    Conductor::new()
1993                        .run_once(
1994                            &cfg,
1995                            dir.path(),
1996                            &queue,
1997                            &questions,
1998                            dir.path(),
1999                            &queued,
2000                            &[],
2001                            &finished,
2002                            2,
2003                        )
2004                        .await
2005                        .unwrap();
2006                    let after = queue.get(&waiting.id).unwrap();
2007                    assert_eq!(after.status, TaskStatus::Blocked);
2008                    assert_eq!(after.blocked_by, [dep.id]);
2009                    assert_eq!(after.blocked_from, Some(waiting.status));
2010                    assert!(!after.block_reason.unwrap().contains("missing"));
2011                    assert!(questions.list().is_empty());
2012                    assert_eq!(after.instruction, waiting.instruction);
2013                }
2014            }
2015        }
2016    }
2017
2018    #[tokio::test]
2019    async fn dependency_resolution_inconclusive_missing_only_decisions_keep_waiting() {
2020        for held in [false, true] {
2021            for unknown in [false, true] {
2022                for question in [false, true] {
2023                    let dir = tempdir().unwrap();
2024                    let queue = Queue::at(dir.path().join("queue"));
2025                    let questions = Questions::at(dir.path().join("questions"));
2026                    let mut dep = task("running prerequisite");
2027                    dep.id = "20261013-193611-ca5b".into();
2028                    dep.start("existing-run".into());
2029                    queue.put(&mut dep).unwrap();
2030                    let reference = if unknown {
2031                        // Even an exact, existing task id is Unknown when
2032                        // the question inventory cannot be scanned.
2033                        std::fs::write(questions.root(), "not a directory").unwrap();
2034                        dep.id.clone()
2035                    } else {
2036                        let mut other = task("another prerequisite");
2037                        other.id = "20261014-193611-ca5b".into();
2038                        queue.put(&mut other).unwrap();
2039                        "ca5b".to_owned()
2040                    };
2041                    assert_eq!(
2042                        crate::queue::dependency(&queue, &questions, &reference),
2043                        if unknown {
2044                            crate::queue::Dependency::Unknown
2045                        } else {
2046                            crate::queue::Dependency::Ambiguous
2047                        }
2048                    );
2049                    let mut waiting = task("waiting");
2050                    if held {
2051                        waiting.hold_machine(Some("dependency could not be located".into()));
2052                    }
2053                    queue.put(&mut waiting).unwrap();
2054                    let reply = serde_json::json!({"decisions": [{
2055                        "id": waiting.id, "blocked_by": [reference],
2056                        "missing_dependencies": [reference],
2057                        "question": question.then_some("ca5b is missing"),
2058                        "recovery": (!question).then_some("hold"),
2059                        "reason": "ca5b is missing"
2060                    }]})
2061                    .to_string();
2062                    let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2063                    let (queued, finished) = if held {
2064                        (vec![], vec![waiting.clone()])
2065                    } else {
2066                        (vec![waiting.clone()], vec![])
2067                    };
2068                    Conductor::new()
2069                        .run_once(
2070                            &cfg,
2071                            dir.path(),
2072                            &queue,
2073                            &questions,
2074                            dir.path(),
2075                            &queued,
2076                            &[],
2077                            &finished,
2078                            2,
2079                        )
2080                        .await
2081                        .unwrap();
2082                    let after = queue.get(&waiting.id).unwrap();
2083                    assert_eq!(after.status, TaskStatus::Blocked);
2084                    assert_eq!(after.blocked_by, [reference]);
2085                    assert_eq!(after.blocked_from, Some(waiting.status));
2086                    assert!(!after.block_reason.unwrap().contains("missing"));
2087                    assert_eq!(after.instruction, waiting.instruction);
2088                    if unknown {
2089                        assert!(questions.root().is_file());
2090                    } else {
2091                        assert!(questions.list().is_empty());
2092                    }
2093                }
2094            }
2095        }
2096    }
2097
2098    #[tokio::test]
2099    async fn dependency_resolution_unverifiable_alias_decisions_are_discarded() {
2100        for held in [false, true] {
2101            for unknown in [false, true] {
2102                for question in [false, true] {
2103                    for short_blocker in [false, true] {
2104                        let dir = tempdir().unwrap();
2105                        let queue = Queue::at(dir.path().join("queue"));
2106                        let questions = Questions::at(dir.path().join("questions"));
2107                        let mut dep = task("running prerequisite");
2108                        dep.id = "20261013-193611-ca5b".into();
2109                        dep.start("existing-run".into());
2110                        queue.put(&mut dep).unwrap();
2111                        if unknown {
2112                            std::fs::write(questions.root(), "not a directory").unwrap();
2113                        } else {
2114                            let mut other = task("another prerequisite");
2115                            other.id = "20261014-193611-ca5b".into();
2116                            queue.put(&mut other).unwrap();
2117                        }
2118                        let (blocker, missing) = if short_blocker {
2119                            ("ca5b", dep.id.as_str())
2120                        } else {
2121                            (dep.id.as_str(), "ca5b")
2122                        };
2123                        let mut waiting = task("waiting");
2124                        if held {
2125                            waiting.hold_machine(Some("dependency could not be located".into()));
2126                        }
2127                        queue.put(&mut waiting).unwrap();
2128                        let reply = serde_json::json!({"decisions": [{
2129                            "id": waiting.id, "blocked_by": [blocker],
2130                            "missing_dependencies": [missing],
2131                            "question": question.then_some("ca5b is missing"),
2132                            "recovery": (!question).then_some("hold"),
2133                            "reason": "ca5b is missing"
2134                        }]})
2135                        .to_string();
2136                        let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2137                        let (queued, finished) = if held {
2138                            (vec![], vec![waiting.clone()])
2139                        } else {
2140                            (vec![waiting.clone()], vec![])
2141                        };
2142                        Conductor::new()
2143                            .run_once(
2144                                &cfg,
2145                                dir.path(),
2146                                &queue,
2147                                &questions,
2148                                dir.path(),
2149                                &queued,
2150                                &[],
2151                                &finished,
2152                                2,
2153                            )
2154                            .await
2155                            .unwrap();
2156                        assert_eq!(
2157                            serde_json::to_value(queue.get(&waiting.id).unwrap()).unwrap(),
2158                            serde_json::to_value(&waiting).unwrap(),
2159                            "unverifiable aliases must leave the entire task unchanged"
2160                        );
2161                        if unknown {
2162                            assert!(questions.root().is_file());
2163                        } else {
2164                            assert!(questions.list().is_empty());
2165                        }
2166                    }
2167                }
2168            }
2169        }
2170    }
2171
2172    #[tokio::test]
2173    async fn dependency_resolution_genuine_missing_reference_still_asks() {
2174        let dir = tempdir().unwrap();
2175        let queue = Queue::at(dir.path().join("queue"));
2176        let questions = Questions::at(dir.path().join("questions"));
2177        let mut waiting = task("missing prerequisite");
2178        queue.put(&mut waiting).unwrap();
2179        let reply = serde_json::json!({"decisions": [{
2180            "id": waiting.id, "blocked_by": ["not-present"],
2181            "missing_dependencies": ["not-present"],
2182            "question": "Which prerequisite replaces the missing task?"
2183        }]})
2184        .to_string();
2185        let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2186        Conductor::new()
2187            .run_once(
2188                &cfg,
2189                dir.path(),
2190                &queue,
2191                &questions,
2192                dir.path(),
2193                &[waiting.clone()],
2194                &[],
2195                &[],
2196                2,
2197            )
2198            .await
2199            .unwrap();
2200        let after = queue.get(&waiting.id).unwrap();
2201        assert_eq!(after.status, TaskStatus::Blocked);
2202        assert_eq!(
2203            questions.get(&after.blocked_by[0]).unwrap().summary,
2204            "Which prerequisite replaces the missing task?"
2205        );
2206    }
2207
2208    #[tokio::test]
2209    async fn dependency_resolution_keeps_decisions_during_unrelated_task_progress() {
2210        let dir = tempdir().unwrap();
2211        let queue = Queue::at(dir.path().join("queue"));
2212        let questions = Questions::at(dir.path().join("questions"));
2213        let mut dep = task("prerequisite");
2214        dep.id = "20261013-193611-ca5b".into();
2215        dep.start("existing-run".into());
2216        queue.put(&mut dep).unwrap();
2217        let mut unrelated = task("unrelated work");
2218        unrelated.start("other-run".into());
2219        queue.put(&mut unrelated).unwrap();
2220        // Make the old and new revision distinguishable without relying on
2221        // the filesystem clock advancing while the mock process starts.
2222        std::fs::File::options()
2223            .write(true)
2224            .open(queue.path_of(&unrelated.id))
2225            .unwrap()
2226            .set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH))
2227            .unwrap();
2228        unrelated.succeed();
2229        let mut waiting = task("waiting");
2230        let mut asking = task("needs a product decision");
2231        let mut holding = task("must stay held");
2232        queue.put(&mut waiting).unwrap();
2233        queue.put(&mut asking).unwrap();
2234        queue.put(&mut holding).unwrap();
2235        let reply = serde_json::json!({"decisions": [
2236            {"id": waiting.id, "blocked_by": ["ca5b"], "reason": "wait for implementation"},
2237            {"id": asking.id, "question": "Which behavior do you want?"},
2238            {"id": holding.id, "recovery": "hold", "reason": "already implemented elsewhere"}
2239        ]})
2240        .to_string();
2241        let mut vars = env(&reply);
2242        vars.insert(
2243            "UNRELATED_PATH".into(),
2244            queue.path_of(&unrelated.id).to_string_lossy().into_owned(),
2245        );
2246        vars.insert(
2247            "UNRELATED_RECORD".into(),
2248            serde_json::to_string(&unrelated).unwrap(),
2249        );
2250        let script = r#"#!/bin/sh
2251cat >/dev/null
2252printf '%s' "$UNRELATED_RECORD" > "$UNRELATED_PATH"
2253printf '%s\n' "$MOCK_REPLY"
2254"#;
2255        let cfg = config(mock_agent(dir.path(), script, vars));
2256        Conductor::new()
2257            .run_once(
2258                &cfg,
2259                dir.path(),
2260                &queue,
2261                &questions,
2262                dir.path(),
2263                &[waiting.clone(), asking.clone(), holding.clone()],
2264                &[],
2265                &[],
2266                2,
2267            )
2268            .await
2269            .unwrap();
2270        assert_eq!(queue.get(&unrelated.id).unwrap().status, TaskStatus::Done);
2271        let after = queue.get(&waiting.id).unwrap();
2272        assert_eq!(after.status, TaskStatus::Blocked);
2273        assert_eq!(after.blocked_by, [dep.id]);
2274        let asked = queue.get(&asking.id).unwrap();
2275        assert_eq!(asked.status, TaskStatus::Blocked);
2276        assert_eq!(
2277            questions.get(&asked.blocked_by[0]).unwrap().summary,
2278            "Which behavior do you want?"
2279        );
2280        assert_eq!(queue.get(&holding.id).unwrap().status, TaskStatus::Held);
2281    }
2282
2283    #[test]
2284    fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
2285        let dir = tempdir().unwrap();
2286        let queue = Queue::at(dir.path().join("queue"));
2287        let questions = Questions::at(dir.path().join("questions"));
2288        let mut a = task("a");
2289        a.priority = 9;
2290        queue.put(&mut a).unwrap();
2291
2292        let verdict = Verdict {
2293            decisions: vec![Decision {
2294                id: a.id.clone(),
2295                blocked_by: vec!["20260101-000000-dead".to_owned()],
2296                reason: Some("waits on the other task".to_owned()),
2297                recovery: None,
2298                question: None,
2299                choices: Vec::new(),
2300                question_decision: None,
2301                evidence: Vec::new(),
2302                missing_dependencies: Vec::new(),
2303            }],
2304        };
2305        apply(&queue, &questions, &verdict).unwrap();
2306
2307        let back = queue.get(&a.id).unwrap();
2308        assert_eq!(back.status, TaskStatus::Blocked);
2309        assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
2310        assert_eq!(
2311            back.priority, 9,
2312            "the conductor's reply cannot carry priority"
2313        );
2314    }
2315
2316    #[test]
2317    fn a_question_decision_files_one_and_blocks_on_its_id() {
2318        let dir = tempdir().unwrap();
2319        let queue = Queue::at(dir.path().join("queue"));
2320        let questions = Questions::at(dir.path().join("questions"));
2321        let mut t = task("ambiguous");
2322        queue.put(&mut t).unwrap();
2323
2324        let verdict = Verdict {
2325            decisions: vec![Decision {
2326                id: t.id.clone(),
2327                blocked_by: Vec::new(),
2328                reason: Some("which backend?".to_owned()),
2329                recovery: None,
2330                question: Some("Which storage backend?".to_owned()),
2331                choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
2332                question_decision: None,
2333                evidence: Vec::new(),
2334                missing_dependencies: Vec::new(),
2335            }],
2336        };
2337        apply(&queue, &questions, &verdict).unwrap();
2338
2339        let back = queue.get(&t.id).unwrap();
2340        assert_eq!(back.status, TaskStatus::Blocked);
2341        assert_eq!(back.blocked_by.len(), 1);
2342        let q = questions.get(&back.blocked_by[0]).unwrap();
2343        assert_eq!(q.summary, "Which storage backend?");
2344        assert_eq!(q.node, NODE);
2345        assert!(q.status.open());
2346    }
2347
2348    #[test]
2349    fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
2350        let dir = tempdir().unwrap();
2351        let queue = Queue::at(dir.path().join("queue"));
2352        let questions = Questions::at(dir.path().join("questions"));
2353        let mut t = task("asked once");
2354        queue.put(&mut t).unwrap();
2355
2356        let decision = Decision {
2357            id: t.id.clone(),
2358            reason: Some("still deciding".to_owned()),
2359            question: Some("Which backend?".to_owned()),
2360            ..Decision::default()
2361        };
2362        apply(
2363            &queue,
2364            &questions,
2365            &Verdict {
2366                decisions: vec![decision.clone()],
2367            },
2368        )
2369        .unwrap();
2370        assert_eq!(questions.list().len(), 1);
2371        let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
2372
2373        // An operator releasing the blocked task by hand, without answering,
2374        // puts it back at `Queued` while the question stays open - exactly
2375        // the case the guard in `apply_one` exists for: a later cycle
2376        // proposing the very same question must reuse it, not file a second.
2377        let mut released = queue.get(&t.id).unwrap();
2378        released.release();
2379        queue.put(&mut released).unwrap();
2380
2381        apply(
2382            &queue,
2383            &questions,
2384            &Verdict {
2385                decisions: vec![decision],
2386            },
2387        )
2388        .unwrap();
2389        assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
2390        let after = queue.get(&t.id).unwrap();
2391        assert_eq!(
2392            after.blocked_by,
2393            [first_question_id],
2394            "the existing open question is reused, not replaced"
2395        );
2396    }
2397
2398    #[test]
2399    fn a_same_id_question_from_another_node_is_not_reused() {
2400        let dir = tempdir().unwrap();
2401        let queue = Queue::at(dir.path().join("queue"));
2402        let questions = Questions::at(dir.path().join("questions"));
2403        let mut t = task("must ask the conductor");
2404        queue.put(&mut t).unwrap();
2405
2406        let mut unrelated = Question::new(
2407            t.id.clone(),
2408            "review".to_owned(),
2409            "reviewer-1".to_owned(),
2410            "An unrelated review question".to_owned(),
2411            String::new(),
2412            Vec::new(),
2413        );
2414        questions.put(&mut unrelated).unwrap();
2415
2416        apply(
2417            &queue,
2418            &questions,
2419            &Verdict {
2420                decisions: vec![Decision {
2421                    id: t.id.clone(),
2422                    question: Some("Which backend?".to_owned()),
2423                    ..Decision::default()
2424                }],
2425            },
2426        )
2427        .unwrap();
2428
2429        let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
2430        assert_eq!(blocked_by.len(), 1);
2431        assert_ne!(blocked_by[0], unrelated.id);
2432        assert!(questions.get(&unrelated.id).unwrap().status.open());
2433        assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
2434    }
2435
2436    #[test]
2437    fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
2438        let dir = tempdir().unwrap();
2439        let queue = Queue::at(dir.path().join("queue"));
2440        let questions = Questions::at(dir.path().join("questions"));
2441        let mut t = task("waits on an answer");
2442        queue.put(&mut t).unwrap();
2443
2444        apply(
2445            &queue,
2446            &questions,
2447            &Verdict {
2448                decisions: vec![Decision {
2449                    id: t.id.clone(),
2450                    blocked_by: Vec::new(),
2451                    reason: None,
2452                    recovery: None,
2453                    question: Some("Which backend?".to_owned()),
2454                    choices: Vec::new(),
2455                    question_decision: None,
2456                    evidence: Vec::new(),
2457                    missing_dependencies: Vec::new(),
2458                }],
2459            },
2460        )
2461        .unwrap();
2462        let blocked = queue.get(&t.id).unwrap();
2463        let question_id = blocked.blocked_by[0].clone();
2464
2465        let mut q = questions.get(&question_id).unwrap();
2466        q.answer(Answer::Text("SQLite".to_owned())).unwrap();
2467        questions.put(&mut q).unwrap();
2468        assert_eq!(q.status, QuestionStatus::Answered);
2469
2470        // `crate::daemon::resolve_blockers` is the deterministic resolver
2471        // that actually does this on the real queue; here it is enough to
2472        // prove the pure steps it is built from behave together.
2473        let mut task_after = queue.get(&t.id).unwrap();
2474        task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
2475        task_after.unblock(&question_id);
2476        assert_eq!(task_after.status, TaskStatus::Queued);
2477        assert_eq!(task_after.answers[0].answer, "SQLite");
2478    }
2479
2480    #[test]
2481    fn a_stalled_task_can_be_requeued_or_held() {
2482        let dir = tempdir().unwrap();
2483        let queue = Queue::at(dir.path().join("queue"));
2484        let questions = Questions::at(dir.path().join("questions"));
2485
2486        let mut requeue_me = task("stuck a");
2487        requeue_me.start("run-1".to_owned());
2488        queue.put(&mut requeue_me).unwrap();
2489
2490        let mut hold_me = task("stuck b");
2491        hold_me.start("run-2".to_owned());
2492        queue.put(&mut hold_me).unwrap();
2493
2494        apply(
2495            &queue,
2496            &questions,
2497            &Verdict {
2498                decisions: vec![
2499                    Decision {
2500                        id: requeue_me.id.clone(),
2501                        recovery: Some(Recovery::Requeue),
2502                        ..Decision::default()
2503                    },
2504                    Decision {
2505                        id: hold_me.id.clone(),
2506                        recovery: Some(Recovery::Hold),
2507                        reason: Some("looks broken".to_owned()),
2508                        ..Decision::default()
2509                    },
2510                ],
2511            },
2512        )
2513        .unwrap();
2514
2515        let requeued = queue.get(&requeue_me.id).unwrap();
2516        assert_eq!(requeued.status, TaskStatus::Queued);
2517        assert_eq!(requeued.attempts, 0);
2518
2519        let held = queue.get(&hold_me.id).unwrap();
2520        assert_eq!(held.status, TaskStatus::Held);
2521        assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
2522    }
2523
2524    #[test]
2525    fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
2526        // Reproduces the reported bug: a task held after burning its
2527        // attempts, once the conductor asks a follow-up question about it,
2528        // must come back `held` - not `queued` - once the question is
2529        // answered, whatever the answer said.
2530        let dir = tempdir().unwrap();
2531        let queue = Queue::at(dir.path().join("queue"));
2532        let questions = Questions::at(dir.path().join("questions"));
2533        let mut t = task("held out of attempts");
2534        t.hold_machine(Some("out of attempts".to_owned()));
2535        queue.put(&mut t).unwrap();
2536
2537        apply(
2538            &queue,
2539            &questions,
2540            &Verdict {
2541                decisions: vec![Decision {
2542                    id: t.id.clone(),
2543                    reason: Some("what should happen to this one?".to_owned()),
2544                    question: Some("Hold it, or try again?".to_owned()),
2545                    ..Decision::default()
2546                }],
2547            },
2548        )
2549        .unwrap();
2550        let blocked = queue.get(&t.id).unwrap();
2551        assert_eq!(blocked.status, TaskStatus::Blocked);
2552        let question_id = blocked.blocked_by[0].clone();
2553
2554        let mut q = questions.get(&question_id).unwrap();
2555        q.answer(Answer::Text("leave it held".to_owned())).unwrap();
2556        questions.put(&mut q).unwrap();
2557
2558        // What `crate::daemon::resolve_blockers` does on the real queue.
2559        let mut after = queue.get(&t.id).unwrap();
2560        after.record_answer(q.summary.clone(), "leave it held".to_owned());
2561        after.unblock(&question_id);
2562        assert_eq!(
2563            after.status,
2564            TaskStatus::Held,
2565            "must not fall back to queued"
2566        );
2567        assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
2568    }
2569
2570    #[test]
2571    fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
2572        // A different path to the same bug round 1 fixed: a task
2573        // machine-held for disk pressure, blocked on a conductor question,
2574        // answered "keep it held", and restored to `held`. If the
2575        // conductor's next decision reconfirms `recovery: hold` with no new
2576        // `reason` - allowed, `Decision::reason` is optional - `hold_machine`
2577        // must not silently leave the stale disk-pressure text in place, or
2578        // `crate::triage::run_once` reads it as an unexamined, auto-resolvable
2579        // hold and releases the task straight through the operator's answer
2580        // the instant disk space looks fine again.
2581        let dir = tempdir().unwrap();
2582        let queue = Queue::at(dir.path().join("queue"));
2583        let questions = Questions::at(dir.path().join("questions"));
2584        let mut t = task("disk pressure, then reconsidered");
2585        t.hold_machine(Some(
2586            "not enough free space to start a run: 10 bytes free, 100 required by \
2587             `[disk] min_free_bytes`"
2588                .to_owned(),
2589        ));
2590        t.record_answer(
2591            "How should this be handled?".to_owned(),
2592            "keep it held, a human will look at it later".to_owned(),
2593        );
2594        queue.put(&mut t).unwrap();
2595
2596        apply(
2597            &queue,
2598            &questions,
2599            &Verdict {
2600                decisions: vec![Decision {
2601                    id: t.id.clone(),
2602                    recovery: Some(Recovery::Hold),
2603                    ..Decision::default()
2604                }],
2605            },
2606        )
2607        .unwrap();
2608
2609        let after = queue.get(&t.id).unwrap();
2610        assert_eq!(after.status, TaskStatus::Held);
2611        assert!(
2612            !after
2613                .hold_reason
2614                .as_deref()
2615                .unwrap_or_default()
2616                .starts_with("not enough free space"),
2617            "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
2618            after.hold_reason
2619        );
2620
2621        // The disk gate would report space is fine now - `crate::triage`
2622        // must not read the old text and release the task through it.
2623        let cfg_dir = tempdir().unwrap();
2624        let config = cfg_dir.path().join("magi.toml");
2625        std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2626        let report =
2627            crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
2628        assert!(report.resumed.is_empty(), "must not be auto-released");
2629        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2630    }
2631
2632    #[test]
2633    fn explicit_resumption_rehold_requires_new_evidence_and_another_answer() {
2634        crate::run::pin_test_home();
2635        let dir = tempdir().unwrap();
2636        let queue = Queue::at(dir.path().join("queue"));
2637        let questions = Questions::at(dir.path().join("questions"));
2638        let config = dir.path().join("magi.toml");
2639        std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2640        let now = jiff::Timestamp::now();
2641        let triage = || crate::triage::run_once(&queue, &questions, Some(&config), now);
2642        let open = || {
2643            questions
2644                .list()
2645                .into_iter()
2646                .filter(|q| q.node == "triage" && q.status.open())
2647                .collect::<Vec<_>>()
2648        };
2649        let hold = Decision {
2650            recovery: Some(Recovery::Hold),
2651            reason: Some("waiting on manual worktree cleanup".to_owned()),
2652            ..Decision::default()
2653        };
2654
2655        let mut t = task("looping hold");
2656        t.hold_machine(Some("waiting on manual worktree cleanup".to_owned()));
2657        queue.put(&mut t).unwrap();
2658        let hold = Decision {
2659            id: t.id.clone(),
2660            ..hold
2661        };
2662
2663        // 1. triage asks the ordinary machine question; the operator resumes.
2664        assert_eq!(triage().asked.len(), 1);
2665        let first = open().remove(0);
2666        let mut q = questions.get(&first.id).unwrap();
2667        let resume = q.choices[0].clone();
2668        q.answer(Answer::Choice(resume)).unwrap();
2669        questions.put(&mut q).unwrap();
2670        assert_eq!(triage().answered.len(), 1);
2671        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2672
2673        // Rewording the old cause cannot undo the first explicit answer.
2674        apply(
2675            &queue,
2676            &questions,
2677            &Verdict {
2678                decisions: vec![hold.clone()],
2679            },
2680        )
2681        .unwrap();
2682        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2683        assert!(triage().asked.is_empty());
2684
2685        // A new independently filed check can justify a later hold.
2686        let mut evidence = Question::new(
2687            t.id.clone(),
2688            "preflight".into(),
2689            "check".into(),
2690            "New permission failure".into(),
2691            String::new(),
2692            Vec::new(),
2693        );
2694        questions.put(&mut evidence).unwrap();
2695        let hold = Decision {
2696            evidence: vec![evidence.id.clone()],
2697            reason: Some("new permission check failed".into()),
2698            ..hold
2699        };
2700        apply(
2701            &queue,
2702            &questions,
2703            &Verdict {
2704                decisions: vec![hold.clone()],
2705            },
2706        )
2707        .unwrap();
2708        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2709
2710        // The new hold earns one question identifying the new evidence.
2711        assert_eq!(triage().asked.len(), 1);
2712        let second = open().remove(0);
2713        assert!(second.detail.contains("new permission check failed"));
2714        assert!(triage().asked.is_empty(), "no duplicate question");
2715        assert_eq!(open().len(), 1);
2716
2717        // Another explicit answer supersedes this later hold too.
2718        let mut q = questions.get(&second.id).unwrap();
2719        let force = q.choices[0].clone();
2720        q.answer(Answer::Choice(force)).unwrap();
2721        questions.put(&mut q).unwrap();
2722        assert_eq!(triage().answered.len(), 1);
2723        apply(
2724            &queue,
2725            &questions,
2726            &Verdict {
2727                decisions: vec![hold],
2728            },
2729        )
2730        .unwrap();
2731        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2732    }
2733
2734    #[test]
2735    fn a_runnable_task_can_be_held_directly_without_a_question() {
2736        let dir = tempdir().unwrap();
2737        let queue = Queue::at(dir.path().join("queue"));
2738        let questions = Questions::at(dir.path().join("questions"));
2739        let mut t = task("already answered, should stay put");
2740        queue.put(&mut t).unwrap();
2741
2742        apply(
2743            &queue,
2744            &questions,
2745            &Verdict {
2746                decisions: vec![Decision {
2747                    id: t.id.clone(),
2748                    recovery: Some(Recovery::Hold),
2749                    reason: Some("operator already said keep this held".to_owned()),
2750                    ..Decision::default()
2751                }],
2752            },
2753        )
2754        .unwrap();
2755
2756        let after = queue.get(&t.id).unwrap();
2757        assert_eq!(after.status, TaskStatus::Held);
2758        assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
2759    }
2760
2761    #[test]
2762    fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
2763        crate::run::pin_test_home();
2764        // Reproduces the other half of the reported bug (task 6081): once
2765        // the operator's answer says the work already happened outside the
2766        // loop - PR merged, worktree cleaned up - the conductor needs an
2767        // actual terminal state to put the task in, not just a hold it will
2768        // keep being re-asked about.
2769        let dir = tempdir().unwrap();
2770        let queue = Queue::at(dir.path().join("queue"));
2771        let questions = Questions::at(dir.path().join("questions"));
2772        let mut t = task("already merged by hand");
2773        t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
2774        t.record_answer(
2775            "Handle this one?".to_owned(),
2776            "already merged and cleaned up, close it".to_owned(),
2777        );
2778        queue.put(&mut t).unwrap();
2779
2780        apply(
2781            &queue,
2782            &questions,
2783            &Verdict {
2784                decisions: vec![Decision {
2785                    id: t.id.clone(),
2786                    recovery: Some(Recovery::Done),
2787                    reason: Some("operator confirmed this already landed".to_owned()),
2788                    ..Decision::default()
2789                }],
2790            },
2791        )
2792        .unwrap();
2793
2794        let after = queue.get(&t.id).unwrap();
2795        assert_eq!(after.status, TaskStatus::Done);
2796        assert!(after.hold_reason.is_none());
2797        assert_eq!(after.answers.len(), 1, "the record of why is kept");
2798    }
2799
2800    #[test]
2801    fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
2802        let dir = tempdir().unwrap();
2803        let queue = Queue::at(dir.path().join("queue"));
2804        let questions = Questions::at(dir.path().join("questions"));
2805
2806        let mut queued = task("never ran yet");
2807        queue.put(&mut queued).unwrap();
2808
2809        let mut running = task("mid-run");
2810        running.start("run-1".to_owned());
2811        queue.put(&mut running).unwrap();
2812
2813        for id in [queued.id.clone(), running.id.clone()] {
2814            apply(
2815                &queue,
2816                &questions,
2817                &Verdict {
2818                    decisions: vec![Decision {
2819                        id,
2820                        recovery: Some(Recovery::Done),
2821                        ..Decision::default()
2822                    }],
2823                },
2824            )
2825            .unwrap();
2826        }
2827
2828        assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
2829        assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
2830    }
2831
2832    #[test]
2833    fn a_question_after_two_settled_answers_is_still_filed_and_blocks() {
2834        // A genuine question must stay answerable however many were settled
2835        // before it: refusing it left `magi answer --list` empty.
2836        let dir = tempdir().unwrap();
2837        let queue = Queue::at(dir.path().join("queue"));
2838        let questions = Questions::at(dir.path().join("questions"));
2839        let mut t = task("asked about repeatedly");
2840        t.hold_machine(Some("out of attempts".to_owned()));
2841        t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
2842        t.record_answer(
2843            "Handle this one? (2)".to_owned(),
2844            "still not yet".to_owned(),
2845        );
2846        queue.put(&mut t).unwrap();
2847        assert_eq!(questions.list().len(), 0);
2848
2849        let verdict = Verdict {
2850            decisions: vec![Decision {
2851                id: t.id.clone(),
2852                question: Some("Branch conflicts with origin/main, how do we proceed?".to_owned()),
2853                ..Decision::default()
2854            }],
2855        };
2856        apply(&queue, &questions, &verdict).unwrap();
2857
2858        let filed = questions.list();
2859        assert_eq!(filed.len(), 1, "the question was filed");
2860        assert_eq!(
2861            filed[0].summary,
2862            "Branch conflicts with origin/main, how do we proceed?"
2863        );
2864        let deputy = filed[0]
2865            .deputy
2866            .as_ref()
2867            .expect("the wait is handed to a deputy");
2868        assert!(deputy.brief.contains(&t.id), "{}", deputy.brief);
2869        assert_eq!(
2870            deputy.starts, 0,
2871            "filing never starts anything: the loop does not wait"
2872        );
2873        assert!(filed[0].status.open());
2874        assert_eq!(filed[0].node, NODE);
2875        let after = queue.get(&t.id).unwrap();
2876        assert_eq!(after.status, TaskStatus::Blocked);
2877        assert_eq!(after.blocked_by, vec![filed[0].id.clone()]);
2878        assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
2879        assert!(
2880            !after
2881                .hold_reason
2882                .clone()
2883                .unwrap_or_default()
2884                .contains("conduct tried to ask"),
2885            "no hold was applied"
2886        );
2887
2888        apply(&queue, &questions, &verdict).unwrap();
2889        let again = questions.list();
2890        assert_eq!(again.len(), 1, "the open question is reused");
2891        assert_eq!(again[0].id, filed[0].id);
2892    }
2893
2894    #[test]
2895    fn question_source_identity_resolves_aliases_and_rejects_invented_targets() {
2896        let dir = tempdir().unwrap();
2897        init_repo_with_branch(dir.path(), "source-a");
2898        let mut t = task("source decision");
2899        t.repo = dir.path().into();
2900        let decision = |reference: &str| QuestionDecision {
2901            operation: QuestionOperation::Recover,
2902            target: QuestionTarget::Source {
2903                reference: reference.into(),
2904            },
2905        };
2906        assert_eq!(
2907            decision("source-a").identity(&t).unwrap(),
2908            decision("refs/heads/source-a").identity(&t).unwrap()
2909        );
2910        assert_ne!(
2911            decision("source-a").identity(&t).unwrap(),
2912            decision("main").identity(&t).unwrap()
2913        );
2914        assert!(decision("invented-source-label").identity(&t).is_err());
2915        let unrelated = QuestionDecision {
2916            operation: QuestionOperation::Resume,
2917            target: QuestionTarget::Run {
2918                id: "unrelated-run".into(),
2919            },
2920        };
2921        assert!(unrelated.identity(&t).is_err());
2922    }
2923
2924    #[test]
2925    fn invalid_question_targets_still_file_and_block_queued_tasks() {
2926        let dir = tempdir().unwrap();
2927        init_repo_with_branch(dir.path(), "source-a");
2928        let queue = Queue::at(dir.path().join("queue"));
2929        let questions = Questions::at(dir.path().join("questions"));
2930        for target in [
2931            QuestionTarget::Source {
2932                reference: "missing-source".into(),
2933            },
2934            QuestionTarget::Run {
2935                id: "unrelated-run".into(),
2936            },
2937        ] {
2938            let mut t = task("decision with invalid metadata");
2939            t.repo = dir.path().into();
2940            queue.put(&mut t).unwrap();
2941            let d = Decision {
2942                id: t.id.clone(),
2943                question: Some("Which recovery should proceed?".into()),
2944                choices: vec!["inspect first".into(), "keep waiting".into()],
2945                question_decision: Some(QuestionDecision {
2946                    operation: QuestionOperation::Recover,
2947                    target,
2948                }),
2949                ..Decision::default()
2950            };
2951            apply_one(&queue, &questions, &d, None).unwrap();
2952            let q = questions
2953                .list()
2954                .into_iter()
2955                .find(|q| q.run == t.id)
2956                .unwrap();
2957            assert_eq!(q.summary, d.question.as_deref().unwrap());
2958            assert_eq!(q.choices, d.choices);
2959            assert!(q.status.open());
2960            let context: serde_json::Value =
2961                serde_json::from_str(q.decision_context.as_ref().unwrap()).unwrap();
2962            assert!(context[3].is_null(), "invalid metadata was discarded");
2963            let after = queue.get(&t.id).unwrap();
2964            assert_eq!(after.status, TaskStatus::Blocked);
2965            assert_eq!(after.blocked_by, vec![q.id.clone()]);
2966            apply_one(&queue, &questions, &d, None).unwrap();
2967            assert_eq!(questions.list().iter().filter(|q| q.run == t.id).count(), 1);
2968        }
2969    }
2970
2971    #[test]
2972    fn answered_conductor_question_needs_changed_facts_before_reasking() {
2973        let dir = tempdir().unwrap();
2974        let queue = Queue::at(dir.path().join("queue"));
2975        let questions = Questions::at(dir.path().join("questions"));
2976        let mut t = task("recovery decision");
2977        t.hold_machine(Some("out of attempts".into()));
2978        t.runs = vec!["source-run-a".into(), "source-run-b".into()];
2979        queue.put(&mut t).unwrap();
2980        let mut d = Decision {
2981            id: t.id.clone(),
2982            question: Some("Recover from source A?".into()),
2983            choices: vec!["approve source A".into(), "keep held".into()],
2984            question_decision: Some(QuestionDecision {
2985                operation: QuestionOperation::Recover,
2986                target: QuestionTarget::Run {
2987                    id: "source-run-a".into(),
2988                },
2989            }),
2990            ..Decision::default()
2991        };
2992        apply_one(&queue, &questions, &d, None).unwrap();
2993        let mut q = questions.list().pop().unwrap();
2994        q.answer(crate::ask::Answer::Choice("keep held".into()))
2995            .unwrap();
2996        questions.put(&mut q).unwrap();
2997        queue
2998            .modify(&t.id, |task| {
2999                task.unblock(&q.id);
3000                task.hold_machine(Some("recovery remains under consideration".into()));
3001                true
3002            })
3003            .unwrap();
3004        d.question = Some("Should we keep waiting on recovery?".into());
3005        d.choices = vec!["recover using source A".into(), "remain on hold".into()];
3006        apply_one(&queue, &questions, &d, None).unwrap();
3007        assert_eq!(questions.list().len(), 1, "no repeated operator turn");
3008        d.question = Some("Recover from source B?".into());
3009        // The labels can remain identical while the approval target changes.
3010        d.question_decision.as_mut().unwrap().target = QuestionTarget::Run {
3011            id: "source-run-b".into(),
3012        };
3013        apply_one(&queue, &questions, &d, None).unwrap();
3014        assert_eq!(
3015            questions.list().len(),
3016            2,
3017            "new decision with unchanged task facts"
3018        );
3019        let after = queue.get(&t.id).unwrap();
3020        assert_eq!(after.instruction, t.instruction);
3021        assert_eq!(after.runs, t.runs);
3022        assert_eq!(after.status, TaskStatus::Blocked);
3023    }
3024
3025    #[test]
3026    fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
3027        let dir = tempdir().unwrap();
3028        let queue = Queue::at(dir.path().join("queue"));
3029        let questions = Questions::at(dir.path().join("questions"));
3030        let mut t = task("asked about once already");
3031        t.hold_machine(Some("out of attempts".to_owned()));
3032        t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
3033        queue.put(&mut t).unwrap();
3034
3035        apply(
3036            &queue,
3037            &questions,
3038            &Verdict {
3039                decisions: vec![Decision {
3040                    id: t.id.clone(),
3041                    question: Some("Still not sure - now what?".to_owned()),
3042                    ..Decision::default()
3043                }],
3044            },
3045        )
3046        .unwrap();
3047
3048        assert_eq!(questions.list().len(), 1, "the second question was filed");
3049        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3050    }
3051
3052    #[test]
3053    fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
3054        // `crate::triage` never calls `Task::block`, and only ever looks at
3055        // tasks still `held` - so a conductor question that moved this task
3056        // to `blocked` would orphan triage's own open question. The guard in
3057        // `apply_one` must leave the task alone until triage's question
3058        // settles.
3059        let dir = tempdir().unwrap();
3060        let queue = Queue::at(dir.path().join("queue"));
3061        let questions = Questions::at(dir.path().join("questions"));
3062        let mut t = task("held, triage already asking about it");
3063        t.hold_machine(Some("cause unclear".to_owned()));
3064        queue.put(&mut t).unwrap();
3065
3066        let mut triage_q = Question::new(
3067            t.id.clone(),
3068            crate::triage::NODE.to_owned(),
3069            "triage".to_owned(),
3070            "Still needed?".to_owned(),
3071            String::new(),
3072            vec![
3073                "resume".to_owned(),
3074                "not yet".to_owned(),
3075                "discard".to_owned(),
3076            ],
3077        );
3078        questions.put(&mut triage_q).unwrap();
3079
3080        for decision in [
3081            Decision {
3082                id: t.id.clone(),
3083                question: Some("what now?".to_owned()),
3084                ..Decision::default()
3085            },
3086            Decision {
3087                id: t.id.clone(),
3088                recovery: Some(Recovery::Requeue),
3089                ..Decision::default()
3090            },
3091        ] {
3092            apply(
3093                &queue,
3094                &questions,
3095                &Verdict {
3096                    decisions: vec![decision],
3097                },
3098            )
3099            .unwrap();
3100        }
3101
3102        let after = queue.get(&t.id).unwrap();
3103        assert_eq!(
3104            after.status,
3105            TaskStatus::Held,
3106            "triage still owns this hold"
3107        );
3108        assert!(after.blocked_by.is_empty());
3109        assert_eq!(
3110            questions.list().len(),
3111            1,
3112            "no second, conductor-owned question was filed"
3113        );
3114    }
3115
3116    #[test]
3117    fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
3118        // The race `crate::triage::pending_for` exists to close: the
3119        // operator has already answered the triage question (it is no
3120        // longer `open`), but `crate::triage::run_once` - which only runs on
3121        // a fully idle daemon tick, far less often than the conductor polls
3122        // - has not had a turn to apply it yet. `apply_one` must not treat
3123        // "not open" as "settled" here, or it would block the task out from
3124        // under an answer triage has not read back yet.
3125        let dir = tempdir().unwrap();
3126        let queue = Queue::at(dir.path().join("queue"));
3127        let questions = Questions::at(dir.path().join("questions"));
3128        let mut t = task("held, triage question answered but not yet applied");
3129        t.hold_machine(Some("cause unclear".to_owned()));
3130        queue.put(&mut t).unwrap();
3131
3132        let mut triage_q = Question::new(
3133            t.id.clone(),
3134            crate::triage::NODE.to_owned(),
3135            "triage".to_owned(),
3136            "Still needed?".to_owned(),
3137            String::new(),
3138            vec![
3139                "resume".to_owned(),
3140                "not yet".to_owned(),
3141                "discard".to_owned(),
3142            ],
3143        );
3144        questions.put(&mut triage_q).unwrap();
3145        triage_q
3146            .answer(Answer::Choice("not yet".to_owned()))
3147            .unwrap();
3148        questions.put(&mut triage_q).unwrap();
3149        assert!(!triage_q.status.open());
3150
3151        apply(
3152            &queue,
3153            &questions,
3154            &Verdict {
3155                decisions: vec![Decision {
3156                    id: t.id.clone(),
3157                    question: Some("what now?".to_owned()),
3158                    ..Decision::default()
3159                }],
3160            },
3161        )
3162        .unwrap();
3163
3164        let after = queue.get(&t.id).unwrap();
3165        assert_eq!(
3166            after.status,
3167            TaskStatus::Held,
3168            "triage's own answer is not yet applied - conduct must wait"
3169        );
3170        assert_eq!(
3171            questions.list().len(),
3172            1,
3173            "no conductor question was filed over the pending triage answer"
3174        );
3175    }
3176
3177    #[test]
3178    fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
3179        let dir = tempdir().unwrap();
3180        let queue = Queue::at(dir.path().join("queue"));
3181        let questions = Questions::at(dir.path().join("questions"));
3182        let mut held = task("manual recovery");
3183        held.priority = 300;
3184        held.runs.push("run20260912-224242-daf5".to_owned());
3185        held.hold_manual(Some(
3186            "active manual recovery run20260912-224242-daf5".to_owned(),
3187        ));
3188        queue.put(&mut held).unwrap();
3189
3190        // Every field a conductor may use to alter lifecycle state is ignored:
3191        // requeue/review would dispatch duplicate work, hold could overwrite
3192        // evidence, and a question would turn the hold into `blocked`.
3193        for decision in [
3194            Decision {
3195                id: held.id.clone(),
3196                recovery: Some(Recovery::Requeue),
3197                ..Decision::default()
3198            },
3199            Decision {
3200                id: held.id.clone(),
3201                recovery: Some(Recovery::Hold),
3202                reason: Some("stale replacement reason".to_owned()),
3203                ..Decision::default()
3204            },
3205            Decision {
3206                id: held.id.clone(),
3207                recovery: Some(Recovery::Review),
3208                ..Decision::default()
3209            },
3210            Decision {
3211                id: held.id.clone(),
3212                blocked_by: vec!["other-task".to_owned()],
3213                question: Some("retry now?".to_owned()),
3214                ..Decision::default()
3215            },
3216        ] {
3217            apply(
3218                &queue,
3219                &questions,
3220                &Verdict {
3221                    decisions: vec![decision],
3222                },
3223            )
3224            .unwrap();
3225        }
3226
3227        let after = queue.get(&held.id).unwrap();
3228        assert_eq!(after.status, TaskStatus::Held);
3229        assert!(after.operator_held());
3230        assert_eq!(after.priority, 300);
3231        assert_eq!(after.runs, ["run20260912-224242-daf5"]);
3232        assert_eq!(
3233            after.hold_reason.as_deref(),
3234            Some("active manual recovery run20260912-224242-daf5")
3235        );
3236        assert!(after.blocked_by.is_empty());
3237        assert!(questions.list().is_empty());
3238        assert!(
3239            queue.next_runnable().is_none(),
3240            "must not dispatch a duplicate"
3241        );
3242    }
3243
3244    #[test]
3245    fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
3246        let dir = tempdir().unwrap();
3247        let queue = Queue::at(dir.path().join("queue"));
3248        let questions = Questions::at(dir.path().join("questions"));
3249
3250        let mut automatic = task("disk gate");
3251        automatic.hold_machine(Some("disk full".to_owned()));
3252        queue.put(&mut automatic).unwrap();
3253        let requeue = || Verdict {
3254            decisions: vec![Decision {
3255                id: automatic.id.clone(),
3256                recovery: Some(Recovery::Requeue),
3257                ..Decision::default()
3258            }],
3259        };
3260        apply(&queue, &questions, &requeue()).unwrap();
3261        assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
3262
3263        let mut manual = task("operator gate");
3264        manual.hold_manual(Some("wait for operator".to_owned()));
3265        queue.put(&mut manual).unwrap();
3266        apply(
3267            &queue,
3268            &questions,
3269            &Verdict {
3270                decisions: vec![Decision {
3271                    id: manual.id.clone(),
3272                    recovery: Some(Recovery::Requeue),
3273                    ..Decision::default()
3274                }],
3275            },
3276        )
3277        .unwrap();
3278        assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
3279
3280        // This mirrors the CLI and web release routes: only an explicit
3281        // operator action clears the manual boundary.
3282        let mut released = queue.get(&manual.id).unwrap();
3283        released.release();
3284        queue.put(&mut released).unwrap();
3285        assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
3286    }
3287
3288    #[test]
3289    fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
3290        let dir = tempdir().unwrap();
3291        let queue = Queue::at(dir.path().join("queue"));
3292        let questions = Questions::at(dir.path().join("questions"));
3293        let mut legacy = task("old explicit hold");
3294        legacy.status = TaskStatus::Held;
3295        legacy.hold_reason = Some("manual recovery already active".to_owned());
3296        legacy.hold_source = None;
3297        legacy.blocked_by = vec!["dependency".to_owned()];
3298        queue.put(&mut legacy).unwrap();
3299
3300        apply(
3301            &queue,
3302            &questions,
3303            &Verdict {
3304                decisions: vec![Decision {
3305                    id: legacy.id.clone(),
3306                    recovery: Some(Recovery::Requeue),
3307                    ..Decision::default()
3308                }],
3309            },
3310        )
3311        .unwrap();
3312
3313        let after = queue.get(&legacy.id).unwrap();
3314        assert_eq!(after.status, TaskStatus::Held);
3315        assert_eq!(after.hold_source, None);
3316        assert_eq!(after.hold_reason, legacy.hold_reason);
3317        assert_eq!(after.blocked_by, legacy.blocked_by);
3318    }
3319
3320    #[test]
3321    fn review_recovery_is_a_no_op_without_a_survivable_branch() {
3322        // `surviving_branch` reaches `RunState::load`, which reaches the
3323        // process-global `run::home()` - a `OnceLock`, so this only wins the
3324        // race the first time it runs in the binary; every other test still
3325        // reaches the same directory whichever call won, and this test's own
3326        // run id never collides with another test's.
3327        crate::run::pin_test_home();
3328        let dir = tempdir().unwrap();
3329        let queue = Queue::at(dir.path().join("queue"));
3330        let questions = Questions::at(dir.path().join("questions"));
3331        let mut t = task("blocked with no readable run");
3332        t.start("20260101-000000-dead".to_owned()); // no such run on disk
3333        t.fail("blocked", 5);
3334        queue.put(&mut t).unwrap();
3335
3336        apply(
3337            &queue,
3338            &questions,
3339            &Verdict {
3340                decisions: vec![Decision {
3341                    id: t.id.clone(),
3342                    recovery: Some(Recovery::Review),
3343                    ..Decision::default()
3344                }],
3345            },
3346        )
3347        .unwrap();
3348
3349        let after = queue.get(&t.id).unwrap();
3350        assert_eq!(
3351            after.status,
3352            TaskStatus::Failed,
3353            "with nothing to reopen, the decision is dropped rather than guessed at"
3354        );
3355        assert!(after.review_branch.is_none());
3356    }
3357
3358    #[test]
3359    fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
3360        // `Hold` is the one exception - see `Decision::recovery`'s doc and
3361        // `a_runnable_task_can_be_held_directly_without_a_question` - but a
3362        // task already in line has nothing for `requeue` or `review` to do.
3363        let dir = tempdir().unwrap();
3364        let queue = Queue::at(dir.path().join("queue"));
3365        let questions = Questions::at(dir.path().join("questions"));
3366
3367        for recovery in [Recovery::Requeue, Recovery::Review] {
3368            let mut t = task("ordinary");
3369            queue.put(&mut t).unwrap();
3370
3371            apply(
3372                &queue,
3373                &questions,
3374                &Verdict {
3375                    decisions: vec![Decision {
3376                        id: t.id.clone(),
3377                        recovery: Some(recovery),
3378                        ..Decision::default()
3379                    }],
3380                },
3381            )
3382            .unwrap();
3383
3384            assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3385        }
3386    }
3387
3388    #[tokio::test]
3389    async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
3390        let dir = tempdir().unwrap();
3391        let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
3392        let queue = Queue::at(dir.path().join("queue"));
3393        let questions = Questions::at(dir.path().join("questions"));
3394        let mut t = task("normal");
3395        queue.put(&mut t).unwrap();
3396
3397        let mut conductor = Conductor::new();
3398        conductor
3399            .maybe_run(
3400                &cfg,
3401                dir.path(),
3402                &queue,
3403                &questions,
3404                dir.path(),
3405                &[t.clone()],
3406                &[],
3407                &[],
3408                2,
3409            )
3410            .await;
3411
3412        assert_eq!(
3413            queue.get(&t.id).unwrap().status,
3414            TaskStatus::Queued,
3415            "a failed invocation must change nothing"
3416        );
3417        assert!(
3418            queue.next_runnable().is_some(),
3419            "the loop must still be able to take the next task"
3420        );
3421    }
3422
3423    #[tokio::test]
3424    async fn a_reply_with_no_json_leaves_the_queue_untouched() {
3425        let dir = tempdir().unwrap();
3426        let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
3427        let queue = Queue::at(dir.path().join("queue"));
3428        let questions = Questions::at(dir.path().join("questions"));
3429        let mut t = task("normal");
3430        queue.put(&mut t).unwrap();
3431
3432        let mut conductor = Conductor::new();
3433        conductor
3434            .maybe_run(
3435                &cfg,
3436                dir.path(),
3437                &queue,
3438                &questions,
3439                dir.path(),
3440                &[t.clone()],
3441                &[],
3442                &[],
3443                2,
3444            )
3445            .await;
3446
3447        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3448    }
3449
3450    #[tokio::test]
3451    async fn a_conductor_chain_falls_to_the_next_agent_once_each() {
3452        let dir = tempdir().unwrap();
3453        let mut t = task("chained");
3454        let reply = format!(
3455            "{{\"decisions\":[{{\"id\":\"{}\",\"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}",
3456            t.id
3457        );
3458        let counted = |id: &str, tail: &str| {
3459            let calls = dir.path().join(format!("{id}.calls"));
3460            let path = dir.path().join(format!("{id}.sh"));
3461            std::fs::write(
3462                &path,
3463                format!(
3464                    "#!/bin/sh\ncat >/dev/null\necho x >> '{}'\n{tail}\n",
3465                    calls.display()
3466                ),
3467            )
3468            .unwrap();
3469            AgentSpec {
3470                id: id.to_owned(),
3471                kind: AgentKind::Command,
3472                model: None,
3473                command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
3474                extra_args: Vec::new(),
3475                env: BTreeMap::new(),
3476                prompt_delivery: None,
3477            }
3478        };
3479        let n_calls = |id: &str| {
3480            std::fs::read_to_string(dir.path().join(format!("{id}.calls")))
3481                .map_or(0, |s| s.lines().count())
3482        };
3483        let mut cfg = config(counted("a", "exit 3"));
3484        cfg.agents = vec![
3485            counted("a", "exit 3"),
3486            counted("b", &format!("printf '%s' '{reply}'")),
3487        ];
3488        cfg.roles.conductor = Some(crate::config::AgentChoice::Chain(vec![
3489            "a".into(),
3490            "b".into(),
3491            "a".into(),
3492        ]));
3493        let queue = Queue::at(dir.path().join("queue"));
3494        let questions = Questions::at(dir.path().join("questions"));
3495        queue.put(&mut t).unwrap();
3496
3497        let mut conductor = Conductor::new();
3498        conductor
3499            .maybe_run(
3500                &cfg,
3501                dir.path(),
3502                &queue,
3503                &questions,
3504                dir.path(),
3505                &[t.clone()],
3506                &[],
3507                &[],
3508                2,
3509            )
3510            .await;
3511
3512        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3513        assert_eq!(n_calls("a"), 1, "each id is tried once");
3514        assert_eq!(n_calls("b"), 1);
3515        let saved = std::fs::read_to_string(seat_path(dir.path())).unwrap();
3516        assert!(
3517            saved.contains("\"b\""),
3518            "the seat on disk is the one that ran: {saved}"
3519        );
3520    }
3521
3522    #[tokio::test]
3523    async fn json_survives_code_fences_and_a_preamble() {
3524        let dir = tempdir().unwrap();
3525        let mut t = task("fenced");
3526        let reply = format!(
3527            "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
3528             \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
3529            t.id
3530        );
3531        let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
3532        let queue = Queue::at(dir.path().join("queue"));
3533        let questions = Questions::at(dir.path().join("questions"));
3534        queue.put(&mut t).unwrap();
3535
3536        let mut conductor = Conductor::new();
3537        conductor
3538            .maybe_run(
3539                &cfg,
3540                dir.path(),
3541                &queue,
3542                &questions,
3543                dir.path(),
3544                &[t.clone()],
3545                &[],
3546                &[],
3547                2,
3548            )
3549            .await;
3550
3551        let back = queue.get(&t.id).unwrap();
3552        assert_eq!(back.status, TaskStatus::Blocked);
3553        assert_eq!(back.blocked_by, ["x"]);
3554    }
3555
3556    #[tokio::test]
3557    async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
3558        // Each real invocation writes its own artifact stem, `turn-<n>`, so
3559        // whether a second one happened is read off the artifacts directory.
3560        let dir = tempdir().unwrap();
3561        let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3562        let queue = Queue::at(dir.path().join("queue"));
3563        let questions = Questions::at(dir.path().join("questions"));
3564        let mut t = task("stable");
3565        queue.put(&mut t).unwrap();
3566        let artifacts = dir.path().join("conduct").join("artifacts");
3567        let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3568
3569        let mut conductor = Conductor::new();
3570        conductor
3571            .maybe_run(
3572                &cfg,
3573                dir.path(),
3574                &queue,
3575                &questions,
3576                dir.path(),
3577                &[t.clone()],
3578                &[],
3579                &[],
3580                2,
3581            )
3582            .await;
3583        assert!(turn(1).is_file(), "the first cycle must call the conductor");
3584
3585        conductor
3586            .maybe_run(
3587                &cfg,
3588                dir.path(),
3589                &queue,
3590                &questions,
3591                dir.path(),
3592                &[t.clone()],
3593                &[],
3594                &[],
3595                2,
3596            )
3597            .await;
3598        assert!(
3599            !turn(2).is_file(),
3600            "an unchanged revision and an unchanged stalled/finished set must not call the \
3601             conductor twice"
3602        );
3603
3604        // Once the queue actually changes, the next `maybe_run` calls again.
3605        t.priority = 1;
3606        queue.put(&mut t).unwrap();
3607        conductor
3608            .maybe_run(
3609                &cfg,
3610                dir.path(),
3611                &queue,
3612                &questions,
3613                dir.path(),
3614                &[t.clone()],
3615                &[],
3616                &[],
3617                2,
3618            )
3619            .await;
3620        assert!(turn(2).is_file(), "a moved revision calls it again");
3621    }
3622
3623    #[tokio::test]
3624    async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
3625        // The queue's own revision has not moved - nothing wrote to it - but
3626        // a task now looks stalled, purely because time passed. Calling
3627        // again here, and never again once this exact set has been shown
3628        // once, is the whole point of keying `worth_a_look` on the id set
3629        // rather than on "is it non-empty".
3630        let dir = tempdir().unwrap();
3631        let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3632        let queue = Queue::at(dir.path().join("queue"));
3633        let questions = Questions::at(dir.path().join("questions"));
3634        let mut t = task("quiet");
3635        queue.put(&mut t).unwrap();
3636        let artifacts = dir.path().join("conduct").join("artifacts");
3637        let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3638
3639        let mut conductor = Conductor::new();
3640        conductor
3641            .maybe_run(
3642                &cfg,
3643                dir.path(),
3644                &queue,
3645                &questions,
3646                dir.path(),
3647                &[t.clone()],
3648                &[],
3649                &[],
3650                2,
3651            )
3652            .await;
3653        assert!(turn(1).is_file());
3654
3655        conductor
3656            .maybe_run(
3657                &cfg,
3658                dir.path(),
3659                &queue,
3660                &questions,
3661                dir.path(),
3662                &[],
3663                &[t.clone()],
3664                &[],
3665                2,
3666            )
3667            .await;
3668        assert!(
3669            turn(2).is_file(),
3670            "a task turning stalled must call the conductor again"
3671        );
3672
3673        // But once shown at this exact revision, showing the *same* stalled
3674        // set again must not call a third time.
3675        conductor
3676            .maybe_run(
3677                &cfg,
3678                dir.path(),
3679                &queue,
3680                &questions,
3681                dir.path(),
3682                &[],
3683                &[t.clone()],
3684                &[],
3685                2,
3686            )
3687            .await;
3688        assert!(
3689            !turn(3).is_file(),
3690            "the same stalled task lingering must not call the conductor every cycle"
3691        );
3692    }
3693
3694    fn rehold(queue: &Queue, questions: &Questions, id: &str, note: &str) {
3695        apply(
3696            queue,
3697            questions,
3698            &Verdict {
3699                decisions: vec![Decision {
3700                    id: id.to_owned(),
3701                    reason: Some(note.to_owned()),
3702                    recovery: Some(Recovery::Hold),
3703                    ..Decision::default()
3704                }],
3705            },
3706        )
3707        .unwrap();
3708    }
3709
3710    #[test]
3711    fn reholding_with_an_identical_note_changes_nothing() {
3712        let dir = tempdir().unwrap();
3713        let queue = Queue::at(dir.path().join("queue"));
3714        let questions = Questions::at(dir.path().join("questions"));
3715        let mut t = task("held");
3716        t.hold_machine(Some("waiting on a human".to_owned()));
3717        queue.put(&mut t).unwrap();
3718
3719        rehold(&queue, &questions, &t.id, "owner said keep it held");
3720        let first = queue.get(&t.id).unwrap();
3721        let rev = queue.revision();
3722        for _ in 0..5 {
3723            rehold(&queue, &questions, &t.id, "owner said keep it held");
3724        }
3725        let after = queue.get(&t.id).unwrap();
3726        assert_eq!(after.hold_reason, first.hold_reason);
3727        assert_eq!(after.updated_at, first.updated_at);
3728        assert_eq!(queue.revision(), rev, "nothing was written");
3729    }
3730
3731    #[test]
3732    fn reholding_with_different_notes_keeps_one_previously_level() {
3733        let dir = tempdir().unwrap();
3734        let queue = Queue::at(dir.path().join("queue"));
3735        let questions = Questions::at(dir.path().join("questions"));
3736        let mut t = task("held");
3737        t.hold_machine(Some("original".to_owned()));
3738        queue.put(&mut t).unwrap();
3739
3740        for i in 0..20 {
3741            rehold(&queue, &questions, &t.id, &format!("note {}", i % 2));
3742        }
3743        let reason = queue.get(&t.id).unwrap().hold_reason.unwrap();
3744        assert_eq!(reason.matches("(previously:").count(), 1, "{reason}");
3745        assert!(reason.starts_with("note "), "the new note leads: {reason}");
3746    }
3747
3748    #[test]
3749    fn an_already_nested_reason_is_matched_by_its_outermost_note() {
3750        let mut t = task("deep");
3751        let nested = "same\n\n(previously: same\n\n(previously: same))";
3752        t.hold_machine(Some(nested.to_owned()));
3753        let d = Decision {
3754            id: t.id.clone(),
3755            reason: Some("same".to_owned()),
3756            ..Decision::default()
3757        };
3758        assert_eq!(reaffirmed_hold_reason(&t, &d), None);
3759        let d2 = Decision {
3760            reason: Some("other".to_owned()),
3761            ..d
3762        };
3763        assert_eq!(
3764            reaffirmed_hold_reason(&t, &d2).as_deref(),
3765            Some("other\n\n(previously: same)")
3766        );
3767    }
3768
3769    #[tokio::test]
3770    async fn a_repeated_hold_decision_settles_instead_of_looping() {
3771        let dir = tempdir().unwrap();
3772        let queue = Queue::at(dir.path().join("queue"));
3773        let questions = Questions::at(dir.path().join("questions"));
3774        let reply = r#"{"decisions":[{"id":"ID","reason":"keep held","recovery":"hold"}]}"#;
3775        let mut t = task("held");
3776        t.hold_machine(Some("first".to_owned()));
3777        queue.put(&mut t).unwrap();
3778        let cfg = config(mock_agent(
3779            dir.path(),
3780            REPLY,
3781            env(&reply.replace("ID", &t.id)),
3782        ));
3783        let mut conductor = Conductor::new();
3784        // Cycle 1 writes the new note; cycle 2 repeats it and writes nothing.
3785        for _ in 0..2 {
3786            let held = queue.get(&t.id).unwrap();
3787            conductor
3788                .maybe_run(
3789                    &cfg,
3790                    dir.path(),
3791                    &queue,
3792                    &questions,
3793                    dir.path(),
3794                    &[],
3795                    &[],
3796                    &[held],
3797                    2,
3798                )
3799                .await;
3800        }
3801        let held = queue.get(&t.id).unwrap();
3802        assert!(
3803            held.hold_reason
3804                .as_deref()
3805                .unwrap()
3806                .starts_with("keep held")
3807        );
3808        assert!(
3809            !conductor.worth_a_look(&queue, &[], &[held]),
3810            "an unchanged held task must stop being reconsidered"
3811        );
3812    }
3813
3814    #[tokio::test]
3815    async fn alternating_wording_does_not_keep_a_held_task_in_the_loop() {
3816        let dir = tempdir().unwrap();
3817        let queue = Queue::at(dir.path().join("queue"));
3818        let questions = Questions::at(dir.path().join("questions"));
3819        let mut t = task("held");
3820        t.hold_machine(Some("first".to_owned()));
3821        queue.put(&mut t).unwrap();
3822        let reply = |note: &str| {
3823            format!(
3824                r#"{{"decisions":[{{"id":"{}","reason":"{note}","recovery":"hold"}}]}}"#,
3825                t.id
3826            )
3827        };
3828        let cfg_a = config(mock_agent(dir.path(), REPLY, env(&reply("keep held"))));
3829        let cfg_b = config(mock_agent(dir.path(), REPLY, env(&reply("leave held"))));
3830        let mut conductor = Conductor::new();
3831        let mut writes = 0;
3832        for i in 0..6 {
3833            let held = queue.get(&t.id).unwrap();
3834            let before = queue.revision();
3835            conductor
3836                .maybe_run(
3837                    if i % 2 == 0 { &cfg_a } else { &cfg_b },
3838                    dir.path(),
3839                    &queue,
3840                    &questions,
3841                    dir.path(),
3842                    &[],
3843                    &[],
3844                    &[held],
3845                    2,
3846                )
3847                .await;
3848            if queue.revision() != before {
3849                writes += 1;
3850            }
3851        }
3852        assert!(writes <= 2, "the loop must settle, saw {writes} writes");
3853        let mut held = queue.get(&t.id).unwrap();
3854        held.instruction.push_str(" (edited)");
3855        queue.put(&mut held).unwrap();
3856        let held = queue.get(&t.id).unwrap();
3857        assert!(conductor.worth_a_look(&queue, &[], &[held]));
3858    }
3859
3860    #[test]
3861    fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
3862        let dir = tempdir().unwrap();
3863        let queue = Queue::at(dir.path().join("queue"));
3864        let mut t = task("t");
3865        queue.put(&mut t).unwrap();
3866
3867        let mut conductor = Conductor::new();
3868        assert!(
3869            conductor.worth_a_look(&queue, &[], &[]),
3870            "a conductor that has never run has something to look at"
3871        );
3872
3873        conductor.last_seen = Some(conductor.snapshot(&queue, &[], &[]));
3874        assert!(
3875            !conductor.worth_a_look(&queue, &[], &[]),
3876            "nothing changed and nothing is stalled or finished"
3877        );
3878        assert!(
3879            conductor.worth_a_look(&queue, &[t.clone()], &[]),
3880            "a stalled task is worth a look even at the same revision"
3881        );
3882        assert!(
3883            conductor.worth_a_look(&queue, &[], &[t.clone()]),
3884            "a finished task is worth a look even at the same revision"
3885        );
3886    }
3887
3888    #[tokio::test]
3889    async fn the_conduct_path_never_calls_ask_and_wait() {
3890        // Structural: grepping this module and `daemon.rs` for
3891        // `ask_and_wait` is the actual assertion this module's own doc
3892        // promises; this test exists so the promise has a name in the test
3893        // output too. `apply_one`'s question path uses `Questions::put`
3894        // exclusively.
3895        let dir = tempdir().unwrap();
3896        let queue = Queue::at(dir.path().join("queue"));
3897        let questions = Questions::at(dir.path().join("questions"));
3898        let mut t = task("asks without blocking");
3899        queue.put(&mut t).unwrap();
3900
3901        apply(
3902            &queue,
3903            &questions,
3904            &Verdict {
3905                decisions: vec![Decision {
3906                    id: t.id.clone(),
3907                    question: Some("ok?".to_owned()),
3908                    ..Decision::default()
3909                }],
3910            },
3911        )
3912        .unwrap();
3913        // Reaching here at all (no hang) is the assertion.
3914        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3915    }
3916
3917    #[test]
3918    fn a_pinned_resume_is_not_held_or_requeued_by_the_conductor() {
3919        let mut t = Task::new("t".into(), "t".into(), PathBuf::new(), Source::Human);
3920        t.resume_override = Some(crate::queue::OperatorResume {
3921            question_id: "q".into(),
3922            at: jiff::Timestamp::now(),
3923            conductor_rehold: None,
3924            forced: false,
3925            pinned_run: Some("run-1".into()),
3926        });
3927        assert!(pinned_resume(&t));
3928        assert!(!may_hold(&mut t, "waiting for magi resume"));
3929        assert!(
3930            t.resume_override
3931                .as_ref()
3932                .unwrap()
3933                .conductor_rehold
3934                .is_none(),
3935            "a refused hold is not recorded as an override"
3936        );
3937        t.resume_override = None;
3938        assert!(!pinned_resume(&t));
3939        assert!(may_hold(&mut t, "no override, so a hold is allowed"));
3940    }
3941
3942    #[test]
3943    fn a_pinned_resume_is_not_blocked_by_a_conductor_question_or_dependency() {
3944        let dir = tempdir().unwrap();
3945        let queue = Queue::at(dir.path().join("queue"));
3946        let questions = Questions::at(dir.path().join("questions"));
3947        let mut t = task("resume me");
3948        t.resume_override = Some(crate::queue::OperatorResume {
3949            question_id: "q".into(),
3950            at: jiff::Timestamp::now(),
3951            conductor_rehold: None,
3952            forced: true,
3953            pinned_run: Some("run-1".into()),
3954        });
3955        queue.put(&mut t).unwrap();
3956
3957        apply(
3958            &queue,
3959            &questions,
3960            &Verdict {
3961                decisions: vec![
3962                    Decision {
3963                        id: t.id.clone(),
3964                        question: Some("really?".to_owned()),
3965                        ..Decision::default()
3966                    },
3967                    Decision {
3968                        id: t.id.clone(),
3969                        blocked_by: vec!["other".to_owned()],
3970                        ..Decision::default()
3971                    },
3972                ],
3973            },
3974        )
3975        .unwrap();
3976
3977        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3978        assert!(questions.list().is_empty());
3979    }
3980}