Skip to main content

magi/
conduct.rs

1//! The conductor: a single agent seat that arranges the queue.
2//!
3//! [`crate::queue`] orders runnable tasks by `priority` alone; nothing in it
4//! can express that one task should wait for another, or that a task whose
5//! last run stopped short deserves a second look before the loop blindly
6//! retries it. Once per polling cycle, [`Conductor::maybe_run`] shows one
7//! agent seat three things - the runnable tasks, the tasks stuck `running`
8//! with no live daemon behind them, and the `failed`/`held` tasks nobody has
9//! decided about yet - and asks it to decide `blocked_by` for the first and a
10//! recovery for the other two. Everything else about the loop - which
11//! unblocked task runs next, one at a time, in `priority` order - is
12//! unchanged; see `crate::daemon`.
13//!
14//! # What the conductor may not do
15//!
16//! [`Decision`] has no field for `priority`, for deleting a task, or for
17//! touching git, a worktree, or a branch directly. [`Recovery::Review`] only
18//! ever reopens a branch `crate::daemon` itself resolved from the task's own
19//! run record ([`surviving_branch`]) - never a name the model wrote - through
20//! `crate::graph::Runner::review`, which reviews and verifies but never
21//! rewrites history.
22//!
23//! # Non-blocking by construction
24//!
25//! [`crate::ask::ask_and_wait`] is never called from here, and the prompt
26//! tells the model the same: that CLI command blocks until a human answers,
27//! and calling it from inside the conductor's own invocation would park the
28//! whole polling loop behind one task's question. Instead a decision that
29//! wants the operator's judgement carries a `question` field, and [`apply`]
30//! files it with [`Question::new`] and [`Questions::put`] and moves on in the
31//! same call. Something still has to read the owner's reply: the question
32//! carries a [`Deputy`] brief, and `magi serve` starts a short-lived seat of
33//! its own for it ([`crate::deputy`]) - never this loop.
34//!
35//! # Fails soft, always
36//!
37//! [`Conductor::maybe_run`] never returns an error: an unusable roster, a
38//! timed-out invocation, or a reply [`verdict::extract_json`] cannot parse are
39//! all logged and treated as "this cycle changes nothing." `crate::daemon`'s
40//! loop always falls through to its own `Queue::next_runnable` regardless of
41//! what happened here.
42
43use std::collections::{BTreeMap, BTreeSet};
44use std::path::{Path, PathBuf};
45use std::time::Duration;
46
47use anyhow::{Context as _, Result};
48use serde::{Deserialize, Serialize};
49
50use crate::agent::{self, Invocation, SeatState};
51use crate::ask::{ChoiceAction, Deputy, Question, Questions};
52use crate::config::Config;
53use crate::prompt;
54use crate::queue::{Queue, Task, TaskStatus};
55use crate::run::RunState;
56use crate::verdict;
57
58/// Seat name for the conductor's own CLI-side conversation, scoped away from
59/// every other seat magi ever opens - the same rule every other seat follows.
60const SEAT: &str = "conduct";
61
62/// Node name reported to the invoked agent (`MAGI_NODE`) and recorded on any
63/// question it files, so an operator reading the questions list can tell a
64/// conductor's question from one a run's own agent asked.
65pub const NODE: &str = "conduct";
66
67/// Wall-clock limit for one conductor turn. The conductor reads a queue
68/// listing and replies with json; it does not implement anything or run a
69/// build, so this is short - the same order of magnitude as
70/// `crate::chat`'s own single-turn, no-write invocations.
71const TURN_TIMEOUT: Duration = Duration::from_secs(300);
72
73/// What the conductor may choose for a `running`-but-stalled or a
74/// `failed`/`held` task.
75#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
76#[serde(rename_all = "lowercase")]
77pub enum Recovery {
78    /// Put it back in line, attempts reset - the same effect as
79    /// `magi task release`.
80    Requeue,
81    /// Leave it for a human, unchanged otherwise - the same effect as
82    /// `magi task hold`.
83    Hold,
84    /// Reopen the task's surviving branch as a review-only pass
85    /// (`crate::graph::Runner::review`) instead of competing from scratch.
86    /// Only takes effect when [`surviving_branch`] can actually name one;
87    /// otherwise `crate::daemon` falls back to [`Recovery::Requeue`].
88    Review,
89    /// Close the task outright, as [`Task::succeed`] - the same effect as
90    /// `magi task done`. For a `failed`/`held` task only, never `running`:
91    /// this is for a task whose own goal is already known to be met outside
92    /// the loop entirely (the branch was merged and the worktree cleaned up
93    /// by hand, say) and competing it again would only spend attempts on
94    /// work that has nothing left to do, not for a task that merely stopped
95    /// mid-competition and might still need to run.
96    Done,
97}
98
99/// The operation for which the operator's decision is requested. These are
100/// fixed protocol values, independent of the language of the displayed choices.
101#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
102#[serde(rename_all = "snake_case")]
103pub enum QuestionOperation {
104    /// Choose an existing implementation as the recovery source.
105    Recover,
106    /// Continue an existing run from its saved state.
107    Resume,
108    /// Authorize repairing existing work before continuing.
109    Repair,
110    /// Confirm that the task may be closed as complete.
111    Complete,
112    /// Request an operator inspection of the target.
113    Inspect,
114    /// Request a configuration change for the target.
115    Configure,
116}
117
118/// The concrete object of an operator decision. Sources resolve to commit ids;
119/// runs must belong to this task. A generated prose label is never an identity.
120#[derive(Debug, Clone, Deserialize)]
121#[serde(tag = "kind", rename_all = "snake_case")]
122pub enum QuestionTarget {
123    /// The task itself, for decisions without a more specific recorded object.
124    Task,
125    /// A run already owned by this task.
126    Run {
127        /// Full recorded run id.
128        id: String,
129    },
130    /// A concrete TOML setting. Different fields on the same task require
131    /// different decisions even when both operations are `configure`.
132    Setting {
133        /// Repository-relative TOML file containing the setting.
134        path: String,
135        /// Exact table/key components copied from that file.
136        keys: Vec<String>,
137    },
138    /// A recovery source which the repository can resolve to a commit.
139    Source {
140        /// Existing branch name or commit reference, never a display label.
141        reference: String,
142    },
143}
144
145/// The structured meaning of a question, separate from its displayed wording.
146#[derive(Debug, Clone, Deserialize)]
147pub struct QuestionDecision {
148    /// The concrete type of operator decision requested.
149    pub operation: QuestionOperation,
150    /// The recorded object the decision concerns.
151    pub target: QuestionTarget,
152}
153
154impl QuestionDecision {
155    fn identity(&self, task: &Task) -> Result<(QuestionOperation, String)> {
156        let target = match &self.target {
157            QuestionTarget::Task => format!("task:{}", task.id),
158            QuestionTarget::Run { id } => {
159                anyhow::ensure!(
160                    task.runs.contains(id),
161                    "question target is not this task's run"
162                );
163                format!("run:{id}")
164            }
165            QuestionTarget::Setting { path, keys } => {
166                use std::path::Component;
167                anyhow::ensure!(
168                    !path.is_empty()
169                        && Path::new(path)
170                            .components()
171                            .all(|c| matches!(c, Component::Normal(_)))
172                        && !keys.is_empty()
173                        && keys.iter().all(|key| !key.is_empty()),
174                    "question setting must name a repository file and concrete keys"
175                );
176                let root = task.repo.canonicalize()?;
177                let file = root.join(path).canonicalize()?;
178                let relative = file
179                    .strip_prefix(&root)
180                    .context("question setting is outside the repository")?;
181                let content = std::fs::read_to_string(&file)?;
182                let value: toml::Value = toml::from_str(&content)?;
183                let mut setting = &value;
184                for key in keys {
185                    setting = setting
186                        .get(key)
187                        .context("question setting key does not exist")?;
188                }
189                format!("setting:{}", serde_json::to_string(&(relative, keys))?)
190            }
191            QuestionTarget::Source { reference } => {
192                use crate::proc::Quiet as _;
193                let output = std::process::Command::new("git")
194                    .args([
195                        "rev-parse",
196                        "--verify",
197                        "--end-of-options",
198                        &format!("{reference}^{{commit}}"),
199                    ])
200                    .current_dir(&task.repo)
201                    .quiet()
202                    .output()
203                    .context("resolve question source")?;
204                anyhow::ensure!(
205                    output.status.success(),
206                    "question source cannot be resolved"
207                );
208                let commit = String::from_utf8(output.stdout)?.trim().to_owned();
209                anyhow::ensure!(
210                    !commit.is_empty() && commit.bytes().all(|c| c.is_ascii_hexdigit()),
211                    "invalid question source commit"
212                );
213                format!("source:{commit}")
214            }
215        };
216        Ok((self.operation, target))
217    }
218}
219
220/// The conductor's decision for one task. Deliberately has no `priority`
221/// field: see this module's doc.
222#[derive(Debug, Clone, Default, Deserialize)]
223pub struct Decision {
224    /// Task id, expected to be copied verbatim from what it was shown.
225    pub id: String,
226    /// Ids this task should be blocked on. Meaningful only for a runnable
227    /// (`queued`) task, or a machine-held task whose missing-only decision
228    /// is unconfirmed against the queue. Ignored otherwise.
229    #[serde(default)]
230    pub blocked_by: Vec<String>,
231    /// Dependencies claimed missing as the sole reason for this question or
232    /// hold. Must name references in `blocked_by`. Empty (including legacy
233    /// replies) means the question/hold may have an independent purpose and
234    /// must be preserved. Never set for decisions needing separate consent.
235    #[serde(default)]
236    pub missing_dependencies: Vec<String>,
237    /// One line explaining the block or the recovery.
238    #[serde(default)]
239    pub reason: Option<String>,
240    /// Recovery for a stalled or finished task. For a runnable (`queued`)
241    /// one, only [`Recovery::Hold`] has any effect. An independent hold takes
242    /// precedence over `blocked_by`, so dependency completion cannot release it.
243    /// Use a hold when the operator already said the task must not compete,
244    /// rather than asking a `question` that restates that answer. `Requeue`, `Review`,
245    /// and `Done` have no meaning for a task that is already in line.
246    #[serde(default)]
247    pub recovery: Option<Recovery>,
248    /// A question for the operator. When present, [`apply`] files it (unless
249    /// one is already open for this task) and blocks the task on its id
250    /// instead of acting on `blocked_by` or `recovery`. Only a missing-only
251    /// decision explicitly marked by [`Self::missing_dependencies`] may be
252    /// replaced with dependency waiting unless the queue confirms missing blockers.
253    #[serde(default)]
254    pub question: Option<String>,
255    /// Fixed answers for `question`, if it has any. Empty means free text.
256    #[serde(default)]
257    pub choices: Vec<String>,
258    /// Explicit operations keyed by exact choice labels; never inferred from prose.
259    #[serde(default)]
260    pub actions: BTreeMap<String, ChoiceAction>,
261    /// Stable operation and concrete target of a question. Missing on legacy
262    /// decisions: those share the task's existing cause, regardless of prose.
263    #[serde(default)]
264    pub question_decision: Option<QuestionDecision>,
265    /// Task or question ids carrying independently verifiable new blocking evidence.
266    #[serde(default)]
267    pub evidence: Vec<String>,
268}
269
270/// The conductor's whole reply for one cycle.
271///
272/// `decisions` is deliberately **not** `#[serde(default)]`, unlike every
273/// other field in this module. [`verdict::extract_json`] disambiguates
274/// between several balanced `{...}` spans in one reply by trying the type the
275/// caller wants against each of them, last first, and keeping the first that
276/// fits - which only works when a span that is not really the answer can
277/// fail to fit. A `Verdict` with no required field at all would make every
278/// span fit, including a `{}` left by stray trailing prose, and the reply's
279/// real `decisions` - earlier in the text - would never be reached. Requiring
280/// the key costs nothing: the prompt already asks for it on every reply, `[]`
281/// included.
282#[derive(Debug, Clone, Default, Deserialize)]
283pub struct Verdict {
284    /// One entry per task the conductor chose to say something about. A task
285    /// left out of this list is left exactly as it was.
286    pub decisions: Vec<Decision>,
287}
288
289/// A view of a task built for [`prompt::conduct`], shared by the runnable and
290/// stalled sections.
291fn view(t: &Task, max_attempts: usize) -> prompt::ConductTask {
292    prompt::ConductTask {
293        id: t.id.clone(),
294        title: t.title.clone(),
295        instruction: t.instruction.clone(),
296        repo: t.repo.display().to_string(),
297        priority: t.priority,
298        status: t.status.as_str().to_owned(),
299        attempts: t.attempts,
300        max_attempts: t.effective_attempt_limit(max_attempts),
301        last_error: t.last_error.clone(),
302        hold_reason: t.hold_reason.clone(),
303        held_at: t.held_at.map(|at| at.to_string()),
304        hold_source: t.hold_source.map(|source| source.label().to_owned()),
305        blocked_by: t.blocked_by.clone(),
306        answers: t
307            .answers
308            .iter()
309            .map(|a| prompt::ConductAnswer {
310                question: a.question.clone(),
311                answer: a.answer.clone(),
312            })
313            .collect(),
314        operator_resume: t.resumption.as_ref().map(|r| format!(
315            "Explicit operator resumption generation {} via {} at {}. Approved instruction: {}. \
316             The first {} historical answers and prior hold ({:?}) are superseded as waiting premises. \
317             To block again cite evidence ids from a later owned question, changed dependency, or new failed run; \
318             wording alone is insufficient.",
319            r.generation, r.source, r.at, r.instruction, r.answers_through, r.cleared_reason
320        )).or_else(|| t.resume_override.as_ref().map(|o| {
321            format!(
322                "the operator explicitly answered \"resume\" at {}; do not hold this \
323                 task again for the same reason unless there is new information",
324                o.at
325            )
326        })),
327    }
328}
329
330/// Did the operator answer a `magi ask` choice with a `resume` action that
331/// this task has not yet acted on? Then `Recovery::Requeue` must not turn the
332/// task into a fresh competition either.
333fn pinned_resume(task: &Task) -> bool {
334    task.resume_override
335        .as_ref()
336        .is_some_and(|o| o.pinned_run.is_some())
337}
338
339/// Verify references independently of the model's reason text. Old evidence
340/// cannot become new merely by being described differently.
341fn new_blocking_evidence(
342    queue: &Queue,
343    questions: &Questions,
344    task: &Task,
345    d: &Decision,
346) -> Vec<String> {
347    let Some(r) = &task.resumption else {
348        return Vec::new();
349    };
350    d.evidence
351        .iter()
352        .filter_map(|id| {
353            let version = if let Ok(q) = questions.get(id) {
354                (q.asked_at > r.at
355                    && (q.run == task.id || task.runs.contains(&q.run))
356                    && q.node != NODE
357                    && q.status != crate::ask::QuestionStatus::Abandoned)
358                    .then(|| {
359                        format!(
360                            "question:{id}:{}:{:?}:{}",
361                            q.asked_at,
362                            q.status,
363                            q.resolution().unwrap_or_default()
364                        )
365                    })
366            } else if d.blocked_by.contains(id) {
367                queue
368                    .get(id)
369                    .ok()
370                    .filter(|dep| {
371                        dep.id != task.id
372                            && dep.status != TaskStatus::Done
373                            && (dep.created_at > r.at
374                                || dep.held_at.is_some_and(|at| at > r.at)
375                                || dep.operator_decided_at.is_some_and(|at| at > r.at)
376                                || dep.passes.iter().any(|p| p.settled_at > r.at)
377                                || (dep.status == TaskStatus::Blocked
378                                    && dep.blocked_at.is_some_and(|at| at > r.at)))
379                    })
380                    .map(|dep| {
381                        format!(
382                            "dependency:{id}:{:?}:{:?}:{:?}",
383                            dep.status,
384                            dep.chat_notice_key(),
385                            dep.hold_notice_cause
386                        )
387                    })
388            } else if task.runs.iter().any(|run| run == id) {
389                RunState::load(id)
390                    .ok()
391                    .filter(|run| {
392                        // A resumed run keeps its id. Only a pass started after
393                        // approval (or a newly created legacy run) is new evidence.
394                        run.updated_at > r.at
395                            && run
396                                .passes
397                                .last()
398                                .map_or(run.created_at > r.at, |p| p.started_at > r.at)
399                            && matches!(
400                                run.status,
401                                crate::run::RunStatus::Failed
402                                    | crate::run::RunStatus::Blocked
403                                    | crate::run::RunStatus::Stalled
404                            )
405                    })
406                    .map(|run| {
407                        format!(
408                            "run:{id}:{:?}:{:?}",
409                            run.status,
410                            run.passes.last().map(|p| p.index)
411                        )
412                    })
413            } else {
414                None
415            };
416            version.filter(|v| !r.blocking_evidence.contains(v))
417        })
418        .collect()
419}
420
421/// Durable approval is enforced by verified evidence at the mutation boundary.
422/// A pinned run stays protected; legacy transient overrides keep their original
423/// bounded contradiction policy until an explicit new approval replaces it.
424fn may_hold(task: &mut Task, reason: &str) -> bool {
425    if pinned_resume(task) {
426        return false;
427    }
428    if task.resumption.is_some() {
429        return true; // The mutation boundary already verified new evidence.
430    }
431    let Some(o) = task.resume_override.as_mut() else {
432        return true;
433    };
434    if o.forced || o.pinned_run.is_some() {
435        tracing::warn!(
436            "conductor tried to hold task {} after the operator forced a resume: {reason}",
437            task.id
438        );
439        return false;
440    }
441    if o.conductor_rehold.is_some() {
442        return false;
443    }
444    o.conductor_rehold = Some(reason.to_owned());
445    true
446}
447
448/// Severity as a lowercase word, matching how `crate::verdict::Severity` is
449/// spelled everywhere else an operator or a model reads it.
450fn severity_str(s: crate::verdict::Severity) -> &'static str {
451    match s {
452        crate::verdict::Severity::Nit => "nit",
453        crate::verdict::Severity::Minor => "minor",
454        crate::verdict::Severity::Major => "major",
455        crate::verdict::Severity::Blocker => "blocker",
456    }
457}
458
459/// The branch a task's last run left behind, if the tally ever ran on it -
460/// what [`Recovery::Review`] reopens. Derived from `crate::run::RunState`
461/// alone, never from anything the conductor wrote, so a hallucinated branch
462/// name can never reach `crate::graph::Runner::review`.
463fn surviving_branch(task: &Task) -> Option<String> {
464    let last = task.runs.last()?;
465    let state = RunState::load(last).ok()?;
466    if !crate::daemon::compatible_run_instruction(&state, task) {
467        return None;
468    }
469    state.winner().map(|c| c.branch.clone())
470}
471
472/// The reason to record when the conductor holds a `failed`/`held` task via
473/// [`Recovery::Hold`]. Always `Some`, never a bare `d.reason.clone()`.
474///
475/// [`Task::hold_machine`] only overwrites [`Task::hold_reason`] when given
476/// one, precisely so a stalled task's existing diagnosis survives a
477/// decision that has nothing new to add. That is the right default when the
478/// task was not already `held` - but a task that *is* already `held`, and
479/// is being held again here, is a different case: its current
480/// `hold_reason` may still read as a cause `crate::triage` knows how to
481/// re-check on its own (a disk-pressure message, say - see
482/// `triage::machine_cause_resolved`), and if the model gives no reason,
483/// leaving that text untouched would make this decision - the conductor
484/// choosing, informed by the operator's own answer, to keep the task held
485/// anyway - indistinguishable from the original, never-reconsidered hold.
486/// `crate::triage` would then read the stale text the next time its one
487/// recognised cause looks resolved and release the task straight through
488/// the decision this call was recording. Prepending (not appending) the new
489/// note keeps the old text as context without leaving the string starting
490/// with whatever pattern `crate::triage` matched before.
491///
492/// Two bounds keep the string from growing, which it once did without limit
493/// (every re-hold nested the whole previous reason one level deeper, and every
494/// change relit the task's notice): a note equal to the current reason - or to
495/// its outermost note - adds nothing, so the result is `None` and the caller
496/// writes nothing; and only ONE `(previously: ...)` level is ever kept, the
497/// prior being cut back to its outermost note before it is embedded.
498fn reaffirmed_hold_reason(task: &Task, d: &Decision) -> Option<String> {
499    let note = match &d.reason {
500        Some(reason) => reason.clone(),
501        None => match task.answers.last() {
502            Some(a) => format!(
503                "conduct held this again with no new reason given; last operator \
504                 answer on record: {}",
505                a.answer
506            ),
507            None => "conduct held this again with no reason given".to_owned(),
508        },
509    };
510    match task.hold_reason.as_deref() {
511        Some(prior) if !prior.is_empty() => {
512            let outer = outermost_hold_note(prior);
513            if note.trim() == prior.trim() || note.trim() == outer {
514                None
515            } else {
516                Some(format!("{note}\n\n(previously: {outer})"))
517            }
518        }
519        _ => Some(note),
520    }
521}
522
523/// Separator [`reaffirmed_hold_reason`] puts between a note and the reason it
524/// replaced.
525const PREVIOUSLY: &str = "\n\n(previously: ";
526
527/// A hold reason without any `(previously: ...)` tail: the text before the
528/// first separator, trimmed.
529fn outermost_hold_note(reason: &str) -> &str {
530    reason.split(PREVIOUSLY).next().unwrap_or(reason).trim()
531}
532
533/// The conductor's stated reason for a hold, or a stand-in when it gave none.
534fn hold_note(d: &Decision) -> String {
535    d.reason
536        .clone()
537        .unwrap_or_else(|| "(no reason given)".to_owned())
538}
539
540/// Everything the conductor is shown about a `failed`/`held` task's last run.
541async fn outcome_for(task: &Task, repo: &Path) -> prompt::ConductOutcome {
542    let Some(run_id) = task.runs.last().cloned() else {
543        return prompt::ConductOutcome {
544            run_id: "(none)".to_owned(),
545            unreadable: Some("this task has not produced a run yet".to_owned()),
546            run_status: None,
547            open_findings: Vec::new(),
548            rounds_used: 0,
549            rounds_max: 0,
550            rounds: Vec::new(),
551            branch: None,
552            branch_head: None,
553            references: None,
554            empty_candidate: false,
555        };
556    };
557    let state = match RunState::load(&run_id) {
558        Ok(s) => s,
559        Err(e) => {
560            // The exact failure this feature exists to stop hiding: a schema
561            // mismatch (or any other unreadable state) must never be treated
562            // as "nothing to recover" - it is surfaced here, verbatim, rather
563            // than swallowed into a quiet re-competition.
564            tracing::warn!(
565                "conductor: could not read run {run_id} for task {}: {e:#}",
566                task.short()
567            );
568            return prompt::ConductOutcome {
569                run_id,
570                unreadable: Some(format!("{e:#}")),
571                run_status: None,
572                open_findings: Vec::new(),
573                rounds_used: 0,
574                rounds_max: 0,
575                rounds: Vec::new(),
576                branch: None,
577                branch_head: None,
578                references: None,
579                empty_candidate: false,
580            };
581        }
582    };
583
584    let finding_view = |f: &crate::verdict::Finding| prompt::ConductFinding {
585        id: f.id.clone(),
586        title: f.title.clone(),
587        severity: severity_str(f.severity).to_owned(),
588    };
589    let open_findings = state
590        .open_findings()
591        .into_iter()
592        .map(finding_view)
593        .collect();
594    let rounds = state
595        .reviews
596        .iter()
597        .map(|r| prompt::ConductRound {
598            round: r.round,
599            findings: r
600                .reviews
601                .iter()
602                .flat_map(|rec| rec.findings.iter())
603                .map(finding_view)
604                .collect(),
605            addressed: r
606                .fix
607                .as_ref()
608                .map(|fx| fx.addressed.clone())
609                .unwrap_or_default(),
610            rejected: r
611                .fix
612                .as_ref()
613                .map(|fx| {
614                    fx.rejected
615                        .iter()
616                        .map(|rej| prompt::ConductRejection {
617                            id: rej.id.clone(),
618                            why: rej.why.clone(),
619                        })
620                        .collect()
621                })
622                .unwrap_or_default(),
623        })
624        .collect();
625    let branch = state.winner().map(|c| c.branch.clone());
626    let branch_head = match &branch {
627        Some(b) => crate::git::rev_parse(repo, b)
628            .await
629            .ok()
630            .map(|h| h.chars().take(8).collect()),
631        None => None,
632    };
633
634    prompt::ConductOutcome {
635        run_id,
636        unreadable: None,
637        run_status: Some(state.status.as_str().to_owned()),
638        open_findings,
639        rounds_used: state.reviews.len(),
640        rounds_max: state.config.graph.review_rounds,
641        rounds,
642        branch,
643        branch_head,
644        references: None,
645        empty_candidate: state.merge.as_ref().is_some_and(|m| m.empty),
646    }
647}
648
649/// Whether the task's latest run's winner branch already has an open pull
650/// request into the run's own base - a fact the repository can answer, so the
651/// operator is never asked it. `None` when there is no run or winner to check.
652async fn open_pr_fact(task: &Task, repo: &Path) -> Option<String> {
653    let id = task.runs.last()?;
654    let state = match RunState::load(id) {
655        Ok(s) => s,
656        Err(e) => return Some(format!("could not check open pull requests: {e:#}")),
657    };
658    let branch = state.winner()?.branch.clone();
659    let base = state.base_branch.clone();
660    match crate::land::find_open_pr(repo, &branch, &base).await {
661        Ok(crate::land::OpenPr::None) => None,
662        Ok(crate::land::OpenPr::One { url, .. }) => Some(format!(
663            "pull request {url} is already open for {branch} into {base}"
664        )),
665        Ok(crate::land::OpenPr::Many(urls)) => Some(format!(
666            "several pull requests are already open for {branch} into {base}: {}",
667            urls.join(" ")
668        )),
669        Err(e) => Some(format!("could not check open pull requests: {e:#}")),
670    }
671}
672
673/// Append what the repository says about the branches and commits a task
674/// names to every question the conductor is about to file, so the operator is
675/// never asked a fact git can answer ("is it already on main?"). Done here,
676/// in code, rather than trusted to the model: a question the model wrote
677/// without checking still reaches the operator with the answer attached.
678/// When it cannot be checked the question says so instead of guessing.
679async fn attach_facts(_cfg: &Config, repo: &Path, queue: &Queue, verdict: &mut Verdict) {
680    for d in verdict
681        .decisions
682        .iter_mut()
683        .filter(|d| d.question.is_some())
684    {
685        let Ok(task) = queue.get(&d.id) else { continue };
686        if let (Some(fact), Some(q)) = (
687            open_pr_fact(&task, &repo_for(&task, repo)).await,
688            d.question.as_mut(),
689        ) {
690            q.push_str(&format!("\n\nChecked against the repository: {fact}"));
691        }
692    }
693}
694
695/// A `failed`/`held` task together with how its last run ended.
696async fn finished_view(t: &Task, repo: &Path, max_attempts: usize) -> prompt::ConductFinished {
697    prompt::ConductFinished {
698        task: view(t, max_attempts),
699        outcome: outcome_for(t, &repo_for(t, repo)).await,
700    }
701}
702
703/// The repository containing a task's branch. A task filed without a
704/// repository uses the daemon's repository, exactly as its later attempt does.
705fn repo_for(task: &Task, fallback: &Path) -> PathBuf {
706    if task.repo.as_os_str().is_empty() || task.repo == Path::new(".") {
707        fallback.to_path_buf()
708    } else {
709        task.repo.clone()
710    }
711}
712
713/// Apply one decision to the queue and the question store.
714///
715/// Takes the task's own claim before touching it: a model call spans a whole
716/// agent turn, and the queue can have moved on by the time its answer comes
717/// back. A claim that cannot be taken means something else owns this task
718/// right now - most often a live daemon mid-competition on it - so the
719/// conductor's now-stale view of it is dropped rather than raced against; see
720/// `crate::queue::Queue::claim`'s own doc on why a claim is proof, not a
721/// guess.
722///
723/// A decision is matched against the task's *current* status, re-read under
724/// the claim, not against whichever section of the prompt it came from: a
725/// `blocked_by` only takes effect on a `queued` task, and `recovery` only on
726/// one `running` (stalled) or `failed`/`held`, so a decision that no longer
727/// matches what the task actually is - it moved on between the read that
728/// built the prompt and this write - changes nothing.
729fn apply_one(
730    queue: &Queue,
731    questions: &Questions,
732    d: &Decision,
733    snapshot: Option<&Task>,
734) -> Result<()> {
735    let _claim = queue
736        .claim(&d.id)
737        .with_context(|| format!("task {} is claimed elsewhere right now", d.id))?;
738    let mut task = queue.get(&d.id).context("no such task")?;
739
740    // Compare the actual input, not a generation echoed by the model. Even
741    // question creation must be discarded after an intervening operator edit.
742    if snapshot.is_some_and(|old| decision_input(old) != decision_input(&task)) {
743        return Ok(());
744    }
745    let blocking =
746        d.question.is_some() || !d.blocked_by.is_empty() || d.recovery == Some(Recovery::Hold);
747    let evidence = new_blocking_evidence(queue, questions, &task, d);
748    if blocking && task.resumption.is_some() && evidence.is_empty() {
749        crate::notices::raise(crate::notices::conductor_resumption(
750            &task.id,
751            task.short(),
752            d.reason.as_deref(),
753            &d.blocked_by,
754            &d.evidence,
755            d.question.is_some(),
756        ));
757        return Ok(());
758    }
759
760    if blocking && !evidence.is_empty() {
761        let mut cause = evidence.clone();
762        cause.sort();
763        cause.dedup();
764        task.notice_cause(
765            &format!("conductor-evidence:{}", cause.join(",")),
766            None,
767            None,
768        );
769    }
770    if blocking {
771        if let Some(r) = task.resumption.as_mut() {
772            r.blocking_evidence.extend(evidence);
773        }
774    }
775
776    // A conductor answer is never operator authorization.  In particular,
777    // do this before questions and blocking too: either would reclassify a
778    // manual hold and let a later deterministic resolver queue it.
779    if task.operator_held() {
780        return Ok(());
781    }
782
783    if crate::base_sync_recovery::held_task(queue, &task)
784        && (d.recovery == Some(Recovery::Requeue)
785            || d.question_decision
786                .as_ref()
787                .is_some_and(|q| q.operation == QuestionOperation::Resume)
788            || d.actions
789                .values()
790                .any(|a| matches!(a, ChoiceAction::Resume { .. })))
791    {
792        // Additional base rebases require their own explicit authorization.
793        return Ok(());
794    }
795
796    // A claimed task may have deferred the pre-scheduling triage sweep.
797    // Leave an open or answered-unapplied question to that sweep rather
798    // than move the task out from under it.
799    if task.status == TaskStatus::Held && crate::triage::pending_for(questions, &task) {
800        return Ok(());
801    }
802
803    // The operator's `resume` action is waiting for its turn: a question or a
804    // dependency would park the task before the daemon can run it.
805    let pinned = pinned_resume(&task);
806    if pinned && (d.question.is_some() || !d.blocked_by.is_empty()) {
807        return Ok(());
808    }
809
810    // A missing-only decision with unverifiable aliases is not evidence for
811    // a question or hold. Do not guess that two ambiguous/unknown references
812    // mean the same record, or treat failure to prove that as proof of absence.
813    if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
814        && !d.missing_dependencies.is_empty()
815        && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
816        && d.missing_dependencies
817            .iter()
818            .any(|id| !d.blocked_by.contains(id))
819        && d.missing_dependencies
820            .iter()
821            .chain(&d.blocked_by)
822            .any(|id| {
823                matches!(
824                    crate::queue::dependency(queue, questions, id),
825                    crate::queue::Dependency::Ambiguous | crate::queue::Dependency::Unknown
826                )
827            })
828    {
829        return Ok(());
830    }
831
832    // Only an explicit missing-only claim can justify replacing a question
833    // or hold. Existing prerequisites alone say nothing about independent
834    // consent, destructive-operation questions, or other reasons to stay held.
835    // Ambiguous or unreadable inventories cannot confirm absence either: keep
836    // the original reference waiting until it can be resolved authoritatively.
837    if matches!(task.status, TaskStatus::Queued | TaskStatus::Held)
838        && !d.missing_dependencies.is_empty()
839        && d.missing_dependencies.iter().all(|id| {
840            let claim = crate::queue::dependency(queue, questions, id);
841            claim != crate::queue::Dependency::Missing
842                && d.blocked_by.iter().any(|blocker| {
843                    blocker == id
844                        || (matches!(
845                            &claim,
846                            crate::queue::Dependency::Task(_)
847                                | crate::queue::Dependency::Question(_)
848                                | crate::queue::Dependency::Deleted(_)
849                        ) && crate::queue::dependency(queue, questions, blocker) == claim)
850                })
851        })
852        && (d.question.is_some() || d.recovery == Some(Recovery::Hold))
853        && crate::queue::missing_blockers(queue, questions, &d.blocked_by).is_empty()
854    {
855        let mut ids = d.blocked_by.clone();
856        crate::queue::normalize_blockers(queue, questions, &mut ids);
857        let reason = format!("Waiting on dependencies: {}", ids.join(", "));
858        task.block(ids, Some(reason));
859        return queue.put(&mut task);
860    }
861
862    if let Some(text) = &d.question {
863        if task.status == TaskStatus::Done {
864            return Ok(());
865        }
866        let mut held = task.clone();
867        held.status = TaskStatus::Held;
868        let targets: BTreeSet<_> = d
869            .blocked_by
870            .iter()
871            .filter_map(|id| {
872                if let Ok(dep) = queue.get(id) {
873                    Some(format!("task:{id}:{:?}", dep.chat_notice_key()))
874                } else if questions.get(id).is_ok() {
875                    Some(format!("question:{id}"))
876                } else {
877                    None
878                }
879            })
880            .collect();
881        let decision = d
882            .question_decision
883            .as_ref()
884            .and_then(|q| match q.identity(&task) {
885                Ok(identity) => Some(identity),
886                Err(error) => {
887                    // Invalid optional metadata must not discard the question
888                    // or leave a runnable task without its decision barrier.
889                    tracing::warn!(task = %task.id, %error, "ignoring invalid question decision target");
890                    None
891                }
892            });
893        let context_value =
894            serde_json::json!([&task.instruction, held.chat_notice_key(), targets, decision]);
895        let context = serde_json::to_string(&context_value)?;
896        // An answer alone does not justify asking the same decision again.
897        // New producer evidence, an instruction edit or a different dependency
898        // target can open another question; rewording cannot.
899        if questions.list().iter().any(|q| {
900            q.node == NODE
901                && q.run == task.id
902                && q.status == crate::ask::QuestionStatus::Answered
903                && q.decision_context.as_ref().is_some_and(|stored| {
904                    if stored == &context {
905                        return true;
906                    }
907                    // Missing metadata cannot manufacture a new target. This
908                    // also baselines older contexts whose last item was labels.
909                    decision.is_none()
910                        && serde_json::from_str::<serde_json::Value>(stored)
911                            .ok()
912                            .and_then(|v| {
913                                v.as_array().map(|items| {
914                                    items.len() == 4
915                                        && items[..3]
916                                            == context_value.as_array().expect("context")[..3]
917                                })
918                            })
919                            == Some(true)
920                })
921        }) {
922            return Ok(());
923        }
924        // `Question::run` is a task id for conductor questions, but ordinary
925        // graph questions use it as a run id. A coincidental equality must
926        // not block this task on an answer meant for another node.
927        let question_id = match questions
928            .list()
929            .into_iter()
930            .find(|q| q.status.open() && q.node == NODE && q.run == task.id)
931        {
932            Some(existing) => existing.id,
933            None => {
934                let mut q = Question::new(
935                    task.id.clone(),
936                    NODE.to_owned(),
937                    SEAT.to_owned(),
938                    text.clone(),
939                    d.reason.clone().unwrap_or_default(),
940                    d.choices.clone(),
941                );
942                q.actions = d.actions.clone();
943                crate::choice_effect::executable_conductor(&mut q);
944                q.decision_context = Some(context);
945                // The wait is handed to a deputy (`crate::deputy`), so a
946                // free-text reply reaches an agent that knows why this was
947                // asked. The conductor still never blocks: this only records
948                // what the deputy will be told, and `serve` starts it.
949                q.deputy = Some(Deputy::new(crate::deputy::brief(
950                    &task.id,
951                    d.reason.as_deref().unwrap_or_default(),
952                    &q.choices,
953                    &q.actions,
954                )));
955                if task.repo.is_dir() && task.repo != Path::new(".") {
956                    q.cwd = Some(task.repo.to_string_lossy().into_owned());
957                }
958                questions.put_executable(&mut q)?;
959                q.id
960            }
961        };
962        task.block(vec![question_id], d.reason.clone());
963        return queue.put(&mut task);
964    }
965
966    match task.status {
967        // See `Decision::recovery`'s doc: the only lever a runnable task has
968        // besides `blocked_by` is holding it outright, for a task whose
969        // answers already say it should not compete again.
970        TaskStatus::Queued if d.recovery == Some(Recovery::Hold) => {
971            if may_hold(&mut task, &hold_note(d)) {
972                task.hold_machine(d.reason.clone());
973                queue.put(&mut task)?;
974            }
975        }
976        TaskStatus::Queued if !d.blocked_by.is_empty() => {
977            let mut ids = d.blocked_by.clone();
978            crate::queue::normalize_blockers(queue, questions, &mut ids);
979            task.block(ids, d.reason.clone());
980            queue.put(&mut task)?;
981        }
982        TaskStatus::Running => match d.recovery {
983            Some(Recovery::Requeue) if pinned_resume(&task) => {}
984            Some(Recovery::Requeue) => {
985                task.requeue();
986                queue.put(&mut task)?;
987            }
988            Some(Recovery::Hold) if may_hold(&mut task, &hold_note(d)) => {
989                task.hold_machine(d.reason.clone());
990                queue.put(&mut task)?;
991            }
992            // `Review` reopens a branch, which only makes sense once a run
993            // has actually stopped; a task still `running` has nothing to
994            // reopen yet.
995            _ => {}
996        },
997        TaskStatus::Failed | TaskStatus::Held => match d.recovery {
998            // The operator picked a specific run to continue; a fresh
999            // competition would throw that run away.
1000            Some(Recovery::Requeue) if pinned_resume(&task) => {}
1001            Some(Recovery::Requeue) => {
1002                task.requeue();
1003                queue.put(&mut task)?;
1004            }
1005            Some(Recovery::Hold) => {
1006                // Decided before `may_hold`, which spends the one override: a
1007                // note that adds nothing writes nothing (and so raises no
1008                // notice and moves no queue revision).
1009                if let Some(reason) = reaffirmed_hold_reason(&task, d) {
1010                    if may_hold(&mut task, &hold_note(d)) {
1011                        task.hold_machine(Some(reason));
1012                        queue.put(&mut task)?;
1013                    }
1014                }
1015            }
1016            Some(Recovery::Review) => {
1017                if let Some(branch) = surviving_branch(&task) {
1018                    task.request_review(branch);
1019                    queue.put(&mut task)?;
1020                }
1021                // No survivable branch: a decision naming `review` here is
1022                // simply not actionable, and is dropped rather than guessed
1023                // at - `crate::daemon` applies the same "no branch, no
1024                // review" rule again, from its own read, right before it
1025                // would actually start the run.
1026            }
1027            Some(Recovery::Done) => {
1028                task.succeed();
1029                // Same reason `magi task done` and the web UI's equivalent
1030                // do this: the conductor closing a task by hand is just as
1031                // much a finished story as the loop's own settle path.
1032                crate::daemon::supersede_prior_runs(&task, &crate::run::home());
1033                queue.put(&mut task)?;
1034            }
1035            None => {}
1036        },
1037        // `queued` with nothing to block on, `done`, or already `blocked`:
1038        // nothing for this decision to do.
1039        _ => {}
1040    }
1041    Ok(())
1042}
1043
1044/// Apply every decision in `verdict`. A single bad decision - a task id that
1045/// no longer exists, one already claimed elsewhere - is logged and skipped
1046/// rather than losing every other decision in the same reply.
1047pub fn apply(queue: &Queue, questions: &Questions, verdict: &Verdict) -> Result<()> {
1048    for d in &verdict.decisions {
1049        if let Err(e) = apply_one(queue, questions, d, None) {
1050            tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1051        }
1052    }
1053    Ok(())
1054}
1055
1056/// Where the conductor's seat is kept between turns, so the daemon waiter can
1057/// resume it when the owner answers a question it asked.
1058///
1059/// A graph seat's session lives in its run's `RunState`; the conductor has no
1060/// run, and until this file its seat existed only in memory.
1061pub fn seat_path(home: &Path) -> PathBuf {
1062    home.join("conduct").join("seat.json")
1063}
1064
1065/// The seat the conductor last used, if it has taken a turn.
1066pub fn load_seat(home: &Path) -> Option<SeatState> {
1067    serde_json::from_str(&std::fs::read_to_string(seat_path(home)).ok()?).ok()
1068}
1069
1070fn busy_path(home: &Path) -> PathBuf {
1071    home.join("conduct").join("busy")
1072}
1073
1074/// Is a conductor turn in flight right now?
1075///
1076/// A marker file rather than shared memory because the waiter reads it and the
1077/// conductor is not necessarily in its process. A marker older than one turn's
1078/// timeout (plus slack) was left by a turn that died and counts for nothing.
1079pub fn busy(home: &Path) -> bool {
1080    std::fs::metadata(busy_path(home))
1081        .and_then(|m| m.modified())
1082        .ok()
1083        .and_then(|t| t.elapsed().ok())
1084        .is_some_and(|age| age < TURN_TIMEOUT + Duration::from_secs(30))
1085}
1086
1087/// Removes the busy marker when a turn ends, however it ends.
1088struct Busy(PathBuf);
1089
1090impl Busy {
1091    fn mark(home: &Path) -> Self {
1092        let path = busy_path(home);
1093        if let Some(dir) = path.parent() {
1094            let _ = std::fs::create_dir_all(dir);
1095        }
1096        let _ = std::fs::write(&path, std::process::id().to_string());
1097        Self(path)
1098    }
1099}
1100
1101impl Drop for Busy {
1102    fn drop(&mut self) {
1103        let _ = std::fs::remove_file(&self.0);
1104    }
1105}
1106
1107/// The conductor's state across polling cycles: its own CLI-side conversation
1108/// and the last (revision, stalled ∪ finished ids) pair it actually acted on.
1109#[derive(Debug, Default, Serialize, Deserialize)]
1110pub struct Conductor {
1111    seat: Option<SeatState>,
1112    last_seen: Option<(u64, BTreeSet<String>)>,
1113    /// Content fingerprint of each held task the model was last shown, see
1114    /// [`input_fingerprint`]. A held task whose fingerprint still matches is
1115    /// settled: the conductor's own rewrites of it do not count as news.
1116    considered: BTreeMap<String, String>,
1117}
1118
1119/// What the conductor's decision about a held task is a function of: the task
1120/// without the fields the conductor itself rewrites (`updated_at`,
1121/// `hold_reason`, the recorded re-hold). Independent of how the model words a
1122/// repeated decision.
1123fn decision_input(task: &Task) -> serde_json::Value {
1124    let mut v = serde_json::to_value(task).unwrap_or_default();
1125    if let Some(o) = v.as_object_mut() {
1126        o.remove("chat_notice");
1127        o.remove("chat_notices");
1128        // A delivery write changes only bookkeeping, not the decision input.
1129        o.remove("updated_at");
1130    }
1131    v
1132}
1133
1134fn input_fingerprint(task: &Task) -> String {
1135    let mut v = decision_input(task);
1136    if let Some(o) = v.as_object_mut() {
1137        o.remove("hold_reason");
1138        if let Some(ro) = o.get_mut("resume_override").and_then(|r| r.as_object_mut()) {
1139            ro.remove("conductor_rehold");
1140        }
1141    }
1142    v.to_string()
1143}
1144
1145impl Conductor {
1146    /// A conductor that has never run.
1147    #[must_use]
1148    pub fn new() -> Self {
1149        Self::default()
1150    }
1151
1152    /// Held tasks among `stalled` / `finished` whose input is unchanged since
1153    /// the model last saw them. Their files are left out of the revision, so
1154    /// the conductor re-holding one (however it words it) is not news, while
1155    /// an owner's answer changes the fingerprint and brings the file back in.
1156    fn settled(
1157        considered: &BTreeMap<String, String>,
1158        stalled: &[Task],
1159        finished: &[Task],
1160    ) -> BTreeSet<String> {
1161        stalled
1162            .iter()
1163            .chain(finished)
1164            .filter(|t| t.status == TaskStatus::Held)
1165            .filter(|t| considered.get(&t.id) == Some(&input_fingerprint(t)))
1166            .map(|t| t.id.clone())
1167            .collect()
1168    }
1169
1170    fn snapshot_with(
1171        considered: &BTreeMap<String, String>,
1172        queue: &Queue,
1173        stalled: &[Task],
1174        finished: &[Task],
1175    ) -> (u64, BTreeSet<String>) {
1176        let ids = stalled
1177            .iter()
1178            .chain(finished)
1179            .map(|t| t.id.clone())
1180            .collect();
1181        let skip = Self::settled(considered, stalled, finished);
1182        (queue.revision_excluding(&skip), ids)
1183    }
1184
1185    fn snapshot(
1186        &self,
1187        queue: &Queue,
1188        stalled: &[Task],
1189        finished: &[Task],
1190    ) -> (u64, BTreeSet<String>) {
1191        Self::snapshot_with(&self.considered, queue, stalled, finished)
1192    }
1193
1194    /// Whether calling the conductor could possibly do anything different
1195    /// from last time: [`Queue::revision`] moved, or the set of stalled and
1196    /// finished task ids changed.
1197    ///
1198    /// Deliberately **not** "stalled or finished is non-empty" - a task
1199    /// sitting stalled or finished with nobody changing anything about it
1200    /// must not be re-shown to the model every single poll forever; only a
1201    /// change in *which* tasks are stalled or finished, or a queue write
1202    /// changing something about a runnable one, is worth another look.
1203    ///
1204    /// Cheap and config-free on purpose, so `crate::daemon`'s poll loop can
1205    /// skip `Config::discover`'s synchronous I/O entirely on a cycle where
1206    /// this says no - which [`Conductor::maybe_run`] would otherwise only
1207    /// discover after paying for that load. Both ask the identical question,
1208    /// from the same [`Conductor::last_seen`], so they can never disagree
1209    /// about whether there is anything to look at.
1210    #[must_use]
1211    pub fn worth_a_look(&self, queue: &Queue, stalled: &[Task], finished: &[Task]) -> bool {
1212        self.last_seen.as_ref() != Some(&self.snapshot(queue, stalled, finished))
1213    }
1214
1215    /// Call the conductor once, unless nothing has changed since the last
1216    /// time it was worth calling - see [`Conductor::worth_a_look`], the exact
1217    /// same test. Never fatal - see this module's doc.
1218    #[allow(clippy::too_many_arguments)]
1219    pub async fn maybe_run(
1220        &mut self,
1221        cfg: &Config,
1222        repo: &Path,
1223        queue: &Queue,
1224        questions: &Questions,
1225        home: &Path,
1226        queued: &[Task],
1227        stalled: &[Task],
1228        finished: &[Task],
1229        max_attempts: usize,
1230    ) {
1231        if self.last_seen.as_ref() == Some(&self.snapshot(queue, stalled, finished)) {
1232            return;
1233        }
1234        // What the model is about to be shown becomes "considered" before the
1235        // snapshot is taken, so its own re-hold writes land in the excluded set.
1236        self.considered = stalled
1237            .iter()
1238            .chain(finished)
1239            .filter(|t| t.status == TaskStatus::Held)
1240            .map(|t| (t.id.clone(), input_fingerprint(t)))
1241            .collect();
1242        self.last_seen = Some(self.snapshot(queue, stalled, finished));
1243        if let Err(e) = self
1244            .run_once(
1245                cfg,
1246                repo,
1247                queue,
1248                questions,
1249                home,
1250                queued,
1251                stalled,
1252                finished,
1253                max_attempts,
1254            )
1255            .await
1256        {
1257            // The conductor's own write moves the revision, so one more cycle
1258            // follows; a repeat decision writes nothing (see
1259            // `reaffirmed_hold_reason`), which is what ends the loop. The
1260            // revision is deliberately never re-read here: it cannot tell our
1261            // write from an owner's concurrent update.
1262            tracing::warn!("conductor: {e:#}");
1263        }
1264    }
1265
1266    #[allow(clippy::too_many_arguments)]
1267    async fn run_once(
1268        &mut self,
1269        cfg: &Config,
1270        repo: &Path,
1271        queue: &Queue,
1272        questions: &Questions,
1273        home: &Path,
1274        queued: &[Task],
1275        stalled: &[Task],
1276        finished: &[Task],
1277        max_attempts: usize,
1278    ) -> Result<()> {
1279        if queued.is_empty() && stalled.is_empty() && finished.is_empty() {
1280            return Ok(());
1281        }
1282
1283        let primary = cfg
1284            .resolve_roles()
1285            .context("resolving the conductor seat")?
1286            .conductor;
1287        // A string keeps today's resolution exactly (no CLI preflight here;
1288        // an unavailable CLI fails at invocation). Only a multi-id array
1289        // builds a chain, whose members are each tried at most once.
1290        let chain = match cfg.roles.conductor.as_ref() {
1291            Some(c) if c.ids().len() > 1 => {
1292                agent::pick_chain(&cfg.agents, Some(c), &agent::installed, "conductor")?
1293            }
1294            _ => vec![primary],
1295        };
1296
1297        let runnable_views: Vec<prompt::ConductTask> =
1298            queued.iter().map(|t| view(t, max_attempts)).collect();
1299        let stalled_views: Vec<prompt::ConductTask> =
1300            stalled.iter().map(|t| view(t, max_attempts)).collect();
1301        let mut finished_views = Vec::with_capacity(finished.len());
1302        for t in finished {
1303            finished_views.push(finished_view(t, repo, max_attempts).await);
1304        }
1305
1306        let body = prompt::with_overlay(
1307            prompt::conduct(
1308                &runnable_views,
1309                &stalled_views,
1310                &finished_views,
1311                &cfg.graph.language,
1312            ),
1313            cfg.prompts.overlay(NODE),
1314        );
1315
1316        let body = format!(
1317            "{body}\n\n{}",
1318            crate::queue::dependency_facts(queue, questions)
1319        );
1320        let artifacts = home.join("conduct").join("artifacts");
1321        // Bound to a local: `Invocation` only borrows the cache path, and the
1322        // `Option<PathBuf>` `cache_dir()` returns has to outlive that borrow.
1323        let cache_dir = cfg.cache_dir();
1324
1325        // One marker for the whole chain: a fallback is still this turn.
1326        let busy = Busy::mark(home);
1327        let mut result: Result<Verdict> = Err(anyhow::anyhow!("no conductor agent ran"));
1328        for spec in &chain {
1329            // The prompt carries the whole queue state every cycle, so a
1330            // fallback agent needs no history - but its seat is its own.
1331            let needs_new_seat = !matches!(&self.seat, Some(s) if s.agent == spec.id);
1332            if needs_new_seat {
1333                self.seat = Some(SeatState::new(SEAT, &spec.id, crate::rng::entropy()));
1334            }
1335            let seat = self.seat.as_mut().expect("just ensured a seat exists");
1336            // The first agent keeps the plain stem; a fallback's own artifacts
1337            // must not overwrite it.
1338            let stem = if std::ptr::eq(spec, &chain[0]) {
1339                format!("turn-{}", seat.turns + 1)
1340            } else {
1341                format!("turn-{}-{}", seat.turns + 1, spec.id)
1342            };
1343            let inv = Invocation {
1344                coordination: cfg.coordination.as_ref(),
1345                fetched_base: None,
1346                cwd: repo,
1347                prompt: &body,
1348                timeout: TURN_TIMEOUT,
1349                // The conductor never edits anything - it only decides what
1350                // blocks a task and what to do about one stuck or finished.
1351                allow_write: false,
1352                sessions: cfg.graph.sessions,
1353                artifacts: &artifacts,
1354                stem: &stem,
1355                run: NODE,
1356                node: NODE,
1357                cache_dir: cache_dir.as_deref(),
1358                attachments: &[],
1359                writable: &[],
1360            };
1361            let out = agent::invoke(spec, seat, &inv).await;
1362            // Kept after every turn, failed or not: the CLI-side conversation is
1363            // what a resumed question needs, and it exists once the turn ran.
1364            if let Ok(body) = serde_json::to_string(&*seat) {
1365                let path = seat_path(home);
1366                if let Some(dir) = path.parent() {
1367                    let _ = std::fs::create_dir_all(dir);
1368                }
1369                let _ = std::fs::write(path, body);
1370            }
1371            let advance = agent::chain_advances(&out);
1372            result = match out {
1373                Err(e) => Err(e.context("invoking the conductor")),
1374                Ok(out) if advance => Err(anyhow::anyhow!(
1375                    "no usable reply ({})",
1376                    crate::notices::turn_status(
1377                        out.exit_code,
1378                        out.timed_out,
1379                        out.quota_exhausted()
1380                    )
1381                )),
1382                Ok(out) => verdict::extract_json(&out.text)
1383                    .context("the conductor's reply could not be parsed"),
1384            };
1385            if result.is_ok() {
1386                break;
1387            }
1388            if chain.len() > 1 {
1389                tracing::warn!("conductor: `{}` failed, trying the next agent", spec.id);
1390            }
1391        }
1392        drop(busy);
1393        let mut verdict = result?;
1394        attach_facts(cfg, repo, queue, &mut verdict).await;
1395        for d in &verdict.decisions {
1396            let snapshot = queued
1397                .iter()
1398                .chain(stalled)
1399                .chain(finished)
1400                .find(|t| t.id == d.id);
1401            if let Some(snapshot) = snapshot {
1402                if let Err(e) = apply_one(queue, questions, d, Some(snapshot)) {
1403                    tracing::warn!("conductor decision for task {}: {e:#}", d.id);
1404                }
1405            }
1406        }
1407        Ok(())
1408    }
1409}
1410
1411#[cfg(test)]
1412mod tests {
1413    use std::collections::BTreeMap;
1414
1415    use tempfile::tempdir;
1416
1417    use super::*;
1418    use crate::ask::{Answer, QuestionStatus};
1419    use crate::config::{AgentKind, AgentSpec, Graph};
1420    use crate::queue::Source;
1421
1422    fn mock_agent(dir: &Path, script: &str, env: BTreeMap<String, String>) -> AgentSpec {
1423        let path = dir.join("mock-conduct-agent.sh");
1424        std::fs::write(&path, script).expect("write mock");
1425        AgentSpec {
1426            id: "mock".to_owned(),
1427            kind: AgentKind::Command,
1428            model: None,
1429            command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
1430            extra_args: Vec::new(),
1431            env,
1432            prompt_delivery: None,
1433            max_turn_secs: None,
1434        }
1435    }
1436
1437    fn config(spec: AgentSpec) -> Config {
1438        Config {
1439            agents: vec![spec],
1440            graph: Graph {
1441                language: "en".to_owned(),
1442                ..Graph::default()
1443            },
1444            ..Config::default()
1445        }
1446    }
1447
1448    fn task(title: &str) -> Task {
1449        Task::new(
1450            title.to_owned(),
1451            format!("do {title}"),
1452            std::path::PathBuf::from("."),
1453            Source::Human,
1454        )
1455    }
1456
1457    #[test]
1458    fn delivery_bookkeeping_does_not_restart_conductor_deliberation() {
1459        let mut t = task("waiting");
1460        t.hold_machine(Some("waiting for recovery".into()));
1461        let before = input_fingerprint(&t);
1462        let snapshot = decision_input(&t);
1463        t.chat_notice = Some(crate::queue::ChatNotice {
1464            key: t.chat_notice_key().unwrap(),
1465            at: jiff::Timestamp::now(),
1466            outcome: crate::queue::ChatNoticeOutcome::Posted,
1467        });
1468        t.chat_notices.push(t.chat_notice_key().unwrap());
1469        t.updated_at = jiff::Timestamp::now();
1470        assert_eq!(before, input_fingerprint(&t));
1471        assert_eq!(snapshot, decision_input(&t));
1472        t.hold_reason = Some("recovery remains under consideration".into());
1473        assert_eq!(before, input_fingerprint(&t));
1474        t.record_answer("a different question".into(), "a new decision".into());
1475        assert_ne!(before, input_fingerprint(&t));
1476    }
1477
1478    const BROKEN: &str = "#!/bin/sh\ncat >/dev/null\nexit 3\n";
1479    const GARBAGE: &str = "#!/bin/sh\ncat >/dev/null\nprintf 'not json at all\\n'\n";
1480
1481    fn env(reply: &str) -> BTreeMap<String, String> {
1482        BTreeMap::from([("MOCK_REPLY".to_owned(), reply.to_owned())])
1483    }
1484
1485    const REPLY: &str = "#!/bin/sh\ncat >/dev/null\nprintf '%s\\n' \"$MOCK_REPLY\"\n";
1486
1487    /// A throwaway repo with one commit on `main` and a second branch ahead
1488    /// of it, so `outcome_for`'s own `git::rev_parse` call has a real head to
1489    /// resolve.
1490    fn init_repo_with_branch(dir: &Path, branch: &str) {
1491        use crate::proc::Quiet as _;
1492        let run = |args: &[&str]| {
1493            let out = std::process::Command::new("git")
1494                .args(args)
1495                .current_dir(dir)
1496                .quiet()
1497                .output()
1498                .expect("spawn git");
1499            assert!(
1500                out.status.success(),
1501                "git {args:?} failed: {}",
1502                String::from_utf8_lossy(&out.stderr)
1503            );
1504        };
1505        run(&["init", "-b", "main"]);
1506        run(&["config", "user.name", "magi test"]);
1507        run(&["config", "user.email", "magi@example.com"]);
1508        std::fs::write(dir.join("README.md"), "# fixture\n").unwrap();
1509        run(&["add", "-A"]);
1510        run(&["commit", "-m", "init"]);
1511        run(&["checkout", "-b", branch]);
1512        std::fs::write(dir.join("change.txt"), "x\n").unwrap();
1513        run(&["add", "-A"]);
1514        run(&["commit", "-m", "candidate work"]);
1515    }
1516
1517    fn review_round_with_finding(
1518        round: usize,
1519        finding_id: &str,
1520        title: &str,
1521        addressed: &[&str],
1522        rejected: &[(&str, &str)],
1523    ) -> crate::run::ReviewRound {
1524        crate::run::ReviewRound {
1525            round,
1526            head: "deadbeef".to_owned(),
1527            verified_head: None,
1528            verified_at: None,
1529            reviews: vec![crate::run::ReviewRecord {
1530                attempts: 0,
1531                reviewer: 1,
1532                agent: "mock".to_owned(),
1533                summary: String::new(),
1534                findings: vec![crate::verdict::Finding {
1535                    id: finding_id.to_owned(),
1536                    severity: crate::verdict::Severity::Major,
1537                    file: None,
1538                    line: None,
1539                    title: title.to_owned(),
1540                    detail: String::new(),
1541                }],
1542                vote: None,
1543                failed: None,
1544                duration_ms: 0,
1545            }],
1546            e2e: Vec::new(),
1547            verify_retried: false,
1548            e2e_deferred: false,
1549            e2e_defer_reason: None,
1550            fix: Some(crate::run::FixRecord {
1551                agent: "mock".to_owned(),
1552                addressed: addressed.iter().map(|s| (*s).to_owned()).collect(),
1553                rejected: rejected
1554                    .iter()
1555                    .map(|(id, why)| crate::verdict::Rejection {
1556                        id: (*id).to_owned(),
1557                        why: (*why).to_owned(),
1558                    })
1559                    .collect(),
1560                notes: String::new(),
1561                committed: false,
1562                failed: None,
1563                duration_ms: 0,
1564                continuation: None,
1565            }),
1566            blocking: 1,
1567            answered: 1,
1568            expected: 1,
1569            clean: false,
1570            progressed: true,
1571            vote_split: false,
1572            reconsideration: Vec::new(),
1573            verdict: None,
1574        }
1575    }
1576
1577    #[test]
1578    fn explicit_resumption_supersedes_history_and_rejects_stale_decisions() {
1579        crate::run::pin_test_home();
1580        let dir = tempdir().unwrap();
1581        let queue = Queue::at(dir.path().join("queue"));
1582        let questions = Questions::at(dir.path().join("questions"));
1583        let mut t = task("recover");
1584        t.answers.push(crate::queue::AnsweredQuestion {
1585            question: "Inspect remote first?".into(),
1586            answer: "Inspect, then await resumption instructions".into(),
1587        });
1588        t.hold_machine(Some("waiting for remote inspection".into()));
1589        queue.put(&mut t).unwrap();
1590        let snapshot = queue.get(&t.id).unwrap();
1591        t.edit(
1592            "recover".into(),
1593            "Reuse the measured PR head; fix Clippy; review".into(),
1594        )
1595        .unwrap();
1596        t.release_operator("cli", None);
1597        queue.put(&mut t).unwrap();
1598        let approval = t.resumption.clone();
1599        let d = Decision {
1600            id: t.id.clone(),
1601            recovery: Some(Recovery::Hold),
1602            reason: Some("remote inspection still required".into()),
1603            ..Decision::default()
1604        };
1605        apply_one(&queue, &questions, &d, Some(&snapshot)).unwrap();
1606        let stale_question = Decision {
1607            question: Some("Inspect remote?".into()),
1608            ..d.clone()
1609        };
1610        apply_one(&queue, &questions, &stale_question, Some(&snapshot)).unwrap();
1611        assert!(questions.list().is_empty());
1612        for reason in [
1613            "waiting for remote inspection",
1614            "inspection has not been confirmed",
1615        ] {
1616            let repeated = Decision {
1617                reason: Some(reason.into()),
1618                question: Some("How should we proceed?".into()),
1619                ..d.clone()
1620            };
1621            apply(
1622                &queue,
1623                &questions,
1624                &Verdict {
1625                    decisions: vec![repeated],
1626                },
1627            )
1628            .unwrap();
1629            assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1630        }
1631        assert!(
1632            questions.list().is_empty(),
1633            "a rejected proposal registers no question"
1634        );
1635        let notice = crate::notices::Notices::open()
1636            .get(&crate::notices::id_of(&format!(
1637                "conductor-resumption:{}",
1638                t.id
1639            )))
1640            .unwrap();
1641        assert!(!notice.message.contains("How should we proceed?"));
1642        assert!(!notice.message.contains("Some("));
1643        assert!(notice.message.contains("No reply to this notice is needed"));
1644        let reloaded = Queue::at(dir.path().join("queue")).get(&t.id).unwrap();
1645        assert_eq!(reloaded.resumption, approval);
1646        assert_eq!(reloaded.answers, t.answers);
1647        let v = view(&reloaded, 3);
1648        assert!(v.operator_resume.unwrap().contains("superseded"));
1649        let mut dep = task("new prerequisite");
1650        dep.hold_machine(Some("new authentication failure".into()));
1651        queue.put(&mut dep).unwrap();
1652        let new_blocker = Decision {
1653            blocked_by: vec![dep.id.clone()],
1654            evidence: vec![dep.id.clone()],
1655            // Exercise dependency waiting, without inheriting a separate hold.
1656            recovery: None,
1657            reason: Some("new prerequisite failed".into()),
1658            ..d.clone()
1659        };
1660        apply(
1661            &queue,
1662            &questions,
1663            &Verdict {
1664                decisions: vec![new_blocker.clone()],
1665            },
1666        )
1667        .unwrap();
1668        let mut after = queue.get(&t.id).unwrap();
1669        assert_eq!(after.status, TaskStatus::Blocked);
1670        // Automatic release does not grant fresh operator authority or make
1671        // the same unchanged evidence reusable.
1672        after.release();
1673        queue.put(&mut after).unwrap();
1674        apply(
1675            &queue,
1676            &questions,
1677            &Verdict {
1678                decisions: vec![new_blocker],
1679            },
1680        )
1681        .unwrap();
1682        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
1683        after.start("new-run".into());
1684        assert_eq!(after.resumption.as_ref().unwrap().generation, 1);
1685        after.hold_manual(Some("operator says wait".into()));
1686        queue.put(&mut after).unwrap();
1687        apply(&queue, &questions, &Verdict { decisions: vec![d] }).unwrap();
1688        assert!(queue.get(&t.id).unwrap().operator_held());
1689    }
1690
1691    #[test]
1692    fn newly_blocked_dependency_is_material_evidence_after_approval() {
1693        let dir = tempdir().unwrap();
1694        let queue = Queue::at(dir.path().join("queue"));
1695        let questions = Questions::at(dir.path().join("questions"));
1696        let mut dep = task("old queued dependency");
1697        queue.put(&mut dep).unwrap();
1698        let mut t = task("approved task");
1699        t.release_operator("test", None);
1700        queue.put(&mut t).unwrap();
1701        let mut q = Question::new(
1702            dep.id.clone(),
1703            "check".into(),
1704            "check".into(),
1705            "New dependency approval".into(),
1706            String::new(),
1707            Vec::new(),
1708        );
1709        questions.put(&mut q).unwrap();
1710        dep.block(vec![q.id.clone()], Some("new decision needed".into()));
1711        let changed = dep.blocked_at;
1712        queue.put(&mut dep).unwrap();
1713        dep.block(vec![q.id], Some("same decision paraphrased".into()));
1714        assert_eq!(
1715            dep.blocked_at, changed,
1716            "prose is not new blocking evidence"
1717        );
1718        queue.put(&mut dep).unwrap();
1719        let d = Decision {
1720            id: t.id.clone(),
1721            blocked_by: vec![dep.id.clone()],
1722            evidence: vec![dep.id],
1723            ..Decision::default()
1724        };
1725        assert_eq!(new_blocking_evidence(&queue, &questions, &t, &d).len(), 1);
1726        apply_one(&queue, &questions, &d, None).unwrap();
1727        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
1728    }
1729
1730    #[test]
1731    fn resumed_run_evidence_uses_approval_time_not_run_list_offsets() {
1732        crate::run::pin_test_home();
1733        let dir = tempdir().unwrap();
1734        let queue = Queue::at(dir.path().join("queue"));
1735        let questions = Questions::at(dir.path().join("questions"));
1736        let mut t = task("resume evidence");
1737        let mut run = RunState::new(
1738            t.repo.clone(),
1739            "main".into(),
1740            "base".into(),
1741            "work".into(),
1742            crate::config::Config::default(),
1743        );
1744        run.begin_pass();
1745        run.status = crate::run::RunStatus::Failed;
1746        run.save().unwrap();
1747        t.start(run.id.clone());
1748        // Model legacy duplicate history, which the next start will deduplicate.
1749        t.runs.push(run.id.clone());
1750        t.release_operator("cli", None);
1751        let approval = t.resumption.as_ref().unwrap().at;
1752        let decision = Decision {
1753            evidence: vec![run.id.clone()],
1754            ..Decision::default()
1755        };
1756        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1757        t.start(run.id.clone());
1758        assert_eq!(t.runs.len(), 1);
1759        run.begin_pass();
1760        run.save().unwrap();
1761        assert!(run.passes.last().unwrap().started_at > approval);
1762        let evidence = new_blocking_evidence(&queue, &questions, &t, &decision);
1763        assert_eq!(evidence.len(), 1);
1764        t.resumption.as_mut().unwrap().blocking_evidence = evidence;
1765        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1766        run.begin_pass();
1767        run.save().unwrap();
1768        assert_eq!(
1769            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1770            1
1771        );
1772
1773        let mut q = Question::new(
1774            run.id.clone(),
1775            "review".into(),
1776            "check".into(),
1777            "New failure".into(),
1778            String::new(),
1779            Vec::new(),
1780        );
1781        questions.put(&mut q).unwrap();
1782        let decision = Decision {
1783            evidence: vec![q.id.clone()],
1784            ..Decision::default()
1785        };
1786        assert_eq!(
1787            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1788            1
1789        );
1790        q.asked_at = approval;
1791        questions.put(&mut q).unwrap();
1792        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1793        q.asked_at = jiff::Timestamp::now();
1794        q.run = "unrelated".into();
1795        questions.put(&mut q).unwrap();
1796        assert!(new_blocking_evidence(&queue, &questions, &t, &decision).is_empty());
1797
1798        let mut fresh = RunState::new(
1799            t.repo.clone(),
1800            "main".into(),
1801            "base".into(),
1802            "fresh work".into(),
1803            crate::config::Config::default(),
1804        );
1805        fresh.status = crate::run::RunStatus::Failed;
1806        fresh.save().unwrap();
1807        t.start(fresh.id.clone());
1808        let decision = Decision {
1809            evidence: vec![fresh.id.clone()],
1810            ..Decision::default()
1811        };
1812        assert_eq!(
1813            new_blocking_evidence(&queue, &questions, &t, &decision).len(),
1814            1
1815        );
1816    }
1817
1818    #[test]
1819    fn outcome_for_carries_every_rounds_findings_and_the_branch_head() {
1820        crate::run::pin_test_home();
1821        let dir = tempdir().unwrap();
1822        let default_repo = dir.path().join("default");
1823        let task_repo = dir.path().join("task");
1824        std::fs::create_dir_all(&default_repo).unwrap();
1825        std::fs::create_dir_all(&task_repo).unwrap();
1826        init_repo_with_branch(&default_repo, "other-branch");
1827        init_repo_with_branch(&task_repo, "magi/f00d/A");
1828
1829        let mut config = Config::default();
1830        config.graph.review_rounds = 6;
1831        let mut state = crate::run::RunState::new(
1832            task_repo.clone(),
1833            "main".to_owned(),
1834            "deadbeef".to_owned(),
1835            "task".to_owned(),
1836            config,
1837        );
1838        state.status = crate::run::RunStatus::Blocked;
1839        state.candidates.push(crate::run::Candidate {
1840            index: 0,
1841            label: 'A',
1842            agent: "mock".to_owned(),
1843            branch: "magi/f00d/A".to_owned(),
1844            worktree: task_repo.clone(),
1845            summary: String::new(),
1846            stat: String::new(),
1847            files: 1,
1848            commits: 1,
1849            empty: false,
1850            failed: None,
1851            verified_noop: None,
1852            duration_ms: 0,
1853            folded: false,
1854        });
1855        state.tally = Some(crate::run::Tally {
1856            first_choice: std::collections::BTreeMap::new(),
1857            borda: std::collections::BTreeMap::new(),
1858            winner: 'A',
1859            rankings: 0,
1860            unanimous_initial: false,
1861            deliberated: false,
1862            changed_votes: 0,
1863            unanimous_final: false,
1864            tie_break: None,
1865            judges: 0,
1866            present: 0,
1867            quorum: 0,
1868            met_quorum: true,
1869            uncontested: Some("solo".to_owned()),
1870        });
1871        state.reviews = vec![
1872            review_round_with_finding(
1873                1,
1874                "R1-1-2",
1875                "answer content is dropped",
1876                &[],
1877                &[("R1-1-2", "the id leaving blocked_by is enough")],
1878            ),
1879            review_round_with_finding(2, "R2-1-3", "answer content is still dropped", &[], &[]),
1880        ];
1881        state.save().unwrap();
1882
1883        let mut t = task("outcome test");
1884        t.repo = task_repo;
1885        t.runs.push(state.id.clone());
1886
1887        let finished = tokio_test_block_on(finished_view(&t, &default_repo, 2));
1888        let outcome = finished.outcome;
1889
1890        assert!(outcome.unreadable.is_none());
1891        assert_eq!(outcome.run_status.as_deref(), Some("blocked"));
1892        assert_eq!(outcome.rounds_used, 2);
1893        assert_eq!(outcome.rounds_max, 6);
1894        assert_eq!(outcome.rounds.len(), 2);
1895        assert_eq!(outcome.rounds[0].findings[0].id, "R1-1-2");
1896        assert_eq!(outcome.rounds[0].rejected[0].id, "R1-1-2");
1897        assert!(outcome.rounds[1].addressed.is_empty());
1898        assert!(outcome.rounds[1].rejected.is_empty());
1899        assert_eq!(outcome.branch.as_deref(), Some("magi/f00d/A"));
1900        assert!(
1901            outcome.branch_head.is_some(),
1902            "a real branch must resolve a head commit: {outcome:?}"
1903        );
1904    }
1905
1906    /// A tiny single-threaded block-on, so an `async fn` can be exercised
1907    /// from a plain `#[test]` without pulling `tokio::test`'s multi-thread
1908    /// runtime into a test that does no other async work.
1909    fn tokio_test_block_on<F: std::future::Future>(f: F) -> F::Output {
1910        tokio::runtime::Builder::new_current_thread()
1911            .enable_all()
1912            .build()
1913            .unwrap()
1914            .block_on(f)
1915    }
1916
1917    #[test]
1918    fn view_resolves_task_attempt_limit_before_daemon_default() {
1919        let mut t = task("budget");
1920        assert_eq!(view(&t, 7).max_attempts, 7);
1921        t.change_attempt_limit(std::num::NonZeroUsize::new(3))
1922            .unwrap();
1923        assert_eq!(view(&t, 7).max_attempts, 3);
1924        t.change_attempt_limit(None).unwrap();
1925        assert_eq!(view(&t, 7).max_attempts, 7);
1926    }
1927
1928    #[test]
1929    fn view_carries_a_tasks_recorded_answers_into_the_conductor_prompt_input() {
1930        let mut t = task("answered");
1931        t.record_answer("Which backend?".to_owned(), "SQLite".to_owned());
1932        let v = view(&t, 2);
1933        assert_eq!(v.answers.len(), 1);
1934        assert_eq!(v.answers[0].question, "Which backend?");
1935        assert_eq!(v.answers[0].answer, "SQLite");
1936    }
1937
1938    #[tokio::test]
1939    async fn dependency_resolution_rechecks_questions_in_the_production_conductor_path() {
1940        for held in [false, true] {
1941            for reference in ["ca5b", "20261013-193611-ca5b"] {
1942                for question in [false, true] {
1943                    let dir = tempdir().unwrap();
1944                    let queue = Queue::at(dir.path().join("queue"));
1945                    let questions = Questions::at(dir.path().join("questions"));
1946                    let mut dep = task("prerequisite");
1947                    dep.id = "20261013-193611-ca5b".into();
1948                    dep.start("existing-run".into());
1949                    queue.put(&mut dep).unwrap();
1950                    let mut waiting = task("waiting");
1951                    if held {
1952                        waiting.hold_machine(Some("dependency could not be located".into()));
1953                    }
1954                    queue.put(&mut waiting).unwrap();
1955                    let missing_reference = if reference == "ca5b" {
1956                        dep.id.as_str()
1957                    } else {
1958                        "ca5b"
1959                    };
1960                    let reply = serde_json::json!({"decisions": [{
1961                        "id": waiting.id, "blocked_by": [reference],
1962                        "missing_dependencies": [missing_reference],
1963                        "question": question.then_some("ca5b is missing; where is its replacement?"),
1964                        "recovery": (!question).then_some("hold"),
1965                        "reason": "ca5b is missing"
1966                    }]}).to_string();
1967                    let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
1968                    let (queued, finished) = if held {
1969                        (vec![], vec![waiting.clone()])
1970                    } else {
1971                        (vec![waiting.clone()], vec![])
1972                    };
1973                    Conductor::new()
1974                        .run_once(
1975                            &cfg,
1976                            dir.path(),
1977                            &queue,
1978                            &questions,
1979                            dir.path(),
1980                            &queued,
1981                            &[],
1982                            &finished,
1983                            2,
1984                        )
1985                        .await
1986                        .unwrap();
1987                    let after = queue.get(&waiting.id).unwrap();
1988                    assert_eq!(after.status, TaskStatus::Blocked);
1989                    assert_eq!(after.blocked_by, [dep.id]);
1990                    assert_eq!(after.blocked_from, Some(waiting.status));
1991                    assert!(!after.block_reason.unwrap().contains("missing"));
1992                    assert!(questions.list().is_empty());
1993                    assert_eq!(after.instruction, waiting.instruction);
1994                }
1995            }
1996        }
1997    }
1998
1999    #[tokio::test]
2000    async fn dependency_resolution_inconclusive_missing_only_decisions_keep_waiting() {
2001        for held in [false, true] {
2002            for unknown in [false, true] {
2003                for question in [false, true] {
2004                    let dir = tempdir().unwrap();
2005                    let queue = Queue::at(dir.path().join("queue"));
2006                    let questions = Questions::at(dir.path().join("questions"));
2007                    let mut dep = task("running prerequisite");
2008                    dep.id = "20261013-193611-ca5b".into();
2009                    dep.start("existing-run".into());
2010                    queue.put(&mut dep).unwrap();
2011                    let reference = if unknown {
2012                        // Even an exact, existing task id is Unknown when
2013                        // the question inventory cannot be scanned.
2014                        std::fs::write(questions.root(), "not a directory").unwrap();
2015                        dep.id.clone()
2016                    } else {
2017                        let mut other = task("another prerequisite");
2018                        other.id = "20261014-193611-ca5b".into();
2019                        queue.put(&mut other).unwrap();
2020                        "ca5b".to_owned()
2021                    };
2022                    assert_eq!(
2023                        crate::queue::dependency(&queue, &questions, &reference),
2024                        if unknown {
2025                            crate::queue::Dependency::Unknown
2026                        } else {
2027                            crate::queue::Dependency::Ambiguous
2028                        }
2029                    );
2030                    let mut waiting = task("waiting");
2031                    if held {
2032                        waiting.hold_machine(Some("dependency could not be located".into()));
2033                    }
2034                    queue.put(&mut waiting).unwrap();
2035                    let reply = serde_json::json!({"decisions": [{
2036                        "id": waiting.id, "blocked_by": [reference],
2037                        "missing_dependencies": [reference],
2038                        "question": question.then_some("ca5b is missing"),
2039                        "recovery": (!question).then_some("hold"),
2040                        "reason": "ca5b is missing"
2041                    }]})
2042                    .to_string();
2043                    let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2044                    let (queued, finished) = if held {
2045                        (vec![], vec![waiting.clone()])
2046                    } else {
2047                        (vec![waiting.clone()], vec![])
2048                    };
2049                    Conductor::new()
2050                        .run_once(
2051                            &cfg,
2052                            dir.path(),
2053                            &queue,
2054                            &questions,
2055                            dir.path(),
2056                            &queued,
2057                            &[],
2058                            &finished,
2059                            2,
2060                        )
2061                        .await
2062                        .unwrap();
2063                    let after = queue.get(&waiting.id).unwrap();
2064                    assert_eq!(after.status, TaskStatus::Blocked);
2065                    assert_eq!(after.blocked_by, [reference]);
2066                    assert_eq!(after.blocked_from, Some(waiting.status));
2067                    assert!(!after.block_reason.unwrap().contains("missing"));
2068                    assert_eq!(after.instruction, waiting.instruction);
2069                    if unknown {
2070                        assert!(questions.root().is_file());
2071                    } else {
2072                        assert!(questions.list().is_empty());
2073                    }
2074                }
2075            }
2076        }
2077    }
2078
2079    #[tokio::test]
2080    async fn dependency_resolution_unverifiable_alias_decisions_are_discarded() {
2081        for held in [false, true] {
2082            for unknown in [false, true] {
2083                for question in [false, true] {
2084                    for short_blocker in [false, true] {
2085                        let dir = tempdir().unwrap();
2086                        let queue = Queue::at(dir.path().join("queue"));
2087                        let questions = Questions::at(dir.path().join("questions"));
2088                        let mut dep = task("running prerequisite");
2089                        dep.id = "20261013-193611-ca5b".into();
2090                        dep.start("existing-run".into());
2091                        queue.put(&mut dep).unwrap();
2092                        if unknown {
2093                            std::fs::write(questions.root(), "not a directory").unwrap();
2094                        } else {
2095                            let mut other = task("another prerequisite");
2096                            other.id = "20261014-193611-ca5b".into();
2097                            queue.put(&mut other).unwrap();
2098                        }
2099                        let (blocker, missing) = if short_blocker {
2100                            ("ca5b", dep.id.as_str())
2101                        } else {
2102                            (dep.id.as_str(), "ca5b")
2103                        };
2104                        let mut waiting = task("waiting");
2105                        if held {
2106                            waiting.hold_machine(Some("dependency could not be located".into()));
2107                        }
2108                        queue.put(&mut waiting).unwrap();
2109                        let reply = serde_json::json!({"decisions": [{
2110                            "id": waiting.id, "blocked_by": [blocker],
2111                            "missing_dependencies": [missing],
2112                            "question": question.then_some("ca5b is missing"),
2113                            "recovery": (!question).then_some("hold"),
2114                            "reason": "ca5b is missing"
2115                        }]})
2116                        .to_string();
2117                        let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2118                        let (queued, finished) = if held {
2119                            (vec![], vec![waiting.clone()])
2120                        } else {
2121                            (vec![waiting.clone()], vec![])
2122                        };
2123                        Conductor::new()
2124                            .run_once(
2125                                &cfg,
2126                                dir.path(),
2127                                &queue,
2128                                &questions,
2129                                dir.path(),
2130                                &queued,
2131                                &[],
2132                                &finished,
2133                                2,
2134                            )
2135                            .await
2136                            .unwrap();
2137                        assert_eq!(
2138                            serde_json::to_value(queue.get(&waiting.id).unwrap()).unwrap(),
2139                            serde_json::to_value(&waiting).unwrap(),
2140                            "unverifiable aliases must leave the entire task unchanged"
2141                        );
2142                        if unknown {
2143                            assert!(questions.root().is_file());
2144                        } else {
2145                            assert!(questions.list().is_empty());
2146                        }
2147                    }
2148                }
2149            }
2150        }
2151    }
2152
2153    #[tokio::test]
2154    async fn dependency_resolution_genuine_missing_reference_still_asks() {
2155        let dir = tempdir().unwrap();
2156        let queue = Queue::at(dir.path().join("queue"));
2157        let questions = Questions::at(dir.path().join("questions"));
2158        let mut waiting = task("missing prerequisite");
2159        queue.put(&mut waiting).unwrap();
2160        let reply = serde_json::json!({"decisions": [{
2161            "id": waiting.id, "blocked_by": ["not-present"],
2162            "missing_dependencies": ["not-present"],
2163            "question": "Which prerequisite replaces the missing task?"
2164        }]})
2165        .to_string();
2166        let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
2167        Conductor::new()
2168            .run_once(
2169                &cfg,
2170                dir.path(),
2171                &queue,
2172                &questions,
2173                dir.path(),
2174                &[waiting.clone()],
2175                &[],
2176                &[],
2177                2,
2178            )
2179            .await
2180            .unwrap();
2181        let after = queue.get(&waiting.id).unwrap();
2182        assert_eq!(after.status, TaskStatus::Blocked);
2183        assert_eq!(
2184            questions.get(&after.blocked_by[0]).unwrap().summary,
2185            "Which prerequisite replaces the missing task?"
2186        );
2187    }
2188
2189    #[tokio::test]
2190    async fn dependency_resolution_keeps_decisions_during_unrelated_task_progress() {
2191        let dir = tempdir().unwrap();
2192        let queue = Queue::at(dir.path().join("queue"));
2193        let questions = Questions::at(dir.path().join("questions"));
2194        let mut dep = task("prerequisite");
2195        dep.id = "20261013-193611-ca5b".into();
2196        dep.start("existing-run".into());
2197        queue.put(&mut dep).unwrap();
2198        let mut unrelated = task("unrelated work");
2199        unrelated.start("other-run".into());
2200        queue.put(&mut unrelated).unwrap();
2201        // Make the old and new revision distinguishable without relying on
2202        // the filesystem clock advancing while the mock process starts.
2203        std::fs::File::options()
2204            .write(true)
2205            .open(queue.path_of(&unrelated.id))
2206            .unwrap()
2207            .set_times(std::fs::FileTimes::new().set_modified(std::time::UNIX_EPOCH))
2208            .unwrap();
2209        unrelated.succeed();
2210        let mut waiting = task("waiting");
2211        let mut asking = task("needs a product decision");
2212        let mut holding = task("must stay held");
2213        queue.put(&mut waiting).unwrap();
2214        queue.put(&mut asking).unwrap();
2215        queue.put(&mut holding).unwrap();
2216        let reply = serde_json::json!({"decisions": [
2217            {"id": waiting.id, "blocked_by": ["ca5b"], "reason": "wait for implementation"},
2218            {"id": asking.id, "question": "Which behavior do you want?"},
2219            {"id": holding.id, "recovery": "hold", "reason": "already implemented elsewhere"}
2220        ]})
2221        .to_string();
2222        let mut vars = env(&reply);
2223        vars.insert(
2224            "UNRELATED_PATH".into(),
2225            queue.path_of(&unrelated.id).to_string_lossy().into_owned(),
2226        );
2227        vars.insert(
2228            "UNRELATED_RECORD".into(),
2229            serde_json::to_string(&unrelated).unwrap(),
2230        );
2231        let script = r#"#!/bin/sh
2232cat >/dev/null
2233printf '%s' "$UNRELATED_RECORD" > "$UNRELATED_PATH"
2234printf '%s\n' "$MOCK_REPLY"
2235"#;
2236        let cfg = config(mock_agent(dir.path(), script, vars));
2237        Conductor::new()
2238            .run_once(
2239                &cfg,
2240                dir.path(),
2241                &queue,
2242                &questions,
2243                dir.path(),
2244                &[waiting.clone(), asking.clone(), holding.clone()],
2245                &[],
2246                &[],
2247                2,
2248            )
2249            .await
2250            .unwrap();
2251        assert_eq!(queue.get(&unrelated.id).unwrap().status, TaskStatus::Done);
2252        let after = queue.get(&waiting.id).unwrap();
2253        assert_eq!(after.status, TaskStatus::Blocked);
2254        assert_eq!(after.blocked_by, [dep.id]);
2255        let asked = queue.get(&asking.id).unwrap();
2256        assert_eq!(asked.status, TaskStatus::Blocked);
2257        assert_eq!(
2258            questions.get(&asked.blocked_by[0]).unwrap().summary,
2259            "Which behavior do you want?"
2260        );
2261        assert_eq!(queue.get(&holding.id).unwrap().status, TaskStatus::Held);
2262    }
2263
2264    #[test]
2265    fn a_dependency_decision_blocks_the_task_and_leaves_priority_alone() {
2266        let dir = tempdir().unwrap();
2267        let queue = Queue::at(dir.path().join("queue"));
2268        let questions = Questions::at(dir.path().join("questions"));
2269        let mut a = task("a");
2270        a.priority = 9;
2271        queue.put(&mut a).unwrap();
2272
2273        let verdict = Verdict {
2274            decisions: vec![Decision {
2275                id: a.id.clone(),
2276                blocked_by: vec!["20260101-000000-dead".to_owned()],
2277                reason: Some("waits on the other task".to_owned()),
2278                recovery: None,
2279                question: None,
2280                choices: Vec::new(),
2281                actions: BTreeMap::new(),
2282                question_decision: None,
2283                evidence: Vec::new(),
2284                missing_dependencies: Vec::new(),
2285            }],
2286        };
2287        apply(&queue, &questions, &verdict).unwrap();
2288
2289        let back = queue.get(&a.id).unwrap();
2290        assert_eq!(back.status, TaskStatus::Blocked);
2291        assert_eq!(back.blocked_by, ["20260101-000000-dead"]);
2292        assert_eq!(
2293            back.priority, 9,
2294            "the conductor's reply cannot carry priority"
2295        );
2296    }
2297
2298    #[test]
2299    fn a_question_decision_files_one_and_blocks_on_its_id() {
2300        let dir = tempdir().unwrap();
2301        let queue = Queue::at(dir.path().join("queue"));
2302        let questions = Questions::at(dir.path().join("questions"));
2303        let mut t = task("ambiguous");
2304        queue.put(&mut t).unwrap();
2305
2306        let verdict = Verdict {
2307            decisions: vec![Decision {
2308                id: t.id.clone(),
2309                blocked_by: Vec::new(),
2310                reason: Some("which backend?".to_owned()),
2311                recovery: None,
2312                question: Some("Which storage backend?".to_owned()),
2313                choices: vec!["SQLite".to_owned(), "Redis".to_owned()],
2314                actions: BTreeMap::new(),
2315                question_decision: None,
2316                evidence: Vec::new(),
2317                missing_dependencies: Vec::new(),
2318            }],
2319        };
2320        apply(&queue, &questions, &verdict).unwrap();
2321
2322        let back = queue.get(&t.id).unwrap();
2323        assert_eq!(back.status, TaskStatus::Blocked);
2324        assert_eq!(back.blocked_by.len(), 1);
2325        let q = questions.get(&back.blocked_by[0]).unwrap();
2326        assert_eq!(q.summary, "Which storage backend?");
2327        assert_eq!(q.node, NODE);
2328        assert!(q.status.open());
2329    }
2330
2331    #[test]
2332    fn a_task_with_an_open_question_already_reuses_it_rather_than_filing_a_second_one() {
2333        let dir = tempdir().unwrap();
2334        let queue = Queue::at(dir.path().join("queue"));
2335        let questions = Questions::at(dir.path().join("questions"));
2336        let mut t = task("asked once");
2337        queue.put(&mut t).unwrap();
2338
2339        let decision = Decision {
2340            id: t.id.clone(),
2341            reason: Some("still deciding".to_owned()),
2342            question: Some("Which backend?".to_owned()),
2343            ..Decision::default()
2344        };
2345        apply(
2346            &queue,
2347            &questions,
2348            &Verdict {
2349                decisions: vec![decision.clone()],
2350            },
2351        )
2352        .unwrap();
2353        assert_eq!(questions.list().len(), 1);
2354        let first_question_id = queue.get(&t.id).unwrap().blocked_by[0].clone();
2355
2356        // An operator releasing the blocked task by hand, without answering,
2357        // puts it back at `Queued` while the question stays open - exactly
2358        // the case the guard in `apply_one` exists for: a later cycle
2359        // proposing the very same question must reuse it, not file a second.
2360        let mut released = queue.get(&t.id).unwrap();
2361        released.release();
2362        queue.put(&mut released).unwrap();
2363
2364        apply(
2365            &queue,
2366            &questions,
2367            &Verdict {
2368                decisions: vec![decision],
2369            },
2370        )
2371        .unwrap();
2372        assert_eq!(questions.list().len(), 1, "no duplicate question was filed");
2373        let after = queue.get(&t.id).unwrap();
2374        assert_eq!(
2375            after.blocked_by,
2376            [first_question_id],
2377            "the existing open question is reused, not replaced"
2378        );
2379    }
2380
2381    #[test]
2382    fn a_same_id_question_from_another_node_is_not_reused() {
2383        let dir = tempdir().unwrap();
2384        let queue = Queue::at(dir.path().join("queue"));
2385        let questions = Questions::at(dir.path().join("questions"));
2386        let mut t = task("must ask the conductor");
2387        queue.put(&mut t).unwrap();
2388
2389        let mut unrelated = Question::new(
2390            t.id.clone(),
2391            "review".to_owned(),
2392            "reviewer-1".to_owned(),
2393            "An unrelated review question".to_owned(),
2394            String::new(),
2395            Vec::new(),
2396        );
2397        questions.put(&mut unrelated).unwrap();
2398
2399        apply(
2400            &queue,
2401            &questions,
2402            &Verdict {
2403                decisions: vec![Decision {
2404                    id: t.id.clone(),
2405                    question: Some("Which backend?".to_owned()),
2406                    ..Decision::default()
2407                }],
2408            },
2409        )
2410        .unwrap();
2411
2412        let blocked_by = &queue.get(&t.id).unwrap().blocked_by;
2413        assert_eq!(blocked_by.len(), 1);
2414        assert_ne!(blocked_by[0], unrelated.id);
2415        assert!(questions.get(&unrelated.id).unwrap().status.open());
2416        assert_eq!(questions.get(&blocked_by[0]).unwrap().node, NODE);
2417    }
2418
2419    #[test]
2420    fn answering_the_question_lets_the_resolver_clear_the_block_with_the_answer_kept() {
2421        let dir = tempdir().unwrap();
2422        let queue = Queue::at(dir.path().join("queue"));
2423        let questions = Questions::at(dir.path().join("questions"));
2424        let mut t = task("waits on an answer");
2425        queue.put(&mut t).unwrap();
2426
2427        apply(
2428            &queue,
2429            &questions,
2430            &Verdict {
2431                decisions: vec![Decision {
2432                    id: t.id.clone(),
2433                    blocked_by: Vec::new(),
2434                    reason: None,
2435                    recovery: None,
2436                    question: Some("Which backend?".to_owned()),
2437                    choices: Vec::new(),
2438                    actions: BTreeMap::new(),
2439                    question_decision: None,
2440                    evidence: Vec::new(),
2441                    missing_dependencies: Vec::new(),
2442                }],
2443            },
2444        )
2445        .unwrap();
2446        let blocked = queue.get(&t.id).unwrap();
2447        let question_id = blocked.blocked_by[0].clone();
2448
2449        let mut q = questions.get(&question_id).unwrap();
2450        q.answer(Answer::Text("SQLite".to_owned())).unwrap();
2451        questions.put(&mut q).unwrap();
2452        assert_eq!(q.status, QuestionStatus::Answered);
2453
2454        // `crate::daemon::resolve_blockers` is the deterministic resolver
2455        // that actually does this on the real queue; here it is enough to
2456        // prove the pure steps it is built from behave together.
2457        let mut task_after = queue.get(&t.id).unwrap();
2458        task_after.record_answer(q.summary.clone(), "SQLite".to_owned());
2459        task_after.unblock(&question_id);
2460        assert_eq!(task_after.status, TaskStatus::Queued);
2461        assert_eq!(task_after.answers[0].answer, "SQLite");
2462    }
2463
2464    #[test]
2465    fn a_stalled_task_can_be_requeued_or_held() {
2466        let dir = tempdir().unwrap();
2467        let queue = Queue::at(dir.path().join("queue"));
2468        let questions = Questions::at(dir.path().join("questions"));
2469
2470        let mut requeue_me = task("stuck a");
2471        requeue_me.start("run-1".to_owned());
2472        queue.put(&mut requeue_me).unwrap();
2473
2474        let mut hold_me = task("stuck b");
2475        hold_me.start("run-2".to_owned());
2476        queue.put(&mut hold_me).unwrap();
2477
2478        apply(
2479            &queue,
2480            &questions,
2481            &Verdict {
2482                decisions: vec![
2483                    Decision {
2484                        id: requeue_me.id.clone(),
2485                        recovery: Some(Recovery::Requeue),
2486                        ..Decision::default()
2487                    },
2488                    Decision {
2489                        id: hold_me.id.clone(),
2490                        recovery: Some(Recovery::Hold),
2491                        reason: Some("looks broken".to_owned()),
2492                        ..Decision::default()
2493                    },
2494                ],
2495            },
2496        )
2497        .unwrap();
2498
2499        let requeued = queue.get(&requeue_me.id).unwrap();
2500        assert_eq!(requeued.status, TaskStatus::Queued);
2501        assert_eq!(requeued.attempts, 0);
2502
2503        let held = queue.get(&hold_me.id).unwrap();
2504        assert_eq!(held.status, TaskStatus::Held);
2505        assert_eq!(held.hold_reason.as_deref(), Some("looks broken"));
2506    }
2507
2508    #[test]
2509    fn a_machine_held_task_asked_about_restores_to_held_once_answered() {
2510        // Reproduces the reported bug: a task held after burning its
2511        // attempts, once the conductor asks a follow-up question about it,
2512        // must come back `held` - not `queued` - once the question is
2513        // answered, whatever the answer said.
2514        let dir = tempdir().unwrap();
2515        let queue = Queue::at(dir.path().join("queue"));
2516        let questions = Questions::at(dir.path().join("questions"));
2517        let mut t = task("held out of attempts");
2518        t.hold_machine(Some("out of attempts".to_owned()));
2519        queue.put(&mut t).unwrap();
2520
2521        apply(
2522            &queue,
2523            &questions,
2524            &Verdict {
2525                decisions: vec![Decision {
2526                    id: t.id.clone(),
2527                    reason: Some("what should happen to this one?".to_owned()),
2528                    question: Some("Hold it, or try again?".to_owned()),
2529                    ..Decision::default()
2530                }],
2531            },
2532        )
2533        .unwrap();
2534        let blocked = queue.get(&t.id).unwrap();
2535        assert_eq!(blocked.status, TaskStatus::Blocked);
2536        let question_id = blocked.blocked_by[0].clone();
2537
2538        let mut q = questions.get(&question_id).unwrap();
2539        q.answer(Answer::Text("leave it held".to_owned())).unwrap();
2540        questions.put(&mut q).unwrap();
2541
2542        // What `crate::daemon::resolve_blockers` does on the real queue.
2543        let mut after = queue.get(&t.id).unwrap();
2544        after.record_answer(q.summary.clone(), "leave it held".to_owned());
2545        after.unblock(&question_id);
2546        assert_eq!(
2547            after.status,
2548            TaskStatus::Held,
2549            "must not fall back to queued"
2550        );
2551        assert_eq!(after.hold_reason.as_deref(), Some("out of attempts"));
2552    }
2553
2554    #[test]
2555    fn a_reaffirmed_hold_with_no_new_reason_is_not_silently_auto_released_by_triage() {
2556        // A different path to the same bug round 1 fixed: a task
2557        // machine-held for disk pressure, blocked on a conductor question,
2558        // answered "keep it held", and restored to `held`. If the
2559        // conductor's next decision reconfirms `recovery: hold` with no new
2560        // `reason` - allowed, `Decision::reason` is optional - `hold_machine`
2561        // must not silently leave the stale disk-pressure text in place, or
2562        // `crate::triage::run_once` reads it as an unexamined, auto-resolvable
2563        // hold and releases the task straight through the operator's answer
2564        // the instant disk space looks fine again.
2565        let dir = tempdir().unwrap();
2566        let queue = Queue::at(dir.path().join("queue"));
2567        let questions = Questions::at(dir.path().join("questions"));
2568        let mut t = task("disk pressure, then reconsidered");
2569        t.hold_machine(Some(
2570            "not enough free space to start a run: 10 bytes free, 100 required by \
2571             `[disk] min_free_bytes`"
2572                .to_owned(),
2573        ));
2574        t.record_answer(
2575            "How should this be handled?".to_owned(),
2576            "keep it held, a human will look at it later".to_owned(),
2577        );
2578        queue.put(&mut t).unwrap();
2579
2580        apply(
2581            &queue,
2582            &questions,
2583            &Verdict {
2584                decisions: vec![Decision {
2585                    id: t.id.clone(),
2586                    recovery: Some(Recovery::Hold),
2587                    ..Decision::default()
2588                }],
2589            },
2590        )
2591        .unwrap();
2592
2593        let after = queue.get(&t.id).unwrap();
2594        assert_eq!(after.status, TaskStatus::Held);
2595        assert!(
2596            !after
2597                .hold_reason
2598                .as_deref()
2599                .unwrap_or_default()
2600                .starts_with("not enough free space"),
2601            "the stale disk-pressure text must not survive a reconfirmed hold: {:?}",
2602            after.hold_reason
2603        );
2604
2605        // The disk gate would report space is fine now - `crate::triage`
2606        // must not read the old text and release the task through it.
2607        let cfg_dir = tempdir().unwrap();
2608        let config = cfg_dir.path().join("magi.toml");
2609        std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2610        let report =
2611            crate::triage::run_once(&queue, &questions, Some(&config), jiff::Timestamp::now());
2612        assert!(report.resumed.is_empty(), "must not be auto-released");
2613        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2614    }
2615
2616    #[test]
2617    fn explicit_resumption_rehold_requires_new_evidence_and_another_answer() {
2618        crate::run::pin_test_home();
2619        let dir = tempdir().unwrap();
2620        let queue = Queue::at(dir.path().join("queue"));
2621        let questions = Questions::at(dir.path().join("questions"));
2622        let config = dir.path().join("magi.toml");
2623        std::fs::write(&config, "[disk]\nmin_free_bytes = 0\n").unwrap();
2624        let now = jiff::Timestamp::now();
2625        let triage = || crate::triage::run_once(&queue, &questions, Some(&config), now);
2626        let open = || {
2627            questions
2628                .list()
2629                .into_iter()
2630                .filter(|q| q.node == "triage" && q.status.open())
2631                .collect::<Vec<_>>()
2632        };
2633        let hold = Decision {
2634            recovery: Some(Recovery::Hold),
2635            reason: Some("waiting on manual worktree cleanup".to_owned()),
2636            ..Decision::default()
2637        };
2638
2639        let mut t = task("looping hold");
2640        t.hold_machine(Some("waiting on manual worktree cleanup".to_owned()));
2641        queue.put(&mut t).unwrap();
2642        let hold = Decision {
2643            id: t.id.clone(),
2644            ..hold
2645        };
2646
2647        // 1. triage asks the ordinary machine question; the operator resumes.
2648        assert_eq!(triage().asked.len(), 1);
2649        let first = open().remove(0);
2650        let mut q = questions.get(&first.id).unwrap();
2651        let resume = q.choices[0].clone();
2652        q.answer(Answer::Choice(resume)).unwrap();
2653        questions.put(&mut q).unwrap();
2654        assert_eq!(triage().answered.len(), 1);
2655        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2656
2657        // Rewording the old cause cannot undo the first explicit answer.
2658        apply(
2659            &queue,
2660            &questions,
2661            &Verdict {
2662                decisions: vec![hold.clone()],
2663            },
2664        )
2665        .unwrap();
2666        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2667        assert!(triage().asked.is_empty());
2668
2669        // A new independently filed check can justify a later hold.
2670        let mut evidence = Question::new(
2671            t.id.clone(),
2672            "preflight".into(),
2673            "check".into(),
2674            "New permission failure".into(),
2675            String::new(),
2676            Vec::new(),
2677        );
2678        questions.put(&mut evidence).unwrap();
2679        let hold = Decision {
2680            evidence: vec![evidence.id.clone()],
2681            reason: Some("new permission check failed".into()),
2682            ..hold
2683        };
2684        apply(
2685            &queue,
2686            &questions,
2687            &Verdict {
2688                decisions: vec![hold.clone()],
2689            },
2690        )
2691        .unwrap();
2692        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
2693
2694        // The new hold earns one question identifying the new evidence.
2695        assert_eq!(triage().asked.len(), 1);
2696        let second = open().remove(0);
2697        assert!(second.detail.contains("new permission check failed"));
2698        assert!(triage().asked.is_empty(), "no duplicate question");
2699        assert_eq!(open().len(), 1);
2700
2701        // Another explicit answer supersedes this later hold too.
2702        let mut q = questions.get(&second.id).unwrap();
2703        let force = q.choices[0].clone();
2704        q.answer(Answer::Choice(force)).unwrap();
2705        questions.put(&mut q).unwrap();
2706        assert_eq!(triage().answered.len(), 1);
2707        apply(
2708            &queue,
2709            &questions,
2710            &Verdict {
2711                decisions: vec![hold],
2712            },
2713        )
2714        .unwrap();
2715        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2716    }
2717
2718    #[test]
2719    fn conductor_resumption_notice_does_not_solicit_unregistered_questions() {
2720        crate::run::pin_test_home();
2721        let dir = tempdir().unwrap();
2722        let queue = Queue::at(dir.path().join("queue"));
2723        let questions = Questions::at(dir.path().join("questions"));
2724
2725        let mut t = task("task with explicit resumption");
2726        t.status = TaskStatus::Queued;
2727        t.resumption = Some(crate::queue::Resumption {
2728            generation: 1,
2729            source: "test".into(),
2730            question_id: Some("q1".into()),
2731            at: jiff::Timestamp::now(),
2732            instruction: "resume".into(),
2733            answers_through: 0,
2734            runs_through: 0,
2735            cleared_reason: None,
2736            blocking_evidence: Vec::new(),
2737        });
2738        queue.put(&mut t).unwrap();
2739
2740        // Conductor suggests a question and dependencies without new evidence
2741        let decision = Decision {
2742            id: t.id.clone(),
2743            question: Some("How should we proceed with this task?".into()),
2744            blocked_by: vec!["dep-x".into()],
2745            reason: Some("needs clarification".into()),
2746            recovery: Some(Recovery::Hold),
2747            ..Decision::default()
2748        };
2749
2750        apply(
2751            &queue,
2752            &questions,
2753            &Verdict {
2754                decisions: vec![decision],
2755            },
2756        )
2757        .unwrap();
2758
2759        // Task stays Queued (not re-blocked on old evidence)
2760        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
2761        // No question was filed in the questions store
2762        assert!(questions.list().is_empty());
2763
2764        // A notice was filed with human-readable text and task link
2765        let notices = crate::notices::Notices::open();
2766        let notice = notices
2767            .get(&crate::notices::id_of(&format!(
2768                "conductor-resumption:{}",
2769                t.id
2770            )))
2771            .expect("conductor-resumption notice filed");
2772
2773        assert_eq!(
2774            notice.link,
2775            Some(crate::notices::Link::Task { id: t.id.clone() })
2776        );
2777        // Message does NOT contain the question prose to avoid asking for non-existent answers
2778        assert!(!notice.message.contains("How should we proceed"));
2779        // Message explicitly notes that no reply is required
2780        assert!(
2781            notice
2782                .message
2783                .contains("No reply to this notice is needed.")
2784        );
2785        // No raw Rust debug representations
2786        assert!(!notice.message.contains("Some("));
2787        assert!(!notice.message.contains("None"));
2788        assert!(!notice.message.contains("[]"));
2789        assert!(!notice.message.contains("{:?}"));
2790        // Mention proposed dependencies cleanly
2791        assert!(notice.message.contains("Proposed dependencies: dep-x."));
2792    }
2793
2794    #[test]
2795    fn a_runnable_task_can_be_held_directly_without_a_question() {
2796        let dir = tempdir().unwrap();
2797        let queue = Queue::at(dir.path().join("queue"));
2798        let questions = Questions::at(dir.path().join("questions"));
2799        let mut t = task("already answered, should stay put");
2800        queue.put(&mut t).unwrap();
2801
2802        apply(
2803            &queue,
2804            &questions,
2805            &Verdict {
2806                decisions: vec![Decision {
2807                    id: t.id.clone(),
2808                    recovery: Some(Recovery::Hold),
2809                    reason: Some("operator already said keep this held".to_owned()),
2810                    ..Decision::default()
2811                }],
2812            },
2813        )
2814        .unwrap();
2815
2816        let after = queue.get(&t.id).unwrap();
2817        assert_eq!(after.status, TaskStatus::Held);
2818        assert_eq!(after.hold_source, Some(crate::queue::HoldSource::Machine));
2819    }
2820
2821    #[test]
2822    fn done_recovery_closes_a_held_task_whose_goal_is_already_met() {
2823        crate::run::pin_test_home();
2824        // Reproduces the other half of the reported bug (task 6081): once
2825        // the operator's answer says the work already happened outside the
2826        // loop - PR merged, worktree cleaned up - the conductor needs an
2827        // actual terminal state to put the task in, not just a hold it will
2828        // keep being re-asked about.
2829        let dir = tempdir().unwrap();
2830        let queue = Queue::at(dir.path().join("queue"));
2831        let questions = Questions::at(dir.path().join("questions"));
2832        let mut t = task("already merged by hand");
2833        t.hold_machine(Some("branch survived, awaiting a decision".to_owned()));
2834        t.record_answer(
2835            "Handle this one?".to_owned(),
2836            "already merged and cleaned up, close it".to_owned(),
2837        );
2838        queue.put(&mut t).unwrap();
2839
2840        apply(
2841            &queue,
2842            &questions,
2843            &Verdict {
2844                decisions: vec![Decision {
2845                    id: t.id.clone(),
2846                    recovery: Some(Recovery::Done),
2847                    reason: Some("operator confirmed this already landed".to_owned()),
2848                    ..Decision::default()
2849                }],
2850            },
2851        )
2852        .unwrap();
2853
2854        let after = queue.get(&t.id).unwrap();
2855        assert_eq!(after.status, TaskStatus::Done);
2856        assert!(after.hold_reason.is_none());
2857        assert_eq!(after.answers.len(), 1, "the record of why is kept");
2858    }
2859
2860    #[test]
2861    fn done_recovery_is_ignored_for_a_runnable_or_running_task() {
2862        let dir = tempdir().unwrap();
2863        let queue = Queue::at(dir.path().join("queue"));
2864        let questions = Questions::at(dir.path().join("questions"));
2865
2866        let mut queued = task("never ran yet");
2867        queue.put(&mut queued).unwrap();
2868
2869        let mut running = task("mid-run");
2870        running.start("run-1".to_owned());
2871        queue.put(&mut running).unwrap();
2872
2873        for id in [queued.id.clone(), running.id.clone()] {
2874            apply(
2875                &queue,
2876                &questions,
2877                &Verdict {
2878                    decisions: vec![Decision {
2879                        id,
2880                        recovery: Some(Recovery::Done),
2881                        ..Decision::default()
2882                    }],
2883                },
2884            )
2885            .unwrap();
2886        }
2887
2888        assert_eq!(queue.get(&queued.id).unwrap().status, TaskStatus::Queued);
2889        assert_eq!(queue.get(&running.id).unwrap().status, TaskStatus::Running);
2890    }
2891
2892    #[test]
2893    fn a_question_after_two_settled_answers_is_still_filed_and_blocks() {
2894        // A genuine question must stay answerable however many were settled
2895        // before it: refusing it left `magi answer --list` empty.
2896        let dir = tempdir().unwrap();
2897        let queue = Queue::at(dir.path().join("queue"));
2898        let questions = Questions::at(dir.path().join("questions"));
2899        let mut t = task("asked about repeatedly");
2900        t.hold_machine(Some("out of attempts".to_owned()));
2901        t.record_answer("Handle this one? (1)".to_owned(), "not yet".to_owned());
2902        t.record_answer(
2903            "Handle this one? (2)".to_owned(),
2904            "still not yet".to_owned(),
2905        );
2906        queue.put(&mut t).unwrap();
2907        assert_eq!(questions.list().len(), 0);
2908
2909        let verdict = Verdict {
2910            decisions: vec![Decision {
2911                id: t.id.clone(),
2912                question: Some("Branch conflicts with origin/main, how do we proceed?".to_owned()),
2913                ..Decision::default()
2914            }],
2915        };
2916        apply(&queue, &questions, &verdict).unwrap();
2917
2918        let filed = questions.list();
2919        assert_eq!(filed.len(), 1, "the question was filed");
2920        assert_eq!(
2921            filed[0].summary,
2922            "Branch conflicts with origin/main, how do we proceed?"
2923        );
2924        let deputy = filed[0]
2925            .deputy
2926            .as_ref()
2927            .expect("the wait is handed to a deputy");
2928        assert!(deputy.brief.contains(&t.id), "{}", deputy.brief);
2929        assert_eq!(
2930            deputy.starts, 0,
2931            "filing never starts anything: the loop does not wait"
2932        );
2933        assert!(filed[0].status.open());
2934        assert_eq!(filed[0].node, NODE);
2935        let after = queue.get(&t.id).unwrap();
2936        assert_eq!(after.status, TaskStatus::Blocked);
2937        assert_eq!(after.blocked_by, vec![filed[0].id.clone()]);
2938        assert_eq!(after.answers.len(), 2, "the prior answers are untouched");
2939        assert!(
2940            !after
2941                .hold_reason
2942                .clone()
2943                .unwrap_or_default()
2944                .contains("conduct tried to ask"),
2945            "no hold was applied"
2946        );
2947
2948        apply(&queue, &questions, &verdict).unwrap();
2949        let again = questions.list();
2950        assert_eq!(again.len(), 1, "the open question is reused");
2951        assert_eq!(again[0].id, filed[0].id);
2952    }
2953
2954    #[test]
2955    fn question_source_identity_resolves_aliases_and_rejects_invented_targets() {
2956        let dir = tempdir().unwrap();
2957        init_repo_with_branch(dir.path(), "source-a");
2958        let mut t = task("source decision");
2959        t.repo = dir.path().into();
2960        let decision = |reference: &str| QuestionDecision {
2961            operation: QuestionOperation::Recover,
2962            target: QuestionTarget::Source {
2963                reference: reference.into(),
2964            },
2965        };
2966        assert_eq!(
2967            decision("source-a").identity(&t).unwrap(),
2968            decision("refs/heads/source-a").identity(&t).unwrap()
2969        );
2970        assert_ne!(
2971            decision("source-a").identity(&t).unwrap(),
2972            decision("main").identity(&t).unwrap()
2973        );
2974        assert!(decision("invented-source-label").identity(&t).is_err());
2975        let unrelated = QuestionDecision {
2976            operation: QuestionOperation::Resume,
2977            target: QuestionTarget::Run {
2978                id: "unrelated-run".into(),
2979            },
2980        };
2981        assert!(unrelated.identity(&t).is_err());
2982    }
2983
2984    #[test]
2985    fn base_recovery_conductor_cannot_offer_or_requeue_an_unchanged_stop() {
2986        let dir = tempdir().unwrap();
2987        let queue = Queue::at(dir.path().join("queue"));
2988        let questions = Questions::at(dir.path().join("questions"));
2989        let mut t = task("retained base stop");
2990        let mut state = RunState::new(
2991            dir.path().into(),
2992            "main".into(),
2993            "base".into(),
2994            "retained work".into(),
2995            Config::default(),
2996        );
2997        state.base_sync = Some(crate::run::BaseSync {
2998            tip: "tip".into(),
2999            behind: 3,
3000            attempts: 4,
3001            conflict: Some("exhausted".into()),
3002            already_in: None,
3003        });
3004        state.save_under(dir.path()).unwrap();
3005        t.link_run(&state.id);
3006        t.hold_machine(Some("base stopped".into()));
3007        queue.put(&mut t).unwrap();
3008        let d = Decision {
3009            id: t.id.clone(),
3010            recovery: Some(Recovery::Requeue),
3011            ..Decision::default()
3012        };
3013        apply_one(&queue, &questions, &d, None).unwrap();
3014        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
3015        let d = Decision {
3016            id: t.id.clone(),
3017            question: Some("Resume?".into()),
3018            question_decision: Some(QuestionDecision {
3019                operation: QuestionOperation::Resume,
3020                target: QuestionTarget::Run { id: state.id },
3021            }),
3022            ..Decision::default()
3023        };
3024        apply_one(&queue, &questions, &d, None).unwrap();
3025        assert!(questions.list().is_empty());
3026        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Held);
3027    }
3028
3029    #[test]
3030    fn invalid_question_targets_still_file_and_block_queued_tasks() {
3031        let dir = tempdir().unwrap();
3032        init_repo_with_branch(dir.path(), "source-a");
3033        let queue = Queue::at(dir.path().join("queue"));
3034        let questions = Questions::at(dir.path().join("questions"));
3035        for target in [
3036            QuestionTarget::Source {
3037                reference: "missing-source".into(),
3038            },
3039            QuestionTarget::Run {
3040                id: "unrelated-run".into(),
3041            },
3042        ] {
3043            let mut t = task("decision with invalid metadata");
3044            t.repo = dir.path().into();
3045            queue.put(&mut t).unwrap();
3046            let d = Decision {
3047                id: t.id.clone(),
3048                question: Some("Which recovery should proceed?".into()),
3049                choices: vec!["inspect first".into(), "keep waiting".into()],
3050                question_decision: Some(QuestionDecision {
3051                    operation: QuestionOperation::Recover,
3052                    target,
3053                }),
3054                ..Decision::default()
3055            };
3056            apply_one(&queue, &questions, &d, None).unwrap();
3057            let q = questions
3058                .list()
3059                .into_iter()
3060                .find(|q| q.run == t.id)
3061                .unwrap();
3062            assert_eq!(q.summary, d.question.as_deref().unwrap());
3063            assert!(
3064                q.choices.iter().all(|c| q.actions.contains_key(c)),
3065                "manual-only choices must become an executable hold"
3066            );
3067            assert!(q.status.open());
3068            let context: serde_json::Value =
3069                serde_json::from_str(q.decision_context.as_ref().unwrap()).unwrap();
3070            assert!(context[3].is_null(), "invalid metadata was discarded");
3071            let after = queue.get(&t.id).unwrap();
3072            assert_eq!(after.status, TaskStatus::Blocked);
3073            assert_eq!(after.blocked_by, vec![q.id.clone()]);
3074            apply_one(&queue, &questions, &d, None).unwrap();
3075            assert_eq!(questions.list().iter().filter(|q| q.run == t.id).count(), 1);
3076        }
3077    }
3078
3079    #[test]
3080    fn answered_conductor_question_needs_changed_facts_before_reasking() {
3081        let dir = tempdir().unwrap();
3082        let queue = Queue::at(dir.path().join("queue"));
3083        let questions = Questions::at(dir.path().join("questions"));
3084        let mut t = task("recovery decision");
3085        t.hold_machine(Some("out of attempts".into()));
3086        t.runs = vec!["source-run-a".into(), "source-run-b".into()];
3087        queue.put(&mut t).unwrap();
3088        let mut d = Decision {
3089            id: t.id.clone(),
3090            question: Some("Recover from source A?".into()),
3091            choices: vec!["approve source A".into(), "keep held".into()],
3092            question_decision: Some(QuestionDecision {
3093                operation: QuestionOperation::Recover,
3094                target: QuestionTarget::Run {
3095                    id: "source-run-a".into(),
3096                },
3097            }),
3098            ..Decision::default()
3099        };
3100        apply_one(&queue, &questions, &d, None).unwrap();
3101        let mut q = questions.list().pop().unwrap();
3102        q.answer(crate::ask::Answer::Choice(q.choices[0].clone()))
3103            .unwrap();
3104        questions.put(&mut q).unwrap();
3105        queue
3106            .modify(&t.id, |task| {
3107                task.unblock(&q.id);
3108                task.hold_machine(Some("recovery remains under consideration".into()));
3109                true
3110            })
3111            .unwrap();
3112        d.question = Some("Should we keep waiting on recovery?".into());
3113        d.choices = vec!["recover using source A".into(), "remain on hold".into()];
3114        apply_one(&queue, &questions, &d, None).unwrap();
3115        assert_eq!(questions.list().len(), 1, "no repeated operator turn");
3116        d.question = Some("Recover from source B?".into());
3117        // The labels can remain identical while the approval target changes.
3118        d.question_decision.as_mut().unwrap().target = QuestionTarget::Run {
3119            id: "source-run-b".into(),
3120        };
3121        apply_one(&queue, &questions, &d, None).unwrap();
3122        assert_eq!(
3123            questions.list().len(),
3124            2,
3125            "new decision with unchanged task facts"
3126        );
3127        let after = queue.get(&t.id).unwrap();
3128        assert_eq!(after.instruction, t.instruction);
3129        assert_eq!(after.runs, t.runs);
3130        assert_eq!(after.status, TaskStatus::Blocked);
3131    }
3132
3133    #[test]
3134    fn a_second_conductor_question_is_still_allowed_after_one_settled_answer() {
3135        let dir = tempdir().unwrap();
3136        let queue = Queue::at(dir.path().join("queue"));
3137        let questions = Questions::at(dir.path().join("questions"));
3138        let mut t = task("asked about once already");
3139        t.hold_machine(Some("out of attempts".to_owned()));
3140        t.record_answer("Handle this one?".to_owned(), "not yet".to_owned());
3141        queue.put(&mut t).unwrap();
3142
3143        apply(
3144            &queue,
3145            &questions,
3146            &Verdict {
3147                decisions: vec![Decision {
3148                    id: t.id.clone(),
3149                    question: Some("Still not sure - now what?".to_owned()),
3150                    ..Decision::default()
3151                }],
3152            },
3153        )
3154        .unwrap();
3155
3156        assert_eq!(questions.list().len(), 1, "the second question was filed");
3157        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3158    }
3159
3160    #[test]
3161    fn a_held_task_with_an_open_triage_question_is_left_to_triage() {
3162        // `crate::triage` never calls `Task::block`, and only ever looks at
3163        // tasks still `held` - so a conductor question that moved this task
3164        // to `blocked` would orphan triage's own open question. The guard in
3165        // `apply_one` must leave the task alone until triage's question
3166        // settles.
3167        let dir = tempdir().unwrap();
3168        let queue = Queue::at(dir.path().join("queue"));
3169        let questions = Questions::at(dir.path().join("questions"));
3170        let mut t = task("held, triage already asking about it");
3171        t.hold_machine(Some("cause unclear".to_owned()));
3172        queue.put(&mut t).unwrap();
3173
3174        let mut triage_q = Question::new(
3175            t.id.clone(),
3176            crate::triage::NODE.to_owned(),
3177            "triage".to_owned(),
3178            "Still needed?".to_owned(),
3179            String::new(),
3180            vec![
3181                "resume".to_owned(),
3182                "not yet".to_owned(),
3183                "discard".to_owned(),
3184            ],
3185        );
3186        questions.put(&mut triage_q).unwrap();
3187
3188        for decision in [
3189            Decision {
3190                id: t.id.clone(),
3191                question: Some("what now?".to_owned()),
3192                ..Decision::default()
3193            },
3194            Decision {
3195                id: t.id.clone(),
3196                recovery: Some(Recovery::Requeue),
3197                ..Decision::default()
3198            },
3199        ] {
3200            apply(
3201                &queue,
3202                &questions,
3203                &Verdict {
3204                    decisions: vec![decision],
3205                },
3206            )
3207            .unwrap();
3208        }
3209
3210        let after = queue.get(&t.id).unwrap();
3211        assert_eq!(
3212            after.status,
3213            TaskStatus::Held,
3214            "triage still owns this hold"
3215        );
3216        assert!(after.blocked_by.is_empty());
3217        assert_eq!(
3218            questions.list().len(),
3219            1,
3220            "no second, conductor-owned question was filed"
3221        );
3222    }
3223
3224    #[test]
3225    fn a_held_task_with_an_answered_but_unapplied_triage_question_is_still_left_alone() {
3226        // The race `crate::triage::pending_for` exists to close: the
3227        // operator has already answered the triage question (it is no
3228        // longer `open`), but `crate::triage::run_once` - which only runs on
3229        // a fully idle daemon tick, far less often than the conductor polls
3230        // - has not had a turn to apply it yet. `apply_one` must not treat
3231        // "not open" as "settled" here, or it would block the task out from
3232        // under an answer triage has not read back yet.
3233        let dir = tempdir().unwrap();
3234        let queue = Queue::at(dir.path().join("queue"));
3235        let questions = Questions::at(dir.path().join("questions"));
3236        let mut t = task("held, triage question answered but not yet applied");
3237        t.hold_machine(Some("cause unclear".to_owned()));
3238        queue.put(&mut t).unwrap();
3239
3240        let mut triage_q = Question::new(
3241            t.id.clone(),
3242            crate::triage::NODE.to_owned(),
3243            "triage".to_owned(),
3244            "Still needed?".to_owned(),
3245            String::new(),
3246            vec![
3247                "resume".to_owned(),
3248                "not yet".to_owned(),
3249                "discard".to_owned(),
3250            ],
3251        );
3252        questions.put(&mut triage_q).unwrap();
3253        triage_q
3254            .answer(Answer::Choice("not yet".to_owned()))
3255            .unwrap();
3256        questions.put(&mut triage_q).unwrap();
3257        assert!(!triage_q.status.open());
3258
3259        apply(
3260            &queue,
3261            &questions,
3262            &Verdict {
3263                decisions: vec![Decision {
3264                    id: t.id.clone(),
3265                    question: Some("what now?".to_owned()),
3266                    ..Decision::default()
3267                }],
3268            },
3269        )
3270        .unwrap();
3271
3272        let after = queue.get(&t.id).unwrap();
3273        assert_eq!(
3274            after.status,
3275            TaskStatus::Held,
3276            "triage's own answer is not yet applied - conduct must wait"
3277        );
3278        assert_eq!(
3279            questions.list().len(),
3280            1,
3281            "no conductor question was filed over the pending triage answer"
3282        );
3283    }
3284
3285    #[test]
3286    fn manual_hold_rejects_hostile_or_stale_conductor_recovery() {
3287        let dir = tempdir().unwrap();
3288        let queue = Queue::at(dir.path().join("queue"));
3289        let questions = Questions::at(dir.path().join("questions"));
3290        let mut held = task("manual recovery");
3291        held.priority = 300;
3292        held.runs.push("run20260912-224242-daf5".to_owned());
3293        held.hold_manual(Some(
3294            "active manual recovery run20260912-224242-daf5".to_owned(),
3295        ));
3296        queue.put(&mut held).unwrap();
3297
3298        // Every field a conductor may use to alter lifecycle state is ignored:
3299        // requeue/review would dispatch duplicate work, hold could overwrite
3300        // evidence, and a question would turn the hold into `blocked`.
3301        for decision in [
3302            Decision {
3303                id: held.id.clone(),
3304                recovery: Some(Recovery::Requeue),
3305                ..Decision::default()
3306            },
3307            Decision {
3308                id: held.id.clone(),
3309                recovery: Some(Recovery::Hold),
3310                reason: Some("stale replacement reason".to_owned()),
3311                ..Decision::default()
3312            },
3313            Decision {
3314                id: held.id.clone(),
3315                recovery: Some(Recovery::Review),
3316                ..Decision::default()
3317            },
3318            Decision {
3319                id: held.id.clone(),
3320                blocked_by: vec!["other-task".to_owned()],
3321                question: Some("retry now?".to_owned()),
3322                ..Decision::default()
3323            },
3324        ] {
3325            apply(
3326                &queue,
3327                &questions,
3328                &Verdict {
3329                    decisions: vec![decision],
3330                },
3331            )
3332            .unwrap();
3333        }
3334
3335        let after = queue.get(&held.id).unwrap();
3336        assert_eq!(after.status, TaskStatus::Held);
3337        assert!(after.operator_held());
3338        assert_eq!(after.priority, 300);
3339        assert_eq!(after.runs, ["run20260912-224242-daf5"]);
3340        assert_eq!(
3341            after.hold_reason.as_deref(),
3342            Some("active manual recovery run20260912-224242-daf5")
3343        );
3344        assert!(after.blocked_by.is_empty());
3345        assert!(questions.list().is_empty());
3346        assert!(
3347            queue.next_runnable().is_none(),
3348            "must not dispatch a duplicate"
3349        );
3350    }
3351
3352    #[test]
3353    fn machine_holds_remain_recoverable_and_manual_release_is_authorization() {
3354        let dir = tempdir().unwrap();
3355        let queue = Queue::at(dir.path().join("queue"));
3356        let questions = Questions::at(dir.path().join("questions"));
3357
3358        let mut automatic = task("disk gate");
3359        automatic.hold_machine(Some("disk full".to_owned()));
3360        queue.put(&mut automatic).unwrap();
3361        let requeue = || Verdict {
3362            decisions: vec![Decision {
3363                id: automatic.id.clone(),
3364                recovery: Some(Recovery::Requeue),
3365                ..Decision::default()
3366            }],
3367        };
3368        apply(&queue, &questions, &requeue()).unwrap();
3369        assert_eq!(queue.get(&automatic.id).unwrap().status, TaskStatus::Queued);
3370
3371        let mut manual = task("operator gate");
3372        manual.hold_manual(Some("wait for operator".to_owned()));
3373        queue.put(&mut manual).unwrap();
3374        apply(
3375            &queue,
3376            &questions,
3377            &Verdict {
3378                decisions: vec![Decision {
3379                    id: manual.id.clone(),
3380                    recovery: Some(Recovery::Requeue),
3381                    ..Decision::default()
3382                }],
3383            },
3384        )
3385        .unwrap();
3386        assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Held);
3387
3388        // This mirrors the CLI and web release routes: only an explicit
3389        // operator action clears the manual boundary.
3390        let mut released = queue.get(&manual.id).unwrap();
3391        released.release();
3392        queue.put(&mut released).unwrap();
3393        assert_eq!(queue.get(&manual.id).unwrap().status, TaskStatus::Queued);
3394    }
3395
3396    #[test]
3397    fn legacy_reasoned_hold_is_protected_without_losing_its_metadata() {
3398        let dir = tempdir().unwrap();
3399        let queue = Queue::at(dir.path().join("queue"));
3400        let questions = Questions::at(dir.path().join("questions"));
3401        let mut legacy = task("old explicit hold");
3402        legacy.status = TaskStatus::Held;
3403        legacy.hold_reason = Some("manual recovery already active".to_owned());
3404        legacy.hold_source = None;
3405        legacy.blocked_by = vec!["dependency".to_owned()];
3406        queue.put(&mut legacy).unwrap();
3407
3408        apply(
3409            &queue,
3410            &questions,
3411            &Verdict {
3412                decisions: vec![Decision {
3413                    id: legacy.id.clone(),
3414                    recovery: Some(Recovery::Requeue),
3415                    ..Decision::default()
3416                }],
3417            },
3418        )
3419        .unwrap();
3420
3421        let after = queue.get(&legacy.id).unwrap();
3422        assert_eq!(after.status, TaskStatus::Held);
3423        assert_eq!(after.hold_source, None);
3424        assert_eq!(after.hold_reason, legacy.hold_reason);
3425        assert_eq!(after.blocked_by, legacy.blocked_by);
3426    }
3427
3428    #[test]
3429    fn review_recovery_is_a_no_op_without_a_survivable_branch() {
3430        // `surviving_branch` reaches `RunState::load`, which reaches the
3431        // process-global `run::home()` - a `OnceLock`, so this only wins the
3432        // race the first time it runs in the binary; every other test still
3433        // reaches the same directory whichever call won, and this test's own
3434        // run id never collides with another test's.
3435        crate::run::pin_test_home();
3436        let dir = tempdir().unwrap();
3437        let queue = Queue::at(dir.path().join("queue"));
3438        let questions = Questions::at(dir.path().join("questions"));
3439        let mut t = task("blocked with no readable run");
3440        t.start("20260101-000000-dead".to_owned()); // no such run on disk
3441        t.fail("blocked", 5);
3442        queue.put(&mut t).unwrap();
3443
3444        apply(
3445            &queue,
3446            &questions,
3447            &Verdict {
3448                decisions: vec![Decision {
3449                    id: t.id.clone(),
3450                    recovery: Some(Recovery::Review),
3451                    ..Decision::default()
3452                }],
3453            },
3454        )
3455        .unwrap();
3456
3457        let after = queue.get(&t.id).unwrap();
3458        assert_eq!(
3459            after.status,
3460            TaskStatus::Failed,
3461            "with nothing to reopen, the decision is dropped rather than guessed at"
3462        );
3463        assert!(after.review_branch.is_none());
3464    }
3465
3466    #[test]
3467    fn requeue_and_review_recovery_are_ignored_for_a_runnable_task() {
3468        // `Hold` is the one exception - see `Decision::recovery`'s doc and
3469        // `a_runnable_task_can_be_held_directly_without_a_question` - but a
3470        // task already in line has nothing for `requeue` or `review` to do.
3471        let dir = tempdir().unwrap();
3472        let queue = Queue::at(dir.path().join("queue"));
3473        let questions = Questions::at(dir.path().join("questions"));
3474
3475        for recovery in [Recovery::Requeue, Recovery::Review] {
3476            let mut t = task("ordinary");
3477            queue.put(&mut t).unwrap();
3478
3479            apply(
3480                &queue,
3481                &questions,
3482                &Verdict {
3483                    decisions: vec![Decision {
3484                        id: t.id.clone(),
3485                        recovery: Some(recovery),
3486                        ..Decision::default()
3487                    }],
3488                },
3489            )
3490            .unwrap();
3491
3492            assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3493        }
3494    }
3495
3496    #[tokio::test]
3497    async fn a_broken_agent_leaves_the_queue_untouched_and_does_not_error() {
3498        let dir = tempdir().unwrap();
3499        let cfg = config(mock_agent(dir.path(), BROKEN, BTreeMap::new()));
3500        let queue = Queue::at(dir.path().join("queue"));
3501        let questions = Questions::at(dir.path().join("questions"));
3502        let mut t = task("normal");
3503        queue.put(&mut t).unwrap();
3504
3505        let mut conductor = Conductor::new();
3506        conductor
3507            .maybe_run(
3508                &cfg,
3509                dir.path(),
3510                &queue,
3511                &questions,
3512                dir.path(),
3513                &[t.clone()],
3514                &[],
3515                &[],
3516                2,
3517            )
3518            .await;
3519
3520        assert_eq!(
3521            queue.get(&t.id).unwrap().status,
3522            TaskStatus::Queued,
3523            "a failed invocation must change nothing"
3524        );
3525        assert!(
3526            queue.next_runnable().is_some(),
3527            "the loop must still be able to take the next task"
3528        );
3529    }
3530
3531    #[tokio::test]
3532    async fn a_reply_with_no_json_leaves_the_queue_untouched() {
3533        let dir = tempdir().unwrap();
3534        let cfg = config(mock_agent(dir.path(), GARBAGE, BTreeMap::new()));
3535        let queue = Queue::at(dir.path().join("queue"));
3536        let questions = Questions::at(dir.path().join("questions"));
3537        let mut t = task("normal");
3538        queue.put(&mut t).unwrap();
3539
3540        let mut conductor = Conductor::new();
3541        conductor
3542            .maybe_run(
3543                &cfg,
3544                dir.path(),
3545                &queue,
3546                &questions,
3547                dir.path(),
3548                &[t.clone()],
3549                &[],
3550                &[],
3551                2,
3552            )
3553            .await;
3554
3555        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
3556    }
3557
3558    #[tokio::test]
3559    async fn a_conductor_chain_falls_to_the_next_agent_once_each() {
3560        let dir = tempdir().unwrap();
3561        let mut t = task("chained");
3562        let reply = format!(
3563            "{{\"decisions\":[{{\"id\":\"{}\",\"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}",
3564            t.id
3565        );
3566        let counted = |id: &str, tail: &str| {
3567            let calls = dir.path().join(format!("{id}.calls"));
3568            let path = dir.path().join(format!("{id}.sh"));
3569            std::fs::write(
3570                &path,
3571                format!(
3572                    "#!/bin/sh\ncat >/dev/null\necho x >> '{}'\n{tail}\n",
3573                    calls.display()
3574                ),
3575            )
3576            .unwrap();
3577            AgentSpec {
3578                id: id.to_owned(),
3579                kind: AgentKind::Command,
3580                model: None,
3581                command: vec!["sh".to_owned(), path.to_string_lossy().into_owned()],
3582                extra_args: Vec::new(),
3583                env: BTreeMap::new(),
3584                prompt_delivery: None,
3585                max_turn_secs: None,
3586            }
3587        };
3588        let n_calls = |id: &str| {
3589            std::fs::read_to_string(dir.path().join(format!("{id}.calls")))
3590                .map_or(0, |s| s.lines().count())
3591        };
3592        let mut cfg = config(counted("a", "exit 3"));
3593        cfg.agents = vec![
3594            counted("a", "exit 3"),
3595            counted("b", &format!("printf '%s' '{reply}'")),
3596        ];
3597        cfg.roles.conductor = Some(crate::config::AgentChoice::Chain(vec![
3598            "a".into(),
3599            "b".into(),
3600            "a".into(),
3601        ]));
3602        let queue = Queue::at(dir.path().join("queue"));
3603        let questions = Questions::at(dir.path().join("questions"));
3604        queue.put(&mut t).unwrap();
3605
3606        let mut conductor = Conductor::new();
3607        conductor
3608            .maybe_run(
3609                &cfg,
3610                dir.path(),
3611                &queue,
3612                &questions,
3613                dir.path(),
3614                &[t.clone()],
3615                &[],
3616                &[],
3617                2,
3618            )
3619            .await;
3620
3621        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
3622        assert_eq!(n_calls("a"), 1, "each id is tried once");
3623        assert_eq!(n_calls("b"), 1);
3624        let saved = std::fs::read_to_string(seat_path(dir.path())).unwrap();
3625        assert!(
3626            saved.contains("\"b\""),
3627            "the seat on disk is the one that ran: {saved}"
3628        );
3629    }
3630
3631    #[tokio::test]
3632    async fn json_survives_code_fences_and_a_preamble() {
3633        let dir = tempdir().unwrap();
3634        let mut t = task("fenced");
3635        let reply = format!(
3636            "Sure, here is my decision.\n\n```json\n{{\"decisions\":[{{\"id\":\"{}\",\
3637             \"blocked_by\":[\"x\"],\"reason\":\"why\"}}]}}\n```\n",
3638            t.id
3639        );
3640        let cfg = config(mock_agent(dir.path(), REPLY, env(&reply)));
3641        let queue = Queue::at(dir.path().join("queue"));
3642        let questions = Questions::at(dir.path().join("questions"));
3643        queue.put(&mut t).unwrap();
3644
3645        let mut conductor = Conductor::new();
3646        conductor
3647            .maybe_run(
3648                &cfg,
3649                dir.path(),
3650                &queue,
3651                &questions,
3652                dir.path(),
3653                &[t.clone()],
3654                &[],
3655                &[],
3656                2,
3657            )
3658            .await;
3659
3660        let back = queue.get(&t.id).unwrap();
3661        assert_eq!(back.status, TaskStatus::Blocked);
3662        assert_eq!(back.blocked_by, ["x"]);
3663    }
3664
3665    #[tokio::test]
3666    async fn the_conductor_is_not_called_again_when_nothing_worth_looking_at_has_changed() {
3667        // Each real invocation writes its own artifact stem, `turn-<n>`, so
3668        // whether a second one happened is read off the artifacts directory.
3669        let dir = tempdir().unwrap();
3670        let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3671        let queue = Queue::at(dir.path().join("queue"));
3672        let questions = Questions::at(dir.path().join("questions"));
3673        let mut t = task("stable");
3674        queue.put(&mut t).unwrap();
3675        let artifacts = dir.path().join("conduct").join("artifacts");
3676        let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3677
3678        let mut conductor = Conductor::new();
3679        conductor
3680            .maybe_run(
3681                &cfg,
3682                dir.path(),
3683                &queue,
3684                &questions,
3685                dir.path(),
3686                &[t.clone()],
3687                &[],
3688                &[],
3689                2,
3690            )
3691            .await;
3692        assert!(turn(1).is_file(), "the first cycle must call the conductor");
3693
3694        conductor
3695            .maybe_run(
3696                &cfg,
3697                dir.path(),
3698                &queue,
3699                &questions,
3700                dir.path(),
3701                &[t.clone()],
3702                &[],
3703                &[],
3704                2,
3705            )
3706            .await;
3707        assert!(
3708            !turn(2).is_file(),
3709            "an unchanged revision and an unchanged stalled/finished set must not call the \
3710             conductor twice"
3711        );
3712
3713        // Once the queue actually changes, the next `maybe_run` calls again.
3714        t.priority = 1;
3715        queue.put(&mut t).unwrap();
3716        conductor
3717            .maybe_run(
3718                &cfg,
3719                dir.path(),
3720                &queue,
3721                &questions,
3722                dir.path(),
3723                &[t.clone()],
3724                &[],
3725                &[],
3726                2,
3727            )
3728            .await;
3729        assert!(turn(2).is_file(), "a moved revision calls it again");
3730    }
3731
3732    #[tokio::test]
3733    async fn a_task_turning_stalled_calls_the_conductor_again_despite_an_unchanged_revision() {
3734        // The queue's own revision has not moved - nothing wrote to it - but
3735        // a task now looks stalled, purely because time passed. Calling
3736        // again here, and never again once this exact set has been shown
3737        // once, is the whole point of keying `worth_a_look` on the id set
3738        // rather than on "is it non-empty".
3739        let dir = tempdir().unwrap();
3740        let cfg = config(mock_agent(dir.path(), REPLY, env("{\"decisions\":[]}")));
3741        let queue = Queue::at(dir.path().join("queue"));
3742        let questions = Questions::at(dir.path().join("questions"));
3743        let mut t = task("quiet");
3744        queue.put(&mut t).unwrap();
3745        let artifacts = dir.path().join("conduct").join("artifacts");
3746        let turn = |n: usize| artifacts.join(format!("turn-{n}.out"));
3747
3748        let mut conductor = Conductor::new();
3749        conductor
3750            .maybe_run(
3751                &cfg,
3752                dir.path(),
3753                &queue,
3754                &questions,
3755                dir.path(),
3756                &[t.clone()],
3757                &[],
3758                &[],
3759                2,
3760            )
3761            .await;
3762        assert!(turn(1).is_file());
3763
3764        conductor
3765            .maybe_run(
3766                &cfg,
3767                dir.path(),
3768                &queue,
3769                &questions,
3770                dir.path(),
3771                &[],
3772                &[t.clone()],
3773                &[],
3774                2,
3775            )
3776            .await;
3777        assert!(
3778            turn(2).is_file(),
3779            "a task turning stalled must call the conductor again"
3780        );
3781
3782        // But once shown at this exact revision, showing the *same* stalled
3783        // set again must not call a third time.
3784        conductor
3785            .maybe_run(
3786                &cfg,
3787                dir.path(),
3788                &queue,
3789                &questions,
3790                dir.path(),
3791                &[],
3792                &[t.clone()],
3793                &[],
3794                2,
3795            )
3796            .await;
3797        assert!(
3798            !turn(3).is_file(),
3799            "the same stalled task lingering must not call the conductor every cycle"
3800        );
3801    }
3802
3803    fn rehold(queue: &Queue, questions: &Questions, id: &str, note: &str) {
3804        apply(
3805            queue,
3806            questions,
3807            &Verdict {
3808                decisions: vec![Decision {
3809                    id: id.to_owned(),
3810                    reason: Some(note.to_owned()),
3811                    recovery: Some(Recovery::Hold),
3812                    ..Decision::default()
3813                }],
3814            },
3815        )
3816        .unwrap();
3817    }
3818
3819    #[test]
3820    fn reholding_with_an_identical_note_changes_nothing() {
3821        let dir = tempdir().unwrap();
3822        let queue = Queue::at(dir.path().join("queue"));
3823        let questions = Questions::at(dir.path().join("questions"));
3824        let mut t = task("held");
3825        t.hold_machine(Some("waiting on a human".to_owned()));
3826        queue.put(&mut t).unwrap();
3827
3828        rehold(&queue, &questions, &t.id, "owner said keep it held");
3829        let first = queue.get(&t.id).unwrap();
3830        let rev = queue.revision();
3831        for _ in 0..5 {
3832            rehold(&queue, &questions, &t.id, "owner said keep it held");
3833        }
3834        let after = queue.get(&t.id).unwrap();
3835        assert_eq!(after.hold_reason, first.hold_reason);
3836        assert_eq!(after.updated_at, first.updated_at);
3837        assert_eq!(queue.revision(), rev, "nothing was written");
3838    }
3839
3840    #[test]
3841    fn reholding_with_different_notes_keeps_one_previously_level() {
3842        let dir = tempdir().unwrap();
3843        let queue = Queue::at(dir.path().join("queue"));
3844        let questions = Questions::at(dir.path().join("questions"));
3845        let mut t = task("held");
3846        t.hold_machine(Some("original".to_owned()));
3847        queue.put(&mut t).unwrap();
3848
3849        for i in 0..20 {
3850            rehold(&queue, &questions, &t.id, &format!("note {}", i % 2));
3851        }
3852        let reason = queue.get(&t.id).unwrap().hold_reason.unwrap();
3853        assert_eq!(reason.matches("(previously:").count(), 1, "{reason}");
3854        assert!(reason.starts_with("note "), "the new note leads: {reason}");
3855    }
3856
3857    #[test]
3858    fn an_already_nested_reason_is_matched_by_its_outermost_note() {
3859        let mut t = task("deep");
3860        let nested = "same\n\n(previously: same\n\n(previously: same))";
3861        t.hold_machine(Some(nested.to_owned()));
3862        let d = Decision {
3863            id: t.id.clone(),
3864            reason: Some("same".to_owned()),
3865            ..Decision::default()
3866        };
3867        assert_eq!(reaffirmed_hold_reason(&t, &d), None);
3868        let d2 = Decision {
3869            reason: Some("other".to_owned()),
3870            ..d
3871        };
3872        assert_eq!(
3873            reaffirmed_hold_reason(&t, &d2).as_deref(),
3874            Some("other\n\n(previously: same)")
3875        );
3876    }
3877
3878    #[tokio::test]
3879    async fn a_repeated_hold_decision_settles_instead_of_looping() {
3880        let dir = tempdir().unwrap();
3881        let queue = Queue::at(dir.path().join("queue"));
3882        let questions = Questions::at(dir.path().join("questions"));
3883        let reply = r#"{"decisions":[{"id":"ID","reason":"keep held","recovery":"hold"}]}"#;
3884        let mut t = task("held");
3885        t.hold_machine(Some("first".to_owned()));
3886        queue.put(&mut t).unwrap();
3887        let cfg = config(mock_agent(
3888            dir.path(),
3889            REPLY,
3890            env(&reply.replace("ID", &t.id)),
3891        ));
3892        let mut conductor = Conductor::new();
3893        // Cycle 1 writes the new note; cycle 2 repeats it and writes nothing.
3894        for _ in 0..2 {
3895            let held = queue.get(&t.id).unwrap();
3896            conductor
3897                .maybe_run(
3898                    &cfg,
3899                    dir.path(),
3900                    &queue,
3901                    &questions,
3902                    dir.path(),
3903                    &[],
3904                    &[],
3905                    &[held],
3906                    2,
3907                )
3908                .await;
3909        }
3910        let held = queue.get(&t.id).unwrap();
3911        assert!(
3912            held.hold_reason
3913                .as_deref()
3914                .unwrap()
3915                .starts_with("keep held")
3916        );
3917        assert!(
3918            !conductor.worth_a_look(&queue, &[], &[held]),
3919            "an unchanged held task must stop being reconsidered"
3920        );
3921    }
3922
3923    #[tokio::test]
3924    async fn alternating_wording_does_not_keep_a_held_task_in_the_loop() {
3925        let dir = tempdir().unwrap();
3926        let queue = Queue::at(dir.path().join("queue"));
3927        let questions = Questions::at(dir.path().join("questions"));
3928        let mut t = task("held");
3929        t.hold_machine(Some("first".to_owned()));
3930        queue.put(&mut t).unwrap();
3931        let reply = |note: &str| {
3932            format!(
3933                r#"{{"decisions":[{{"id":"{}","reason":"{note}","recovery":"hold"}}]}}"#,
3934                t.id
3935            )
3936        };
3937        let cfg_a = config(mock_agent(dir.path(), REPLY, env(&reply("keep held"))));
3938        let cfg_b = config(mock_agent(dir.path(), REPLY, env(&reply("leave held"))));
3939        let mut conductor = Conductor::new();
3940        let mut writes = 0;
3941        for i in 0..6 {
3942            let held = queue.get(&t.id).unwrap();
3943            let before = queue.revision();
3944            conductor
3945                .maybe_run(
3946                    if i % 2 == 0 { &cfg_a } else { &cfg_b },
3947                    dir.path(),
3948                    &queue,
3949                    &questions,
3950                    dir.path(),
3951                    &[],
3952                    &[],
3953                    &[held],
3954                    2,
3955                )
3956                .await;
3957            if queue.revision() != before {
3958                writes += 1;
3959            }
3960        }
3961        assert!(writes <= 2, "the loop must settle, saw {writes} writes");
3962        let mut held = queue.get(&t.id).unwrap();
3963        held.instruction.push_str(" (edited)");
3964        queue.put(&mut held).unwrap();
3965        let held = queue.get(&t.id).unwrap();
3966        assert!(conductor.worth_a_look(&queue, &[], &[held]));
3967    }
3968
3969    #[test]
3970    fn worth_a_look_is_config_free_and_matches_maybe_runs_own_gate() {
3971        let dir = tempdir().unwrap();
3972        let queue = Queue::at(dir.path().join("queue"));
3973        let mut t = task("t");
3974        queue.put(&mut t).unwrap();
3975
3976        let mut conductor = Conductor::new();
3977        assert!(
3978            conductor.worth_a_look(&queue, &[], &[]),
3979            "a conductor that has never run has something to look at"
3980        );
3981
3982        conductor.last_seen = Some(conductor.snapshot(&queue, &[], &[]));
3983        assert!(
3984            !conductor.worth_a_look(&queue, &[], &[]),
3985            "nothing changed and nothing is stalled or finished"
3986        );
3987        assert!(
3988            conductor.worth_a_look(&queue, &[t.clone()], &[]),
3989            "a stalled task is worth a look even at the same revision"
3990        );
3991        assert!(
3992            conductor.worth_a_look(&queue, &[], &[t.clone()]),
3993            "a finished task is worth a look even at the same revision"
3994        );
3995    }
3996
3997    #[tokio::test]
3998    async fn the_conduct_path_never_calls_ask_and_wait() {
3999        // Structural: grepping this module and `daemon.rs` for
4000        // `ask_and_wait` is the actual assertion this module's own doc
4001        // promises; this test exists so the promise has a name in the test
4002        // output too. `apply_one`'s question path uses `Questions::put`
4003        // exclusively.
4004        let dir = tempdir().unwrap();
4005        let queue = Queue::at(dir.path().join("queue"));
4006        let questions = Questions::at(dir.path().join("questions"));
4007        let mut t = task("asks without blocking");
4008        queue.put(&mut t).unwrap();
4009
4010        apply(
4011            &queue,
4012            &questions,
4013            &Verdict {
4014                decisions: vec![Decision {
4015                    id: t.id.clone(),
4016                    question: Some("ok?".to_owned()),
4017                    ..Decision::default()
4018                }],
4019            },
4020        )
4021        .unwrap();
4022        // Reaching here at all (no hang) is the assertion.
4023        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Blocked);
4024    }
4025
4026    #[test]
4027    fn a_pinned_resume_is_not_held_or_requeued_by_the_conductor() {
4028        let mut t = Task::new("t".into(), "t".into(), PathBuf::new(), Source::Human);
4029        t.resume_override = Some(crate::queue::OperatorResume {
4030            question_id: "q".into(),
4031            at: jiff::Timestamp::now(),
4032            conductor_rehold: None,
4033            forced: false,
4034            pinned_run: Some("run-1".into()),
4035        });
4036        assert!(pinned_resume(&t));
4037        assert!(!may_hold(&mut t, "waiting for magi resume"));
4038        assert!(
4039            t.resume_override
4040                .as_ref()
4041                .unwrap()
4042                .conductor_rehold
4043                .is_none(),
4044            "a refused hold is not recorded as an override"
4045        );
4046        t.resume_override = None;
4047        assert!(!pinned_resume(&t));
4048        assert!(may_hold(&mut t, "no override, so a hold is allowed"));
4049    }
4050
4051    #[test]
4052    fn a_pinned_resume_is_not_blocked_by_a_conductor_question_or_dependency() {
4053        let dir = tempdir().unwrap();
4054        let queue = Queue::at(dir.path().join("queue"));
4055        let questions = Questions::at(dir.path().join("questions"));
4056        let mut t = task("resume me");
4057        t.resume_override = Some(crate::queue::OperatorResume {
4058            question_id: "q".into(),
4059            at: jiff::Timestamp::now(),
4060            conductor_rehold: None,
4061            forced: true,
4062            pinned_run: Some("run-1".into()),
4063        });
4064        queue.put(&mut t).unwrap();
4065
4066        apply(
4067            &queue,
4068            &questions,
4069            &Verdict {
4070                decisions: vec![
4071                    Decision {
4072                        id: t.id.clone(),
4073                        question: Some("really?".to_owned()),
4074                        ..Decision::default()
4075                    },
4076                    Decision {
4077                        id: t.id.clone(),
4078                        blocked_by: vec!["other".to_owned()],
4079                        ..Decision::default()
4080                    },
4081                ],
4082            },
4083        )
4084        .unwrap();
4085
4086        assert_eq!(queue.get(&t.id).unwrap().status, TaskStatus::Queued);
4087        assert!(questions.list().is_empty());
4088    }
4089}