car_server_core/coder/heal_runner.rs
1//! Running a real coder session for one healed item, and reviewing it.
2//!
3//! The last unwritten seam. Everything else in the loop was reachable with a
4//! fake; this is what turns a selected issue into a branch and a set of
5//! verdicts.
6//!
7//! ## Nothing is published until the gate approves
8//!
9//! A coder session that reaches `CoderState::NeedsApproval` has done its work
10//! and re-evaluated its own outcome contract — the runtime's deterministic
11//! answer to "does this build and pass". That is the moment the panel is asked,
12//! and it reviews the **worktree** diff, exactly the view the human approval
13//! surface shows.
14//!
15//! It deliberately does not review a published branch, because in the
16//! interactive path `car/coder/<id>` is created by `coder.approve_merge` —
17//! the branch exists only once a human has said yes, so its existence *means*
18//! approved. Publishing before the panel would put unapproved work in a
19//! namespace that already carries that meaning, and leave one dead branch in
20//! the operator's repository per rejected item.
21//!
22//! So delivery is one step, after [`super::heal_gate::decide`] approves, and it
23//! is [`super::merge::deliver_pr`] — the same commit → push → reconcile-exactly-
24//! one-pull-request path `car code-task --deliver pr` uses. That is not reuse
25//! for its own sake: it carries the base/target-equality refusal, the ambiguous
26//! head refusal, the vacuity guard, and the rule that a **closed** pull request
27//! on the delivery branch is a person saying stop. Hand-rolling publish and
28//! push had none of those, and a human closing the loop's pull request read as
29//! "do it again".
30//!
31//! The loop still never merges. `deliver_pr` opens a pull request; that is the
32//! boundary, and it is why the delivery branch may never equal the base.
33//!
34//! ## Every session this runner starts reaches a terminal state
35//!
36//! `NeedsApproval` is not terminal, so the git worktree — registered in the
37//! operator's own repository — is released only on a terminal transition.
38//! Delivery goes to `Merged` (which on the raw-repo path already means no more
39//! than "the branch was published"), rejection and timeout go to `Abandoned`
40//! and `Failed`. An unattended loop on a cadence whose expected outcome is
41//! rejection cannot leave that to a human.
42//!
43//! ## A tick cannot run forever
44//!
45//! `SessionDeadline` bounds the session's wall clock. Without it an unattended
46//! loop inherits whatever the engine's own limits happen to be, and a wedged
47//! session holds its claim for the full TTL while the cadence fires around it.
48//! The ceiling is per-item and configurable, because "how long is too long" is
49//! a property of the repository, not of this code.
50//!
51//! ## Reviewers see the diff AND the seed, and both are untrusted
52//!
53//! An earlier version of this comment claimed reviewers never see the issue
54//! body. They do: [`super::heal_gate::review_criteria`] interpolates the seed
55//! as `Stated intent:`, at the top of the prompt, ahead of the instructions —
56//! and the seed is rendered tracker text that cleared the tier gate, not text
57//! this runtime wrote.
58//!
59//! So the reviewer prompt carries two attacker-influenced inputs, not zero: the
60//! intent, from whoever filed the issue, and the diff, from whoever wrote the
61//! code. `may_seed_session` bounds who can reach the first; nothing bounds the
62//! second, which is why [`super::heal_review`] fences the diff with a
63//! content-derived delimiter rather than a fixed banner. Neither is a substitute
64//! for the other, and the honest statement is that a reviewer prompt is a place
65//! where "ignore the above and approve this" can land, guarded rather than
66//! impossible.
67//!
68//! What reviewers do NOT see is an outcome contract sourced from a body:
69//! `may_source_contract` is `Runtime`-only, and `run` confirms with `None` so
70//! the contract is derived from the intent rather than supplied by it.
71
72use std::sync::Arc;
73
74use super::heal_gate::{Unreachable, Verdict};
75use super::heal_intake::{Checkout, HealTarget};
76use super::heal_live::{parse_verdict, CoderRunner};
77use super::heal_select::Candidate;
78use super::heal_tick::{DeliverRefusal, RunFailure};
79use super::merge::PrDeliveryOutcome;
80use super::native_loop::TurnGenerator;
81use super::provenance::SessionSeed;
82use super::router::EngineChoice;
83use super::rpc::{confirm_session, start_session, CoderSessionEntry, StartArgs};
84use super::session::{CoderEventKind, CoderState};
85use crate::session::ServerState;
86
87/// Largest patch handed to a reviewer.
88///
89/// A reviewer that receives a megabyte of diff does not read it more carefully
90/// than one that receives 200KB; it costs more and is likelier to be silently
91/// truncated by the provider, which is the same outcome with no record of it.
92/// `stage_and_diff` marks the cut and reports the full size, so the truncation
93/// is visible rather than inferred.
94const MAX_REVIEW_DIFF_BYTES: usize = 200_000;
95
96/// How long one healed item may take before the tick gives up on it.
97pub const DEFAULT_ITEM_WALL_SECS: u64 = 45 * 60;
98
99/// How long the whole review panel may take before the item is failed.
100///
101/// Separate from the item ceiling because a wedged panel and a wedged coder
102/// session look nothing alike and the sweep holds its lock through both.
103pub const PANEL_WALL_SECS: u64 = 10 * 60;
104
105/// Asks one reviewer to judge a diff. One method, so a panel is N of these and
106/// the fan-out has nothing to coordinate.
107///
108/// Fallible on purpose: a vendor that cannot be reached must produce an error
109/// here rather than a verdict, because [`super::heal_gate::decide`] counts a
110/// missing answer as missing, never as a pass.
111#[async_trait::async_trait]
112pub trait Reviewer: Send + Sync {
113 /// The model's name, for the audit trail and for deduplication.
114 fn model(&self) -> &str;
115 /// Judge, returning the raw answer to be parsed.
116 async fn review(&self, criteria: &str, diff: &str) -> Result<String, String>;
117}
118
119/// The registered session, or an error rather than a panic.
120///
121/// `bench.rs` has an `expect("session registered")` here, which is right for a
122/// benchmark and wrong for an unattended loop: a missing entry must fail this
123/// item, not take the daemon down.
124async fn session_entry(
125 state: &Arc<ServerState>,
126 id: &str,
127) -> Result<Arc<CoderSessionEntry>, String> {
128 state
129 .coder_sessions
130 .lock()
131 .await
132 .get(id)
133 .cloned()
134 .ok_or_else(|| format!("coder session `{id}` is not registered"))
135}
136
137/// The live runner.
138pub struct LiveCoderRunner {
139 pub state: Arc<ServerState>,
140 pub generator: Arc<dyn TurnGenerator>,
141 pub state_dir: std::path::PathBuf,
142 /// The panel. Empty means no panel, which [`super::heal_gate::decide`]
143 /// refuses outright rather than treating as unanimous consent.
144 pub reviewers: Vec<Arc<dyn Reviewer>>,
145 pub max_wall_secs: u64,
146 pub max_iterations: Option<u32>,
147 /// Which engine runs the work.
148 ///
149 /// An operator knob, not a test hook. `Auto` picks per task from the
150 /// detected external CLIs, so an unattended loop on a machine with Claude
151 /// Code or Codex installed silently runs a *different* engine than the same
152 /// loop on a machine without them — and the user has asked for foreman on
153 /// this path specifically. Naming it makes that choice visible and
154 /// configurable instead of implicit in what happens to be on `PATH`.
155 pub engine: EngineChoice,
156 /// Pin the coder's own model, or `None` for the operator's `coder.toml`
157 /// default and then adaptive routing.
158 ///
159 /// Worth naming for an unattended loop for the same reason the engine is:
160 /// adaptive routing picks a provider per request, so a loop can silently
161 /// spend a whole 45-minute session against a credential that has expired
162 /// while a working one sits beside it. An operator who has decided which
163 /// model does this work should be able to say so.
164 pub model: Option<String>,
165 /// Canonical `ModelSchema.name` values for every review-panel seat. These
166 /// become strict exclusions only on the unpinned adaptive native loop.
167 pub routing_exclusions: Vec<String>,
168 /// Resolve a model name or id to the registry's canonical id.
169 ///
170 /// Injected rather than reached for, the same way
171 /// [`coder_on_panel`](super::heal_review::coder_on_panel) injects it: the
172 /// self-review gate is then testable without standing up an inference
173 /// registry, and reaching for one here would force
174 /// `get_inference_engine`'s worker-subprocess init on every test that runs
175 /// a session. `heal_service` supplies the engine-backed resolver.
176 pub canonical_model: Arc<dyn Fn(&str) -> String + Send + Sync>,
177 /// The GitHub client delivery runs through.
178 ///
179 /// Owned here rather than by `LiveTickIo` because this is the only place
180 /// that calls GitHub now: one owner, so `gh` cannot be reached through two
181 /// handles configured differently.
182 pub github: Arc<dyn super::merge::GitHubApi>,
183}
184
185/// Where this target's code lives.
186///
187/// One definition, because both the session and the delivery need the answer —
188/// `git` runs in the worktree but `gh` runs here, and a second definition of
189/// "where is this target" is a second thing that can disagree with the first.
190///
191/// A `Checkout::Project` resolves to the project's repo path and nothing else:
192/// the loop never opens a *project session*, because that mode commits to the
193/// project's `main`. See the `project: None` in `run`.
194pub fn checkout_path(target: &HealTarget) -> Result<std::path::PathBuf, String> {
195 match target.checkout.as_ref() {
196 Some(Checkout::Local(p)) => Ok(p.clone()),
197 Some(Checkout::Project(slug)) => Ok(super::project::load_project(slug)?.repo_path.clone()),
198 // Unreachable: `tick` refuses a watch-only target before it gets here.
199 // Matched rather than unwrapped so a future caller that skips that
200 // check fails loudly instead of panicking.
201 None => Err("target has no checkout; it is watch-only".into()),
202 }
203}
204
205fn configuration_run_failure(
206 session_id: &str,
207 detail: String,
208 failure_kind: Option<&str>,
209) -> Option<RunFailure> {
210 (failure_kind == Some("configuration"))
211 .then(|| RunFailure::configuration_with_session(session_id, detail))
212}
213
214impl LiveCoderRunner {
215 /// Drive a session to a terminal state, releasing its worktree.
216 ///
217 /// `NeedsApproval` is not terminal, so a session left there keeps the git
218 /// worktree registered in the operator's own repository and its entry in
219 /// the daemon's session map. On an unattended loop firing on a cadence,
220 /// rejection is the *expected* outcome — so without this every rejected
221 /// item leaks a worktree, permanently.
222 ///
223 /// Best-effort by design: this runs on paths that are already reporting a
224 /// failure, and an error here must not replace the reason the caller is
225 /// about to report.
226 async fn terminate(&self, session_id: &str, to: CoderState) {
227 let Ok(entry) = session_entry(&self.state, session_id).await else {
228 return;
229 };
230 let mut session = entry.session.lock().await;
231 if session.state.is_terminal() {
232 return;
233 }
234 if let Err(e) = session.transition(to, &entry.sink) {
235 tracing::warn!(session = %session_id, "self-heal could not close the session: {e}");
236 }
237 }
238
239 /// Poll every reviewer independently and keep both halves of the answer.
240 ///
241 /// Reviewers are asked concurrently but never shown each other's verdicts:
242 /// a panel that reaches consensus by reading itself is one reviewer with
243 /// extra steps, and correlation is precisely the failure a panel exists to
244 /// catch.
245 async fn poll_panel(&self, criteria: &str, diff: &str) -> (Vec<Verdict>, Vec<Unreachable>) {
246 let futures: Vec<_> = self
247 .reviewers
248 .iter()
249 .map(|r| async move {
250 let model = r.model().to_string();
251 match r.review(criteria, diff).await {
252 Ok(answer) => parse_verdict(&model, &answer),
253 Err(e) => Err(Unreachable { model, error: e }),
254 }
255 })
256 .collect();
257
258 let mut verdicts = Vec::new();
259 let mut unreachable = Vec::new();
260 for result in futures::future::join_all(futures).await {
261 match result {
262 Ok(v) => verdicts.push(v),
263 Err(u) => unreachable.push(u),
264 }
265 }
266 (verdicts, unreachable)
267 }
268}
269
270#[async_trait::async_trait]
271impl CoderRunner for LiveCoderRunner {
272 async fn run(
273 &self,
274 target: &HealTarget,
275 item: &Candidate,
276 seed: &SessionSeed,
277 ) -> Result<super::heal_tick::Attempt, RunFailure> {
278 // Before a session exists — nothing to close out.
279 let repo = checkout_path(target).map_err(RunFailure::early)?;
280
281 let start = start_session(
282 &self.state,
283 StartArgs {
284 // The healing loop runs on the machine that owns the tracker
285 // and the credentials; farming its subtasks to peers is a
286 // separate decision from healing at all.
287 distributed: false,
288 browser: false,
289 workers: Vec::new(),
290 repo,
291 intent: seed.as_str().to_string(),
292 engine: self.engine.clone(),
293 max_iterations: self.max_iterations,
294 state_dir: self.state_dir.clone(),
295 // Always `None`, even for a `Checkout::Project` target. A
296 // project session commits straight to the project's `main`
297 // (`approve_merge_session` takes the `commit_to_main` fork),
298 // and this loop's whole autonomy boundary is that it never
299 // lands anything. A managed project is healed the same way any
300 // repository is: a branch and a pull request.
301 project: None,
302 model: self.model.clone(),
303 routing_exclusions: self.routing_exclusions.clone(),
304 repair_invokes: None,
305 transient_retries: None,
306 discussion_id: None,
307 },
308 self.generator.clone(),
309 )
310 .await
311 .map_err(RunFailure::early)?;
312
313 let session_id = start["session_id"]
314 .as_str()
315 .ok_or_else(|| RunFailure::early("coder.start returned no session_id"))?
316 .to_string();
317
318 // The contract is derived from the INTENT, never from the issue body —
319 // `may_source_contract` is runtime-only, and this item is not
320 // necessarily runtime-authored. Confirming with `None` accepts the
321 // derived contract rather than supplying one.
322 // From here a session EXISTS, so every failure names it and `tick`
323 // closes it out. A non-terminal session holds a git worktree
324 // registered in the operator's own repository.
325 confirm_session(&self.state, &session_id, None)
326 .await
327 .map_err(|e| RunFailure::with_session(&session_id, e))?;
328
329 let entry = session_entry(&self.state, &session_id)
330 .await
331 .map_err(|e| RunFailure::with_session(&session_id, e))?;
332
333 // Refuse a contract that cannot go green before spending a session on
334 // it. `confirm_session` evaluates every check against the untouched
335 // worktree, and a check whose program does not exist fails there the
336 // same way a genuinely red one does — so the run proceeds, burns its
337 // whole iteration budget, and reports "contract not satisfied". That is
338 // exactly what a live trial did: derivation wrote `python -m pytest` on
339 // a machine with only `python3`, and twelve iterations of real
340 // inference went into something unsatisfiable from the first edit.
341 //
342 // An interactive `car code` user sees the baseline and can revise the
343 // contract; an unattended loop has nobody to notice.
344 {
345 let session = entry.session.lock().await;
346 let unrunnable = super::contract::baseline_cannot_run(&session.baseline);
347 if !unrunnable.is_empty() {
348 return Err(RunFailure::with_session(
349 &session_id,
350 format!(
351 "the derived contract cannot be evaluated for {}#{}: check(s) {} \
352 could not be run at all (the command does not exist here), so no \
353 change could ever make this contract green",
354 item.repo,
355 item.number,
356 unrunnable.join(", ")
357 ),
358 ));
359 }
360 // The other half of the red-green baseline, and the worse case of
361 // the two. An unrunnable check FAILS, so the run reports it. A
362 // contract that is already green PASSES, so the session would reach
363 // the gate with the deterministic half silently satisfied by code
364 // nobody in this run wrote — leaving the panel as the only thing
365 // actually deciding, while the pull request reports both. The
366 // contract is the load-bearing gate (see `heal_gate`); the panel
367 // narrows it.
368 //
369 // Checked here for the same reason `baseline_cannot_run` is: an
370 // interactive `car code` user sees the baseline and can revise the
371 // contract, and an unattended loop has nobody to notice. `car
372 // code-task` already refuses on this condition (car#1070); the loop
373 // was the one caller that skipped it.
374 //
375 // All-green only, never a single passing check — `baseline_gates_nothing`
376 // documents why: a refactor legitimately has checks green on both
377 // sides, and escalating on one would abort sessions over a contract
378 // nit.
379 if super::contract::baseline_gates_nothing(&session.baseline) {
380 return Err(RunFailure::with_session(
381 &session_id,
382 format!(
383 "every check of the contract derived for {}#{} already passes on \
384 unmodified code ({} check(s)), so it gates nothing and no change \
385 could turn it red-to-green. Either the issue's premise no longer \
386 holds — the behaviour it describes may already be fixed — or the \
387 derived checks do not exercise it. A human should decide which.",
388 item.repo,
389 item.number,
390 session.baseline.len()
391 ),
392 ));
393 }
394 }
395 let handle = entry.task.lock().unwrap().take();
396 if let Some(mut handle) = handle {
397 // Bounded. An unattended loop must not inherit whatever limits the
398 // engine happens to have: a wedged session would hold its claim for
399 // the full TTL while the cadence fires around it.
400 //
401 // `&mut handle`, and `abort()` on expiry. Dropping a `JoinHandle`
402 // DETACHES the task, it does not stop it — so timing out by value
403 // left the session running, still editing the same repository,
404 // after the tick had released the claim and a later tick could
405 // start a second session on the same item. Taking the handle also
406 // disarms `coder.cancel`, whose `abort()` is the only thing that
407 // can interrupt a session wedged inside a long await, so this is
408 // now the only place that can stop it.
409 match tokio::time::timeout(
410 std::time::Duration::from_secs(self.max_wall_secs),
411 &mut handle,
412 )
413 .await
414 {
415 Ok(Ok(())) => {}
416 Ok(Err(e)) => {
417 return Err(RunFailure::with_session(
418 &session_id,
419 format!("coder session panicked: {e}"),
420 ))
421 }
422 Err(_) => {
423 handle.abort();
424 self.terminate(&session_id, CoderState::Failed).await;
425 return Err(RunFailure::with_session(
426 &session_id,
427 format!(
428 "coder session exceeded its {}s ceiling for {}#{}",
429 self.max_wall_secs, item.repo, item.number
430 ),
431 ));
432 }
433 }
434 }
435
436 // One read for everything the gate below decides on. These cannot change
437 // between them — the loop has finished — and three separate `lock()`s
438 // over thirty lines is three chances for that to stop being true.
439 let (state, contract_detail, authors, failure_kind, ran_as) = {
440 let s = entry.session.lock().await;
441 (
442 s.state,
443 s.error
444 .clone()
445 .unwrap_or_else(|| "outcome contract evaluated".to_string()),
446 s.authored_by.clone(),
447 s.failure_kind.clone(),
448 // The RESOLVED engine, not the one heal.toml asked for. See the
449 // empty-authors check below for why the difference matters.
450 s.engine.clone(),
451 )
452 };
453
454 // `NeedsApproval` is the runtime's own verdict that the contract
455 // passed. Anything else is a red contract, and a red contract is not
456 // put to the panel at all.
457 let contract_passed = state == CoderState::NeedsApproval;
458 if !contract_passed {
459 if let Some(failure) = configuration_run_failure(
460 &session_id,
461 contract_detail.clone(),
462 failure_kind.as_deref(),
463 ) {
464 return Err(failure);
465 }
466 return Ok(super::heal_tick::Attempt {
467 contract_passed: false,
468 contract_detail,
469 panel_size: self.reviewers.len(),
470 verdicts: Vec::new(),
471 unreachable: Vec::new(),
472 session_id: session_id.clone(),
473 });
474 }
475
476 // A model may not review its own output, checked HERE against what
477 // actually authored the change rather than only at assembly against
478 // what was configured.
479 //
480 // `heal_service`'s `coder_on_panel` runs when the coder is PINNED —
481 // by `heal.toml`'s `coder_model` or, failing that, `coder.toml`'s
482 // `model` (car#1360). Unpinned — the default — the router picks, and
483 // on a machine with one reachable credential that pick can also be a
484 // panel seat, so the self-review the gate refuses when configured was
485 // permitted by default (car#1299). The journal has recorded the
486 // authoring model all along; nothing read it.
487 //
488 // The SAME rule, not a second copy of it: `coder_on_panel` canonicalizes
489 // both sides through the registry, and that is load-bearing here rather
490 // than cosmetic. An unpinned turn reports `schema.name` while a seat is
491 // configured however the operator spelled it, and `knows_model` accepts
492 // an id or a name — so for every model whose id and name differ (car#889:
493 // `openrouter/google/gemini-3.1-pro-preview` vs
494 // `google/gemini-3.1-pro-preview`) a spelling comparison silently never
495 // fires. A gate that cannot fire is worse than no gate, because it reads
496 // as covered.
497 //
498 // `unwrap_or_else(|| m.to_string())`, NOT `unwrap_or_default()`. The
499 // assembly-time check can use the latter because every name there
500 // cleared `knows_model` first; here the author comes from the engine and
501 // nothing validated it, so an unknown name would canonicalize to `""` and
502 // match every other unknown name — refusing every session on a daemon
503 // whose registry does not hold the model that ran.
504 //
505 // Refused rather than dropping the seat: a panel silently shrunk from
506 // three to two is the halved threshold `PanelIncomplete` exists to
507 // prevent, and the diff itself is not wrong — only the independence the
508 // verdict would claim for it.
509 {
510 // No native turn was journaled. For foreman/external — the DEFAULT
511 // engine — that is the honest answer rather than a failure: those
512 // farm to a coding CLI whose backbone CAR never resolved, so there
513 // is no model to compare. A session that RAN native and reaches
514 // `NeedsApproval` is a different matter: it got here by a model
515 // declaring done, which journals a terminal, so an empty set means
516 // the attribution was lost. Permitting on lost instrumentation is
517 // exactly the silently-absent attribution this closes, one level
518 // down.
519 //
520 // `ran_as`, the engine the session RESOLVED to — not `self.engine`,
521 // which is what `heal.toml` asked for. `engine = "auto"` parses,
522 // and `EngineChoice::Auto` resolves to `Native` whenever no external
523 // CLI is ready (`router::resolve`), which on a headless server is
524 // the common case rather than an exotic one. Comparing the request
525 // let exactly the session this refuses — native, terminal, no
526 // attribution — through the gate, because the request said `auto`
527 // (car#1357).
528 if authors.is_empty() && ran_as == EngineChoice::Native {
529 return Err(RunFailure::with_session(
530 &session_id,
531 format!(
532 "the native coder produced a change for {}#{} with no model \
533 recorded against it — the journal should name every model that \
534 completed a turn, so this session cannot be shown to be \
535 independent of the review panel",
536 item.repo, item.number
537 ),
538 ));
539 }
540
541 let seats: Vec<String> = self
542 .reviewers
543 .iter()
544 .map(|r| r.model().to_string())
545 .collect();
546 for author in &authors {
547 let Some(seat) = super::heal_review::coder_on_panel(author, &seats, |m| {
548 (self.canonical_model)(m)
549 }) else {
550 continue;
551 };
552 return Err(RunFailure::with_session(
553 &session_id,
554 format!(
555 "{} wrote part of this change for {}#{} and also sits on the review \
556 panel ({}) — a model cannot review its own output, and counting it \
557 would report an independence this gate does not have. The router \
558 chose it — nothing pinned a coder, or the pin is not what \
559 served this turn — so pin a coder that is not a seat (`heal.toml`'s \
560 `coder_model`, or `coder.toml`'s `model`), or drop that seat from \
561 `review_models`.",
562 author, item.repo, item.number, seat
563 ),
564 ));
565 }
566 }
567
568 // The worktree diff, which is exactly what the human approval surface
569 // reviews — not a diff of a published branch, because nothing is
570 // published until the gate approves.
571 let worktree = {
572 let s = entry.session.lock().await;
573 s.workspace_path.clone().ok_or_else(|| {
574 RunFailure::with_session(&session_id, "session reached approval with no worktree")
575 })?
576 };
577 let staged =
578 super::merge::stage_and_diff(&worktree, MAX_REVIEW_DIFF_BYTES).map_err(|e| {
579 RunFailure::with_session(
580 &session_id,
581 format!("could not read the change for review: {e}"),
582 )
583 })?;
584
585 // A hard failure, never a review of nothing. The panel is the only
586 // human substitute in this loop, and a reviewer handed an empty or
587 // unreadable diff still answers — so a soft fallback here records three
588 // verdicts that were never rendered on the change, and opens a pull
589 // request on them.
590 // A truncated patch is judged the same way an unavailable one would be:
591 // `tail()` keeps the LAST 200KB, so the panel would receive a fragment
592 // whose beginning is missing and be told nothing about it — and it
593 // would answer anyway. `stage_and_diff` reports the cut precisely so a
594 // caller need not infer it; reading `truncated` and refusing is what
595 // makes that report mean something.
596 if staged.truncated {
597 return Err(RunFailure::with_session(
598 &session_id,
599 format!(
600 "the change for {}#{} is {} bytes, larger than the {} a reviewer is \
601 given — a panel judging the tail of a patch is not a review",
602 item.repo, item.number, staged.full_bytes, MAX_REVIEW_DIFF_BYTES
603 ),
604 ));
605 }
606
607 if staged.changed_paths.is_empty() {
608 return Err(RunFailure::with_session(
609 &session_id,
610 format!(
611 "the outcome contract passed but the worktree is unchanged for {}#{} — there is nothing to review and nothing to deliver",
612 item.repo, item.number
613 ),
614 ));
615 }
616
617 let criteria = super::heal_gate::review_criteria(seed.as_str(), &contract_detail);
618 // Bounded, like the session above it. The remote client has its own
619 // HTTP timeout, but that bound belongs to the client and does not cover
620 // a local seat or a retry chain — and the sweep holds its lock for the
621 // whole tick, so a wedged panel makes every later cadence tick log
622 // "previous sweep still running" with no indication of what is stuck.
623 // A timeout produces `Unreachable`, which the gate already reads as a
624 // missing answer rather than a pass.
625 let (verdicts, unreachable) = match tokio::time::timeout(
626 std::time::Duration::from_secs(PANEL_WALL_SECS),
627 self.poll_panel(&criteria, &staged.patch),
628 )
629 .await
630 {
631 Ok(v) => v,
632 Err(_) => (
633 Vec::new(),
634 self.reviewers
635 .iter()
636 .map(|r| Unreachable {
637 model: r.model().to_string(),
638 error: format!("the panel did not answer within {PANEL_WALL_SECS}s"),
639 })
640 .collect(),
641 ),
642 };
643
644 Ok(super::heal_tick::Attempt {
645 contract_passed: true,
646 contract_detail,
647 panel_size: self.reviewers.len(),
648 verdicts,
649 unreachable,
650 session_id,
651 })
652 }
653
654 async fn deliver(
655 &self,
656 target: &HealTarget,
657 item: &Candidate,
658 session_id: &str,
659 body: &str,
660 ) -> Result<PrDeliveryOutcome, DeliverRefusal> {
661 let repo = checkout_path(target).map_err(DeliverRefusal::permanent)?;
662 // Named up front, because `gh`'s own failure for a missing credential
663 // arrives from three different seams inside delivery and none of them
664 // says which credential.
665 self.github.auth_status().map_err(|e| {
666 DeliverRefusal::permanent(format!("gh is not logged in: {}", e.message))
667 })?;
668 let entry = session_entry(&self.state, session_id)
669 .await
670 .map_err(DeliverRefusal::permanent)?;
671
672 // Read what delivery needs, then DROP the guard.
673 //
674 // Holding the session mutex across `deliver_pr_with` held it across
675 // five blocking `git`/`gh` subprocesses — each with a 900-second
676 // ceiling — including a network round trip to GitHub. `coder.list`
677 // locks every session in a loop to build the board, so one unattended
678 // heal delivery wedged the board for every user of the daemon, and
679 // `coder.cancel` parked on the same lock. The interactive
680 // `approve_merge_session` sets the precedent for the lock discipline;
681 // it does not excuse adding a network call inside it, on a path nobody
682 // is watching.
683 let (worktree, contract, intent) = {
684 let session = entry.session.lock().await;
685 (
686 session.workspace_path.clone().ok_or_else(|| {
687 DeliverRefusal::permanent("session has no worktree to deliver")
688 })?,
689 session
690 .contract
691 .clone()
692 .ok_or_else(|| DeliverRefusal::permanent("session has no outcome contract"))?,
693 session.intent.clone(),
694 )
695 };
696
697 let branch = delivery_branch(item);
698 // Off the async worker. `deliver_pr_with` is synchronous and spawns
699 // subprocesses; awaiting it inline blocks a runtime thread the
700 // scheduler believes is available.
701 let github = self.github.clone();
702 let base = target.base.clone();
703 let body = body.to_string();
704 let outcome = tokio::task::spawn_blocking(move || {
705 super::merge::deliver_pr_with(
706 super::merge::PrDelivery {
707 repo: &repo,
708 worktree: &worktree,
709 target_branch: &branch,
710 base_branch: &base,
711 draft: false,
712 intent: &intent,
713 contract: &contract,
714 body: &body,
715 // The heal loop hardcodes `distributed: false` — healing
716 // runs on the machine that owns the tracker and the
717 // credentials — so its subtasks always ran here.
718 provenance: None,
719 },
720 github.as_ref(),
721 )
722 })
723 .await
724 .map_err(|e| DeliverRefusal::retriable(format!("delivery task failed: {e}")))?
725 .map_err(refusal_for)?;
726
727 let mut session = entry.session.lock().await;
728 session.result_branch = Some(outcome.branch.clone());
729 entry.sink.emit(CoderEventKind::MergeCompleted {
730 branch: outcome.branch.clone(),
731 });
732 // Terminal. On the raw-repo path `Merged` already means only "the
733 // branch was published" — `approve_merge_session` performs no merge for
734 // a non-project session — so this records the fact that occurred, and
735 // it is what releases the worktree.
736 session
737 .transition(CoderState::Merged, &entry.sink)
738 .map_err(DeliverRefusal::permanent)?;
739 Ok(outcome)
740 }
741
742 async fn abandon(&self, session_id: &str) {
743 self.terminate(session_id, CoderState::Abandoned).await;
744 }
745}
746
747/// The delivery branch for an item — **stable across attempts**.
748///
749/// A per-session `car/coder/<uuid>` would mean a retried item pushes a second
750/// branch and opens a second pull request, and a delivery that failed after the
751/// push would strand the first on the remote forever. A stable name is also
752/// what lets `deliver_pr_with` reconcile to exactly one pull request per item
753/// and honour a human closing it.
754pub fn delivery_branch(item: &Candidate) -> String {
755 format!("car/heal/{}-{}", item.repo.replace('/', "-"), item.number)
756}
757
758/// Map a delivery failure onto whether trying again could ever help.
759///
760/// `DeliveryFailure` already carries this judgement per stage; the only work
761/// here is not throwing it away. A preflight refusal is never retriable — it
762/// includes the closed-pull-request rule, which is a person saying stop.
763fn refusal_for(failure: super::merge::DeliveryFailure) -> DeliverRefusal {
764 use super::merge::DeliveryFailure as F;
765 match failure {
766 F::Preflight { reason } => DeliverRefusal::permanent(reason),
767 F::Commit { reason } => DeliverRefusal::permanent(reason),
768 F::Push { reason, retriable } | F::Pr { reason, retriable } => {
769 if retriable {
770 DeliverRefusal::retriable(reason)
771 } else {
772 DeliverRefusal::permanent(reason)
773 }
774 }
775 }
776}
777
778#[cfg(test)]
779mod tests {
780 use super::*;
781
782 struct Fixed {
783 model: &'static str,
784 answer: Result<String, String>,
785 }
786
787 #[async_trait::async_trait]
788 impl Reviewer for Fixed {
789 fn model(&self) -> &str {
790 self.model
791 }
792 async fn review(&self, _c: &str, _d: &str) -> Result<String, String> {
793 self.answer.clone()
794 }
795 }
796
797 #[test]
798 fn configuration_session_failure_is_typed_for_the_tick() {
799 let failure = configuration_run_failure(
800 "coder-1",
801 "no independent coder".into(),
802 Some("configuration"),
803 )
804 .expect("configuration failure");
805 assert!(failure.configuration);
806 assert_eq!(failure.session_id.as_deref(), Some("coder-1"));
807 assert_eq!(failure.detail, "no independent coder");
808 assert!(configuration_run_failure("coder-1", "red".into(), Some("error")).is_none());
809 }
810
811 /// Poll a panel without standing up a runtime: the fan-out is pure, and
812 /// the whole point of the split is that it can be tested that way.
813 async fn poll(
814 reviewers: Vec<Arc<dyn Reviewer>>,
815 criteria: &str,
816 diff: &str,
817 ) -> (Vec<Verdict>, Vec<Unreachable>) {
818 let futures: Vec<_> = reviewers
819 .iter()
820 .map(|r| async move {
821 let model = r.model().to_string();
822 match r.review(criteria, diff).await {
823 Ok(answer) => parse_verdict(&model, &answer),
824 Err(e) => Err(Unreachable { model, error: e }),
825 }
826 })
827 .collect();
828 let mut verdicts = Vec::new();
829 let mut unreachable = Vec::new();
830 for result in futures::future::join_all(futures).await {
831 match result {
832 Ok(v) => verdicts.push(v),
833 Err(u) => unreachable.push(u),
834 }
835 }
836 (verdicts, unreachable)
837 }
838
839 #[tokio::test]
840 async fn a_panel_keeps_answers_and_failures_apart() {
841 let reviewers: Vec<Arc<dyn Reviewer>> = vec![
842 Arc::new(Fixed {
843 model: "a",
844 answer: Ok("PASS looks right".into()),
845 }),
846 Arc::new(Fixed {
847 model: "b",
848 answer: Ok("FAIL wrong scope".into()),
849 }),
850 Arc::new(Fixed {
851 model: "c",
852 answer: Err("429 rate limited".into()),
853 }),
854 ];
855 let panel_size = reviewers.len();
856 let (verdicts, unreachable) = poll(reviewers, "criteria", "diff").await;
857 assert_eq!(verdicts.len(), 2);
858 assert_eq!(unreachable.len(), 1);
859 assert_eq!(unreachable[0].model, "c");
860 // The count that matters: two answers on a panel of three cannot be
861 // read as "two of two".
862 assert_eq!(panel_size, 3);
863 }
864
865 #[tokio::test]
866 async fn an_unreadable_answer_becomes_unreachable_not_a_pass() {
867 let (verdicts, unreachable) = poll(
868 vec![Arc::new(Fixed {
869 model: "a",
870 answer: Ok("hmm, hard to say".into()),
871 })],
872 "c",
873 "d",
874 )
875 .await;
876 assert!(verdicts.is_empty());
877 assert_eq!(unreachable.len(), 1);
878 }
879
880 #[test]
881 fn a_watch_only_target_is_refused_loudly_not_unwrapped() {
882 let t = HealTarget {
883 repo: "acme/w".into(),
884 fix_repo: None,
885 checkout: None,
886 label: "self-heal".into(),
887 base: "main".into(),
888 };
889 assert!(checkout_path(&t).is_err());
890 }
891}