car_server_core/coder/heal_runner.rs
1//! Running a real coder session for one healed item, and reviewing it.
2//!
3//! The last unwritten seam. Everything else in the loop was reachable with a
4//! fake; this is what turns a selected issue into a branch and a set of
5//! verdicts.
6//!
7//! ## Nothing is published until the gate approves
8//!
9//! A coder session that reaches `CoderState::NeedsApproval` has done its work
10//! and re-evaluated its own outcome contract — the runtime's deterministic
11//! answer to "does this build and pass". That is the moment the panel is asked,
12//! and it reviews the **worktree** diff, exactly the view the human approval
13//! surface shows.
14//!
15//! It deliberately does not review a published branch, because in the
16//! interactive path `car/coder/<id>` is created by `coder.approve_merge` —
17//! the branch exists only once a human has said yes, so its existence *means*
18//! approved. Publishing before the panel would put unapproved work in a
19//! namespace that already carries that meaning, and leave one dead branch in
20//! the operator's repository per rejected item.
21//!
22//! So delivery is one step, after [`super::heal_gate::decide`] approves, and it
23//! is [`super::merge::deliver_pr`] — the same commit → push → reconcile-exactly-
24//! one-pull-request path `car code-task --deliver pr` uses. That is not reuse
25//! for its own sake: it carries the base/target-equality refusal, the ambiguous
26//! head refusal, the vacuity guard, and the rule that a **closed** pull request
27//! on the delivery branch is a person saying stop. Hand-rolling publish and
28//! push had none of those, and a human closing the loop's pull request read as
29//! "do it again".
30//!
31//! The loop still never merges. `deliver_pr` opens a pull request; that is the
32//! boundary, and it is why the delivery branch may never equal the base.
33//!
34//! ## Every session this runner starts reaches a terminal state
35//!
36//! `NeedsApproval` is not terminal, so the git worktree — registered in the
37//! operator's own repository — is released only on a terminal transition.
38//! Delivery goes to `Merged` (which on the raw-repo path already means no more
39//! than "the branch was published"), rejection and timeout go to `Abandoned`
40//! and `Failed`. An unattended loop on a cadence whose expected outcome is
41//! rejection cannot leave that to a human.
42//!
43//! ## A tick cannot run forever
44//!
45//! `SessionDeadline` bounds the session's wall clock. Without it an unattended
46//! loop inherits whatever the engine's own limits happen to be, and a wedged
47//! session holds its claim for the full TTL while the cadence fires around it.
48//! The ceiling is per-item and configurable, because "how long is too long" is
49//! a property of the repository, not of this code.
50//!
51//! ## Reviewers see the diff AND the seed, and both are untrusted
52//!
53//! An earlier version of this comment claimed reviewers never see the issue
54//! body. They do: [`super::heal_gate::review_criteria`] interpolates the seed
55//! as `Stated intent:`, at the top of the prompt, ahead of the instructions —
56//! and the seed is rendered tracker text that cleared the tier gate, not text
57//! this runtime wrote.
58//!
59//! So the reviewer prompt carries two attacker-influenced inputs, not zero: the
60//! intent, from whoever filed the issue, and the diff, from whoever wrote the
61//! code. `may_seed_session` bounds who can reach the first; nothing bounds the
62//! second, which is why [`super::heal_review`] fences the diff with a
63//! content-derived delimiter rather than a fixed banner. Neither is a substitute
64//! for the other, and the honest statement is that a reviewer prompt is a place
65//! where "ignore the above and approve this" can land, guarded rather than
66//! impossible.
67//!
68//! What reviewers do NOT see is an outcome contract sourced from a body:
69//! `may_source_contract` is `Runtime`-only, and `run` confirms with `None` so
70//! the contract is derived from the intent rather than supplied by it.
71
72use std::sync::Arc;
73
74use super::heal_gate::{Unreachable, Verdict};
75use super::heal_intake::{Checkout, HealTarget};
76use super::heal_live::{parse_verdict, CoderRunner};
77use super::heal_select::Candidate;
78use super::heal_tick::{DeliverRefusal, RunFailure};
79use super::merge::PrDeliveryOutcome;
80use super::native_loop::TurnGenerator;
81use super::provenance::SessionSeed;
82use super::router::EngineChoice;
83use super::rpc::{confirm_session, start_session, CoderSessionEntry, StartArgs};
84use super::session::{CoderEventKind, CoderState};
85use crate::session::ServerState;
86
87/// Largest patch handed to a reviewer.
88///
89/// A reviewer that receives a megabyte of diff does not read it more carefully
90/// than one that receives 200KB; it costs more and is likelier to be silently
91/// truncated by the provider, which is the same outcome with no record of it.
92/// `stage_and_diff` marks the cut and reports the full size, so the truncation
93/// is visible rather than inferred.
94const MAX_REVIEW_DIFF_BYTES: usize = 200_000;
95
96/// How long one healed item may take before the tick gives up on it.
97pub const DEFAULT_ITEM_WALL_SECS: u64 = 45 * 60;
98
99/// How long the whole review panel may take before the item is failed.
100///
101/// Separate from the item ceiling because a wedged panel and a wedged coder
102/// session look nothing alike and the sweep holds its lock through both.
103pub const PANEL_WALL_SECS: u64 = 10 * 60;
104
105/// Asks one reviewer to judge a diff. One method, so a panel is N of these and
106/// the fan-out has nothing to coordinate.
107///
108/// Fallible on purpose: a vendor that cannot be reached must produce an error
109/// here rather than a verdict, because [`super::heal_gate::decide`] counts a
110/// missing answer as missing, never as a pass.
111#[async_trait::async_trait]
112pub trait Reviewer: Send + Sync {
113 /// The model's name, for the audit trail and for deduplication.
114 fn model(&self) -> &str;
115 /// Judge, returning the raw answer to be parsed.
116 async fn review(&self, criteria: &str, diff: &str) -> Result<String, String>;
117}
118
119/// The registered session, or an error rather than a panic.
120///
121/// `bench.rs` has an `expect("session registered")` here, which is right for a
122/// benchmark and wrong for an unattended loop: a missing entry must fail this
123/// item, not take the daemon down.
124async fn session_entry(
125 state: &Arc<ServerState>,
126 id: &str,
127) -> Result<Arc<CoderSessionEntry>, String> {
128 state
129 .coder_sessions
130 .lock()
131 .await
132 .get(id)
133 .cloned()
134 .ok_or_else(|| format!("coder session `{id}` is not registered"))
135}
136
137/// The live runner.
138pub struct LiveCoderRunner {
139 pub state: Arc<ServerState>,
140 pub generator: Arc<dyn TurnGenerator>,
141 pub state_dir: std::path::PathBuf,
142 /// The panel. Empty means no panel, which [`super::heal_gate::decide`]
143 /// refuses outright rather than treating as unanimous consent.
144 pub reviewers: Vec<Arc<dyn Reviewer>>,
145 pub max_wall_secs: u64,
146 pub max_iterations: Option<u32>,
147 /// Which engine runs the work.
148 ///
149 /// An operator knob, not a test hook. `Auto` picks per task from the
150 /// detected external CLIs, so an unattended loop on a machine with Claude
151 /// Code or Codex installed silently runs a *different* engine than the same
152 /// loop on a machine without them — and the user has asked for foreman on
153 /// this path specifically. Naming it makes that choice visible and
154 /// configurable instead of implicit in what happens to be on `PATH`.
155 pub engine: EngineChoice,
156 /// Pin the coder's own model, or `None` for the operator's `coder.toml`
157 /// default and then adaptive routing.
158 ///
159 /// Worth naming for an unattended loop for the same reason the engine is:
160 /// adaptive routing picks a provider per request, so a loop can silently
161 /// spend a whole 45-minute session against a credential that has expired
162 /// while a working one sits beside it. An operator who has decided which
163 /// model does this work should be able to say so.
164 pub model: Option<String>,
165 /// Canonical `ModelSchema.name` values for every review-panel seat. These
166 /// become strict exclusions only on the unpinned adaptive native loop.
167 pub routing_exclusions: Vec<String>,
168 /// Resolve a model name or id to the registry's canonical id.
169 ///
170 /// Injected rather than reached for, the same way
171 /// [`coder_on_panel`](super::heal_review::coder_on_panel) injects it: the
172 /// self-review gate is then testable without standing up an inference
173 /// registry, and reaching for one here would force
174 /// `get_inference_engine`'s worker-subprocess init on every test that runs
175 /// a session. `heal_service` supplies the engine-backed resolver.
176 pub canonical_model: Arc<dyn Fn(&str) -> String + Send + Sync>,
177 /// The GitHub client delivery runs through.
178 ///
179 /// Owned here rather than by `LiveTickIo` because this is the only place
180 /// that calls GitHub now: one owner, so `gh` cannot be reached through two
181 /// handles configured differently.
182 pub github: Arc<dyn super::merge::GitHubApi>,
183}
184
185/// Where this target's code lives.
186///
187/// One definition, because both the session and the delivery need the answer —
188/// `git` runs in the worktree but `gh` runs here, and a second definition of
189/// "where is this target" is a second thing that can disagree with the first.
190///
191/// A `Checkout::Project` resolves to the project's repo path and nothing else:
192/// the loop never opens a *project session*, because that mode commits to the
193/// project's `main`. See the `project: None` in `run`.
194pub fn checkout_path(target: &HealTarget) -> Result<std::path::PathBuf, String> {
195 match target.checkout.as_ref() {
196 Some(Checkout::Local(p)) => Ok(p.clone()),
197 Some(Checkout::Project(slug)) => Ok(super::project::load_project(slug)?.repo_path.clone()),
198 // Unreachable: `tick` refuses a watch-only target before it gets here.
199 // Matched rather than unwrapped so a future caller that skips that
200 // check fails loudly instead of panicking.
201 None => Err("target has no checkout; it is watch-only".into()),
202 }
203}
204
205fn configuration_run_failure(
206 session_id: &str,
207 detail: String,
208 failure_kind: Option<&str>,
209) -> Option<RunFailure> {
210 (failure_kind == Some("configuration"))
211 .then(|| RunFailure::configuration_with_session(session_id, detail))
212}
213
214impl LiveCoderRunner {
215 /// Drive a session to a terminal state, releasing its worktree.
216 ///
217 /// `NeedsApproval` is not terminal, so a session left there keeps the git
218 /// worktree registered in the operator's own repository and its entry in
219 /// the daemon's session map. On an unattended loop firing on a cadence,
220 /// rejection is the *expected* outcome — so without this every rejected
221 /// item leaks a worktree, permanently.
222 ///
223 /// Best-effort by design: this runs on paths that are already reporting a
224 /// failure, and an error here must not replace the reason the caller is
225 /// about to report.
226 async fn terminate(&self, session_id: &str, to: CoderState) {
227 let Ok(entry) = session_entry(&self.state, session_id).await else {
228 return;
229 };
230 let mut session = entry.session.lock().await;
231 if session.state.is_terminal() {
232 return;
233 }
234 if let Err(e) = session.transition(to, &entry.sink) {
235 tracing::warn!(session = %session_id, "self-heal could not close the session: {e}");
236 }
237 }
238
239 /// Poll every reviewer independently and keep both halves of the answer.
240 ///
241 /// Reviewers are asked concurrently but never shown each other's verdicts:
242 /// a panel that reaches consensus by reading itself is one reviewer with
243 /// extra steps, and correlation is precisely the failure a panel exists to
244 /// catch.
245 async fn poll_panel(&self, criteria: &str, diff: &str) -> (Vec<Verdict>, Vec<Unreachable>) {
246 let futures: Vec<_> = self
247 .reviewers
248 .iter()
249 .map(|r| async move {
250 let model = r.model().to_string();
251 match r.review(criteria, diff).await {
252 Ok(answer) => parse_verdict(&model, &answer),
253 Err(e) => Err(Unreachable { model, error: e }),
254 }
255 })
256 .collect();
257
258 let mut verdicts = Vec::new();
259 let mut unreachable = Vec::new();
260 for result in futures::future::join_all(futures).await {
261 match result {
262 Ok(v) => verdicts.push(v),
263 Err(u) => unreachable.push(u),
264 }
265 }
266 (verdicts, unreachable)
267 }
268}
269
270#[async_trait::async_trait]
271impl CoderRunner for LiveCoderRunner {
272 async fn run(
273 &self,
274 target: &HealTarget,
275 item: &Candidate,
276 seed: &SessionSeed,
277 ) -> Result<super::heal_tick::Attempt, RunFailure> {
278 // Before a session exists — nothing to close out.
279 let repo = checkout_path(target).map_err(RunFailure::early)?;
280
281 let start = start_session(
282 &self.state,
283 StartArgs {
284 // The healing loop runs on the machine that owns the tracker
285 // and the credentials; farming its subtasks to peers is a
286 // separate decision from healing at all.
287 distributed: false,
288 browser: false,
289 workers: Vec::new(),
290 repo,
291 intent: seed.as_str().to_string(),
292 engine: self.engine.clone(),
293 max_iterations: self.max_iterations,
294 state_dir: self.state_dir.clone(),
295 // Always `None`, even for a `Checkout::Project` target. A
296 // project session commits straight to the project's `main`
297 // (`approve_merge_session` takes the `commit_to_main` fork),
298 // and this loop's whole autonomy boundary is that it never
299 // lands anything. A managed project is healed the same way any
300 // repository is: a branch and a pull request.
301 project: None,
302 model: self.model.clone(),
303 routing_exclusions: self.routing_exclusions.clone(),
304 repair_invokes: None,
305 transient_retries: None,
306 discussion_id: None,
307 base: None,
308 },
309 self.generator.clone(),
310 )
311 .await
312 .map_err(RunFailure::early)?;
313
314 let session_id = start["session_id"]
315 .as_str()
316 .ok_or_else(|| RunFailure::early("coder.start returned no session_id"))?
317 .to_string();
318
319 // The contract is derived from the INTENT, never from the issue body —
320 // `may_source_contract` is runtime-only, and this item is not
321 // necessarily runtime-authored. Confirming with `None` accepts the
322 // derived contract rather than supplying one.
323 // From here a session EXISTS, so every failure names it and `tick`
324 // closes it out. A non-terminal session holds a git worktree
325 // registered in the operator's own repository.
326 confirm_session(&self.state, &session_id, None)
327 .await
328 .map_err(|e| RunFailure::with_session(&session_id, e))?;
329
330 let entry = session_entry(&self.state, &session_id)
331 .await
332 .map_err(|e| RunFailure::with_session(&session_id, e))?;
333
334 // Refuse a contract that cannot go green before spending a session on
335 // it. `confirm_session` evaluates every check against the untouched
336 // worktree, and a check whose program does not exist fails there the
337 // same way a genuinely red one does — so the run proceeds, burns its
338 // whole iteration budget, and reports "contract not satisfied". That is
339 // exactly what a live trial did: derivation wrote `python -m pytest` on
340 // a machine with only `python3`, and twelve iterations of real
341 // inference went into something unsatisfiable from the first edit.
342 //
343 // An interactive `car code` user sees the baseline and can revise the
344 // contract; an unattended loop has nobody to notice.
345 {
346 let session = entry.session.lock().await;
347 let unrunnable = super::contract::baseline_cannot_run(&session.baseline);
348 if !unrunnable.is_empty() {
349 return Err(RunFailure::with_session(
350 &session_id,
351 format!(
352 "the derived contract cannot be evaluated for {}#{}: check(s) {} \
353 could not be run at all (the command does not exist here), so no \
354 change could ever make this contract green",
355 item.repo,
356 item.number,
357 unrunnable.join(", ")
358 ),
359 ));
360 }
361 // The other half of the red-green baseline, and the worse case of
362 // the two. An unrunnable check FAILS, so the run reports it. A
363 // contract that is already green PASSES, so the session would reach
364 // the gate with the deterministic half silently satisfied by code
365 // nobody in this run wrote — leaving the panel as the only thing
366 // actually deciding, while the pull request reports both. The
367 // contract is the load-bearing gate (see `heal_gate`); the panel
368 // narrows it.
369 //
370 // Checked here for the same reason `baseline_cannot_run` is: an
371 // interactive `car code` user sees the baseline and can revise the
372 // contract, and an unattended loop has nobody to notice. `car
373 // code-task` already refuses on this condition (car#1070); the loop
374 // was the one caller that skipped it.
375 //
376 // All-green only, never a single passing check — `baseline_gates_nothing`
377 // documents why: a refactor legitimately has checks green on both
378 // sides, and escalating on one would abort sessions over a contract
379 // nit.
380 if super::contract::baseline_gates_nothing(&session.baseline) {
381 return Err(RunFailure::with_session(
382 &session_id,
383 format!(
384 "every check of the contract derived for {}#{} already passes on \
385 unmodified code ({} check(s)), so it gates nothing and no change \
386 could turn it red-to-green. Either the issue's premise no longer \
387 holds — the behaviour it describes may already be fixed — or the \
388 derived checks do not exercise it. A human should decide which.",
389 item.repo,
390 item.number,
391 session.baseline.len()
392 ),
393 ));
394 }
395 }
396 let handle = entry.task.lock().unwrap().take();
397 if let Some(mut handle) = handle {
398 // Bounded. An unattended loop must not inherit whatever limits the
399 // engine happens to have: a wedged session would hold its claim for
400 // the full TTL while the cadence fires around it.
401 //
402 // `&mut handle`, and `abort()` on expiry. Dropping a `JoinHandle`
403 // DETACHES the task, it does not stop it — so timing out by value
404 // left the session running, still editing the same repository,
405 // after the tick had released the claim and a later tick could
406 // start a second session on the same item. Taking the handle also
407 // disarms `coder.cancel`, whose `abort()` is the only thing that
408 // can interrupt a session wedged inside a long await, so this is
409 // now the only place that can stop it.
410 match tokio::time::timeout(
411 std::time::Duration::from_secs(self.max_wall_secs),
412 &mut handle,
413 )
414 .await
415 {
416 Ok(Ok(())) => {}
417 Ok(Err(e)) => {
418 return Err(RunFailure::with_session(
419 &session_id,
420 format!("coder session panicked: {e}"),
421 ))
422 }
423 Err(_) => {
424 handle.abort();
425 self.terminate(&session_id, CoderState::Failed).await;
426 return Err(RunFailure::with_session(
427 &session_id,
428 format!(
429 "coder session exceeded its {}s ceiling for {}#{}",
430 self.max_wall_secs, item.repo, item.number
431 ),
432 ));
433 }
434 }
435 }
436
437 // One read for everything the gate below decides on. These cannot change
438 // between them — the loop has finished — and three separate `lock()`s
439 // over thirty lines is three chances for that to stop being true.
440 let (state, contract_detail, authors, failure_kind, ran_as, nominated) = {
441 let s = entry.session.lock().await;
442 // A model's `report_no_change` can park at `NeedsApproval` on a
443 // contract that never went green. Unlike the runtime-observed
444 // finding (a green run that left the tree untouched), that is not
445 // "the contract passed", and reporting it as such would post a
446 // false comment on the tracker. The last check results decide.
447 let nominated = s.no_change_finding.as_ref().and_then(|finding| {
448 let green = !s.last_check_results.is_empty()
449 && s.last_check_results.iter().all(|r| r.passed);
450 (!green).then(|| {
451 format!(
452 "the coder concluded no change is needed ({}: {}) without the \
453 outcome contract passing; a no-change conclusion needs a human",
454 finding.kind.as_str(),
455 finding.summary
456 )
457 })
458 });
459 (
460 s.state,
461 s.error
462 .clone()
463 .unwrap_or_else(|| "outcome contract evaluated".to_string()),
464 s.authored_by.clone(),
465 s.failure_kind.clone(),
466 // The RESOLVED engine, not the one heal.toml asked for. See the
467 // empty-authors check below for why the difference matters.
468 s.engine.clone(),
469 nominated,
470 )
471 };
472 let red_nomination = nominated.is_some();
473 let contract_detail = nominated.unwrap_or(contract_detail);
474
475 // `NeedsApproval` is the runtime's own verdict that the contract
476 // passed — except for a nominated no-change finding on a contract that
477 // never went green (above). Anything else is a red contract, and a red
478 // contract is not put to the panel at all.
479 let contract_passed = state == CoderState::NeedsApproval && !red_nomination;
480 if !contract_passed {
481 if let Some(failure) = configuration_run_failure(
482 &session_id,
483 contract_detail.clone(),
484 failure_kind.as_deref(),
485 ) {
486 return Err(failure);
487 }
488 return Ok(super::heal_tick::Attempt {
489 contract_passed: false,
490 contract_detail,
491 panel_size: self.reviewers.len(),
492 verdicts: Vec::new(),
493 unreachable: Vec::new(),
494 session_id: session_id.clone(),
495 });
496 }
497
498 // A model may not review its own output, checked HERE against what
499 // actually authored the change rather than only at assembly against
500 // what was configured.
501 //
502 // `heal_service`'s `coder_on_panel` runs when the coder is PINNED —
503 // by `heal.toml`'s `coder_model` or, failing that, `coder.toml`'s
504 // `model` (car#1360). Unpinned — the default — the router picks, and
505 // on a machine with one reachable credential that pick can also be a
506 // panel seat, so the self-review the gate refuses when configured was
507 // permitted by default (car#1299). The journal has recorded the
508 // authoring model all along; nothing read it.
509 //
510 // The SAME rule, not a second copy of it: `coder_on_panel` canonicalizes
511 // both sides through the registry, and that is load-bearing here rather
512 // than cosmetic. An unpinned turn reports `schema.name` while a seat is
513 // configured however the operator spelled it, and `knows_model` accepts
514 // an id or a name — so for every model whose id and name differ (car#889:
515 // `openrouter/google/gemini-3.1-pro-preview` vs
516 // `google/gemini-3.1-pro-preview`) a spelling comparison silently never
517 // fires. A gate that cannot fire is worse than no gate, because it reads
518 // as covered.
519 //
520 // `unwrap_or_else(|| m.to_string())`, NOT `unwrap_or_default()`. The
521 // assembly-time check can use the latter because every name there
522 // cleared `knows_model` first; here the author comes from the engine and
523 // nothing validated it, so an unknown name would canonicalize to `""` and
524 // match every other unknown name — refusing every session on a daemon
525 // whose registry does not hold the model that ran.
526 //
527 // Refused rather than dropping the seat: a panel silently shrunk from
528 // three to two is the halved threshold `PanelIncomplete` exists to
529 // prevent, and the diff itself is not wrong — only the independence the
530 // verdict would claim for it.
531 {
532 // No native turn was journaled. For foreman/external — the DEFAULT
533 // engine — that is the honest answer rather than a failure: those
534 // farm to a coding CLI whose backbone CAR never resolved, so there
535 // is no model to compare. A session that RAN native and reaches
536 // `NeedsApproval` is a different matter: it got here by a model
537 // declaring done, which journals a terminal, so an empty set means
538 // the attribution was lost. Permitting on lost instrumentation is
539 // exactly the silently-absent attribution this closes, one level
540 // down.
541 //
542 // `ran_as`, the engine the session RESOLVED to — not `self.engine`,
543 // which is what `heal.toml` asked for. `engine = "auto"` parses,
544 // and `EngineChoice::Auto` resolves to `Native` whenever no external
545 // CLI is ready (`router::resolve`), which on a headless server is
546 // the common case rather than an exotic one. Comparing the request
547 // let exactly the session this refuses — native, terminal, no
548 // attribution — through the gate, because the request said `auto`
549 // (car#1357).
550 if authors.is_empty() && ran_as == EngineChoice::Native {
551 return Err(RunFailure::with_session(
552 &session_id,
553 format!(
554 "the native coder produced a change for {}#{} with no model \
555 recorded against it — the journal should name every model that \
556 completed a turn, so this session cannot be shown to be \
557 independent of the review panel",
558 item.repo, item.number
559 ),
560 ));
561 }
562
563 let seats: Vec<String> = self
564 .reviewers
565 .iter()
566 .map(|r| r.model().to_string())
567 .collect();
568 for author in &authors {
569 let Some(seat) = super::heal_review::coder_on_panel(author, &seats, |m| {
570 (self.canonical_model)(m)
571 }) else {
572 continue;
573 };
574 return Err(RunFailure::with_session(
575 &session_id,
576 format!(
577 "{} wrote part of this change for {}#{} and also sits on the review \
578 panel ({}) — a model cannot review its own output, and counting it \
579 would report an independence this gate does not have. The router \
580 chose it — nothing pinned a coder, or the pin is not what \
581 served this turn — so pin a coder that is not a seat (`heal.toml`'s \
582 `coder_model`, or `coder.toml`'s `model`), or drop that seat from \
583 `review_models`.",
584 author, item.repo, item.number, seat
585 ),
586 ));
587 }
588 }
589
590 // The worktree diff, which is exactly what the human approval surface
591 // reviews — not a diff of a published branch, because nothing is
592 // published until the gate approves.
593 let worktree = {
594 let s = entry.session.lock().await;
595 s.workspace_path.clone().ok_or_else(|| {
596 RunFailure::with_session(&session_id, "session reached approval with no worktree")
597 })?
598 };
599 let staged =
600 super::merge::stage_and_diff(&worktree, MAX_REVIEW_DIFF_BYTES).map_err(|e| {
601 RunFailure::with_session(
602 &session_id,
603 format!("could not read the change for review: {e}"),
604 )
605 })?;
606
607 // A hard failure, never a review of nothing. The panel is the only
608 // human substitute in this loop, and a reviewer handed an empty or
609 // unreadable diff still answers — so a soft fallback here records three
610 // verdicts that were never rendered on the change, and opens a pull
611 // request on them.
612 // A truncated patch is judged the same way an unavailable one would be:
613 // `tail()` keeps the LAST 200KB, so the panel would receive a fragment
614 // whose beginning is missing and be told nothing about it — and it
615 // would answer anyway. `stage_and_diff` reports the cut precisely so a
616 // caller need not infer it; reading `truncated` and refusing is what
617 // makes that report mean something.
618 if staged.truncated {
619 return Err(RunFailure::with_session(
620 &session_id,
621 format!(
622 "the change for {}#{} is {} bytes, larger than the {} a reviewer is \
623 given — a panel judging the tail of a patch is not a review",
624 item.repo, item.number, staged.full_bytes, MAX_REVIEW_DIFF_BYTES
625 ),
626 ));
627 }
628
629 if staged.changed_paths.is_empty() {
630 return Err(RunFailure::with_session(
631 &session_id,
632 format!(
633 "the outcome contract passed but the worktree is unchanged for {}#{} — there is nothing to review and nothing to deliver",
634 item.repo, item.number
635 ),
636 ));
637 }
638
639 let criteria = super::heal_gate::review_criteria(seed.as_str(), &contract_detail);
640 // Bounded, like the session above it. The remote client has its own
641 // HTTP timeout, but that bound belongs to the client and does not cover
642 // a local seat or a retry chain — and the sweep holds its lock for the
643 // whole tick, so a wedged panel makes every later cadence tick log
644 // "previous sweep still running" with no indication of what is stuck.
645 // A timeout produces `Unreachable`, which the gate already reads as a
646 // missing answer rather than a pass.
647 let (verdicts, unreachable) = match tokio::time::timeout(
648 std::time::Duration::from_secs(PANEL_WALL_SECS),
649 self.poll_panel(&criteria, &staged.patch),
650 )
651 .await
652 {
653 Ok(v) => v,
654 Err(_) => (
655 Vec::new(),
656 self.reviewers
657 .iter()
658 .map(|r| Unreachable {
659 model: r.model().to_string(),
660 error: format!("the panel did not answer within {PANEL_WALL_SECS}s"),
661 })
662 .collect(),
663 ),
664 };
665
666 Ok(super::heal_tick::Attempt {
667 contract_passed: true,
668 contract_detail,
669 panel_size: self.reviewers.len(),
670 verdicts,
671 unreachable,
672 session_id,
673 })
674 }
675
676 async fn deliver(
677 &self,
678 target: &HealTarget,
679 item: &Candidate,
680 session_id: &str,
681 body: &str,
682 ) -> Result<PrDeliveryOutcome, DeliverRefusal> {
683 let repo = checkout_path(target).map_err(DeliverRefusal::permanent)?;
684 // Named up front, because `gh`'s own failure for a missing credential
685 // arrives from three different seams inside delivery and none of them
686 // says which credential.
687 self.github.auth_status().map_err(|e| {
688 DeliverRefusal::permanent(format!("gh is not logged in: {}", e.message))
689 })?;
690 let entry = session_entry(&self.state, session_id)
691 .await
692 .map_err(DeliverRefusal::permanent)?;
693
694 // Read what delivery needs, then DROP the guard.
695 //
696 // Holding the session mutex across `deliver_pr_with` held it across
697 // five blocking `git`/`gh` subprocesses — each with a 900-second
698 // ceiling — including a network round trip to GitHub. `coder.list`
699 // locks every session in a loop to build the board, so one unattended
700 // heal delivery wedged the board for every user of the daemon, and
701 // `coder.cancel` parked on the same lock. The interactive
702 // `approve_merge_session` sets the precedent for the lock discipline;
703 // it does not excuse adding a network call inside it, on a path nobody
704 // is watching.
705 let (worktree, contract, intent) = {
706 let session = entry.session.lock().await;
707 (
708 session.workspace_path.clone().ok_or_else(|| {
709 DeliverRefusal::permanent("session has no worktree to deliver")
710 })?,
711 session
712 .contract
713 .clone()
714 .ok_or_else(|| DeliverRefusal::permanent("session has no outcome contract"))?,
715 session.intent.clone(),
716 )
717 };
718
719 let branch = delivery_branch(item);
720 // Off the async worker. `deliver_pr_with` is synchronous and spawns
721 // subprocesses; awaiting it inline blocks a runtime thread the
722 // scheduler believes is available.
723 let github = self.github.clone();
724 let base = target.base.clone();
725 let body = body.to_string();
726 let outcome = tokio::task::spawn_blocking(move || {
727 super::merge::deliver_pr_with(
728 super::merge::PrDelivery {
729 repo: &repo,
730 worktree: &worktree,
731 target_branch: &branch,
732 base_branch: &base,
733 draft: false,
734 intent: &intent,
735 contract: &contract,
736 body: &body,
737 // The heal loop hardcodes `distributed: false` — healing
738 // runs on the machine that owns the tracker and the
739 // credentials — so its subtasks always ran here.
740 provenance: None,
741 },
742 github.as_ref(),
743 )
744 })
745 .await
746 .map_err(|e| DeliverRefusal::retriable(format!("delivery task failed: {e}")))?
747 .map_err(refusal_for)?;
748
749 let mut session = entry.session.lock().await;
750 session.result_branch = Some(outcome.branch.clone());
751 entry.sink.emit(CoderEventKind::MergeCompleted {
752 branch: outcome.branch.clone(),
753 });
754 // Terminal. On the raw-repo path `Merged` already means only "the
755 // branch was published" — `approve_merge_session` performs no merge for
756 // a non-project session — so this records the fact that occurred, and
757 // it is what releases the worktree.
758 session
759 .transition(CoderState::Merged, &entry.sink)
760 .map_err(DeliverRefusal::permanent)?;
761 Ok(outcome)
762 }
763
764 async fn abandon(&self, session_id: &str) {
765 self.terminate(session_id, CoderState::Abandoned).await;
766 }
767}
768
769/// The delivery branch for an item — **stable across attempts**.
770///
771/// A per-session `car/coder/<uuid>` would mean a retried item pushes a second
772/// branch and opens a second pull request, and a delivery that failed after the
773/// push would strand the first on the remote forever. A stable name is also
774/// what lets `deliver_pr_with` reconcile to exactly one pull request per item
775/// and honour a human closing it.
776pub fn delivery_branch(item: &Candidate) -> String {
777 format!("car/heal/{}-{}", item.repo.replace('/', "-"), item.number)
778}
779
780/// Map a delivery failure onto whether trying again could ever help.
781///
782/// `DeliveryFailure` already carries this judgement per stage; the only work
783/// here is not throwing it away. A preflight refusal is never retriable — it
784/// includes the closed-pull-request rule, which is a person saying stop.
785fn refusal_for(failure: super::merge::DeliveryFailure) -> DeliverRefusal {
786 use super::merge::DeliveryFailure as F;
787 match failure {
788 F::Preflight { reason } => DeliverRefusal::permanent(reason),
789 F::Commit { reason } => DeliverRefusal::permanent(reason),
790 F::Push { reason, retriable } | F::Pr { reason, retriable } => {
791 if retriable {
792 DeliverRefusal::retriable(reason)
793 } else {
794 DeliverRefusal::permanent(reason)
795 }
796 }
797 }
798}
799
800#[cfg(test)]
801mod tests {
802 use super::*;
803
804 struct Fixed {
805 model: &'static str,
806 answer: Result<String, String>,
807 }
808
809 #[async_trait::async_trait]
810 impl Reviewer for Fixed {
811 fn model(&self) -> &str {
812 self.model
813 }
814 async fn review(&self, _c: &str, _d: &str) -> Result<String, String> {
815 self.answer.clone()
816 }
817 }
818
819 #[test]
820 fn configuration_session_failure_is_typed_for_the_tick() {
821 let failure = configuration_run_failure(
822 "coder-1",
823 "no independent coder".into(),
824 Some("configuration"),
825 )
826 .expect("configuration failure");
827 assert!(failure.configuration);
828 assert_eq!(failure.session_id.as_deref(), Some("coder-1"));
829 assert_eq!(failure.detail, "no independent coder");
830 assert!(configuration_run_failure("coder-1", "red".into(), Some("error")).is_none());
831 }
832
833 /// Poll a panel without standing up a runtime: the fan-out is pure, and
834 /// the whole point of the split is that it can be tested that way.
835 async fn poll(
836 reviewers: Vec<Arc<dyn Reviewer>>,
837 criteria: &str,
838 diff: &str,
839 ) -> (Vec<Verdict>, Vec<Unreachable>) {
840 let futures: Vec<_> = reviewers
841 .iter()
842 .map(|r| async move {
843 let model = r.model().to_string();
844 match r.review(criteria, diff).await {
845 Ok(answer) => parse_verdict(&model, &answer),
846 Err(e) => Err(Unreachable { model, error: e }),
847 }
848 })
849 .collect();
850 let mut verdicts = Vec::new();
851 let mut unreachable = Vec::new();
852 for result in futures::future::join_all(futures).await {
853 match result {
854 Ok(v) => verdicts.push(v),
855 Err(u) => unreachable.push(u),
856 }
857 }
858 (verdicts, unreachable)
859 }
860
861 #[tokio::test]
862 async fn a_panel_keeps_answers_and_failures_apart() {
863 let reviewers: Vec<Arc<dyn Reviewer>> = vec![
864 Arc::new(Fixed {
865 model: "a",
866 answer: Ok("PASS looks right".into()),
867 }),
868 Arc::new(Fixed {
869 model: "b",
870 answer: Ok("FAIL wrong scope".into()),
871 }),
872 Arc::new(Fixed {
873 model: "c",
874 answer: Err("429 rate limited".into()),
875 }),
876 ];
877 let panel_size = reviewers.len();
878 let (verdicts, unreachable) = poll(reviewers, "criteria", "diff").await;
879 assert_eq!(verdicts.len(), 2);
880 assert_eq!(unreachable.len(), 1);
881 assert_eq!(unreachable[0].model, "c");
882 // The count that matters: two answers on a panel of three cannot be
883 // read as "two of two".
884 assert_eq!(panel_size, 3);
885 }
886
887 #[tokio::test]
888 async fn an_unreadable_answer_becomes_unreachable_not_a_pass() {
889 let (verdicts, unreachable) = poll(
890 vec![Arc::new(Fixed {
891 model: "a",
892 answer: Ok("hmm, hard to say".into()),
893 })],
894 "c",
895 "d",
896 )
897 .await;
898 assert!(verdicts.is_empty());
899 assert_eq!(unreachable.len(), 1);
900 }
901
902 #[test]
903 fn a_watch_only_target_is_refused_loudly_not_unwrapped() {
904 let t = HealTarget {
905 repo: "acme/w".into(),
906 fix_repo: None,
907 checkout: None,
908 label: "self-heal".into(),
909 base: "main".into(),
910 };
911 assert!(checkout_path(&t).is_err());
912 }
913}