car_server_core/coder/heal_service.rs
1//! The thing that actually runs the self-healing loop.
2//!
3//! T7 of `docs/proposals/self-healing-issue-loop.md`. Until this existed,
4//! [`super::heal_tick::tick`] had no caller — a working half, not a feature.
5//!
6//! ## A daemon subsystem, not a scheduler task
7//!
8//! The proposal originally said `scheduler.os_install`, arguing against a
9//! Claude Code `/loop` because a `/loop` dies with its session. True, and
10//! beside the point: a daemon subsystem does not have that problem, because
11//! the daemon *is* the durable process. `car-scheduler`'s tasks are either
12//! prompt-driven through an agent runner or a shell `CommandSpec`, and this
13//! tick is neither — it is deterministic Rust composition that needs
14//! `coder.start`, so a shell task would mean re-entering by subprocess to
15//! reach a process we were already inside.
16//!
17//! The precedent this follows is [`crate::selfheal`]: a service on
18//! `ServerState`, a cadence spawner, an interval with an env override, and a
19//! manual-run RPC.
20//!
21//! ## Three things the reviews said this had to settle
22//!
23//! **Who persists the ledger.** This service does, around every tick, before
24//! and after. The claim must reach disk *before* the coder session starts, or a
25//! crash mid-session loses it and the next start re-picks an item whose work is
26//! already in flight.
27//!
28//! **What `run_id` is.** A fresh id per tick. A stable one would make every
29//! tick the holder of every claim it ever took — `claim()` treats a matching
30//! run id as a refresh — and the mechanism would silently do nothing.
31//!
32//! **Overlap.** `CLAIM_TTL_MS` is 90 minutes and the cadence is shorter, so
33//! ticks *will* collide. This try-locks and skips rather than queueing: a
34//! queue of ticks against one repository is just a slower way to do the same
35//! work twice.
36
37use std::sync::Arc;
38
39use tokio::sync::Mutex;
40
41use super::heal_claims::ClaimStore;
42use super::heal_config::HealConfig;
43use super::heal_config::HEAL_CONFIG_FILE;
44use super::heal_tick::{tick, ClaimSink, TickIo, TickOutcome};
45use crate::session::ServerState;
46
47/// The engine the loop uses when the configuration does not name one.
48///
49/// `foreman` rather than `auto`: the work is unattended, so decomposing an
50/// intent and gating each patch plus the integrated union is worth more here
51/// than it is in an interactive session someone is watching. Foreman declines
52/// to single-session when the plan has no parallelism, so this is not a cost on
53/// small fixes.
54pub const DEFAULT_ENGINE: &str = "foreman";
55
56/// Env override for the cadence, mirroring `CAR_SELFHEAL_INTERVAL_SECS`.
57pub const HEAL_INTERVAL_ENV: &str = "CAR_HEAL_INTERVAL_SECS";
58
59/// Default cadence. Deliberately unhurried: the queue is human-authored, so
60/// polling faster mostly means asking GitHub the same question more often.
61pub const DEFAULT_INTERVAL_SECS: u64 = 15 * 60;
62
63/// The loop's state inside the daemon.
64pub struct HealService {
65 /// The configuration as of the last read. Behind a lock because the file is
66 /// re-read every sweep: an operator who fixes a typo'd repo spec and reruns
67 /// `car heal status` must not be shown the boot-time config presented as
68 /// current, and enabling the loop must not require a daemon restart.
69 ///
70 /// Re-reading is one small TOML at a 15-minute cadence. The alternative —
71 /// read once at `ServerState` construction — made "the loop never does
72 /// anything" indistinguishable from "your edit has not been picked up",
73 /// which is the failure mode this whole file's diagnostics exist to
74 /// prevent.
75 config: std::sync::RwLock<HealConfig>,
76 /// Where `heal.toml` lives, so a sweep can re-read it.
77 ///
78 /// `None` means the configuration was supplied directly and there is no
79 /// file behind it: re-reading would discard what the caller passed in, so
80 /// this service simply never reloads.
81 config_dir: Option<std::path::PathBuf>,
82 /// Held across a whole tick. See the module docs: try-lock and skip.
83 running: Mutex<()>,
84 state_dir: std::path::PathBuf,
85 interval_secs: u64,
86 /// Why the boot-time assembly failed, when it did.
87 ///
88 /// `is_enabled` reads the config; assembly additionally validates the model
89 /// names, refuses a coder that sits on its own panel, and requires at least
90 /// two serving providers. A configuration
91 /// that passes the first and fails the second leaves the loop enabled and
92 /// dead — which is precisely the state `heal.status` exists to make
93 /// impossible ("an idle loop and a misconfigured one are indistinguishable
94 /// from outside"). Recorded here so `disabled_reason` can say it.
95 assembly_error: std::sync::RwLock<Option<String>>,
96 /// Why the most recent MANUAL `heal.run` refused to assemble, cleared when
97 /// one succeeds.
98 ///
99 /// Separate from `assembly_error` because the two have different lifetimes
100 /// and a single slot gets both wrong. The cadence assembles ONCE at boot
101 /// and does not start on failure, so its error is true until the daemon
102 /// restarts — sticky is correct. `run_tick` re-assembles per call, so its
103 /// failure is a statement about the config as it stands now. Writing that
104 /// into the boot slot made a healthy, sweeping cadence report
105 /// `disabled_reason` permanently, which is the same lie this field exists
106 /// to prevent, pointed the other way.
107 run_refusal: std::sync::RwLock<Option<String>>,
108}
109
110/// What one pass over every configured target did.
111#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
112pub struct SweepReport {
113 pub outcomes: Vec<(String, TickOutcome)>,
114 /// True when another sweep held the lock and this one stood down.
115 pub skipped_overlap: bool,
116}
117
118/// What `heal.status` reports.
119///
120/// Carries `disabled_reason` and `rejected` because a loop that is idle and a
121/// loop that is misconfigured look identical from outside, and on first setup
122/// the second is the common case. "It never does anything" is the hardest
123/// failure to notice in a subsystem whose normal state is doing nothing.
124#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
125pub struct HealStatus {
126 /// Where the configuration was read from, so an operator editing the wrong
127 /// file finds out from `heal.status` instead of from silence.
128 pub config_path: String,
129 pub enabled: bool,
130 pub cadence_secs: u64,
131 /// Why the loop will not run, when it is configured but disabled.
132 pub disabled_reason: Option<String>,
133 /// `owner/name` per configured target.
134 pub targets: Vec<String>,
135 /// Targets that were named but could not be used, and why.
136 pub rejected: Vec<RejectedTargetView>,
137 /// The review panel, by model id.
138 pub review_models: Vec<String>,
139 /// The panel with each seat resolved to the vendor that serves it, after
140 /// the same deduplication the live panel applies.
141 pub panel: Vec<super::heal_review::PanelSeat>,
142 /// Why a panel that clears the two-provider construction floor still
143 /// cannot be shown to be independent of itself — see `correlation_warning`.
144 pub panel_warning: Option<String>,
145 /// Why the most recent manual `heal.run` refused to assemble, if one did
146 /// and none has succeeded since.
147 ///
148 /// NOT `disabled_reason`: a cadence that assembled cleanly at boot keeps
149 /// sweeping on its boot-time io, so a manual refusal says the config AS IT
150 /// STANDS NOW would not assemble — a warning about the next state, not a
151 /// statement that the loop is off.
152 pub run_refusal: Option<String>,
153 /// The model the coder will run on, and which file pinned it.
154 ///
155 /// Two states, not three. `None` means unpinned — neither `heal.toml`'s
156 /// `coder_model` nor `coder.toml`'s `[coder] model` names one, so adaptive
157 /// routing picks per request and nothing before the run can say what it
158 /// will pick. ("Pinned to something the panel rejects" is a
159 /// `disabled_reason` state, not one of this field's.)
160 ///
161 /// **What the RUNNING cadence uses may differ.** `spawn_heal_cadence`
162 /// calls `live_io` once at boot and bakes the pin into the runner, so this
163 /// reports the current files while a cadence started earlier keeps using
164 /// the boot-time value — the same freeze `panel_composition` documents for
165 /// the panel. Read it as what the next `heal.run` would use.
166 ///
167 /// Reported because car#1334 made this pin load-bearing: the panel
168 /// independence check refuses a sweep when the coder is also a review seat,
169 /// and the pin it checks lives in whichever of two files won. Deriving that
170 /// by hand means reading both and re-implementing the precedence — which is
171 /// exactly the re-derivation that produced car#1360.
172 pub coder_pin: Option<CoderPinView>,
173 pub engine: String,
174}
175
176/// The resolved coder pin, as `heal.status` reports it.
177#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
178pub struct CoderPinView {
179 /// The model id, verbatim — not canonicalized. A `coder.toml` value is
180 /// legitimately outside CAR's registry (on the external rung it is
181 /// forwarded to a third-party CLI's own namespace), so showing the
182 /// operator what the file says is the useful answer.
183 pub model: String,
184 /// Which file won, as a machine value: `heal_toml` or `coder_toml`.
185 ///
186 /// Not prose. Every non-CLI consumer — a host app, a script, a human
187 /// reading raw JSON — would otherwise have to strip markdown backticks out
188 /// of a JSON-RPC field; the operator-facing wording is the renderer's job.
189 pub source: String,
190}
191
192#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
193pub struct RejectedTargetView {
194 pub repo: String,
195 pub reason: String,
196}
197
198impl HealService {
199 /// Construct with a config already loaded, and the directory it came from.
200 pub fn with_config_dir(
201 config: HealConfig,
202 state_dir: std::path::PathBuf,
203 config_dir: std::path::PathBuf,
204 ) -> Self {
205 let mut s = Self::new(config, state_dir);
206 s.config_dir = Some(config_dir);
207 s
208 }
209
210 /// Re-read `heal.toml`, returning the current configuration.
211 ///
212 /// A read failure is not distinguishable from an absent file by design
213 /// (`HealConfig::load` returns an empty config either way), which is why
214 /// the empty case disables the loop rather than doing anything.
215 fn reload(&self) -> HealConfig {
216 // No file behind this service: the held config IS the configuration,
217 // and re-reading would throw away what the caller supplied.
218 let Some(dir) = &self.config_dir else {
219 return self.held();
220 };
221 let fresh = HealConfig::load(dir);
222 if let Ok(mut held) = self.config.write() {
223 *held = fresh.clone();
224 }
225 fresh
226 }
227
228 /// The configuration as last read, without touching the disk.
229 fn held(&self) -> HealConfig {
230 match self.config.read() {
231 Ok(c) => c.clone(),
232 // A poisoned lock means a panic while holding it. An empty config
233 // disables the loop, which is the safe answer when the state that
234 // says what to do cannot be trusted.
235 Err(_) => HealConfig::default(),
236 }
237 }
238
239 fn config(&self) -> HealConfig {
240 self.held()
241 }
242
243 pub fn new(config: HealConfig, state_dir: std::path::PathBuf) -> Self {
244 let interval_secs = std::env::var(HEAL_INTERVAL_ENV)
245 .ok()
246 .and_then(|v| v.parse::<u64>().ok())
247 .filter(|v| *v > 0)
248 .unwrap_or(DEFAULT_INTERVAL_SECS);
249 Self {
250 config: std::sync::RwLock::new(config),
251 config_dir: None,
252 running: Mutex::new(()),
253 state_dir,
254 interval_secs,
255 assembly_error: std::sync::RwLock::new(None),
256 run_refusal: std::sync::RwLock::new(None),
257 }
258 }
259
260 /// Record why the loop could not be assembled, so `heal.status` can report
261 /// an enabled-but-dead loop instead of leaving it to a boot log line.
262 pub fn record_assembly_error(&self, error: &str) {
263 if let Ok(mut slot) = self.assembly_error.write() {
264 *slot = Some(error.to_string());
265 }
266 }
267
268 /// The coder pin in force, resolved ONCE for both `status` and
269 /// `live_io_for`.
270 ///
271 /// Used by both so the two cannot disagree — the same reason
272 /// `panel_composition` exists. `config::session_model` is the single copy
273 /// of the PRECEDENCE rule; this is the single copy of "read `coder.toml`
274 /// only when `heal.toml` does not pin", which is a second rule and was
275 /// briefly a second copy.
276 ///
277 /// `coder.toml` is read only when it can matter: a malformed one should not
278 /// warn on every sweep for a value `heal.toml` overrides. Its unreadable,
279 /// malformed and absent cases all collapse to an empty config, so all three
280 /// read as unpinned here.
281 fn resolved_coder_pin(config: &HealConfig) -> Option<(String, super::config::PinSource)> {
282 let coder_toml = config
283 .coder_model
284 .is_none()
285 .then(super::config::CoderConfig::load);
286 super::config::session_model(
287 config.coder_model.as_deref(),
288 coder_toml.as_ref().and_then(|c| c.model.as_deref()),
289 )
290 .map(|(m, src)| (m.to_string(), src))
291 }
292
293 /// Record why a manual `heal.run` refused, or clear it on a run that got
294 /// past assembly. Both arms matter: without the clearing one an operator
295 /// who fixes the config keeps seeing the old refusal until they restart the
296 /// daemon.
297 fn set_run_refusal(&self, error: Option<&str>) {
298 if let Ok(mut slot) = self.run_refusal.write() {
299 *slot = error.map(str::to_string);
300 }
301 }
302
303 pub fn is_enabled(&self) -> bool {
304 self.config().is_enabled()
305 }
306
307 pub fn interval_secs(&self) -> u64 {
308 self.interval_secs
309 }
310
311 /// Every target that could not be used, so a misconfiguration is visible
312 /// rather than presenting as a loop that never does anything.
313 pub fn rejected(&self) -> Vec<super::heal_config::RejectedTarget> {
314 self.config().rejected
315 }
316
317 /// What the loop is configured to do, and why it is not doing it.
318 pub fn status(&self, engine: &Arc<car_inference::InferenceEngine>) -> HealStatus {
319 // Re-read, so an operator who just fixed `heal.toml` is shown what the
320 // next sweep will actually use rather than what booted.
321 let config = self.reload();
322 let panel = self.panel_composition(engine, &config.review_models);
323 let coder_pin = Self::resolved_coder_pin(&config);
324 HealStatus {
325 config_path: self
326 .config_dir
327 .as_ref()
328 .map(|d| d.join(HEAL_CONFIG_FILE).display().to_string())
329 .unwrap_or_else(|| "(supplied directly; no file)".into()),
330 enabled: config.is_enabled(),
331 cadence_secs: self.interval_secs,
332 // Config first (it names a missing panel or missing targets), then
333 // the assembly failure — which is the only one that can leave
334 // `enabled` true.
335 disabled_reason: config.disabled_reason().map(str::to_string).or_else(|| {
336 self.assembly_error
337 .read()
338 .ok()
339 .and_then(|e| e.clone())
340 .map(|e| format!("the loop could not be assembled: {e}"))
341 }),
342 run_refusal: self.run_refusal.read().ok().and_then(|e| e.clone()),
343 targets: config.targets.iter().map(|t| t.repo.clone()).collect(),
344 rejected: config
345 .rejected
346 .iter()
347 .map(|r| RejectedTargetView {
348 repo: r.repo.clone(),
349 reason: r.reason.clone(),
350 })
351 .collect(),
352 review_models: config.review_models.clone(),
353 // Composition, not just names: a single-vendor panel is invisible in
354 // a list of model ids, and the operator reading `3/3 approved` is
355 // the person who needs to know it was one vendor three times.
356 panel: panel.clone(),
357 panel_warning: super::heal_review::correlation_warning(&panel),
358 // Resolved through the same `session_model` the sweep's check uses,
359 // not a second copy of the precedence rule. `coder.toml` is read
360 // only when `heal.toml` does not pin, matching the sweep — an
361 // unreadable `coder.toml` that nothing consults must not turn into
362 // a status field that disagrees with what will run.
363 coder_pin: coder_pin.map(|(model, source)| CoderPinView {
364 model,
365 // Named for the FILE, not for `PinSource`'s generic
366 // request/config spelling: heal passes `heal.toml`'s
367 // `coder_model` as the request pin, so "request" would tell an
368 // operator nothing about which file to open.
369 source: match source {
370 super::config::PinSource::Request => "heal_toml",
371 super::config::PinSource::Config => "coder_toml",
372 }
373 .to_string(),
374 }),
375 engine: config
376 .engine
377 .clone()
378 .unwrap_or_else(|| DEFAULT_ENGINE.to_string()),
379 }
380 }
381
382 /// Resolve each configured seat to its serving vendor.
383 ///
384 /// Used by both `status` and `live_io_for` so the two cannot disagree about
385 /// how a seat resolves. They can still describe different PANELS: `status`
386 /// re-reads `heal.toml` every call, while the cadence's `TickIo` is
387 /// assembled once at boot and reused, so an edit shows up in `heal.status`
388 /// before the running loop picks it up (it takes a restart). That freeze
389 /// predates this and is not what this function is claiming to fix.
390 fn panel_composition(
391 &self,
392 engine: &std::sync::Arc<car_inference::InferenceEngine>,
393 models: &[String],
394 ) -> Vec<super::heal_review::PanelSeat> {
395 super::heal_review::composition(models, |m| {
396 engine
397 .model_schema(m)
398 .and_then(|s| s.vendor())
399 .map(str::to_string)
400 })
401 }
402
403 /// Assemble the live loop: real GitHub, real coder, real review panel.
404 ///
405 /// **One constructor, one call site.** Every dependency below already
406 /// existed and none of them were ever wired together — `LiveTickIo` had no
407 /// construction site in the workspace at all, so the loop was a complete,
408 /// tested, unreachable subsystem. A second assembly point (a CLI building
409 /// its own in-process) would be the next copy of that mistake, and would
410 /// also bypass the single-sweep try-lock, so `heal.run` and the cadence
411 /// both come through here.
412 pub fn live_io(&self, state: &Arc<ServerState>) -> Result<Arc<dyn TickIo>, String> {
413 self.live_io_for(state, &self.config())
414 }
415
416 fn live_io_for(
417 &self,
418 state: &Arc<ServerState>,
419 config: &HealConfig,
420 ) -> Result<Arc<dyn TickIo>, String> {
421 if config.review_models.is_empty() {
422 // `decide` refuses a panel of zero, so continuing would run a full
423 // coder session per item and reject every one of them.
424 return Err("no `review_models` configured; refusing to run without a panel".into());
425 }
426 // Validate the panel HERE, where the catalog is reachable, rather than
427 // discovering a typo after a full coder session: an unknown id errors
428 // at generation, which the gate reads as an unreachable seat, so a
429 // misspelled model costs a session and a backoff per tick before
430 // anyone learns why. Refusing is not the same as dropping the seat — a
431 // panel silently shrunk from three to two is the halved threshold the
432 // `PanelIncomplete` design exists to prevent.
433 //
434 // `knows_model`, not `list_models`: the latter is the ON-DEVICE
435 // catalog, so checking a cloud model id against it reports every
436 // frontier model as unknown and would refuse a correct configuration —
437 // a false refusal disables the whole feature, which is worse than the
438 // cost it was added to avoid.
439 let engine_handle = crate::handler::get_inference_engine(state);
440 // The coder is validated with the seats, not after them: the argument
441 // above applies with more force to the model that runs the EXPENSIVE
442 // half, and an unvalidated coder name is also what would force the
443 // disjointness check below to guess at a canonical form.
444 let unknown: Vec<&str> = config
445 .review_models
446 .iter()
447 .chain(config.coder_model.iter())
448 .filter(|m| !engine_handle.knows_model(m))
449 .map(String::as_str)
450 .collect();
451 if !unknown.is_empty() {
452 return Err(format!(
453 "unknown model(s) in `heal.toml`: {} — a seat that cannot be reached is \
454 not a reviewer, and dropping it would quietly shrink the panel. Run \
455 `car models list` for the names this daemon knows.",
456 unknown.join(", ")
457 ));
458 }
459
460 // Which model the coder will run on, resolved ONCE. `coder.start`
461 // takes `coder_model` as its request pin and falls back to
462 // `~/.car/coder.toml`'s `[coder] model`, so an unset `coder_model` does
463 // not mean the coder is unpinned — it means it runs on that file's
464 // pin, which nothing checked against the panel (car#1360). Read only
465 // when it can matter; a malformed coder.toml should not warn on every
466 // sweep for a value heal.toml overrides.
467 let coder_pin = Self::resolved_coder_pin(config);
468
469 // Refused rather than dropped: a seat quietly removed lowers the
470 // majority threshold without saying so, which is what `PanelIncomplete`
471 // exists to prevent. The rule and its message live in
472 // `check_coder_pin`, where they are table-tested; the registry reaches
473 // it as the one closure.
474 //
475 // Every SEAT cleared `knows_model` above, so seats always canonicalize
476 // to a real id — but the coder pin may not have, because a coder.toml
477 // value is legitimately outside CAR's registry: on the external rung it
478 // is forwarded verbatim to a third-party CLI's own namespace. That is
479 // what `canonicalizer`'s fallback is for.
480 check_coder_pin(
481 coder_pin.as_ref().map(|(m, src)| (m.as_str(), *src)),
482 &config.review_models,
483 &canonicalizer(|m| engine_handle.model_schema(m).map(|s| s.id.clone())),
484 )?;
485
486 let engine = match config.engine.as_deref() {
487 Some(name) => super::router::EngineChoice::parse(name)?,
488 None => super::router::EngineChoice::parse(DEFAULT_ENGINE)?,
489 };
490
491 // The minimum diversity floor is enforced here, after every seat has a
492 // catalog-backed serving vendor. Refuse rather than dropping seats: a
493 // smaller panel has a different majority threshold and is a different
494 // configuration. The stronger majority-capture condition remains a
495 // warning for panels that do span two providers but can still be
496 // carried by one of them.
497 let panel = self.panel_composition(engine_handle, &config.review_models);
498 if let Some(error) = super::heal_review::panel_diversity_error(&panel) {
499 return Err(error);
500 }
501 if let Some(warning) = super::heal_review::correlation_warning(&panel) {
502 tracing::warn!(target: "car::heal", "{warning}");
503 }
504
505 // Adaptive inference reports `ModelSchema.name`, so pass the panel in
506 // that same namespace. `build_exclude_set` resolves either names or ids,
507 // but using the result namespace here also makes the list directly
508 // comparable with the durable author attribution the delivery backstop
509 // reads. Validation above guarantees every seat resolves.
510 let mut routing_exclusions = Vec::new();
511 for seat in &config.review_models {
512 if let Some(name) = engine_handle.model_schema(seat).map(|schema| &schema.name) {
513 if !routing_exclusions.contains(name) {
514 routing_exclusions.push(name.clone());
515 }
516 }
517 }
518
519 let runner = super::heal_runner::LiveCoderRunner {
520 state: state.clone(),
521 // The production `TurnGenerator` is the inference engine itself.
522 generator: crate::handler::get_inference_engine(state).clone(),
523 state_dir: self.state_dir.clone(),
524 reviewers: super::heal_review::panel(state, &config.review_models),
525 max_wall_secs: super::heal_runner::DEFAULT_ITEM_WALL_SECS,
526 max_iterations: None,
527 engine,
528 // The value that was CHECKED above, not `coder_model` — otherwise
529 // `coder.start` re-reads coder.toml at session time and the pin
530 // the runner ends up on is not the one the panel check saw.
531 model: coder_pin.map(|(m, _)| m),
532 routing_exclusions,
533 // The registry resolver the gate's self-review check canonicalizes
534 // through, on the same fallback policy as the assembly check above
535 // — and it matters more here, because the gate's author comes from
536 // the engine unvalidated.
537 canonical_model: {
538 let engine_handle = engine_handle.clone();
539 Arc::new(canonicalizer(move |m: &str| {
540 engine_handle.model_schema(m).map(|s| s.id.clone())
541 }))
542 },
543 github: Arc::new(super::merge::GhCli::default()),
544 };
545
546 Ok(Arc::new(super::heal_live::LiveTickIo {
547 issues: Arc::new(super::fix_issues::GhIssues),
548 prs: Arc::new(super::heal_intake::GhPullRequests),
549 oracle: Arc::new(super::provenance::GhPermissions),
550 coder: Arc::new(runner),
551 // EMPTY, deliberately, and it narrows what the loop will act on.
552 //
553 // `LocalSignatures` holds the signatures of issues this runtime
554 // itself filed, and it is the only route to the `Runtime` tier —
555 // the one tier that may source an outcome contract from an issue
556 // body. Nothing in the daemon files issues today (`car-selfheal` is
557 // watch-only by design), so there are no such signatures to supply
558 // and inventing a non-empty set would be asserting authorship this
559 // process cannot demonstrate.
560 //
561 // The consequence, stated rather than discovered: every item the
562 // loop acts on is maintainer-authored or better, and no issue body
563 // ever sources a contract. That is the safer posture, and it is the
564 // one the proposal asks be revisited *before* runtime filing is
565 // enabled — not after.
566 local_signatures: super::provenance::LocalSignatures::from_proposals(&[]),
567 redactor: car_selfheal::redact::Redactor::from_env(std::env::vars()),
568 panel,
569 }))
570 }
571
572 /// One sweep, assembling the live loop first. The manual-run entry point.
573 pub async fn run_tick(&self, state: &Arc<ServerState>) -> Result<SweepReport, String> {
574 // Re-read first: a manual run is exactly when an operator has just
575 // edited the file.
576 let config = self.reload();
577 // Recorded, not just returned. `spawn_heal_cadence` records its
578 // assembly failure so a boot-time refusal still reaches `heal.status` a
579 // week later; a manual `heal.run` refusing for the same reason left no
580 // trace, so `heal.status` could show a config `heal.run` will not run
581 // on. The caller still gets the error — this only stops it from being
582 // the ONLY place it appears.
583 //
584 // A `match` rather than `inspect_err`: this needs a SUCCESS arm. The
585 // slot has to be cleared by a run that assembles, or an operator who
586 // fixes the config keeps reading the old refusal forever.
587 let io = match self.live_io_for(state, &config) {
588 Ok(io) => {
589 self.set_run_refusal(None);
590 io
591 }
592 Err(e) => {
593 self.set_run_refusal(Some(&e));
594 return Err(e);
595 }
596 };
597 Ok(self.sweep_with(&io, &config).await)
598 }
599
600 /// One pass over every configured target, at most one item each.
601 ///
602 /// Every target, not "the first with work": stopping early would let a busy
603 /// repository starve every entry after it in the config, and the operator
604 /// who listed them has no way to see that happening.
605 pub async fn sweep(&self, io: &Arc<dyn TickIo>) -> SweepReport {
606 let config = self.reload();
607 self.sweep_with(io, &config).await
608 }
609
610 async fn sweep_with(&self, io: &Arc<dyn TickIo>, config: &HealConfig) -> SweepReport {
611 let Ok(_guard) = self.running.try_lock() else {
612 // A tick is already running. Queueing would just do the same work
613 // twice, more slowly.
614 return SweepReport {
615 outcomes: Vec::new(),
616 skipped_overlap: true,
617 };
618 };
619
620 let mut claims = ClaimStore::load(&self.state_dir);
621 let mut outcomes = Vec::new();
622 let sink = FileClaimSink {
623 dir: self.state_dir.clone(),
624 };
625
626 for target in &config.targets {
627 // A fresh id per tick. A stable one would make this run the holder
628 // of every claim it ever took, and claiming would silently do
629 // nothing.
630 let run_id = format!("heal-{}", uuid::Uuid::new_v4().simple());
631 let out = tick(io, target, &mut claims, &run_id, &sink).await;
632
633 // Again after the target, to record the outcome — the CLAIM
634 // already reached disk inside `tick`, before any work started.
635 // Saving only here lost it whenever the daemon died during the
636 // 45-minute session the claim was taken for, and the next start
637 // re-picked an item whose work was still in flight.
638 sink.persist(&claims, io.now_ms());
639 outcomes.push((target.repo.clone(), out));
640 }
641
642 SweepReport {
643 outcomes,
644 skipped_overlap: false,
645 }
646 }
647}
648
649// There is deliberately NO worktree reaper here.
650//
651// There was one, and it was worse than nothing: it swept
652// `<state_dir>/heal-worktrees`, a directory nothing has ever created. Coder
653// sessions provision under `<state_dir>/worktrees` (`CoderSession::
654// provision_workspace`). So the guard whose rationale was the 102 GB incident
655// scanned an empty path, always returned 0, and logged nothing — a
656// disk-exhaustion protection that reported success while protecting nothing.
657//
658// Repointing it at `worktrees` would have been worse still: that directory is
659// shared with every *interactive* `coder.start`, so an age-based sweep would
660// delete a human's overnight `NeedsApproval` worktree.
661//
662// The real fix was upstream. `LiveCoderRunner` now drives every session it
663// starts to a terminal state — `Merged` on delivery, `Abandoned` on rejection,
664// `Failed` on timeout — and a terminal transition drops the `AgentWorkspace`
665// RAII handle, which is what removes the worktree and its `git worktree`
666// registration. Sessions clean up after themselves; nothing has to sweep.
667
668/// Writes the claim ledger to the loop's state directory.
669///
670/// A failure is logged, not propagated: a disk that refuses the write must not
671/// fail a sweep that is otherwise fine, and the in-memory ledger still holds
672/// the claim for the rest of this sweep.
673struct FileClaimSink {
674 dir: std::path::PathBuf,
675}
676
677impl ClaimSink for FileClaimSink {
678 fn persist(&self, claims: &ClaimStore, now_ms: u64) {
679 if let Err(e) = claims.save(&self.dir, now_ms) {
680 tracing::warn!(error = %e, "could not persist heal claims");
681 }
682 }
683}
684
685/// Run the loop on its cadence until the daemon stops.
686///
687/// Returns `None` when no targets are configured, so a daemon with no
688/// `heal.toml` spawns nothing at all rather than a task that wakes to do
689/// nothing forever.
690pub fn spawn_heal_cadence(state: Arc<ServerState>) -> Option<tokio::task::JoinHandle<()>> {
691 let service = state.heal.clone();
692 if !service.is_enabled() {
693 // Named, not silent. `disabled_reason` distinguishes "nothing is
694 // configured" from "targets are configured but there is no review
695 // panel", and the second reads exactly like the first from a log that
696 // only says "not starting".
697 tracing::info!(
698 reason = service
699 .config()
700 .disabled_reason()
701 .unwrap_or("no targets configured"),
702 "self-healing loop not started"
703 );
704 return None;
705 }
706 for r in service.rejected() {
707 tracing::warn!(repo = %r.repo, reason = %r.reason, "heal target ignored");
708 }
709
710 // Assemble ONCE, at startup, so a missing credential or an unparseable
711 // engine name is a boot-time error in the log rather than a failure
712 // rediscovered on every tick forever.
713 let io = match service.live_io(&state) {
714 Ok(io) => io,
715 Err(e) => {
716 // Recorded before returning: a log line at boot is not reachable by
717 // the operator who runs `car heal status` a week later.
718 service.record_assembly_error(&e);
719 tracing::warn!(error = %e, "self-healing loop could not be assembled; not starting");
720 return None;
721 }
722 };
723 let secs = service.interval_secs();
724 tracing::info!(interval_secs = secs, "self-healing loop started");
725 Some(tokio::spawn(async move {
726 let mut ticker = tokio::time::interval(std::time::Duration::from_secs(secs));
727 // The first tick fires immediately; skip it so a daemon restart does
728 // not start a coder session before the operator has seen it come up.
729 ticker.tick().await;
730 loop {
731 ticker.tick().await;
732 let report = service.sweep(&io).await;
733 if report.skipped_overlap {
734 tracing::debug!("heal sweep skipped: previous sweep still running");
735 continue;
736 }
737 for (repo, out) in &report.outcomes {
738 match out {
739 TickOutcome::Opened {
740 number,
741 pr_url,
742 ci,
743 delivery,
744 ..
745 } => tracing::info!(
746 %repo,
747 number,
748 %pr_url,
749 head_sha = %ci.head_sha,
750 ci_state = ?ci.state,
751 %delivery,
752 "self-heal opened a pull request"
753 ),
754 TickOutcome::Rejected { number, gate, .. } => {
755 tracing::info!(%repo, number, %gate, "self-heal stopped at the gate")
756 }
757 TickOutcome::Failed { detail } => {
758 tracing::warn!(%repo, %detail, "self-heal tick failed")
759 }
760 TickOutcome::Idle { .. } => {}
761 }
762 }
763 }
764 }))
765}
766
767/// The registry canonicalizer both self-review checks compare through.
768///
769/// `unwrap_or_else(|| m.to_string())`, never `unwrap_or_default()`. Two names
770/// the registry cannot resolve would both collapse to `""` and match each
771/// other, and here that means refusing every session on a daemon whose catalog
772/// does not hold the model that ran. Falling back to the name itself degrades
773/// to a spelling comparison instead — the honest answer when the registry has
774/// nothing to say. Written once so the policy has one place to be wrong.
775fn canonicalizer(resolve: impl Fn(&str) -> Option<String>) -> impl Fn(&str) -> String {
776 move |m| resolve(m).unwrap_or_else(|| m.to_string())
777}
778
779/// Refuse an assembly where the model that will WRITE the change also sits on
780/// the panel that will judge it.
781///
782/// Takes the pin already resolved — `(model, source)` from
783/// [`config::session_model`](super::config::session_model) — rather than the
784/// two raw sources. Two adjacent `Option<&str>` parameters can be transposed
785/// silently, and getting the precedence backwards is the defect this function
786/// exists to fix; there is nothing to transpose here. The source rides along so
787/// the message can name the file the operator has to edit, without this
788/// re-deriving precedence from its own copy of the rule.
789///
790/// Refused rather than dropping the seat: a panel silently shrunk from three
791/// to two is the halved threshold `PanelIncomplete` exists to prevent. Two
792/// cases are deliberately NOT refused here:
793///
794/// - **An unpinned coder** (neither file pins, the default). The router picks
795/// per request, so only `heal_runner`'s gate, reading what actually authored
796/// the change, can answer it (car#1299).
797/// - **A pin this daemon's registry does not know.** heal validates its own
798/// `coder_model` against the catalog, but `coder.toml` is a shared file no
799/// other consumer validates, and on the external rung its value is forwarded
800/// verbatim to a CLI's own namespace — so a name `car models list` has never
801/// heard of can be exactly right for the rung that runs. Refusing would take
802/// the whole loop offline over a working configuration, which is the false
803/// refusal `correlation_warning` and the `knows_model` comment upstream both
804/// decline to make.
805fn check_coder_pin(
806 pin: Option<(&str, super::config::PinSource)>,
807 seats: &[String],
808 canonical: &dyn Fn(&str) -> String,
809) -> Result<(), String> {
810 let Some((coder, source)) = pin else {
811 return Ok(());
812 };
813 let Some(seat) = super::heal_review::coder_on_panel(coder, seats, canonical) else {
814 return Ok(());
815 };
816 Err(format!(
817 "the coder model {} ({}) is also a review seat ({}) — a model cannot review its \
818 own output, and counting it as a reviewer reports an independence the panel does \
819 not have. Remove it from `review_models`, or pin a different coder.",
820 coder,
821 match source {
822 super::config::PinSource::Request => "`coder_model` in `heal.toml`",
823 super::config::PinSource::Config => "`model` in `coder.toml`",
824 },
825 seat
826 ))
827}
828
829#[cfg(test)]
830mod tests {
831 use super::*;
832 use crate::coder::heal_intake::{Checkout, HealTarget};
833
834 struct NoopIo;
835
836 #[async_trait::async_trait]
837 impl TickIo for NoopIo {
838 async fn candidates(
839 &self,
840 _t: &HealTarget,
841 ) -> Result<Vec<crate::coder::heal_select::Candidate>, String> {
842 Ok(vec![])
843 }
844 async fn open_prs(
845 &self,
846 _t: &HealTarget,
847 ) -> Result<Vec<crate::coder::heal_intake::RawPullRequest>, String> {
848 Ok(vec![])
849 }
850 async fn intent_for(
851 &self,
852 _i: &crate::coder::heal_select::Candidate,
853 ) -> Result<crate::coder::heal_tick::Intent, String> {
854 Ok(crate::coder::heal_tick::Intent::Gone)
855 }
856 fn redact(&self, text: &str) -> String {
857 text.to_string()
858 }
859 async fn run_coder(
860 &self,
861 _t: &HealTarget,
862 _i: &crate::coder::heal_select::Candidate,
863 _s: &crate::coder::provenance::SessionSeed,
864 ) -> Result<crate::coder::heal_tick::Attempt, crate::coder::heal_tick::RunFailure> {
865 Err(crate::coder::heal_tick::RunFailure::early("not reached"))
866 }
867 async fn deliver(
868 &self,
869 _t: &HealTarget,
870 _i: &crate::coder::heal_select::Candidate,
871 _s: &str,
872 _g: &crate::coder::heal_gate::GateOutcome,
873 ) -> Result<crate::coder::merge::PrDeliveryOutcome, crate::coder::heal_tick::DeliverRefusal>
874 {
875 Err(crate::coder::heal_tick::DeliverRefusal::retriable(
876 "not reached",
877 ))
878 }
879 async fn abandon(&self, _s: &str) {}
880 async fn comment(
881 &self,
882 _i: &crate::coder::heal_select::Candidate,
883 _t: &str,
884 ) -> Result<(), String> {
885 Ok(())
886 }
887 fn now_ms(&self) -> u64 {
888 1_000_000
889 }
890 }
891
892 fn cfg(targets: Vec<HealTarget>) -> HealConfig {
893 HealConfig {
894 targets,
895 rejected: vec![],
896 review_models: vec!["reviewer-a".into()],
897 engine: None,
898 coder_model: None,
899 }
900 }
901
902 fn target(repo: &str) -> HealTarget {
903 HealTarget {
904 repo: repo.into(),
905 fix_repo: None,
906 checkout: Some(Checkout::Project("p".into())),
907 label: "self-heal".into(),
908 base: "main".into(),
909 }
910 }
911
912 /// `status` resolves each seat's vendor through the registry, so it needs
913 /// the engine — and only the engine.
914 fn engine() -> Arc<car_inference::InferenceEngine> {
915 Arc::new(car_inference::InferenceEngine::new(Default::default()))
916 }
917
918 /// A daemon with no `heal.toml` spawns nothing at all — not a task that
919 /// wakes every fifteen minutes to find no targets.
920 #[test]
921 fn no_targets_means_the_loop_is_disabled() {
922 let dir = tempfile::tempdir().unwrap();
923 let s = HealService::new(cfg(vec![]), dir.path().into());
924 assert!(!s.is_enabled());
925 assert_eq!(
926 s.status(&engine()).disabled_reason.as_deref(),
927 Some("no usable targets are configured")
928 );
929 }
930
931 /// A registry stand-in with the car#889 hazard in it: `alias` is a name
932 /// whose canonical id is something else, so a spelling comparison between
933 /// the two never fires. Everything else is unknown to it.
934 ///
935 /// Built through the PRODUCTION `canonicalizer`, not hand-rolled — so the
936 /// unresolvable-name policy these tests rely on is the one that ships, and
937 /// changing that fallback turns them red.
938 fn canon(m: &str) -> String {
939 canonicalizer(|m: &str| (m == "alias").then(|| "vendor/real".to_string()))(m)
940 }
941
942 fn seats() -> Vec<String> {
943 vec!["gpt-5.6".into(), "vendor/real".into(), "gpt-5.4".into()]
944 }
945
946 /// Resolve exactly as `live_io_for` does, so these cases exercise the
947 /// precedence rule and the check together rather than the check alone.
948 fn check(heal: Option<&str>, coder_toml: Option<&str>) -> Result<(), String> {
949 check_coder_pin(
950 super::super::config::session_model(heal, coder_toml),
951 &seats(),
952 &canon,
953 )
954 }
955
956 /// The defect car#1360 is: heal.toml pinning nothing does not mean the
957 /// coder is unpinned, it means `coder.toml`'s `model` is the pin — and
958 /// that source was never checked against the panel.
959 #[test]
960 fn a_coder_toml_pin_that_is_a_review_seat_is_refused() {
961 let err = check(None, Some("gpt-5.6")).expect_err("coder.toml pinned a seat");
962 assert!(err.contains("gpt-5.6"), "{err}");
963 assert!(err.contains("`model` in `coder.toml`"), "{err}");
964 // Naming the wrong file sends the operator to edit a key that isn't
965 // set, which is how a correct refusal still costs an afternoon.
966 assert!(!err.contains("`coder_model` in `heal.toml`"), "{err}");
967 }
968
969 /// Canonicalization is load-bearing, not cosmetic: the pin is spelled as a
970 /// name and the seat as an id. A gate that cannot fire reads as covered.
971 #[test]
972 fn a_coder_toml_pin_is_matched_through_the_registry_not_by_spelling() {
973 let err = check(None, Some("alias")).expect_err("alias canonicalizes onto vendor/real");
974 assert!(err.contains("vendor/real"), "{err}");
975 }
976
977 /// heal.toml still wins where it is set, and the message says so — the
978 /// coder.toml value is not what runs, so refusing on it would be wrong.
979 #[test]
980 fn the_heal_toml_pin_wins_and_is_the_one_checked() {
981 let err = check(Some("gpt-5.4"), Some("claude-sonnet-5")).expect_err("heal.toml pinned");
982 assert!(err.contains("`coder_model` in `heal.toml`"), "{err}");
983
984 // And the converse: a coder.toml pin sitting on the panel is harmless
985 // when heal.toml overrides it, because it never runs.
986 check(Some("claude-sonnet-5"), Some("gpt-5.6")).expect("the pin that runs is not a seat");
987 }
988
989 /// A blank `coder_model` is "unset", not "pin blank" — the only input where
990 /// precedence and the source label could disagree and name the wrong file.
991 #[test]
992 fn a_blank_heal_pin_falls_through_and_the_message_names_coder_toml() {
993 let err = check(Some(" "), Some("gpt-5.6")).expect_err("blank falls through");
994 assert!(err.contains("`model` in `coder.toml`"), "{err}");
995 }
996
997 /// Neither source pins: the router picks per request, so nothing here can
998 /// say what it will pick. `heal_runner`'s gate answers that one, against
999 /// what actually authored the change.
1000 #[test]
1001 fn an_unpinned_coder_is_not_this_checks_to_refuse() {
1002 check(None, None).expect("unpinned is not a seat");
1003 }
1004
1005 /// A pin outside CAR's registry is NOT refused. `coder.toml` is shared with
1006 /// `coder.start` and `car code-task`, neither of which validates it, and on
1007 /// the external rung the value is handed verbatim to a CLI's own namespace
1008 /// — so an unknown name can be exactly right for the rung that runs, and
1009 /// taking the whole loop offline over it is the false refusal this file
1010 /// declines to make elsewhere.
1011 ///
1012 /// It still has to be COMPARED, though. The canonicalizer falls back to the
1013 /// name itself rather than `""`, so an unresolvable pin degrades to a
1014 /// spelling match instead of silently matching nothing.
1015 #[test]
1016 fn a_pin_outside_the_registry_is_compared_not_refused() {
1017 // Unknown and not a seat: allowed through.
1018 check(None, Some("codex-mini")).expect("an unknown pin is not by itself a refusal");
1019 // Unknown to `canon` but spelled exactly like a seat: still caught. A
1020 // canonicalizer collapsing the unknown to `""` would miss this.
1021 let err = check(None, Some("gpt-5.4")).expect_err("spelling still matches a seat");
1022 assert!(err.contains("gpt-5.4"), "{err}");
1023 }
1024
1025 /// Targets but no panel is a DIFFERENT disabled /// Targets but no panel is a DIFFERENT disabled, and the difference is the
1026 /// whole point of reporting a reason: `decide` refuses a panel of zero, so
1027 /// running would mean a full coder session per item followed by a
1028 /// guaranteed rejection.
1029 #[test]
1030 fn targets_without_a_review_panel_do_not_enable_the_loop() {
1031 let dir = tempfile::tempdir().unwrap();
1032 let mut c = cfg(vec![target("acme/one")]);
1033 c.review_models.clear();
1034 let s = HealService::new(c, dir.path().into());
1035 assert!(!s.is_enabled());
1036 assert!(s
1037 .status(&engine())
1038 .disabled_reason
1039 .unwrap()
1040 .contains("review_models"));
1041 }
1042
1043 /// `heal.status` must report the panel it will actually use, resolved.
1044 ///
1045 /// The wiring, not the rule: `correlation_warning` is table-tested next
1046 /// door, and every defect this file has had was in getting the right value
1047 /// to it.
1048 #[test]
1049 fn status_resolves_the_panel_and_reports_a_correlated_one() {
1050 let dir = tempfile::tempdir().unwrap();
1051 let mut c = cfg(vec![target("acme/one")]);
1052 // Real catalog ids: both are OpenAI, so one vendor holds every seat.
1053 c.review_models = vec!["gpt-5.4".into(), "gpt-5.5".into()];
1054 let s = HealService::new(c, dir.path().into());
1055 let st = s.status(&engine());
1056
1057 assert_eq!(
1058 st.panel
1059 .iter()
1060 .map(|p| p.model.as_str())
1061 .collect::<Vec<_>>(),
1062 vec!["gpt-5.4", "gpt-5.5"]
1063 );
1064 assert!(
1065 st.panel
1066 .iter()
1067 .all(|p| p.vendor.as_deref() == Some("openai")),
1068 "both seats must resolve to openai: {:?}",
1069 st.panel
1070 );
1071 let w = st
1072 .panel_warning
1073 .expect("a one-vendor panel must be reported");
1074 assert!(w.contains("openai serves 2 of the 2 seats"), "{w}");
1075 }
1076
1077 /// An independent panel carries no warning, so the warning means something.
1078 #[test]
1079 fn status_does_not_warn_about_a_panel_spanning_vendors() {
1080 let dir = tempfile::tempdir().unwrap();
1081 let mut c = cfg(vec![target("acme/one")]);
1082 c.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1083 let s = HealService::new(c, dir.path().into());
1084 let st = s.status(&engine());
1085 assert_eq!(
1086 st.panel
1087 .iter()
1088 .filter_map(|p| p.vendor.as_deref())
1089 .collect::<Vec<_>>(),
1090 vec!["openai", "anthropic"]
1091 );
1092 assert_eq!(st.panel_warning, None);
1093 }
1094
1095 #[test]
1096 fn assembly_refuses_a_three_seat_single_provider_panel() {
1097 let dir = tempfile::tempdir().unwrap();
1098 let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1099 let mut c = cfg(vec![target("acme/one")]);
1100 c.review_models = vec!["gpt-5.4".into(), "gpt-5.5".into(), "gpt-5.6-sol".into()];
1101 c.coder_model = Some("claude-opus-5".into());
1102 let s = HealService::new(c.clone(), dir.path().join("coder"));
1103
1104 let error = s
1105 .live_io_for(&state, &c)
1106 .err()
1107 .expect("one serving provider must refuse assembly");
1108 assert!(error.contains("openai"), "{error}");
1109 for model in &c.review_models {
1110 assert!(error.contains(model), "{model} is missing from: {error}");
1111 }
1112 }
1113
1114 #[test]
1115 fn assembly_refuses_a_coder_that_is_also_a_review_seat() {
1116 let dir = tempfile::tempdir().unwrap();
1117 let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1118 let mut c = cfg(vec![target("acme/one")]);
1119 c.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1120 c.coder_model = Some("gpt-5.4".into());
1121 let s = HealService::new(c.clone(), dir.path().join("coder"));
1122
1123 let error = s
1124 .live_io_for(&state, &c)
1125 .err()
1126 .expect("a coder on its panel must refuse assembly");
1127 assert!(error.contains("gpt-5.4"), "{error}");
1128 assert!(error.contains("also a review seat"), "{error}");
1129 }
1130
1131 #[test]
1132 fn assembly_accepts_a_two_provider_panel_with_a_disjoint_coder() {
1133 let dir = tempfile::tempdir().unwrap();
1134 let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1135 let mut c = cfg(vec![target("acme/one")]);
1136 c.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1137 c.coder_model = Some("gpt-5.5".into());
1138 let s = HealService::new(c.clone(), dir.path().join("coder"));
1139
1140 assert!(s.live_io_for(&state, &c).is_ok());
1141 }
1142
1143 /// A loop that is enabled and DEAD is the state `heal.status` exists to
1144 /// make impossible. Assembly validates things `is_enabled` never sees — an
1145 /// unknown model, a coder sitting on its own panel — so its failure has to
1146 /// reach `disabled_reason` rather than only the boot log.
1147 #[test]
1148 fn an_assembly_failure_is_reported_as_the_disabled_reason() {
1149 let dir = tempfile::tempdir().unwrap();
1150 let s = HealService::new(cfg(vec![target("acme/one")]), dir.path().into());
1151 assert_eq!(s.status(&engine()).disabled_reason, None);
1152
1153 s.record_assembly_error("`coder_model` x is also a review seat (x)");
1154 let st = s.status(&engine());
1155 assert!(
1156 st.enabled,
1157 "the config is still valid; the assembly was not"
1158 );
1159 assert!(
1160 st.disabled_reason
1161 .as_deref()
1162 .is_some_and(|r| r.contains("also a review seat")),
1163 "{:?}",
1164 st.disabled_reason
1165 );
1166 }
1167
1168 /// car#1334 made this pin decide whether a sweep runs at all, and the value
1169 /// lives in whichever of two files won. An operator reading a refusal had
1170 /// to open both and re-derive the precedence by hand — the re-derivation
1171 /// that produced car#1360.
1172 #[test]
1173 fn status_reports_the_coder_pin_from_heal_toml() {
1174 let dir = tempfile::tempdir().unwrap();
1175 let mut c = cfg(vec![target("acme/one")]);
1176 c.coder_model = Some("gpt-5.5".into());
1177 let s = HealService::new(c, dir.path().into());
1178
1179 let pin = s
1180 .status(&engine())
1181 .coder_pin
1182 .expect("a pinned coder must be reported");
1183 assert_eq!(pin.model, "gpt-5.5");
1184 // Naming the wrong file sends the operator to edit a line that is not
1185 // the one in force, which is worse than saying nothing.
1186 assert_eq!(pin.source, "heal_toml");
1187 }
1188
1189 /// An absent `coder_model` does NOT mean unpinned — it means `coder.toml`
1190 /// decides, and that is the case the operator is least able to work out
1191 /// from `heal.toml` alone.
1192 #[test]
1193 fn status_falls_through_to_coder_toml_and_says_so() {
1194 let _guard = crate::coder::config::config_env_lock()
1195 .lock()
1196 .unwrap_or_else(|e| e.into_inner());
1197 let dir = tempfile::tempdir().unwrap();
1198 let cfg_path = dir.path().join("coder.toml");
1199 std::fs::write(&cfg_path, "[coder]\nmodel = \"claude-opus-5\"\n").unwrap();
1200 let prev = std::env::var_os("CAR_CODER_CONFIG");
1201 std::env::set_var("CAR_CODER_CONFIG", &cfg_path);
1202
1203 let mut c = cfg(vec![target("acme/one")]);
1204 c.coder_model = None;
1205 let st = HealService::new(c, dir.path().into()).status(&engine());
1206
1207 match prev {
1208 Some(v) => std::env::set_var("CAR_CODER_CONFIG", v),
1209 None => std::env::remove_var("CAR_CODER_CONFIG"),
1210 }
1211
1212 let pin = st.coder_pin.expect("coder.toml pins it");
1213 assert_eq!(pin.model, "claude-opus-5");
1214 assert_eq!(pin.source, "coder_toml");
1215 }
1216
1217 /// Unpinned is a THIRD state, not a missing value. Adaptive routing picks
1218 /// per request, so nothing before the run can say what it will pick — and
1219 /// an operator has to be able to tell that from "pinned to something the
1220 /// panel rejects".
1221 #[test]
1222 fn status_reports_no_pin_when_neither_file_names_one() {
1223 let _guard = crate::coder::config::config_env_lock()
1224 .lock()
1225 .unwrap_or_else(|e| e.into_inner());
1226 let dir = tempfile::tempdir().unwrap();
1227 let cfg_path = dir.path().join("coder.toml");
1228 // A file that EXISTS and parses, with `[coder]` present and no `model`.
1229 // Pointing at an absent path would pass identically if `status` never
1230 // read the file at all, if the `.then()` guard were inverted, or if the
1231 // env override were ignored — the test would then be agreeing with the
1232 // behaviour rather than pinning it.
1233 std::fs::write(&cfg_path, "[coder]\ndefault_max_iterations = 3\n").unwrap();
1234 let prev = std::env::var_os("CAR_CODER_CONFIG");
1235 std::env::set_var("CAR_CODER_CONFIG", &cfg_path);
1236
1237 let mut c = cfg(vec![target("acme/one")]);
1238 c.coder_model = None;
1239 let st = HealService::new(c, dir.path().into()).status(&engine());
1240
1241 match prev {
1242 Some(v) => std::env::set_var("CAR_CODER_CONFIG", v),
1243 None => std::env::remove_var("CAR_CODER_CONFIG"),
1244 }
1245
1246 assert_eq!(st.coder_pin, None);
1247 }
1248
1249 /// `heal.run` refusing for a reason `heal.status` will not show leaves an
1250 /// operator with a status that describes a config the manual door will not
1251 /// run on.
1252 ///
1253 /// Reported as `run_refusal`, NOT as `disabled_reason`. A cadence that
1254 /// assembled cleanly at boot keeps sweeping on its boot-time io, so a
1255 /// manual refusal is a warning about the config as it stands now — calling
1256 /// it "disabled" would report a dead loop that is in fact running.
1257 #[tokio::test]
1258 async fn a_manual_run_records_its_assembly_refusal() {
1259 let dir = tempfile::tempdir().unwrap();
1260 let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1261 let mut c = cfg(vec![target("acme/one")]);
1262 c.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1263 // The coder sitting on its own panel: assembly refuses.
1264 c.coder_model = Some("gpt-5.4".into());
1265 let s = HealService::new(c, dir.path().join("coder"));
1266
1267 assert_eq!(
1268 s.status(&engine()).run_refusal,
1269 None,
1270 "nothing has refused yet"
1271 );
1272 let err = s
1273 .run_tick(&state)
1274 .await
1275 .expect_err("a coder on its panel must refuse");
1276 assert!(err.contains("also a review seat"), "{err}");
1277
1278 let st = s.status(&engine());
1279 assert!(
1280 st.run_refusal
1281 .as_deref()
1282 .is_some_and(|r| r.contains("also a review seat")),
1283 "the refusal must reach heal.status, not just the caller: {:?}",
1284 st.run_refusal
1285 );
1286 assert_eq!(
1287 st.disabled_reason, None,
1288 "the cadence was never assembled here, and a manual refusal is not \
1289 a statement that the loop is disabled"
1290 );
1291 }
1292
1293 /// The other half of the manual-refusal contract: a run that gets past
1294 /// assembly clears the refusal.
1295 ///
1296 /// Without this the first version traded one lie for a worse one — an
1297 /// operator who fixed the config kept reading the old refusal until the
1298 /// daemon restarted, because the only writer was `record_*` and nothing
1299 /// cleared.
1300 #[tokio::test]
1301 async fn a_successful_manual_run_clears_the_refusal() {
1302 let dir = tempfile::tempdir().unwrap();
1303 let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1304 let mut bad = cfg(vec![target("acme/one")]);
1305 bad.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1306 bad.coder_model = Some("gpt-5.4".into());
1307 let s = HealService::new(bad, dir.path().join("coder"));
1308
1309 s.run_tick(&state)
1310 .await
1311 .expect_err("a coder on its panel refuses");
1312 assert!(s.status(&engine()).run_refusal.is_some());
1313
1314 // The operator fixes it: same panel, a coder that is not a seat.
1315 let mut good = cfg(vec![target("acme/one")]);
1316 good.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1317 good.coder_model = Some("gpt-5.5".into());
1318 if let Ok(mut held) = s.config.write() {
1319 *held = good;
1320 }
1321
1322 s.run_tick(&state)
1323 .await
1324 .expect("the fixed config assembles");
1325 assert_eq!(
1326 s.status(&engine()).run_refusal,
1327 None,
1328 "a run that assembled must clear the refusal it is contradicting"
1329 );
1330 }
1331
1332 /// A manual run must not touch the BOOT slot. The two have different
1333 /// lifetimes: the cadence assembles once and does not start on failure, so
1334 /// its error is true until the daemon restarts, while a manual run's is a
1335 /// statement about the config right now.
1336 ///
1337 /// Conflating them broke both directions — a stale refusal that never
1338 /// cleared, AND a healthy sweeping cadence reporting `disabled_reason`
1339 /// permanently because someone ran `heal.run` on a bad edit.
1340 #[tokio::test]
1341 async fn a_manual_run_neither_sets_nor_clears_the_boot_assembly_error() {
1342 let dir = tempfile::tempdir().unwrap();
1343 let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1344 let mut c = cfg(vec![target("acme/one")]);
1345 c.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1346 c.coder_model = Some("gpt-5.4".into());
1347 let s = HealService::new(c, dir.path().join("coder"));
1348
1349 // The cadence died at boot. That stays true until a restart.
1350 s.record_assembly_error("boot: something the cadence could not assemble");
1351
1352 s.run_tick(&state).await.expect_err("still refuses");
1353 let st = s.status(&engine());
1354 assert!(
1355 st.disabled_reason
1356 .as_deref()
1357 .is_some_and(|r| r.contains("boot: something")),
1358 "a manual run must not overwrite the boot error: {:?}",
1359 st.disabled_reason
1360 );
1361 assert!(st.run_refusal.is_some(), "and must record its own");
1362
1363 // Now a manual run succeeds. The cadence is STILL dead — it returned
1364 // early at boot and nothing restarted it — so clearing the boot error
1365 // here would report a working loop that does not exist.
1366 let mut good = cfg(vec![target("acme/one")]);
1367 good.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1368 good.coder_model = Some("gpt-5.5".into());
1369 if let Ok(mut held) = s.config.write() {
1370 *held = good;
1371 }
1372 s.run_tick(&state).await.expect("assembles now");
1373
1374 let st = s.status(&engine());
1375 assert_eq!(st.run_refusal, None);
1376 assert!(
1377 st.disabled_reason
1378 .as_deref()
1379 .is_some_and(|r| r.contains("boot: something")),
1380 "the cadence is still dead until restart: {:?}",
1381 st.disabled_reason
1382 );
1383 }
1384
1385 /// The status a `heal.status` caller gets before anything has run.
1386 #[test]
1387 fn status_names_the_targets_the_panel_and_the_engine() {
1388 let dir = tempfile::tempdir().unwrap();
1389 let s = HealService::new(cfg(vec![target("acme/one")]), dir.path().into());
1390 let st = s.status(&engine());
1391 assert!(st.enabled);
1392 assert_eq!(st.disabled_reason, None);
1393 assert_eq!(st.targets, vec!["acme/one".to_string()]);
1394 assert_eq!(st.review_models, vec!["reviewer-a".to_string()]);
1395 // The engine is reported even when it was defaulted, so an operator
1396 // never has to know what the default is to know what will run.
1397 assert_eq!(st.engine, DEFAULT_ENGINE);
1398 }
1399
1400 /// The default must be an engine that actually parses, or every daemon
1401 /// with a configured target refuses to assemble the loop at boot.
1402 #[test]
1403 fn the_default_engine_is_a_real_engine() {
1404 assert!(crate::coder::router::EngineChoice::parse(DEFAULT_ENGINE).is_ok());
1405 }
1406
1407 #[tokio::test]
1408 async fn a_sweep_visits_every_target_not_just_the_first() {
1409 // Stopping at the first with work would starve every later entry, and
1410 // the operator who listed them could not see it happening.
1411 let dir = tempfile::tempdir().unwrap();
1412 let s = HealService::new(
1413 cfg(vec![target("acme/one"), target("acme/two")]),
1414 dir.path().into(),
1415 );
1416 let io: Arc<dyn TickIo> = Arc::new(NoopIo);
1417 let report = s.sweep(&io).await;
1418 assert_eq!(report.outcomes.len(), 2);
1419 assert_eq!(report.outcomes[0].0, "acme/one");
1420 assert_eq!(report.outcomes[1].0, "acme/two");
1421 }
1422
1423 #[tokio::test]
1424 async fn an_overlapping_sweep_stands_down_rather_than_queueing() {
1425 let dir = tempfile::tempdir().unwrap();
1426 let s = Arc::new(HealService::new(
1427 cfg(vec![target("acme/one")]),
1428 dir.path().into(),
1429 ));
1430 let io: Arc<dyn TickIo> = Arc::new(NoopIo);
1431
1432 // Hold the lock as a concurrent sweep would.
1433 let held = s.running.lock().await;
1434 let report = s.sweep(&io).await;
1435 assert!(report.skipped_overlap);
1436 assert!(report.outcomes.is_empty());
1437 drop(held);
1438
1439 // And it runs again once the lock is free.
1440 assert!(!s.sweep(&io).await.skipped_overlap);
1441 }
1442
1443 #[tokio::test]
1444 async fn the_ledger_is_written_even_when_nothing_was_claimed() {
1445 // The file's existence is what makes the next start's load meaningful.
1446 let dir = tempfile::tempdir().unwrap();
1447 let s = HealService::new(cfg(vec![target("acme/one")]), dir.path().into());
1448 let io: Arc<dyn TickIo> = Arc::new(NoopIo);
1449 let _ = s.sweep(&io).await;
1450 assert!(
1451 dir.path().join("heal-claims.json").exists(),
1452 "the sweep persists the ledger around every target"
1453 );
1454 }
1455
1456 #[test]
1457 fn the_interval_can_be_overridden_but_never_to_zero() {
1458 // A zero interval is a busy loop against someone's API quota.
1459 let dir = tempfile::tempdir().unwrap();
1460 std::env::set_var(HEAL_INTERVAL_ENV, "0");
1461 let s = HealService::new(cfg(vec![]), dir.path().into());
1462 assert_eq!(s.interval_secs(), DEFAULT_INTERVAL_SECS);
1463 std::env::set_var(HEAL_INTERVAL_ENV, "60");
1464 let s = HealService::new(cfg(vec![]), dir.path().into());
1465 assert_eq!(s.interval_secs(), 60);
1466 std::env::remove_var(HEAL_INTERVAL_ENV);
1467 }
1468}