Skip to main content

car_server_core/coder/
heal_service.rs

1//! The thing that actually runs the self-healing loop.
2//!
3//! T7 of `docs/proposals/self-healing-issue-loop.md`. Until this existed,
4//! [`super::heal_tick::tick`] had no caller — a working half, not a feature.
5//!
6//! ## A daemon subsystem, not a scheduler task
7//!
8//! The proposal originally said `scheduler.os_install`, arguing against a
9//! Claude Code `/loop` because a `/loop` dies with its session. True, and
10//! beside the point: a daemon subsystem does not have that problem, because
11//! the daemon *is* the durable process. `car-scheduler`'s tasks are either
12//! prompt-driven through an agent runner or a shell `CommandSpec`, and this
13//! tick is neither — it is deterministic Rust composition that needs
14//! `coder.start`, so a shell task would mean re-entering by subprocess to
15//! reach a process we were already inside.
16//!
17//! The precedent this follows is [`crate::selfheal`]: a service on
18//! `ServerState`, a cadence spawner, an interval with an env override, and a
19//! manual-run RPC.
20//!
21//! ## Three things the reviews said this had to settle
22//!
23//! **Who persists the ledger.** This service does, around every tick, before
24//! and after. The claim must reach disk *before* the coder session starts, or a
25//! crash mid-session loses it and the next start re-picks an item whose work is
26//! already in flight.
27//!
28//! **What `run_id` is.** A fresh id per tick. A stable one would make every
29//! tick the holder of every claim it ever took — `claim()` treats a matching
30//! run id as a refresh — and the mechanism would silently do nothing.
31//!
32//! **Overlap.** `CLAIM_TTL_MS` is 90 minutes and the cadence is shorter, so
33//! ticks *will* collide. This try-locks and skips rather than queueing: a
34//! queue of ticks against one repository is just a slower way to do the same
35//! work twice.
36
37use std::sync::Arc;
38
39use tokio::sync::Mutex;
40
41use super::heal_claims::ClaimStore;
42use super::heal_config::HealConfig;
43use super::heal_config::HEAL_CONFIG_FILE;
44use super::heal_tick::{tick, ClaimSink, TickIo, TickOutcome};
45use crate::session::ServerState;
46
47/// The engine the loop uses when the configuration does not name one.
48///
49/// `foreman` rather than `auto`: the work is unattended, so decomposing an
50/// intent and gating each patch plus the integrated union is worth more here
51/// than it is in an interactive session someone is watching. Foreman declines
52/// to single-session when the plan has no parallelism, so this is not a cost on
53/// small fixes.
54pub const DEFAULT_ENGINE: &str = "foreman";
55
56/// Env override for the cadence, mirroring `CAR_SELFHEAL_INTERVAL_SECS`.
57pub const HEAL_INTERVAL_ENV: &str = "CAR_HEAL_INTERVAL_SECS";
58
59/// Default cadence. Deliberately unhurried: the queue is human-authored, so
60/// polling faster mostly means asking GitHub the same question more often.
61pub const DEFAULT_INTERVAL_SECS: u64 = 15 * 60;
62
63/// The loop's state inside the daemon.
64pub struct HealService {
65    /// The configuration as of the last read. Behind a lock because the file is
66    /// re-read every sweep: an operator who fixes a typo'd repo spec and reruns
67    /// `car heal status` must not be shown the boot-time config presented as
68    /// current, and enabling the loop must not require a daemon restart.
69    ///
70    /// Re-reading is one small TOML at a 15-minute cadence. The alternative —
71    /// read once at `ServerState` construction — made "the loop never does
72    /// anything" indistinguishable from "your edit has not been picked up",
73    /// which is the failure mode this whole file's diagnostics exist to
74    /// prevent.
75    config: std::sync::RwLock<HealConfig>,
76    /// Where `heal.toml` lives, so a sweep can re-read it.
77    ///
78    /// `None` means the configuration was supplied directly and there is no
79    /// file behind it: re-reading would discard what the caller passed in, so
80    /// this service simply never reloads.
81    config_dir: Option<std::path::PathBuf>,
82    /// Held across a whole tick. See the module docs: try-lock and skip.
83    running: Mutex<()>,
84    state_dir: std::path::PathBuf,
85    interval_secs: u64,
86    /// Why the boot-time assembly failed, when it did.
87    ///
88    /// `is_enabled` reads the config; assembly additionally validates the model
89    /// names, refuses a coder that sits on its own panel, and requires at least
90    /// two serving providers. A configuration
91    /// that passes the first and fails the second leaves the loop enabled and
92    /// dead — which is precisely the state `heal.status` exists to make
93    /// impossible ("an idle loop and a misconfigured one are indistinguishable
94    /// from outside"). Recorded here so `disabled_reason` can say it.
95    assembly_error: std::sync::RwLock<Option<String>>,
96    /// Why the most recent MANUAL `heal.run` refused to assemble, cleared when
97    /// one succeeds.
98    ///
99    /// Separate from `assembly_error` because the two have different lifetimes
100    /// and a single slot gets both wrong. The cadence assembles ONCE at boot
101    /// and does not start on failure, so its error is true until the daemon
102    /// restarts — sticky is correct. `run_tick` re-assembles per call, so its
103    /// failure is a statement about the config as it stands now. Writing that
104    /// into the boot slot made a healthy, sweeping cadence report
105    /// `disabled_reason` permanently, which is the same lie this field exists
106    /// to prevent, pointed the other way.
107    run_refusal: std::sync::RwLock<Option<String>>,
108}
109
110/// What one pass over every configured target did.
111#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
112pub struct SweepReport {
113    pub outcomes: Vec<(String, TickOutcome)>,
114    /// True when another sweep held the lock and this one stood down.
115    pub skipped_overlap: bool,
116}
117
118/// What `heal.status` reports.
119///
120/// Carries `disabled_reason` and `rejected` because a loop that is idle and a
121/// loop that is misconfigured look identical from outside, and on first setup
122/// the second is the common case. "It never does anything" is the hardest
123/// failure to notice in a subsystem whose normal state is doing nothing.
124#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
125pub struct HealStatus {
126    /// Where the configuration was read from, so an operator editing the wrong
127    /// file finds out from `heal.status` instead of from silence.
128    pub config_path: String,
129    pub enabled: bool,
130    pub cadence_secs: u64,
131    /// Why the loop will not run, when it is configured but disabled.
132    pub disabled_reason: Option<String>,
133    /// `owner/name` per configured target.
134    pub targets: Vec<String>,
135    /// Targets that were named but could not be used, and why.
136    pub rejected: Vec<RejectedTargetView>,
137    /// The review panel, by model id.
138    pub review_models: Vec<String>,
139    /// The panel with each seat resolved to the vendor that serves it, after
140    /// the same deduplication the live panel applies.
141    pub panel: Vec<super::heal_review::PanelSeat>,
142    /// Why a panel that clears the two-provider construction floor still
143    /// cannot be shown to be independent of itself — see `correlation_warning`.
144    pub panel_warning: Option<String>,
145    /// Why the most recent manual `heal.run` refused to assemble, if one did
146    /// and none has succeeded since.
147    ///
148    /// NOT `disabled_reason`: a cadence that assembled cleanly at boot keeps
149    /// sweeping on its boot-time io, so a manual refusal says the config AS IT
150    /// STANDS NOW would not assemble — a warning about the next state, not a
151    /// statement that the loop is off.
152    pub run_refusal: Option<String>,
153    /// The model the coder will run on, and which file pinned it.
154    ///
155    /// Two states, not three. `None` means unpinned — neither `heal.toml`'s
156    /// `coder_model` nor `coder.toml`'s `[coder] model` names one, so adaptive
157    /// routing picks per request and nothing before the run can say what it
158    /// will pick. ("Pinned to something the panel rejects" is a
159    /// `disabled_reason` state, not one of this field's.)
160    ///
161    /// **What the RUNNING cadence uses may differ.** `spawn_heal_cadence`
162    /// calls `live_io` once at boot and bakes the pin into the runner, so this
163    /// reports the current files while a cadence started earlier keeps using
164    /// the boot-time value — the same freeze `panel_composition` documents for
165    /// the panel. Read it as what the next `heal.run` would use.
166    ///
167    /// Reported because car#1334 made this pin load-bearing: the panel
168    /// independence check refuses a sweep when the coder is also a review seat,
169    /// and the pin it checks lives in whichever of two files won. Deriving that
170    /// by hand means reading both and re-implementing the precedence — which is
171    /// exactly the re-derivation that produced car#1360.
172    pub coder_pin: Option<CoderPinView>,
173    pub engine: String,
174}
175
176/// The resolved coder pin, as `heal.status` reports it.
177#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
178pub struct CoderPinView {
179    /// The model id, verbatim — not canonicalized. A `coder.toml` value is
180    /// legitimately outside CAR's registry (on the external rung it is
181    /// forwarded to a third-party CLI's own namespace), so showing the
182    /// operator what the file says is the useful answer.
183    pub model: String,
184    /// Which file won, as a machine value: `heal_toml` or `coder_toml`.
185    ///
186    /// Not prose. Every non-CLI consumer — a host app, a script, a human
187    /// reading raw JSON — would otherwise have to strip markdown backticks out
188    /// of a JSON-RPC field; the operator-facing wording is the renderer's job.
189    pub source: String,
190}
191
192#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
193pub struct RejectedTargetView {
194    pub repo: String,
195    pub reason: String,
196}
197
198impl HealService {
199    /// Construct with a config already loaded, and the directory it came from.
200    pub fn with_config_dir(
201        config: HealConfig,
202        state_dir: std::path::PathBuf,
203        config_dir: std::path::PathBuf,
204    ) -> Self {
205        let mut s = Self::new(config, state_dir);
206        s.config_dir = Some(config_dir);
207        s
208    }
209
210    /// Re-read `heal.toml`, returning the current configuration.
211    ///
212    /// A read failure is not distinguishable from an absent file by design
213    /// (`HealConfig::load` returns an empty config either way), which is why
214    /// the empty case disables the loop rather than doing anything.
215    fn reload(&self) -> HealConfig {
216        // No file behind this service: the held config IS the configuration,
217        // and re-reading would throw away what the caller supplied.
218        let Some(dir) = &self.config_dir else {
219            return self.held();
220        };
221        let fresh = HealConfig::load(dir);
222        if let Ok(mut held) = self.config.write() {
223            *held = fresh.clone();
224        }
225        fresh
226    }
227
228    /// The configuration as last read, without touching the disk.
229    fn held(&self) -> HealConfig {
230        match self.config.read() {
231            Ok(c) => c.clone(),
232            // A poisoned lock means a panic while holding it. An empty config
233            // disables the loop, which is the safe answer when the state that
234            // says what to do cannot be trusted.
235            Err(_) => HealConfig::default(),
236        }
237    }
238
239    fn config(&self) -> HealConfig {
240        self.held()
241    }
242
243    pub fn new(config: HealConfig, state_dir: std::path::PathBuf) -> Self {
244        let interval_secs = std::env::var(HEAL_INTERVAL_ENV)
245            .ok()
246            .and_then(|v| v.parse::<u64>().ok())
247            .filter(|v| *v > 0)
248            .unwrap_or(DEFAULT_INTERVAL_SECS);
249        Self {
250            config: std::sync::RwLock::new(config),
251            config_dir: None,
252            running: Mutex::new(()),
253            state_dir,
254            interval_secs,
255            assembly_error: std::sync::RwLock::new(None),
256            run_refusal: std::sync::RwLock::new(None),
257        }
258    }
259
260    /// Record why the loop could not be assembled, so `heal.status` can report
261    /// an enabled-but-dead loop instead of leaving it to a boot log line.
262    pub fn record_assembly_error(&self, error: &str) {
263        if let Ok(mut slot) = self.assembly_error.write() {
264            *slot = Some(error.to_string());
265        }
266    }
267
268    /// The coder pin in force, resolved ONCE for both `status` and
269    /// `live_io_for`.
270    ///
271    /// Used by both so the two cannot disagree — the same reason
272    /// `panel_composition` exists. `config::session_model` is the single copy
273    /// of the PRECEDENCE rule; this is the single copy of "read `coder.toml`
274    /// only when `heal.toml` does not pin", which is a second rule and was
275    /// briefly a second copy.
276    ///
277    /// `coder.toml` is read only when it can matter: a malformed one should not
278    /// warn on every sweep for a value `heal.toml` overrides. Its unreadable,
279    /// malformed and absent cases all collapse to an empty config, so all three
280    /// read as unpinned here.
281    fn resolved_coder_pin(config: &HealConfig) -> Option<(String, super::config::PinSource)> {
282        let coder_toml = config
283            .coder_model
284            .is_none()
285            .then(super::config::CoderConfig::load);
286        super::config::session_model(
287            config.coder_model.as_deref(),
288            coder_toml.as_ref().and_then(|c| c.model.as_deref()),
289        )
290        .map(|(m, src)| (m.to_string(), src))
291    }
292
293    /// Record why a manual `heal.run` refused, or clear it on a run that got
294    /// past assembly. Both arms matter: without the clearing one an operator
295    /// who fixes the config keeps seeing the old refusal until they restart the
296    /// daemon.
297    fn set_run_refusal(&self, error: Option<&str>) {
298        if let Ok(mut slot) = self.run_refusal.write() {
299            *slot = error.map(str::to_string);
300        }
301    }
302
303    pub fn is_enabled(&self) -> bool {
304        self.config().is_enabled()
305    }
306
307    pub fn interval_secs(&self) -> u64 {
308        self.interval_secs
309    }
310
311    /// Every target that could not be used, so a misconfiguration is visible
312    /// rather than presenting as a loop that never does anything.
313    pub fn rejected(&self) -> Vec<super::heal_config::RejectedTarget> {
314        self.config().rejected
315    }
316
317    /// What the loop is configured to do, and why it is not doing it.
318    pub fn status(&self, engine: &Arc<car_inference::InferenceEngine>) -> HealStatus {
319        // Re-read, so an operator who just fixed `heal.toml` is shown what the
320        // next sweep will actually use rather than what booted.
321        let config = self.reload();
322        let panel = self.panel_composition(engine, &config.review_models);
323        let coder_pin = Self::resolved_coder_pin(&config);
324        HealStatus {
325            config_path: self
326                .config_dir
327                .as_ref()
328                .map(|d| d.join(HEAL_CONFIG_FILE).display().to_string())
329                .unwrap_or_else(|| "(supplied directly; no file)".into()),
330            enabled: config.is_enabled(),
331            cadence_secs: self.interval_secs,
332            // Config first (it names a missing panel or missing targets), then
333            // the assembly failure — which is the only one that can leave
334            // `enabled` true.
335            disabled_reason: config.disabled_reason().map(str::to_string).or_else(|| {
336                self.assembly_error
337                    .read()
338                    .ok()
339                    .and_then(|e| e.clone())
340                    .map(|e| format!("the loop could not be assembled: {e}"))
341            }),
342            run_refusal: self.run_refusal.read().ok().and_then(|e| e.clone()),
343            targets: config.targets.iter().map(|t| t.repo.clone()).collect(),
344            rejected: config
345                .rejected
346                .iter()
347                .map(|r| RejectedTargetView {
348                    repo: r.repo.clone(),
349                    reason: r.reason.clone(),
350                })
351                .collect(),
352            review_models: config.review_models.clone(),
353            // Composition, not just names: a single-vendor panel is invisible in
354            // a list of model ids, and the operator reading `3/3 approved` is
355            // the person who needs to know it was one vendor three times.
356            panel: panel.clone(),
357            panel_warning: super::heal_review::correlation_warning(&panel),
358            // Resolved through the same `session_model` the sweep's check uses,
359            // not a second copy of the precedence rule. `coder.toml` is read
360            // only when `heal.toml` does not pin, matching the sweep — an
361            // unreadable `coder.toml` that nothing consults must not turn into
362            // a status field that disagrees with what will run.
363            coder_pin: coder_pin.map(|(model, source)| CoderPinView {
364                model,
365                // Named for the FILE, not for `PinSource`'s generic
366                // request/config spelling: heal passes `heal.toml`'s
367                // `coder_model` as the request pin, so "request" would tell an
368                // operator nothing about which file to open.
369                source: match source {
370                    super::config::PinSource::Request => "heal_toml",
371                    super::config::PinSource::Config => "coder_toml",
372                }
373                .to_string(),
374            }),
375            engine: config
376                .engine
377                .clone()
378                .unwrap_or_else(|| DEFAULT_ENGINE.to_string()),
379        }
380    }
381
382    /// Resolve each configured seat to its serving vendor.
383    ///
384    /// Used by both `status` and `live_io_for` so the two cannot disagree about
385    /// how a seat resolves. They can still describe different PANELS: `status`
386    /// re-reads `heal.toml` every call, while the cadence's `TickIo` is
387    /// assembled once at boot and reused, so an edit shows up in `heal.status`
388    /// before the running loop picks it up (it takes a restart). That freeze
389    /// predates this and is not what this function is claiming to fix.
390    fn panel_composition(
391        &self,
392        engine: &std::sync::Arc<car_inference::InferenceEngine>,
393        models: &[String],
394    ) -> Vec<super::heal_review::PanelSeat> {
395        super::heal_review::composition(models, |m| {
396            engine
397                .model_schema(m)
398                .and_then(|s| s.vendor())
399                .map(str::to_string)
400        })
401    }
402
403    /// Assemble the live loop: real GitHub, real coder, real review panel.
404    ///
405    /// **One constructor, one call site.** Every dependency below already
406    /// existed and none of them were ever wired together — `LiveTickIo` had no
407    /// construction site in the workspace at all, so the loop was a complete,
408    /// tested, unreachable subsystem. A second assembly point (a CLI building
409    /// its own in-process) would be the next copy of that mistake, and would
410    /// also bypass the single-sweep try-lock, so `heal.run` and the cadence
411    /// both come through here.
412    pub fn live_io(&self, state: &Arc<ServerState>) -> Result<Arc<dyn TickIo>, String> {
413        self.live_io_for(state, &self.config())
414    }
415
416    fn live_io_for(
417        &self,
418        state: &Arc<ServerState>,
419        config: &HealConfig,
420    ) -> Result<Arc<dyn TickIo>, String> {
421        if config.review_models.is_empty() {
422            // `decide` refuses a panel of zero, so continuing would run a full
423            // coder session per item and reject every one of them.
424            return Err("no `review_models` configured; refusing to run without a panel".into());
425        }
426        // Validate the panel HERE, where the catalog is reachable, rather than
427        // discovering a typo after a full coder session: an unknown id errors
428        // at generation, which the gate reads as an unreachable seat, so a
429        // misspelled model costs a session and a backoff per tick before
430        // anyone learns why. Refusing is not the same as dropping the seat — a
431        // panel silently shrunk from three to two is the halved threshold the
432        // `PanelIncomplete` design exists to prevent.
433        //
434        // `knows_model`, not `list_models`: the latter is the ON-DEVICE
435        // catalog, so checking a cloud model id against it reports every
436        // frontier model as unknown and would refuse a correct configuration —
437        // a false refusal disables the whole feature, which is worse than the
438        // cost it was added to avoid.
439        let engine_handle = crate::handler::get_inference_engine(state);
440        // The coder is validated with the seats, not after them: the argument
441        // above applies with more force to the model that runs the EXPENSIVE
442        // half, and an unvalidated coder name is also what would force the
443        // disjointness check below to guess at a canonical form.
444        let unknown: Vec<&str> = config
445            .review_models
446            .iter()
447            .chain(config.coder_model.iter())
448            .filter(|m| !engine_handle.knows_model(m))
449            .map(String::as_str)
450            .collect();
451        if !unknown.is_empty() {
452            return Err(format!(
453                "unknown model(s) in `heal.toml`: {} — a seat that cannot be reached is \
454                 not a reviewer, and dropping it would quietly shrink the panel. Run \
455                 `car models list` for the names this daemon knows.",
456                unknown.join(", ")
457            ));
458        }
459
460        // Which model the coder will run on, resolved ONCE. `coder.start`
461        // takes `coder_model` as its request pin and falls back to
462        // `~/.car/coder.toml`'s `[coder] model`, so an unset `coder_model` does
463        // not mean the coder is unpinned — it means it runs on that file's
464        // pin, which nothing checked against the panel (car#1360). Read only
465        // when it can matter; a malformed coder.toml should not warn on every
466        // sweep for a value heal.toml overrides.
467        let coder_pin = Self::resolved_coder_pin(config);
468
469        // Refused rather than dropped: a seat quietly removed lowers the
470        // majority threshold without saying so, which is what `PanelIncomplete`
471        // exists to prevent. The rule and its message live in
472        // `check_coder_pin`, where they are table-tested; the registry reaches
473        // it as the one closure.
474        //
475        // Every SEAT cleared `knows_model` above, so seats always canonicalize
476        // to a real id — but the coder pin may not have, because a coder.toml
477        // value is legitimately outside CAR's registry: on the external rung it
478        // is forwarded verbatim to a third-party CLI's own namespace. That is
479        // what `canonicalizer`'s fallback is for.
480        check_coder_pin(
481            coder_pin.as_ref().map(|(m, src)| (m.as_str(), *src)),
482            &config.review_models,
483            &canonicalizer(|m| engine_handle.model_schema(m).map(|s| s.id.clone())),
484        )?;
485
486        let engine = match config.engine.as_deref() {
487            Some(name) => super::router::EngineChoice::parse(name)?,
488            None => super::router::EngineChoice::parse(DEFAULT_ENGINE)?,
489        };
490
491        // The minimum diversity floor is enforced here, after every seat has a
492        // catalog-backed serving vendor. Refuse rather than dropping seats: a
493        // smaller panel has a different majority threshold and is a different
494        // configuration. The stronger majority-capture condition remains a
495        // warning for panels that do span two providers but can still be
496        // carried by one of them.
497        let panel = self.panel_composition(engine_handle, &config.review_models);
498        if let Some(error) = super::heal_review::panel_diversity_error(&panel) {
499            return Err(error);
500        }
501        if let Some(warning) = super::heal_review::correlation_warning(&panel) {
502            tracing::warn!(target: "car::heal", "{warning}");
503        }
504
505        // Adaptive inference reports `ModelSchema.name`, so pass the panel in
506        // that same namespace. `build_exclude_set` resolves either names or ids,
507        // but using the result namespace here also makes the list directly
508        // comparable with the durable author attribution the delivery backstop
509        // reads. Validation above guarantees every seat resolves.
510        let mut routing_exclusions = Vec::new();
511        for seat in &config.review_models {
512            if let Some(name) = engine_handle.model_schema(seat).map(|schema| &schema.name) {
513                if !routing_exclusions.contains(name) {
514                    routing_exclusions.push(name.clone());
515                }
516            }
517        }
518
519        let runner = super::heal_runner::LiveCoderRunner {
520            state: state.clone(),
521            // The production `TurnGenerator` is the inference engine itself.
522            generator: crate::handler::get_inference_engine(state).clone(),
523            state_dir: self.state_dir.clone(),
524            reviewers: super::heal_review::panel(state, &config.review_models),
525            max_wall_secs: super::heal_runner::DEFAULT_ITEM_WALL_SECS,
526            max_iterations: None,
527            engine,
528            // The value that was CHECKED above, not `coder_model` — otherwise
529            // `coder.start` re-reads coder.toml at session time and the pin
530            // the runner ends up on is not the one the panel check saw.
531            model: coder_pin.map(|(m, _)| m),
532            routing_exclusions,
533            // The registry resolver the gate's self-review check canonicalizes
534            // through, on the same fallback policy as the assembly check above
535            // — and it matters more here, because the gate's author comes from
536            // the engine unvalidated.
537            canonical_model: {
538                let engine_handle = engine_handle.clone();
539                Arc::new(canonicalizer(move |m: &str| {
540                    engine_handle.model_schema(m).map(|s| s.id.clone())
541                }))
542            },
543            github: Arc::new(super::merge::GhCli::default()),
544        };
545
546        Ok(Arc::new(super::heal_live::LiveTickIo {
547            issues: Arc::new(super::fix_issues::GhIssues),
548            prs: Arc::new(super::heal_intake::GhPullRequests),
549            oracle: Arc::new(super::provenance::GhPermissions),
550            coder: Arc::new(runner),
551            // EMPTY, deliberately, and it narrows what the loop will act on.
552            //
553            // `LocalSignatures` holds the signatures of issues this runtime
554            // itself filed, and it is the only route to the `Runtime` tier —
555            // the one tier that may source an outcome contract from an issue
556            // body. Nothing in the daemon files issues today (`car-selfheal` is
557            // watch-only by design), so there are no such signatures to supply
558            // and inventing a non-empty set would be asserting authorship this
559            // process cannot demonstrate.
560            //
561            // The consequence, stated rather than discovered: every item the
562            // loop acts on is maintainer-authored or better, and no issue body
563            // ever sources a contract. That is the safer posture, and it is the
564            // one the proposal asks be revisited *before* runtime filing is
565            // enabled — not after.
566            local_signatures: super::provenance::LocalSignatures::from_proposals(&[]),
567            redactor: car_selfheal::redact::Redactor::from_env(std::env::vars()),
568            panel,
569        }))
570    }
571
572    /// One sweep, assembling the live loop first. The manual-run entry point.
573    pub async fn run_tick(&self, state: &Arc<ServerState>) -> Result<SweepReport, String> {
574        // Re-read first: a manual run is exactly when an operator has just
575        // edited the file.
576        let config = self.reload();
577        // Recorded, not just returned. `spawn_heal_cadence` records its
578        // assembly failure so a boot-time refusal still reaches `heal.status` a
579        // week later; a manual `heal.run` refusing for the same reason left no
580        // trace, so `heal.status` could show a config `heal.run` will not run
581        // on. The caller still gets the error — this only stops it from being
582        // the ONLY place it appears.
583        //
584        // A `match` rather than `inspect_err`: this needs a SUCCESS arm. The
585        // slot has to be cleared by a run that assembles, or an operator who
586        // fixes the config keeps reading the old refusal forever.
587        let io = match self.live_io_for(state, &config) {
588            Ok(io) => {
589                self.set_run_refusal(None);
590                io
591            }
592            Err(e) => {
593                self.set_run_refusal(Some(&e));
594                return Err(e);
595            }
596        };
597        Ok(self.sweep_with(&io, &config).await)
598    }
599
600    /// One pass over every configured target, at most one item each.
601    ///
602    /// Every target, not "the first with work": stopping early would let a busy
603    /// repository starve every entry after it in the config, and the operator
604    /// who listed them has no way to see that happening.
605    pub async fn sweep(&self, io: &Arc<dyn TickIo>) -> SweepReport {
606        let config = self.reload();
607        self.sweep_with(io, &config).await
608    }
609
610    async fn sweep_with(&self, io: &Arc<dyn TickIo>, config: &HealConfig) -> SweepReport {
611        let Ok(_guard) = self.running.try_lock() else {
612            // A tick is already running. Queueing would just do the same work
613            // twice, more slowly.
614            return SweepReport {
615                outcomes: Vec::new(),
616                skipped_overlap: true,
617            };
618        };
619
620        let mut claims = ClaimStore::load(&self.state_dir);
621        let mut outcomes = Vec::new();
622        let sink = FileClaimSink {
623            dir: self.state_dir.clone(),
624        };
625
626        for target in &config.targets {
627            // A fresh id per tick. A stable one would make this run the holder
628            // of every claim it ever took, and claiming would silently do
629            // nothing.
630            let run_id = format!("heal-{}", uuid::Uuid::new_v4().simple());
631            let out = tick(io, target, &mut claims, &run_id, &sink).await;
632
633            // Again after the target, to record the outcome — the CLAIM
634            // already reached disk inside `tick`, before any work started.
635            // Saving only here lost it whenever the daemon died during the
636            // 45-minute session the claim was taken for, and the next start
637            // re-picked an item whose work was still in flight.
638            sink.persist(&claims, io.now_ms());
639            outcomes.push((target.repo.clone(), out));
640        }
641
642        SweepReport {
643            outcomes,
644            skipped_overlap: false,
645        }
646    }
647}
648
649// There is deliberately NO worktree reaper here.
650//
651// There was one, and it was worse than nothing: it swept
652// `<state_dir>/heal-worktrees`, a directory nothing has ever created. Coder
653// sessions provision under `<state_dir>/worktrees` (`CoderSession::
654// provision_workspace`). So the guard whose rationale was the 102 GB incident
655// scanned an empty path, always returned 0, and logged nothing — a
656// disk-exhaustion protection that reported success while protecting nothing.
657//
658// Repointing it at `worktrees` would have been worse still: that directory is
659// shared with every *interactive* `coder.start`, so an age-based sweep would
660// delete a human's overnight `NeedsApproval` worktree.
661//
662// The real fix was upstream. `LiveCoderRunner` now drives every session it
663// starts to a terminal state — `Merged` on delivery, `Abandoned` on rejection,
664// `Failed` on timeout — and a terminal transition drops the `AgentWorkspace`
665// RAII handle, which is what removes the worktree and its `git worktree`
666// registration. Sessions clean up after themselves; nothing has to sweep.
667
668/// Writes the claim ledger to the loop's state directory.
669///
670/// A failure is logged, not propagated: a disk that refuses the write must not
671/// fail a sweep that is otherwise fine, and the in-memory ledger still holds
672/// the claim for the rest of this sweep.
673struct FileClaimSink {
674    dir: std::path::PathBuf,
675}
676
677impl ClaimSink for FileClaimSink {
678    fn persist(&self, claims: &ClaimStore, now_ms: u64) {
679        if let Err(e) = claims.save(&self.dir, now_ms) {
680            tracing::warn!(error = %e, "could not persist heal claims");
681        }
682    }
683}
684
685/// Run the loop on its cadence until the daemon stops.
686///
687/// Returns `None` when no targets are configured, so a daemon with no
688/// `heal.toml` spawns nothing at all rather than a task that wakes to do
689/// nothing forever.
690pub fn spawn_heal_cadence(state: Arc<ServerState>) -> Option<tokio::task::JoinHandle<()>> {
691    let service = state.heal.clone();
692    if !service.is_enabled() {
693        // Named, not silent. `disabled_reason` distinguishes "nothing is
694        // configured" from "targets are configured but there is no review
695        // panel", and the second reads exactly like the first from a log that
696        // only says "not starting".
697        tracing::info!(
698            reason = service
699                .config()
700                .disabled_reason()
701                .unwrap_or("no targets configured"),
702            "self-healing loop not started"
703        );
704        return None;
705    }
706    for r in service.rejected() {
707        tracing::warn!(repo = %r.repo, reason = %r.reason, "heal target ignored");
708    }
709
710    // Assemble ONCE, at startup, so a missing credential or an unparseable
711    // engine name is a boot-time error in the log rather than a failure
712    // rediscovered on every tick forever.
713    let io = match service.live_io(&state) {
714        Ok(io) => io,
715        Err(e) => {
716            // Recorded before returning: a log line at boot is not reachable by
717            // the operator who runs `car heal status` a week later.
718            service.record_assembly_error(&e);
719            tracing::warn!(error = %e, "self-healing loop could not be assembled; not starting");
720            return None;
721        }
722    };
723    let secs = service.interval_secs();
724    tracing::info!(interval_secs = secs, "self-healing loop started");
725    Some(tokio::spawn(async move {
726        let mut ticker = tokio::time::interval(std::time::Duration::from_secs(secs));
727        // The first tick fires immediately; skip it so a daemon restart does
728        // not start a coder session before the operator has seen it come up.
729        ticker.tick().await;
730        loop {
731            ticker.tick().await;
732            let report = service.sweep(&io).await;
733            if report.skipped_overlap {
734                tracing::debug!("heal sweep skipped: previous sweep still running");
735                continue;
736            }
737            for (repo, out) in &report.outcomes {
738                match out {
739                    TickOutcome::Opened {
740                        number,
741                        pr_url,
742                        ci,
743                        delivery,
744                        ..
745                    } => tracing::info!(
746                        %repo,
747                        number,
748                        %pr_url,
749                        head_sha = %ci.head_sha,
750                        ci_state = ?ci.state,
751                        %delivery,
752                        "self-heal opened a pull request"
753                    ),
754                    TickOutcome::Rejected { number, gate, .. } => {
755                        tracing::info!(%repo, number, %gate, "self-heal stopped at the gate")
756                    }
757                    TickOutcome::Failed { detail } => {
758                        tracing::warn!(%repo, %detail, "self-heal tick failed")
759                    }
760                    TickOutcome::Idle { .. } => {}
761                }
762            }
763        }
764    }))
765}
766
767/// The registry canonicalizer both self-review checks compare through.
768///
769/// `unwrap_or_else(|| m.to_string())`, never `unwrap_or_default()`. Two names
770/// the registry cannot resolve would both collapse to `""` and match each
771/// other, and here that means refusing every session on a daemon whose catalog
772/// does not hold the model that ran. Falling back to the name itself degrades
773/// to a spelling comparison instead — the honest answer when the registry has
774/// nothing to say. Written once so the policy has one place to be wrong.
775fn canonicalizer(resolve: impl Fn(&str) -> Option<String>) -> impl Fn(&str) -> String {
776    move |m| resolve(m).unwrap_or_else(|| m.to_string())
777}
778
779/// Refuse an assembly where the model that will WRITE the change also sits on
780/// the panel that will judge it.
781///
782/// Takes the pin already resolved — `(model, source)` from
783/// [`config::session_model`](super::config::session_model) — rather than the
784/// two raw sources. Two adjacent `Option<&str>` parameters can be transposed
785/// silently, and getting the precedence backwards is the defect this function
786/// exists to fix; there is nothing to transpose here. The source rides along so
787/// the message can name the file the operator has to edit, without this
788/// re-deriving precedence from its own copy of the rule.
789///
790/// Refused rather than dropping the seat: a panel silently shrunk from three
791/// to two is the halved threshold `PanelIncomplete` exists to prevent. Two
792/// cases are deliberately NOT refused here:
793///
794/// - **An unpinned coder** (neither file pins, the default). The router picks
795///   per request, so only `heal_runner`'s gate, reading what actually authored
796///   the change, can answer it (car#1299).
797/// - **A pin this daemon's registry does not know.** heal validates its own
798///   `coder_model` against the catalog, but `coder.toml` is a shared file no
799///   other consumer validates, and on the external rung its value is forwarded
800///   verbatim to a CLI's own namespace — so a name `car models list` has never
801///   heard of can be exactly right for the rung that runs. Refusing would take
802///   the whole loop offline over a working configuration, which is the false
803///   refusal `correlation_warning` and the `knows_model` comment upstream both
804///   decline to make.
805fn check_coder_pin(
806    pin: Option<(&str, super::config::PinSource)>,
807    seats: &[String],
808    canonical: &dyn Fn(&str) -> String,
809) -> Result<(), String> {
810    let Some((coder, source)) = pin else {
811        return Ok(());
812    };
813    let Some(seat) = super::heal_review::coder_on_panel(coder, seats, canonical) else {
814        return Ok(());
815    };
816    Err(format!(
817        "the coder model {} ({}) is also a review seat ({}) — a model cannot review its \
818         own output, and counting it as a reviewer reports an independence the panel does \
819         not have. Remove it from `review_models`, or pin a different coder.",
820        coder,
821        match source {
822            super::config::PinSource::Request => "`coder_model` in `heal.toml`",
823            super::config::PinSource::Config => "`model` in `coder.toml`",
824        },
825        seat
826    ))
827}
828
829#[cfg(test)]
830mod tests {
831    use super::*;
832    use crate::coder::heal_intake::{Checkout, HealTarget};
833
834    struct NoopIo;
835
836    #[async_trait::async_trait]
837    impl TickIo for NoopIo {
838        async fn candidates(
839            &self,
840            _t: &HealTarget,
841        ) -> Result<Vec<crate::coder::heal_select::Candidate>, String> {
842            Ok(vec![])
843        }
844        async fn open_prs(
845            &self,
846            _t: &HealTarget,
847        ) -> Result<Vec<crate::coder::heal_intake::RawPullRequest>, String> {
848            Ok(vec![])
849        }
850        async fn intent_for(
851            &self,
852            _i: &crate::coder::heal_select::Candidate,
853        ) -> Result<crate::coder::heal_tick::Intent, String> {
854            Ok(crate::coder::heal_tick::Intent::Gone)
855        }
856        fn redact(&self, text: &str) -> String {
857            text.to_string()
858        }
859        async fn run_coder(
860            &self,
861            _t: &HealTarget,
862            _i: &crate::coder::heal_select::Candidate,
863            _s: &crate::coder::provenance::SessionSeed,
864        ) -> Result<crate::coder::heal_tick::Attempt, crate::coder::heal_tick::RunFailure> {
865            Err(crate::coder::heal_tick::RunFailure::early("not reached"))
866        }
867        async fn deliver(
868            &self,
869            _t: &HealTarget,
870            _i: &crate::coder::heal_select::Candidate,
871            _s: &str,
872            _g: &crate::coder::heal_gate::GateOutcome,
873        ) -> Result<crate::coder::merge::PrDeliveryOutcome, crate::coder::heal_tick::DeliverRefusal>
874        {
875            Err(crate::coder::heal_tick::DeliverRefusal::retriable(
876                "not reached",
877            ))
878        }
879        async fn abandon(&self, _s: &str) {}
880        async fn comment(
881            &self,
882            _i: &crate::coder::heal_select::Candidate,
883            _t: &str,
884        ) -> Result<(), String> {
885            Ok(())
886        }
887        fn now_ms(&self) -> u64 {
888            1_000_000
889        }
890    }
891
892    fn cfg(targets: Vec<HealTarget>) -> HealConfig {
893        HealConfig {
894            targets,
895            rejected: vec![],
896            review_models: vec!["reviewer-a".into()],
897            engine: None,
898            coder_model: None,
899        }
900    }
901
902    fn target(repo: &str) -> HealTarget {
903        HealTarget {
904            repo: repo.into(),
905            fix_repo: None,
906            checkout: Some(Checkout::Project("p".into())),
907            label: "self-heal".into(),
908            base: "main".into(),
909        }
910    }
911
912    /// `status` resolves each seat's vendor through the registry, so it needs
913    /// the engine — and only the engine.
914    fn engine() -> Arc<car_inference::InferenceEngine> {
915        Arc::new(car_inference::InferenceEngine::new(Default::default()))
916    }
917
918    /// A daemon with no `heal.toml` spawns nothing at all — not a task that
919    /// wakes every fifteen minutes to find no targets.
920    #[test]
921    fn no_targets_means_the_loop_is_disabled() {
922        let dir = tempfile::tempdir().unwrap();
923        let s = HealService::new(cfg(vec![]), dir.path().into());
924        assert!(!s.is_enabled());
925        assert_eq!(
926            s.status(&engine()).disabled_reason.as_deref(),
927            Some("no usable targets are configured")
928        );
929    }
930
931    /// A registry stand-in with the car#889 hazard in it: `alias` is a name
932    /// whose canonical id is something else, so a spelling comparison between
933    /// the two never fires. Everything else is unknown to it.
934    ///
935    /// Built through the PRODUCTION `canonicalizer`, not hand-rolled — so the
936    /// unresolvable-name policy these tests rely on is the one that ships, and
937    /// changing that fallback turns them red.
938    fn canon(m: &str) -> String {
939        canonicalizer(|m: &str| (m == "alias").then(|| "vendor/real".to_string()))(m)
940    }
941
942    fn seats() -> Vec<String> {
943        vec!["gpt-5.6".into(), "vendor/real".into(), "gpt-5.4".into()]
944    }
945
946    /// Resolve exactly as `live_io_for` does, so these cases exercise the
947    /// precedence rule and the check together rather than the check alone.
948    fn check(heal: Option<&str>, coder_toml: Option<&str>) -> Result<(), String> {
949        check_coder_pin(
950            super::super::config::session_model(heal, coder_toml),
951            &seats(),
952            &canon,
953        )
954    }
955
956    /// The defect car#1360 is: heal.toml pinning nothing does not mean the
957    /// coder is unpinned, it means `coder.toml`'s `model` is the pin — and
958    /// that source was never checked against the panel.
959    #[test]
960    fn a_coder_toml_pin_that_is_a_review_seat_is_refused() {
961        let err = check(None, Some("gpt-5.6")).expect_err("coder.toml pinned a seat");
962        assert!(err.contains("gpt-5.6"), "{err}");
963        assert!(err.contains("`model` in `coder.toml`"), "{err}");
964        // Naming the wrong file sends the operator to edit a key that isn't
965        // set, which is how a correct refusal still costs an afternoon.
966        assert!(!err.contains("`coder_model` in `heal.toml`"), "{err}");
967    }
968
969    /// Canonicalization is load-bearing, not cosmetic: the pin is spelled as a
970    /// name and the seat as an id. A gate that cannot fire reads as covered.
971    #[test]
972    fn a_coder_toml_pin_is_matched_through_the_registry_not_by_spelling() {
973        let err = check(None, Some("alias")).expect_err("alias canonicalizes onto vendor/real");
974        assert!(err.contains("vendor/real"), "{err}");
975    }
976
977    /// heal.toml still wins where it is set, and the message says so — the
978    /// coder.toml value is not what runs, so refusing on it would be wrong.
979    #[test]
980    fn the_heal_toml_pin_wins_and_is_the_one_checked() {
981        let err = check(Some("gpt-5.4"), Some("claude-sonnet-5")).expect_err("heal.toml pinned");
982        assert!(err.contains("`coder_model` in `heal.toml`"), "{err}");
983
984        // And the converse: a coder.toml pin sitting on the panel is harmless
985        // when heal.toml overrides it, because it never runs.
986        check(Some("claude-sonnet-5"), Some("gpt-5.6")).expect("the pin that runs is not a seat");
987    }
988
989    /// A blank `coder_model` is "unset", not "pin blank" — the only input where
990    /// precedence and the source label could disagree and name the wrong file.
991    #[test]
992    fn a_blank_heal_pin_falls_through_and_the_message_names_coder_toml() {
993        let err = check(Some("   "), Some("gpt-5.6")).expect_err("blank falls through");
994        assert!(err.contains("`model` in `coder.toml`"), "{err}");
995    }
996
997    /// Neither source pins: the router picks per request, so nothing here can
998    /// say what it will pick. `heal_runner`'s gate answers that one, against
999    /// what actually authored the change.
1000    #[test]
1001    fn an_unpinned_coder_is_not_this_checks_to_refuse() {
1002        check(None, None).expect("unpinned is not a seat");
1003    }
1004
1005    /// A pin outside CAR's registry is NOT refused. `coder.toml` is shared with
1006    /// `coder.start` and `car code-task`, neither of which validates it, and on
1007    /// the external rung the value is handed verbatim to a CLI's own namespace
1008    /// — so an unknown name can be exactly right for the rung that runs, and
1009    /// taking the whole loop offline over it is the false refusal this file
1010    /// declines to make elsewhere.
1011    ///
1012    /// It still has to be COMPARED, though. The canonicalizer falls back to the
1013    /// name itself rather than `""`, so an unresolvable pin degrades to a
1014    /// spelling match instead of silently matching nothing.
1015    #[test]
1016    fn a_pin_outside_the_registry_is_compared_not_refused() {
1017        // Unknown and not a seat: allowed through.
1018        check(None, Some("codex-mini")).expect("an unknown pin is not by itself a refusal");
1019        // Unknown to `canon` but spelled exactly like a seat: still caught. A
1020        // canonicalizer collapsing the unknown to `""` would miss this.
1021        let err = check(None, Some("gpt-5.4")).expect_err("spelling still matches a seat");
1022        assert!(err.contains("gpt-5.4"), "{err}");
1023    }
1024
1025    /// Targets but no panel is a DIFFERENT disabled    /// Targets but no panel is a DIFFERENT disabled, and the difference is the
1026    /// whole point of reporting a reason: `decide` refuses a panel of zero, so
1027    /// running would mean a full coder session per item followed by a
1028    /// guaranteed rejection.
1029    #[test]
1030    fn targets_without_a_review_panel_do_not_enable_the_loop() {
1031        let dir = tempfile::tempdir().unwrap();
1032        let mut c = cfg(vec![target("acme/one")]);
1033        c.review_models.clear();
1034        let s = HealService::new(c, dir.path().into());
1035        assert!(!s.is_enabled());
1036        assert!(s
1037            .status(&engine())
1038            .disabled_reason
1039            .unwrap()
1040            .contains("review_models"));
1041    }
1042
1043    /// `heal.status` must report the panel it will actually use, resolved.
1044    ///
1045    /// The wiring, not the rule: `correlation_warning` is table-tested next
1046    /// door, and every defect this file has had was in getting the right value
1047    /// to it.
1048    #[test]
1049    fn status_resolves_the_panel_and_reports_a_correlated_one() {
1050        let dir = tempfile::tempdir().unwrap();
1051        let mut c = cfg(vec![target("acme/one")]);
1052        // Real catalog ids: both are OpenAI, so one vendor holds every seat.
1053        c.review_models = vec!["gpt-5.4".into(), "gpt-5.5".into()];
1054        let s = HealService::new(c, dir.path().into());
1055        let st = s.status(&engine());
1056
1057        assert_eq!(
1058            st.panel
1059                .iter()
1060                .map(|p| p.model.as_str())
1061                .collect::<Vec<_>>(),
1062            vec!["gpt-5.4", "gpt-5.5"]
1063        );
1064        assert!(
1065            st.panel
1066                .iter()
1067                .all(|p| p.vendor.as_deref() == Some("openai")),
1068            "both seats must resolve to openai: {:?}",
1069            st.panel
1070        );
1071        let w = st
1072            .panel_warning
1073            .expect("a one-vendor panel must be reported");
1074        assert!(w.contains("openai serves 2 of the 2 seats"), "{w}");
1075    }
1076
1077    /// An independent panel carries no warning, so the warning means something.
1078    #[test]
1079    fn status_does_not_warn_about_a_panel_spanning_vendors() {
1080        let dir = tempfile::tempdir().unwrap();
1081        let mut c = cfg(vec![target("acme/one")]);
1082        c.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1083        let s = HealService::new(c, dir.path().into());
1084        let st = s.status(&engine());
1085        assert_eq!(
1086            st.panel
1087                .iter()
1088                .filter_map(|p| p.vendor.as_deref())
1089                .collect::<Vec<_>>(),
1090            vec!["openai", "anthropic"]
1091        );
1092        assert_eq!(st.panel_warning, None);
1093    }
1094
1095    #[test]
1096    fn assembly_refuses_a_three_seat_single_provider_panel() {
1097        let dir = tempfile::tempdir().unwrap();
1098        let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1099        let mut c = cfg(vec![target("acme/one")]);
1100        c.review_models = vec!["gpt-5.4".into(), "gpt-5.5".into(), "gpt-5.6-sol".into()];
1101        c.coder_model = Some("claude-opus-5".into());
1102        let s = HealService::new(c.clone(), dir.path().join("coder"));
1103
1104        let error = s
1105            .live_io_for(&state, &c)
1106            .err()
1107            .expect("one serving provider must refuse assembly");
1108        assert!(error.contains("openai"), "{error}");
1109        for model in &c.review_models {
1110            assert!(error.contains(model), "{model} is missing from: {error}");
1111        }
1112    }
1113
1114    #[test]
1115    fn assembly_refuses_a_coder_that_is_also_a_review_seat() {
1116        let dir = tempfile::tempdir().unwrap();
1117        let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1118        let mut c = cfg(vec![target("acme/one")]);
1119        c.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1120        c.coder_model = Some("gpt-5.4".into());
1121        let s = HealService::new(c.clone(), dir.path().join("coder"));
1122
1123        let error = s
1124            .live_io_for(&state, &c)
1125            .err()
1126            .expect("a coder on its panel must refuse assembly");
1127        assert!(error.contains("gpt-5.4"), "{error}");
1128        assert!(error.contains("also a review seat"), "{error}");
1129    }
1130
1131    #[test]
1132    fn assembly_accepts_a_two_provider_panel_with_a_disjoint_coder() {
1133        let dir = tempfile::tempdir().unwrap();
1134        let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1135        let mut c = cfg(vec![target("acme/one")]);
1136        c.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1137        c.coder_model = Some("gpt-5.5".into());
1138        let s = HealService::new(c.clone(), dir.path().join("coder"));
1139
1140        assert!(s.live_io_for(&state, &c).is_ok());
1141    }
1142
1143    /// A loop that is enabled and DEAD is the state `heal.status` exists to
1144    /// make impossible. Assembly validates things `is_enabled` never sees — an
1145    /// unknown model, a coder sitting on its own panel — so its failure has to
1146    /// reach `disabled_reason` rather than only the boot log.
1147    #[test]
1148    fn an_assembly_failure_is_reported_as_the_disabled_reason() {
1149        let dir = tempfile::tempdir().unwrap();
1150        let s = HealService::new(cfg(vec![target("acme/one")]), dir.path().into());
1151        assert_eq!(s.status(&engine()).disabled_reason, None);
1152
1153        s.record_assembly_error("`coder_model` x is also a review seat (x)");
1154        let st = s.status(&engine());
1155        assert!(
1156            st.enabled,
1157            "the config is still valid; the assembly was not"
1158        );
1159        assert!(
1160            st.disabled_reason
1161                .as_deref()
1162                .is_some_and(|r| r.contains("also a review seat")),
1163            "{:?}",
1164            st.disabled_reason
1165        );
1166    }
1167
1168    /// car#1334 made this pin decide whether a sweep runs at all, and the value
1169    /// lives in whichever of two files won. An operator reading a refusal had
1170    /// to open both and re-derive the precedence by hand — the re-derivation
1171    /// that produced car#1360.
1172    #[test]
1173    fn status_reports_the_coder_pin_from_heal_toml() {
1174        let dir = tempfile::tempdir().unwrap();
1175        let mut c = cfg(vec![target("acme/one")]);
1176        c.coder_model = Some("gpt-5.5".into());
1177        let s = HealService::new(c, dir.path().into());
1178
1179        let pin = s
1180            .status(&engine())
1181            .coder_pin
1182            .expect("a pinned coder must be reported");
1183        assert_eq!(pin.model, "gpt-5.5");
1184        // Naming the wrong file sends the operator to edit a line that is not
1185        // the one in force, which is worse than saying nothing.
1186        assert_eq!(pin.source, "heal_toml");
1187    }
1188
1189    /// An absent `coder_model` does NOT mean unpinned — it means `coder.toml`
1190    /// decides, and that is the case the operator is least able to work out
1191    /// from `heal.toml` alone.
1192    #[test]
1193    fn status_falls_through_to_coder_toml_and_says_so() {
1194        let _guard = crate::coder::config::config_env_lock()
1195            .lock()
1196            .unwrap_or_else(|e| e.into_inner());
1197        let dir = tempfile::tempdir().unwrap();
1198        let cfg_path = dir.path().join("coder.toml");
1199        std::fs::write(&cfg_path, "[coder]\nmodel = \"claude-opus-5\"\n").unwrap();
1200        let prev = std::env::var_os("CAR_CODER_CONFIG");
1201        std::env::set_var("CAR_CODER_CONFIG", &cfg_path);
1202
1203        let mut c = cfg(vec![target("acme/one")]);
1204        c.coder_model = None;
1205        let st = HealService::new(c, dir.path().into()).status(&engine());
1206
1207        match prev {
1208            Some(v) => std::env::set_var("CAR_CODER_CONFIG", v),
1209            None => std::env::remove_var("CAR_CODER_CONFIG"),
1210        }
1211
1212        let pin = st.coder_pin.expect("coder.toml pins it");
1213        assert_eq!(pin.model, "claude-opus-5");
1214        assert_eq!(pin.source, "coder_toml");
1215    }
1216
1217    /// Unpinned is a THIRD state, not a missing value. Adaptive routing picks
1218    /// per request, so nothing before the run can say what it will pick — and
1219    /// an operator has to be able to tell that from "pinned to something the
1220    /// panel rejects".
1221    #[test]
1222    fn status_reports_no_pin_when_neither_file_names_one() {
1223        let _guard = crate::coder::config::config_env_lock()
1224            .lock()
1225            .unwrap_or_else(|e| e.into_inner());
1226        let dir = tempfile::tempdir().unwrap();
1227        let cfg_path = dir.path().join("coder.toml");
1228        // A file that EXISTS and parses, with `[coder]` present and no `model`.
1229        // Pointing at an absent path would pass identically if `status` never
1230        // read the file at all, if the `.then()` guard were inverted, or if the
1231        // env override were ignored — the test would then be agreeing with the
1232        // behaviour rather than pinning it.
1233        std::fs::write(&cfg_path, "[coder]\ndefault_max_iterations = 3\n").unwrap();
1234        let prev = std::env::var_os("CAR_CODER_CONFIG");
1235        std::env::set_var("CAR_CODER_CONFIG", &cfg_path);
1236
1237        let mut c = cfg(vec![target("acme/one")]);
1238        c.coder_model = None;
1239        let st = HealService::new(c, dir.path().into()).status(&engine());
1240
1241        match prev {
1242            Some(v) => std::env::set_var("CAR_CODER_CONFIG", v),
1243            None => std::env::remove_var("CAR_CODER_CONFIG"),
1244        }
1245
1246        assert_eq!(st.coder_pin, None);
1247    }
1248
1249    /// `heal.run` refusing for a reason `heal.status` will not show leaves an
1250    /// operator with a status that describes a config the manual door will not
1251    /// run on.
1252    ///
1253    /// Reported as `run_refusal`, NOT as `disabled_reason`. A cadence that
1254    /// assembled cleanly at boot keeps sweeping on its boot-time io, so a
1255    /// manual refusal is a warning about the config as it stands now — calling
1256    /// it "disabled" would report a dead loop that is in fact running.
1257    #[tokio::test]
1258    async fn a_manual_run_records_its_assembly_refusal() {
1259        let dir = tempfile::tempdir().unwrap();
1260        let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1261        let mut c = cfg(vec![target("acme/one")]);
1262        c.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1263        // The coder sitting on its own panel: assembly refuses.
1264        c.coder_model = Some("gpt-5.4".into());
1265        let s = HealService::new(c, dir.path().join("coder"));
1266
1267        assert_eq!(
1268            s.status(&engine()).run_refusal,
1269            None,
1270            "nothing has refused yet"
1271        );
1272        let err = s
1273            .run_tick(&state)
1274            .await
1275            .expect_err("a coder on its panel must refuse");
1276        assert!(err.contains("also a review seat"), "{err}");
1277
1278        let st = s.status(&engine());
1279        assert!(
1280            st.run_refusal
1281                .as_deref()
1282                .is_some_and(|r| r.contains("also a review seat")),
1283            "the refusal must reach heal.status, not just the caller: {:?}",
1284            st.run_refusal
1285        );
1286        assert_eq!(
1287            st.disabled_reason, None,
1288            "the cadence was never assembled here, and a manual refusal is not \
1289             a statement that the loop is disabled"
1290        );
1291    }
1292
1293    /// The other half of the manual-refusal contract: a run that gets past
1294    /// assembly clears the refusal.
1295    ///
1296    /// Without this the first version traded one lie for a worse one — an
1297    /// operator who fixed the config kept reading the old refusal until the
1298    /// daemon restarted, because the only writer was `record_*` and nothing
1299    /// cleared.
1300    #[tokio::test]
1301    async fn a_successful_manual_run_clears_the_refusal() {
1302        let dir = tempfile::tempdir().unwrap();
1303        let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1304        let mut bad = cfg(vec![target("acme/one")]);
1305        bad.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1306        bad.coder_model = Some("gpt-5.4".into());
1307        let s = HealService::new(bad, dir.path().join("coder"));
1308
1309        s.run_tick(&state)
1310            .await
1311            .expect_err("a coder on its panel refuses");
1312        assert!(s.status(&engine()).run_refusal.is_some());
1313
1314        // The operator fixes it: same panel, a coder that is not a seat.
1315        let mut good = cfg(vec![target("acme/one")]);
1316        good.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1317        good.coder_model = Some("gpt-5.5".into());
1318        if let Ok(mut held) = s.config.write() {
1319            *held = good;
1320        }
1321
1322        s.run_tick(&state)
1323            .await
1324            .expect("the fixed config assembles");
1325        assert_eq!(
1326            s.status(&engine()).run_refusal,
1327            None,
1328            "a run that assembled must clear the refusal it is contradicting"
1329        );
1330    }
1331
1332    /// A manual run must not touch the BOOT slot. The two have different
1333    /// lifetimes: the cadence assembles once and does not start on failure, so
1334    /// its error is true until the daemon restarts, while a manual run's is a
1335    /// statement about the config right now.
1336    ///
1337    /// Conflating them broke both directions — a stale refusal that never
1338    /// cleared, AND a healthy sweeping cadence reporting `disabled_reason`
1339    /// permanently because someone ran `heal.run` on a bad edit.
1340    #[tokio::test]
1341    async fn a_manual_run_neither_sets_nor_clears_the_boot_assembly_error() {
1342        let dir = tempfile::tempdir().unwrap();
1343        let state = Arc::new(ServerState::standalone(dir.path().join("journal")));
1344        let mut c = cfg(vec![target("acme/one")]);
1345        c.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1346        c.coder_model = Some("gpt-5.4".into());
1347        let s = HealService::new(c, dir.path().join("coder"));
1348
1349        // The cadence died at boot. That stays true until a restart.
1350        s.record_assembly_error("boot: something the cadence could not assemble");
1351
1352        s.run_tick(&state).await.expect_err("still refuses");
1353        let st = s.status(&engine());
1354        assert!(
1355            st.disabled_reason
1356                .as_deref()
1357                .is_some_and(|r| r.contains("boot: something")),
1358            "a manual run must not overwrite the boot error: {:?}",
1359            st.disabled_reason
1360        );
1361        assert!(st.run_refusal.is_some(), "and must record its own");
1362
1363        // Now a manual run succeeds. The cadence is STILL dead — it returned
1364        // early at boot and nothing restarted it — so clearing the boot error
1365        // here would report a working loop that does not exist.
1366        let mut good = cfg(vec![target("acme/one")]);
1367        good.review_models = vec!["gpt-5.4".into(), "claude-opus-5".into()];
1368        good.coder_model = Some("gpt-5.5".into());
1369        if let Ok(mut held) = s.config.write() {
1370            *held = good;
1371        }
1372        s.run_tick(&state).await.expect("assembles now");
1373
1374        let st = s.status(&engine());
1375        assert_eq!(st.run_refusal, None);
1376        assert!(
1377            st.disabled_reason
1378                .as_deref()
1379                .is_some_and(|r| r.contains("boot: something")),
1380            "the cadence is still dead until restart: {:?}",
1381            st.disabled_reason
1382        );
1383    }
1384
1385    /// The status a `heal.status` caller gets before anything has run.
1386    #[test]
1387    fn status_names_the_targets_the_panel_and_the_engine() {
1388        let dir = tempfile::tempdir().unwrap();
1389        let s = HealService::new(cfg(vec![target("acme/one")]), dir.path().into());
1390        let st = s.status(&engine());
1391        assert!(st.enabled);
1392        assert_eq!(st.disabled_reason, None);
1393        assert_eq!(st.targets, vec!["acme/one".to_string()]);
1394        assert_eq!(st.review_models, vec!["reviewer-a".to_string()]);
1395        // The engine is reported even when it was defaulted, so an operator
1396        // never has to know what the default is to know what will run.
1397        assert_eq!(st.engine, DEFAULT_ENGINE);
1398    }
1399
1400    /// The default must be an engine that actually parses, or every daemon
1401    /// with a configured target refuses to assemble the loop at boot.
1402    #[test]
1403    fn the_default_engine_is_a_real_engine() {
1404        assert!(crate::coder::router::EngineChoice::parse(DEFAULT_ENGINE).is_ok());
1405    }
1406
1407    #[tokio::test]
1408    async fn a_sweep_visits_every_target_not_just_the_first() {
1409        // Stopping at the first with work would starve every later entry, and
1410        // the operator who listed them could not see it happening.
1411        let dir = tempfile::tempdir().unwrap();
1412        let s = HealService::new(
1413            cfg(vec![target("acme/one"), target("acme/two")]),
1414            dir.path().into(),
1415        );
1416        let io: Arc<dyn TickIo> = Arc::new(NoopIo);
1417        let report = s.sweep(&io).await;
1418        assert_eq!(report.outcomes.len(), 2);
1419        assert_eq!(report.outcomes[0].0, "acme/one");
1420        assert_eq!(report.outcomes[1].0, "acme/two");
1421    }
1422
1423    #[tokio::test]
1424    async fn an_overlapping_sweep_stands_down_rather_than_queueing() {
1425        let dir = tempfile::tempdir().unwrap();
1426        let s = Arc::new(HealService::new(
1427            cfg(vec![target("acme/one")]),
1428            dir.path().into(),
1429        ));
1430        let io: Arc<dyn TickIo> = Arc::new(NoopIo);
1431
1432        // Hold the lock as a concurrent sweep would.
1433        let held = s.running.lock().await;
1434        let report = s.sweep(&io).await;
1435        assert!(report.skipped_overlap);
1436        assert!(report.outcomes.is_empty());
1437        drop(held);
1438
1439        // And it runs again once the lock is free.
1440        assert!(!s.sweep(&io).await.skipped_overlap);
1441    }
1442
1443    #[tokio::test]
1444    async fn the_ledger_is_written_even_when_nothing_was_claimed() {
1445        // The file's existence is what makes the next start's load meaningful.
1446        let dir = tempfile::tempdir().unwrap();
1447        let s = HealService::new(cfg(vec![target("acme/one")]), dir.path().into());
1448        let io: Arc<dyn TickIo> = Arc::new(NoopIo);
1449        let _ = s.sweep(&io).await;
1450        assert!(
1451            dir.path().join("heal-claims.json").exists(),
1452            "the sweep persists the ledger around every target"
1453        );
1454    }
1455
1456    #[test]
1457    fn the_interval_can_be_overridden_but_never_to_zero() {
1458        // A zero interval is a busy loop against someone's API quota.
1459        let dir = tempfile::tempdir().unwrap();
1460        std::env::set_var(HEAL_INTERVAL_ENV, "0");
1461        let s = HealService::new(cfg(vec![]), dir.path().into());
1462        assert_eq!(s.interval_secs(), DEFAULT_INTERVAL_SECS);
1463        std::env::set_var(HEAL_INTERVAL_ENV, "60");
1464        let s = HealService::new(cfg(vec![]), dir.path().into());
1465        assert_eq!(s.interval_secs(), 60);
1466        std::env::remove_var(HEAL_INTERVAL_ENV);
1467    }
1468}