Skip to main content

car_server_core/
selfheal.rs

1//! Daemon wiring for deterministic self-healing detection and bounded PR fixes.
2//!
3//! The subsystem reads evidence CAR already owns, runs the pure
4//! `car-selfheal` detectors, and appends detections/dismissals/ticks/coder
5//! attempts to one private JSONL ledger. A bounded source probe renders local
6//! handoff documents; only a redactor-clean recurring builtin failure reaches
7//! the registry-owned PR template. Auto-fix never edits the maintainer's working
8//! tree or merges: it fetches `origin/main`, creates an isolated worktree, and
9//! delegates one contract-gated round to `car code-task`.
10
11use car_eventlog::{Alert, AlertThresholds, EventKind};
12use car_ir::ActionProposal;
13use car_selfheal::{
14    agent_gave_up, agent_log_errors, capability_miss, metrics_alerts, recurring_tool_failure,
15    AgentDetectorConfig, AgentLogDetectorConfig, Detection, DetectionKind, EventEvidence,
16    EvidenceSource, Redactor, Severity, SupervisorAgentState, SupervisorSnapshot,
17    ToolFailureConfig,
18};
19use chrono::{DateTime, Duration, Utc};
20use serde::{Deserialize, Serialize};
21use std::collections::{BTreeMap, BTreeSet, VecDeque};
22use std::io::{BufRead, BufReader, Read, Write};
23use std::path::{Path, PathBuf};
24use std::sync::Arc;
25use tokio::sync::Mutex;
26
27use crate::session::ServerState;
28
29pub const DEFAULT_SELFHEAL_INTERVAL_SECS: u64 = 15 * 60;
30pub const SELFHEAL_INTERVAL_ENV: &str = "CAR_SELFHEAL_INTERVAL_SECS";
31const MAX_PAGE_SIZE: usize = 500;
32const DEFAULT_PAGE_SIZE: usize = 100;
33const ACTIVITY_TAIL_LINES: usize = 40;
34const STDERR_TAIL_LINES: usize = 50;
35const EXPECTED_ORIGIN: &str = "Parslee-ai/car";
36const TRUST_TIER: &str = "trusted";
37const DEFAULT_AUTO_FIX: bool = true;
38const DEFAULT_MAX_CONCURRENT: u32 = 1;
39const DEFAULT_MAX_PER_DAY: u32 = 3;
40const DEFAULT_MAX_ROUNDS_PER_KEY: u32 = 3;
41// The JSON-RPC dispatcher abandons a handler after 1800 seconds. Bound the
42// child well inside that ceiling so the daemon can persist its terminal result.
43const SELFHEAL_CODER_WALL_SECS: u64 = 15 * 60;
44
45pub const DETECTOR_IDS: [&str; 5] = [
46    car_selfheal::detectors::metrics::DETECTOR_ID,
47    car_selfheal::detectors::agent::DETECTOR_ID,
48    car_selfheal::detectors::agent_logs::DETECTOR_ID,
49    car_selfheal::detectors::tools::DETECTOR_ID,
50    car_selfheal::detectors::capability::DETECTOR_ID,
51];
52
53/// Bounded filesystem inputs for source-checkout detection. The standalone
54/// daemon builds this once from its explicit CAR_HOME config, running-binary
55/// ancestors, and `.car/` project discovery. Each tick validates these paths
56/// using file reads only: never a subprocess, network request, or broad scan.
57#[derive(Debug, Clone, Default)]
58pub struct SelfhealSourceProbe {
59    explicit_checkout: Option<PathBuf>,
60    candidates: Vec<PathBuf>,
61    setup_refusal: Option<String>,
62}
63
64impl SelfhealSourceProbe {
65    /// Fail-closed probe with no source candidates. Suitable for embedders that
66    /// have not explicitly established a CAR source checkout.
67    pub fn empty() -> Self {
68        Self::default()
69    }
70
71    /// An explicit checkout is authoritative: if it fails validation, auto
72    /// candidates are not consulted.
73    pub fn explicit(path: PathBuf) -> Self {
74        Self {
75            explicit_checkout: Some(path),
76            ..Self::default()
77        }
78    }
79
80    /// Deterministic candidate order for embedders and tests.
81    pub fn candidates(paths: impl IntoIterator<Item = PathBuf>) -> Self {
82        Self {
83            candidates: paths.into_iter().collect(),
84            ..Self::default()
85        }
86    }
87
88    /// Add bounded fallback candidates. They are ignored when an explicit
89    /// checkout was configured, including when that explicit path is refused.
90    pub fn with_candidates(mut self, paths: impl IntoIterator<Item = PathBuf>) -> Self {
91        self.candidates.extend(paths);
92        dedup_paths(&mut self.candidates);
93        self
94    }
95
96    /// Build the production probe without launching `git` or touching the
97    /// network. Explicit config is `[selfheal] source_checkout = "..."` in
98    /// `<CAR_HOME>/config.toml`; relative values resolve under CAR_HOME.
99    /// Auto-candidates are bounded to running-binary ancestors and the root of
100    /// the `.car/` project discovered from `$CAR_PROJECT_DIR` or cwd.
101    pub fn from_process() -> Self {
102        let car_home = car_home::root_or_relative();
103        let executable = std::env::current_exe().ok();
104        let anchor = std::env::var_os("CAR_PROJECT_DIR")
105            .map(PathBuf::from)
106            .or_else(|| std::env::current_dir().ok());
107        Self::from_local_paths(&car_home, executable.as_deref(), anchor.as_deref())
108    }
109
110    /// Deterministic seam for verifying the exact production discovery path.
111    /// `executable` is the binary file, and `project_anchor` is where `.car/`
112    /// walk-up starts.
113    #[doc(hidden)]
114    pub fn from_local_paths(
115        car_home: &Path,
116        executable: Option<&Path>,
117        project_anchor: Option<&Path>,
118    ) -> Self {
119        let config_path = car_home.join("config.toml");
120        let mut probe = Self::default();
121        match read_source_checkout_config(&config_path) {
122            Ok(Some(path)) => {
123                probe.explicit_checkout = Some(if path.is_absolute() {
124                    path
125                } else {
126                    car_home.join(path)
127                });
128            }
129            Ok(None) => {}
130            Err(error) => probe.setup_refusal = Some(error),
131        }
132
133        if let Some(parent) = executable.and_then(Path::parent) {
134            probe
135                .candidates
136                .extend(parent.ancestors().map(Path::to_path_buf));
137        }
138        if let Some(car_dir) = project_anchor.and_then(car_memgine::project::discover_project) {
139            if let Some(project_root) = car_dir.parent() {
140                probe.candidates.push(project_root.to_path_buf());
141            }
142        }
143        dedup_paths(&mut probe.candidates);
144        probe
145    }
146
147    fn resolve(&self) -> SourceRouteDecision {
148        if let Some(reason) = &self.setup_refusal {
149            return SourceRouteDecision::ledger_only(reason.clone());
150        }
151        if let Some(candidate) = &self.explicit_checkout {
152            return match validate_source_checkout(candidate) {
153                Ok(path) => SourceRouteDecision::local(path),
154                Err(reason) => SourceRouteDecision::ledger_only(format!(
155                    "explicit selfheal.source_checkout {} refused: {reason}",
156                    candidate.display()
157                )),
158            };
159        }
160
161        let mut refusals = Vec::new();
162        for candidate in &self.candidates {
163            match validate_source_checkout(candidate) {
164                Ok(path) => return SourceRouteDecision::local(path),
165                Err(reason) => refusals.push(format!("{}: {reason}", candidate.display())),
166            }
167        }
168        if refusals.is_empty() {
169            SourceRouteDecision::ledger_only(
170                "no bounded source-checkout candidates were discovered".to_string(),
171            )
172        } else {
173            SourceRouteDecision::ledger_only(format!(
174                "no candidate validated as {EXPECTED_ORIGIN}; refusals: {}",
175                refusals.join("; ")
176            ))
177        }
178    }
179}
180
181/// Where a self-heal detection was routed. `feedback` is reserved in the wire
182/// shape for the separately gated feedback sink; this implementation never
183/// selects it and fails closed to `ledger-only`.
184#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
185#[serde(rename_all = "kebab-case")]
186pub enum SelfhealRoute {
187    Local,
188    Feedback,
189    #[default]
190    LedgerOnly,
191}
192
193#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
194pub struct SourceRouteDecision {
195    pub route: SelfhealRoute,
196    #[serde(default, skip_serializing_if = "Option::is_none")]
197    pub source_checkout: Option<PathBuf>,
198    #[serde(default, skip_serializing_if = "Option::is_none")]
199    pub refusal_reason: Option<String>,
200}
201
202impl SourceRouteDecision {
203    fn local(path: PathBuf) -> Self {
204        Self {
205            route: SelfhealRoute::Local,
206            source_checkout: Some(path),
207            refusal_reason: None,
208        }
209    }
210
211    fn ledger_only(reason: String) -> Self {
212        Self {
213            route: SelfhealRoute::LedgerOnly,
214            source_checkout: None,
215            refusal_reason: Some(reason),
216        }
217    }
218}
219
220/// How a tick proves that its validated checkout already contains the replay
221/// verb. Production builds compile that checkout's CLI and invoke its help;
222/// the fixed variants are deterministic seams for integration tests.
223#[derive(Debug, Clone, Default)]
224pub enum SelfhealReplayVerbProbe {
225    #[default]
226    CheckoutBuild,
227    #[doc(hidden)]
228    Available,
229    #[doc(hidden)]
230    Unavailable(String),
231}
232
233impl SelfhealReplayVerbProbe {
234    #[doc(hidden)]
235    pub fn available_for_tests() -> Self {
236        Self::Available
237    }
238
239    #[doc(hidden)]
240    pub fn unavailable_for_tests(reason: impl Into<String>) -> Self {
241        Self::Unavailable(reason.into())
242    }
243
244    async fn check(&self, checkout: &Path, target_dir: &Path) -> Result<(), String> {
245        match self {
246            Self::Available => Ok(()),
247            Self::Unavailable(reason) => Err(reason.clone()),
248            Self::CheckoutBuild => check_checkout_replay_verb(checkout, target_dir).await,
249        }
250    }
251}
252
253/// Evidence assembled by the daemon for one tick. Public only so embedders and
254/// integration tests can supply deterministic evidence without touching the
255/// real process supervisor or user event journals.
256#[derive(Debug, Clone, Default)]
257pub struct SelfhealEvidence {
258    pub events: Vec<EventEvidence>,
259    pub metric_alerts: Vec<Alert>,
260    pub metric_provenance: Vec<EvidenceSource>,
261    pub supervisor_snapshot: Option<SupervisorSnapshot>,
262    pub registry_agents: usize,
263}
264
265#[derive(Debug, Clone, Serialize, Deserialize)]
266pub struct TickSummary {
267    pub started_at: DateTime<Utc>,
268    pub completed_at: DateTime<Utc>,
269    #[serde(default)]
270    pub route: SelfhealRoute,
271    #[serde(default, skip_serializing_if = "Option::is_none")]
272    pub source_checkout: Option<PathBuf>,
273    #[serde(default, skip_serializing_if = "Option::is_none")]
274    pub refusal_reason: Option<String>,
275    pub events_scanned: usize,
276    pub supervisor_agents: usize,
277    pub registry_agents: usize,
278    pub detections_found: usize,
279    pub appended: usize,
280    pub changed: usize,
281    pub suppressed_dismissed: usize,
282    pub filing_mode: String,
283}
284
285#[derive(Debug, Clone, Serialize)]
286pub struct SelfhealStatus {
287    pub cadence_secs: u64,
288    pub auto_fix_enabled: bool,
289    pub max_concurrent: u32,
290    pub max_per_day: u32,
291    pub max_rounds_per_key: u32,
292    #[serde(default, skip_serializing_if = "Option::is_none")]
293    pub auto_fix_refusal_reason: Option<String>,
294    pub last_tick_at: Option<DateTime<Utc>>,
295    pub route: SelfhealRoute,
296    #[serde(default, skip_serializing_if = "Option::is_none")]
297    pub source_checkout: Option<PathBuf>,
298    #[serde(default, skip_serializing_if = "Option::is_none")]
299    pub refusal_reason: Option<String>,
300    pub detectors: Vec<&'static str>,
301    pub detection_count: usize,
302    pub warning_count: usize,
303    pub critical_count: usize,
304    pub dismissed_count: usize,
305    pub filing_mode: &'static str,
306}
307
308#[derive(Debug, Clone, Deserialize, Default)]
309pub struct DetectionQuery {
310    #[serde(default)]
311    pub kind: Option<DetectionKind>,
312    #[serde(default)]
313    pub severity: Option<Severity>,
314    #[serde(default)]
315    pub since: Option<DateTime<Utc>>,
316    #[serde(default)]
317    pub offset: usize,
318    #[serde(default)]
319    pub limit: Option<usize>,
320}
321
322/// A secret-safe tool invocation reconstructed from the accepted proposal
323/// preimage in the same journal as its `ActionFailed` event.
324#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
325pub struct ReconstructedToolCall {
326    pub tool: String,
327    pub params: BTreeMap<String, serde_json::Value>,
328}
329
330#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
331pub struct RoutedDetection {
332    #[serde(flatten)]
333    pub detection: Detection,
334    pub route: SelfhealRoute,
335    #[serde(default, skip_serializing_if = "Option::is_none")]
336    pub local_issue_path: Option<PathBuf>,
337    /// Present only for `recurring_tool_failure`. True requires a recovered
338    /// call, builtin provenance, and a byte-unchanged redactor pass.
339    #[serde(default, skip_serializing_if = "Option::is_none")]
340    pub eligible: Option<bool>,
341    /// Omitted when parameters were unavailable or the redactor changed them;
342    /// unsafe raw parameters are never copied out of `ProposalReceived`.
343    #[serde(default, skip_serializing_if = "Option::is_none")]
344    pub reconstructed_call: Option<ReconstructedToolCall>,
345    /// Owner-private sidecar beside `local_issue_path`, when routed locally.
346    #[serde(default, skip_serializing_if = "Option::is_none")]
347    pub reconstructed_call_path: Option<PathBuf>,
348    /// Number of durable coder rounds started for this stable key.
349    #[serde(default, skip_serializing_if = "is_zero_u32")]
350    pub auto_fix_attempts: u32,
351    /// True once the configured/hard three-round ceiling is reached.
352    #[serde(default, skip_serializing_if = "is_false")]
353    pub auto_fix_exhausted: bool,
354    /// True when a durable start marker has no terminal attempt record yet.
355    #[serde(default, skip_serializing_if = "is_false")]
356    pub auto_fix_in_progress: bool,
357    #[serde(default, skip_serializing_if = "Option::is_none")]
358    pub last_auto_fix_attempt: Option<FixAttemptResult>,
359    /// An already-open stable-key PR exists remotely; the review loop owns it.
360    #[serde(default, skip_serializing_if = "is_false")]
361    pub auto_fix_awaiting_review: bool,
362    /// The stable-key PR was closed without merge and must never be reopened.
363    #[serde(default, skip_serializing_if = "is_false")]
364    pub auto_fix_parked: bool,
365    #[serde(default, skip_serializing_if = "Option::is_none")]
366    pub remote_pr_number: Option<u64>,
367    #[serde(default, skip_serializing_if = "Option::is_none")]
368    pub remote_pr_url: Option<String>,
369}
370
371fn is_zero_u32(value: &u32) -> bool {
372    *value == 0
373}
374
375fn is_false(value: &bool) -> bool {
376    !*value
377}
378
379#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
380#[serde(rename_all = "snake_case")]
381pub enum FixTrigger {
382    Cadence,
383    Manual,
384}
385
386#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
387#[serde(rename_all = "snake_case")]
388pub enum RemotePrDisposition {
389    AwaitingReview,
390    Parked,
391}
392
393#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
394pub struct FixAttemptResult {
395    pub dedup_key: String,
396    pub round: u32,
397    pub trigger: FixTrigger,
398    pub started_at: DateTime<Utc>,
399    pub completed_at: DateTime<Utc>,
400    pub target_branch: String,
401    pub workspace_dir: PathBuf,
402    pub spawned: bool,
403    pub exit_code: Option<i32>,
404    pub failure_class: String,
405}
406
407impl RoutedDetection {
408    pub fn dedup_key(&self) -> &str {
409        self.detection.dedup_key()
410    }
411}
412
413#[derive(Debug, Clone, Serialize)]
414pub struct DetectionPage {
415    pub detections: Vec<RoutedDetection>,
416    pub total: usize,
417    pub offset: usize,
418    pub limit: usize,
419    pub next_offset: Option<usize>,
420}
421
422#[derive(Debug, Clone, Serialize)]
423pub struct DismissResult {
424    pub dedup_key: String,
425    pub dismissed: bool,
426    pub already_dismissed: bool,
427    pub dismissed_at: DateTime<Utc>,
428}
429
430#[derive(Debug, Clone, Serialize, Deserialize)]
431#[serde(tag = "record_type", rename_all = "snake_case")]
432enum LedgerRecord {
433    Detection {
434        recorded_at: DateTime<Utc>,
435        detection: Detection,
436        #[serde(default)]
437        route: SelfhealRoute,
438        #[serde(default, skip_serializing_if = "Option::is_none")]
439        local_issue_path: Option<PathBuf>,
440        #[serde(default, skip_serializing_if = "Option::is_none")]
441        eligible: Option<bool>,
442        #[serde(default, skip_serializing_if = "Option::is_none")]
443        reconstructed_call: Option<ReconstructedToolCall>,
444        #[serde(default, skip_serializing_if = "Option::is_none")]
445        reconstructed_call_path: Option<PathBuf>,
446    },
447    Dismissal {
448        dismissed_at: DateTime<Utc>,
449        dedup_key: String,
450    },
451    Tick {
452        summary: TickSummary,
453    },
454    FixStarted {
455        started_at: DateTime<Utc>,
456        dedup_key: String,
457        round: u32,
458        trigger: FixTrigger,
459        target_branch: String,
460        workspace_dir: PathBuf,
461    },
462    FixAttempt {
463        result: FixAttemptResult,
464    },
465    RemotePr {
466        observed_at: DateTime<Utc>,
467        dedup_key: String,
468        disposition: RemotePrDisposition,
469        pr_number: u64,
470        pr_url: String,
471    },
472}
473
474#[derive(Default)]
475struct LedgerState {
476    records: Vec<LedgerRecord>,
477    supervisor_snapshots: VecDeque<SupervisorSnapshot>,
478}
479
480/// Daemon-wide detector, private ledger, and bounded PR-fix owner.
481pub struct SelfhealService {
482    ledger_path: PathBuf,
483    state_root: PathBuf,
484    interval_secs: u64,
485    state: Mutex<LedgerState>,
486    operation: Mutex<()>,
487    evidence_override: Option<SelfhealEvidence>,
488    source_probe: SelfhealSourceProbe,
489    replay_verb_probe: SelfhealReplayVerbProbe,
490    auto_fix: AutoFixConfig,
491    auto_fix_refusal_reason: Option<String>,
492}
493
494impl SelfhealService {
495    pub fn open(
496        ledger_path: PathBuf,
497        interval_secs: u64,
498        evidence_override: Option<SelfhealEvidence>,
499        source_probe: SelfhealSourceProbe,
500        replay_verb_probe: SelfhealReplayVerbProbe,
501    ) -> Result<Self, String> {
502        let records = load_records(&ledger_path)?;
503        let state_root = ledger_path
504            .parent()
505            .and_then(Path::parent)
506            .map(Path::to_path_buf)
507            .ok_or_else(|| "self-heal ledger must live under <CAR_HOME>/selfheal".to_string())?;
508        let (auto_fix, auto_fix_refusal_reason) =
509            match read_auto_fix_config(&state_root.join("config.toml")) {
510                Ok(config) => (config, None),
511                Err(reason) => (AutoFixConfig::disabled(), Some(reason)),
512            };
513        Ok(Self {
514            ledger_path,
515            state_root,
516            interval_secs: interval_secs.max(1),
517            state: Mutex::new(LedgerState {
518                records,
519                supervisor_snapshots: VecDeque::new(),
520            }),
521            operation: Mutex::new(()),
522            evidence_override,
523            source_probe,
524            replay_verb_probe,
525            auto_fix,
526            auto_fix_refusal_reason,
527        })
528    }
529
530    pub fn interval_secs(&self) -> u64 {
531        self.interval_secs
532    }
533
534    pub fn ledger_path(&self) -> &Path {
535        &self.ledger_path
536    }
537
538    pub async fn status(&self) -> SelfhealStatus {
539        let state = self.state.lock().await;
540        let (active, dismissed) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
541        let mut warning_count = 0;
542        let mut critical_count = 0;
543        for detection in active.values() {
544            match detection.detection.severity() {
545                Severity::Warning => warning_count += 1,
546                Severity::Critical => critical_count += 1,
547            }
548        }
549        let latest_tick = state.records.iter().rev().find_map(|record| match record {
550            LedgerRecord::Tick { summary } => Some(summary),
551            LedgerRecord::Detection { .. }
552            | LedgerRecord::Dismissal { .. }
553            | LedgerRecord::FixStarted { .. }
554            | LedgerRecord::FixAttempt { .. }
555            | LedgerRecord::RemotePr { .. } => None,
556        });
557        let route = latest_tick.map_or_else(
558            || SourceRouteDecision::ledger_only("source route not evaluated yet".to_string()),
559            |summary| SourceRouteDecision {
560                route: summary.route,
561                source_checkout: summary.source_checkout.clone(),
562                refusal_reason: summary.refusal_reason.clone(),
563            },
564        );
565        SelfhealStatus {
566            cadence_secs: self.interval_secs,
567            auto_fix_enabled: self.auto_fix.auto_fix && self.auto_fix_refusal_reason.is_none(),
568            max_concurrent: self.auto_fix.max_concurrent,
569            max_per_day: self.auto_fix.max_per_day,
570            max_rounds_per_key: self.auto_fix.max_rounds_per_key,
571            auto_fix_refusal_reason: self.auto_fix_refusal_reason.clone(),
572            last_tick_at: latest_tick.map(|summary| summary.completed_at),
573            route: route.route,
574            source_checkout: route.source_checkout,
575            refusal_reason: route.refusal_reason,
576            detectors: DETECTOR_IDS.to_vec(),
577            detection_count: active.len(),
578            warning_count,
579            critical_count,
580            dismissed_count: dismissed.len(),
581            filing_mode: if self.auto_fix.auto_fix
582                && self.auto_fix_refusal_reason.is_none()
583                && route.route == SelfhealRoute::Local
584            {
585                "pr-only"
586            } else {
587                "watch-only"
588            },
589        }
590    }
591
592    pub async fn detections(&self, query: DetectionQuery) -> DetectionPage {
593        let state = self.state.lock().await;
594        let (active, _) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
595        let mut detections: Vec<_> = active
596            .into_values()
597            .filter(|detection| {
598                query
599                    .kind
600                    .is_none_or(|kind| detection.detection.kind() == kind)
601                    && query
602                        .severity
603                        .is_none_or(|severity| detection.detection.severity() == severity)
604                    && query
605                        .since
606                        .is_none_or(|since| detection.detection.last_observed_at() >= since)
607            })
608            .collect();
609        detections.sort_by(|a, b| {
610            b.detection
611                .last_observed_at()
612                .cmp(&a.detection.last_observed_at())
613                .then_with(|| a.dedup_key().cmp(b.dedup_key()))
614        });
615        let total = detections.len();
616        let limit = query
617            .limit
618            .unwrap_or(DEFAULT_PAGE_SIZE)
619            .clamp(1, MAX_PAGE_SIZE);
620        let page = detections
621            .into_iter()
622            .skip(query.offset)
623            .take(limit)
624            .collect::<Vec<_>>();
625        let consumed = query.offset.saturating_add(page.len());
626        DetectionPage {
627            detections: page,
628            total,
629            offset: query.offset,
630            limit,
631            next_offset: (consumed < total).then_some(consumed),
632        }
633    }
634
635    pub async fn dismiss(&self, dedup_key: &str) -> Result<DismissResult, String> {
636        validate_dedup_key(dedup_key)?;
637        let _operation = self.operation.lock().await;
638        let dismissed_at = Utc::now();
639        let mut state = self.state.lock().await;
640        let known = state.records.iter().any(|record| {
641            matches!(
642                record,
643                LedgerRecord::Detection { detection, .. } if detection.dedup_key() == dedup_key
644            )
645        });
646        if !known {
647            return Err(format!(
648                "unknown self-heal detection dedup_key '{dedup_key}'"
649            ));
650        }
651        let (_, dismissed) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
652        let already_dismissed = dismissed.contains(dedup_key);
653        let record = LedgerRecord::Dismissal {
654            dismissed_at,
655            dedup_key: dedup_key.to_string(),
656        };
657        append_records(&self.ledger_path, std::slice::from_ref(&record))?;
658        state.records.push(record);
659        Ok(DismissResult {
660            dedup_key: dedup_key.to_string(),
661            dismissed: true,
662            already_dismissed,
663            dismissed_at,
664        })
665    }
666
667    /// Start one explicit coder round for an eligible recurring-tool key.
668    /// The off switch controls unattended cadence only; an operator invoking
669    /// `selfheal.fix` may still run a bounded round deliberately.
670    pub async fn fix(&self, dedup_key: &str) -> Result<FixAttemptResult, String> {
671        validate_dedup_key(dedup_key)?;
672        let _operation = self
673            .operation
674            .try_lock()
675            .map_err(|_| "another self-heal operation is already running".to_string())?;
676        match self.reconcile_remote_pr_unlocked(dedup_key).await? {
677            RemoteDedupResult::Proceed => {
678                self.run_fix_unlocked(dedup_key, FixTrigger::Manual).await
679            }
680            RemoteDedupResult::AwaitingReview => Err(format!(
681                "self-heal detection '{dedup_key}' is awaiting review on its open remote PR"
682            )),
683            RemoteDedupResult::Parked => Err(format!(
684                "self-heal detection '{dedup_key}' is parked after its PR closed unmerged"
685            )),
686        }
687    }
688
689    async fn run_auto_fix_unlocked(&self) -> Result<Option<FixAttemptResult>, String> {
690        if !self.auto_fix.auto_fix || self.auto_fix_refusal_reason.is_some() {
691            return Ok(None);
692        }
693        let candidate = {
694            let state = self.state.lock().await;
695            if starts_today(&state.records, Utc::now()) >= self.auto_fix.max_per_day {
696                return Ok(None);
697            }
698            let (active, _) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
699            active
700                .into_values()
701                .filter(|detection| detection.eligible == Some(true))
702                // Cadence owns only the initial round. Review-driven/manual
703                // rounds reuse the key and stable PR branch explicitly.
704                .filter(|detection| detection.auto_fix_attempts == 0)
705                .filter(|detection| {
706                    !detection.auto_fix_awaiting_review && !detection.auto_fix_parked
707                })
708                .min_by(|left, right| {
709                    left.detection
710                        .last_observed_at()
711                        .cmp(&right.detection.last_observed_at())
712                        .then_with(|| left.dedup_key().cmp(right.dedup_key()))
713                })
714                .map(|detection| detection.dedup_key().to_string())
715        };
716        match candidate {
717            Some(key) => match self.reconcile_remote_pr_unlocked(&key).await? {
718                RemoteDedupResult::Proceed => self
719                    .run_fix_unlocked(&key, FixTrigger::Cadence)
720                    .await
721                    .map(Some),
722                RemoteDedupResult::AwaitingReview | RemoteDedupResult::Parked => Ok(None),
723            },
724            None => Ok(None),
725        }
726    }
727
728    async fn reconcile_remote_pr_unlocked(
729        &self,
730        dedup_key: &str,
731    ) -> Result<RemoteDedupResult, String> {
732        let route = self.source_probe.resolve();
733        let Some(checkout) = route.source_checkout else {
734            return Ok(RemoteDedupResult::Proceed);
735        };
736        let attempts = {
737            let state = self.state.lock().await;
738            let (active, _) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
739            active
740                .get(dedup_key)
741                .map_or(0, |detection| detection.auto_fix_attempts)
742        };
743        if attempts > 0 {
744            return Ok(RemoteDedupResult::Proceed);
745        }
746
747        let Some(remote) = query_remote_selfheal_pr(&checkout, dedup_key).await? else {
748            return Ok(RemoteDedupResult::Proceed);
749        };
750        let disposition = match (remote.state.as_str(), remote.merged_at.as_deref()) {
751            ("OPEN", _) => RemotePrDisposition::AwaitingReview,
752            ("CLOSED", None) => RemotePrDisposition::Parked,
753            // A merged stable-key PR also makes a fresh initial round invalid;
754            // never recreate the branch after delivery completed.
755            ("CLOSED", Some(_)) => RemotePrDisposition::Parked,
756            (state, _) => {
757                return Err(format!("remote self-heal PR has unknown state '{state}'"));
758            }
759        };
760        let record = LedgerRecord::RemotePr {
761            observed_at: Utc::now(),
762            dedup_key: dedup_key.to_string(),
763            disposition,
764            pr_number: remote.number,
765            pr_url: remote.url,
766        };
767        let mut state = self.state.lock().await;
768        append_records(&self.ledger_path, std::slice::from_ref(&record))?;
769        state.records.push(record);
770        Ok(match disposition {
771            RemotePrDisposition::AwaitingReview => RemoteDedupResult::AwaitingReview,
772            RemotePrDisposition::Parked => RemoteDedupResult::Parked,
773        })
774    }
775
776    async fn run_fix_unlocked(
777        &self,
778        dedup_key: &str,
779        trigger: FixTrigger,
780    ) -> Result<FixAttemptResult, String> {
781        if let Some(reason) = &self.auto_fix_refusal_reason {
782            return Err(format!("self-heal auto-fix config refused: {reason}"));
783        }
784        let route = self.source_probe.resolve();
785        let checkout = route.source_checkout.ok_or_else(|| {
786            format!(
787                "self-heal key '{dedup_key}' cannot auto-fix from route {}: {}",
788                route_name(route.route),
789                route
790                    .refusal_reason
791                    .unwrap_or_else(|| "no validated source checkout".to_string())
792            )
793        })?;
794        let (detection, round) = {
795            let state = self.state.lock().await;
796            let (active, _) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
797            let detection = active
798                .get(dedup_key)
799                .cloned()
800                .ok_or_else(|| format!("unknown or dismissed self-heal detection '{dedup_key}'"))?;
801            if detection.eligible != Some(true) {
802                return Err(format!(
803                    "self-heal detection '{dedup_key}' is not eligible for auto-fix"
804                ));
805            }
806            if detection.auto_fix_in_progress {
807                return Err(format!(
808                    "self-heal detection '{dedup_key}' already has a coder round in progress"
809                ));
810            }
811            if detection.auto_fix_attempts >= self.auto_fix.max_rounds_per_key {
812                return Err(format!(
813                    "self-heal detection '{dedup_key}' is auto-fix exhausted"
814                ));
815            }
816            if starts_today(&state.records, Utc::now()) >= self.auto_fix.max_per_day {
817                return Err(format!(
818                    "self-heal daily auto-fix limit ({}) is exhausted",
819                    self.auto_fix.max_per_day
820                ));
821            }
822            let round = detection.auto_fix_attempts + 1;
823            (detection, round)
824        };
825        let call = detection
826            .reconstructed_call
827            .as_ref()
828            .ok_or_else(|| format!("eligible self-heal key '{dedup_key}' has no safe call"))?;
829
830        let attempt_dir = self
831            .state_root
832            .join("selfheal")
833            .join("attempts")
834            .join(dedup_key);
835        car_secrets::ensure_private_dir(&attempt_dir)
836            .map_err(|error| format!("create self-heal attempt directory: {error}"))?;
837        let params_path = attempt_dir.join("params.json");
838        persist_private_json(&params_path, &call.params)?;
839        let rendered =
840            crate::selfheal_templates::render(&crate::selfheal_templates::TemplateContext {
841                dedup_key,
842                detector_id: detection.detection.detector_id(),
843                locator: detection.detection.locator(),
844                tool: &call.tool,
845                params_path: &params_path,
846            })?;
847        let intent_path = attempt_dir.join("intent.md");
848        persist_private_bytes(&intent_path, rendered.intent.as_bytes())?;
849        let contract_path = attempt_dir.join("contract.json");
850        persist_private_json(&contract_path, &rendered.contract)?;
851
852        let target_branch = format!("car/selfheal/{dedup_key}");
853        let workspace_dir = checkout
854            .join(".worktrees")
855            .join(format!("selfheal-{dedup_key}"));
856        let target_dir = self.state_root.join("selfheal").join("target");
857        car_secrets::ensure_private_dir(&target_dir)
858            .map_err(|error| format!("create self-heal cargo target directory: {error}"))?;
859        // Resolve the orchestrator before prepending the isolated target to its
860        // child PATH. Otherwise round two could launch round one's freshly
861        // built (and not yet re-verified) `car` binary as the orchestrator.
862        let car_executable = resolve_car_cli_executable();
863        let mut path_entries = vec![target_dir.join("debug")];
864        if let Some(path) = std::env::var_os("PATH") {
865            path_entries.extend(std::env::split_paths(&path));
866        }
867        let code_task_path = std::env::join_paths(path_entries)
868            .map_err(|error| format!("build self-heal code-task PATH: {error}"))?;
869        let started_at = Utc::now();
870        let started = LedgerRecord::FixStarted {
871            started_at,
872            dedup_key: dedup_key.to_string(),
873            round,
874            trigger: trigger.clone(),
875            target_branch: target_branch.clone(),
876            workspace_dir: workspace_dir.clone(),
877        };
878        {
879            let mut state = self.state.lock().await;
880            append_records(&self.ledger_path, std::slice::from_ref(&started))?;
881            state.records.push(started);
882        }
883
884        let fetch = tokio::process::Command::new("git")
885            .current_dir(&checkout)
886            .args(["fetch", "origin", "main"])
887            .kill_on_drop(true)
888            .output()
889            .await;
890        match fetch {
891            Err(_) => {
892                return self
893                    .finish_attempt(
894                        dedup_key,
895                        round,
896                        trigger,
897                        started_at,
898                        target_branch,
899                        workspace_dir,
900                        false,
901                        None,
902                        "fetch_spawn_failed".to_string(),
903                    )
904                    .await;
905            }
906            Ok(output) if !output.status.success() => {
907                return self
908                    .finish_attempt(
909                        dedup_key,
910                        round,
911                        trigger,
912                        started_at,
913                        target_branch,
914                        workspace_dir,
915                        false,
916                        output.status.code(),
917                        "fetch_failed".to_string(),
918                    )
919                    .await;
920            }
921            Ok(_) => {}
922        }
923
924        let Some(car_executable) = car_executable else {
925            return self
926                .finish_attempt(
927                    dedup_key,
928                    round,
929                    trigger,
930                    started_at,
931                    target_branch,
932                    workspace_dir,
933                    false,
934                    None,
935                    "spawn_failed".to_string(),
936                )
937                .await;
938        };
939        let marker = format!("<!-- car-selfheal:key={dedup_key} -->");
940        let transcript_path = attempt_dir.join(format!("code-task-round-{round}.jsonl"));
941        persist_private_bytes(&transcript_path, b"")?;
942        let argv = vec![
943            "code-task".to_string(),
944            "--repo".to_string(),
945            checkout.to_string_lossy().into_owned(),
946            "--intent-file".to_string(),
947            intent_path.to_string_lossy().into_owned(),
948            "--contract-file".to_string(),
949            contract_path.to_string_lossy().into_owned(),
950            "--deliver".to_string(),
951            "pr".to_string(),
952            "--target-branch".to_string(),
953            target_branch.clone(),
954            "--pr-base".to_string(),
955            "main".to_string(),
956            "--workspace-dir".to_string(),
957            workspace_dir.to_string_lossy().into_owned(),
958            "--body-prefix".to_string(),
959            marker,
960            "--max-session-wall-secs".to_string(),
961            SELFHEAL_CODER_WALL_SECS.to_string(),
962            "--transcript".to_string(),
963            transcript_path.to_string_lossy().into_owned(),
964            "--json".to_string(),
965        ];
966        let output = tokio::process::Command::new(car_executable)
967            .args(&argv)
968            .env("CARGO_TARGET_DIR", &target_dir)
969            // The first contract line builds this worktree's CLI into the
970            // isolated target. The second line's literal `car tools call`
971            // must therefore resolve that fresh binary, not the installed
972            // daemon binary whose source predates the repair.
973            .env("PATH", code_task_path)
974            .kill_on_drop(true)
975            .output()
976            .await;
977        match output {
978            Err(_) => {
979                self.finish_attempt(
980                    dedup_key,
981                    round,
982                    trigger,
983                    started_at,
984                    target_branch,
985                    workspace_dir,
986                    false,
987                    None,
988                    "spawn_failed".to_string(),
989                )
990                .await
991            }
992            Ok(output) => {
993                let failure_class = classify_code_task_output(&output.stdout);
994                self.finish_attempt(
995                    dedup_key,
996                    round,
997                    trigger,
998                    started_at,
999                    target_branch,
1000                    workspace_dir,
1001                    true,
1002                    output.status.code(),
1003                    failure_class,
1004                )
1005                .await
1006            }
1007        }
1008    }
1009
1010    #[allow(clippy::too_many_arguments)]
1011    async fn finish_attempt(
1012        &self,
1013        dedup_key: &str,
1014        round: u32,
1015        trigger: FixTrigger,
1016        started_at: DateTime<Utc>,
1017        target_branch: String,
1018        workspace_dir: PathBuf,
1019        spawned: bool,
1020        exit_code: Option<i32>,
1021        failure_class: String,
1022    ) -> Result<FixAttemptResult, String> {
1023        let result = FixAttemptResult {
1024            dedup_key: dedup_key.to_string(),
1025            round,
1026            trigger,
1027            started_at,
1028            completed_at: Utc::now(),
1029            target_branch,
1030            workspace_dir,
1031            spawned,
1032            exit_code,
1033            failure_class,
1034        };
1035        let record = LedgerRecord::FixAttempt {
1036            result: result.clone(),
1037        };
1038        let mut state = self.state.lock().await;
1039        append_records(&self.ledger_path, std::slice::from_ref(&record))?;
1040        state.records.push(record);
1041        Ok(result)
1042    }
1043
1044    pub async fn run_tick(&self, server: &Arc<ServerState>) -> Result<TickSummary, String> {
1045        let _operation = self
1046            .operation
1047            .try_lock()
1048            .map_err(|_| "self-heal tick already running".to_string())?;
1049        let started_at = Utc::now();
1050        // Use tick start as the event-log watermark, not completion. Events
1051        // appended while a slow scan is in flight are therefore deferred to
1052        // (not skipped by) the next slice.
1053        let since = {
1054            let state = self.state.lock().await;
1055            state.records.iter().rev().find_map(|record| match record {
1056                LedgerRecord::Tick { summary } => Some(summary.started_at),
1057                LedgerRecord::Detection { .. }
1058                | LedgerRecord::Dismissal { .. }
1059                | LedgerRecord::FixStarted { .. }
1060                | LedgerRecord::FixAttempt { .. }
1061                | LedgerRecord::RemotePr { .. } => None,
1062            })
1063        };
1064        let evidence = match self.evidence_override.clone() {
1065            Some(evidence) => evidence,
1066            None => gather_live_evidence(server, &self.state_root, since, started_at).await,
1067        };
1068        // Validate once for this tick, then reuse the immutable result for its
1069        // summary and every routed detection.
1070        let route = self.source_probe.resolve();
1071        let summary = self.run_with_evidence(started_at, evidence, route).await?;
1072        if let Err(error) = self.run_auto_fix_unlocked().await {
1073            tracing::warn!(target: "car::selfheal", %error, "self-heal auto-fix round failed");
1074        }
1075        Ok(summary)
1076    }
1077
1078    async fn run_with_evidence(
1079        &self,
1080        started_at: DateTime<Utc>,
1081        evidence: SelfhealEvidence,
1082        route: SourceRouteDecision,
1083    ) -> Result<TickSummary, String> {
1084        let redactor = Redactor::from_env(std::env::vars());
1085        let car_version = env!("CARGO_PKG_VERSION");
1086        let observed_at = Utc::now();
1087        let mut found = Vec::new();
1088        found.extend(metrics_alerts(
1089            &evidence.metric_alerts,
1090            car_version,
1091            observed_at,
1092            &evidence.metric_provenance,
1093            &redactor,
1094        ));
1095        found.extend(recurring_tool_failure(
1096            &evidence.events,
1097            ToolFailureConfig::default(),
1098            car_version,
1099            &redactor,
1100        ));
1101        found.extend(capability_miss(&evidence.events, car_version, &redactor));
1102
1103        let supervisor_agents = evidence
1104            .supervisor_snapshot
1105            .as_ref()
1106            .map_or(0, |snapshot| snapshot.agents.len());
1107        if let Some(snapshot) = evidence.supervisor_snapshot.as_ref() {
1108            found.extend(agent_log_errors(
1109                snapshot,
1110                AgentLogDetectorConfig::default(),
1111                car_version,
1112                &redactor,
1113            ));
1114        }
1115        let snapshots = {
1116            let mut state = self.state.lock().await;
1117            if let Some(snapshot) = evidence.supervisor_snapshot {
1118                state.supervisor_snapshots.push_back(snapshot);
1119                let cutoff = observed_at
1120                    - Duration::seconds(AgentDetectorConfig::default().window_secs as i64);
1121                while state
1122                    .supervisor_snapshots
1123                    .front()
1124                    .is_some_and(|snapshot| snapshot.captured_at < cutoff)
1125                {
1126                    state.supervisor_snapshots.pop_front();
1127                }
1128            }
1129            state
1130                .supervisor_snapshots
1131                .iter()
1132                .cloned()
1133                .collect::<Vec<_>>()
1134        };
1135        found.extend(agent_gave_up(
1136            &snapshots,
1137            AgentDetectorConfig::default(),
1138            car_version,
1139            &redactor,
1140        ));
1141
1142        // One latest value per stable key. This also collapses the same
1143        // machine-wide metrics signal observed in multiple session logs.
1144        let mut unique = BTreeMap::new();
1145        for detection in found {
1146            unique.insert(detection.dedup_key().to_string(), detection);
1147        }
1148        let detections_found = unique.len();
1149        let replay_verb_refusal = if route.route == SelfhealRoute::Local
1150            && unique.values().any(|detection| {
1151                detection.detector_id() == car_selfheal::detectors::tools::DETECTOR_ID
1152            }) {
1153            let checkout = route
1154                .source_checkout
1155                .as_ref()
1156                .expect("local route always carries a checkout");
1157            self.replay_verb_probe
1158                .check(
1159                    checkout,
1160                    &self.state_root.join("selfheal/replay-probe-target"),
1161                )
1162                .await
1163                .err()
1164                .map(|reason| format!("checkout predates the replay verb: {reason}"))
1165        } else {
1166            None
1167        };
1168        let mut state = self.state.lock().await;
1169        let (previous, dismissed) =
1170            fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
1171        let mut appended = 0;
1172        let mut changed = 0;
1173        let mut suppressed_dismissed = 0;
1174        let mut records = Vec::new();
1175        for (key, detection) in unique {
1176            if dismissed.contains(&key) {
1177                suppressed_dismissed += 1;
1178                continue;
1179            }
1180            let assessment = assess_reconstructed_call(&detection, &evidence.events, &redactor);
1181            let mut eligible = assessment.as_ref().map(|assessment| assessment.eligible);
1182            let reconstructed_call =
1183                assessment.and_then(|assessment| assessment.reconstructed_call);
1184            let ineligible_reason = if eligible == Some(true) {
1185                replay_verb_refusal.as_deref()
1186            } else {
1187                None
1188            };
1189            if ineligible_reason.is_some() {
1190                eligible = Some(false);
1191            }
1192            let local_issue_path = if route.route == SelfhealRoute::Local {
1193                let path = self
1194                    .state_root
1195                    .join("selfheal")
1196                    .join("issues")
1197                    .join(format!("{key}.md"));
1198                render_local_issue(&path, &detection, eligible, ineligible_reason)?;
1199                Some(path)
1200            } else {
1201                None
1202            };
1203            let reconstructed_call_path =
1204                if route.route == SelfhealRoute::Local && eligible.is_some() {
1205                    let path = self
1206                        .state_root
1207                        .join("selfheal")
1208                        .join("issues")
1209                        .join(format!("{key}.call.json"));
1210                    persist_reconstructed_call(
1211                        &path,
1212                        eligible.unwrap_or(false),
1213                        reconstructed_call.as_ref(),
1214                    )?;
1215                    Some(path)
1216                } else {
1217                    None
1218                };
1219            let mut routed = RoutedDetection {
1220                detection,
1221                route: route.route,
1222                local_issue_path,
1223                eligible,
1224                reconstructed_call,
1225                reconstructed_call_path,
1226                auto_fix_attempts: 0,
1227                auto_fix_exhausted: false,
1228                auto_fix_in_progress: false,
1229                last_auto_fix_attempt: None,
1230                auto_fix_awaiting_review: false,
1231                auto_fix_parked: false,
1232                remote_pr_number: None,
1233                remote_pr_url: None,
1234            };
1235            apply_attempt_state(
1236                &mut routed,
1237                &state.records,
1238                self.auto_fix.max_rounds_per_key,
1239            );
1240            apply_remote_pr_state(&mut routed, &state.records);
1241            let should_append = match previous.get(&key) {
1242                None => {
1243                    appended += 1;
1244                    true
1245                }
1246                Some(old) if old != &routed => {
1247                    appended += 1;
1248                    changed += 1;
1249                    true
1250                }
1251                Some(_) => false,
1252            };
1253            if should_append {
1254                records.push(LedgerRecord::Detection {
1255                    recorded_at: observed_at,
1256                    detection: routed.detection,
1257                    route: routed.route,
1258                    local_issue_path: routed.local_issue_path,
1259                    eligible: routed.eligible,
1260                    reconstructed_call: routed.reconstructed_call,
1261                    reconstructed_call_path: routed.reconstructed_call_path,
1262                });
1263            }
1264        }
1265        let summary = TickSummary {
1266            started_at,
1267            completed_at: Utc::now(),
1268            route: route.route,
1269            source_checkout: route.source_checkout,
1270            refusal_reason: route.refusal_reason,
1271            events_scanned: evidence.events.len(),
1272            supervisor_agents,
1273            registry_agents: evidence.registry_agents,
1274            detections_found,
1275            appended,
1276            changed,
1277            suppressed_dismissed,
1278            filing_mode: if self.auto_fix.auto_fix
1279                && self.auto_fix_refusal_reason.is_none()
1280                && route.route == SelfhealRoute::Local
1281            {
1282                "pr-only"
1283            } else {
1284                "watch-only"
1285            }
1286            .to_string(),
1287        };
1288        records.push(LedgerRecord::Tick {
1289            summary: summary.clone(),
1290        });
1291        append_records(&self.ledger_path, &records)?;
1292        state.records.extend(records);
1293        Ok(summary)
1294    }
1295}
1296
1297/// Spawn the default-on detection and eligible auto-fix cadence. The first tick runs immediately so
1298/// an already-Errored supervised agent is visible after daemon boot. A slow
1299/// tick is never overlapped: `run_tick` refuses while its operation guard is
1300/// held. The task dies with the daemon runtime.
1301pub fn spawn_selfheal_cadence(
1302    state: Arc<ServerState>,
1303    interval_secs: u64,
1304) -> tokio::task::JoinHandle<()> {
1305    tokio::spawn(async move {
1306        let mut ticker =
1307            tokio::time::interval(std::time::Duration::from_secs(interval_secs.max(1)));
1308        ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1309        loop {
1310            ticker.tick().await;
1311            if let Err(error) = state.selfheal.run_tick(&state).await {
1312                tracing::warn!(target: "car::selfheal", %error, "self-heal detection tick failed");
1313            }
1314        }
1315    })
1316}
1317
1318async fn log_file_observation(path: &str) -> (Option<DateTime<Utc>>, Option<u64>) {
1319    let Ok(metadata) = tokio::fs::metadata(path).await else {
1320        return (None, None);
1321    };
1322    let modified_at = metadata.modified().ok().map(DateTime::<Utc>::from);
1323    (modified_at, Some(metadata.len()))
1324}
1325
1326async fn gather_live_evidence(
1327    server: &Arc<ServerState>,
1328    state_root: &Path,
1329    since: Option<DateTime<Utc>>,
1330    until: DateTime<Utc>,
1331) -> SelfhealEvidence {
1332    let sessions = server
1333        .sessions
1334        .lock()
1335        .await
1336        .values()
1337        .cloned()
1338        .collect::<Vec<_>>();
1339    let mut events = Vec::new();
1340    let mut metric_alerts_found = Vec::new();
1341    let mut metric_provenance = Vec::new();
1342    let thresholds = AlertThresholds {
1343        max_cost_usd: None,
1344        max_error_rate: Some(0.5),
1345        max_avg_latency_ms: None,
1346        max_goals_ungrounded: Some(0),
1347        min_actions: Some(5),
1348    };
1349    for session in sessions {
1350        let handle = session.runtime.event_log_handle();
1351        let log = handle.lock().await;
1352        let slice = log
1353            .events()
1354            .iter()
1355            .filter(|event| {
1356                since.is_none_or(|cutoff| event.timestamp > cutoff) && event.timestamp <= until
1357            })
1358            .cloned()
1359            .collect::<Vec<_>>();
1360        // A tool may start before the prior tick's watermark and fail after
1361        // it. Include only older ProposalReceived rows as reconstruction
1362        // context; detectors ignore this kind, so prior failures are never
1363        // made active again.
1364        let proposal_context = since.map_or_else(Vec::new, |cutoff| {
1365            log.events()
1366                .iter()
1367                .filter(|event| {
1368                    event.kind == EventKind::ProposalReceived
1369                        && event.timestamp <= cutoff
1370                        && event.timestamp <= until
1371                })
1372                .cloned()
1373                .collect::<Vec<_>>()
1374        });
1375        let summary = car_eventlog::summarize(&slice);
1376        metric_alerts_found.extend(car_eventlog::evaluate_alerts(&summary, &thresholds));
1377        let path = Some(format!("journals/{}.jsonl", session.client_id));
1378        metric_provenance.extend(slice.iter().map(|event| EvidenceSource {
1379            event_id: None,
1380            run_id: event.run_id.clone(),
1381            path: path.clone(),
1382        }));
1383        events.extend(slice.into_iter().map(|event| EventEvidence {
1384            event,
1385            path: path.clone(),
1386        }));
1387        events.extend(proposal_context.into_iter().map(|event| EventEvidence {
1388            event,
1389            path: path.clone(),
1390        }));
1391    }
1392
1393    let supervisor_snapshot = if let Some(supervisor) = server.supervisor_if_installed() {
1394        let managed = supervisor.list().await;
1395        let mut agents = Vec::with_capacity(managed.len());
1396        for agent in &managed {
1397            let activity = supervisor
1398                .read_log(
1399                    &agent.spec.id,
1400                    car_registry::supervisor::LogStream::Stdout,
1401                    ACTIVITY_TAIL_LINES,
1402                    0,
1403                )
1404                .await
1405                .ok();
1406            let stderr = supervisor
1407                .read_log(
1408                    &agent.spec.id,
1409                    car_registry::supervisor::LogStream::Stderr,
1410                    STDERR_TAIL_LINES,
1411                    0,
1412                )
1413                .await
1414                .ok();
1415            let (activity_modified_at, activity_bytes) = match activity.as_ref() {
1416                Some(tail) => log_file_observation(&tail.stdout_path).await,
1417                None => (None, None),
1418            };
1419            let stderr_bytes = match stderr.as_ref() {
1420                Some(tail) => log_file_observation(&tail.stderr_path).await.1,
1421                None => None,
1422            };
1423            let mut state = SupervisorAgentState::from_managed(
1424                agent,
1425                stderr
1426                    .as_ref()
1427                    .map(|tail| tail.stderr.join("\n"))
1428                    .unwrap_or_default(),
1429                Some(format!("logs/{}.stderr.log", agent.spec.id)),
1430            );
1431            state.activity_tail = activity
1432                .as_ref()
1433                .map(|tail| tail.stdout.join("\n"))
1434                .unwrap_or_default();
1435            state.activity_path = Some(format!("logs/{}.stdout.log", agent.spec.id));
1436            state.activity_modified_at = activity_modified_at;
1437            state.activity_bytes = activity_bytes;
1438            state.stderr_bytes = stderr_bytes;
1439            agents.push(state);
1440        }
1441        Some(SupervisorSnapshot {
1442            captured_at: Utc::now(),
1443            agents,
1444        })
1445    } else if let Some(manifest) = server.observer_manifest_path() {
1446        car_registry::supervisor::Supervisor::list_from_manifest(manifest)
1447            .ok()
1448            .map(|managed| SupervisorSnapshot {
1449                captured_at: Utc::now(),
1450                agents: managed
1451                    .iter()
1452                    .map(|agent| SupervisorAgentState::from_managed(agent, "", None))
1453                    .collect(),
1454            })
1455    } else {
1456        None
1457    };
1458
1459    SelfhealEvidence {
1460        events,
1461        metric_alerts: metric_alerts_found,
1462        metric_provenance,
1463        supervisor_snapshot,
1464        registry_agents: count_registry_agents(&state_root.join("registry")),
1465    }
1466}
1467
1468#[derive(Deserialize)]
1469struct SelfhealConfigFile {
1470    #[serde(default)]
1471    selfheal: Option<SelfhealConfigSection>,
1472}
1473
1474#[derive(Deserialize)]
1475struct SelfhealConfigSection {
1476    #[serde(default)]
1477    source_checkout: Option<PathBuf>,
1478    #[serde(default = "default_auto_fix")]
1479    auto_fix: bool,
1480    #[serde(default = "default_max_concurrent")]
1481    max_concurrent: u32,
1482    #[serde(default = "default_max_per_day")]
1483    max_per_day: u32,
1484    #[serde(default = "default_max_rounds_per_key")]
1485    max_rounds_per_key: u32,
1486}
1487
1488impl Default for SelfhealConfigSection {
1489    fn default() -> Self {
1490        Self {
1491            source_checkout: None,
1492            auto_fix: DEFAULT_AUTO_FIX,
1493            max_concurrent: DEFAULT_MAX_CONCURRENT,
1494            max_per_day: DEFAULT_MAX_PER_DAY,
1495            max_rounds_per_key: DEFAULT_MAX_ROUNDS_PER_KEY,
1496        }
1497    }
1498}
1499
1500#[derive(Debug, Clone)]
1501struct AutoFixConfig {
1502    auto_fix: bool,
1503    max_concurrent: u32,
1504    max_per_day: u32,
1505    max_rounds_per_key: u32,
1506}
1507
1508impl Default for AutoFixConfig {
1509    fn default() -> Self {
1510        Self {
1511            auto_fix: DEFAULT_AUTO_FIX,
1512            max_concurrent: DEFAULT_MAX_CONCURRENT,
1513            max_per_day: DEFAULT_MAX_PER_DAY,
1514            max_rounds_per_key: DEFAULT_MAX_ROUNDS_PER_KEY,
1515        }
1516    }
1517}
1518
1519impl AutoFixConfig {
1520    fn disabled() -> Self {
1521        Self {
1522            auto_fix: false,
1523            ..Self::default()
1524        }
1525    }
1526}
1527
1528const fn default_auto_fix() -> bool {
1529    DEFAULT_AUTO_FIX
1530}
1531
1532const fn default_max_concurrent() -> u32 {
1533    DEFAULT_MAX_CONCURRENT
1534}
1535
1536const fn default_max_per_day() -> u32 {
1537    DEFAULT_MAX_PER_DAY
1538}
1539
1540const fn default_max_rounds_per_key() -> u32 {
1541    DEFAULT_MAX_ROUNDS_PER_KEY
1542}
1543
1544fn read_auto_fix_config(path: &Path) -> Result<AutoFixConfig, String> {
1545    let text = match std::fs::read_to_string(path) {
1546        Ok(text) => text,
1547        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
1548            return Ok(AutoFixConfig::default())
1549        }
1550        Err(error) => {
1551            return Err(format!(
1552                "could not read self-heal config {}: {error}",
1553                path.display()
1554            ))
1555        }
1556    };
1557    let config: SelfhealConfigFile = toml::from_str(&text).map_err(|error| {
1558        format!(
1559            "could not parse self-heal config {}: {error}",
1560            path.display()
1561        )
1562    })?;
1563    let section = config.selfheal.unwrap_or_default();
1564    if section.max_concurrent != DEFAULT_MAX_CONCURRENT {
1565        return Err(format!(
1566            "[selfheal] max_concurrent must be 1, got {}",
1567            section.max_concurrent
1568        ));
1569    }
1570    if !(1..=DEFAULT_MAX_PER_DAY).contains(&section.max_per_day) {
1571        return Err(format!(
1572            "[selfheal] max_per_day must be between 1 and {DEFAULT_MAX_PER_DAY}, got {}",
1573            section.max_per_day
1574        ));
1575    }
1576    if !(1..=DEFAULT_MAX_ROUNDS_PER_KEY).contains(&section.max_rounds_per_key) {
1577        return Err(format!(
1578            "[selfheal] max_rounds_per_key must be between 1 and {DEFAULT_MAX_ROUNDS_PER_KEY}, got {}",
1579            section.max_rounds_per_key
1580        ));
1581    }
1582    Ok(AutoFixConfig {
1583        auto_fix: section.auto_fix,
1584        max_concurrent: section.max_concurrent,
1585        max_per_day: section.max_per_day,
1586        max_rounds_per_key: section.max_rounds_per_key,
1587    })
1588}
1589
1590fn read_source_checkout_config(path: &Path) -> Result<Option<PathBuf>, String> {
1591    let text = match std::fs::read_to_string(path) {
1592        Ok(text) => text,
1593        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1594        Err(error) => {
1595            return Err(format!(
1596                "could not read self-heal config {}: {error}",
1597                path.display()
1598            ))
1599        }
1600    };
1601    let config: SelfhealConfigFile = toml::from_str(&text).map_err(|error| {
1602        format!(
1603            "could not parse self-heal config {}: {error}",
1604            path.display()
1605        )
1606    })?;
1607    Ok(config.selfheal.and_then(|section| section.source_checkout))
1608}
1609
1610fn dedup_paths(paths: &mut Vec<PathBuf>) {
1611    let mut seen = BTreeSet::new();
1612    paths.retain(|path| seen.insert(path.clone()));
1613}
1614
1615fn validate_source_checkout(candidate: &Path) -> Result<PathBuf, String> {
1616    if !candidate.is_dir() {
1617        return Err("directory does not exist".to_string());
1618    }
1619    let checkout = std::fs::canonicalize(candidate)
1620        .map_err(|error| format!("canonicalize candidate: {error}"))?;
1621    let dot_git = checkout.join(".git");
1622    if !dot_git.is_dir() && !dot_git.is_file() {
1623        return Err("missing .git directory or worktree file".to_string());
1624    }
1625    let config_path = git_config_path(&checkout, &dot_git)?;
1626    let config = std::fs::read_to_string(&config_path)
1627        .map_err(|error| format!("read git config {}: {error}", config_path.display()))?;
1628    let remote = origin_remote(&config)
1629        .ok_or_else(|| format!("git config {} has no origin remote", config_path.display()))?;
1630    if !is_expected_origin(&remote) {
1631        return Err(format!(
1632            "origin remote is '{}', expected {EXPECTED_ORIGIN}",
1633            remote_for_display(&remote)
1634        ));
1635    }
1636    Ok(checkout)
1637}
1638
1639fn git_config_path(checkout: &Path, dot_git: &Path) -> Result<PathBuf, String> {
1640    if dot_git.is_dir() {
1641        return Ok(dot_git.join("config"));
1642    }
1643    let marker = std::fs::read_to_string(dot_git)
1644        .map_err(|error| format!("read worktree marker {}: {error}", dot_git.display()))?;
1645    let raw_git_dir = marker
1646        .lines()
1647        .find_map(|line| line.trim().strip_prefix("gitdir:"))
1648        .map(str::trim)
1649        .filter(|path| !path.is_empty())
1650        .ok_or_else(|| format!("invalid worktree marker {}", dot_git.display()))?;
1651    let git_dir = resolve_relative(checkout, Path::new(raw_git_dir));
1652    let common_dir_path = git_dir.join("commondir");
1653    if let Ok(raw_common_dir) = std::fs::read_to_string(&common_dir_path) {
1654        let common_dir = resolve_relative(&git_dir, Path::new(raw_common_dir.trim()));
1655        Ok(common_dir.join("config"))
1656    } else {
1657        Ok(git_dir.join("config"))
1658    }
1659}
1660
1661fn resolve_relative(base: &Path, path: &Path) -> PathBuf {
1662    if path.is_absolute() {
1663        path.to_path_buf()
1664    } else {
1665        base.join(path)
1666    }
1667}
1668
1669fn origin_remote(config: &str) -> Option<String> {
1670    let mut in_origin = false;
1671    for raw_line in config.lines() {
1672        let line = raw_line.trim();
1673        if line.starts_with('[') && line.ends_with(']') {
1674            in_origin = line.eq_ignore_ascii_case(r#"[remote "origin"]"#);
1675            continue;
1676        }
1677        if !in_origin || line.starts_with('#') || line.starts_with(';') {
1678            continue;
1679        }
1680        let Some((key, value)) = line.split_once('=') else {
1681            continue;
1682        };
1683        if key.trim().eq_ignore_ascii_case("url") {
1684            return Some(value.trim().trim_matches('"').to_string());
1685        }
1686    }
1687    None
1688}
1689
1690fn remote_for_display(remote: &str) -> String {
1691    let Some((scheme, rest)) = remote.split_once("://") else {
1692        return remote.to_string();
1693    };
1694    let Some((authority, tail)) = rest.split_once('/') else {
1695        return remote.to_string();
1696    };
1697    if authority.contains('@') {
1698        format!(
1699            "{scheme}://[REDACTED]@{}/{tail}",
1700            authority.rsplit('@').next().unwrap_or(authority)
1701        )
1702    } else {
1703        remote.to_string()
1704    }
1705}
1706
1707fn is_expected_origin(remote: &str) -> bool {
1708    let remote = remote.trim().trim_end_matches('/').trim_end_matches(".git");
1709    let (host, path) = if remote.contains("://") {
1710        let without_scheme = remote.split_once("://").map_or(remote, |(_, rest)| rest);
1711        let Some((host, path)) = without_scheme.split_once('/') else {
1712            return false;
1713        };
1714        (host.rsplit('@').next().unwrap_or(host), path)
1715    } else if let Some((left, path)) = remote.split_once(':') {
1716        (left.rsplit('@').next().unwrap_or(left), path)
1717    } else {
1718        let Some((host, path)) = remote.split_once('/') else {
1719            return false;
1720        };
1721        (host.rsplit('@').next().unwrap_or(host), path)
1722    };
1723    host.eq_ignore_ascii_case("github.com") && path.eq_ignore_ascii_case(EXPECTED_ORIGIN)
1724}
1725
1726#[derive(Debug)]
1727struct CallAssessment {
1728    eligible: bool,
1729    reconstructed_call: Option<ReconstructedToolCall>,
1730}
1731
1732fn assess_reconstructed_call(
1733    detection: &Detection,
1734    events: &[EventEvidence],
1735    redactor: &Redactor,
1736) -> Option<CallAssessment> {
1737    if detection.kind() != DetectionKind::RecurringToolFailure {
1738        return None;
1739    }
1740    let Some((builtin, reconstructed_call)) = reconstruct_failed_call(detection, events) else {
1741        return Some(CallAssessment {
1742            eligible: false,
1743            reconstructed_call: None,
1744        });
1745    };
1746    if !params_pass_redactor(&reconstructed_call.params, redactor) {
1747        return Some(CallAssessment {
1748            eligible: false,
1749            reconstructed_call: None,
1750        });
1751    }
1752    Some(CallAssessment {
1753        eligible: auto_fix_tool_allowed(&reconstructed_call.tool, builtin),
1754        reconstructed_call: Some(reconstructed_call),
1755    })
1756}
1757
1758/// Auto-fix may replay data-oriented commodity builtins, never the nested GUI
1759/// agent that can click and type in an ambient desktop session. Provenance
1760/// alone identifies who owns a tool; it does not make every builtin safe to
1761/// replay unattended.
1762fn auto_fix_tool_allowed(tool: &str, builtin: bool) -> bool {
1763    builtin && tool != "delegate_gui"
1764}
1765
1766fn params_pass_redactor(params: &BTreeMap<String, serde_json::Value>, redactor: &Redactor) -> bool {
1767    let Ok(params_json) = serde_json::to_string(params) else {
1768        return false;
1769    };
1770    redactor.redact(&params_json) == params_json
1771        && params.iter().all(|(key, value)| {
1772            redactor.redact(key) == *key && json_value_passes_redactor(value, redactor)
1773        })
1774}
1775
1776fn json_value_passes_redactor(value: &serde_json::Value, redactor: &Redactor) -> bool {
1777    match value {
1778        serde_json::Value::String(value) => redactor.redact(value) == *value,
1779        serde_json::Value::Array(values) => values
1780            .iter()
1781            .all(|value| json_value_passes_redactor(value, redactor)),
1782        serde_json::Value::Object(values) => values.iter().all(|(key, value)| {
1783            redactor.redact(key) == *key && json_value_passes_redactor(value, redactor)
1784        }),
1785        serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => true,
1786    }
1787}
1788
1789fn reconstruct_failed_call(
1790    detection: &Detection,
1791    events: &[EventEvidence],
1792) -> Option<(bool, ReconstructedToolCall)> {
1793    let failure = events
1794        .iter()
1795        .filter(|record| record.event.kind == EventKind::ActionFailed)
1796        .filter(|record| failure_belongs_to_detection(detection, record))
1797        .max_by(|left, right| {
1798            left.event
1799                .timestamp
1800                .cmp(&right.event.timestamp)
1801                .then_with(|| left.event.proposal_id.cmp(&right.event.proposal_id))
1802                .then_with(|| left.event.action_id.cmp(&right.event.action_id))
1803        })?;
1804    let journal = failure.path.as_deref()?;
1805    let proposal_id = failure.event.proposal_id.as_deref()?;
1806    let action_id = failure.event.action_id.as_deref()?;
1807    let failed_tool = failure.event.data.get("tool")?.as_str()?;
1808    let builtin = failure
1809        .event
1810        .data
1811        .get("tool_source")
1812        .and_then(serde_json::Value::as_str)
1813        == Some("builtin");
1814
1815    let proposal_event = events
1816        .iter()
1817        .filter(|record| {
1818            record.event.kind == EventKind::ProposalReceived
1819                && record.path.as_deref() == Some(journal)
1820                && record.event.proposal_id.as_deref() == Some(proposal_id)
1821                && record.event.timestamp <= failure.event.timestamp
1822        })
1823        .max_by_key(|record| record.event.timestamp)?;
1824    let proposal: ActionProposal =
1825        serde_json::from_value(proposal_event.event.data.get("proposal")?.clone()).ok()?;
1826    if proposal.id != proposal_id {
1827        return None;
1828    }
1829    let action = proposal
1830        .actions
1831        .into_iter()
1832        .find(|action| action.id == action_id)?;
1833    let tool = action.tool?;
1834    if tool != failed_tool {
1835        return None;
1836    }
1837    Some((
1838        builtin,
1839        ReconstructedToolCall {
1840            tool,
1841            params: action.parameters.into_iter().collect(),
1842        },
1843    ))
1844}
1845
1846fn failure_belongs_to_detection(detection: &Detection, failure: &EventEvidence) -> bool {
1847    let Some(journal) = failure.path.as_deref() else {
1848        return false;
1849    };
1850    detection.provenance().iter().any(|provenance| {
1851        provenance.path() == Some(journal)
1852            && provenance.run_id() == failure.event.run_id.as_deref()
1853            && provenance.event_id().is_some_and(|event_id| {
1854                failure.event.hash.as_deref() == Some(event_id)
1855                    || failure.event.action_id.as_deref() == Some(event_id)
1856            })
1857    })
1858}
1859
1860#[derive(Serialize)]
1861struct PersistedReconstructedCall<'a> {
1862    eligible: bool,
1863    reconstructed_call: Option<&'a ReconstructedToolCall>,
1864}
1865
1866fn persist_reconstructed_call(
1867    path: &Path,
1868    eligible: bool,
1869    reconstructed_call: Option<&ReconstructedToolCall>,
1870) -> Result<(), String> {
1871    let parent = path
1872        .parent()
1873        .ok_or_else(|| "reconstructed self-heal call path has no parent".to_string())?;
1874    car_secrets::ensure_private_dir(parent)
1875        .map_err(|error| format!("create reconstructed self-heal call directory: {error}"))?;
1876    let mut bytes = serde_json::to_vec_pretty(&PersistedReconstructedCall {
1877        eligible,
1878        reconstructed_call,
1879    })
1880    .map_err(|error| format!("serialize reconstructed self-heal call: {error}"))?;
1881    bytes.push(b'\n');
1882    let mut file = if path.exists() {
1883        car_secrets::open_private_truncate(path)
1884    } else {
1885        car_secrets::create_private_file(path)
1886    }
1887    .map_err(|error| {
1888        format!(
1889            "open reconstructed self-heal call {}: {error}",
1890            path.display()
1891        )
1892    })?;
1893    file.write_all(&bytes)
1894        .map_err(|error| format!("write reconstructed self-heal call: {error}"))?;
1895    file.flush()
1896        .map_err(|error| format!("flush reconstructed self-heal call: {error}"))?;
1897    file.sync_all()
1898        .map_err(|error| format!("sync reconstructed self-heal call: {error}"))?;
1899    car_secrets::revalidate_private_path(path, &file)
1900        .map_err(|error| format!("revalidate reconstructed self-heal call: {error}"))
1901}
1902
1903fn persist_private_json<T: Serialize>(path: &Path, value: &T) -> Result<(), String> {
1904    let mut bytes = serde_json::to_vec_pretty(value)
1905        .map_err(|error| format!("serialize private self-heal artifact: {error}"))?;
1906    bytes.push(b'\n');
1907    persist_private_bytes(path, &bytes)
1908}
1909
1910fn persist_private_bytes(path: &Path, bytes: &[u8]) -> Result<(), String> {
1911    let parent = path
1912        .parent()
1913        .ok_or_else(|| "private self-heal artifact path has no parent".to_string())?;
1914    car_secrets::ensure_private_dir(parent)
1915        .map_err(|error| format!("create private self-heal artifact directory: {error}"))?;
1916    let mut file = if path.exists() {
1917        car_secrets::open_private_truncate(path)
1918    } else {
1919        car_secrets::create_private_file(path)
1920    }
1921    .map_err(|error| {
1922        format!(
1923            "open private self-heal artifact {}: {error}",
1924            path.display()
1925        )
1926    })?;
1927    file.write_all(bytes)
1928        .map_err(|error| format!("write private self-heal artifact: {error}"))?;
1929    file.flush()
1930        .map_err(|error| format!("flush private self-heal artifact: {error}"))?;
1931    file.sync_all()
1932        .map_err(|error| format!("sync private self-heal artifact: {error}"))?;
1933    car_secrets::revalidate_private_path(path, &file)
1934        .map_err(|error| format!("revalidate private self-heal artifact: {error}"))
1935}
1936
1937fn resolve_car_cli_executable() -> Option<PathBuf> {
1938    let executable_name = if cfg!(windows) { "car.exe" } else { "car" };
1939    if let Ok(current) = std::env::current_exe() {
1940        if let Some(sibling) = current.parent().map(|parent| parent.join(executable_name)) {
1941            if sibling.is_file() {
1942                return Some(sibling);
1943            }
1944        }
1945    }
1946    std::env::var_os("PATH").and_then(|path| {
1947        std::env::split_paths(&path)
1948            .map(|entry| entry.join(executable_name))
1949            .find(|candidate| candidate.is_file())
1950    })
1951}
1952
1953#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1954enum RemoteDedupResult {
1955    Proceed,
1956    AwaitingReview,
1957    Parked,
1958}
1959
1960#[derive(Debug, Deserialize)]
1961struct RemoteSelfhealPr {
1962    number: u64,
1963    url: String,
1964    state: String,
1965    #[serde(rename = "mergedAt")]
1966    merged_at: Option<String>,
1967    body: String,
1968}
1969
1970async fn check_checkout_replay_verb(checkout: &Path, target_dir: &Path) -> Result<(), String> {
1971    car_secrets::ensure_private_dir(target_dir)
1972        .map_err(|error| format!("create replay probe target: {error}"))?;
1973    let build = tokio::process::Command::new("cargo")
1974        .current_dir(checkout.join("car-rs"))
1975        .args(["build", "-p", "car-cli"])
1976        .env("CARGO_TARGET_DIR", target_dir)
1977        .kill_on_drop(true)
1978        .output()
1979        .await
1980        .map_err(|error| format!("could not build checkout car CLI: {error}"))?;
1981    if !build.status.success() {
1982        return Err(format!(
1983            "checkout car CLI build failed with exit {}: {}",
1984            build.status.code().unwrap_or(-1),
1985            output_tail(&build.stderr, 8)
1986        ));
1987    }
1988
1989    #[cfg(windows)]
1990    let car = target_dir.join("debug/car.exe");
1991    #[cfg(not(windows))]
1992    let car = target_dir.join("debug/car");
1993    let help = tokio::process::Command::new(&car)
1994        .args(["tools", "call", "--help"])
1995        .kill_on_drop(true)
1996        .output()
1997        .await
1998        .map_err(|error| format!("checkout-built car has no runnable replay verb: {error}"))?;
1999    if help.status.success() {
2000        Ok(())
2001    } else {
2002        Err(format!(
2003            "checkout-built `car tools call --help` exited {}: {}",
2004            help.status.code().unwrap_or(-1),
2005            output_tail(&help.stderr, 8)
2006        ))
2007    }
2008}
2009
2010async fn query_remote_selfheal_pr(
2011    checkout: &Path,
2012    dedup_key: &str,
2013) -> Result<Option<RemoteSelfhealPr>, String> {
2014    let head = format!("car/selfheal/{dedup_key}");
2015    let output = tokio::process::Command::new("gh")
2016        .current_dir(checkout)
2017        .args([
2018            "pr",
2019            "list",
2020            "--repo",
2021            "Parslee-ai/car",
2022            "--head",
2023            &head,
2024            "--state",
2025            "all",
2026            "--json",
2027            "number,url,state,mergedAt,body",
2028        ])
2029        .kill_on_drop(true)
2030        .output()
2031        .await
2032        .map_err(|error| format!("remote self-heal PR lookup could not start: {error}"))?;
2033    if !output.status.success() {
2034        return Err(format!(
2035            "remote self-heal PR lookup failed with exit {}: {}",
2036            output.status.code().unwrap_or(-1),
2037            output_tail(&output.stderr, 8)
2038        ));
2039    }
2040    let mut prs: Vec<RemoteSelfhealPr> = serde_json::from_slice(&output.stdout)
2041        .map_err(|error| format!("parse remote self-heal PR lookup: {error}"))?;
2042    let marker = format!("<!-- car-selfheal:key={dedup_key} -->");
2043    if prs.iter().any(|pr| !pr.body.contains(&marker)) {
2044        return Err(format!(
2045            "remote branch '{head}' has a PR without the trusted self-heal marker"
2046        ));
2047    }
2048    prs.sort_by_key(|pr| if pr.state == "OPEN" { 0 } else { 1 });
2049    Ok(prs.into_iter().next())
2050}
2051
2052fn output_tail(bytes: &[u8], lines: usize) -> String {
2053    let text = String::from_utf8_lossy(bytes);
2054    text.lines()
2055        .rev()
2056        .take(lines)
2057        .collect::<Vec<_>>()
2058        .into_iter()
2059        .rev()
2060        .collect::<Vec<_>>()
2061        .join("\n")
2062}
2063
2064fn classify_code_task_output(stdout: &[u8]) -> String {
2065    code_task_failure_class(stdout).unwrap_or_else(|| "missing_run_end".to_string())
2066}
2067
2068fn code_task_failure_class(stdout: &[u8]) -> Option<String> {
2069    String::from_utf8_lossy(stdout)
2070        .lines()
2071        .rev()
2072        .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
2073        .find(|event| event.get("type").and_then(serde_json::Value::as_str) == Some("run_end"))
2074        .and_then(|event| {
2075            event
2076                .get("failure_class")
2077                .and_then(serde_json::Value::as_str)
2078                .map(str::to_string)
2079        })
2080}
2081
2082fn route_name(route: SelfhealRoute) -> &'static str {
2083    match route {
2084        SelfhealRoute::Local => "local",
2085        SelfhealRoute::Feedback => "feedback",
2086        SelfhealRoute::LedgerOnly => "ledger-only",
2087    }
2088}
2089
2090fn render_local_issue(
2091    path: &Path,
2092    detection: &Detection,
2093    eligible: Option<bool>,
2094    ineligible_reason: Option<&str>,
2095) -> Result<(), String> {
2096    let occurrence_count = local_issue_occurrence_count(path)?.saturating_add(1);
2097    let parent = path
2098        .parent()
2099        .ok_or_else(|| "local self-heal issue path has no parent".to_string())?;
2100    car_secrets::ensure_private_dir(parent)
2101        .map_err(|error| format!("create local self-heal issue directory: {error}"))?;
2102
2103    let event_ids = detection
2104        .provenance()
2105        .iter()
2106        .filter_map(|source| source.event_id())
2107        .collect::<Vec<_>>();
2108    let run_ids = detection
2109        .provenance()
2110        .iter()
2111        .filter_map(|source| source.run_id())
2112        .collect::<Vec<_>>();
2113    let paths = detection
2114        .provenance()
2115        .iter()
2116        .filter_map(|source| source.path())
2117        .collect::<Vec<_>>();
2118    let evidence = if detection.evidence().is_empty() {
2119        "- (none)".to_string()
2120    } else {
2121        detection
2122            .evidence()
2123            .iter()
2124            .map(|excerpt| format!("- {excerpt}"))
2125            .collect::<Vec<_>>()
2126            .join("\n")
2127    };
2128    let issue = format!(
2129        "# CAR self-heal detection: {}\n\n\
2130Trust-Tier: {TRUST_TIER}\n\
2131Dedup-Key: {}\n\
2132Detector-ID: {}\n\
2133Severity: {:?}\n\
2134Route: local\n\
2135Auto-Fix-Eligible: {}\n\
2136Auto-Fix-Ineligible-Reason: {}\n\
2137Occurrence-Count: {occurrence_count}\n\
2138First-Observed: {}\n\
2139Last-Observed: {}\n\n\
2140## Provenance\n\n\
2141- CAR-Version: {}\n\
2142- Platform: {}/{}\n\
2143- Event-IDs: {}\n\
2144- Run-IDs: {}\n\
2145- Evidence-Paths: {}\n\n\
2146## REDACTED Evidence Excerpt\n\n\
2147{evidence}\n\n\
2148## Repro Hints\n\n\
2149- Re-run `selfheal.run` and correlate the detector identity and provenance above.\n\
2150- Inspect the named local CAR evidence source; do not send it off-machine.\n",
2151        detection.locator(),
2152        detection.dedup_key(),
2153        detection.detector_id(),
2154        detection.severity(),
2155        eligible.map_or("not-applicable", |value| if value {
2156            "true"
2157        } else {
2158            "false"
2159        }),
2160        ineligible_reason.unwrap_or("(none)"),
2161        detection.first_observed_at().to_rfc3339(),
2162        detection.last_observed_at().to_rfc3339(),
2163        detection.car_version(),
2164        std::env::consts::OS,
2165        std::env::consts::ARCH,
2166        display_list(&event_ids),
2167        display_list(&run_ids),
2168        display_list(&paths),
2169    );
2170
2171    let mut file = if path.exists() {
2172        car_secrets::open_private_truncate(path)
2173    } else {
2174        car_secrets::create_private_file(path)
2175    }
2176    .map_err(|error| format!("open local self-heal issue {}: {error}", path.display()))?;
2177    file.write_all(issue.as_bytes())
2178        .map_err(|error| format!("write local self-heal issue: {error}"))?;
2179    file.flush()
2180        .map_err(|error| format!("flush local self-heal issue: {error}"))?;
2181    file.sync_all()
2182        .map_err(|error| format!("sync local self-heal issue: {error}"))?;
2183    car_secrets::revalidate_private_path(path, &file)
2184        .map_err(|error| format!("revalidate local self-heal issue: {error}"))
2185}
2186
2187fn local_issue_occurrence_count(path: &Path) -> Result<u64, String> {
2188    let mut file = match car_secrets::open_private_read(path) {
2189        Ok(file) => file,
2190        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
2191        Err(error) => {
2192            return Err(format!(
2193                "open existing local self-heal issue {}: {error}",
2194                path.display()
2195            ))
2196        }
2197    };
2198    let mut text = String::new();
2199    file.read_to_string(&mut text)
2200        .map_err(|error| format!("read existing local self-heal issue: {error}"))?;
2201    text.lines()
2202        .find_map(|line| line.strip_prefix("Occurrence-Count: "))
2203        .ok_or_else(|| {
2204            format!(
2205                "existing local self-heal issue {} has no occurrence count",
2206                path.display()
2207            )
2208        })?
2209        .parse::<u64>()
2210        .map_err(|error| format!("parse local self-heal issue occurrence count: {error}"))
2211}
2212
2213fn display_list(values: &[&str]) -> String {
2214    if values.is_empty() {
2215        "(none)".to_string()
2216    } else {
2217        values.join(", ")
2218    }
2219}
2220
2221fn count_registry_agents(registry_dir: &Path) -> usize {
2222    let Ok(entries) = std::fs::read_dir(registry_dir) else {
2223        return 0;
2224    };
2225    entries
2226        .filter_map(Result::ok)
2227        .filter(|entry| entry.path().extension().and_then(|ext| ext.to_str()) == Some("json"))
2228        .filter(|entry| {
2229            std::fs::read(entry.path())
2230                .ok()
2231                .and_then(|bytes| serde_json::from_slice::<car_registry::AgentEntry>(&bytes).ok())
2232                .is_some()
2233        })
2234        .count()
2235}
2236
2237fn fold_detections(
2238    records: &[LedgerRecord],
2239    max_rounds_per_key: u32,
2240) -> (BTreeMap<String, RoutedDetection>, BTreeSet<String>) {
2241    let mut detections = BTreeMap::new();
2242    let mut dismissed = BTreeSet::new();
2243    for record in records {
2244        match record {
2245            LedgerRecord::Detection {
2246                detection,
2247                route,
2248                local_issue_path,
2249                eligible,
2250                reconstructed_call,
2251                reconstructed_call_path,
2252                ..
2253            } => {
2254                detections.insert(
2255                    detection.dedup_key().to_string(),
2256                    RoutedDetection {
2257                        detection: detection.clone(),
2258                        route: *route,
2259                        local_issue_path: local_issue_path.clone(),
2260                        eligible: *eligible,
2261                        reconstructed_call: reconstructed_call.clone(),
2262                        reconstructed_call_path: reconstructed_call_path.clone(),
2263                        auto_fix_attempts: 0,
2264                        auto_fix_exhausted: false,
2265                        auto_fix_in_progress: false,
2266                        last_auto_fix_attempt: None,
2267                        auto_fix_awaiting_review: false,
2268                        auto_fix_parked: false,
2269                        remote_pr_number: None,
2270                        remote_pr_url: None,
2271                    },
2272                );
2273            }
2274            LedgerRecord::Dismissal { dedup_key, .. } => {
2275                dismissed.insert(dedup_key.clone());
2276            }
2277            LedgerRecord::Tick { .. }
2278            | LedgerRecord::FixStarted { .. }
2279            | LedgerRecord::FixAttempt { .. }
2280            | LedgerRecord::RemotePr { .. } => {}
2281        }
2282    }
2283    for (key, detection) in &mut detections {
2284        apply_attempt_state(detection, records, max_rounds_per_key);
2285        apply_remote_pr_state(detection, records);
2286        if dismissed.contains(key) {
2287            detection.auto_fix_in_progress = false;
2288        }
2289    }
2290    for key in &dismissed {
2291        detections.remove(key);
2292    }
2293    (detections, dismissed)
2294}
2295
2296fn apply_attempt_state(
2297    detection: &mut RoutedDetection,
2298    records: &[LedgerRecord],
2299    max_rounds_per_key: u32,
2300) {
2301    let key = detection.dedup_key().to_string();
2302    let started = records
2303        .iter()
2304        .filter_map(|record| match record {
2305            LedgerRecord::FixStarted {
2306                dedup_key, round, ..
2307            } if dedup_key == &key => Some(*round),
2308            _ => None,
2309        })
2310        .collect::<BTreeSet<_>>();
2311    let finished = records
2312        .iter()
2313        .filter_map(|record| match record {
2314            LedgerRecord::FixAttempt { result } if result.dedup_key == key => Some(result.round),
2315            _ => None,
2316        })
2317        .collect::<BTreeSet<_>>();
2318    detection.auto_fix_attempts = u32::try_from(started.len()).unwrap_or(u32::MAX);
2319    detection.auto_fix_exhausted = detection.auto_fix_attempts >= max_rounds_per_key;
2320    detection.auto_fix_in_progress = started.iter().any(|round| !finished.contains(round));
2321    detection.last_auto_fix_attempt = records.iter().rev().find_map(|record| match record {
2322        LedgerRecord::FixAttempt { result } if result.dedup_key == key => Some(result.clone()),
2323        _ => None,
2324    });
2325}
2326
2327fn apply_remote_pr_state(detection: &mut RoutedDetection, records: &[LedgerRecord]) {
2328    let key = detection.dedup_key().to_string();
2329    if let Some((disposition, number, url)) = records.iter().rev().find_map(|record| match record {
2330        LedgerRecord::RemotePr {
2331            dedup_key,
2332            disposition,
2333            pr_number,
2334            pr_url,
2335            ..
2336        } if dedup_key == &key => Some((*disposition, *pr_number, pr_url.clone())),
2337        _ => None,
2338    }) {
2339        detection.auto_fix_awaiting_review = disposition == RemotePrDisposition::AwaitingReview;
2340        detection.auto_fix_parked = disposition == RemotePrDisposition::Parked;
2341        detection.remote_pr_number = Some(number);
2342        detection.remote_pr_url = Some(url);
2343    }
2344}
2345
2346fn starts_today(records: &[LedgerRecord], now: DateTime<Utc>) -> u32 {
2347    u32::try_from(
2348        records
2349            .iter()
2350            .filter(|record| {
2351                matches!(
2352                    record,
2353                    LedgerRecord::FixStarted { started_at, .. }
2354                        if started_at.date_naive() == now.date_naive()
2355                )
2356            })
2357            .count(),
2358    )
2359    .unwrap_or(u32::MAX)
2360}
2361
2362fn validate_dedup_key(key: &str) -> Result<(), String> {
2363    if key.len() == 64 && key.bytes().all(|byte| byte.is_ascii_hexdigit()) {
2364        Ok(())
2365    } else {
2366        Err("dedup_key must be a 64-character SHA-256 hex string".to_string())
2367    }
2368}
2369
2370fn load_records(path: &Path) -> Result<Vec<LedgerRecord>, String> {
2371    let file = match car_secrets::open_private_read(path) {
2372        Ok(file) => file,
2373        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
2374        Err(error) => return Err(format!("open self-heal ledger: {error}")),
2375    };
2376    car_secrets::revalidate_private_path(path, &file)
2377        .map_err(|error| format!("validate self-heal ledger: {error}"))?;
2378    let mut records = Vec::new();
2379    for (index, line) in BufReader::new(file).lines().enumerate() {
2380        let line =
2381            line.map_err(|error| format!("read self-heal ledger line {}: {error}", index + 1))?;
2382        if line.trim().is_empty() {
2383            continue;
2384        }
2385        records.push(
2386            serde_json::from_str(&line)
2387                .map_err(|error| format!("parse self-heal ledger line {}: {error}", index + 1))?,
2388        );
2389    }
2390    Ok(records)
2391}
2392
2393fn append_records(path: &Path, records: &[LedgerRecord]) -> Result<(), String> {
2394    if records.is_empty() {
2395        return Ok(());
2396    }
2397    let parent = path
2398        .parent()
2399        .ok_or_else(|| "self-heal ledger path has no parent".to_string())?;
2400    car_secrets::ensure_private_dir(parent)
2401        .map_err(|error| format!("create self-heal ledger directory: {error}"))?;
2402    let mut file = car_secrets::open_private_append(path)
2403        .map_err(|error| format!("open self-heal ledger for append: {error}"))?;
2404    let original_len = file
2405        .metadata()
2406        .map_err(|error| format!("stat self-heal ledger: {error}"))?
2407        .len();
2408    let write_result = (|| -> Result<(), String> {
2409        for record in records {
2410            serde_json::to_writer(&mut file, record)
2411                .map_err(|error| format!("serialize self-heal ledger record: {error}"))?;
2412            file.write_all(b"\n")
2413                .map_err(|error| format!("append self-heal ledger newline: {error}"))?;
2414        }
2415        file.flush()
2416            .map_err(|error| format!("flush self-heal ledger: {error}"))?;
2417        file.sync_all()
2418            .map_err(|error| format!("sync self-heal ledger: {error}"))?;
2419        car_secrets::revalidate_private_path(path, &file)
2420            .map_err(|error| format!("revalidate self-heal ledger: {error}"))?;
2421        Ok(())
2422    })();
2423    if let Err(error) = write_result {
2424        let _ = file.set_len(original_len);
2425        return Err(error);
2426    }
2427    Ok(())
2428}
2429
2430#[cfg(test)]
2431mod tests {
2432    use super::*;
2433
2434    #[test]
2435    fn auto_fix_config_defaults_on_and_refuses_raised_hard_limits() {
2436        let temp = tempfile::TempDir::new().unwrap();
2437        let path = temp.path().join("config.toml");
2438        let default = read_auto_fix_config(&path).unwrap();
2439        assert!(default.auto_fix);
2440        assert_eq!(default.max_concurrent, 1);
2441        assert_eq!(default.max_per_day, 3);
2442        assert_eq!(default.max_rounds_per_key, 3);
2443
2444        std::fs::write(
2445            &path,
2446            "[selfheal]\nauto_fix=false\nmax_per_day=2\nmax_rounds_per_key=1\n",
2447        )
2448        .unwrap();
2449        let lowered = read_auto_fix_config(&path).unwrap();
2450        assert!(!lowered.auto_fix);
2451        assert_eq!(lowered.max_per_day, 2);
2452        assert_eq!(lowered.max_rounds_per_key, 1);
2453
2454        for (config, refusal) in [
2455            ("[selfheal]\nmax_concurrent=2\n", "max_concurrent must be 1"),
2456            (
2457                "[selfheal]\nmax_per_day=4\n",
2458                "max_per_day must be between 1 and 3",
2459            ),
2460            (
2461                "[selfheal]\nmax_rounds_per_key=4\n",
2462                "max_rounds_per_key must be between 1 and 3",
2463            ),
2464        ] {
2465            std::fs::write(&path, config).unwrap();
2466            assert!(read_auto_fix_config(&path).unwrap_err().contains(refusal));
2467        }
2468    }
2469
2470    #[test]
2471    fn auto_fix_excludes_side_effecting_gui_delegation_even_when_builtin() {
2472        assert!(auto_fix_tool_allowed("read_file", true));
2473        assert!(!auto_fix_tool_allowed("delegate_gui", true));
2474        assert!(!auto_fix_tool_allowed("read_file", false));
2475    }
2476
2477    #[test]
2478    fn code_task_terminal_failure_class_comes_from_run_end_record() {
2479        assert_eq!(
2480            code_task_failure_class(
2481                b"{\"type\":\"progress\"}\n{\"type\":\"run_end\",\"failure_class\":\"contract_not_green\"}\n"
2482            )
2483            .as_deref(),
2484            Some("contract_not_green")
2485        );
2486        assert!(code_task_failure_class(b"not json\n").is_none());
2487        assert_eq!(classify_code_task_output(b"not json\n"), "missing_run_end");
2488    }
2489}