1use car_eventlog::{Alert, AlertThresholds, EventKind};
12use car_ir::ActionProposal;
13use car_selfheal::{
14 agent_gave_up, agent_log_errors, capability_miss, metrics_alerts, recurring_tool_failure,
15 AgentDetectorConfig, AgentLogDetectorConfig, Detection, DetectionKind, EventEvidence,
16 EvidenceSource, Redactor, Severity, SupervisorAgentState, SupervisorSnapshot,
17 ToolFailureConfig,
18};
19use chrono::{DateTime, Duration, Utc};
20use serde::{Deserialize, Serialize};
21use std::collections::{BTreeMap, BTreeSet, VecDeque};
22use std::io::{BufRead, BufReader, Read, Write};
23use std::path::{Path, PathBuf};
24use std::sync::Arc;
25use tokio::sync::Mutex;
26
27use crate::session::ServerState;
28
29pub const DEFAULT_SELFHEAL_INTERVAL_SECS: u64 = 15 * 60;
30pub const SELFHEAL_INTERVAL_ENV: &str = "CAR_SELFHEAL_INTERVAL_SECS";
31const MAX_PAGE_SIZE: usize = 500;
32const DEFAULT_PAGE_SIZE: usize = 100;
33const ACTIVITY_TAIL_LINES: usize = 40;
34const STDERR_TAIL_LINES: usize = 50;
35const EXPECTED_ORIGIN: &str = "Parslee-ai/car";
36const TRUST_TIER: &str = "trusted";
37const DEFAULT_AUTO_FIX: bool = true;
38const DEFAULT_MAX_CONCURRENT: u32 = 1;
39const DEFAULT_MAX_PER_DAY: u32 = 3;
40const DEFAULT_MAX_ROUNDS_PER_KEY: u32 = 3;
41const SELFHEAL_CODER_WALL_SECS: u64 = 15 * 60;
44
45pub const DETECTOR_IDS: [&str; 5] = [
46 car_selfheal::detectors::metrics::DETECTOR_ID,
47 car_selfheal::detectors::agent::DETECTOR_ID,
48 car_selfheal::detectors::agent_logs::DETECTOR_ID,
49 car_selfheal::detectors::tools::DETECTOR_ID,
50 car_selfheal::detectors::capability::DETECTOR_ID,
51];
52
53#[derive(Debug, Clone, Default)]
58pub struct SelfhealSourceProbe {
59 explicit_checkout: Option<PathBuf>,
60 candidates: Vec<PathBuf>,
61 setup_refusal: Option<String>,
62}
63
64impl SelfhealSourceProbe {
65 pub fn empty() -> Self {
68 Self::default()
69 }
70
71 pub fn explicit(path: PathBuf) -> Self {
74 Self {
75 explicit_checkout: Some(path),
76 ..Self::default()
77 }
78 }
79
80 pub fn candidates(paths: impl IntoIterator<Item = PathBuf>) -> Self {
82 Self {
83 candidates: paths.into_iter().collect(),
84 ..Self::default()
85 }
86 }
87
88 pub fn with_candidates(mut self, paths: impl IntoIterator<Item = PathBuf>) -> Self {
91 self.candidates.extend(paths);
92 dedup_paths(&mut self.candidates);
93 self
94 }
95
96 pub fn from_process() -> Self {
102 let car_home = car_home::root_or_relative();
103 let executable = std::env::current_exe().ok();
104 let anchor = std::env::var_os("CAR_PROJECT_DIR")
105 .map(PathBuf::from)
106 .or_else(|| std::env::current_dir().ok());
107 Self::from_local_paths(&car_home, executable.as_deref(), anchor.as_deref())
108 }
109
110 #[doc(hidden)]
114 pub fn from_local_paths(
115 car_home: &Path,
116 executable: Option<&Path>,
117 project_anchor: Option<&Path>,
118 ) -> Self {
119 let config_path = car_home.join("config.toml");
120 let mut probe = Self::default();
121 match read_source_checkout_config(&config_path) {
122 Ok(Some(path)) => {
123 probe.explicit_checkout = Some(if path.is_absolute() {
124 path
125 } else {
126 car_home.join(path)
127 });
128 }
129 Ok(None) => {}
130 Err(error) => probe.setup_refusal = Some(error),
131 }
132
133 if let Some(parent) = executable.and_then(Path::parent) {
134 probe
135 .candidates
136 .extend(parent.ancestors().map(Path::to_path_buf));
137 }
138 if let Some(car_dir) = project_anchor.and_then(car_memgine::project::discover_project) {
139 if let Some(project_root) = car_dir.parent() {
140 probe.candidates.push(project_root.to_path_buf());
141 }
142 }
143 dedup_paths(&mut probe.candidates);
144 probe
145 }
146
147 fn resolve(&self) -> SourceRouteDecision {
148 if let Some(reason) = &self.setup_refusal {
149 return SourceRouteDecision::ledger_only(reason.clone());
150 }
151 if let Some(candidate) = &self.explicit_checkout {
152 return match validate_source_checkout(candidate) {
153 Ok(path) => SourceRouteDecision::local(path),
154 Err(reason) => SourceRouteDecision::ledger_only(format!(
155 "explicit selfheal.source_checkout {} refused: {reason}",
156 candidate.display()
157 )),
158 };
159 }
160
161 let mut refusals = Vec::new();
162 for candidate in &self.candidates {
163 match validate_source_checkout(candidate) {
164 Ok(path) => return SourceRouteDecision::local(path),
165 Err(reason) => refusals.push(format!("{}: {reason}", candidate.display())),
166 }
167 }
168 if refusals.is_empty() {
169 SourceRouteDecision::ledger_only(
170 "no bounded source-checkout candidates were discovered".to_string(),
171 )
172 } else {
173 SourceRouteDecision::ledger_only(format!(
174 "no candidate validated as {EXPECTED_ORIGIN}; refusals: {}",
175 refusals.join("; ")
176 ))
177 }
178 }
179}
180
181#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
185#[serde(rename_all = "kebab-case")]
186pub enum SelfhealRoute {
187 Local,
188 Feedback,
189 #[default]
190 LedgerOnly,
191}
192
193#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
194pub struct SourceRouteDecision {
195 pub route: SelfhealRoute,
196 #[serde(default, skip_serializing_if = "Option::is_none")]
197 pub source_checkout: Option<PathBuf>,
198 #[serde(default, skip_serializing_if = "Option::is_none")]
199 pub refusal_reason: Option<String>,
200}
201
202impl SourceRouteDecision {
203 fn local(path: PathBuf) -> Self {
204 Self {
205 route: SelfhealRoute::Local,
206 source_checkout: Some(path),
207 refusal_reason: None,
208 }
209 }
210
211 fn ledger_only(reason: String) -> Self {
212 Self {
213 route: SelfhealRoute::LedgerOnly,
214 source_checkout: None,
215 refusal_reason: Some(reason),
216 }
217 }
218}
219
220#[derive(Debug, Clone, Default)]
224pub enum SelfhealReplayVerbProbe {
225 #[default]
226 CheckoutBuild,
227 #[doc(hidden)]
228 Available,
229 #[doc(hidden)]
230 Unavailable(String),
231}
232
233impl SelfhealReplayVerbProbe {
234 #[doc(hidden)]
235 pub fn available_for_tests() -> Self {
236 Self::Available
237 }
238
239 #[doc(hidden)]
240 pub fn unavailable_for_tests(reason: impl Into<String>) -> Self {
241 Self::Unavailable(reason.into())
242 }
243
244 async fn check(&self, checkout: &Path, target_dir: &Path) -> Result<(), String> {
245 match self {
246 Self::Available => Ok(()),
247 Self::Unavailable(reason) => Err(reason.clone()),
248 Self::CheckoutBuild => check_checkout_replay_verb(checkout, target_dir).await,
249 }
250 }
251}
252
253#[derive(Debug, Clone, Default)]
257pub struct SelfhealEvidence {
258 pub events: Vec<EventEvidence>,
259 pub metric_alerts: Vec<Alert>,
260 pub metric_provenance: Vec<EvidenceSource>,
261 pub supervisor_snapshot: Option<SupervisorSnapshot>,
262 pub registry_agents: usize,
263}
264
265#[derive(Debug, Clone, Serialize, Deserialize)]
266pub struct TickSummary {
267 pub started_at: DateTime<Utc>,
268 pub completed_at: DateTime<Utc>,
269 #[serde(default)]
270 pub route: SelfhealRoute,
271 #[serde(default, skip_serializing_if = "Option::is_none")]
272 pub source_checkout: Option<PathBuf>,
273 #[serde(default, skip_serializing_if = "Option::is_none")]
274 pub refusal_reason: Option<String>,
275 pub events_scanned: usize,
276 pub supervisor_agents: usize,
277 pub registry_agents: usize,
278 pub detections_found: usize,
279 pub appended: usize,
280 pub changed: usize,
281 pub suppressed_dismissed: usize,
282 pub filing_mode: String,
283}
284
285#[derive(Debug, Clone, Serialize)]
286pub struct SelfhealStatus {
287 pub cadence_secs: u64,
288 pub auto_fix_enabled: bool,
289 pub max_concurrent: u32,
290 pub max_per_day: u32,
291 pub max_rounds_per_key: u32,
292 #[serde(default, skip_serializing_if = "Option::is_none")]
293 pub auto_fix_refusal_reason: Option<String>,
294 pub last_tick_at: Option<DateTime<Utc>>,
295 pub route: SelfhealRoute,
296 #[serde(default, skip_serializing_if = "Option::is_none")]
297 pub source_checkout: Option<PathBuf>,
298 #[serde(default, skip_serializing_if = "Option::is_none")]
299 pub refusal_reason: Option<String>,
300 pub detectors: Vec<&'static str>,
301 pub detection_count: usize,
302 pub warning_count: usize,
303 pub critical_count: usize,
304 pub dismissed_count: usize,
305 pub filing_mode: &'static str,
306}
307
308#[derive(Debug, Clone, Deserialize, Default)]
309pub struct DetectionQuery {
310 #[serde(default)]
311 pub kind: Option<DetectionKind>,
312 #[serde(default)]
313 pub severity: Option<Severity>,
314 #[serde(default)]
315 pub since: Option<DateTime<Utc>>,
316 #[serde(default)]
317 pub offset: usize,
318 #[serde(default)]
319 pub limit: Option<usize>,
320}
321
322#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
325pub struct ReconstructedToolCall {
326 pub tool: String,
327 pub params: BTreeMap<String, serde_json::Value>,
328}
329
330#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
331pub struct RoutedDetection {
332 #[serde(flatten)]
333 pub detection: Detection,
334 pub route: SelfhealRoute,
335 #[serde(default, skip_serializing_if = "Option::is_none")]
336 pub local_issue_path: Option<PathBuf>,
337 #[serde(default, skip_serializing_if = "Option::is_none")]
340 pub eligible: Option<bool>,
341 #[serde(default, skip_serializing_if = "Option::is_none")]
344 pub reconstructed_call: Option<ReconstructedToolCall>,
345 #[serde(default, skip_serializing_if = "Option::is_none")]
347 pub reconstructed_call_path: Option<PathBuf>,
348 #[serde(default, skip_serializing_if = "is_zero_u32")]
350 pub auto_fix_attempts: u32,
351 #[serde(default, skip_serializing_if = "is_false")]
353 pub auto_fix_exhausted: bool,
354 #[serde(default, skip_serializing_if = "is_false")]
356 pub auto_fix_in_progress: bool,
357 #[serde(default, skip_serializing_if = "Option::is_none")]
358 pub last_auto_fix_attempt: Option<FixAttemptResult>,
359 #[serde(default, skip_serializing_if = "is_false")]
361 pub auto_fix_awaiting_review: bool,
362 #[serde(default, skip_serializing_if = "is_false")]
364 pub auto_fix_parked: bool,
365 #[serde(default, skip_serializing_if = "Option::is_none")]
366 pub remote_pr_number: Option<u64>,
367 #[serde(default, skip_serializing_if = "Option::is_none")]
368 pub remote_pr_url: Option<String>,
369}
370
371fn is_zero_u32(value: &u32) -> bool {
372 *value == 0
373}
374
375fn is_false(value: &bool) -> bool {
376 !*value
377}
378
379#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
380#[serde(rename_all = "snake_case")]
381pub enum FixTrigger {
382 Cadence,
383 Manual,
384}
385
386#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
387#[serde(rename_all = "snake_case")]
388pub enum RemotePrDisposition {
389 AwaitingReview,
390 Parked,
391}
392
393#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
394pub struct FixAttemptResult {
395 pub dedup_key: String,
396 pub round: u32,
397 pub trigger: FixTrigger,
398 pub started_at: DateTime<Utc>,
399 pub completed_at: DateTime<Utc>,
400 pub target_branch: String,
401 pub workspace_dir: PathBuf,
402 pub spawned: bool,
403 pub exit_code: Option<i32>,
404 pub failure_class: String,
405}
406
407impl RoutedDetection {
408 pub fn dedup_key(&self) -> &str {
409 self.detection.dedup_key()
410 }
411}
412
413#[derive(Debug, Clone, Serialize)]
414pub struct DetectionPage {
415 pub detections: Vec<RoutedDetection>,
416 pub total: usize,
417 pub offset: usize,
418 pub limit: usize,
419 pub next_offset: Option<usize>,
420}
421
422#[derive(Debug, Clone, Serialize)]
423pub struct DismissResult {
424 pub dedup_key: String,
425 pub dismissed: bool,
426 pub already_dismissed: bool,
427 pub dismissed_at: DateTime<Utc>,
428}
429
430#[derive(Debug, Clone, Serialize, Deserialize)]
431#[serde(tag = "record_type", rename_all = "snake_case")]
432enum LedgerRecord {
433 Detection {
434 recorded_at: DateTime<Utc>,
435 detection: Detection,
436 #[serde(default)]
437 route: SelfhealRoute,
438 #[serde(default, skip_serializing_if = "Option::is_none")]
439 local_issue_path: Option<PathBuf>,
440 #[serde(default, skip_serializing_if = "Option::is_none")]
441 eligible: Option<bool>,
442 #[serde(default, skip_serializing_if = "Option::is_none")]
443 reconstructed_call: Option<ReconstructedToolCall>,
444 #[serde(default, skip_serializing_if = "Option::is_none")]
445 reconstructed_call_path: Option<PathBuf>,
446 },
447 Dismissal {
448 dismissed_at: DateTime<Utc>,
449 dedup_key: String,
450 },
451 Tick {
452 summary: TickSummary,
453 },
454 FixStarted {
455 started_at: DateTime<Utc>,
456 dedup_key: String,
457 round: u32,
458 trigger: FixTrigger,
459 target_branch: String,
460 workspace_dir: PathBuf,
461 },
462 FixAttempt {
463 result: FixAttemptResult,
464 },
465 RemotePr {
466 observed_at: DateTime<Utc>,
467 dedup_key: String,
468 disposition: RemotePrDisposition,
469 pr_number: u64,
470 pr_url: String,
471 },
472}
473
474#[derive(Default)]
475struct LedgerState {
476 records: Vec<LedgerRecord>,
477 supervisor_snapshots: VecDeque<SupervisorSnapshot>,
478}
479
480pub struct SelfhealService {
482 ledger_path: PathBuf,
483 state_root: PathBuf,
484 interval_secs: u64,
485 state: Mutex<LedgerState>,
486 operation: Mutex<()>,
487 evidence_override: Option<SelfhealEvidence>,
488 source_probe: SelfhealSourceProbe,
489 replay_verb_probe: SelfhealReplayVerbProbe,
490 auto_fix: AutoFixConfig,
491 auto_fix_refusal_reason: Option<String>,
492}
493
494impl SelfhealService {
495 pub fn open(
496 ledger_path: PathBuf,
497 interval_secs: u64,
498 evidence_override: Option<SelfhealEvidence>,
499 source_probe: SelfhealSourceProbe,
500 replay_verb_probe: SelfhealReplayVerbProbe,
501 ) -> Result<Self, String> {
502 let records = load_records(&ledger_path)?;
503 let state_root = ledger_path
504 .parent()
505 .and_then(Path::parent)
506 .map(Path::to_path_buf)
507 .ok_or_else(|| "self-heal ledger must live under <CAR_HOME>/selfheal".to_string())?;
508 let (auto_fix, auto_fix_refusal_reason) =
509 match read_auto_fix_config(&state_root.join("config.toml")) {
510 Ok(config) => (config, None),
511 Err(reason) => (AutoFixConfig::disabled(), Some(reason)),
512 };
513 Ok(Self {
514 ledger_path,
515 state_root,
516 interval_secs: interval_secs.max(1),
517 state: Mutex::new(LedgerState {
518 records,
519 supervisor_snapshots: VecDeque::new(),
520 }),
521 operation: Mutex::new(()),
522 evidence_override,
523 source_probe,
524 replay_verb_probe,
525 auto_fix,
526 auto_fix_refusal_reason,
527 })
528 }
529
530 pub fn interval_secs(&self) -> u64 {
531 self.interval_secs
532 }
533
534 pub fn ledger_path(&self) -> &Path {
535 &self.ledger_path
536 }
537
538 pub async fn status(&self) -> SelfhealStatus {
539 let state = self.state.lock().await;
540 let (active, dismissed) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
541 let mut warning_count = 0;
542 let mut critical_count = 0;
543 for detection in active.values() {
544 match detection.detection.severity() {
545 Severity::Warning => warning_count += 1,
546 Severity::Critical => critical_count += 1,
547 }
548 }
549 let latest_tick = state.records.iter().rev().find_map(|record| match record {
550 LedgerRecord::Tick { summary } => Some(summary),
551 LedgerRecord::Detection { .. }
552 | LedgerRecord::Dismissal { .. }
553 | LedgerRecord::FixStarted { .. }
554 | LedgerRecord::FixAttempt { .. }
555 | LedgerRecord::RemotePr { .. } => None,
556 });
557 let route = latest_tick.map_or_else(
558 || SourceRouteDecision::ledger_only("source route not evaluated yet".to_string()),
559 |summary| SourceRouteDecision {
560 route: summary.route,
561 source_checkout: summary.source_checkout.clone(),
562 refusal_reason: summary.refusal_reason.clone(),
563 },
564 );
565 SelfhealStatus {
566 cadence_secs: self.interval_secs,
567 auto_fix_enabled: self.auto_fix.auto_fix && self.auto_fix_refusal_reason.is_none(),
568 max_concurrent: self.auto_fix.max_concurrent,
569 max_per_day: self.auto_fix.max_per_day,
570 max_rounds_per_key: self.auto_fix.max_rounds_per_key,
571 auto_fix_refusal_reason: self.auto_fix_refusal_reason.clone(),
572 last_tick_at: latest_tick.map(|summary| summary.completed_at),
573 route: route.route,
574 source_checkout: route.source_checkout,
575 refusal_reason: route.refusal_reason,
576 detectors: DETECTOR_IDS.to_vec(),
577 detection_count: active.len(),
578 warning_count,
579 critical_count,
580 dismissed_count: dismissed.len(),
581 filing_mode: if self.auto_fix.auto_fix
582 && self.auto_fix_refusal_reason.is_none()
583 && route.route == SelfhealRoute::Local
584 {
585 "pr-only"
586 } else {
587 "watch-only"
588 },
589 }
590 }
591
592 pub async fn detections(&self, query: DetectionQuery) -> DetectionPage {
593 let state = self.state.lock().await;
594 let (active, _) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
595 let mut detections: Vec<_> = active
596 .into_values()
597 .filter(|detection| {
598 query
599 .kind
600 .is_none_or(|kind| detection.detection.kind() == kind)
601 && query
602 .severity
603 .is_none_or(|severity| detection.detection.severity() == severity)
604 && query
605 .since
606 .is_none_or(|since| detection.detection.last_observed_at() >= since)
607 })
608 .collect();
609 detections.sort_by(|a, b| {
610 b.detection
611 .last_observed_at()
612 .cmp(&a.detection.last_observed_at())
613 .then_with(|| a.dedup_key().cmp(b.dedup_key()))
614 });
615 let total = detections.len();
616 let limit = query
617 .limit
618 .unwrap_or(DEFAULT_PAGE_SIZE)
619 .clamp(1, MAX_PAGE_SIZE);
620 let page = detections
621 .into_iter()
622 .skip(query.offset)
623 .take(limit)
624 .collect::<Vec<_>>();
625 let consumed = query.offset.saturating_add(page.len());
626 DetectionPage {
627 detections: page,
628 total,
629 offset: query.offset,
630 limit,
631 next_offset: (consumed < total).then_some(consumed),
632 }
633 }
634
635 pub async fn dismiss(&self, dedup_key: &str) -> Result<DismissResult, String> {
636 validate_dedup_key(dedup_key)?;
637 let _operation = self.operation.lock().await;
638 let dismissed_at = Utc::now();
639 let mut state = self.state.lock().await;
640 let known = state.records.iter().any(|record| {
641 matches!(
642 record,
643 LedgerRecord::Detection { detection, .. } if detection.dedup_key() == dedup_key
644 )
645 });
646 if !known {
647 return Err(format!(
648 "unknown self-heal detection dedup_key '{dedup_key}'"
649 ));
650 }
651 let (_, dismissed) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
652 let already_dismissed = dismissed.contains(dedup_key);
653 let record = LedgerRecord::Dismissal {
654 dismissed_at,
655 dedup_key: dedup_key.to_string(),
656 };
657 append_records(&self.ledger_path, std::slice::from_ref(&record))?;
658 state.records.push(record);
659 Ok(DismissResult {
660 dedup_key: dedup_key.to_string(),
661 dismissed: true,
662 already_dismissed,
663 dismissed_at,
664 })
665 }
666
667 pub async fn fix(&self, dedup_key: &str) -> Result<FixAttemptResult, String> {
671 validate_dedup_key(dedup_key)?;
672 let _operation = self
673 .operation
674 .try_lock()
675 .map_err(|_| "another self-heal operation is already running".to_string())?;
676 match self.reconcile_remote_pr_unlocked(dedup_key).await? {
677 RemoteDedupResult::Proceed => {
678 self.run_fix_unlocked(dedup_key, FixTrigger::Manual).await
679 }
680 RemoteDedupResult::AwaitingReview => Err(format!(
681 "self-heal detection '{dedup_key}' is awaiting review on its open remote PR"
682 )),
683 RemoteDedupResult::Parked => Err(format!(
684 "self-heal detection '{dedup_key}' is parked after its PR closed unmerged"
685 )),
686 }
687 }
688
689 async fn run_auto_fix_unlocked(&self) -> Result<Option<FixAttemptResult>, String> {
690 if !self.auto_fix.auto_fix || self.auto_fix_refusal_reason.is_some() {
691 return Ok(None);
692 }
693 let candidate = {
694 let state = self.state.lock().await;
695 if starts_today(&state.records, Utc::now()) >= self.auto_fix.max_per_day {
696 return Ok(None);
697 }
698 let (active, _) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
699 active
700 .into_values()
701 .filter(|detection| detection.eligible == Some(true))
702 .filter(|detection| detection.auto_fix_attempts == 0)
705 .filter(|detection| {
706 !detection.auto_fix_awaiting_review && !detection.auto_fix_parked
707 })
708 .min_by(|left, right| {
709 left.detection
710 .last_observed_at()
711 .cmp(&right.detection.last_observed_at())
712 .then_with(|| left.dedup_key().cmp(right.dedup_key()))
713 })
714 .map(|detection| detection.dedup_key().to_string())
715 };
716 match candidate {
717 Some(key) => match self.reconcile_remote_pr_unlocked(&key).await? {
718 RemoteDedupResult::Proceed => self
719 .run_fix_unlocked(&key, FixTrigger::Cadence)
720 .await
721 .map(Some),
722 RemoteDedupResult::AwaitingReview | RemoteDedupResult::Parked => Ok(None),
723 },
724 None => Ok(None),
725 }
726 }
727
728 async fn reconcile_remote_pr_unlocked(
729 &self,
730 dedup_key: &str,
731 ) -> Result<RemoteDedupResult, String> {
732 let route = self.source_probe.resolve();
733 let Some(checkout) = route.source_checkout else {
734 return Ok(RemoteDedupResult::Proceed);
735 };
736 let attempts = {
737 let state = self.state.lock().await;
738 let (active, _) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
739 active
740 .get(dedup_key)
741 .map_or(0, |detection| detection.auto_fix_attempts)
742 };
743 if attempts > 0 {
744 return Ok(RemoteDedupResult::Proceed);
745 }
746
747 let Some(remote) = query_remote_selfheal_pr(&checkout, dedup_key).await? else {
748 return Ok(RemoteDedupResult::Proceed);
749 };
750 let disposition = match (remote.state.as_str(), remote.merged_at.as_deref()) {
751 ("OPEN", _) => RemotePrDisposition::AwaitingReview,
752 ("CLOSED", None) => RemotePrDisposition::Parked,
753 ("CLOSED", Some(_)) => RemotePrDisposition::Parked,
756 (state, _) => {
757 return Err(format!("remote self-heal PR has unknown state '{state}'"));
758 }
759 };
760 let record = LedgerRecord::RemotePr {
761 observed_at: Utc::now(),
762 dedup_key: dedup_key.to_string(),
763 disposition,
764 pr_number: remote.number,
765 pr_url: remote.url,
766 };
767 let mut state = self.state.lock().await;
768 append_records(&self.ledger_path, std::slice::from_ref(&record))?;
769 state.records.push(record);
770 Ok(match disposition {
771 RemotePrDisposition::AwaitingReview => RemoteDedupResult::AwaitingReview,
772 RemotePrDisposition::Parked => RemoteDedupResult::Parked,
773 })
774 }
775
776 async fn run_fix_unlocked(
777 &self,
778 dedup_key: &str,
779 trigger: FixTrigger,
780 ) -> Result<FixAttemptResult, String> {
781 if let Some(reason) = &self.auto_fix_refusal_reason {
782 return Err(format!("self-heal auto-fix config refused: {reason}"));
783 }
784 let route = self.source_probe.resolve();
785 let checkout = route.source_checkout.ok_or_else(|| {
786 format!(
787 "self-heal key '{dedup_key}' cannot auto-fix from route {}: {}",
788 route_name(route.route),
789 route
790 .refusal_reason
791 .unwrap_or_else(|| "no validated source checkout".to_string())
792 )
793 })?;
794 let (detection, round) = {
795 let state = self.state.lock().await;
796 let (active, _) = fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
797 let detection = active
798 .get(dedup_key)
799 .cloned()
800 .ok_or_else(|| format!("unknown or dismissed self-heal detection '{dedup_key}'"))?;
801 if detection.eligible != Some(true) {
802 return Err(format!(
803 "self-heal detection '{dedup_key}' is not eligible for auto-fix"
804 ));
805 }
806 if detection.auto_fix_in_progress {
807 return Err(format!(
808 "self-heal detection '{dedup_key}' already has a coder round in progress"
809 ));
810 }
811 if detection.auto_fix_attempts >= self.auto_fix.max_rounds_per_key {
812 return Err(format!(
813 "self-heal detection '{dedup_key}' is auto-fix exhausted"
814 ));
815 }
816 if starts_today(&state.records, Utc::now()) >= self.auto_fix.max_per_day {
817 return Err(format!(
818 "self-heal daily auto-fix limit ({}) is exhausted",
819 self.auto_fix.max_per_day
820 ));
821 }
822 let round = detection.auto_fix_attempts + 1;
823 (detection, round)
824 };
825 let call = detection
826 .reconstructed_call
827 .as_ref()
828 .ok_or_else(|| format!("eligible self-heal key '{dedup_key}' has no safe call"))?;
829
830 let attempt_dir = self
831 .state_root
832 .join("selfheal")
833 .join("attempts")
834 .join(dedup_key);
835 car_secrets::ensure_private_dir(&attempt_dir)
836 .map_err(|error| format!("create self-heal attempt directory: {error}"))?;
837 let params_path = attempt_dir.join("params.json");
838 persist_private_json(¶ms_path, &call.params)?;
839 let rendered =
840 crate::selfheal_templates::render(&crate::selfheal_templates::TemplateContext {
841 dedup_key,
842 detector_id: detection.detection.detector_id(),
843 locator: detection.detection.locator(),
844 tool: &call.tool,
845 params_path: ¶ms_path,
846 })?;
847 let intent_path = attempt_dir.join("intent.md");
848 persist_private_bytes(&intent_path, rendered.intent.as_bytes())?;
849 let contract_path = attempt_dir.join("contract.json");
850 persist_private_json(&contract_path, &rendered.contract)?;
851
852 let target_branch = format!("car/selfheal/{dedup_key}");
853 let workspace_dir = checkout
854 .join(".worktrees")
855 .join(format!("selfheal-{dedup_key}"));
856 let target_dir = self.state_root.join("selfheal").join("target");
857 car_secrets::ensure_private_dir(&target_dir)
858 .map_err(|error| format!("create self-heal cargo target directory: {error}"))?;
859 let car_executable = resolve_car_cli_executable();
863 let mut path_entries = vec![target_dir.join("debug")];
864 if let Some(path) = std::env::var_os("PATH") {
865 path_entries.extend(std::env::split_paths(&path));
866 }
867 let code_task_path = std::env::join_paths(path_entries)
868 .map_err(|error| format!("build self-heal code-task PATH: {error}"))?;
869 let started_at = Utc::now();
870 let started = LedgerRecord::FixStarted {
871 started_at,
872 dedup_key: dedup_key.to_string(),
873 round,
874 trigger: trigger.clone(),
875 target_branch: target_branch.clone(),
876 workspace_dir: workspace_dir.clone(),
877 };
878 {
879 let mut state = self.state.lock().await;
880 append_records(&self.ledger_path, std::slice::from_ref(&started))?;
881 state.records.push(started);
882 }
883
884 let fetch = tokio::process::Command::new("git")
885 .current_dir(&checkout)
886 .args(["fetch", "origin", "main"])
887 .kill_on_drop(true)
888 .output()
889 .await;
890 match fetch {
891 Err(_) => {
892 return self
893 .finish_attempt(
894 dedup_key,
895 round,
896 trigger,
897 started_at,
898 target_branch,
899 workspace_dir,
900 false,
901 None,
902 "fetch_spawn_failed".to_string(),
903 )
904 .await;
905 }
906 Ok(output) if !output.status.success() => {
907 return self
908 .finish_attempt(
909 dedup_key,
910 round,
911 trigger,
912 started_at,
913 target_branch,
914 workspace_dir,
915 false,
916 output.status.code(),
917 "fetch_failed".to_string(),
918 )
919 .await;
920 }
921 Ok(_) => {}
922 }
923
924 let Some(car_executable) = car_executable else {
925 return self
926 .finish_attempt(
927 dedup_key,
928 round,
929 trigger,
930 started_at,
931 target_branch,
932 workspace_dir,
933 false,
934 None,
935 "spawn_failed".to_string(),
936 )
937 .await;
938 };
939 let marker = format!("<!-- car-selfheal:key={dedup_key} -->");
940 let transcript_path = attempt_dir.join(format!("code-task-round-{round}.jsonl"));
941 persist_private_bytes(&transcript_path, b"")?;
942 let argv = vec![
943 "code-task".to_string(),
944 "--repo".to_string(),
945 checkout.to_string_lossy().into_owned(),
946 "--intent-file".to_string(),
947 intent_path.to_string_lossy().into_owned(),
948 "--contract-file".to_string(),
949 contract_path.to_string_lossy().into_owned(),
950 "--deliver".to_string(),
951 "pr".to_string(),
952 "--target-branch".to_string(),
953 target_branch.clone(),
954 "--pr-base".to_string(),
955 "main".to_string(),
956 "--workspace-dir".to_string(),
957 workspace_dir.to_string_lossy().into_owned(),
958 "--body-prefix".to_string(),
959 marker,
960 "--max-session-wall-secs".to_string(),
961 SELFHEAL_CODER_WALL_SECS.to_string(),
962 "--transcript".to_string(),
963 transcript_path.to_string_lossy().into_owned(),
964 "--json".to_string(),
965 ];
966 let output = tokio::process::Command::new(car_executable)
967 .args(&argv)
968 .env("CARGO_TARGET_DIR", &target_dir)
969 .env("PATH", code_task_path)
974 .kill_on_drop(true)
975 .output()
976 .await;
977 match output {
978 Err(_) => {
979 self.finish_attempt(
980 dedup_key,
981 round,
982 trigger,
983 started_at,
984 target_branch,
985 workspace_dir,
986 false,
987 None,
988 "spawn_failed".to_string(),
989 )
990 .await
991 }
992 Ok(output) => {
993 let failure_class = classify_code_task_output(&output.stdout);
994 self.finish_attempt(
995 dedup_key,
996 round,
997 trigger,
998 started_at,
999 target_branch,
1000 workspace_dir,
1001 true,
1002 output.status.code(),
1003 failure_class,
1004 )
1005 .await
1006 }
1007 }
1008 }
1009
1010 #[allow(clippy::too_many_arguments)]
1011 async fn finish_attempt(
1012 &self,
1013 dedup_key: &str,
1014 round: u32,
1015 trigger: FixTrigger,
1016 started_at: DateTime<Utc>,
1017 target_branch: String,
1018 workspace_dir: PathBuf,
1019 spawned: bool,
1020 exit_code: Option<i32>,
1021 failure_class: String,
1022 ) -> Result<FixAttemptResult, String> {
1023 let result = FixAttemptResult {
1024 dedup_key: dedup_key.to_string(),
1025 round,
1026 trigger,
1027 started_at,
1028 completed_at: Utc::now(),
1029 target_branch,
1030 workspace_dir,
1031 spawned,
1032 exit_code,
1033 failure_class,
1034 };
1035 let record = LedgerRecord::FixAttempt {
1036 result: result.clone(),
1037 };
1038 let mut state = self.state.lock().await;
1039 append_records(&self.ledger_path, std::slice::from_ref(&record))?;
1040 state.records.push(record);
1041 Ok(result)
1042 }
1043
1044 pub async fn run_tick(&self, server: &Arc<ServerState>) -> Result<TickSummary, String> {
1045 let _operation = self
1046 .operation
1047 .try_lock()
1048 .map_err(|_| "self-heal tick already running".to_string())?;
1049 let started_at = Utc::now();
1050 let since = {
1054 let state = self.state.lock().await;
1055 state.records.iter().rev().find_map(|record| match record {
1056 LedgerRecord::Tick { summary } => Some(summary.started_at),
1057 LedgerRecord::Detection { .. }
1058 | LedgerRecord::Dismissal { .. }
1059 | LedgerRecord::FixStarted { .. }
1060 | LedgerRecord::FixAttempt { .. }
1061 | LedgerRecord::RemotePr { .. } => None,
1062 })
1063 };
1064 let evidence = match self.evidence_override.clone() {
1065 Some(evidence) => evidence,
1066 None => gather_live_evidence(server, &self.state_root, since, started_at).await,
1067 };
1068 let route = self.source_probe.resolve();
1071 let summary = self.run_with_evidence(started_at, evidence, route).await?;
1072 if let Err(error) = self.run_auto_fix_unlocked().await {
1073 tracing::warn!(target: "car::selfheal", %error, "self-heal auto-fix round failed");
1074 }
1075 Ok(summary)
1076 }
1077
1078 async fn run_with_evidence(
1079 &self,
1080 started_at: DateTime<Utc>,
1081 evidence: SelfhealEvidence,
1082 route: SourceRouteDecision,
1083 ) -> Result<TickSummary, String> {
1084 let redactor = Redactor::from_env(std::env::vars());
1085 let car_version = env!("CARGO_PKG_VERSION");
1086 let observed_at = Utc::now();
1087 let mut found = Vec::new();
1088 found.extend(metrics_alerts(
1089 &evidence.metric_alerts,
1090 car_version,
1091 observed_at,
1092 &evidence.metric_provenance,
1093 &redactor,
1094 ));
1095 found.extend(recurring_tool_failure(
1096 &evidence.events,
1097 ToolFailureConfig::default(),
1098 car_version,
1099 &redactor,
1100 ));
1101 found.extend(capability_miss(&evidence.events, car_version, &redactor));
1102
1103 let supervisor_agents = evidence
1104 .supervisor_snapshot
1105 .as_ref()
1106 .map_or(0, |snapshot| snapshot.agents.len());
1107 if let Some(snapshot) = evidence.supervisor_snapshot.as_ref() {
1108 found.extend(agent_log_errors(
1109 snapshot,
1110 AgentLogDetectorConfig::default(),
1111 car_version,
1112 &redactor,
1113 ));
1114 }
1115 let snapshots = {
1116 let mut state = self.state.lock().await;
1117 if let Some(snapshot) = evidence.supervisor_snapshot {
1118 state.supervisor_snapshots.push_back(snapshot);
1119 let cutoff = observed_at
1120 - Duration::seconds(AgentDetectorConfig::default().window_secs as i64);
1121 while state
1122 .supervisor_snapshots
1123 .front()
1124 .is_some_and(|snapshot| snapshot.captured_at < cutoff)
1125 {
1126 state.supervisor_snapshots.pop_front();
1127 }
1128 }
1129 state
1130 .supervisor_snapshots
1131 .iter()
1132 .cloned()
1133 .collect::<Vec<_>>()
1134 };
1135 found.extend(agent_gave_up(
1136 &snapshots,
1137 AgentDetectorConfig::default(),
1138 car_version,
1139 &redactor,
1140 ));
1141
1142 let mut unique = BTreeMap::new();
1145 for detection in found {
1146 unique.insert(detection.dedup_key().to_string(), detection);
1147 }
1148 let detections_found = unique.len();
1149 let replay_verb_refusal = if route.route == SelfhealRoute::Local
1150 && unique.values().any(|detection| {
1151 detection.detector_id() == car_selfheal::detectors::tools::DETECTOR_ID
1152 }) {
1153 let checkout = route
1154 .source_checkout
1155 .as_ref()
1156 .expect("local route always carries a checkout");
1157 self.replay_verb_probe
1158 .check(
1159 checkout,
1160 &self.state_root.join("selfheal/replay-probe-target"),
1161 )
1162 .await
1163 .err()
1164 .map(|reason| format!("checkout predates the replay verb: {reason}"))
1165 } else {
1166 None
1167 };
1168 let mut state = self.state.lock().await;
1169 let (previous, dismissed) =
1170 fold_detections(&state.records, self.auto_fix.max_rounds_per_key);
1171 let mut appended = 0;
1172 let mut changed = 0;
1173 let mut suppressed_dismissed = 0;
1174 let mut records = Vec::new();
1175 for (key, detection) in unique {
1176 if dismissed.contains(&key) {
1177 suppressed_dismissed += 1;
1178 continue;
1179 }
1180 let assessment = assess_reconstructed_call(&detection, &evidence.events, &redactor);
1181 let mut eligible = assessment.as_ref().map(|assessment| assessment.eligible);
1182 let reconstructed_call =
1183 assessment.and_then(|assessment| assessment.reconstructed_call);
1184 let ineligible_reason = if eligible == Some(true) {
1185 replay_verb_refusal.as_deref()
1186 } else {
1187 None
1188 };
1189 if ineligible_reason.is_some() {
1190 eligible = Some(false);
1191 }
1192 let local_issue_path = if route.route == SelfhealRoute::Local {
1193 let path = self
1194 .state_root
1195 .join("selfheal")
1196 .join("issues")
1197 .join(format!("{key}.md"));
1198 render_local_issue(&path, &detection, eligible, ineligible_reason)?;
1199 Some(path)
1200 } else {
1201 None
1202 };
1203 let reconstructed_call_path =
1204 if route.route == SelfhealRoute::Local && eligible.is_some() {
1205 let path = self
1206 .state_root
1207 .join("selfheal")
1208 .join("issues")
1209 .join(format!("{key}.call.json"));
1210 persist_reconstructed_call(
1211 &path,
1212 eligible.unwrap_or(false),
1213 reconstructed_call.as_ref(),
1214 )?;
1215 Some(path)
1216 } else {
1217 None
1218 };
1219 let mut routed = RoutedDetection {
1220 detection,
1221 route: route.route,
1222 local_issue_path,
1223 eligible,
1224 reconstructed_call,
1225 reconstructed_call_path,
1226 auto_fix_attempts: 0,
1227 auto_fix_exhausted: false,
1228 auto_fix_in_progress: false,
1229 last_auto_fix_attempt: None,
1230 auto_fix_awaiting_review: false,
1231 auto_fix_parked: false,
1232 remote_pr_number: None,
1233 remote_pr_url: None,
1234 };
1235 apply_attempt_state(
1236 &mut routed,
1237 &state.records,
1238 self.auto_fix.max_rounds_per_key,
1239 );
1240 apply_remote_pr_state(&mut routed, &state.records);
1241 let should_append = match previous.get(&key) {
1242 None => {
1243 appended += 1;
1244 true
1245 }
1246 Some(old) if old != &routed => {
1247 appended += 1;
1248 changed += 1;
1249 true
1250 }
1251 Some(_) => false,
1252 };
1253 if should_append {
1254 records.push(LedgerRecord::Detection {
1255 recorded_at: observed_at,
1256 detection: routed.detection,
1257 route: routed.route,
1258 local_issue_path: routed.local_issue_path,
1259 eligible: routed.eligible,
1260 reconstructed_call: routed.reconstructed_call,
1261 reconstructed_call_path: routed.reconstructed_call_path,
1262 });
1263 }
1264 }
1265 let summary = TickSummary {
1266 started_at,
1267 completed_at: Utc::now(),
1268 route: route.route,
1269 source_checkout: route.source_checkout,
1270 refusal_reason: route.refusal_reason,
1271 events_scanned: evidence.events.len(),
1272 supervisor_agents,
1273 registry_agents: evidence.registry_agents,
1274 detections_found,
1275 appended,
1276 changed,
1277 suppressed_dismissed,
1278 filing_mode: if self.auto_fix.auto_fix
1279 && self.auto_fix_refusal_reason.is_none()
1280 && route.route == SelfhealRoute::Local
1281 {
1282 "pr-only"
1283 } else {
1284 "watch-only"
1285 }
1286 .to_string(),
1287 };
1288 records.push(LedgerRecord::Tick {
1289 summary: summary.clone(),
1290 });
1291 append_records(&self.ledger_path, &records)?;
1292 state.records.extend(records);
1293 Ok(summary)
1294 }
1295}
1296
1297pub fn spawn_selfheal_cadence(
1302 state: Arc<ServerState>,
1303 interval_secs: u64,
1304) -> tokio::task::JoinHandle<()> {
1305 tokio::spawn(async move {
1306 let mut ticker =
1307 tokio::time::interval(std::time::Duration::from_secs(interval_secs.max(1)));
1308 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1309 loop {
1310 ticker.tick().await;
1311 if let Err(error) = state.selfheal.run_tick(&state).await {
1312 tracing::warn!(target: "car::selfheal", %error, "self-heal detection tick failed");
1313 }
1314 }
1315 })
1316}
1317
1318async fn log_file_observation(path: &str) -> (Option<DateTime<Utc>>, Option<u64>) {
1319 let Ok(metadata) = tokio::fs::metadata(path).await else {
1320 return (None, None);
1321 };
1322 let modified_at = metadata.modified().ok().map(DateTime::<Utc>::from);
1323 (modified_at, Some(metadata.len()))
1324}
1325
1326async fn gather_live_evidence(
1327 server: &Arc<ServerState>,
1328 state_root: &Path,
1329 since: Option<DateTime<Utc>>,
1330 until: DateTime<Utc>,
1331) -> SelfhealEvidence {
1332 let sessions = server
1333 .sessions
1334 .lock()
1335 .await
1336 .values()
1337 .cloned()
1338 .collect::<Vec<_>>();
1339 let mut events = Vec::new();
1340 let mut metric_alerts_found = Vec::new();
1341 let mut metric_provenance = Vec::new();
1342 let thresholds = AlertThresholds {
1343 max_cost_usd: None,
1344 max_error_rate: Some(0.5),
1345 max_avg_latency_ms: None,
1346 max_goals_ungrounded: Some(0),
1347 min_actions: Some(5),
1348 };
1349 for session in sessions {
1350 let handle = session.runtime.event_log_handle();
1351 let log = handle.lock().await;
1352 let slice = log
1353 .events()
1354 .iter()
1355 .filter(|event| {
1356 since.is_none_or(|cutoff| event.timestamp > cutoff) && event.timestamp <= until
1357 })
1358 .cloned()
1359 .collect::<Vec<_>>();
1360 let proposal_context = since.map_or_else(Vec::new, |cutoff| {
1365 log.events()
1366 .iter()
1367 .filter(|event| {
1368 event.kind == EventKind::ProposalReceived
1369 && event.timestamp <= cutoff
1370 && event.timestamp <= until
1371 })
1372 .cloned()
1373 .collect::<Vec<_>>()
1374 });
1375 let summary = car_eventlog::summarize(&slice);
1376 metric_alerts_found.extend(car_eventlog::evaluate_alerts(&summary, &thresholds));
1377 let path = Some(format!("journals/{}.jsonl", session.client_id));
1378 metric_provenance.extend(slice.iter().map(|event| EvidenceSource {
1379 event_id: None,
1380 run_id: event.run_id.clone(),
1381 path: path.clone(),
1382 }));
1383 events.extend(slice.into_iter().map(|event| EventEvidence {
1384 event,
1385 path: path.clone(),
1386 }));
1387 events.extend(proposal_context.into_iter().map(|event| EventEvidence {
1388 event,
1389 path: path.clone(),
1390 }));
1391 }
1392
1393 let supervisor_snapshot = if let Some(supervisor) = server.supervisor_if_installed() {
1394 let managed = supervisor.list().await;
1395 let mut agents = Vec::with_capacity(managed.len());
1396 for agent in &managed {
1397 let activity = supervisor
1398 .read_log(
1399 &agent.spec.id,
1400 car_registry::supervisor::LogStream::Stdout,
1401 ACTIVITY_TAIL_LINES,
1402 0,
1403 )
1404 .await
1405 .ok();
1406 let stderr = supervisor
1407 .read_log(
1408 &agent.spec.id,
1409 car_registry::supervisor::LogStream::Stderr,
1410 STDERR_TAIL_LINES,
1411 0,
1412 )
1413 .await
1414 .ok();
1415 let (activity_modified_at, activity_bytes) = match activity.as_ref() {
1416 Some(tail) => log_file_observation(&tail.stdout_path).await,
1417 None => (None, None),
1418 };
1419 let stderr_bytes = match stderr.as_ref() {
1420 Some(tail) => log_file_observation(&tail.stderr_path).await.1,
1421 None => None,
1422 };
1423 let mut state = SupervisorAgentState::from_managed(
1424 agent,
1425 stderr
1426 .as_ref()
1427 .map(|tail| tail.stderr.join("\n"))
1428 .unwrap_or_default(),
1429 Some(format!("logs/{}.stderr.log", agent.spec.id)),
1430 );
1431 state.activity_tail = activity
1432 .as_ref()
1433 .map(|tail| tail.stdout.join("\n"))
1434 .unwrap_or_default();
1435 state.activity_path = Some(format!("logs/{}.stdout.log", agent.spec.id));
1436 state.activity_modified_at = activity_modified_at;
1437 state.activity_bytes = activity_bytes;
1438 state.stderr_bytes = stderr_bytes;
1439 agents.push(state);
1440 }
1441 Some(SupervisorSnapshot {
1442 captured_at: Utc::now(),
1443 agents,
1444 })
1445 } else if let Some(manifest) = server.observer_manifest_path() {
1446 car_registry::supervisor::Supervisor::list_from_manifest(manifest)
1447 .ok()
1448 .map(|managed| SupervisorSnapshot {
1449 captured_at: Utc::now(),
1450 agents: managed
1451 .iter()
1452 .map(|agent| SupervisorAgentState::from_managed(agent, "", None))
1453 .collect(),
1454 })
1455 } else {
1456 None
1457 };
1458
1459 SelfhealEvidence {
1460 events,
1461 metric_alerts: metric_alerts_found,
1462 metric_provenance,
1463 supervisor_snapshot,
1464 registry_agents: count_registry_agents(&state_root.join("registry")),
1465 }
1466}
1467
1468#[derive(Deserialize)]
1469struct SelfhealConfigFile {
1470 #[serde(default)]
1471 selfheal: Option<SelfhealConfigSection>,
1472}
1473
1474#[derive(Deserialize)]
1475struct SelfhealConfigSection {
1476 #[serde(default)]
1477 source_checkout: Option<PathBuf>,
1478 #[serde(default = "default_auto_fix")]
1479 auto_fix: bool,
1480 #[serde(default = "default_max_concurrent")]
1481 max_concurrent: u32,
1482 #[serde(default = "default_max_per_day")]
1483 max_per_day: u32,
1484 #[serde(default = "default_max_rounds_per_key")]
1485 max_rounds_per_key: u32,
1486}
1487
1488impl Default for SelfhealConfigSection {
1489 fn default() -> Self {
1490 Self {
1491 source_checkout: None,
1492 auto_fix: DEFAULT_AUTO_FIX,
1493 max_concurrent: DEFAULT_MAX_CONCURRENT,
1494 max_per_day: DEFAULT_MAX_PER_DAY,
1495 max_rounds_per_key: DEFAULT_MAX_ROUNDS_PER_KEY,
1496 }
1497 }
1498}
1499
1500#[derive(Debug, Clone)]
1501struct AutoFixConfig {
1502 auto_fix: bool,
1503 max_concurrent: u32,
1504 max_per_day: u32,
1505 max_rounds_per_key: u32,
1506}
1507
1508impl Default for AutoFixConfig {
1509 fn default() -> Self {
1510 Self {
1511 auto_fix: DEFAULT_AUTO_FIX,
1512 max_concurrent: DEFAULT_MAX_CONCURRENT,
1513 max_per_day: DEFAULT_MAX_PER_DAY,
1514 max_rounds_per_key: DEFAULT_MAX_ROUNDS_PER_KEY,
1515 }
1516 }
1517}
1518
1519impl AutoFixConfig {
1520 fn disabled() -> Self {
1521 Self {
1522 auto_fix: false,
1523 ..Self::default()
1524 }
1525 }
1526}
1527
1528const fn default_auto_fix() -> bool {
1529 DEFAULT_AUTO_FIX
1530}
1531
1532const fn default_max_concurrent() -> u32 {
1533 DEFAULT_MAX_CONCURRENT
1534}
1535
1536const fn default_max_per_day() -> u32 {
1537 DEFAULT_MAX_PER_DAY
1538}
1539
1540const fn default_max_rounds_per_key() -> u32 {
1541 DEFAULT_MAX_ROUNDS_PER_KEY
1542}
1543
1544fn read_auto_fix_config(path: &Path) -> Result<AutoFixConfig, String> {
1545 let text = match std::fs::read_to_string(path) {
1546 Ok(text) => text,
1547 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
1548 return Ok(AutoFixConfig::default())
1549 }
1550 Err(error) => {
1551 return Err(format!(
1552 "could not read self-heal config {}: {error}",
1553 path.display()
1554 ))
1555 }
1556 };
1557 let config: SelfhealConfigFile = toml::from_str(&text).map_err(|error| {
1558 format!(
1559 "could not parse self-heal config {}: {error}",
1560 path.display()
1561 )
1562 })?;
1563 let section = config.selfheal.unwrap_or_default();
1564 if section.max_concurrent != DEFAULT_MAX_CONCURRENT {
1565 return Err(format!(
1566 "[selfheal] max_concurrent must be 1, got {}",
1567 section.max_concurrent
1568 ));
1569 }
1570 if !(1..=DEFAULT_MAX_PER_DAY).contains(§ion.max_per_day) {
1571 return Err(format!(
1572 "[selfheal] max_per_day must be between 1 and {DEFAULT_MAX_PER_DAY}, got {}",
1573 section.max_per_day
1574 ));
1575 }
1576 if !(1..=DEFAULT_MAX_ROUNDS_PER_KEY).contains(§ion.max_rounds_per_key) {
1577 return Err(format!(
1578 "[selfheal] max_rounds_per_key must be between 1 and {DEFAULT_MAX_ROUNDS_PER_KEY}, got {}",
1579 section.max_rounds_per_key
1580 ));
1581 }
1582 Ok(AutoFixConfig {
1583 auto_fix: section.auto_fix,
1584 max_concurrent: section.max_concurrent,
1585 max_per_day: section.max_per_day,
1586 max_rounds_per_key: section.max_rounds_per_key,
1587 })
1588}
1589
1590fn read_source_checkout_config(path: &Path) -> Result<Option<PathBuf>, String> {
1591 let text = match std::fs::read_to_string(path) {
1592 Ok(text) => text,
1593 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
1594 Err(error) => {
1595 return Err(format!(
1596 "could not read self-heal config {}: {error}",
1597 path.display()
1598 ))
1599 }
1600 };
1601 let config: SelfhealConfigFile = toml::from_str(&text).map_err(|error| {
1602 format!(
1603 "could not parse self-heal config {}: {error}",
1604 path.display()
1605 )
1606 })?;
1607 Ok(config.selfheal.and_then(|section| section.source_checkout))
1608}
1609
1610fn dedup_paths(paths: &mut Vec<PathBuf>) {
1611 let mut seen = BTreeSet::new();
1612 paths.retain(|path| seen.insert(path.clone()));
1613}
1614
1615fn validate_source_checkout(candidate: &Path) -> Result<PathBuf, String> {
1616 if !candidate.is_dir() {
1617 return Err("directory does not exist".to_string());
1618 }
1619 let checkout = std::fs::canonicalize(candidate)
1620 .map_err(|error| format!("canonicalize candidate: {error}"))?;
1621 let dot_git = checkout.join(".git");
1622 if !dot_git.is_dir() && !dot_git.is_file() {
1623 return Err("missing .git directory or worktree file".to_string());
1624 }
1625 let config_path = git_config_path(&checkout, &dot_git)?;
1626 let config = std::fs::read_to_string(&config_path)
1627 .map_err(|error| format!("read git config {}: {error}", config_path.display()))?;
1628 let remote = origin_remote(&config)
1629 .ok_or_else(|| format!("git config {} has no origin remote", config_path.display()))?;
1630 if !is_expected_origin(&remote) {
1631 return Err(format!(
1632 "origin remote is '{}', expected {EXPECTED_ORIGIN}",
1633 remote_for_display(&remote)
1634 ));
1635 }
1636 Ok(checkout)
1637}
1638
1639fn git_config_path(checkout: &Path, dot_git: &Path) -> Result<PathBuf, String> {
1640 if dot_git.is_dir() {
1641 return Ok(dot_git.join("config"));
1642 }
1643 let marker = std::fs::read_to_string(dot_git)
1644 .map_err(|error| format!("read worktree marker {}: {error}", dot_git.display()))?;
1645 let raw_git_dir = marker
1646 .lines()
1647 .find_map(|line| line.trim().strip_prefix("gitdir:"))
1648 .map(str::trim)
1649 .filter(|path| !path.is_empty())
1650 .ok_or_else(|| format!("invalid worktree marker {}", dot_git.display()))?;
1651 let git_dir = resolve_relative(checkout, Path::new(raw_git_dir));
1652 let common_dir_path = git_dir.join("commondir");
1653 if let Ok(raw_common_dir) = std::fs::read_to_string(&common_dir_path) {
1654 let common_dir = resolve_relative(&git_dir, Path::new(raw_common_dir.trim()));
1655 Ok(common_dir.join("config"))
1656 } else {
1657 Ok(git_dir.join("config"))
1658 }
1659}
1660
1661fn resolve_relative(base: &Path, path: &Path) -> PathBuf {
1662 if path.is_absolute() {
1663 path.to_path_buf()
1664 } else {
1665 base.join(path)
1666 }
1667}
1668
1669fn origin_remote(config: &str) -> Option<String> {
1670 let mut in_origin = false;
1671 for raw_line in config.lines() {
1672 let line = raw_line.trim();
1673 if line.starts_with('[') && line.ends_with(']') {
1674 in_origin = line.eq_ignore_ascii_case(r#"[remote "origin"]"#);
1675 continue;
1676 }
1677 if !in_origin || line.starts_with('#') || line.starts_with(';') {
1678 continue;
1679 }
1680 let Some((key, value)) = line.split_once('=') else {
1681 continue;
1682 };
1683 if key.trim().eq_ignore_ascii_case("url") {
1684 return Some(value.trim().trim_matches('"').to_string());
1685 }
1686 }
1687 None
1688}
1689
1690fn remote_for_display(remote: &str) -> String {
1691 let Some((scheme, rest)) = remote.split_once("://") else {
1692 return remote.to_string();
1693 };
1694 let Some((authority, tail)) = rest.split_once('/') else {
1695 return remote.to_string();
1696 };
1697 if authority.contains('@') {
1698 format!(
1699 "{scheme}://[REDACTED]@{}/{tail}",
1700 authority.rsplit('@').next().unwrap_or(authority)
1701 )
1702 } else {
1703 remote.to_string()
1704 }
1705}
1706
1707fn is_expected_origin(remote: &str) -> bool {
1708 let remote = remote.trim().trim_end_matches('/').trim_end_matches(".git");
1709 let (host, path) = if remote.contains("://") {
1710 let without_scheme = remote.split_once("://").map_or(remote, |(_, rest)| rest);
1711 let Some((host, path)) = without_scheme.split_once('/') else {
1712 return false;
1713 };
1714 (host.rsplit('@').next().unwrap_or(host), path)
1715 } else if let Some((left, path)) = remote.split_once(':') {
1716 (left.rsplit('@').next().unwrap_or(left), path)
1717 } else {
1718 let Some((host, path)) = remote.split_once('/') else {
1719 return false;
1720 };
1721 (host.rsplit('@').next().unwrap_or(host), path)
1722 };
1723 host.eq_ignore_ascii_case("github.com") && path.eq_ignore_ascii_case(EXPECTED_ORIGIN)
1724}
1725
1726#[derive(Debug)]
1727struct CallAssessment {
1728 eligible: bool,
1729 reconstructed_call: Option<ReconstructedToolCall>,
1730}
1731
1732fn assess_reconstructed_call(
1733 detection: &Detection,
1734 events: &[EventEvidence],
1735 redactor: &Redactor,
1736) -> Option<CallAssessment> {
1737 if detection.kind() != DetectionKind::RecurringToolFailure {
1738 return None;
1739 }
1740 let Some((builtin, reconstructed_call)) = reconstruct_failed_call(detection, events) else {
1741 return Some(CallAssessment {
1742 eligible: false,
1743 reconstructed_call: None,
1744 });
1745 };
1746 if !params_pass_redactor(&reconstructed_call.params, redactor) {
1747 return Some(CallAssessment {
1748 eligible: false,
1749 reconstructed_call: None,
1750 });
1751 }
1752 Some(CallAssessment {
1753 eligible: auto_fix_tool_allowed(&reconstructed_call.tool, builtin),
1754 reconstructed_call: Some(reconstructed_call),
1755 })
1756}
1757
1758fn auto_fix_tool_allowed(tool: &str, builtin: bool) -> bool {
1763 builtin && tool != "delegate_gui"
1764}
1765
1766fn params_pass_redactor(params: &BTreeMap<String, serde_json::Value>, redactor: &Redactor) -> bool {
1767 let Ok(params_json) = serde_json::to_string(params) else {
1768 return false;
1769 };
1770 redactor.redact(¶ms_json) == params_json
1771 && params.iter().all(|(key, value)| {
1772 redactor.redact(key) == *key && json_value_passes_redactor(value, redactor)
1773 })
1774}
1775
1776fn json_value_passes_redactor(value: &serde_json::Value, redactor: &Redactor) -> bool {
1777 match value {
1778 serde_json::Value::String(value) => redactor.redact(value) == *value,
1779 serde_json::Value::Array(values) => values
1780 .iter()
1781 .all(|value| json_value_passes_redactor(value, redactor)),
1782 serde_json::Value::Object(values) => values.iter().all(|(key, value)| {
1783 redactor.redact(key) == *key && json_value_passes_redactor(value, redactor)
1784 }),
1785 serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => true,
1786 }
1787}
1788
1789fn reconstruct_failed_call(
1790 detection: &Detection,
1791 events: &[EventEvidence],
1792) -> Option<(bool, ReconstructedToolCall)> {
1793 let failure = events
1794 .iter()
1795 .filter(|record| record.event.kind == EventKind::ActionFailed)
1796 .filter(|record| failure_belongs_to_detection(detection, record))
1797 .max_by(|left, right| {
1798 left.event
1799 .timestamp
1800 .cmp(&right.event.timestamp)
1801 .then_with(|| left.event.proposal_id.cmp(&right.event.proposal_id))
1802 .then_with(|| left.event.action_id.cmp(&right.event.action_id))
1803 })?;
1804 let journal = failure.path.as_deref()?;
1805 let proposal_id = failure.event.proposal_id.as_deref()?;
1806 let action_id = failure.event.action_id.as_deref()?;
1807 let failed_tool = failure.event.data.get("tool")?.as_str()?;
1808 let builtin = failure
1809 .event
1810 .data
1811 .get("tool_source")
1812 .and_then(serde_json::Value::as_str)
1813 == Some("builtin");
1814
1815 let proposal_event = events
1816 .iter()
1817 .filter(|record| {
1818 record.event.kind == EventKind::ProposalReceived
1819 && record.path.as_deref() == Some(journal)
1820 && record.event.proposal_id.as_deref() == Some(proposal_id)
1821 && record.event.timestamp <= failure.event.timestamp
1822 })
1823 .max_by_key(|record| record.event.timestamp)?;
1824 let proposal: ActionProposal =
1825 serde_json::from_value(proposal_event.event.data.get("proposal")?.clone()).ok()?;
1826 if proposal.id != proposal_id {
1827 return None;
1828 }
1829 let action = proposal
1830 .actions
1831 .into_iter()
1832 .find(|action| action.id == action_id)?;
1833 let tool = action.tool?;
1834 if tool != failed_tool {
1835 return None;
1836 }
1837 Some((
1838 builtin,
1839 ReconstructedToolCall {
1840 tool,
1841 params: action.parameters.into_iter().collect(),
1842 },
1843 ))
1844}
1845
1846fn failure_belongs_to_detection(detection: &Detection, failure: &EventEvidence) -> bool {
1847 let Some(journal) = failure.path.as_deref() else {
1848 return false;
1849 };
1850 detection.provenance().iter().any(|provenance| {
1851 provenance.path() == Some(journal)
1852 && provenance.run_id() == failure.event.run_id.as_deref()
1853 && provenance.event_id().is_some_and(|event_id| {
1854 failure.event.hash.as_deref() == Some(event_id)
1855 || failure.event.action_id.as_deref() == Some(event_id)
1856 })
1857 })
1858}
1859
1860#[derive(Serialize)]
1861struct PersistedReconstructedCall<'a> {
1862 eligible: bool,
1863 reconstructed_call: Option<&'a ReconstructedToolCall>,
1864}
1865
1866fn persist_reconstructed_call(
1867 path: &Path,
1868 eligible: bool,
1869 reconstructed_call: Option<&ReconstructedToolCall>,
1870) -> Result<(), String> {
1871 let parent = path
1872 .parent()
1873 .ok_or_else(|| "reconstructed self-heal call path has no parent".to_string())?;
1874 car_secrets::ensure_private_dir(parent)
1875 .map_err(|error| format!("create reconstructed self-heal call directory: {error}"))?;
1876 let mut bytes = serde_json::to_vec_pretty(&PersistedReconstructedCall {
1877 eligible,
1878 reconstructed_call,
1879 })
1880 .map_err(|error| format!("serialize reconstructed self-heal call: {error}"))?;
1881 bytes.push(b'\n');
1882 let mut file = if path.exists() {
1883 car_secrets::open_private_truncate(path)
1884 } else {
1885 car_secrets::create_private_file(path)
1886 }
1887 .map_err(|error| {
1888 format!(
1889 "open reconstructed self-heal call {}: {error}",
1890 path.display()
1891 )
1892 })?;
1893 file.write_all(&bytes)
1894 .map_err(|error| format!("write reconstructed self-heal call: {error}"))?;
1895 file.flush()
1896 .map_err(|error| format!("flush reconstructed self-heal call: {error}"))?;
1897 file.sync_all()
1898 .map_err(|error| format!("sync reconstructed self-heal call: {error}"))?;
1899 car_secrets::revalidate_private_path(path, &file)
1900 .map_err(|error| format!("revalidate reconstructed self-heal call: {error}"))
1901}
1902
1903fn persist_private_json<T: Serialize>(path: &Path, value: &T) -> Result<(), String> {
1904 let mut bytes = serde_json::to_vec_pretty(value)
1905 .map_err(|error| format!("serialize private self-heal artifact: {error}"))?;
1906 bytes.push(b'\n');
1907 persist_private_bytes(path, &bytes)
1908}
1909
1910fn persist_private_bytes(path: &Path, bytes: &[u8]) -> Result<(), String> {
1911 let parent = path
1912 .parent()
1913 .ok_or_else(|| "private self-heal artifact path has no parent".to_string())?;
1914 car_secrets::ensure_private_dir(parent)
1915 .map_err(|error| format!("create private self-heal artifact directory: {error}"))?;
1916 let mut file = if path.exists() {
1917 car_secrets::open_private_truncate(path)
1918 } else {
1919 car_secrets::create_private_file(path)
1920 }
1921 .map_err(|error| {
1922 format!(
1923 "open private self-heal artifact {}: {error}",
1924 path.display()
1925 )
1926 })?;
1927 file.write_all(bytes)
1928 .map_err(|error| format!("write private self-heal artifact: {error}"))?;
1929 file.flush()
1930 .map_err(|error| format!("flush private self-heal artifact: {error}"))?;
1931 file.sync_all()
1932 .map_err(|error| format!("sync private self-heal artifact: {error}"))?;
1933 car_secrets::revalidate_private_path(path, &file)
1934 .map_err(|error| format!("revalidate private self-heal artifact: {error}"))
1935}
1936
1937fn resolve_car_cli_executable() -> Option<PathBuf> {
1938 let executable_name = if cfg!(windows) { "car.exe" } else { "car" };
1939 if let Ok(current) = std::env::current_exe() {
1940 if let Some(sibling) = current.parent().map(|parent| parent.join(executable_name)) {
1941 if sibling.is_file() {
1942 return Some(sibling);
1943 }
1944 }
1945 }
1946 std::env::var_os("PATH").and_then(|path| {
1947 std::env::split_paths(&path)
1948 .map(|entry| entry.join(executable_name))
1949 .find(|candidate| candidate.is_file())
1950 })
1951}
1952
1953#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1954enum RemoteDedupResult {
1955 Proceed,
1956 AwaitingReview,
1957 Parked,
1958}
1959
1960#[derive(Debug, Deserialize)]
1961struct RemoteSelfhealPr {
1962 number: u64,
1963 url: String,
1964 state: String,
1965 #[serde(rename = "mergedAt")]
1966 merged_at: Option<String>,
1967 body: String,
1968}
1969
1970async fn check_checkout_replay_verb(checkout: &Path, target_dir: &Path) -> Result<(), String> {
1971 car_secrets::ensure_private_dir(target_dir)
1972 .map_err(|error| format!("create replay probe target: {error}"))?;
1973 let build = tokio::process::Command::new("cargo")
1974 .current_dir(checkout.join("car-rs"))
1975 .args(["build", "-p", "car-cli"])
1976 .env("CARGO_TARGET_DIR", target_dir)
1977 .kill_on_drop(true)
1978 .output()
1979 .await
1980 .map_err(|error| format!("could not build checkout car CLI: {error}"))?;
1981 if !build.status.success() {
1982 return Err(format!(
1983 "checkout car CLI build failed with exit {}: {}",
1984 build.status.code().unwrap_or(-1),
1985 output_tail(&build.stderr, 8)
1986 ));
1987 }
1988
1989 #[cfg(windows)]
1990 let car = target_dir.join("debug/car.exe");
1991 #[cfg(not(windows))]
1992 let car = target_dir.join("debug/car");
1993 let help = tokio::process::Command::new(&car)
1994 .args(["tools", "call", "--help"])
1995 .kill_on_drop(true)
1996 .output()
1997 .await
1998 .map_err(|error| format!("checkout-built car has no runnable replay verb: {error}"))?;
1999 if help.status.success() {
2000 Ok(())
2001 } else {
2002 Err(format!(
2003 "checkout-built `car tools call --help` exited {}: {}",
2004 help.status.code().unwrap_or(-1),
2005 output_tail(&help.stderr, 8)
2006 ))
2007 }
2008}
2009
2010async fn query_remote_selfheal_pr(
2011 checkout: &Path,
2012 dedup_key: &str,
2013) -> Result<Option<RemoteSelfhealPr>, String> {
2014 let head = format!("car/selfheal/{dedup_key}");
2015 let output = tokio::process::Command::new("gh")
2016 .current_dir(checkout)
2017 .args([
2018 "pr",
2019 "list",
2020 "--repo",
2021 "Parslee-ai/car",
2022 "--head",
2023 &head,
2024 "--state",
2025 "all",
2026 "--json",
2027 "number,url,state,mergedAt,body",
2028 ])
2029 .kill_on_drop(true)
2030 .output()
2031 .await
2032 .map_err(|error| format!("remote self-heal PR lookup could not start: {error}"))?;
2033 if !output.status.success() {
2034 return Err(format!(
2035 "remote self-heal PR lookup failed with exit {}: {}",
2036 output.status.code().unwrap_or(-1),
2037 output_tail(&output.stderr, 8)
2038 ));
2039 }
2040 let mut prs: Vec<RemoteSelfhealPr> = serde_json::from_slice(&output.stdout)
2041 .map_err(|error| format!("parse remote self-heal PR lookup: {error}"))?;
2042 let marker = format!("<!-- car-selfheal:key={dedup_key} -->");
2043 if prs.iter().any(|pr| !pr.body.contains(&marker)) {
2044 return Err(format!(
2045 "remote branch '{head}' has a PR without the trusted self-heal marker"
2046 ));
2047 }
2048 prs.sort_by_key(|pr| if pr.state == "OPEN" { 0 } else { 1 });
2049 Ok(prs.into_iter().next())
2050}
2051
2052fn output_tail(bytes: &[u8], lines: usize) -> String {
2053 let text = String::from_utf8_lossy(bytes);
2054 text.lines()
2055 .rev()
2056 .take(lines)
2057 .collect::<Vec<_>>()
2058 .into_iter()
2059 .rev()
2060 .collect::<Vec<_>>()
2061 .join("\n")
2062}
2063
2064fn classify_code_task_output(stdout: &[u8]) -> String {
2065 code_task_failure_class(stdout).unwrap_or_else(|| "missing_run_end".to_string())
2066}
2067
2068fn code_task_failure_class(stdout: &[u8]) -> Option<String> {
2069 String::from_utf8_lossy(stdout)
2070 .lines()
2071 .rev()
2072 .filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
2073 .find(|event| event.get("type").and_then(serde_json::Value::as_str) == Some("run_end"))
2074 .and_then(|event| {
2075 event
2076 .get("failure_class")
2077 .and_then(serde_json::Value::as_str)
2078 .map(str::to_string)
2079 })
2080}
2081
2082fn route_name(route: SelfhealRoute) -> &'static str {
2083 match route {
2084 SelfhealRoute::Local => "local",
2085 SelfhealRoute::Feedback => "feedback",
2086 SelfhealRoute::LedgerOnly => "ledger-only",
2087 }
2088}
2089
2090fn render_local_issue(
2091 path: &Path,
2092 detection: &Detection,
2093 eligible: Option<bool>,
2094 ineligible_reason: Option<&str>,
2095) -> Result<(), String> {
2096 let occurrence_count = local_issue_occurrence_count(path)?.saturating_add(1);
2097 let parent = path
2098 .parent()
2099 .ok_or_else(|| "local self-heal issue path has no parent".to_string())?;
2100 car_secrets::ensure_private_dir(parent)
2101 .map_err(|error| format!("create local self-heal issue directory: {error}"))?;
2102
2103 let event_ids = detection
2104 .provenance()
2105 .iter()
2106 .filter_map(|source| source.event_id())
2107 .collect::<Vec<_>>();
2108 let run_ids = detection
2109 .provenance()
2110 .iter()
2111 .filter_map(|source| source.run_id())
2112 .collect::<Vec<_>>();
2113 let paths = detection
2114 .provenance()
2115 .iter()
2116 .filter_map(|source| source.path())
2117 .collect::<Vec<_>>();
2118 let evidence = if detection.evidence().is_empty() {
2119 "- (none)".to_string()
2120 } else {
2121 detection
2122 .evidence()
2123 .iter()
2124 .map(|excerpt| format!("- {excerpt}"))
2125 .collect::<Vec<_>>()
2126 .join("\n")
2127 };
2128 let issue = format!(
2129 "# CAR self-heal detection: {}\n\n\
2130Trust-Tier: {TRUST_TIER}\n\
2131Dedup-Key: {}\n\
2132Detector-ID: {}\n\
2133Severity: {:?}\n\
2134Route: local\n\
2135Auto-Fix-Eligible: {}\n\
2136Auto-Fix-Ineligible-Reason: {}\n\
2137Occurrence-Count: {occurrence_count}\n\
2138First-Observed: {}\n\
2139Last-Observed: {}\n\n\
2140## Provenance\n\n\
2141- CAR-Version: {}\n\
2142- Platform: {}/{}\n\
2143- Event-IDs: {}\n\
2144- Run-IDs: {}\n\
2145- Evidence-Paths: {}\n\n\
2146## REDACTED Evidence Excerpt\n\n\
2147{evidence}\n\n\
2148## Repro Hints\n\n\
2149- Re-run `selfheal.run` and correlate the detector identity and provenance above.\n\
2150- Inspect the named local CAR evidence source; do not send it off-machine.\n",
2151 detection.locator(),
2152 detection.dedup_key(),
2153 detection.detector_id(),
2154 detection.severity(),
2155 eligible.map_or("not-applicable", |value| if value {
2156 "true"
2157 } else {
2158 "false"
2159 }),
2160 ineligible_reason.unwrap_or("(none)"),
2161 detection.first_observed_at().to_rfc3339(),
2162 detection.last_observed_at().to_rfc3339(),
2163 detection.car_version(),
2164 std::env::consts::OS,
2165 std::env::consts::ARCH,
2166 display_list(&event_ids),
2167 display_list(&run_ids),
2168 display_list(&paths),
2169 );
2170
2171 let mut file = if path.exists() {
2172 car_secrets::open_private_truncate(path)
2173 } else {
2174 car_secrets::create_private_file(path)
2175 }
2176 .map_err(|error| format!("open local self-heal issue {}: {error}", path.display()))?;
2177 file.write_all(issue.as_bytes())
2178 .map_err(|error| format!("write local self-heal issue: {error}"))?;
2179 file.flush()
2180 .map_err(|error| format!("flush local self-heal issue: {error}"))?;
2181 file.sync_all()
2182 .map_err(|error| format!("sync local self-heal issue: {error}"))?;
2183 car_secrets::revalidate_private_path(path, &file)
2184 .map_err(|error| format!("revalidate local self-heal issue: {error}"))
2185}
2186
2187fn local_issue_occurrence_count(path: &Path) -> Result<u64, String> {
2188 let mut file = match car_secrets::open_private_read(path) {
2189 Ok(file) => file,
2190 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
2191 Err(error) => {
2192 return Err(format!(
2193 "open existing local self-heal issue {}: {error}",
2194 path.display()
2195 ))
2196 }
2197 };
2198 let mut text = String::new();
2199 file.read_to_string(&mut text)
2200 .map_err(|error| format!("read existing local self-heal issue: {error}"))?;
2201 text.lines()
2202 .find_map(|line| line.strip_prefix("Occurrence-Count: "))
2203 .ok_or_else(|| {
2204 format!(
2205 "existing local self-heal issue {} has no occurrence count",
2206 path.display()
2207 )
2208 })?
2209 .parse::<u64>()
2210 .map_err(|error| format!("parse local self-heal issue occurrence count: {error}"))
2211}
2212
2213fn display_list(values: &[&str]) -> String {
2214 if values.is_empty() {
2215 "(none)".to_string()
2216 } else {
2217 values.join(", ")
2218 }
2219}
2220
2221fn count_registry_agents(registry_dir: &Path) -> usize {
2222 let Ok(entries) = std::fs::read_dir(registry_dir) else {
2223 return 0;
2224 };
2225 entries
2226 .filter_map(Result::ok)
2227 .filter(|entry| entry.path().extension().and_then(|ext| ext.to_str()) == Some("json"))
2228 .filter(|entry| {
2229 std::fs::read(entry.path())
2230 .ok()
2231 .and_then(|bytes| serde_json::from_slice::<car_registry::AgentEntry>(&bytes).ok())
2232 .is_some()
2233 })
2234 .count()
2235}
2236
2237fn fold_detections(
2238 records: &[LedgerRecord],
2239 max_rounds_per_key: u32,
2240) -> (BTreeMap<String, RoutedDetection>, BTreeSet<String>) {
2241 let mut detections = BTreeMap::new();
2242 let mut dismissed = BTreeSet::new();
2243 for record in records {
2244 match record {
2245 LedgerRecord::Detection {
2246 detection,
2247 route,
2248 local_issue_path,
2249 eligible,
2250 reconstructed_call,
2251 reconstructed_call_path,
2252 ..
2253 } => {
2254 detections.insert(
2255 detection.dedup_key().to_string(),
2256 RoutedDetection {
2257 detection: detection.clone(),
2258 route: *route,
2259 local_issue_path: local_issue_path.clone(),
2260 eligible: *eligible,
2261 reconstructed_call: reconstructed_call.clone(),
2262 reconstructed_call_path: reconstructed_call_path.clone(),
2263 auto_fix_attempts: 0,
2264 auto_fix_exhausted: false,
2265 auto_fix_in_progress: false,
2266 last_auto_fix_attempt: None,
2267 auto_fix_awaiting_review: false,
2268 auto_fix_parked: false,
2269 remote_pr_number: None,
2270 remote_pr_url: None,
2271 },
2272 );
2273 }
2274 LedgerRecord::Dismissal { dedup_key, .. } => {
2275 dismissed.insert(dedup_key.clone());
2276 }
2277 LedgerRecord::Tick { .. }
2278 | LedgerRecord::FixStarted { .. }
2279 | LedgerRecord::FixAttempt { .. }
2280 | LedgerRecord::RemotePr { .. } => {}
2281 }
2282 }
2283 for (key, detection) in &mut detections {
2284 apply_attempt_state(detection, records, max_rounds_per_key);
2285 apply_remote_pr_state(detection, records);
2286 if dismissed.contains(key) {
2287 detection.auto_fix_in_progress = false;
2288 }
2289 }
2290 for key in &dismissed {
2291 detections.remove(key);
2292 }
2293 (detections, dismissed)
2294}
2295
2296fn apply_attempt_state(
2297 detection: &mut RoutedDetection,
2298 records: &[LedgerRecord],
2299 max_rounds_per_key: u32,
2300) {
2301 let key = detection.dedup_key().to_string();
2302 let started = records
2303 .iter()
2304 .filter_map(|record| match record {
2305 LedgerRecord::FixStarted {
2306 dedup_key, round, ..
2307 } if dedup_key == &key => Some(*round),
2308 _ => None,
2309 })
2310 .collect::<BTreeSet<_>>();
2311 let finished = records
2312 .iter()
2313 .filter_map(|record| match record {
2314 LedgerRecord::FixAttempt { result } if result.dedup_key == key => Some(result.round),
2315 _ => None,
2316 })
2317 .collect::<BTreeSet<_>>();
2318 detection.auto_fix_attempts = u32::try_from(started.len()).unwrap_or(u32::MAX);
2319 detection.auto_fix_exhausted = detection.auto_fix_attempts >= max_rounds_per_key;
2320 detection.auto_fix_in_progress = started.iter().any(|round| !finished.contains(round));
2321 detection.last_auto_fix_attempt = records.iter().rev().find_map(|record| match record {
2322 LedgerRecord::FixAttempt { result } if result.dedup_key == key => Some(result.clone()),
2323 _ => None,
2324 });
2325}
2326
2327fn apply_remote_pr_state(detection: &mut RoutedDetection, records: &[LedgerRecord]) {
2328 let key = detection.dedup_key().to_string();
2329 if let Some((disposition, number, url)) = records.iter().rev().find_map(|record| match record {
2330 LedgerRecord::RemotePr {
2331 dedup_key,
2332 disposition,
2333 pr_number,
2334 pr_url,
2335 ..
2336 } if dedup_key == &key => Some((*disposition, *pr_number, pr_url.clone())),
2337 _ => None,
2338 }) {
2339 detection.auto_fix_awaiting_review = disposition == RemotePrDisposition::AwaitingReview;
2340 detection.auto_fix_parked = disposition == RemotePrDisposition::Parked;
2341 detection.remote_pr_number = Some(number);
2342 detection.remote_pr_url = Some(url);
2343 }
2344}
2345
2346fn starts_today(records: &[LedgerRecord], now: DateTime<Utc>) -> u32 {
2347 u32::try_from(
2348 records
2349 .iter()
2350 .filter(|record| {
2351 matches!(
2352 record,
2353 LedgerRecord::FixStarted { started_at, .. }
2354 if started_at.date_naive() == now.date_naive()
2355 )
2356 })
2357 .count(),
2358 )
2359 .unwrap_or(u32::MAX)
2360}
2361
2362fn validate_dedup_key(key: &str) -> Result<(), String> {
2363 if key.len() == 64 && key.bytes().all(|byte| byte.is_ascii_hexdigit()) {
2364 Ok(())
2365 } else {
2366 Err("dedup_key must be a 64-character SHA-256 hex string".to_string())
2367 }
2368}
2369
2370fn load_records(path: &Path) -> Result<Vec<LedgerRecord>, String> {
2371 let file = match car_secrets::open_private_read(path) {
2372 Ok(file) => file,
2373 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
2374 Err(error) => return Err(format!("open self-heal ledger: {error}")),
2375 };
2376 car_secrets::revalidate_private_path(path, &file)
2377 .map_err(|error| format!("validate self-heal ledger: {error}"))?;
2378 let mut records = Vec::new();
2379 for (index, line) in BufReader::new(file).lines().enumerate() {
2380 let line =
2381 line.map_err(|error| format!("read self-heal ledger line {}: {error}", index + 1))?;
2382 if line.trim().is_empty() {
2383 continue;
2384 }
2385 records.push(
2386 serde_json::from_str(&line)
2387 .map_err(|error| format!("parse self-heal ledger line {}: {error}", index + 1))?,
2388 );
2389 }
2390 Ok(records)
2391}
2392
2393fn append_records(path: &Path, records: &[LedgerRecord]) -> Result<(), String> {
2394 if records.is_empty() {
2395 return Ok(());
2396 }
2397 let parent = path
2398 .parent()
2399 .ok_or_else(|| "self-heal ledger path has no parent".to_string())?;
2400 car_secrets::ensure_private_dir(parent)
2401 .map_err(|error| format!("create self-heal ledger directory: {error}"))?;
2402 let mut file = car_secrets::open_private_append(path)
2403 .map_err(|error| format!("open self-heal ledger for append: {error}"))?;
2404 let original_len = file
2405 .metadata()
2406 .map_err(|error| format!("stat self-heal ledger: {error}"))?
2407 .len();
2408 let write_result = (|| -> Result<(), String> {
2409 for record in records {
2410 serde_json::to_writer(&mut file, record)
2411 .map_err(|error| format!("serialize self-heal ledger record: {error}"))?;
2412 file.write_all(b"\n")
2413 .map_err(|error| format!("append self-heal ledger newline: {error}"))?;
2414 }
2415 file.flush()
2416 .map_err(|error| format!("flush self-heal ledger: {error}"))?;
2417 file.sync_all()
2418 .map_err(|error| format!("sync self-heal ledger: {error}"))?;
2419 car_secrets::revalidate_private_path(path, &file)
2420 .map_err(|error| format!("revalidate self-heal ledger: {error}"))?;
2421 Ok(())
2422 })();
2423 if let Err(error) = write_result {
2424 let _ = file.set_len(original_len);
2425 return Err(error);
2426 }
2427 Ok(())
2428}
2429
2430#[cfg(test)]
2431mod tests {
2432 use super::*;
2433
2434 #[test]
2435 fn auto_fix_config_defaults_on_and_refuses_raised_hard_limits() {
2436 let temp = tempfile::TempDir::new().unwrap();
2437 let path = temp.path().join("config.toml");
2438 let default = read_auto_fix_config(&path).unwrap();
2439 assert!(default.auto_fix);
2440 assert_eq!(default.max_concurrent, 1);
2441 assert_eq!(default.max_per_day, 3);
2442 assert_eq!(default.max_rounds_per_key, 3);
2443
2444 std::fs::write(
2445 &path,
2446 "[selfheal]\nauto_fix=false\nmax_per_day=2\nmax_rounds_per_key=1\n",
2447 )
2448 .unwrap();
2449 let lowered = read_auto_fix_config(&path).unwrap();
2450 assert!(!lowered.auto_fix);
2451 assert_eq!(lowered.max_per_day, 2);
2452 assert_eq!(lowered.max_rounds_per_key, 1);
2453
2454 for (config, refusal) in [
2455 ("[selfheal]\nmax_concurrent=2\n", "max_concurrent must be 1"),
2456 (
2457 "[selfheal]\nmax_per_day=4\n",
2458 "max_per_day must be between 1 and 3",
2459 ),
2460 (
2461 "[selfheal]\nmax_rounds_per_key=4\n",
2462 "max_rounds_per_key must be between 1 and 3",
2463 ),
2464 ] {
2465 std::fs::write(&path, config).unwrap();
2466 assert!(read_auto_fix_config(&path).unwrap_err().contains(refusal));
2467 }
2468 }
2469
2470 #[test]
2471 fn auto_fix_excludes_side_effecting_gui_delegation_even_when_builtin() {
2472 assert!(auto_fix_tool_allowed("read_file", true));
2473 assert!(!auto_fix_tool_allowed("delegate_gui", true));
2474 assert!(!auto_fix_tool_allowed("read_file", false));
2475 }
2476
2477 #[test]
2478 fn code_task_terminal_failure_class_comes_from_run_end_record() {
2479 assert_eq!(
2480 code_task_failure_class(
2481 b"{\"type\":\"progress\"}\n{\"type\":\"run_end\",\"failure_class\":\"contract_not_green\"}\n"
2482 )
2483 .as_deref(),
2484 Some("contract_not_green")
2485 );
2486 assert!(code_task_failure_class(b"not json\n").is_none());
2487 assert_eq!(classify_code_task_output(b"not json\n"), "missing_run_end");
2488 }
2489}