rch_common/
incident.rs

1//! Shared incident event schema and stable reason-code registry.
2//!
3//! Every RCH subsystem that can steer, reject, defer, or fall back a build —
4//! selection, admission, fallback, proof, doctor, telemetry, artifact
5//! retrieval, and worker lifecycle — emits a uniform [`IncidentEvent`] keyed by
6//! a stable [`IncidentReasonCode`]. A single registry prevents free-form reason
7//! drift across components and gives postmortems one vocabulary.
8//!
9//! The reason-code registry is intentionally 1:1 with the operator-facing
10//! failure classes recorded in the session-history report (e.g. "no admissible
11//! workers", "critical pressure", "disk full"). Each variant exposes:
12//! - a stable `RCH-Innn` [`code`](IncidentReasonCode::code) (the serialized form),
13//! - the exact operator-facing [`failure_class`](IncidentReasonCode::failure_class)
14//!   string, and
15//! - an optional mapped AGENTS-style [`ErrorCode`](IncidentReasonCode::error_code)
16//!   for components that already speak the catalog.
17
18use std::collections::BTreeMap;
19use std::fmt;
20
21use serde::{Deserialize, Serialize};
22
23use crate::errors::ErrorCode;
24use crate::schema_versions::{SchemaComponent, current_version};
25
26/// Stable reason-code registry. One variant per session-history failure class.
27///
28/// Serializes to its canonical `RCH-Innn` string and deserializes the same way,
29/// so the wire form is decoupled from the Rust variant name.
30#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
31pub enum IncidentReasonCode {
32    /// No worker passed admission (all candidates rejected).
33    NoAdmissibleWorkers,
34    /// A worker (or the pool) was under critical resource pressure.
35    CriticalPressure,
36    /// Not enough free build slots to admit the request.
37    InsufficientSlots,
38    /// A hard preflight check rejected the candidate.
39    HardPreflight,
40    /// The active project root was excluded from offload.
41    ActiveProjectExclusion,
42    /// A required runtime, toolchain, or Rust target was missing.
43    MissingRuntimeToolchainTarget,
44    /// The worker's OS/arch did not match the artifact/target triple.
45    OsArchMismatch,
46    /// Telemetry was stale or its age could not be determined.
47    TelemetryStale,
48    /// The worker's circuit breaker was open.
49    CircuitOpen,
50    /// The daemon Unix socket refused the connection.
51    DaemonSocketRefused,
52    /// The build fell back to local execution.
53    LocalFallback,
54    /// Proof-mode fail-closed policy refused to proceed.
55    ProofRefusal,
56    /// rsync reported a source file that vanished during transfer.
57    RsyncVanishedFile,
58    /// An expected build artifact was missing on retrieval.
59    ArtifactMiss,
60    /// Ambiguous queue/job identity prevented a clean correlation.
61    QueueAmbiguity,
62    /// The target disk was full.
63    DiskFull,
64    /// A wrong-user or wrong-path/arch `rch-wkr` binary was detected.
65    WrongUserPathWorkerBinary,
66    /// A self-test canary built successfully but its bytes differ from the
67    /// orchestrator's reference because the worker's toolchain differs — a
68    /// healthy worker on a heterogeneous fleet, not a failure (bd-784xt).
69    /// Advisory only; distinct from a genuine same-toolchain miscompile.
70    ToolchainDrift,
71}
72
73impl IncidentReasonCode {
74    /// Every reason code, in stable declaration order. Used for registry
75    /// coverage/uniqueness tests and reason enumeration.
76    pub const ALL: &'static [IncidentReasonCode] = &[
77        Self::NoAdmissibleWorkers,
78        Self::CriticalPressure,
79        Self::InsufficientSlots,
80        Self::HardPreflight,
81        Self::ActiveProjectExclusion,
82        Self::MissingRuntimeToolchainTarget,
83        Self::OsArchMismatch,
84        Self::TelemetryStale,
85        Self::CircuitOpen,
86        Self::DaemonSocketRefused,
87        Self::LocalFallback,
88        Self::ProofRefusal,
89        Self::RsyncVanishedFile,
90        Self::ArtifactMiss,
91        Self::QueueAmbiguity,
92        Self::DiskFull,
93        Self::WrongUserPathWorkerBinary,
94        Self::ToolchainDrift,
95    ];
96
97    /// Canonical `RCH-Innn` code string. Stable across releases — never
98    /// renumber an existing variant; only append.
99    #[must_use]
100    pub const fn code(self) -> &'static str {
101        match self {
102            Self::NoAdmissibleWorkers => "RCH-I001",
103            Self::CriticalPressure => "RCH-I002",
104            Self::InsufficientSlots => "RCH-I003",
105            Self::HardPreflight => "RCH-I004",
106            Self::ActiveProjectExclusion => "RCH-I005",
107            Self::MissingRuntimeToolchainTarget => "RCH-I006",
108            Self::OsArchMismatch => "RCH-I007",
109            Self::TelemetryStale => "RCH-I008",
110            Self::CircuitOpen => "RCH-I009",
111            Self::DaemonSocketRefused => "RCH-I010",
112            Self::LocalFallback => "RCH-I011",
113            Self::ProofRefusal => "RCH-I012",
114            Self::RsyncVanishedFile => "RCH-I013",
115            Self::ArtifactMiss => "RCH-I014",
116            Self::QueueAmbiguity => "RCH-I015",
117            Self::DiskFull => "RCH-I016",
118            Self::WrongUserPathWorkerBinary => "RCH-I017",
119            Self::ToolchainDrift => "RCH-I018",
120        }
121    }
122
123    /// Exact operator-facing failure-class string from the session-history
124    /// report. These strings are part of the contract — dashboards and the
125    /// validation matrix key off them.
126    #[must_use]
127    pub const fn failure_class(self) -> &'static str {
128        match self {
129            Self::NoAdmissibleWorkers => "no admissible workers",
130            Self::CriticalPressure => "critical pressure",
131            Self::InsufficientSlots => "insufficient slots",
132            Self::HardPreflight => "hard preflight",
133            Self::ActiveProjectExclusion => "active project exclusion",
134            Self::MissingRuntimeToolchainTarget => "missing runtime/toolchain/Rust target",
135            Self::OsArchMismatch => "OS/arch mismatch",
136            Self::TelemetryStale => "telemetry stale/age unknown",
137            Self::CircuitOpen => "circuit open",
138            Self::DaemonSocketRefused => "daemon socket refused",
139            Self::LocalFallback => "local fallback",
140            Self::ProofRefusal => "proof refusal",
141            Self::RsyncVanishedFile => "rsync vanished file",
142            Self::ArtifactMiss => "artifact miss",
143            Self::QueueAmbiguity => "queue ambiguity",
144            Self::DiskFull => "disk full",
145            Self::WrongUserPathWorkerBinary => "wrong user/path worker binary",
146            Self::ToolchainDrift => "toolchain drift",
147        }
148    }
149
150    /// Mapped AGENTS-style [`ErrorCode`] where the catalog already has a
151    /// matching concept. `None` for policy/operational reasons (e.g. local
152    /// fallback, proof refusal) that are incident-only by design.
153    #[must_use]
154    pub fn error_code(self) -> Option<ErrorCode> {
155        Some(match self {
156            Self::NoAdmissibleWorkers => ErrorCode::WorkerNoneAvailable,
157            Self::CriticalPressure => ErrorCode::WorkerDiskPressureCritical,
158            Self::InsufficientSlots => ErrorCode::WorkerAtCapacity,
159            Self::MissingRuntimeToolchainTarget => ErrorCode::WorkerMissingToolchain,
160            Self::TelemetryStale => ErrorCode::WorkerTelemetryGap,
161            Self::CircuitOpen => ErrorCode::WorkerCircuitOpen,
162            Self::DaemonSocketRefused => ErrorCode::InternalDaemonSocket,
163            Self::RsyncVanishedFile => ErrorCode::TransferSourceMissing,
164            Self::ArtifactMiss => ErrorCode::BuildArtifactMissing,
165            Self::DiskFull => ErrorCode::TransferDiskFull,
166            // Incident-only reasons with no single catalog analogue.
167            Self::HardPreflight
168            | Self::ActiveProjectExclusion
169            | Self::OsArchMismatch
170            | Self::LocalFallback
171            | Self::ProofRefusal
172            | Self::QueueAmbiguity
173            | Self::WrongUserPathWorkerBinary
174            // Advisory; a healthy-worker codegen difference has no error analogue.
175            | Self::ToolchainDrift => return None,
176        })
177    }
178
179    /// Parse a canonical `RCH-Innn` code string back to a reason code.
180    #[must_use]
181    pub fn from_code_str(code: &str) -> Option<Self> {
182        Self::ALL.iter().copied().find(|r| r.code() == code)
183    }
184}
185
186impl fmt::Display for IncidentReasonCode {
187    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
188        f.write_str(self.code())
189    }
190}
191
192impl Serialize for IncidentReasonCode {
193    fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
194        s.serialize_str(self.code())
195    }
196}
197
198impl<'de> Deserialize<'de> for IncidentReasonCode {
199    fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
200        let raw = String::deserialize(d)?;
201        Self::from_code_str(&raw)
202            .ok_or_else(|| serde::de::Error::custom(format!("unknown incident reason code: {raw}")))
203    }
204}
205
206/// The subsystem that produced an incident.
207#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
208#[serde(rename_all = "snake_case")]
209pub enum IncidentEventType {
210    Selection,
211    Admission,
212    Fallback,
213    Proof,
214    Doctor,
215    Telemetry,
216    ArtifactRetrieval,
217    WorkerLifecycle,
218}
219
220/// The process/component that emitted the event (for provenance).
221#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
222#[serde(rename_all = "snake_case")]
223pub enum IncidentSource {
224    Hook,
225    Daemon,
226    Worker,
227    Doctor,
228    Cli,
229}
230
231/// Where the build ultimately ran (or was steered).
232#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
233#[serde(rename_all = "snake_case")]
234pub enum SelectedMode {
235    Local,
236    Remote,
237    Deferred,
238}
239
240/// Canonical control-state snapshot (env/flag-derived) so postmortems never
241/// lose *why* a build was steered, queued, or made fail-closed. Mirrors the
242/// explicit-control surface: `requested_worker`, `requested_profile`,
243/// `strict_remote_policy`, `queue_policy`, `visibility_mode`, `wait_timeout_ms`,
244/// `target_dir_policy`. All fields optional and omitted from the wire form when
245/// unset, keeping the event compact.
246#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
247pub struct ControlState {
248    /// `RCH_WORKER` / `--worker` explicit worker request.
249    #[serde(default, skip_serializing_if = "Option::is_none")]
250    pub requested_worker: Option<String>,
251    /// `RCH_PRESET` / requested profile.
252    #[serde(default, skip_serializing_if = "Option::is_none")]
253    pub requested_profile: Option<String>,
254    /// `RCH_REQUIRE_REMOTE` / `RCH_FORCE_REMOTE` fail-closed remote policy.
255    #[serde(default, skip_serializing_if = "is_false")]
256    pub strict_remote_policy: bool,
257    /// `RCH_QUEUE_WHEN_BUSY` queue policy.
258    #[serde(default, skip_serializing_if = "Option::is_none")]
259    pub queue_policy: Option<String>,
260    /// `RCH_VISIBILITY` visibility mode.
261    #[serde(default, skip_serializing_if = "Option::is_none")]
262    pub visibility_mode: Option<String>,
263    /// Queue wait timeout in milliseconds.
264    #[serde(default, skip_serializing_if = "Option::is_none")]
265    pub wait_timeout_ms: Option<u64>,
266    /// Worker-scoped target-dir policy.
267    #[serde(default, skip_serializing_if = "Option::is_none")]
268    pub target_dir_policy: Option<String>,
269}
270
271impl ControlState {
272    /// True when no control field is set (used to omit the section entirely).
273    #[must_use]
274    pub fn is_empty(&self) -> bool {
275        *self == ControlState::default()
276    }
277}
278
279/// serde `skip_serializing_if` helper for `bool` fields that default to false.
280#[allow(clippy::trivially_copy_pass_by_ref)]
281fn is_false(b: &bool) -> bool {
282    !*b
283}
284
285/// A single incident event. The uniform record every remediation-relevant
286/// subsystem emits, keyed by a stable [`IncidentReasonCode`].
287#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
288pub struct IncidentEvent {
289    /// Schema version (`SchemaComponent::IncidentLedger`).
290    pub schema_version: String,
291    /// Emitting subsystem.
292    pub event_type: IncidentEventType,
293    /// Stable reason (serialized as `RCH-Innn`).
294    pub reason_code: IncidentReasonCode,
295    /// Project key (e.g. blake3 of the canonical project root).
296    pub project_id: String,
297    /// Classified command fingerprint.
298    pub command_fingerprint: String,
299    /// Worker id where applicable.
300    #[serde(default, skip_serializing_if = "Option::is_none")]
301    pub worker_id: Option<String>,
302    /// Where the build ran / was steered.
303    pub selected_mode: SelectedMode,
304    /// Whether local fallback was permitted for this request.
305    pub local_fallback_allowed: bool,
306    /// Emitting process/component.
307    pub source: IncidentSource,
308    /// Event time as Unix epoch milliseconds. Caller-supplied so the schema is
309    /// deterministic in tests and golden artifacts.
310    pub occurred_at_unix_ms: u64,
311    /// Canonical control-state snapshot (omitted from the wire form when empty).
312    #[serde(default, skip_serializing_if = "ControlState::is_empty")]
313    pub control: ControlState,
314    /// Compact, ordered free-form details (small, bounded; not for secrets).
315    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
316    pub details: BTreeMap<String, String>,
317}
318
319impl IncidentEvent {
320    /// Construct an event with the required core fields; optional fields
321    /// (`worker_id`, `control`, `details`) default to empty and can be set
322    /// afterward.
323    ///
324    /// These are the genuinely-required record fields — every one is a
325    /// distinct, non-defaultable dimension of an incident — so a flat
326    /// constructor is clearer here than a params struct.
327    #[allow(clippy::too_many_arguments)]
328    #[must_use]
329    pub fn new(
330        event_type: IncidentEventType,
331        reason_code: IncidentReasonCode,
332        source: IncidentSource,
333        project_id: impl Into<String>,
334        command_fingerprint: impl Into<String>,
335        selected_mode: SelectedMode,
336        local_fallback_allowed: bool,
337        occurred_at_unix_ms: u64,
338    ) -> Self {
339        Self {
340            schema_version: incident_schema_version().to_string(),
341            event_type,
342            reason_code,
343            project_id: project_id.into(),
344            command_fingerprint: command_fingerprint.into(),
345            worker_id: None,
346            selected_mode,
347            local_fallback_allowed,
348            source,
349            occurred_at_unix_ms,
350            control: ControlState::default(),
351            details: BTreeMap::new(),
352        }
353    }
354
355    /// Set the worker id (builder style).
356    #[must_use]
357    pub fn with_worker_id(mut self, worker_id: impl Into<String>) -> Self {
358        self.worker_id = Some(worker_id.into());
359        self
360    }
361
362    /// Set the control-state snapshot (builder style).
363    #[must_use]
364    pub fn with_control(mut self, control: ControlState) -> Self {
365        self.control = control;
366        self
367    }
368
369    /// Insert a compact detail key/value (builder style).
370    #[must_use]
371    pub fn with_detail(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
372        self.details.insert(key.into(), value.into());
373        self
374    }
375
376    /// Return a copy with the free-form `details` *values* routed through the
377    /// shared secret redactor (bd-53ga7).
378    ///
379    /// The structured fields (ids, hashes, fingerprints, enums, timestamps) are
380    /// non-sensitive by construction; only the free-form `details` map can carry
381    /// an injected secret. The ledger redacts it at write time as defense in
382    /// depth even though callers are asked not to put secrets here. Detail
383    /// *keys* are stable field names and are left intact.
384    #[must_use]
385    pub fn redacted(&self) -> Self {
386        let mut out = self.clone();
387        out.details = self
388            .details
389            .iter()
390            .map(|(k, v)| (k.clone(), crate::redaction::redact_secrets(v)))
391            .collect();
392        out
393    }
394}
395
396/// The current incident-schema version string.
397#[must_use]
398pub fn incident_schema_version() -> &'static str {
399    current_version(SchemaComponent::IncidentLedger)
400}
401
402#[cfg(test)]
403mod tests {
404    use super::*;
405
406    #[test]
407    fn registry_covers_session_history_failure_classes_plus_appended() {
408        // The session-history report enumerates the first 17 classes (RCH-I001..
409        // RCH-I017); the registry maps each 1:1 for the validation matrix. Newer
410        // operational reasons are APPENDED with the next sequential code and never
411        // renumber an existing one — e.g. RCH-I018 (toolchain drift, bd-784xt).
412        assert_eq!(IncidentReasonCode::ALL.len(), 18);
413        assert_eq!(
414            IncidentReasonCode::WrongUserPathWorkerBinary.code(),
415            "RCH-I017"
416        );
417        assert_eq!(IncidentReasonCode::ToolchainDrift.code(), "RCH-I018");
418        assert!(
419            !IncidentReasonCode::ToolchainDrift
420                .failure_class()
421                .is_empty()
422        );
423    }
424
425    #[test]
426    fn reason_codes_are_unique() {
427        let mut codes: Vec<&str> = IncidentReasonCode::ALL.iter().map(|r| r.code()).collect();
428        codes.sort_unstable();
429        let before = codes.len();
430        codes.dedup();
431        assert_eq!(before, codes.len(), "duplicate RCH-I codes in registry");
432    }
433
434    #[test]
435    fn failure_class_strings_are_unique() {
436        let mut classes: Vec<&str> = IncidentReasonCode::ALL
437            .iter()
438            .map(|r| r.failure_class())
439            .collect();
440        classes.sort_unstable();
441        let before = classes.len();
442        classes.dedup();
443        assert_eq!(before, classes.len(), "duplicate failure-class strings");
444    }
445
446    #[test]
447    fn codes_follow_rch_i_format_and_sequence() {
448        for (i, reason) in IncidentReasonCode::ALL.iter().enumerate() {
449            let expected = format!("RCH-I{:03}", i + 1);
450            assert_eq!(reason.code(), expected, "code out of sequence at {i}");
451        }
452    }
453
454    #[test]
455    fn reason_code_roundtrips_through_code_str() {
456        for reason in IncidentReasonCode::ALL {
457            assert_eq!(
458                IncidentReasonCode::from_code_str(reason.code()),
459                Some(*reason)
460            );
461        }
462        assert_eq!(IncidentReasonCode::from_code_str("RCH-I999"), None);
463        assert_eq!(IncidentReasonCode::from_code_str("nonsense"), None);
464    }
465
466    #[test]
467    fn reason_code_serializes_as_stable_string() {
468        let json = serde_json::to_string(&IncidentReasonCode::DiskFull).unwrap();
469        assert_eq!(json, "\"RCH-I016\"");
470        let back: IncidentReasonCode = serde_json::from_str("\"RCH-I016\"").unwrap();
471        assert_eq!(back, IncidentReasonCode::DiskFull);
472    }
473
474    #[test]
475    fn unknown_reason_code_deserialization_errors() {
476        let res: Result<IncidentReasonCode, _> = serde_json::from_str("\"RCH-I404\"");
477        assert!(res.is_err());
478    }
479
480    #[test]
481    fn error_code_mapping_is_present_where_expected() {
482        // A representative sample of the catalog-mapped reasons.
483        assert_eq!(
484            IncidentReasonCode::CircuitOpen.error_code(),
485            Some(ErrorCode::WorkerCircuitOpen)
486        );
487        assert_eq!(
488            IncidentReasonCode::DiskFull.error_code(),
489            Some(ErrorCode::TransferDiskFull)
490        );
491        assert_eq!(
492            IncidentReasonCode::ArtifactMiss.error_code(),
493            Some(ErrorCode::BuildArtifactMissing)
494        );
495        // Incident-only reasons have no catalog code.
496        assert_eq!(IncidentReasonCode::LocalFallback.error_code(), None);
497        assert_eq!(IncidentReasonCode::ProofRefusal.error_code(), None);
498    }
499
500    fn sample_event() -> IncidentEvent {
501        IncidentEvent::new(
502            IncidentEventType::Admission,
503            IncidentReasonCode::NoAdmissibleWorkers,
504            IncidentSource::Daemon,
505            "proj-abc",
506            "cargo build --release",
507            SelectedMode::Local,
508            true,
509            1_700_000_000_000,
510        )
511    }
512
513    #[test]
514    fn incident_event_roundtrips() {
515        let event = sample_event()
516            .with_worker_id("css")
517            .with_detail("candidates", "3")
518            .with_control(ControlState {
519                strict_remote_policy: true,
520                requested_worker: Some("bil".to_string()),
521                wait_timeout_ms: Some(5000),
522                ..ControlState::default()
523            });
524        let json = serde_json::to_string(&event).unwrap();
525        let back: IncidentEvent = serde_json::from_str(&json).unwrap();
526        assert_eq!(event, back);
527    }
528
529    #[test]
530    fn incident_event_serialization_is_stable_golden() {
531        let event = sample_event();
532        let value = serde_json::to_value(&event).unwrap();
533        // Required core fields present with expected values.
534        assert_eq!(value["schema_version"], "1.0.0");
535        assert_eq!(value["event_type"], "admission");
536        assert_eq!(value["reason_code"], "RCH-I001");
537        assert_eq!(value["project_id"], "proj-abc");
538        assert_eq!(value["command_fingerprint"], "cargo build --release");
539        assert_eq!(value["selected_mode"], "local");
540        assert_eq!(value["local_fallback_allowed"], true);
541        assert_eq!(value["source"], "daemon");
542        assert_eq!(value["occurred_at_unix_ms"], 1_700_000_000_000u64);
543        // Empty optional sections are omitted from the wire form.
544        assert!(value.get("worker_id").is_none());
545        assert!(value.get("control").is_none());
546        assert!(value.get("details").is_none());
547    }
548
549    #[test]
550    fn control_state_fields_round_trip_and_omit_when_empty() {
551        // Every canonical control field from the FIFTH AUDIT NOTE.
552        let control = ControlState {
553            requested_worker: Some("css".to_string()),
554            requested_profile: Some("fast".to_string()),
555            strict_remote_policy: true,
556            queue_policy: Some("queue_when_busy".to_string()),
557            visibility_mode: Some("visible".to_string()),
558            wait_timeout_ms: Some(10_000),
559            target_dir_policy: Some("pooled".to_string()),
560        };
561        assert!(!control.is_empty());
562        let value = serde_json::to_value(&control).unwrap();
563        for key in [
564            "requested_worker",
565            "requested_profile",
566            "strict_remote_policy",
567            "queue_policy",
568            "visibility_mode",
569            "wait_timeout_ms",
570            "target_dir_policy",
571        ] {
572            assert!(value.get(key).is_some(), "missing control field {key}");
573        }
574        // Default control omits everything.
575        assert!(ControlState::default().is_empty());
576        let empty = serde_json::to_value(ControlState::default()).unwrap();
577        assert_eq!(empty.as_object().unwrap().len(), 0);
578    }
579
580    #[test]
581    fn incident_event_carries_current_schema_version() {
582        assert_eq!(incident_schema_version(), "1.0.0");
583        assert_eq!(sample_event().schema_version, incident_schema_version());
584    }
585}
rch_common/incident.rs

rch_common/
incident.rs
