rch_common/errors/
reliability.rs

1//! Reliability-doctor reason-code catalog (`RCH-Rnnn`).
2//!
3//! This module mirrors the [`super::catalog::ErrorCode`] pattern but for the
4//! `rch doctor --reliability` diagnostics surface. Every diagnostic emitted
5//! by the reliability doctor carries one of these variants. The CLI/JSON
6//! representation is a stable `RCH-Rnnn` string, so agents and dashboards
7//! can branch on the code without parsing free-form snake_case strings.
8//!
9//! # Code Ranges
10//!
11//! | Range      | Category              | Description                            |
12//! |------------|-----------------------|----------------------------------------|
13//! | R001-R099  | Topology              | Worker config + daemon worker capacity |
14//! | R100-R199  | DiskPressure          | Worker disk-pressure tiers + telemetry |
15//! | R200-R299  | ProcessTriage         | Cancellation cleanup + process debt    |
16//! | R300-R399  | RepoConvergence       | Worker repo-state convergence          |
17//! | R400-R499  | HelperCompatibility   | rsync / ssh / cargo / zstd availability |
18//! | R600-R699  | SchemaCompatibility   | Cross-binary schema-version drift      |
19//! | R700-R799  | MirrorOwnership       | Worker mirror-tree ownership drift     |
20//! Discipline:
21//! - Variant identifiers are CamelCase.
22//! - Each variant has a fixed `RCH-Rnnn` code returned by [`ReliabilityReasonCode::code`].
23//! - Each variant has a category (per the table above) for analytics grouping.
24//! - Each variant declares whether its remediation requires a daemon restart
25//!   via [`ReliabilityReasonCode::requires_restart`] (consumed by the
26//!   reliability doctor's `requires_restart` field on `RemediationStep`).
27//! - Each variant carries a one-line `remediation_hint` for the diagnostic's
28//!   default suggestion text.
29//!
30//! Adding a new variant requires:
31//! 1. Pick the next free code in the right range.
32//! 2. Add the variant to the enum below.
33//! 3. Add an arm to every `match self` block (Rust's exhaustiveness check
34//!    will not let you forget — that's the whole point of the enum).
35//! 4. The unit tests in this module enforce uniqueness, format, and
36//!    range-membership at `cargo test` time.
37
38use serde::{Deserialize, Serialize};
39use std::fmt;
40
41/// Stable category groups for reliability reason codes.
42#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
43#[serde(rename_all = "snake_case")]
44pub enum ReliabilityCategoryKind {
45    Topology,
46    DiskPressure,
47    ProcessTriage,
48    RepoConvergence,
49    HelperCompatibility,
50    RolloutPosture,
51    SchemaCompatibility,
52    MirrorOwnership,
53}
54
55impl ReliabilityCategoryKind {
56    /// Stable snake_case identifier — same as the serde tag, exposed
57    /// directly so callers don't need `serde_json::to_value` to render
58    /// the label. Used by `rch doctor --runbook-list` (br-62u24.20)
59    /// to print a per-code category column.
60    #[must_use]
61    pub const fn as_str(self) -> &'static str {
62        match self {
63            Self::Topology => "topology",
64            Self::DiskPressure => "disk_pressure",
65            Self::ProcessTriage => "process_triage",
66            Self::RepoConvergence => "repo_convergence",
67            Self::HelperCompatibility => "helper_compatibility",
68            Self::RolloutPosture => "rollout_posture",
69            Self::SchemaCompatibility => "schema_compatibility",
70            Self::MirrorOwnership => "mirror_ownership",
71        }
72    }
73}
74
75/// One reason code per emitted reliability diagnostic. Serializes to its
76/// canonical `RCH-Rnnn` string form via [`Serialize`]; deserializes the same
77/// form via [`Deserialize`].
78#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
79pub enum ReliabilityReasonCode {
80    // ---- Topology (R001-R099) ----
81    /// Workers configuration could not be loaded.
82    WorkersConfigUnreadable,
83    /// No workers configured (every build runs locally).
84    NoWorkersConfigured,
85    /// Workers are configured (Pass).
86    WorkersConfigured,
87    /// Daemon status surface is unavailable.
88    DaemonStatusUnavailable,
89    /// Daemon has no registered workers.
90    DaemonHasNoWorkers,
91    /// Every worker is unhealthy.
92    AllWorkersUnhealthy,
93    /// Some workers are healthy, some are not.
94    PartialWorkerCapacity,
95    /// All workers are healthy (Pass).
96    WorkersHealthy,
97    /// A worker's circuit breaker is open.
98    WorkerCircuitOpen,
99    /// A worker is unreachable / offline / failed.
100    WorkerUnreachable,
101    /// A worker is degraded (half-open circuit or not-ready).
102    WorkerDegraded,
103    /// A worker is ready (Pass).
104    WorkerReady,
105    /// A worker reported an unrecognized `ready_status` value (defensive parse).
106    WorkerStatusUnrecognized,
107    /// A worker reported an unrecognized `circuit_state` value (defensive parse).
108    WorkerCircuitStateUnrecognized,
109
110    // ---- DiskPressure (R100-R199) ----
111    /// Disk-pressure surface is unavailable.
112    DiskPressureUnavailable,
113    /// Worker disk pressure has reached critical.
114    WorkerDiskPressureCritical,
115    /// Worker disk pressure has reached the warning threshold.
116    WorkerDiskPressureWarning,
117    /// Worker disk pressure is healthy (Pass).
118    WorkerDiskPressureHealthy,
119    /// Worker is missing fresh disk telemetry.
120    WorkerDiskPressureTelemetryGap,
121    /// No workers reported disk-pressure telemetry (Info; common for empty fleets).
122    DiskPressureNoWorkers,
123
124    // ---- ProcessTriage (R200-R299) ----
125    /// Process-debt surface is unavailable.
126    ProcessDebtUnavailable,
127    /// Cancellation cleanup is healthy (Pass).
128    CancellationCleanupHealthy,
129    /// Cancellation cleanup was skipped (no jobs to triage).
130    CancellationCleanupSkipped,
131    /// Cancellation cleanup is degraded (some warnings).
132    CancellationCleanupDegraded,
133    /// Cancellation cleanup is failing.
134    CancellationCleanupFailed,
135    /// Unmanaged compiler processes are running on a dispatcher.
136    LocalBuildsDetected,
137    /// Local compiler process observation could not complete.
138    LocalBuildScanUnavailable,
139    /// The shared local-build alarm latch could not be persisted.
140    LocalBuildAlarmStateUnavailable,
141    /// No unmanaged compiler processes were observed on the dispatcher.
142    LocalBuildsAbsent,
143
144    // ---- RepoConvergence (R300-R399) ----
145    /// Repo-convergence surface is unavailable.
146    RepoConvergenceUnavailable,
147    /// One or more workers failed repo convergence.
148    RepoConvergenceFailed,
149    /// Workers are drifting / stale on repo convergence.
150    RepoConvergenceDrift,
151    /// No workers reported repo-convergence records (Info).
152    RepoConvergenceNoWorkers,
153    /// All workers are repo-converged (Pass).
154    RepoConvergenceReady,
155    /// A specific worker's repo state is not ready.
156    WorkerRepoNotReady,
157
158    // ---- HelperCompatibility (R400-R499) ----
159    /// A required helper binary is available (Pass).
160    HelperAvailable,
161    /// A required helper binary is missing.
162    HelperMissing,
163    /// The helper compatibility probe itself did not complete.
164    HelperProbeUnavailable,
165
166    // ---- RolloutPosture (R500-R599) ----
167    /// `self_healing.hook_starts_daemon` is enabled (Pass).
168    HookAutoStartEnabled,
169    /// `self_healing.hook_starts_daemon` is disabled.
170    HookAutoStartDisabled,
171    /// `self_healing.daemon_installs_hooks` is enabled (Pass).
172    DaemonHookRepairEnabled,
173    /// `self_healing.daemon_installs_hooks` is disabled.
174    DaemonHookRepairDisabled,
175    /// Configuration could not be loaded.
176    ConfigLoadFailed,
177    /// Unified status surface is compiled in (Pass).
178    StatusSurfaceAvailable,
179    /// Repo-convergence status endpoint is wired into the CLI (Pass).
180    RepoConvergenceSurfaceAvailable,
181    /// Disk-pressure fields are wired into worker status (Pass).
182    DiskPressureSurfaceAvailable,
183    /// The Claude Code PreToolUse `rch` hook is installed (Pass).
184    HookInstalled,
185    /// The Claude Code PreToolUse `rch` hook is missing; builds will not offload.
186    HookNotInstalled,
187    /// The hook/CLI socket path matches the canonical daemon socket (Pass).
188    SocketPathConsistent,
189    /// The configured socket path diverges from the canonical daemon socket;
190    /// the hook may not reach the daemon.
191    SocketPathMismatch,
192
193    // ---- SchemaCompatibility (R600-R699) ----
194    /// Schema versions are compatible (Pass).
195    SchemaCompatible,
196    /// Schema versions are incompatible.
197    SchemaIncompatible,
198
199    // ---- MirrorOwnership (R700-R799) ----
200    /// A worker's canonical mirror tree has no root-owned entries (Pass).
201    WorkerMirrorOwnershipHealthy,
202    /// A worker's canonical mirror tree contains root-owned entries that
203    /// will make rsync-as-ssh-user fail exit 23 until repaired. The
204    /// dispatch-time preflight self-heals these (bd-8iwkm); this code
205    /// surfaces the drift proactively, before a dispatch hits it.
206    WorkerMirrorOwnershipDrift,
207    /// The worker's ownership probe could not run because passwordless
208    /// sudo is unavailable to the SSH user; dispatch-time repair fails
209    /// open silently in this state.
210    WorkerMirrorOwnershipCheckUnavailable,
211    /// The per-worker ownership probe did not complete (SSH transport
212    /// error or timeout); drift state on that worker is unknown.
213    WorkerMirrorOwnershipUnprobeable,
214    /// No workers are configured, so there is nothing to probe (Info;
215    /// the topology probe reports this condition as its own code).
216    MirrorOwnershipNoWorkers,
217}
218
219impl ReliabilityReasonCode {
220    /// The CamelCase variant identifier as a static string. Used by
221    /// `rch error explain` to render a human-readable name alongside
222    /// the `RCH-Rnnn` code.
223    #[must_use]
224    pub const fn name(self) -> &'static str {
225        match self {
226            Self::WorkersConfigUnreadable => "WorkersConfigUnreadable",
227            Self::NoWorkersConfigured => "NoWorkersConfigured",
228            Self::WorkersConfigured => "WorkersConfigured",
229            Self::DaemonStatusUnavailable => "DaemonStatusUnavailable",
230            Self::DaemonHasNoWorkers => "DaemonHasNoWorkers",
231            Self::AllWorkersUnhealthy => "AllWorkersUnhealthy",
232            Self::PartialWorkerCapacity => "PartialWorkerCapacity",
233            Self::WorkersHealthy => "WorkersHealthy",
234            Self::WorkerCircuitOpen => "WorkerCircuitOpen",
235            Self::WorkerUnreachable => "WorkerUnreachable",
236            Self::WorkerDegraded => "WorkerDegraded",
237            Self::WorkerReady => "WorkerReady",
238            Self::WorkerStatusUnrecognized => "WorkerStatusUnrecognized",
239            Self::WorkerCircuitStateUnrecognized => "WorkerCircuitStateUnrecognized",
240            Self::DiskPressureUnavailable => "DiskPressureUnavailable",
241            Self::WorkerDiskPressureCritical => "WorkerDiskPressureCritical",
242            Self::WorkerDiskPressureWarning => "WorkerDiskPressureWarning",
243            Self::WorkerDiskPressureHealthy => "WorkerDiskPressureHealthy",
244            Self::WorkerDiskPressureTelemetryGap => "WorkerDiskPressureTelemetryGap",
245            Self::DiskPressureNoWorkers => "DiskPressureNoWorkers",
246            Self::ProcessDebtUnavailable => "ProcessDebtUnavailable",
247            Self::CancellationCleanupHealthy => "CancellationCleanupHealthy",
248            Self::CancellationCleanupSkipped => "CancellationCleanupSkipped",
249            Self::CancellationCleanupDegraded => "CancellationCleanupDegraded",
250            Self::CancellationCleanupFailed => "CancellationCleanupFailed",
251            Self::LocalBuildsDetected => "LocalBuildsDetected",
252            Self::LocalBuildScanUnavailable => "LocalBuildScanUnavailable",
253            Self::LocalBuildAlarmStateUnavailable => "LocalBuildAlarmStateUnavailable",
254            Self::LocalBuildsAbsent => "LocalBuildsAbsent",
255            Self::RepoConvergenceUnavailable => "RepoConvergenceUnavailable",
256            Self::RepoConvergenceFailed => "RepoConvergenceFailed",
257            Self::RepoConvergenceDrift => "RepoConvergenceDrift",
258            Self::RepoConvergenceNoWorkers => "RepoConvergenceNoWorkers",
259            Self::RepoConvergenceReady => "RepoConvergenceReady",
260            Self::WorkerRepoNotReady => "WorkerRepoNotReady",
261            Self::HelperAvailable => "HelperAvailable",
262            Self::HelperMissing => "HelperMissing",
263            Self::HelperProbeUnavailable => "HelperProbeUnavailable",
264            Self::HookAutoStartEnabled => "HookAutoStartEnabled",
265            Self::HookAutoStartDisabled => "HookAutoStartDisabled",
266            Self::DaemonHookRepairEnabled => "DaemonHookRepairEnabled",
267            Self::DaemonHookRepairDisabled => "DaemonHookRepairDisabled",
268            Self::ConfigLoadFailed => "ConfigLoadFailed",
269            Self::StatusSurfaceAvailable => "StatusSurfaceAvailable",
270            Self::RepoConvergenceSurfaceAvailable => "RepoConvergenceSurfaceAvailable",
271            Self::DiskPressureSurfaceAvailable => "DiskPressureSurfaceAvailable",
272            Self::HookInstalled => "HookInstalled",
273            Self::HookNotInstalled => "HookNotInstalled",
274            Self::SocketPathConsistent => "SocketPathConsistent",
275            Self::SocketPathMismatch => "SocketPathMismatch",
276            Self::SchemaCompatible => "SchemaCompatible",
277            Self::SchemaIncompatible => "SchemaIncompatible",
278            Self::WorkerMirrorOwnershipHealthy => "WorkerMirrorOwnershipHealthy",
279            Self::WorkerMirrorOwnershipDrift => "WorkerMirrorOwnershipDrift",
280            Self::WorkerMirrorOwnershipCheckUnavailable => "WorkerMirrorOwnershipCheckUnavailable",
281            Self::WorkerMirrorOwnershipUnprobeable => "WorkerMirrorOwnershipUnprobeable",
282            Self::MirrorOwnershipNoWorkers => "MirrorOwnershipNoWorkers",
283        }
284    }
285
286    /// The canonical `RCH-Rnnn` code string for this variant.
287    #[must_use]
288    pub const fn code(self) -> &'static str {
289        match self {
290            // R001-R099 — Topology
291            Self::WorkersConfigUnreadable => "RCH-R001",
292            Self::NoWorkersConfigured => "RCH-R002",
293            Self::WorkersConfigured => "RCH-R003",
294            Self::DaemonStatusUnavailable => "RCH-R004",
295            Self::DaemonHasNoWorkers => "RCH-R005",
296            Self::AllWorkersUnhealthy => "RCH-R006",
297            Self::PartialWorkerCapacity => "RCH-R007",
298            Self::WorkersHealthy => "RCH-R008",
299            Self::WorkerCircuitOpen => "RCH-R009",
300            Self::WorkerUnreachable => "RCH-R010",
301            Self::WorkerDegraded => "RCH-R011",
302            Self::WorkerReady => "RCH-R012",
303            Self::WorkerStatusUnrecognized => "RCH-R013",
304            Self::WorkerCircuitStateUnrecognized => "RCH-R014",
305
306            // R100-R199 — DiskPressure
307            Self::DiskPressureUnavailable => "RCH-R100",
308            Self::WorkerDiskPressureCritical => "RCH-R101",
309            Self::WorkerDiskPressureWarning => "RCH-R102",
310            Self::WorkerDiskPressureHealthy => "RCH-R103",
311            Self::WorkerDiskPressureTelemetryGap => "RCH-R104",
312            Self::DiskPressureNoWorkers => "RCH-R105",
313
314            // R200-R299 — ProcessTriage
315            Self::ProcessDebtUnavailable => "RCH-R200",
316            Self::CancellationCleanupHealthy => "RCH-R201",
317            Self::CancellationCleanupSkipped => "RCH-R202",
318            Self::CancellationCleanupDegraded => "RCH-R203",
319            Self::CancellationCleanupFailed => "RCH-R204",
320            Self::LocalBuildsDetected => "RCH-R206",
321            Self::LocalBuildScanUnavailable => "RCH-R207",
322            Self::LocalBuildAlarmStateUnavailable => "RCH-R208",
323            Self::LocalBuildsAbsent => "RCH-R209",
324
325            // R300-R399 — RepoConvergence
326            Self::RepoConvergenceUnavailable => "RCH-R300",
327            Self::RepoConvergenceFailed => "RCH-R301",
328            Self::RepoConvergenceDrift => "RCH-R302",
329            Self::RepoConvergenceNoWorkers => "RCH-R303",
330            Self::RepoConvergenceReady => "RCH-R304",
331            Self::WorkerRepoNotReady => "RCH-R305",
332
333            // R400-R499 — HelperCompatibility
334            Self::HelperAvailable => "RCH-R400",
335            Self::HelperMissing => "RCH-R401",
336            Self::HelperProbeUnavailable => "RCH-R402",
337
338            // R500-R599 — RolloutPosture
339            Self::HookAutoStartEnabled => "RCH-R500",
340            Self::HookAutoStartDisabled => "RCH-R501",
341            Self::DaemonHookRepairEnabled => "RCH-R502",
342            Self::DaemonHookRepairDisabled => "RCH-R503",
343            Self::ConfigLoadFailed => "RCH-R504",
344            Self::StatusSurfaceAvailable => "RCH-R505",
345            Self::RepoConvergenceSurfaceAvailable => "RCH-R506",
346            Self::DiskPressureSurfaceAvailable => "RCH-R507",
347            Self::HookInstalled => "RCH-R508",
348            Self::HookNotInstalled => "RCH-R509",
349            Self::SocketPathConsistent => "RCH-R510",
350            Self::SocketPathMismatch => "RCH-R511",
351
352            // R600-R699 — SchemaCompatibility
353            Self::SchemaCompatible => "RCH-R600",
354            Self::SchemaIncompatible => "RCH-R601",
355
356            // R700-R799 — MirrorOwnership
357            Self::WorkerMirrorOwnershipHealthy => "RCH-R700",
358            Self::WorkerMirrorOwnershipDrift => "RCH-R701",
359            Self::WorkerMirrorOwnershipCheckUnavailable => "RCH-R702",
360            Self::WorkerMirrorOwnershipUnprobeable => "RCH-R703",
361            Self::MirrorOwnershipNoWorkers => "RCH-R704",
362        }
363    }
364
365    /// The category this variant belongs to.
366    #[must_use]
367    pub const fn category(self) -> ReliabilityCategoryKind {
368        use ReliabilityCategoryKind as C;
369        match self {
370            Self::WorkersConfigUnreadable
371            | Self::NoWorkersConfigured
372            | Self::WorkersConfigured
373            | Self::DaemonStatusUnavailable
374            | Self::DaemonHasNoWorkers
375            | Self::AllWorkersUnhealthy
376            | Self::PartialWorkerCapacity
377            | Self::WorkersHealthy
378            | Self::WorkerCircuitOpen
379            | Self::WorkerUnreachable
380            | Self::WorkerDegraded
381            | Self::WorkerReady
382            | Self::WorkerStatusUnrecognized
383            | Self::WorkerCircuitStateUnrecognized => C::Topology,
384
385            Self::DiskPressureUnavailable
386            | Self::WorkerDiskPressureCritical
387            | Self::WorkerDiskPressureWarning
388            | Self::WorkerDiskPressureHealthy
389            | Self::WorkerDiskPressureTelemetryGap
390            | Self::DiskPressureNoWorkers => C::DiskPressure,
391
392            Self::ProcessDebtUnavailable
393            | Self::CancellationCleanupHealthy
394            | Self::CancellationCleanupSkipped
395            | Self::CancellationCleanupDegraded
396            | Self::CancellationCleanupFailed
397            | Self::LocalBuildsDetected
398            | Self::LocalBuildScanUnavailable
399            | Self::LocalBuildAlarmStateUnavailable
400            | Self::LocalBuildsAbsent => C::ProcessTriage,
401
402            Self::RepoConvergenceUnavailable
403            | Self::RepoConvergenceFailed
404            | Self::RepoConvergenceDrift
405            | Self::RepoConvergenceNoWorkers
406            | Self::RepoConvergenceReady
407            | Self::WorkerRepoNotReady => C::RepoConvergence,
408
409            Self::HelperAvailable | Self::HelperMissing | Self::HelperProbeUnavailable => {
410                C::HelperCompatibility
411            }
412
413            Self::HookAutoStartEnabled
414            | Self::HookAutoStartDisabled
415            | Self::DaemonHookRepairEnabled
416            | Self::DaemonHookRepairDisabled
417            | Self::ConfigLoadFailed
418            | Self::StatusSurfaceAvailable
419            | Self::RepoConvergenceSurfaceAvailable
420            | Self::DiskPressureSurfaceAvailable
421            | Self::HookInstalled
422            | Self::HookNotInstalled
423            | Self::SocketPathConsistent
424            | Self::SocketPathMismatch => C::RolloutPosture,
425
426            Self::SchemaCompatible | Self::SchemaIncompatible => C::SchemaCompatibility,
427
428            Self::WorkerMirrorOwnershipHealthy
429            | Self::WorkerMirrorOwnershipDrift
430            | Self::WorkerMirrorOwnershipCheckUnavailable
431            | Self::WorkerMirrorOwnershipUnprobeable
432            | Self::MirrorOwnershipNoWorkers => C::MirrorOwnership,
433        }
434    }
435
436    /// Whether the configured remediation requires a process restart for the
437    /// fix to take effect. Consumed by the reliability doctor when building
438    /// `RemediationStep::requires_restart` (sibling bead `2s99h.15`).
439    ///
440    /// Policy:
441    /// - `true` when the fix changes a flag/state read at daemon startup, OR
442    ///   when the underlying subsystem caches state at process start.
443    /// - `false` when the fix is purely external (e.g., disk space, key
444    ///   permissions) OR when the daemon hot-reloads the relevant state.
445    #[must_use]
446    pub const fn requires_restart(self) -> bool {
447        match self {
448            // Topology — daemon parses workers.toml at startup; no SIGHUP yet.
449            Self::WorkersConfigUnreadable
450            | Self::NoWorkersConfigured
451            | Self::DaemonStatusUnavailable
452            | Self::DaemonHasNoWorkers => true,
453            Self::WorkersConfigured | Self::WorkersHealthy | Self::WorkerReady => false,
454            // Worker-level conditions are upstream; no rch restart fixes them.
455            Self::AllWorkersUnhealthy
456            | Self::PartialWorkerCapacity
457            | Self::WorkerCircuitOpen
458            | Self::WorkerUnreachable
459            | Self::WorkerDegraded
460            | Self::WorkerStatusUnrecognized
461            | Self::WorkerCircuitStateUnrecognized => false,
462
463            // Disk pressure is external — operator cleans up disk; daemon picks
464            // up the new free-space numbers on next probe.
465            Self::DiskPressureUnavailable
466            | Self::WorkerDiskPressureCritical
467            | Self::WorkerDiskPressureWarning
468            | Self::WorkerDiskPressureHealthy
469            | Self::WorkerDiskPressureTelemetryGap
470            | Self::DiskPressureNoWorkers => false,
471
472            // Process-triage stale subprocess cleanup may require daemon restart
473            // to clear pgid handles.
474            Self::ProcessDebtUnavailable | Self::CancellationCleanupFailed => true,
475            Self::CancellationCleanupHealthy
476            | Self::CancellationCleanupSkipped
477            | Self::CancellationCleanupDegraded
478            | Self::LocalBuildsDetected
479            | Self::LocalBuildScanUnavailable
480            | Self::LocalBuildAlarmStateUnavailable
481            | Self::LocalBuildsAbsent => false,
482
483            // Repo-convergence checks are read-only; remediation is git-side.
484            Self::RepoConvergenceUnavailable
485            | Self::RepoConvergenceFailed
486            | Self::RepoConvergenceDrift
487            | Self::RepoConvergenceNoWorkers
488            | Self::RepoConvergenceReady
489            | Self::WorkerRepoNotReady => false,
490
491            // Helper install (cargo install / package manager) doesn't require
492            // daemon restart.
493            Self::HelperAvailable | Self::HelperMissing | Self::HelperProbeUnavailable => false,
494
495            // Rollout posture flags are cached at startup.
496            Self::HookAutoStartEnabled => false,
497            Self::HookAutoStartDisabled => true,
498            Self::DaemonHookRepairEnabled => false,
499            Self::DaemonHookRepairDisabled => true,
500            Self::ConfigLoadFailed => false,
501            Self::StatusSurfaceAvailable
502            | Self::RepoConvergenceSurfaceAvailable
503            | Self::DiskPressureSurfaceAvailable => false,
504            // Installing the hook / fixing the socket path takes effect on the
505            // next hook invocation; no daemon restart needed.
506            Self::HookInstalled | Self::SocketPathConsistent => false,
507            Self::HookNotInstalled | Self::SocketPathMismatch => false,
508
509            // Schema versions are pinned at compile/bind time; mismatch
510            // requires a fresh process.
511            Self::SchemaCompatible => false,
512            Self::SchemaIncompatible => true,
513
514            // Mirror ownership is external worker state; the dispatch-time
515            // preflight self-heals it without any rch/rchd process restart.
516            Self::WorkerMirrorOwnershipHealthy
517            | Self::WorkerMirrorOwnershipDrift
518            | Self::WorkerMirrorOwnershipCheckUnavailable
519            | Self::WorkerMirrorOwnershipUnprobeable
520            | Self::MirrorOwnershipNoWorkers => false,
521        }
522    }
523
524    /// One-line operator-facing remediation hint. Used as the default
525    /// suggestion text when the diagnostic doesn't override it.
526    #[must_use]
527    pub const fn remediation_hint(self) -> &'static str {
528        match self {
529            Self::WorkersConfigUnreadable => {
530                "Inspect ~/.config/rch/workers.toml for a parse error."
531            }
532            Self::NoWorkersConfigured => "Run `rch workers init` to register a worker.",
533            Self::WorkersConfigured => "No action needed.",
534            Self::DaemonStatusUnavailable => "Start the daemon with `rch daemon start` and retry.",
535            Self::DaemonHasNoWorkers => "Run `rch workers init` to register a worker.",
536            Self::AllWorkersUnhealthy => {
537                "Run `rch workers probe --all` to diagnose worker connectivity."
538            }
539            Self::PartialWorkerCapacity => {
540                "Run `rch workers list --json` to identify the unhealthy worker."
541            }
542            Self::WorkersHealthy => "No action needed.",
543            Self::WorkerCircuitOpen => {
544                "Run `rch workers reset-circuit <worker>` once the underlying issue is fixed."
545            }
546            Self::WorkerUnreachable => "Verify SSH connectivity with `rch workers probe <worker>`.",
547            Self::WorkerDegraded => {
548                "Run `rch workers probe <worker>` to refresh worker health state."
549            }
550            Self::WorkerReady => "No action needed.",
551            Self::WorkerStatusUnrecognized => {
552                "Daemon and rch versions may have drifted; reinstall both binaries."
553            }
554            Self::WorkerCircuitStateUnrecognized => {
555                "Daemon and rch versions may have drifted; reinstall both binaries."
556            }
557            Self::DiskPressureUnavailable => "Start the daemon with `rch daemon start` and retry.",
558            Self::WorkerDiskPressureCritical => {
559                "Run `rch worker disk-cleanup --worker <name>` immediately."
560            }
561            Self::WorkerDiskPressureWarning => {
562                "Plan a `rch worker disk-cleanup --worker <name>` cycle."
563            }
564            Self::WorkerDiskPressureHealthy => "No action needed.",
565            Self::WorkerDiskPressureTelemetryGap => {
566                // Telemetry ingest is daemon-driven; no `rch workers ...`
567                // subcommand calls TelemetryStore::ingest(). Pointing agents
568                // at `workers probe` was confident-wrong (issue #16).
569                "Telemetry refresh is automatic; wait for the next daemon poll cycle (telemetry ingest is daemon-driven, not an `rch workers` action)."
570            }
571            Self::DiskPressureNoWorkers => "No action needed.",
572            Self::ProcessDebtUnavailable => "Start the daemon with `rch daemon start` and retry.",
573            Self::CancellationCleanupHealthy => "No action needed.",
574            Self::CancellationCleanupSkipped => "No action needed.",
575            Self::CancellationCleanupDegraded => {
576                "Run `rch status --jobs --json` to inspect process-triage state."
577            }
578            Self::CancellationCleanupFailed => {
579                "Restart the daemon with `rch daemon restart` to reset stale pgid handles."
580            }
581            Self::LocalBuildsDetected => {
582                "Run `rch shim status`; check PATH order and absolute-path toolchain Cargo invocations."
583            }
584            Self::LocalBuildScanUnavailable => {
585                "Check access to /proc and retry `rch doctor --reliability --scope=triage`."
586            }
587            Self::LocalBuildAlarmStateUnavailable => {
588                "Check permissions on the rch cache directory; current process warnings remain valid."
589            }
590            Self::LocalBuildsAbsent => "No action needed.",
591            Self::RepoConvergenceUnavailable => {
592                "Start the daemon with `rch daemon start` and retry."
593            }
594            Self::RepoConvergenceFailed => "Run `rch repo sync --all` to drive convergence.",
595            Self::RepoConvergenceDrift => "Run `rch repo sync --all` to refresh worker state.",
596            Self::RepoConvergenceNoWorkers => "No action needed.",
597            Self::RepoConvergenceReady => "No action needed.",
598            Self::WorkerRepoNotReady => "Run `rch repo sync --worker <name>` to converge.",
599            Self::HelperAvailable => "No action needed.",
600            Self::HelperMissing => "Install the missing helper via the system package manager.",
601            Self::HelperProbeUnavailable => {
602                "Rerun the helper probe after checking for stuck local helper subprocesses."
603            }
604            Self::HookAutoStartEnabled => "No action needed.",
605            Self::HookAutoStartDisabled => {
606                "Run `rch config set self_healing.hook_starts_daemon true`."
607            }
608            Self::DaemonHookRepairEnabled => "No action needed.",
609            Self::DaemonHookRepairDisabled => {
610                "Run `rch config set self_healing.daemon_installs_hooks true`."
611            }
612            Self::ConfigLoadFailed => "Run `rch config doctor --json` to diagnose.",
613            Self::StatusSurfaceAvailable
614            | Self::RepoConvergenceSurfaceAvailable
615            | Self::DiskPressureSurfaceAvailable => "No action needed.",
616            Self::HookInstalled | Self::SocketPathConsistent => "No action needed.",
617            Self::HookNotInstalled => "Install the hook with `rch hook install`.",
618            Self::SocketPathMismatch => {
619                "Align the socket: set `general.socket_path` to the daemon's socket \
620                (`rch config set general.socket_path <path>`) or relaunch the daemon on the \
621                configured socket."
622            }
623            Self::SchemaCompatible => "No action needed.",
624            Self::SchemaIncompatible => {
625                "Upgrade rch / rchd / rch-wkr binaries to the same release."
626            }
627            Self::WorkerMirrorOwnershipHealthy => "No action needed.",
628            Self::WorkerMirrorOwnershipDrift => {
629                "Root-owned entries under the worker's canonical mirror tree block \
630                 rsync-as-ssh-user; the dispatch-time preflight self-heals them \
631                 (bd-8iwkm), or chown them to the SSH user on the worker now."
632            }
633            Self::WorkerMirrorOwnershipCheckUnavailable => {
634                "Verify passwordless sudo (`sudo -n true`) for the SSH user on the \
635                 worker; ownership detection and repair fail open until then."
636            }
637            Self::WorkerMirrorOwnershipUnprobeable => {
638                "Run `rch workers probe --all`; the ownership probe could not reach \
639                 the worker over SSH."
640            }
641            Self::MirrorOwnershipNoWorkers => "No action needed.",
642        }
643    }
644
645    /// Every variant of this enum, useful for exhaustive iteration in tests.
646    pub const ALL: &'static [ReliabilityReasonCode] = &[
647        Self::WorkersConfigUnreadable,
648        Self::NoWorkersConfigured,
649        Self::WorkersConfigured,
650        Self::DaemonStatusUnavailable,
651        Self::DaemonHasNoWorkers,
652        Self::AllWorkersUnhealthy,
653        Self::PartialWorkerCapacity,
654        Self::WorkersHealthy,
655        Self::WorkerCircuitOpen,
656        Self::WorkerUnreachable,
657        Self::WorkerDegraded,
658        Self::WorkerReady,
659        Self::WorkerStatusUnrecognized,
660        Self::WorkerCircuitStateUnrecognized,
661        Self::DiskPressureUnavailable,
662        Self::WorkerDiskPressureCritical,
663        Self::WorkerDiskPressureWarning,
664        Self::WorkerDiskPressureHealthy,
665        Self::WorkerDiskPressureTelemetryGap,
666        Self::DiskPressureNoWorkers,
667        Self::ProcessDebtUnavailable,
668        Self::CancellationCleanupHealthy,
669        Self::CancellationCleanupSkipped,
670        Self::CancellationCleanupDegraded,
671        Self::CancellationCleanupFailed,
672        Self::LocalBuildsDetected,
673        Self::LocalBuildScanUnavailable,
674        Self::LocalBuildAlarmStateUnavailable,
675        Self::LocalBuildsAbsent,
676        Self::RepoConvergenceUnavailable,
677        Self::RepoConvergenceFailed,
678        Self::RepoConvergenceDrift,
679        Self::RepoConvergenceNoWorkers,
680        Self::RepoConvergenceReady,
681        Self::WorkerRepoNotReady,
682        Self::HelperAvailable,
683        Self::HelperMissing,
684        Self::HelperProbeUnavailable,
685        Self::HookAutoStartEnabled,
686        Self::HookAutoStartDisabled,
687        Self::DaemonHookRepairEnabled,
688        Self::DaemonHookRepairDisabled,
689        Self::ConfigLoadFailed,
690        Self::StatusSurfaceAvailable,
691        Self::RepoConvergenceSurfaceAvailable,
692        Self::DiskPressureSurfaceAvailable,
693        Self::HookInstalled,
694        Self::HookNotInstalled,
695        Self::SocketPathConsistent,
696        Self::SocketPathMismatch,
697        Self::SchemaCompatible,
698        Self::SchemaIncompatible,
699        Self::WorkerMirrorOwnershipHealthy,
700        Self::WorkerMirrorOwnershipDrift,
701        Self::WorkerMirrorOwnershipCheckUnavailable,
702        Self::WorkerMirrorOwnershipUnprobeable,
703        Self::MirrorOwnershipNoWorkers,
704    ];
705}
706
707impl fmt::Display for ReliabilityReasonCode {
708    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
709        f.write_str(self.code())
710    }
711}
712
713impl Serialize for ReliabilityReasonCode {
714    fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
715        s.serialize_str(self.code())
716    }
717}
718
719impl<'de> Deserialize<'de> for ReliabilityReasonCode {
720    fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
721        let raw = String::deserialize(d)?;
722        Self::from_code_str(&raw).ok_or_else(|| {
723            serde::de::Error::custom(format!("unknown reliability reason code {raw:?}"))
724        })
725    }
726}
727
728impl ReliabilityReasonCode {
729    /// Reverse-lookup helper for deserialization.
730    #[must_use]
731    pub fn from_code_str(code: &str) -> Option<Self> {
732        Self::ALL.iter().copied().find(|c| c.code() == code)
733    }
734}
735
736// =============================================================================
737// Operator Runbook Authoring (br-62u24.20)
738// =============================================================================
739
740/// Per-code operator runbook — symptoms, diagnosis, remediation, escalation.
741/// Consumed by `rch doctor --runbook <code>` to render paste-ready Markdown
742/// (or JSON) that an SRE can drop into a PagerDuty / Slack / wiki page when
743/// the corresponding diagnostic fires. Generated from code so it can't go
744/// stale relative to the registry that produces the codes themselves.
745#[derive(Debug, Clone, Copy)]
746pub struct RunbookEntry {
747    /// Bulletized operator-visible signs: log lines, dashboard panels, the
748    /// `rch doctor` output the operator likely just saw before reaching for
749    /// the runbook. Used to confirm "this is the right runbook" before
750    /// running diagnosis.
751    pub symptoms: &'static [&'static str],
752    /// Numbered procedure with exact commands. Each step is a single
753    /// paste-ready command — no embedded prose. Diagnosis steps are
754    /// READ-ONLY: they should never mutate state.
755    pub diagnosis_steps: &'static [&'static str],
756    /// Numbered remediation procedure. Each step is paste-ready. Steps
757    /// MAY mutate state (restart daemon, reset circuit, evict cache, etc.).
758    pub remediation_steps: &'static [&'static str],
759    /// Final command to confirm the issue resolved. Typically
760    /// `rch doctor --reliability --scope=<applicable>` so the operator
761    /// can re-check the exact diagnostic without re-running the full
762    /// doctor suite.
763    pub verification_command: &'static str,
764    /// Optional escalation guidance. `None` = use the generic default
765    /// ("open a GitHub issue with diagnostic output"). Set this for
766    /// codes that have a specific subject-matter-expert routing.
767    pub escalation: Option<&'static str>,
768    /// External references: AGENTS.md sections, related git commits, RFC
769    /// links. Rendered as a Markdown link list.
770    pub references: &'static [&'static str],
771    /// ISO 8601 date the runbook entry was authored. Used by
772    /// `--runbook --since=<ts>` (deferred) to surface newly-authored
773    /// runbooks. Hand-maintained on entry edits.
774    pub authored_at: &'static str,
775}
776
777impl ReliabilityReasonCode {
778    /// Operator runbook for this reason code. Returns `Some(entry)` when
779    /// an authored runbook exists; `None` for Pass/Info-state codes that
780    /// don't need a runbook (they're not the kind of thing an operator
781    /// gets paged on). Per br-62u24.20 MVP scope, the highest-leverage
782    /// operational codes are authored first; remaining codes return
783    /// `None` and will be filled in as follow-up work.
784    #[must_use]
785    pub fn runbook(self) -> Option<RunbookEntry> {
786        use ReliabilityReasonCode::*;
787        // Ordered to mirror the variant enumeration for grep-ability.
788        // Each authored entry is the operator-facing source of truth
789        // for incident response on that code.
790        match self {
791            // ---------------- Topology ----------------
792            NoWorkersConfigured => Some(RunbookEntry {
793                symptoms: &[
794                    "`rch doctor --reliability` reports `RCH-R002` (Critical)",
795                    "All builds run locally regardless of project size",
796                    "`rch workers list` returns an empty list",
797                ],
798                diagnosis_steps: &[
799                    "rch workers list --json | jq '.data | length'",
800                    "cat ~/.config/rch/workers.toml",
801                ],
802                remediation_steps: &[
803                    "rch workers init                # interactive wizard: host, SSH key, slots",
804                    "rch workers probe --all        # confirm SSH + tools reachable",
805                ],
806                verification_command: "rch doctor --reliability --scope=topology --json",
807                escalation: Some(
808                    "If no remote worker is available, the project may need a local-only \
809                     config: `rch config set general.enabled=false` to skip the hook entirely.",
810                ),
811                references: &[
812                    "AGENTS.md > Architecture > Worker Fleet Management",
813                    "rch workers init --help",
814                ],
815                authored_at: "2026-05-16",
816            }),
817            AllWorkersUnhealthy => Some(RunbookEntry {
818                symptoms: &[
819                    "`rch doctor --reliability` reports `RCH-R006` (Critical)",
820                    "0/N workers healthy in the topology summary",
821                    "Every build falls back to local execution",
822                    "Hook latency spikes (no fast remote target)",
823                ],
824                diagnosis_steps: &[
825                    "rch status --workers --json | jq '.data.workers[] | {id, status, last_error}'",
826                    "rch workers probe --all --verbose",
827                    "for w in $(rch workers list --json | jq -r '.data[].id'); do echo \"--- $w ---\"; rch workers probe --worker=$w --verbose; done",
828                ],
829                remediation_steps: &[
830                    "# 1. Check network from this host:",
831                    "rch workers probe --all  # if it hangs, network is the issue",
832                    "# 2. Restart the daemon to clear stale circuit breakers:",
833                    "rch daemon restart",
834                    "# 3. If specific workers are down, evict them temporarily:",
835                    "rch workers disable <id>",
836                ],
837                verification_command: "rch doctor --reliability --scope=topology",
838                escalation: Some(
839                    "If all workers are reachable via raw SSH but RCH still reports \
840                     unhealthy, capture daemon logs with `rch daemon logs --tail=200 --json` \
841                     and file an issue. This is likely a daemon-side regression.",
842                ),
843                references: &[
844                    "AGENTS.md > Performance > Worker Selection Algorithm",
845                    "RCH-R007 (PartialWorkerCapacity) — same diagnosis path, partial scope",
846                ],
847                authored_at: "2026-05-16",
848            }),
849            PartialWorkerCapacity => Some(RunbookEntry {
850                symptoms: &[
851                    "`rch doctor --reliability` reports `RCH-R007` (Warning)",
852                    "M/N workers healthy where M < N",
853                    "Builds still succeed but on a shrunk fleet",
854                ],
855                diagnosis_steps: &[
856                    "rch status --workers --json | jq '.data.workers[] | select(.status != \"ready\") | {id, status, last_error}'",
857                    "rch workers probe --all --verbose 2>&1 | grep -E 'FAIL|ERROR'",
858                ],
859                remediation_steps: &[
860                    "# Identify the unhealthy worker(s):",
861                    "UNHEALTHY=$(rch status --workers --json | jq -r '.data.workers[] | select(.status != \"ready\") | .id')",
862                    "# Probe each in turn for forensics:",
863                    "for w in $UNHEALTHY; do rch workers probe --worker=$w --verbose; done",
864                    "# If the cause is a transient network blip, the daemon may auto-recover.",
865                    "# Force-clear the circuit breaker if needed:",
866                    "rch workers reset-circuit --worker=<id>",
867                ],
868                verification_command: "rch doctor --reliability --scope=topology",
869                escalation: None,
870                references: &["RCH-R006 (AllWorkersUnhealthy) — same playbook, full-outage scope"],
871                authored_at: "2026-05-16",
872            }),
873            WorkerUnreachable => Some(RunbookEntry {
874                symptoms: &[
875                    "Single worker reports `RCH-R010` (Critical) in topology",
876                    "`rch workers probe --worker=<id>` fails with SSH error",
877                    "Builds skip this worker via selection but fleet still works",
878                ],
879                diagnosis_steps: &[
880                    "rch workers probe --worker=<id> --verbose",
881                    "ssh -i <identity> <user>@<host> 'echo ok'  # raw SSH test",
882                    "rch workers list --json | jq '.data[] | select(.id==\"<id>\")'",
883                ],
884                remediation_steps: &[
885                    "# If raw SSH works but rch probe fails, the daemon's SSH cache may be stale:",
886                    "rch daemon restart",
887                    "# If raw SSH also fails, check the host/network/firewall:",
888                    "ping <host>",
889                    "ssh -vvv -i <identity> <user>@<host>",
890                ],
891                verification_command: "rch workers probe --worker=<id>",
892                escalation: None,
893                references: &["AGENTS.md > Worker Fleet Management"],
894                authored_at: "2026-05-16",
895            }),
896            WorkerCircuitOpen => Some(RunbookEntry {
897                symptoms: &[
898                    "Worker reports `RCH-R009` (Warning)",
899                    "Selection algorithm refuses to route builds to this worker",
900                    "Build failures from this worker recently exceeded the threshold",
901                ],
902                diagnosis_steps: &[
903                    "rch status --workers --json | jq '.data.workers[] | select(.id==\"<id>\") | {circuit_state, consecutive_failures, last_error}'",
904                    "rch daemon logs --tail=200 | grep <worker-id>",
905                ],
906                remediation_steps: &[
907                    "# Manual reset (use after confirming the underlying cause is resolved):",
908                    "rch workers reset-circuit --worker=<id>",
909                    "# Or wait for auto-recovery (typically 5 minutes after the last failure)",
910                ],
911                verification_command: "rch doctor --reliability --scope=topology",
912                escalation: None,
913                references: &["rchd/src/selection.rs > circuit-breaker policy"],
914                authored_at: "2026-05-16",
915            }),
916
917            // ---------------- Daemon ----------------
918            DaemonStatusUnavailable => Some(RunbookEntry {
919                symptoms: &[
920                    "`rch doctor --reliability` reports `RCH-R004` (Warning)",
921                    "Daemon socket not responding to `/status`",
922                    "`rch status` hangs or errors",
923                ],
924                diagnosis_steps: &[
925                    "ls -la /tmp/rch.sock        # socket exists?",
926                    "pgrep -af rchd              # process running?",
927                    "rch daemon logs --tail=50",
928                ],
929                remediation_steps: &[
930                    "rch daemon start            # cold-start if not running",
931                    "rch daemon restart          # restart if process is wedged",
932                    "# If autostart is disabled in config, enable it:",
933                    "rch config set self_healing.hook_starts_daemon true",
934                ],
935                verification_command: "rch status --json",
936                escalation: Some(
937                    "If the daemon repeatedly fails to start, capture full diagnostics: \
938                     `rch doctor --reliability --json > /tmp/diag.json` and inspect with \
939                     `jq '.data.diagnostics[] | select(.severity != \"pass\")'`.",
940                ),
941                references: &["AGENTS.md > Architecture > Daemon Communication"],
942                authored_at: "2026-05-16",
943            }),
944
945            // ---------------- Disk pressure ----------------
946            WorkerDiskPressureCritical => Some(RunbookEntry {
947                symptoms: &[
948                    "Worker reports `RCH-R101` (Critical)",
949                    "Free space on the worker's build volume is below the critical threshold",
950                    "Subsequent rsync upload may fail with ENOSPC",
951                ],
952                diagnosis_steps: &[
953                    "rch status --workers --json | jq '.data.workers[] | select(.id==\"<id>\") | {pressure_disk_free_gb, pressure_confidence, pressure_state}'",
954                    "ssh -i <identity> <user>@<host> 'df -h /tmp /var'",
955                ],
956                remediation_steps: &[
957                    "# Stop new jobs from landing on the saturated worker:",
958                    "rch workers disable <id>",
959                    "# Confirm the remaining fleet can absorb builds while the worker is remediated:",
960                    "rch doctor --reliability --scope=topology",
961                ],
962                verification_command: "rch doctor --reliability --scope=pressure",
963                escalation: Some(
964                    "If disk pressure persists after the worker is disabled, perform \
965                     out-of-band worker cache cleanup only after confirming no jobs are \
966                     active and obtaining explicit approval for the cleanup command. If \
967                     disk fills again within hours, grow the volume or reduce \
968                     CARGO_TARGET_DIR retention.",
969                ),
970                references: &[
971                    "RCH-R102 (WorkerDiskPressureWarning) — same playbook, less urgent",
972                    "rchd/src/workers.rs > disk_pressure tracking",
973                ],
974                authored_at: "2026-05-16",
975            }),
976
977            // ---------------- Repo convergence ----------------
978            RepoConvergenceFailed => Some(RunbookEntry {
979                symptoms: &[
980                    "`rch doctor --reliability` reports `RCH-R301` (Critical)",
981                    "One or more workers have stale repo state vs the local checkout",
982                    "Builds may compile against the wrong revision",
983                ],
984                diagnosis_steps: &[
985                    "rch status --json | jq '.data.repo_convergence'",
986                    "rch workers probe --all --verbose | grep -E 'commit|revision'",
987                ],
988                remediation_steps: &[
989                    "# Force a fresh sync to all workers:",
990                    "rch cache warm",
991                    "# If a specific worker is the outlier, target it:",
992                    "rch cache warm --workers <id>",
993                ],
994                verification_command: "rch doctor --reliability --scope=convergence",
995                escalation: None,
996                references: &["br-4zm6u (rch cache warm subcommand)"],
997                authored_at: "2026-05-16",
998            }),
999
1000            // ---------------- Rollout posture ----------------
1001            ConfigLoadFailed => Some(RunbookEntry {
1002                symptoms: &[
1003                    "`rch doctor --reliability` reports `RCH-R504` (Critical)",
1004                    "rch commands fail to start with a config-parse error",
1005                    "Hook may be falling open (allowing local execution)",
1006                ],
1007                diagnosis_steps: &[
1008                    "rch config doctor --json",
1009                    "rch config show --sources",
1010                    "cat ~/.config/rch/config.toml | head -40",
1011                ],
1012                remediation_steps: &[
1013                    "# Validate the TOML structure first:",
1014                    "rch config lint",
1015                    "# If a key has the wrong type, the error message names the line.",
1016                    "# To start from a known-good template:",
1017                    "rch config init --force",
1018                ],
1019                verification_command: "rch doctor --reliability --scope=rollout",
1020                escalation: None,
1021                references: &["rch config doctor --help"],
1022                authored_at: "2026-05-16",
1023            }),
1024
1025            // ---------------- Helper compatibility ----------------
1026            HelperMissing => Some(RunbookEntry {
1027                symptoms: &[
1028                    "`rch doctor --reliability` reports `RCH-R401` (Critical/Warning)",
1029                    "rsync / ssh / zstd / cargo missing from PATH",
1030                    "Builds fail with `command not found`",
1031                ],
1032                diagnosis_steps: &[
1033                    "which rsync ssh zstd cargo",
1034                    "rch doctor --reliability --scope=helpers --json | jq '.data.diagnostics[] | select(.code==\"RCH-R401\")'",
1035                ],
1036                remediation_steps: &[
1037                    "# Install missing helpers via the system package manager:",
1038                    "# Ubuntu / Debian:",
1039                    "sudo apt-get install rsync zstd openssh-client",
1040                    "# macOS:",
1041                    "brew install rsync zstd",
1042                    "# Cargo: install via rustup (https://rustup.rs)",
1043                ],
1044                verification_command: "rch doctor --reliability --scope=helpers",
1045                escalation: None,
1046                references: &["AGENTS.md > Toolchain: Rust & Cargo"],
1047                authored_at: "2026-05-16",
1048            }),
1049
1050            // ---------------- Mirror ownership ----------------
1051            WorkerMirrorOwnershipDrift => Some(RunbookEntry {
1052                symptoms: &[
1053                    "`rch doctor --reliability` reports `RCH-R701` (Warning) per affected worker",
1054                    "Root-owned entries under the worker's canonical mirror tree",
1055                    "rsync uploads to this worker can fail exit 23 on replace/unlink",
1056                ],
1057                diagnosis_steps: &[
1058                    "rch doctor --reliability --scope=ownership --json | jq '.data.diagnostics[] | select(.code==\"RCH-R701\")'",
1059                    "ssh <user>@<host> 'sudo find /data/projects -xdev -user root -print | head -20'",
1060                ],
1061                remediation_steps: &[
1062                    "# The dispatch-time preflight self-heals drift automatically (bd-8iwkm);",
1063                    "# any offloaded build triggers the repair. To fix immediately:",
1064                    "ssh <user>@<host> 'sudo find /data/projects -xdev -user root -exec chown -h <user> {} +'",
1065                ],
1066                verification_command: "rch doctor --reliability --scope=ownership",
1067                escalation: None,
1068                references: &[
1069                    "bd-8iwkm (dispatch-time detect+repair)",
1070                    "RCH-R702 — sudo unavailable, repair fails open",
1071                ],
1072                authored_at: "2026-08-23",
1073            }),
1074
1075            // ---------------- All other variants return None ----------------
1076            //
1077            // Pass / Info codes describe healthy states and don't merit
1078            // a runbook (an operator wouldn't be paged on them).
1079            // Warning/Critical codes not yet authored fall through here
1080            // and are filed as follow-up MVP scope.
1081            _ => None,
1082        }
1083    }
1084
1085    /// Convenience: list every variant that has an authored runbook.
1086    /// Used by `rch doctor --runbook-list` to enumerate the codes
1087    /// an operator can request a runbook for.
1088    #[must_use]
1089    pub fn authored_runbook_codes() -> Vec<Self> {
1090        Self::ALL
1091            .iter()
1092            .copied()
1093            .filter(|c| c.runbook().is_some())
1094            .collect()
1095    }
1096}
1097
1098#[cfg(test)]
1099mod tests {
1100    use super::*;
1101    use std::collections::HashSet;
1102
1103    #[test]
1104    fn test_reliability_reason_codes_unique() {
1105        let mut seen = HashSet::new();
1106        for &c in ReliabilityReasonCode::ALL {
1107            assert!(
1108                seen.insert(c.code()),
1109                "duplicate code {} for variant {:?}",
1110                c.code(),
1111                c
1112            );
1113        }
1114        assert_eq!(seen.len(), ReliabilityReasonCode::ALL.len());
1115    }
1116
1117    #[test]
1118    fn test_reliability_reason_codes_format() {
1119        for &c in ReliabilityReasonCode::ALL {
1120            let code = c.code();
1121            assert!(
1122                code.starts_with("RCH-R")
1123                    && code[5..].len() == 3
1124                    && code[5..].chars().all(|ch| ch.is_ascii_digit()),
1125                "invalid code format {code} for {c:?}"
1126            );
1127        }
1128    }
1129
1130    #[test]
1131    fn test_reliability_reason_codes_in_documented_ranges() {
1132        for &c in ReliabilityReasonCode::ALL {
1133            let n: u32 = c.code()[5..].parse().expect("3-digit numeric");
1134            let cat = c.category();
1135            let expected_range = match cat {
1136                ReliabilityCategoryKind::Topology => 1..=99,
1137                ReliabilityCategoryKind::DiskPressure => 100..=199,
1138                ReliabilityCategoryKind::ProcessTriage => 200..=299,
1139                ReliabilityCategoryKind::RepoConvergence => 300..=399,
1140                ReliabilityCategoryKind::HelperCompatibility => 400..=499,
1141                ReliabilityCategoryKind::RolloutPosture => 500..=599,
1142                ReliabilityCategoryKind::SchemaCompatibility => 600..=699,
1143                ReliabilityCategoryKind::MirrorOwnership => 700..=799,
1144            };
1145            assert!(
1146                expected_range.contains(&n),
1147                "{c:?} code {} (n={n}) not in expected range {:?} for category {:?}",
1148                c.code(),
1149                expected_range,
1150                cat
1151            );
1152        }
1153    }
1154
1155    #[test]
1156    fn test_reliability_reason_codes_serde_roundtrip() {
1157        for &c in ReliabilityReasonCode::ALL {
1158            let s = serde_json::to_string(&c).unwrap();
1159            let d: ReliabilityReasonCode = serde_json::from_str(&s).unwrap();
1160            assert_eq!(c, d, "round-trip mismatch for {c:?}");
1161            // Also confirm the on-the-wire form is the RCH-Rnnn string.
1162            let expected = format!("\"{}\"", c.code());
1163            assert_eq!(s, expected);
1164        }
1165    }
1166
1167    #[test]
1168    fn test_reliability_reason_codes_remediation_non_empty() {
1169        for &c in ReliabilityReasonCode::ALL {
1170            let hint = c.remediation_hint();
1171            assert!(!hint.is_empty(), "empty remediation hint for {c:?}");
1172        }
1173    }
1174
1175    #[test]
1176    fn test_authored_runbook_primary_symptom_names_own_code() {
1177        for code in ReliabilityReasonCode::authored_runbook_codes() {
1178            let entry = code.runbook().expect("authored code has runbook");
1179            let primary = entry
1180                .symptoms
1181                .first()
1182                .expect("authored runbook should have at least one symptom");
1183            assert!(
1184                primary.contains(code.code()),
1185                "primary symptom for {:?} should mention its own code {}; got {primary:?}",
1186                code,
1187                code.code()
1188            );
1189        }
1190    }
1191
1192    #[test]
1193    fn test_authored_runbooks_do_not_suggest_destructive_cleanup_commands() {
1194        const FORBIDDEN_COMMANDS: &[&str] = &["rm -rf", "git reset --hard", "git clean -fd"];
1195
1196        for code in ReliabilityReasonCode::authored_runbook_codes() {
1197            let entry = code.runbook().expect("authored code has runbook");
1198            let lines = entry
1199                .symptoms
1200                .iter()
1201                .chain(entry.diagnosis_steps)
1202                .chain(entry.remediation_steps)
1203                .chain(std::iter::once(&entry.verification_command))
1204                .chain(entry.escalation.iter())
1205                .chain(entry.references);
1206
1207            for line in lines {
1208                for forbidden in FORBIDDEN_COMMANDS {
1209                    assert!(
1210                        !line.contains(forbidden),
1211                        "runbook {} must not suggest forbidden command {forbidden:?}: {line}",
1212                        code.code()
1213                    );
1214                }
1215            }
1216        }
1217    }
1218
1219    #[test]
1220    fn test_unknown_code_deserialize_fails_clearly() {
1221        let r: Result<ReliabilityReasonCode, _> = serde_json::from_str("\"RCH-R999\"");
1222        let err = r.expect_err("RCH-R999 should not deserialize");
1223        let msg = err.to_string();
1224        assert!(
1225            msg.contains("RCH-R999"),
1226            "error should name the unknown code, got: {msg}"
1227        );
1228    }
1229
1230    #[test]
1231    fn test_display_matches_code() {
1232        for &c in ReliabilityReasonCode::ALL {
1233            assert_eq!(format!("{c}"), c.code().to_string());
1234        }
1235    }
1236
1237    #[test]
1238    fn test_requires_restart_explicit_for_every_reason() {
1239        // Rust's exhaustive match enforces this; the test exists to catch a
1240        // future "_ => false" wildcard from regressing the discipline.
1241        for &c in ReliabilityReasonCode::ALL {
1242            // Just call it; if there's a panic-on-miss, this catches it.
1243            let _ = c.requires_restart();
1244        }
1245    }
1246
1247    /// Pinned policy table mirroring the [bead body's table for
1248    /// `2s99h.15`](https://example.invalid). The implementation
1249    /// [`ReliabilityReasonCode::requires_restart`] is the canonical authority;
1250    /// this table is duplicate state used to:
1251    /// 1. Detect implementation drift (reviewer sees both edits in the diff).
1252    /// 2. Provide a paste-ready reference table for documentation.
1253    ///
1254    /// Adding a new variant requires updating BOTH the impl AND this table —
1255    /// surfacing the rationale in code review. The
1256    /// [`test_requires_restart_table_matches_impl`] test enforces the match.
1257    const REQUIRES_RESTART_TABLE: &[(ReliabilityReasonCode, bool)] = &[
1258        // Topology
1259        (ReliabilityReasonCode::WorkersConfigUnreadable, true),
1260        (ReliabilityReasonCode::NoWorkersConfigured, true),
1261        (ReliabilityReasonCode::WorkersConfigured, false),
1262        (ReliabilityReasonCode::DaemonStatusUnavailable, true),
1263        (ReliabilityReasonCode::DaemonHasNoWorkers, true),
1264        (ReliabilityReasonCode::AllWorkersUnhealthy, false),
1265        (ReliabilityReasonCode::PartialWorkerCapacity, false),
1266        (ReliabilityReasonCode::WorkersHealthy, false),
1267        (ReliabilityReasonCode::WorkerCircuitOpen, false),
1268        (ReliabilityReasonCode::WorkerUnreachable, false),
1269        (ReliabilityReasonCode::WorkerDegraded, false),
1270        (ReliabilityReasonCode::WorkerReady, false),
1271        (ReliabilityReasonCode::WorkerStatusUnrecognized, false),
1272        (ReliabilityReasonCode::WorkerCircuitStateUnrecognized, false),
1273        // DiskPressure
1274        (ReliabilityReasonCode::DiskPressureUnavailable, false),
1275        (ReliabilityReasonCode::WorkerDiskPressureCritical, false),
1276        (ReliabilityReasonCode::WorkerDiskPressureWarning, false),
1277        (ReliabilityReasonCode::WorkerDiskPressureHealthy, false),
1278        (ReliabilityReasonCode::WorkerDiskPressureTelemetryGap, false),
1279        (ReliabilityReasonCode::DiskPressureNoWorkers, false),
1280        // ProcessTriage
1281        (ReliabilityReasonCode::ProcessDebtUnavailable, true),
1282        (ReliabilityReasonCode::CancellationCleanupHealthy, false),
1283        (ReliabilityReasonCode::CancellationCleanupSkipped, false),
1284        (ReliabilityReasonCode::CancellationCleanupDegraded, false),
1285        (ReliabilityReasonCode::CancellationCleanupFailed, true),
1286        (ReliabilityReasonCode::LocalBuildsDetected, false),
1287        (ReliabilityReasonCode::LocalBuildScanUnavailable, false),
1288        (
1289            ReliabilityReasonCode::LocalBuildAlarmStateUnavailable,
1290            false,
1291        ),
1292        (ReliabilityReasonCode::LocalBuildsAbsent, false),
1293        // RepoConvergence
1294        (ReliabilityReasonCode::RepoConvergenceUnavailable, false),
1295        (ReliabilityReasonCode::RepoConvergenceFailed, false),
1296        (ReliabilityReasonCode::RepoConvergenceDrift, false),
1297        (ReliabilityReasonCode::RepoConvergenceNoWorkers, false),
1298        (ReliabilityReasonCode::RepoConvergenceReady, false),
1299        (ReliabilityReasonCode::WorkerRepoNotReady, false),
1300        // HelperCompatibility
1301        (ReliabilityReasonCode::HelperAvailable, false),
1302        (ReliabilityReasonCode::HelperMissing, false),
1303        (ReliabilityReasonCode::HelperProbeUnavailable, false),
1304        // RolloutPosture
1305        (ReliabilityReasonCode::HookAutoStartEnabled, false),
1306        (ReliabilityReasonCode::HookAutoStartDisabled, true),
1307        (ReliabilityReasonCode::DaemonHookRepairEnabled, false),
1308        (ReliabilityReasonCode::DaemonHookRepairDisabled, true),
1309        (ReliabilityReasonCode::ConfigLoadFailed, false),
1310        (ReliabilityReasonCode::StatusSurfaceAvailable, false),
1311        (
1312            ReliabilityReasonCode::RepoConvergenceSurfaceAvailable,
1313            false,
1314        ),
1315        (ReliabilityReasonCode::DiskPressureSurfaceAvailable, false),
1316        (ReliabilityReasonCode::HookInstalled, false),
1317        (ReliabilityReasonCode::HookNotInstalled, false),
1318        (ReliabilityReasonCode::SocketPathConsistent, false),
1319        (ReliabilityReasonCode::SocketPathMismatch, false),
1320        // SchemaCompatibility
1321        (ReliabilityReasonCode::SchemaCompatible, false),
1322        (ReliabilityReasonCode::SchemaIncompatible, true),
1323        // MirrorOwnership
1324        (ReliabilityReasonCode::WorkerMirrorOwnershipHealthy, false),
1325        (ReliabilityReasonCode::WorkerMirrorOwnershipDrift, false),
1326        (
1327            ReliabilityReasonCode::WorkerMirrorOwnershipCheckUnavailable,
1328            false,
1329        ),
1330        (
1331            ReliabilityReasonCode::WorkerMirrorOwnershipUnprobeable,
1332            false,
1333        ),
1334        (ReliabilityReasonCode::MirrorOwnershipNoWorkers, false),
1335    ];
1336
1337    #[test]
1338    fn test_requires_restart_table_matches_impl() {
1339        // Every entry in the pinned table must agree with the impl AND
1340        // every variant must appear in the table (in the same order as
1341        // ALL_COMPONENTS). Editing requires_restart() without updating the
1342        // table — or vice versa — triggers a clear failure.
1343        assert_eq!(
1344            REQUIRES_RESTART_TABLE.len(),
1345            ReliabilityReasonCode::ALL.len(),
1346            "REQUIRES_RESTART_TABLE has {} entries but {} variants exist. \
1347             Add or remove the corresponding entry when changing the variant set.",
1348            REQUIRES_RESTART_TABLE.len(),
1349            ReliabilityReasonCode::ALL.len()
1350        );
1351
1352        for (i, ((variant, expected), &impl_variant)) in REQUIRES_RESTART_TABLE
1353            .iter()
1354            .zip(ReliabilityReasonCode::ALL.iter())
1355            .enumerate()
1356        {
1357            assert_eq!(
1358                *variant, impl_variant,
1359                "Position {i}: REQUIRES_RESTART_TABLE has {variant:?} but ALL has {impl_variant:?}. \
1360                 Tables must be in identical order — easier diff review.",
1361            );
1362            let actual = variant.requires_restart();
1363            assert_eq!(
1364                *expected, actual,
1365                "Policy mismatch for {variant:?}: table says {expected}, impl says {actual}. \
1366                 Update BOTH or NEITHER.",
1367            );
1368        }
1369    }
1370
1371    #[test]
1372    fn test_requires_restart_consistency_with_remediation_hint() {
1373        // Heuristic: if the remediation hint mentions "restart", the variant
1374        // SHOULD have requires_restart=true. Catches drift between the
1375        // operator-facing hint and the policy bool. (Limited to the obvious
1376        // case — no false-positive on hints that mention "rch daemon
1377        // restart" only as a remediation command for a non-restart variant.)
1378        for &c in ReliabilityReasonCode::ALL {
1379            let hint = c.remediation_hint().to_lowercase();
1380            // Only flag the case where hint says "restart" but bool says false.
1381            // The reverse (bool=true, no "restart" in hint) is fine since the
1382            // hint may use a different idiom (e.g., "reinstall both binaries").
1383            if hint.contains("restart") {
1384                assert!(
1385                    c.requires_restart(),
1386                    "Variant {c:?} hint mentions 'restart' but requires_restart() returns false. \
1387                     Either update the hint to NOT say 'restart' or set requires_restart=true."
1388                );
1389            }
1390        }
1391    }
1392}
rch_common/errors/reliability.rs

rch_common/errors/
reliability.rs
