Skip to main content

rch_common/
types.rs

1//! Common types used across RCH components.
2
3use crate::{CompilationKind, toolchain::ToolchainInfo};
4use rand::RngExt;
5use schemars::JsonSchema;
6use serde::{Deserialize, Serialize};
7use std::path::PathBuf;
8
9/// Unique identifier for a worker in the fleet.
10#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
11pub struct WorkerId(pub String);
12
13impl WorkerId {
14    pub fn new(id: impl Into<String>) -> Self {
15        Self(id.into())
16    }
17
18    pub fn as_str(&self) -> &str {
19        &self.0
20    }
21}
22
23impl std::fmt::Display for WorkerId {
24    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
25        write!(f, "{}", self.0)
26    }
27}
28
29/// Status of a worker in the fleet.
30///
31/// State transitions:
32/// ```text
33/// HEALTHY ←→ DEGRADED (automatic based on response times)
34///    ↓           ↓
35///    ↓       UNREACHABLE (automatic on heartbeat failure)
36///    ↓
37/// DRAINING (via `rch workers drain`) - finishing current jobs
38///    ↓
39/// DRAINED (automatic when all jobs complete) - idle, ready to disable
40///    ↓
41/// DISABLED (via `rch workers disable`) - completely offline
42///
43/// Use `rch workers enable` to return from DRAINING/DRAINED/DISABLED → HEALTHY
44/// ```
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
46#[serde(rename_all = "snake_case")]
47pub enum WorkerStatus {
48    /// Worker is healthy and accepting jobs.
49    #[default]
50    Healthy,
51    /// Worker is responding slowly.
52    Degraded,
53    /// Worker failed to respond to heartbeat.
54    Unreachable,
55    /// Worker is not accepting new jobs (finishing current).
56    Draining,
57    /// Worker has finished draining (no active jobs, idle).
58    Drained,
59    /// Worker is manually disabled.
60    Disabled,
61}
62
63/// Circuit breaker state for a worker.
64#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
65#[serde(rename_all = "snake_case")]
66pub enum CircuitState {
67    /// Normal operation.
68    #[default]
69    Closed,
70    /// Circuit is open (short-circuit).
71    Open,
72    /// Circuit is half-open (probing).
73    HalfOpen,
74}
75
76/// Required runtime for command execution.
77#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
78#[serde(rename_all = "lowercase")]
79pub enum RequiredRuntime {
80    /// No specific runtime required (default).
81    #[default]
82    None,
83    /// Requires Rust toolchain.
84    Rust,
85    /// Requires Bun runtime.
86    Bun,
87    /// Requires Node.js runtime.
88    Node,
89    /// Requires the Nix package manager (nix binary + populated `/nix/store`).
90    Nix,
91    /// Requires the Go toolchain (`go` binary).
92    Go,
93    /// Requires the zig cross-compilation toolchain (`cargo-zigbuild` + `zig`).
94    Zig,
95}
96
97/// Per-command priority hint for worker selection.
98///
99/// This is an *input hint* from the caller (typically the hook) and should not
100/// change default behavior when omitted.
101#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
102#[serde(rename_all = "lowercase")]
103pub enum CommandPriority {
104    Low,
105    #[default]
106    Normal,
107    High,
108}
109
110impl std::fmt::Display for CommandPriority {
111    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112        let value = match self {
113            Self::Low => "low",
114            Self::Normal => "normal",
115            Self::High => "high",
116        };
117        write!(f, "{}", value)
118    }
119}
120
121impl std::str::FromStr for CommandPriority {
122    type Err = ();
123
124    fn from_str(s: &str) -> Result<Self, Self::Err> {
125        match s.trim().to_lowercase().as_str() {
126            "low" => Ok(Self::Low),
127            "normal" => Ok(Self::Normal),
128            "high" => Ok(Self::High),
129            _ => Err(()),
130        }
131    }
132}
133
134/// Worker selection request sent from hook to daemon.
135#[derive(Debug, Clone, Serialize, Deserialize)]
136pub struct SelectionRequest {
137    /// Project identifier (usually directory name or hash).
138    pub project: String,
139    /// Full command being executed (optional, for active build tracking).
140    #[serde(default, skip_serializing_if = "Option::is_none")]
141    pub command: Option<String>,
142    /// Priority hint for this request.
143    #[serde(default)]
144    pub command_priority: CommandPriority,
145    /// Estimated CPU cores needed for this compilation.
146    pub estimated_cores: u32,
147    /// Additional free GiB required on the worker's reported build filesystem.
148    /// A positive value is a hard admission budget, retained by the daemon for
149    /// this build's lifetime; zero leaves disk admission to the ordinary policy.
150    #[serde(default, skip_serializing_if = "disk_headroom_disabled")]
151    pub disk_headroom_gib: u32,
152    /// Preferred worker IDs (e.g., from project config).
153    #[serde(default)]
154    pub preferred_workers: Vec<WorkerId>,
155    /// Rust toolchain information for the project.
156    #[serde(default)]
157    pub toolchain: Option<ToolchainInfo>,
158    /// Required runtime for command execution.
159    #[serde(default)]
160    pub required_runtime: RequiredRuntime,
161    /// Classification decision latency in microseconds (for AGENTS.md compliance).
162    /// This tracks how long the 5-tier classification took on the hook side.
163    #[serde(default, skip_serializing_if = "Option::is_none")]
164    pub classification_duration_us: Option<u64>,
165    /// Job-mode admission (bd-g7rpy): set by `rch exec --job` only. When the
166    /// one-active-job-per-project-per-worker guard excludes every otherwise-
167    /// eligible worker, a job request may QUEUE on that exclusion instead of
168    /// failing over to local execution. Compilation requests never set it,
169    /// preserving their immediate NoAdmissibleWorkers behavior.
170    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
171    pub job_mode: bool,
172    /// Operator-declared named tools this work requires VERIFIED on the worker
173    /// (`rch exec --job --require-tool NAME`, or a project's
174    /// `jobs.required_tools`).
175    ///
176    /// Job mode only — the compilation classifier never sets it, so ordinary
177    /// offloaded builds keep their existing admission behavior exactly. Empty
178    /// means no gate; a name no worker has verified excludes every worker,
179    /// because the caller asked for the gate precisely because the job cannot
180    /// run without the tool.
181    #[serde(default, skip_serializing_if = "Vec::is_empty")]
182    pub required_tools: Vec<String>,
183    /// Process ID of the hook (for active build tracking).
184    #[serde(default, skip_serializing_if = "Option::is_none")]
185    pub hook_pid: Option<u32>,
186}
187
188fn disk_headroom_disabled(gib: &u32) -> bool {
189    *gib == 0
190}
191
192/// Reason for worker selection result.
193///
194/// Provides context when no worker is available, enabling informative
195/// fallback messages in the hook.
196#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
197#[serde(rename_all = "snake_case")]
198pub enum SelectionReason {
199    /// Worker assigned successfully.
200    Success,
201    /// No workers configured in workers.toml.
202    NoWorkersConfigured,
203    /// All workers are unreachable (failed health checks).
204    AllWorkersUnreachable,
205    /// All workers have circuits open (after repeated failures).
206    AllCircuitsOpen,
207    /// All workers are at capacity (no available slots).
208    AllWorkersBusy,
209    /// Workers were present but none passed selection health thresholds.
210    NoWorkersPassedHealth,
211    /// All candidate workers failed hard preflight checks.
212    AllWorkersFailedPreflight,
213    /// All candidate workers failed repo convergence checks (repos missing/stale/failed).
214    AllWorkersFailedConvergence,
215    /// Workers exist but admission was blocked by concrete capacity/preflight reasons.
216    NoAdmissibleWorkers(String),
217    /// No workers match required tags or preferences.
218    NoMatchingWorkers,
219    /// No workers have the required runtime (e.g., Bun, Node).
220    NoWorkersWithRuntime(String),
221    /// Internal error during selection.
222    SelectionError(String),
223    /// Worker assigned via affinity pinning (recent successful build).
224    AffinityPinned,
225    /// Worker assigned via last-success fallback (all others unavailable).
226    AffinityFallback,
227}
228
229/// Wire protocol version for daemon `/select-worker` responses.
230///
231/// This is intentionally separate from the broader CLI API envelope version:
232/// selection responses are emitted as a raw [`SelectionResponse`] on the hook
233/// hot path, so the client needs a small explicit compatibility marker.
234pub const SELECTION_RESPONSE_PROTOCOL_VERSION: u64 = 1;
235
236impl std::fmt::Display for SelectionReason {
237    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
238        match self {
239            Self::Success => write!(f, "worker assigned successfully"),
240            Self::NoWorkersConfigured => write!(f, "no workers configured"),
241            Self::AllWorkersUnreachable => write!(f, "all workers unreachable"),
242            Self::AllCircuitsOpen => write!(f, "all worker circuits open"),
243            Self::AllWorkersBusy => write!(f, "all workers at capacity"),
244            Self::NoWorkersPassedHealth => write!(f, "no workers passed health thresholds"),
245            Self::AllWorkersFailedPreflight => write!(f, "all workers failed preflight checks"),
246            Self::AllWorkersFailedConvergence => {
247                write!(f, "all workers failed repo convergence checks")
248            }
249            Self::NoAdmissibleWorkers(summary) => {
250                write!(f, "no admissible workers: {}", summary)
251            }
252            Self::NoMatchingWorkers => write!(f, "no matching workers found"),
253            Self::NoWorkersWithRuntime(rt) => write!(f, "no workers with {} installed", rt),
254            Self::SelectionError(e) => write!(f, "selection error: {}", e),
255            Self::AffinityPinned => write!(f, "worker assigned via affinity pinning"),
256            Self::AffinityFallback => write!(f, "worker assigned via last-success fallback"),
257        }
258    }
259}
260
261// ============================================================================
262// Worker Selection Strategy
263// ============================================================================
264
265/// Worker selection strategy determining how workers are chosen for jobs.
266#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
267#[serde(rename_all = "snake_case")]
268pub enum SelectionStrategy {
269    /// Legacy/manual control: prioritize worker.priority first, then break ties
270    /// using command-aware cache/speed hints.
271    Priority,
272    /// Select worker with highest SpeedScore.
273    /// Best for performance-critical builds with homogeneous workers.
274    Fastest,
275    /// Balance all factors: SpeedScore, load, health, cache affinity.
276    /// Default for general use across diverse worker pools.
277    #[default]
278    Balanced,
279    /// Prefer workers with warm caches for the project.
280    /// Best for incremental builds on large codebases.
281    CacheAffinity,
282    /// Weighted random selection favoring fast workers but ensuring fairness.
283    /// Prevents hot-spotting when many workers are available.
284    FairFastest,
285}
286
287impl std::fmt::Display for SelectionStrategy {
288    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
289        let name = match self {
290            Self::Priority => "priority",
291            Self::Fastest => "fastest",
292            Self::Balanced => "balanced",
293            Self::CacheAffinity => "cache_affinity",
294            Self::FairFastest => "fair_fastest",
295        };
296        write!(f, "{}", name)
297    }
298}
299
300impl std::str::FromStr for SelectionStrategy {
301    type Err = String;
302
303    fn from_str(s: &str) -> Result<Self, Self::Err> {
304        match s.trim().to_lowercase().as_str() {
305            "priority" => Ok(Self::Priority),
306            "fastest" => Ok(Self::Fastest),
307            "balanced" => Ok(Self::Balanced),
308            "cache_affinity" | "cache-affinity" | "cacheaffinity" => Ok(Self::CacheAffinity),
309            "fair_fastest" | "fair-fastest" | "fairfastest" => Ok(Self::FairFastest),
310            _ => Err(format!(
311                "unknown selection strategy '{}', expected one of: priority, fastest, balanced, cache_affinity, fair_fastest",
312                s
313            )),
314        }
315    }
316}
317
318/// Configuration for the worker selection algorithm.
319#[derive(Debug, Clone, Serialize, Deserialize)]
320pub struct SelectionConfig {
321    /// Selection strategy to use.
322    #[serde(default)]
323    pub strategy: SelectionStrategy,
324    /// Minimum success rate (0.0-1.0) for a worker to be considered.
325    #[serde(default = "default_min_success_rate")]
326    pub min_success_rate: f64,
327    /// Factor weights for the balanced strategy.
328    #[serde(default)]
329    pub weights: SelectionWeightConfig,
330    /// Fairness settings for fair_fastest strategy.
331    #[serde(default)]
332    pub fairness: FairnessConfig,
333    /// Cache affinity settings for project-to-worker pinning.
334    #[serde(default)]
335    pub affinity: AffinityConfig,
336
337    // Preflight health guards (bd-3eaa)
338    /// Maximum load-per-core threshold. Workers exceeding this are skipped.
339    /// Set to None to disable load-based filtering.
340    #[serde(default = "default_max_load_per_core")]
341    pub max_load_per_core: Option<f64>,
342    /// Minimum free disk space in GB. Workers below this are skipped.
343    /// Set to None to disable the free-space floor; per-slot disk derating still applies.
344    #[serde(default = "default_min_free_gb")]
345    pub min_free_gb: Option<f64>,
346    /// Disk budget in GiB per concurrent slot, after reserving `min_free_gb`.
347    /// Must be finite and positive. Unknown disk telemetry preserves configured slots.
348    #[serde(
349        default = "default_disk_gb_per_slot",
350        deserialize_with = "deserialize_disk_gb_per_slot"
351    )]
352    pub disk_gb_per_slot: f64,
353    /// Free-disk ratio at or below which rchd marks a worker Critical and
354    /// gives it no slots (bd-xxck3). Read by rchd at startup.
355    #[serde(default = "default_disk_critical_free_ratio")]
356    pub disk_critical_free_ratio: f64,
357    /// When set, the ratio rule above trips only while free space is also at
358    /// or below this many GB, so a large disk is judged by absolute headroom
359    /// (5% of 900 GB is 45 GB). Unset keeps the ratio rule unconditional.
360    #[serde(default, skip_serializing_if = "Option::is_none")]
361    pub disk_critical_ratio_cap_gb: Option<f64>,
362}
363
364fn default_disk_critical_free_ratio() -> f64 {
365    0.05
366}
367
368impl Default for SelectionConfig {
369    fn default() -> Self {
370        Self {
371            strategy: SelectionStrategy::default(),
372            min_success_rate: default_min_success_rate(),
373            weights: SelectionWeightConfig::default(),
374            fairness: FairnessConfig::default(),
375            affinity: AffinityConfig::default(),
376            max_load_per_core: default_max_load_per_core(),
377            min_free_gb: default_min_free_gb(),
378            disk_gb_per_slot: default_disk_gb_per_slot(),
379            disk_critical_free_ratio: default_disk_critical_free_ratio(),
380            disk_critical_ratio_cap_gb: None,
381        }
382    }
383}
384
385fn default_min_success_rate() -> f64 {
386    0.8
387}
388
389fn default_max_load_per_core() -> Option<f64> {
390    Some(2.0) // Skip workers with load/core > 2.0
391}
392
393fn default_min_free_gb() -> Option<f64> {
394    Some(10.0) // Skip workers with < 10 GB free
395}
396
397fn default_disk_gb_per_slot() -> f64 {
398    10.0
399}
400
401fn deserialize_disk_gb_per_slot<'de, D>(deserializer: D) -> Result<f64, D::Error>
402where
403    D: serde::Deserializer<'de>,
404{
405    let budget = f64::deserialize(deserializer)?;
406    if budget.is_finite() && budget > 0.0 {
407        Ok(budget)
408    } else {
409        Err(serde::de::Error::custom(
410            "disk_gb_per_slot must be finite and greater than zero",
411        ))
412    }
413}
414
415/// Weight configuration for the balanced selection strategy.
416#[derive(Debug, Clone, Serialize, Deserialize)]
417pub struct SelectionWeightConfig {
418    /// Weight for SpeedScore (0.0-1.0).
419    #[serde(default = "default_weight_speedscore")]
420    pub speedscore: f64,
421    /// Weight for available slots (0.0-1.0).
422    #[serde(default = "default_weight_slots")]
423    pub slots: f64,
424    /// Weight for health/success rate (0.0-1.0).
425    #[serde(default = "default_weight_health")]
426    pub health: f64,
427    /// Weight for cache affinity (0.0-1.0).
428    #[serde(default = "default_weight_cache")]
429    pub cache: f64,
430    /// Weight for network latency (0.0-1.0).
431    #[serde(default = "default_weight_network")]
432    pub network: f64,
433    /// Weight for worker priority (0.0-1.0).
434    #[serde(default = "default_weight_priority")]
435    pub priority: f64,
436    /// Weight for disk headroom (0.0-1.0); zero disables disk ranking.
437    #[serde(
438        default = "default_weight_disk",
439        deserialize_with = "deserialize_weight_disk"
440    )]
441    pub disk: f64,
442    /// Penalty multiplier for half-open circuit workers (0.0-1.0).
443    #[serde(default = "default_half_open_penalty")]
444    pub half_open_penalty: f64,
445}
446
447impl Default for SelectionWeightConfig {
448    fn default() -> Self {
449        Self {
450            speedscore: default_weight_speedscore(),
451            slots: default_weight_slots(),
452            health: default_weight_health(),
453            cache: default_weight_cache(),
454            network: default_weight_network(),
455            priority: default_weight_priority(),
456            disk: default_weight_disk(),
457            half_open_penalty: default_half_open_penalty(),
458        }
459    }
460}
461
462fn default_weight_speedscore() -> f64 {
463    0.5
464}
465fn default_weight_slots() -> f64 {
466    0.4
467}
468fn default_weight_health() -> f64 {
469    0.3
470}
471fn default_weight_cache() -> f64 {
472    0.2
473}
474fn default_weight_network() -> f64 {
475    0.1
476}
477fn default_weight_priority() -> f64 {
478    // Worker priority is a first-class selection factor: operators set high
479    // priority on preferred boxes (e.g. fast dedicated workers) and expect them
480    // to actually receive work. At the old 0.1 a top-priority worker gained at
481    // most +0.1 to its balanced score — too weak to overcome a warm-cache or
482    // lower-latency competitor, so the highest-priority workers could sit idle
483    // indefinitely. Load/slots still throttle a high-priority worker as it fills.
484    0.5
485}
486fn default_half_open_penalty() -> f64 {
487    0.5
488}
489
490fn default_weight_disk() -> f64 {
491    0.2
492}
493
494fn deserialize_weight_disk<'de, D>(deserializer: D) -> Result<f64, D::Error>
495where
496    D: serde::Deserializer<'de>,
497{
498    let weight = f64::deserialize(deserializer)?;
499    if weight.is_finite() && (0.0..=1.0).contains(&weight) {
500        Ok(weight)
501    } else {
502        Err(serde::de::Error::custom(
503            "disk weight must be finite and between zero and one",
504        ))
505    }
506}
507
508/// Fairness settings for the fair_fastest selection strategy.
509#[derive(Debug, Clone, Serialize, Deserialize)]
510pub struct FairnessConfig {
511    /// Lookback window in seconds for tracking recent selections.
512    #[serde(default = "default_fairness_lookback_secs")]
513    pub lookback_secs: u64,
514    /// Maximum consecutive selections for a single worker before penalty.
515    #[serde(default = "default_max_consecutive_selections")]
516    pub max_consecutive_selections: u32,
517}
518
519impl Default for FairnessConfig {
520    fn default() -> Self {
521        Self {
522            lookback_secs: default_fairness_lookback_secs(),
523            max_consecutive_selections: default_max_consecutive_selections(),
524        }
525    }
526}
527
528fn default_fairness_lookback_secs() -> u64 {
529    300 // 5 minutes
530}
531
532fn default_max_consecutive_selections() -> u32 {
533    3
534}
535
536/// Cache affinity settings for project-to-worker pinning.
537#[derive(Debug, Clone, Serialize, Deserialize)]
538pub struct AffinityConfig {
539    /// Enable affinity pinning (pin projects to last successful worker).
540    #[serde(default = "default_affinity_enabled")]
541    pub enabled: bool,
542    /// Time window in minutes to pin a project to its last successful worker.
543    /// After this window expires, normal selection resumes.
544    #[serde(default = "default_affinity_pin_minutes")]
545    pub pin_minutes: u64,
546    /// Enable last-success fallback when no workers match selection criteria.
547    /// If enabled, the daemon will attempt the last successful worker for a project
548    /// when all other workers are unavailable or fail selection.
549    #[serde(default = "default_last_success_fallback")]
550    pub enable_last_success_fallback: bool,
551    /// Minimum success rate for the last-success worker to be used as fallback.
552    /// Must be lower than min_success_rate to be meaningful as a fallback.
553    #[serde(default = "default_fallback_min_success_rate")]
554    pub fallback_min_success_rate: f64,
555}
556
557impl Default for AffinityConfig {
558    fn default() -> Self {
559        Self {
560            enabled: default_affinity_enabled(),
561            pin_minutes: default_affinity_pin_minutes(),
562            enable_last_success_fallback: default_last_success_fallback(),
563            fallback_min_success_rate: default_fallback_min_success_rate(),
564        }
565    }
566}
567
568fn default_affinity_enabled() -> bool {
569    true
570}
571
572fn default_affinity_pin_minutes() -> u64 {
573    60 // 1 hour default
574}
575
576fn default_last_success_fallback() -> bool {
577    true
578}
579
580fn default_fallback_min_success_rate() -> f64 {
581    0.5 // More lenient than normal min_success_rate (0.8)
582}
583
584/// Details about a selected worker for remote compilation.
585#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
586pub struct SelectedWorker {
587    /// Selected worker ID.
588    pub id: WorkerId,
589    /// Host address for SSH.
590    pub host: String,
591    /// SSH user.
592    pub user: String,
593    /// Path to SSH identity file.
594    pub identity_file: String,
595    /// Number of slots available on this worker after reservation.
596    pub slots_available: u32,
597    /// Worker's speed score (0-100).
598    pub speed_score: f64,
599    /// Operator-declared host OS (e.g. `windows`), if any. Propagated so the
600    /// hook can pick the right transport (Windows workers use `C:/rch` +
601    /// tar-over-ssh instead of the Unix rsync path). `None` for the historical
602    /// unlabelled workers. `#[serde(default)]` keeps old daemons/hooks
603    /// wire-compatible.
604    #[serde(default)]
605    pub declared_os: Option<String>,
606}
607
608/// Worker selection response from daemon to hook.
609#[derive(Debug, Clone, Serialize, Deserialize)]
610pub struct SelectionResponse {
611    /// Selected worker details, if available.
612    pub worker: Option<SelectedWorker>,
613    /// Reason for the selection result.
614    pub reason: SelectionReason,
615    /// Optional build ID assigned by the daemon (for active build tracking).
616    #[serde(default, skip_serializing_if = "Option::is_none")]
617    pub build_id: Option<u64>,
618    /// Optional per-worker selector diagnostics for machine-readable triage.
619    #[serde(default, skip_serializing_if = "Option::is_none")]
620    pub diagnostics: Option<SelectionDiagnostics>,
621}
622
623/// Final selector decision for a worker considered during routing.
624#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
625#[serde(rename_all = "snake_case")]
626pub enum WorkerSelectionDiagnosticDecision {
627    /// Worker passed hard gates and normal health threshold.
628    Allow,
629    /// Worker failed normal health threshold but passed fallback threshold.
630    FallbackCandidate,
631    /// Worker was denied by a hard gate or both health thresholds.
632    Deny,
633}
634
635/// Per-worker selector facts captured at worker-selection time.
636#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
637pub struct WorkerSelectionDiagnostic {
638    /// Worker ID.
639    pub worker_id: WorkerId,
640    /// Worker lifecycle/status label.
641    pub status: String,
642    /// Circuit breaker state label.
643    pub circuit_state: String,
644    /// Current normalized pressure state.
645    pub pressure_state: String,
646    /// Stable pressure reason code.
647    pub pressure_reason_code: String,
648    /// Rolling selector success rate, where available.
649    #[serde(skip_serializing_if = "Option::is_none")]
650    pub success_rate: Option<f64>,
651    /// Normal selection success-rate threshold.
652    pub min_success_rate: f64,
653    /// Last-success fallback success-rate threshold.
654    pub fallback_min_success_rate: f64,
655    /// Required runtime for this command.
656    pub required_runtime: RequiredRuntime,
657    /// Whether this worker has the required runtime.
658    pub runtime_available: bool,
659    /// Available slots at selection time.
660    pub available_slots: u32,
661    /// Total configured slots.
662    pub total_slots: u32,
663    /// Estimated slots requested by this command.
664    pub estimated_cores: u32,
665    /// Whether this worker was excluded because it already runs this project.
666    pub active_project_excluded: bool,
667    /// Final per-worker selector decision.
668    pub final_decision: WorkerSelectionDiagnosticDecision,
669    /// Human-readable final reason.
670    pub final_reason: String,
671    /// Stable machine-readable reasons collected for this worker.
672    pub reason_codes: Vec<String>,
673}
674
675/// Machine-readable worker selector diagnostics for a selection request.
676#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
677pub struct SelectionDiagnostics {
678    /// Required runtime for this command.
679    pub required_runtime: RequiredRuntime,
680    /// Estimated slots requested by this command.
681    pub estimated_cores: u32,
682    /// Normal success-rate threshold.
683    pub min_success_rate: f64,
684    /// Last-success fallback success-rate threshold.
685    pub fallback_min_success_rate: f64,
686    /// Count of workers excluded because they already run this project.
687    pub active_project_exclusion_count: usize,
688    /// One diagnostic entry per configured worker.
689    pub workers: Vec<WorkerSelectionDiagnostic>,
690}
691
692/// Request to release reserved worker slots.
693#[derive(Debug, Clone, Serialize, Deserialize)]
694pub struct ReleaseRequest {
695    /// ID of the worker to release slots on.
696    pub worker_id: WorkerId,
697    /// Number of slots to release.
698    pub slots: u32,
699    /// Optional build ID to mark complete.
700    #[serde(default, skip_serializing_if = "Option::is_none")]
701    pub build_id: Option<u64>,
702    /// Exact client wrapper identity associated with the admitted build.
703    #[serde(default, skip_serializing_if = "Option::is_none")]
704    pub local_wrapper_id: Option<String>,
705    /// Optional exit code for the build (used to finalize active build tracking).
706    #[serde(default, skip_serializing_if = "Option::is_none")]
707    pub exit_code: Option<i32>,
708    /// Optional total duration in milliseconds (pipeline or execution).
709    ///
710    /// If omitted, the daemon will compute duration from the active build's start time.
711    #[serde(default, skip_serializing_if = "Option::is_none")]
712    pub duration_ms: Option<u64>,
713    /// Optional bytes transferred during the pipeline (upload + artifact download).
714    #[serde(default, skip_serializing_if = "Option::is_none")]
715    pub bytes_transferred: Option<u64>,
716    /// Optional per-phase timing breakdown for the build pipeline.
717    #[serde(default, skip_serializing_if = "Option::is_none")]
718    pub timing: Option<CommandTimingBreakdown>,
719    /// The hook classified this non-zero exit as a worker fault (missing
720    /// toolchain or system library, CPU-capability signal, full disk), so the
721    /// daemon records no cache warmth for it (review of GH #81).
722    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
723    pub worker_fault: bool,
724    /// The failed remote command reported exhausted disk space or quota.
725    /// After matching the durable build owner, the daemon temporarily bypasses
726    /// this worker until fresh recovery probes establish disk headroom.
727    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
728    pub worker_disk_full: bool,
729    /// Actual remote source and output roots used by the failed attempt.
730    /// Recovery must measure these filesystems as well as configured roots.
731    #[serde(default, skip_serializing_if = "Vec::is_empty")]
732    pub worker_disk_roots: Vec<String>,
733    /// The worker's source receiver refused the sync because its copy of the
734    /// project is a declared live checkout (`RCH_SYNC_LIVE_CHECKOUT_REFUSED`,
735    /// bd-fbtws). The daemon keeps that project off the worker for a while.
736    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
737    pub worker_live_checkout_refused: bool,
738}
739
740/// Build execution phase for daemon heartbeat tracking.
741#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
742#[serde(rename_all = "snake_case")]
743pub enum BuildHeartbeatPhase {
744    /// Uploading sources/dependencies to the worker.
745    SyncUp,
746    /// Running the remote compilation/test command.
747    Execute,
748    /// Downloading build artifacts from the worker.
749    SyncDown,
750    /// Build is completing or finalizing.
751    Finalize,
752}
753
754/// Request to update liveness/progress for an active build.
755#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
756pub struct BuildHeartbeatRequest {
757    /// Active build identifier assigned by the daemon.
758    pub build_id: u64,
759    /// Worker executing the build.
760    pub worker_id: WorkerId,
761    /// Hook process ID sending the heartbeat.
762    #[serde(default, skip_serializing_if = "Option::is_none")]
763    pub hook_pid: Option<u32>,
764    /// Client-minted wrapper identity for local durable-lease correlation.
765    #[serde(default, skip_serializing_if = "Option::is_none")]
766    pub local_wrapper_id: Option<String>,
767    /// Remote file containing the process-group leader PID for cancellation.
768    #[serde(default, skip_serializing_if = "Option::is_none")]
769    pub remote_pgid_file: Option<String>,
770    /// Current build execution phase.
771    pub phase: BuildHeartbeatPhase,
772    /// Optional human-readable progress detail.
773    #[serde(default, skip_serializing_if = "Option::is_none")]
774    pub detail: Option<String>,
775    /// Optional monotonic progress counter.
776    ///
777    /// The hook increments this when it observes meaningful progress
778    /// (phase transitions, new output, or richer progress signals).
779    #[serde(default, skip_serializing_if = "Option::is_none")]
780    pub progress_counter: Option<u64>,
781    /// Optional progress estimate in [0,100].
782    #[serde(default, skip_serializing_if = "Option::is_none")]
783    pub progress_percent: Option<f64>,
784}
785
786/// Configuration for a remote worker.
787#[derive(Debug, Clone, Serialize, Deserialize)]
788pub struct WorkerConfig {
789    /// Unique identifier for this worker.
790    pub id: WorkerId,
791    /// SSH hostname or IP address.
792    pub host: String,
793    /// SSH username.
794    pub user: String,
795    /// Path to SSH private key.
796    pub identity_file: String,
797    /// Total CPU slots available on this worker.
798    pub total_slots: u32,
799    /// Priority for worker selection (higher = preferred).
800    #[serde(default = "default_priority")]
801    pub priority: u32,
802    /// Optional tags for filtering.
803    ///
804    /// Mostly descriptive, with one reserved form: `os:<name>` (see
805    /// [`crate::declared_os`]) is a hard admission gate, written in
806    /// `workers.toml` as `os = "<name>"`.
807    #[serde(default)]
808    pub tags: Vec<String>,
809    /// Operator-declared named tool probes for this worker, written in
810    /// `workers.toml` as
811    /// `tools = [{ name = "clang", command = ["clang", "--version"] }]`.
812    ///
813    /// Unlike a tag, which is an unverified naming convention, each entry is
814    /// VERIFIED by running its fixed argv on the worker as the configured user;
815    /// only a zero exit makes the tool present. That is what lets
816    /// `--require-tool` gate selection on evidence instead of on a promise.
817    #[serde(default, skip_serializing_if = "Vec::is_empty")]
818    pub tools: Vec<WorkerToolProbe>,
819}
820
821/// One operator-declared tool probe as it appears in `workers.toml`.
822///
823/// Kept as plain configuration data; [`crate::capability_probe::NamedToolProbe`]
824/// is the validated form, and the conversion is where a malformed name is
825/// refused.
826#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
827pub struct WorkerToolProbe {
828    /// The name `--require-tool` and `jobs.required_tools` refer to.
829    pub name: String,
830    /// The exact argv to run. Zero exit = present; anything else = absent.
831    pub command: Vec<String>,
832}
833
834fn default_priority() -> u32 {
835    100
836}
837
838/// Tag prefix reserved for the operator-declared host OS.
839pub const OS_TAG_PREFIX: &str = "os:";
840
841/// The host OS an operator declared for a worker, from its tags.
842///
843/// Written in `workers.toml` as `os = "windows"` and normalized into the
844/// reserved `os:windows` tag on load, so it rides the existing tag vector
845/// through every path that already carries a [`WorkerConfig`].
846///
847/// Declaring it makes the worker **exclusive**: selection admits it only for
848/// commands that require that OS, so a Windows box cannot quietly absorb a
849/// `cargo check` dispatched from Linux and return wrong-platform artifacts.
850/// Workers without the tag — every historical worker — keep taking anything.
851#[must_use]
852pub fn declared_os(tags: &[String]) -> Option<String> {
853    // `find` over the mapped values, not `find_map` over the tags: a malformed
854    // `os:` entry must not mask a valid declaration later in the list, because
855    // failing open here is precisely the "Windows worker silently takes Linux
856    // jobs" outcome the gate exists to prevent.
857    tags.iter()
858        .filter_map(|tag| tag.strip_prefix(OS_TAG_PREFIX))
859        .map(|os| os.trim().to_ascii_lowercase())
860        .find(|os| !os.is_empty())
861}
862
863/// Build the reserved tag for a declared OS.
864#[must_use]
865pub fn os_tag(os: &str) -> String {
866    format!("{OS_TAG_PREFIX}{}", os.trim().to_ascii_lowercase())
867}
868
869impl Default for WorkerConfig {
870    fn default() -> Self {
871        Self {
872            id: WorkerId::new("default-worker"),
873            host: "localhost".to_string(),
874            user: "user".to_string(),
875            identity_file: "~/.ssh/id_rsa".to_string(),
876            total_slots: 4,
877            priority: default_priority(),
878            tags: Vec::new(),
879            tools: Vec::new(),
880        }
881    }
882}
883
884/// Runtime capabilities detected on a worker.
885///
886/// These are probed during health checks and cached for routing decisions.
887/// Commands requiring specific runtimes (e.g., `bun test`) can be routed
888/// only to workers with the corresponding capability.
889#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)]
890pub struct WorkerCapabilities {
891    /// Rust compiler version (from `rustc --version`).
892    #[serde(default, skip_serializing_if = "Option::is_none")]
893    pub rustc_version: Option<String>,
894    /// Installed rustup toolchains, including their host triple when rustup
895    /// reports one (for example `nightly-2026-07-05-x86_64-unknown-linux-gnu`).
896    #[serde(default, skip_serializing_if = "Vec::is_empty")]
897    pub rustup_toolchains: Vec<String>,
898    /// Installed rustup components as `<toolchain>:<component>` facts. Component
899    /// names are normalized by removing only the probed host-triple suffix, so
900    /// `clippy-preview` can never satisfy a `clippy` requirement.
901    #[serde(default, skip_serializing_if = "Vec::is_empty")]
902    pub rustup_components: Vec<String>,
903    /// Non-fatal diagnostics gathered while probing (for example a tool binary
904    /// that had to be resolved from a well-known fallback location because the
905    /// caller's PATH was minimal). Empty in the common case; clients may
906    /// surface these so "no facts" is distinguishable from "facts show
907    /// absence" (bd-deft5).
908    #[serde(default, skip_serializing_if = "Vec::is_empty")]
909    pub probe_warnings: Vec<String>,
910    /// Bun runtime version (from `bun --version`).
911    #[serde(default, skip_serializing_if = "Option::is_none")]
912    pub bun_version: Option<String>,
913    /// Node.js version (from `node --version`).
914    #[serde(default, skip_serializing_if = "Option::is_none")]
915    pub node_version: Option<String>,
916    /// npm version (from `npm --version`).
917    #[serde(default, skip_serializing_if = "Option::is_none")]
918    pub npm_version: Option<String>,
919    /// Nix version (from `nix --version`). Presence implies a usable nix binary
920    /// with an accompanying `/nix/store`, gating `nix build`/`nix develop -c`
921    /// routing to this worker.
922    #[serde(default, skip_serializing_if = "Option::is_none")]
923    pub nix_version: Option<String>,
924    /// Go toolchain version (from `go version`). Presence gates `go build`/
925    /// `go test`/`go vet` routing to this worker.
926    #[serde(default, skip_serializing_if = "Option::is_none")]
927    pub go_version: Option<String>,
928    /// Zig version (from `zig version`). `cargo zigbuild` shells out to `zig`
929    /// as the linker/C toolchain, so the bare `zig` binary is necessary but not
930    /// sufficient — see [`WorkerCapabilities::has_zig`].
931    #[serde(default, skip_serializing_if = "Option::is_none")]
932    pub zig_version: Option<String>,
933    /// cargo-zigbuild version (from `cargo-zigbuild --version`). Probed as the
934    /// hyphenated binary rather than `cargo zigbuild --version`, which the
935    /// subcommand's own parser rejects.
936    #[serde(default, skip_serializing_if = "Option::is_none")]
937    pub cargo_zigbuild_version: Option<String>,
938    /// x86-64 microarchitecture level (1..=4) parsed from `/proc/cpuinfo`
939    /// flags on the worker (bd-6qchz / bd-68hon item 4). `Some(2)` marks a
940    /// pre-AVX2 box (e.g. ovh-b's Ivy Bridge) whose build scripts SIGILL on
941    /// x86-64-v3 codegen; selection deprioritizes such workers proactively
942    /// while the reactive SIGILL quarantine remains the primary mechanism.
943    /// `None` on non-x86 workers or when the probe cannot read cpuinfo.
944    #[serde(default, skip_serializing_if = "Option::is_none")]
945    pub cpu_microarch_level: Option<u8>,
946    /// Operator-declared named tools whose probe exited ZERO on this worker.
947    ///
948    /// The list is evidence, not configuration: a name appears here only
949    /// because the worker ran the declared argv and it succeeded. Selection
950    /// gates `--require-tool` on this, which is the difference between a
951    /// verified capability and a tag somebody wrote by hand.
952    #[serde(default, skip_serializing_if = "Vec::is_empty")]
953    pub tools_present: Vec<String>,
954    /// Operator-declared named tools whose probe RAN and did not exit zero.
955    ///
956    /// Kept separate from "never declared" so an operator can tell a worker
957    /// that is missing a tool from a fleet that never probed for it — the two
958    /// have different fixes.
959    #[serde(default, skip_serializing_if = "Vec::is_empty")]
960    pub tools_absent: Vec<String>,
961
962    // Health metrics (bd-3eaa)
963    /// Number of CPU cores on the worker.
964    #[serde(default, skip_serializing_if = "Option::is_none")]
965    pub num_cpus: Option<u32>,
966    /// 1-minute load average.
967    #[serde(default, skip_serializing_if = "Option::is_none")]
968    pub load_avg_1: Option<f64>,
969    /// 5-minute load average.
970    #[serde(default, skip_serializing_if = "Option::is_none")]
971    pub load_avg_5: Option<f64>,
972    /// 15-minute load average.
973    #[serde(default, skip_serializing_if = "Option::is_none")]
974    pub load_avg_15: Option<f64>,
975    /// Free space in GB on the tightest mount a build touches: whichever of
976    /// the projects root, its alias and `/tmp` has the lowest free ratio.
977    /// Drives pressure classification (a full `/tmp` breaks scp/mktemp).
978    #[serde(default, skip_serializing_if = "Option::is_none")]
979    pub disk_free_gb: Option<f64>,
980    /// Total size in GB of the mount reported in `disk_free_gb`.
981    #[serde(default, skip_serializing_if = "Option::is_none")]
982    pub disk_total_gb: Option<f64>,
983    /// Free space in GB on the filesystem that holds build trees (the fuller
984    /// of the projects root and its alias; `/tmp` excluded). Sizes capacity:
985    /// slot derating, disk headroom and the free-space floor (GH #78).
986    /// `None` from an `rch-wkr` that predates the field.
987    #[serde(default, skip_serializing_if = "Option::is_none")]
988    pub build_disk_free_gb: Option<f64>,
989    /// Total size in GB of the mount reported in `build_disk_free_gb`.
990    #[serde(default, skip_serializing_if = "Option::is_none")]
991    pub build_disk_total_gb: Option<f64>,
992    /// Canonical path-topology preflight status (`/data/projects` + `/dp` alias).
993    ///
994    /// `Some(false)` indicates a hard preflight failure that should exclude this
995    /// worker from remote scheduling until a subsequent successful revalidation.
996    #[serde(default, skip_serializing_if = "Option::is_none")]
997    pub projects_root_ok: Option<bool>,
998    /// Machine-readable failure reason for path-topology preflight.
999    #[serde(default, skip_serializing_if = "Option::is_none")]
1000    pub projects_root_issue: Option<String>,
1001    /// Unix timestamp (ms) when topology preflight last executed.
1002    #[serde(default, skip_serializing_if = "Option::is_none")]
1003    pub projects_root_checked_at_unix_ms: Option<i64>,
1004}
1005
1006impl WorkerCapabilities {
1007    /// Create new empty capabilities.
1008    pub fn new() -> Self {
1009        Self::default()
1010    }
1011
1012    /// Create mock capabilities with Rust installed.
1013    ///
1014    /// Used by mock transport to simulate a worker with Rust toolchain.
1015    pub fn mock_with_rust() -> Self {
1016        Self {
1017            rustc_version: Some("1.85.0-nightly (mock)".to_string()),
1018            projects_root_ok: Some(true),
1019            ..Self::default()
1020        }
1021    }
1022
1023    /// Whether this is a known pre-x86-64-v3 (no AVX2) x86 worker — the class
1024    /// whose build-script/proc-macro binaries SIGILL on v3 codegen (bd-68hon).
1025    /// `None` (unknown/non-x86) is NOT pre-v3: only affirmative evidence
1026    /// deprioritizes a worker.
1027    pub fn is_pre_v3_x86(&self) -> bool {
1028        matches!(self.cpu_microarch_level, Some(1 | 2))
1029    }
1030
1031    /// Check if this worker has Bun installed.
1032    pub fn has_bun(&self) -> bool {
1033        self.bun_version.is_some()
1034    }
1035
1036    /// Check if this worker has Node.js installed.
1037    pub fn has_node(&self) -> bool {
1038        self.node_version.is_some()
1039    }
1040
1041    /// Check if this worker has Rust installed.
1042    pub fn has_rust(&self) -> bool {
1043        self.rustc_version.is_some()
1044    }
1045
1046    /// Check whether `component` is installed for `toolchain`.
1047    ///
1048    /// Rustup reports pinned toolchains with a host-triple suffix. The request
1049    /// carries the project pin without that suffix, so the owner half uses the
1050    /// same exact-or-`-`-boundary prefix rule as toolchain admission. The
1051    /// component half is exact.
1052    pub fn has_rustup_component(&self, toolchain: &str, component: &str) -> bool {
1053        let owner_prefix = format!("{toolchain}-");
1054        self.rustup_components.iter().any(|fact| {
1055            fact.split_once(':').is_some_and(|(owner, name)| {
1056                name == component && (owner == toolchain || owner.starts_with(&owner_prefix))
1057            })
1058        })
1059    }
1060
1061    /// Check if this worker has Nix installed (nix binary + `/nix/store`).
1062    pub fn has_nix(&self) -> bool {
1063        self.nix_version.is_some()
1064    }
1065
1066    /// Check if this worker has the Go toolchain installed.
1067    pub fn has_go(&self) -> bool {
1068        self.go_version.is_some()
1069    }
1070
1071    /// Check if this worker can run `cargo zigbuild` cross-compiles.
1072    ///
1073    /// Requires BOTH halves of the toolchain: the `cargo-zigbuild` cargo
1074    /// subcommand and the `zig` binary it drives as the linker. Either one
1075    /// alone fails at build time with an error the remote-result heuristics do
1076    /// not recognize as infrastructure (`error: no such command: 'zigbuild'` is
1077    /// not a rustup message), so the exit code would be surfaced verbatim
1078    /// instead of falling open to local. Gating both here keeps that failure
1079    /// out of the user's build.
1080    pub fn has_zig(&self) -> bool {
1081        self.zig_version.is_some() && self.cargo_zigbuild_version.is_some()
1082    }
1083
1084    /// Calculate load per core (1-minute load average / num_cpus).
1085    /// Returns None if metrics are unavailable.
1086    pub fn load_per_core(&self) -> Option<f64> {
1087        match (self.load_avg_1, self.num_cpus) {
1088            (Some(load), Some(cpus)) if cpus > 0 => Some(load / cpus as f64),
1089            _ => None,
1090        }
1091    }
1092
1093    /// Check if worker has high load (exceeds threshold).
1094    /// Returns None if metrics unavailable (fail-open).
1095    pub fn is_high_load(&self, max_load_per_core: f64) -> Option<bool> {
1096        self.load_per_core().map(|lpc| lpc > max_load_per_core)
1097    }
1098
1099    /// `(free_gb, total_gb)` of the filesystem that holds build trees.
1100    ///
1101    /// Capacity questions (how many builds fit, is the free-space floor met)
1102    /// are about that filesystem, not about a small tmpfs `/tmp` that happens
1103    /// to be the tightest mount. Falls back to the tightest-mount sample when
1104    /// the worker does not report a build-disk sample (older `rch-wkr`), which
1105    /// is the conservative pre-GH-#78 behaviour.
1106    pub fn build_disk_gb(&self) -> (Option<f64>, Option<f64>) {
1107        if self.build_disk_free_gb.is_some() {
1108            (self.build_disk_free_gb, self.build_disk_total_gb)
1109        } else {
1110            (self.disk_free_gb, self.disk_total_gb)
1111        }
1112    }
1113
1114    /// Check if the build disk is below the free-space threshold.
1115    /// Returns None if metrics unavailable (fail-open).
1116    pub fn is_low_disk(&self, min_free_gb: f64) -> Option<bool> {
1117        self.build_disk_gb().0.map(|free| free < min_free_gb)
1118    }
1119
1120    /// Check if canonical path-topology preflight is healthy.
1121    ///
1122    /// Returns `None` when probe data is unavailable (fail-open behavior).
1123    pub fn is_topology_healthy(&self) -> Option<bool> {
1124        self.projects_root_ok
1125    }
1126}
1127
1128/// Classify an x86-64 microarchitecture level (1..=4) from a `/proc/cpuinfo`
1129/// `flags` line (bd-6qchz).
1130///
1131/// Levels follow the psABI x86-64-v2/v3/v4 feature baselines, restricted to
1132/// flags as they appear in Linux cpuinfo. A CPU missing any flag of a level is
1133/// classified at the level below — e.g. ovh-b's Ivy Bridge E3-1245 V2 has AVX
1134/// but no `avx2`/`bmi2`/`fma`, so it classifies as 2 and its v3-codegen build
1135/// scripts SIGILL (the bd-68hon incident class).
1136#[must_use]
1137pub fn x86_64_microarch_level_from_flags(flags: &str) -> u8 {
1138    let set: std::collections::HashSet<&str> = flags.split_whitespace().collect();
1139    let has_all = |required: &[&str]| required.iter().all(|flag| set.contains(flag));
1140    // x86-64-v2: CMPXCHG16B, LAHF/SAHF, POPCNT, SSE3/SSSE3/SSE4.x.
1141    const V2: &[&str] = &["cx16", "lahf_lm", "popcnt", "sse4_1", "sse4_2", "ssse3"];
1142    // x86-64-v3: AVX/AVX2, BMI1/2, F16C, FMA, MOVBE, XSAVE (LZCNT is `abm`).
1143    const V3: &[&str] = &[
1144        "abm", "avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "movbe", "xsave",
1145    ];
1146    // x86-64-v4: the AVX-512 F/BW/CD/DQ/VL baseline.
1147    const V4: &[&str] = &["avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl"];
1148    if !has_all(V2) {
1149        return 1;
1150    }
1151    if !has_all(V3) {
1152        return 2;
1153    }
1154    if !has_all(V4) {
1155        return 3;
1156    }
1157    4
1158}
1159
1160/// Path topology configuration.
1161///
1162/// Overrides the default canonical and alias project roots used by
1163/// [`crate::path_topology::PathTopologyPolicy`].  This is useful on platforms
1164/// such as macOS where SIP prevents using `/data/projects` and the user must
1165/// choose a different root (e.g. `~/Projects`).
1166///
1167/// ```toml
1168/// [path_topology]
1169/// canonical_root = "/Users/me/Projects"
1170/// alias_root = "/Users/me/p"
1171/// ```
1172#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1173pub struct PathTopologyConfig {
1174    /// Canonical project root directory.
1175    /// Defaults to `/data/projects` when absent.
1176    #[serde(default)]
1177    pub canonical_root: Option<String>,
1178
1179    /// Alias (symlink) root that points at the canonical root.
1180    /// Defaults to `/dp` when absent.
1181    #[serde(default)]
1182    pub alias_root: Option<String>,
1183}
1184
1185impl PathTopologyConfig {
1186    /// Build a [`crate::path_topology::PathTopologyPolicy`] from this config,
1187    /// falling back to the compiled-in defaults for any field left unset.
1188    ///
1189    /// Empty strings are treated as unset (fall back to defaults).
1190    /// Tilde prefixes (`~/...`) are expanded to the user's home directory.
1191    pub fn to_policy(&self) -> crate::path_topology::PathTopologyPolicy {
1192        let canonical = self
1193            .canonical_root
1194            .as_deref()
1195            .filter(|s| !s.is_empty())
1196            .map(|s| shellexpand::tilde(s).into_owned())
1197            .unwrap_or_else(|| crate::path_topology::DEFAULT_CANONICAL_PROJECT_ROOT.to_string());
1198        let alias = self
1199            .alias_root
1200            .as_deref()
1201            .filter(|s| !s.is_empty())
1202            .map(|s| shellexpand::tilde(s).into_owned())
1203            .unwrap_or_else(|| crate::path_topology::DEFAULT_ALIAS_PROJECT_ROOT.to_string());
1204        crate::path_topology::PathTopologyPolicy::new(
1205            PathBuf::from(canonical),
1206            PathBuf::from(alias),
1207        )
1208    }
1209}
1210
1211/// RCH configuration.
1212#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1213pub struct RchConfig {
1214    #[serde(default)]
1215    pub general: GeneralConfig,
1216    #[serde(default)]
1217    pub compilation: CompilationConfig,
1218    #[serde(default)]
1219    pub transfer: TransferConfig,
1220    #[serde(default)]
1221    pub environment: EnvironmentConfig,
1222    #[serde(default)]
1223    pub circuit: CircuitBreakerConfig,
1224    #[serde(default)]
1225    pub output: OutputConfig,
1226    #[serde(default)]
1227    pub self_healing: SelfHealingConfig,
1228    #[serde(default)]
1229    pub self_test: SelfTestConfig,
1230    /// Worker selection algorithm configuration.
1231    #[serde(default)]
1232    pub selection: SelectionConfig,
1233    /// Execution allowlist configuration (bd-785w).
1234    #[serde(default)]
1235    pub execution: ExecutionConfig,
1236    /// Worker health alerting configuration (daemon).
1237    #[serde(default)]
1238    pub alerts: AlertsConfig,
1239    /// Fleet operation configuration (bd-rs7w.2).
1240    #[serde(default)]
1241    pub fleet: FleetConfig,
1242    /// Path topology overrides for project root directories.
1243    #[serde(default)]
1244    pub path_topology: PathTopologyConfig,
1245    /// Doctor / reliability subsystem configuration (verdict webhooks).
1246    #[serde(default)]
1247    pub doctor: DoctorConfig,
1248    /// Session-history remediation knobs: the central schema and default policy
1249    /// for temporary bypass, auto-rejoin, reconciliation, proof, incident
1250    /// ledger, build-root, pooled targets, telemetry freshness, log retention,
1251    /// disk pressure, and smoke defaults (bd-...remediation-ocv9i.17.1).
1252    #[serde(default)]
1253    pub remediation: crate::remediation_config::RemediationConfig,
1254    /// Layer 0 configuration pack knobs (bd-bqu38 / M-1 §149). Defaults OFF;
1255    /// injected as CARGO_PROFILE_* env vars into remote cargo builds.
1256    #[serde(default)]
1257    pub layer0: Layer0Config,
1258    /// Daemon status API over TCP for other machines on the tailnet
1259    /// (`[api]`, bd-2f5ms). Off unless `bind` is set.
1260    #[serde(default)]
1261    pub api: ApiConfig,
1262    /// Where the fleet dashboard lives (`[dashboard]`), for `rch web`.
1263    #[serde(default)]
1264    pub dashboard: DashboardConfig,
1265    /// Job-mode defaults (`[jobs]`), for `rch exec --job`.
1266    #[serde(default)]
1267    pub jobs: JobsConfig,
1268}
1269
1270/// Job-mode configuration (`[jobs]`).
1271#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1272pub struct JobsConfig {
1273    /// Named tools every job from this project requires a worker to have
1274    /// VERIFIED, as though each were passed with `--require-tool`.
1275    ///
1276    /// This is the project saying "my jobs cannot run on a worker without
1277    /// these", so it applies on top of any `--require-tool` given on the
1278    /// command line rather than being replaced by it. It affects job mode
1279    /// ONLY: ordinary offloaded compilation keeps its existing admission
1280    /// behavior, because a project-wide requirement that silently narrowed
1281    /// every build's worker pool would be a surprising way to lose the fleet.
1282    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1283    pub required_tools: Vec<String>,
1284}
1285
1286/// Daemon status API over TCP (`[api]`).
1287///
1288/// rchd's rich state (`/status`, `/workers/capabilities`) lives on a `0600`
1289/// Unix socket, which only the local user can read. This section opens the
1290/// same JSON on a TCP listener so agents and the fleet dashboard on OTHER
1291/// machines can ask a dispatcher "why are you local-only right now?" without
1292/// ssh. It is meant for a tailnet: the bind address must be loopback or a
1293/// Tailscale address (100.64.0.0/10, fd7a:115c:a1e0::/48) unless
1294/// `allow_any_addr` is set, and every status route requires the bearer token
1295/// unless `token` is unset AND `no_token` is explicitly true.
1296///
1297/// ```toml
1298/// [api]
1299/// bind = "tailscale"       # or "tailscale:9101", "100.68.51.94:9101", "127.0.0.1:9101"
1300/// token = "…"              # or token_file = "~/.config/rch/api.token"
1301/// ```
1302#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1303#[serde(default)]
1304pub struct ApiConfig {
1305    /// Where to listen. `"tailscale"` resolves this machine's Tailscale IPv4
1306    /// at startup (default port 9101); `"tailscale:PORT"`, `"IP:PORT"` or
1307    /// `"[v6]:PORT"` are taken literally. Unset (empty) = API off.
1308    pub bind: String,
1309    /// Bearer token required on `/status` and `/workers/capabilities`.
1310    pub token: Option<String>,
1311    /// File holding the bearer token (trimmed). Takes precedence over `token`.
1312    pub token_file: Option<String>,
1313    /// Serve the status routes with NO token. Only honoured when no token is
1314    /// configured; must be set explicitly so an empty token is never an
1315    /// accidental open door.
1316    pub no_token: bool,
1317    /// Permit a bind address outside loopback and the Tailscale ranges.
1318    pub allow_any_addr: bool,
1319}
1320
1321/// Where the fleet dashboard is served (`[dashboard]`).
1322#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1323#[serde(default)]
1324pub struct DashboardConfig {
1325    /// URL `rch web` opens, e.g. `https://rch-fleet.vercel.app`.
1326    pub url: Option<String>,
1327}
1328
1329/// Doctor reliability subsystem configuration.
1330///
1331/// Currently carries the verdict-transition webhook surface. Kept as its own
1332/// section (`[doctor]`) so future doctor knobs land here rather than bloating
1333/// the top-level config.
1334#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1335pub struct DoctorConfig {
1336    /// Verdict-transition webhook destinations.
1337    #[serde(default)]
1338    pub webhooks: DoctorWebhooksConfig,
1339}
1340
1341/// Webhook notification surface for reliability verdict transitions.
1342///
1343/// Fires when `rch doctor --reliability --watch` observes a verdict change
1344/// (`Healthy → Degraded`, `Degraded → Failing`, etc.). Each endpoint selects
1345/// which transitions it cares about and how the payload is formatted.
1346#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1347pub struct DoctorWebhooksConfig {
1348    /// Named webhook destinations. Empty = no webhook dispatch.
1349    #[serde(default)]
1350    pub endpoints: Vec<DoctorWebhookEndpoint>,
1351    /// Maximum number of in-flight webhook deliveries. Bounds memory and
1352    /// outbound connection pressure: when the queue is full, a transition's
1353    /// delivery is dropped (and journaled) rather than growing unboundedly.
1354    /// Per-process, shared across all endpoints.
1355    #[serde(default = "default_webhook_max_inflight")]
1356    pub max_inflight: usize,
1357}
1358
1359// Manual `Default` so the in-code default matches the serde default
1360// (`#[derive(Default)]` would give `max_inflight = 0`, diverging from a
1361// from-TOML load that fills in 16).
1362impl Default for DoctorWebhooksConfig {
1363    fn default() -> Self {
1364        Self {
1365            endpoints: Vec::new(),
1366            max_inflight: default_webhook_max_inflight(),
1367        }
1368    }
1369}
1370
1371/// A single webhook destination for verdict transitions.
1372///
1373/// SECRETS ARE NEVER STORED HERE. Routing keys, bearer tokens, and signing
1374/// secrets are referenced by the *name* of an environment variable
1375/// (`*_env` fields) read at dispatch time. There is deliberately no
1376/// `secret` / `routing_key` / `bearer_token` / `password` field on this
1377/// struct (enforced by `test_no_inline_secret_fields`).
1378#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1379pub struct DoctorWebhookEndpoint {
1380    /// Operator-facing name (appears in logs and the audit trail).
1381    pub name: String,
1382    /// Destination URL.
1383    pub url: String,
1384    /// Payload format.
1385    #[serde(default)]
1386    pub format: DoctorWebhookFormat,
1387    /// Transition predicates that arm this endpoint. Empty = never fire.
1388    /// Supported: `healthy_to_degraded`, `degraded_to_failing`,
1389    /// `any_to_failing`, `any_to_degraded`, `failing_to_healthy`,
1390    /// `degraded_to_healthy`, `any_to_healthy`, `any_to_any`.
1391    #[serde(default)]
1392    pub on_transitions: Vec<String>,
1393    /// Maximum retry attempts after the first try fails.
1394    #[serde(default = "default_webhook_retry_max")]
1395    pub retry_max: u32,
1396    /// Base backoff between retries (milliseconds); doubles each attempt.
1397    #[serde(default = "default_webhook_retry_backoff_ms")]
1398    pub retry_backoff_ms: u64,
1399    /// Per-attempt request timeout (milliseconds).
1400    #[serde(default = "default_webhook_timeout_ms")]
1401    pub timeout_ms: u64,
1402    /// Per-endpoint kill switch.
1403    #[serde(default = "default_true")]
1404    pub enabled: bool,
1405    /// Name of the env var holding the PagerDuty Events API routing key.
1406    /// Required for `format = "pagerduty"`. Read at runtime, never stored.
1407    #[serde(default)]
1408    pub routing_key_env: Option<String>,
1409    /// Name of the env var holding a bearer token sent as
1410    /// `Authorization: Bearer <token>`. Read at runtime, never stored.
1411    #[serde(default)]
1412    pub bearer_token_env: Option<String>,
1413    /// Name of the env var holding an HMAC-SHA256 signing secret. When set,
1414    /// the body is signed and the signature sent as `X-RCH-Signature:
1415    /// sha256=<hex>`. Read at runtime, never stored.
1416    #[serde(default)]
1417    pub signing_secret_env: Option<String>,
1418}
1419
1420/// Built-in webhook payload formats.
1421#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
1422#[serde(rename_all = "snake_case")]
1423pub enum DoctorWebhookFormat {
1424    /// Opinionated RCH JSON schema (default).
1425    #[default]
1426    GenericJson,
1427    /// Slack Block Kit payload.
1428    Slack,
1429    /// PagerDuty Events API v2 payload.
1430    Pagerduty,
1431}
1432
1433fn default_webhook_max_inflight() -> usize {
1434    16
1435}
1436
1437fn default_webhook_retry_max() -> u32 {
1438    3
1439}
1440
1441fn default_webhook_retry_backoff_ms() -> u64 {
1442    200
1443}
1444
1445fn default_webhook_timeout_ms() -> u64 {
1446    5000
1447}
1448
1449/// Worker health alerting configuration.
1450#[derive(Debug, Clone, Serialize, Deserialize)]
1451pub struct AlertsConfig {
1452    /// Enable or disable alert generation in the daemon.
1453    #[serde(default = "default_true")]
1454    pub enabled: bool,
1455    /// Suppress duplicate alerts for this many seconds.
1456    #[serde(default = "default_alert_suppress_duplicates_secs")]
1457    pub suppress_duplicates_secs: u64,
1458    /// How long to keep a cleared alert visible (with state
1459    /// `cleared_pending_clean`) before removing it entirely. Prevents
1460    /// transient, self-healing circuit-breaker warnings from lingering on
1461    /// `rch status` long after the fleet returned to healthy (bd-3ogaz).
1462    #[serde(default = "default_alert_cleared_retention_secs")]
1463    pub cleared_retention_secs: u64,
1464    /// Webhook configuration for external notifications.
1465    #[serde(default)]
1466    pub webhook: Option<WebhookConfig>,
1467}
1468
1469/// Webhook configuration for alert dispatch.
1470#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1471pub struct WebhookConfig {
1472    /// Webhook endpoint URL (e.g., Slack/Discord webhook URL).
1473    pub url: Option<String>,
1474    /// Secret for HMAC-SHA256 signing (optional).
1475    #[serde(default)]
1476    pub secret: Option<String>,
1477    /// Timeout in seconds for webhook requests.
1478    #[serde(default = "default_webhook_timeout_secs")]
1479    pub timeout_secs: u64,
1480    /// Number of retries on failure.
1481    #[serde(default = "default_webhook_retry_count")]
1482    pub retry_count: u32,
1483    /// Events to send (empty = all events).
1484    /// Supported: worker_offline, worker_degraded, circuit_open, all_workers_offline
1485    #[serde(default)]
1486    pub events: Vec<String>,
1487}
1488
1489fn default_webhook_timeout_secs() -> u64 {
1490    5
1491}
1492
1493fn default_webhook_retry_count() -> u32 {
1494    3
1495}
1496
1497fn default_alert_suppress_duplicates_secs() -> u64 {
1498    300
1499}
1500
1501fn default_alert_cleared_retention_secs() -> u64 {
1502    300
1503}
1504
1505impl Default for AlertsConfig {
1506    fn default() -> Self {
1507        Self {
1508            enabled: true,
1509            suppress_duplicates_secs: default_alert_suppress_duplicates_secs(),
1510            cleared_retention_secs: default_alert_cleared_retention_secs(),
1511            webhook: None,
1512        }
1513    }
1514}
1515
1516/// Fleet operation configuration.
1517///
1518/// Controls timeouts, thresholds, and behavior for fleet management commands
1519/// like `rch fleet preflight`, `rch fleet status`, and `rch fleet deploy`.
1520#[derive(Debug, Clone, Serialize, Deserialize)]
1521pub struct FleetConfig {
1522    /// SSH connection timeout in seconds.
1523    #[serde(default = "default_fleet_ssh_connect_timeout_secs")]
1524    pub ssh_connect_timeout_secs: u64,
1525    /// SSH command execution timeout in seconds.
1526    #[serde(default = "default_fleet_ssh_command_timeout_secs")]
1527    pub ssh_command_timeout_secs: u64,
1528    /// Minimum required disk space in MB for worker health.
1529    #[serde(default = "default_fleet_min_disk_space_mb")]
1530    pub min_disk_space_mb: u64,
1531    /// Maximum acceptable load average for worker health.
1532    #[serde(default = "default_fleet_max_load_average")]
1533    pub max_load_average: f64,
1534    /// Maximum number of concurrent SSH connections.
1535    #[serde(default = "default_fleet_max_concurrent_workers")]
1536    pub max_concurrent_workers: usize,
1537    /// Number of retry attempts for transient failures.
1538    #[serde(default = "default_fleet_retry_count")]
1539    pub retry_count: u32,
1540    /// Delay between retries in milliseconds.
1541    #[serde(default = "default_fleet_retry_delay_ms")]
1542    pub retry_delay_ms: u64,
1543}
1544
1545fn default_fleet_ssh_connect_timeout_secs() -> u64 {
1546    10
1547}
1548
1549fn default_fleet_ssh_command_timeout_secs() -> u64 {
1550    30
1551}
1552
1553fn default_fleet_min_disk_space_mb() -> u64 {
1554    500
1555}
1556
1557fn default_fleet_max_load_average() -> f64 {
1558    10.0
1559}
1560
1561fn default_fleet_max_concurrent_workers() -> usize {
1562    10
1563}
1564
1565fn default_fleet_retry_count() -> u32 {
1566    2
1567}
1568
1569fn default_fleet_retry_delay_ms() -> u64 {
1570    1000
1571}
1572
1573impl Default for FleetConfig {
1574    fn default() -> Self {
1575        Self {
1576            ssh_connect_timeout_secs: default_fleet_ssh_connect_timeout_secs(),
1577            ssh_command_timeout_secs: default_fleet_ssh_command_timeout_secs(),
1578            min_disk_space_mb: default_fleet_min_disk_space_mb(),
1579            max_load_average: default_fleet_max_load_average(),
1580            max_concurrent_workers: default_fleet_max_concurrent_workers(),
1581            retry_count: default_fleet_retry_count(),
1582            retry_delay_ms: default_fleet_retry_delay_ms(),
1583        }
1584    }
1585}
1586
1587impl FleetConfig {
1588    /// Get SSH connect timeout as Duration.
1589    pub fn ssh_connect_timeout(&self) -> std::time::Duration {
1590        std::time::Duration::from_secs(self.ssh_connect_timeout_secs)
1591    }
1592
1593    /// Get SSH command timeout as Duration.
1594    pub fn ssh_command_timeout(&self) -> std::time::Duration {
1595        std::time::Duration::from_secs(self.ssh_command_timeout_secs)
1596    }
1597
1598    /// Get retry delay as Duration.
1599    pub fn retry_delay(&self) -> std::time::Duration {
1600        std::time::Duration::from_millis(self.retry_delay_ms)
1601    }
1602}
1603
1604/// Box-level role (bd-wywsj): what this MACHINE is for, so "never
1605/// build locally" is configuration rather than a per-call env var.
1606#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
1607#[serde(rename_all = "lowercase")]
1608pub enum BoxRole {
1609    /// Orchestrator box: offloadable builds default FAIL-CLOSED +
1610    /// queue (RCH_REQUIRE_REMOTE semantics) — a saturated fleet waits
1611    /// for a worker instead of repopulating local rustc. Explicit
1612    /// per-call env still overrides.
1613    Dispatcher,
1614    /// The box IS the compute: fail-open/local; `rch shim install`
1615    /// refuses here.
1616    Worker,
1617    /// Today's behavior (fully backward compatible).
1618    #[default]
1619    Hybrid,
1620}
1621
1622impl BoxRole {
1623    /// Stable lowercase token.
1624    #[must_use]
1625    pub fn as_str(self) -> &'static str {
1626        match self {
1627            BoxRole::Dispatcher => "dispatcher",
1628            BoxRole::Worker => "worker",
1629            BoxRole::Hybrid => "hybrid",
1630        }
1631    }
1632}
1633
1634#[derive(Debug, Clone, Serialize, Deserialize)]
1635pub struct GeneralConfig {
1636    /// Whether RCH is enabled.
1637    #[serde(default = "default_true")]
1638    pub enabled: bool,
1639    /// Box-level role (bd-wywsj): dispatcher boxes default fail-closed
1640    /// + queue for offloadable builds; worker boxes are the compute.
1641    #[serde(default)]
1642    pub role: BoxRole,
1643    /// Force local execution for this project (never offload), even if the command is classified.
1644    ///
1645    /// Intended for sensitive projects or cases where local state must be authoritative.
1646    #[serde(default)]
1647    pub force_local: bool,
1648    /// Force remote execution for this project (always attempt offload) when safe.
1649    ///
1650    /// This bypasses heuristic gating (e.g. confidence threshold overrides) but still respects
1651    /// structural safety checks and NEVER_INTERCEPT patterns.
1652    #[serde(default)]
1653    pub force_remote: bool,
1654    /// Log level (trace, debug, info, warn, error).
1655    #[serde(default = "default_log_level")]
1656    pub log_level: String,
1657    /// Path to Unix socket for daemon communication.
1658    #[serde(default = "default_socket_path")]
1659    pub socket_path: String,
1660}
1661
1662/// Environment variable passthrough configuration.
1663#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1664pub struct EnvironmentConfig {
1665    /// Allowlist of environment variables to forward to remote workers.
1666    #[serde(default)]
1667    pub allowlist: Vec<String>,
1668}
1669
1670impl Default for GeneralConfig {
1671    fn default() -> Self {
1672        Self {
1673            enabled: true,
1674            role: BoxRole::default(),
1675            force_local: false,
1676            force_remote: false,
1677            log_level: "info".to_string(),
1678            socket_path: default_socket_path(),
1679        }
1680    }
1681}
1682
1683/// Visibility level for hook output.
1684#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
1685#[serde(rename_all = "lowercase")]
1686pub enum OutputVisibility {
1687    #[default]
1688    None,
1689    Summary,
1690    Verbose,
1691}
1692
1693impl std::fmt::Display for OutputVisibility {
1694    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1695        let value = match self {
1696            OutputVisibility::None => "none",
1697            OutputVisibility::Summary => "summary",
1698            OutputVisibility::Verbose => "verbose",
1699        };
1700        write!(f, "{}", value)
1701    }
1702}
1703
1704impl std::str::FromStr for OutputVisibility {
1705    type Err = ();
1706
1707    fn from_str(s: &str) -> Result<Self, Self::Err> {
1708        match s.trim().to_lowercase().as_str() {
1709            "none" | "silent" | "quiet" => Ok(Self::None),
1710            "summary" | "short" => Ok(Self::Summary),
1711            "verbose" | "debug" => Ok(Self::Verbose),
1712            _ => Err(()),
1713        }
1714    }
1715}
1716
1717/// Color mode for remote command output.
1718///
1719/// Controls whether ANSI color codes are preserved when streaming
1720/// output from remote compilation commands.
1721#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1722#[serde(rename_all = "lowercase")]
1723pub enum ColorMode {
1724    /// Force color output regardless of terminal detection.
1725    /// Sets CARGO_TERM_COLOR=always and similar environment variables.
1726    #[default]
1727    Always,
1728    /// Let the remote command detect terminal capabilities (may lose colors).
1729    Auto,
1730    /// Disable color output entirely.
1731    Never,
1732}
1733
1734impl std::fmt::Display for ColorMode {
1735    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1736        let value = match self {
1737            ColorMode::Always => "always",
1738            ColorMode::Auto => "auto",
1739            ColorMode::Never => "never",
1740        };
1741        write!(f, "{}", value)
1742    }
1743}
1744
1745impl std::str::FromStr for ColorMode {
1746    type Err = ();
1747
1748    fn from_str(s: &str) -> Result<Self, Self::Err> {
1749        match s.trim().to_lowercase().as_str() {
1750            "always" | "force" | "yes" | "true" => Ok(Self::Always),
1751            "auto" | "detect" => Ok(Self::Auto),
1752            "never" | "none" | "no" | "false" => Ok(Self::Never),
1753            _ => Err(()),
1754        }
1755    }
1756}
1757
1758fn default_color_mode() -> ColorMode {
1759    ColorMode::default()
1760}
1761
1762/// Hook output configuration.
1763#[derive(Debug, Clone, Serialize, Deserialize)]
1764pub struct OutputConfig {
1765    /// Output visibility level for hook messages.
1766    #[serde(default = "default_output_visibility")]
1767    pub visibility: OutputVisibility,
1768    /// Whether the first-run success message has been shown.
1769    #[serde(default)]
1770    pub first_run_complete: bool,
1771    /// Color mode for remote command output.
1772    /// Controls whether ANSI color codes are preserved in remote output.
1773    #[serde(default = "default_color_mode")]
1774    pub color_mode: ColorMode,
1775}
1776
1777impl Default for OutputConfig {
1778    fn default() -> Self {
1779        Self {
1780            visibility: OutputVisibility::None,
1781            first_run_complete: false,
1782            color_mode: ColorMode::default(),
1783        }
1784    }
1785}
1786
1787// ============================================================================
1788// Self-Healing Configuration
1789// ============================================================================
1790
1791fn default_autostart_cooldown_secs() -> u64 {
1792    30
1793}
1794
1795fn default_autostart_timeout_secs() -> u64 {
1796    3
1797}
1798
1799/// Log level for self-healing diagnostic output.
1800///
1801/// Used to filter `tracing` events emitted by the self-healing subsystem
1802/// (hook auto-start, daemon hook-install). Read from
1803/// `RCH_SELF_HEALING_LOG_LEVEL=debug|info|warn|error` (case-insensitive).
1804#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
1805#[serde(rename_all = "lowercase")]
1806pub enum SelfHealingLogLevel {
1807    Debug,
1808    #[default]
1809    Info,
1810    Warn,
1811    Error,
1812}
1813
1814impl SelfHealingLogLevel {
1815    /// Parse a human-friendly level name. Returns `None` for unknown input.
1816    pub fn from_env_str(s: &str) -> Option<Self> {
1817        match s.trim().to_ascii_lowercase().as_str() {
1818            "debug" => Some(Self::Debug),
1819            "info" => Some(Self::Info),
1820            "warn" | "warning" => Some(Self::Warn),
1821            "error" => Some(Self::Error),
1822            _ => None,
1823        }
1824    }
1825}
1826
1827/// Self-healing behaviors for the hook and daemon.
1828#[derive(Debug, Clone, Serialize, Deserialize)]
1829pub struct SelfHealingConfig {
1830    /// Whether the hook should attempt to auto-start the daemon.
1831    #[serde(default = "default_true")]
1832    pub hook_starts_daemon: bool,
1833    /// Whether the daemon should auto-install Claude Code hooks on startup.
1834    #[serde(default = "default_true")]
1835    pub daemon_installs_hooks: bool,
1836    /// Cooldown between auto-start attempts in seconds.
1837    #[serde(default = "default_autostart_cooldown_secs")]
1838    pub auto_start_cooldown_secs: u64,
1839    /// Time to wait for the daemon socket after spawning in seconds.
1840    /// Also aliased as `daemon_start_timeout` for backwards compatibility.
1841    #[serde(
1842        default = "default_autostart_timeout_secs",
1843        alias = "daemon_start_timeout"
1844    )]
1845    pub auto_start_timeout_secs: u64,
1846    /// Log level for self-healing diagnostic events. Default: info.
1847    /// Override via `RCH_SELF_HEALING_LOG_LEVEL=debug|info|warn|error`.
1848    #[serde(default)]
1849    pub self_healing_log_level: SelfHealingLogLevel,
1850}
1851
1852impl Default for SelfHealingConfig {
1853    fn default() -> Self {
1854        Self {
1855            hook_starts_daemon: default_true(),
1856            daemon_installs_hooks: default_true(),
1857            auto_start_cooldown_secs: default_autostart_cooldown_secs(),
1858            auto_start_timeout_secs: default_autostart_timeout_secs(),
1859            self_healing_log_level: SelfHealingLogLevel::default(),
1860        }
1861    }
1862}
1863
1864impl SelfHealingConfig {
1865    /// Apply environment variable overrides to this config.
1866    ///
1867    /// Supported environment variables:
1868    /// - `RCH_NO_SELF_HEALING=1` - Master switch to disable all self-healing
1869    /// - `RCH_HOOK_STARTS_DAEMON=0|1` - Control hook auto-starting daemon
1870    /// - `RCH_DAEMON_INSTALLS_HOOKS=0|1` - Control daemon auto-installing hooks
1871    /// - `RCH_AUTO_START_TIMEOUT_SECS=<seconds>` - Max wait for daemon start
1872    /// - `RCH_AUTO_START_COOLDOWN_SECS=<seconds>` - Min time between auto-starts
1873    pub fn with_env_overrides(mut self) -> Self {
1874        // Master disable switch
1875        if let Ok(val) = std::env::var("RCH_NO_SELF_HEALING")
1876            && (val == "1" || val.eq_ignore_ascii_case("true"))
1877        {
1878            self.hook_starts_daemon = false;
1879            self.daemon_installs_hooks = false;
1880            return self;
1881        }
1882
1883        // Individual toggles
1884        if let Ok(val) = std::env::var("RCH_HOOK_STARTS_DAEMON") {
1885            self.hook_starts_daemon = val != "0" && !val.eq_ignore_ascii_case("false");
1886        }
1887        if let Ok(val) = std::env::var("RCH_DAEMON_INSTALLS_HOOKS") {
1888            self.daemon_installs_hooks = val != "0" && !val.eq_ignore_ascii_case("false");
1889        }
1890
1891        // Numeric settings
1892        if let Ok(val) = std::env::var("RCH_AUTO_START_TIMEOUT_SECS")
1893            && let Ok(secs) = val.parse()
1894        {
1895            self.auto_start_timeout_secs = secs;
1896        }
1897        if let Ok(val) = std::env::var("RCH_AUTO_START_COOLDOWN_SECS")
1898            && let Ok(secs) = val.parse()
1899        {
1900            self.auto_start_cooldown_secs = secs;
1901        }
1902        // br-4zf3p: log-level override.
1903        if let Ok(val) = std::env::var("RCH_SELF_HEALING_LOG_LEVEL") {
1904            if let Some(level) = SelfHealingLogLevel::from_env_str(&val) {
1905                self.self_healing_log_level = level;
1906            } else {
1907                tracing::warn!(
1908                    target: "rch::self_healing",
1909                    invalid = %val,
1910                    "RCH_SELF_HEALING_LOG_LEVEL has invalid value; using default=info"
1911                );
1912            }
1913        }
1914
1915        self
1916    }
1917}
1918
1919// ============================================================================
1920// Self-Test Configuration
1921// ============================================================================
1922
1923/// Action to take when a scheduled self-test fails.
1924#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1925#[serde(rename_all = "snake_case")]
1926pub enum SelfTestFailureAction {
1927    /// Only emit an alert/log entry.
1928    #[default]
1929    Alert,
1930    /// Disable the failing worker automatically.
1931    DisableWorker,
1932    /// Alert and disable the worker.
1933    AlertAndDisable,
1934}
1935
1936/// Which workers to include in self-test runs.
1937#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1938#[serde(untagged)]
1939pub enum SelfTestWorkers {
1940    /// Special value "all" (or a single worker id string).
1941    All(String),
1942    /// Explicit worker list.
1943    List(Vec<WorkerId>),
1944}
1945
1946impl Default for SelfTestWorkers {
1947    fn default() -> Self {
1948        Self::All("all".to_string())
1949    }
1950}
1951
1952impl SelfTestWorkers {
1953    /// Resolve to an explicit list if configured, or None to indicate "all".
1954    pub fn resolve(&self) -> Option<Vec<WorkerId>> {
1955        match self {
1956            SelfTestWorkers::All(value) => {
1957                if value.eq_ignore_ascii_case("all") {
1958                    None
1959                } else {
1960                    Some(vec![WorkerId::new(value.clone())])
1961                }
1962            }
1963            SelfTestWorkers::List(list) => Some(list.clone()),
1964        }
1965    }
1966}
1967
1968fn default_self_test_retry_count() -> u32 {
1969    3
1970}
1971
1972fn default_self_test_retry_delay() -> String {
1973    "5m".to_string()
1974}
1975
1976fn default_self_test_enabled() -> bool {
1977    false
1978}
1979
1980/// Self-test scheduling and behavior configuration.
1981#[derive(Debug, Clone, Serialize, Deserialize)]
1982pub struct SelfTestConfig {
1983    /// Enable scheduled self-tests.
1984    #[serde(default = "default_self_test_enabled")]
1985    pub enabled: bool,
1986    /// Cron schedule (e.g., "0 3 * * *").
1987    #[serde(default)]
1988    pub schedule: Option<String>,
1989    /// Interval duration (e.g., "24h").
1990    #[serde(default)]
1991    pub interval: Option<String>,
1992    /// Which workers to test (default: "all").
1993    #[serde(default)]
1994    pub workers: SelfTestWorkers,
1995    /// Action on failure.
1996    #[serde(default)]
1997    pub on_failure: SelfTestFailureAction,
1998    /// Retry count for failed tests.
1999    #[serde(default = "default_self_test_retry_count")]
2000    pub retry_count: u32,
2001    /// Delay between retries.
2002    #[serde(default = "default_self_test_retry_delay")]
2003    pub retry_delay: String,
2004}
2005
2006impl Default for SelfTestConfig {
2007    fn default() -> Self {
2008        Self {
2009            enabled: default_self_test_enabled(),
2010            schedule: None,
2011            interval: None,
2012            workers: SelfTestWorkers::default(),
2013            on_failure: SelfTestFailureAction::default(),
2014            retry_count: default_self_test_retry_count(),
2015            retry_delay: default_self_test_retry_delay(),
2016        }
2017    }
2018}
2019
2020/// Circuit breaker configuration.
2021#[derive(Debug, Clone, Serialize, Deserialize)]
2022pub struct CircuitBreakerConfig {
2023    /// Consecutive failures to open circuit.
2024    #[serde(default = "default_circuit_failure_threshold")]
2025    pub failure_threshold: u32,
2026    /// Consecutive successes to close circuit.
2027    #[serde(default = "default_circuit_success_threshold")]
2028    pub success_threshold: u32,
2029    /// Error rate threshold (0.0-1.0) to open circuit.
2030    #[serde(default = "default_circuit_error_rate_threshold")]
2031    pub error_rate_threshold: f64,
2032    /// Rolling window size in seconds.
2033    #[serde(default = "default_circuit_window_secs")]
2034    pub window_secs: u64,
2035    /// Cooldown duration before half-open (seconds).
2036    #[serde(default = "default_circuit_open_cooldown_secs")]
2037    pub open_cooldown_secs: u64,
2038    /// Max concurrent probes in half-open.
2039    #[serde(default = "default_circuit_half_open_max_probes")]
2040    pub half_open_max_probes: u32,
2041    /// Consecutive *command-class* failures to open the circuit.
2042    ///
2043    /// Command-class evidence is an operation of the same shape as dispatched
2044    /// work — a remote binary that must run to completion (the telemetry poll,
2045    /// or a health probe whose session connected but whose command never
2046    /// finished). It is tracked separately from the plain liveness probe
2047    /// because a host under fork exhaustion or swap thrash keeps answering
2048    /// `echo` over a warm connection while every real command times out;
2049    /// interleaved liveness successes would otherwise keep resetting
2050    /// `failure_threshold` and the breaker would never trip.
2051    #[serde(default = "default_circuit_command_failure_threshold")]
2052    pub command_failure_threshold: u32,
2053    /// Upper bound (seconds) for the open cooldown after repeated reopens.
2054    ///
2055    /// Each Open -> HalfOpen -> Open cycle without an intervening Closed
2056    /// doubles the cooldown (`open_cooldown_secs`, x2, x4, ...) up to this
2057    /// cap, so a flapping worker converges to Open instead of being
2058    /// re-admitted every cooldown. Set equal to `open_cooldown_secs` to
2059    /// disable the backoff.
2060    #[serde(default = "default_circuit_open_cooldown_max_secs")]
2061    pub open_cooldown_max_secs: u64,
2062}
2063
2064impl Default for CircuitBreakerConfig {
2065    fn default() -> Self {
2066        Self {
2067            failure_threshold: default_circuit_failure_threshold(),
2068            success_threshold: default_circuit_success_threshold(),
2069            error_rate_threshold: default_circuit_error_rate_threshold(),
2070            window_secs: default_circuit_window_secs(),
2071            open_cooldown_secs: default_circuit_open_cooldown_secs(),
2072            half_open_max_probes: default_circuit_half_open_max_probes(),
2073            command_failure_threshold: default_circuit_command_failure_threshold(),
2074            open_cooldown_max_secs: default_circuit_open_cooldown_max_secs(),
2075        }
2076    }
2077}
2078
2079impl CircuitBreakerConfig {
2080    /// Effective cooldown (ms) for the given reopen streak, with exponential
2081    /// backoff capped at `open_cooldown_max_secs`. `reopen_streak` counts
2082    /// Open transitions since the circuit last closed (1 = first open).
2083    pub fn cooldown_ms_for_streak(&self, reopen_streak: u32) -> u64 {
2084        let base_ms = self.open_cooldown_secs.saturating_mul(1000);
2085        let cap_ms = self
2086            .open_cooldown_max_secs
2087            .saturating_mul(1000)
2088            .max(base_ms);
2089        // Exponent bounded so the shift cannot overflow; the cap clamps anyway.
2090        let exponent = reopen_streak.saturating_sub(1).min(16);
2091        base_ms.saturating_mul(1u64 << exponent).min(cap_ms)
2092    }
2093}
2094
2095/// Circuit breaker statistics for tracking state transitions.
2096///
2097/// Maintains a rolling window of successes and failures to determine
2098/// when the circuit should open, close, or enter half-open state.
2099#[derive(Debug, Clone, Default)]
2100pub struct CircuitStats {
2101    /// Current circuit state.
2102    state: CircuitState,
2103    /// Consecutive failure count (reset on success).
2104    consecutive_failures: u32,
2105    /// Consecutive success count in half-open state (reset on failure).
2106    consecutive_successes: u32,
2107    /// Total successes in the current rolling window.
2108    window_successes: u32,
2109    /// Total failures in the current rolling window.
2110    window_failures: u32,
2111    /// Timestamp when circuit entered Open state (epoch millis).
2112    opened_at: Option<u64>,
2113    /// Timestamp of last state change (epoch millis).
2114    last_state_change: u64,
2115    /// Current active probes in half-open state.
2116    active_probes: u32,
2117    /// Consecutive command-class failures (telemetry poll exhaustion, probe
2118    /// sessions that connected but never completed). Reset only by a
2119    /// command-class success or a circuit close — NOT by a liveness success,
2120    /// which is the whole point (see `CircuitBreakerConfig::command_failure_threshold`).
2121    consecutive_command_failures: u32,
2122    /// Number of Open transitions since the circuit last closed. Drives the
2123    /// cooldown backoff; 0 while Closed-since-recovery.
2124    reopen_streak: u32,
2125    /// Recent health check results (true=success, false=failure).
2126    /// Used for history visualization in status display.
2127    recent_results: Vec<bool>,
2128}
2129
2130/// Maximum number of recent results to keep for history visualization.
2131const CIRCUIT_HISTORY_SIZE: usize = 10;
2132
2133impl CircuitStats {
2134    /// Create new circuit stats in the closed state.
2135    pub fn new() -> Self {
2136        Self {
2137            state: CircuitState::Closed,
2138            consecutive_failures: 0,
2139            consecutive_successes: 0,
2140            window_successes: 0,
2141            window_failures: 0,
2142            opened_at: None,
2143            last_state_change: Self::now_millis(),
2144            active_probes: 0,
2145            consecutive_command_failures: 0,
2146            reopen_streak: 0,
2147            recent_results: Vec::with_capacity(CIRCUIT_HISTORY_SIZE),
2148        }
2149    }
2150
2151    /// Consecutive command-class failures observed since the last
2152    /// command-class success (or circuit close).
2153    pub fn consecutive_command_failures(&self) -> u32 {
2154        self.consecutive_command_failures
2155    }
2156
2157    /// Open transitions since the circuit last closed (0 when it has not
2158    /// opened since recovery). Exposed for diagnostics.
2159    pub fn reopen_streak(&self) -> u32 {
2160        self.reopen_streak
2161    }
2162
2163    /// Cooldown currently in force (ms), including reopen backoff.
2164    fn effective_cooldown_ms(&self, config: &CircuitBreakerConfig) -> u64 {
2165        config.cooldown_ms_for_streak(self.reopen_streak.max(1))
2166    }
2167
2168    /// Get the current circuit state.
2169    pub fn state(&self) -> CircuitState {
2170        self.state
2171    }
2172
2173    /// Get the timestamp when the circuit opened (if open).
2174    pub fn opened_at(&self) -> Option<u64> {
2175        self.opened_at
2176    }
2177
2178    /// Get the timestamp of the last state change.
2179    pub fn last_state_change(&self) -> u64 {
2180        self.last_state_change
2181    }
2182
2183    /// Get the consecutive failure count.
2184    pub fn consecutive_failures(&self) -> u32 {
2185        self.consecutive_failures
2186    }
2187
2188    /// Get the error rate in the current window (0.0-1.0).
2189    ///
2190    /// Computed over the bounded recent-results window (the last
2191    /// `CIRCUIT_HISTORY_SIZE` health checks) rather than the unbounded lifetime
2192    /// `window_successes`/`window_failures` counters. A long-lived daemon polling
2193    /// distant workers accumulates occasional transient failures over days; with a
2194    /// lifetime ratio a currently-healthy worker could stay gated (circuit kept
2195    /// open, depressed health score) until the daemon was restarted. A recency
2196    /// window lets the rate self-heal as fresh successes age out old failures.
2197    pub fn error_rate(&self) -> f64 {
2198        if self.recent_results.is_empty() {
2199            return 0.0;
2200        }
2201        let failures = self.recent_results.iter().filter(|&&ok| !ok).count();
2202        failures as f64 / self.recent_results.len() as f64
2203    }
2204
2205    /// Record a successful operation.
2206    ///
2207    /// In closed state: resets consecutive failures, increments window successes.
2208    /// In half-open state: increments consecutive successes.
2209    pub fn record_success(&mut self) {
2210        self.window_successes += 1;
2211        self.consecutive_failures = 0;
2212        self.push_result(true);
2213
2214        if self.state == CircuitState::HalfOpen {
2215            self.consecutive_successes += 1;
2216            if self.active_probes > 0 {
2217                self.active_probes -= 1;
2218            }
2219        }
2220    }
2221
2222    /// Record a failed operation.
2223    ///
2224    /// Increments consecutive failures and window failures.
2225    /// In half-open state, resets consecutive successes.
2226    pub fn record_failure(&mut self) {
2227        self.window_failures += 1;
2228        self.consecutive_failures += 1;
2229        self.push_result(false);
2230
2231        if self.state == CircuitState::HalfOpen {
2232            self.consecutive_successes = 0;
2233            if self.active_probes > 0 {
2234                self.active_probes -= 1;
2235            }
2236        }
2237    }
2238
2239    /// Check if the circuit should open based on config thresholds.
2240    ///
2241    /// Returns true if:
2242    /// - Consecutive failures exceed threshold, OR
2243    /// - Error rate exceeds threshold (with minimum sample size)
2244    pub fn should_open(&self, config: &CircuitBreakerConfig) -> bool {
2245        if self.state != CircuitState::Closed {
2246            return false;
2247        }
2248
2249        // Open if consecutive failures exceed threshold
2250        if self.consecutive_failures >= config.failure_threshold {
2251            return true;
2252        }
2253
2254        // Open if error rate exceeds threshold (with a minimum sample size).
2255        // Counts samples in the bounded recent-results window so the gate
2256        // reflects recent behavior, matching error_rate() above.
2257        if self.recent_results.len() >= 5 && self.error_rate() >= config.error_rate_threshold {
2258            return true;
2259        }
2260
2261        false
2262    }
2263
2264    /// Check if the circuit should transition to half-open.
2265    ///
2266    /// Returns true if the circuit is open and the cooldown period has elapsed.
2267    pub fn should_half_open(&self, config: &CircuitBreakerConfig) -> bool {
2268        if self.state != CircuitState::Open {
2269            return false;
2270        }
2271
2272        let now = Self::now_millis();
2273        let cooldown_ms = self.effective_cooldown_ms(config);
2274
2275        if let Some(opened_at) = self.opened_at {
2276            now.saturating_sub(opened_at) >= cooldown_ms
2277        } else {
2278            false
2279        }
2280    }
2281
2282    /// Check if the circuit should close.
2283    ///
2284    /// Returns true if in half-open state, consecutive successes meet or
2285    /// exceed the success threshold, AND no command-class failure is
2286    /// outstanding. A worker whose last real command timed out is not
2287    /// recovered no matter how many liveness probes it answers; it stays
2288    /// HalfOpen (degraded, probe-limited) until a command-class success
2289    /// clears the evidence.
2290    pub fn should_close(&self, config: &CircuitBreakerConfig) -> bool {
2291        if self.state != CircuitState::HalfOpen {
2292            return false;
2293        }
2294
2295        self.consecutive_successes >= config.success_threshold
2296            && self.consecutive_command_failures == 0
2297    }
2298
2299    /// Check if a probe request can be made in half-open state.
2300    ///
2301    /// Returns true if active probes are below the maximum allowed.
2302    pub fn can_probe(&self, config: &CircuitBreakerConfig) -> bool {
2303        if self.state != CircuitState::HalfOpen {
2304            return false;
2305        }
2306
2307        self.active_probes < config.half_open_max_probes
2308    }
2309
2310    /// Start a probe request in half-open state.
2311    ///
2312    /// Returns true if the probe was started, false if already at max probes.
2313    pub fn start_probe(&mut self, config: &CircuitBreakerConfig) -> bool {
2314        if !self.can_probe(config) {
2315            return false;
2316        }
2317        self.active_probes += 1;
2318        true
2319    }
2320
2321    /// Transition the circuit to open state.
2322    pub fn open(&mut self) {
2323        if self.state != CircuitState::Open {
2324            self.state = CircuitState::Open;
2325            self.opened_at = Some(Self::now_millis());
2326            self.last_state_change = Self::now_millis();
2327            self.consecutive_successes = 0;
2328            self.active_probes = 0;
2329            // Every open without an intervening close lengthens the next
2330            // cooldown (see `CircuitBreakerConfig::cooldown_ms_for_streak`).
2331            self.reopen_streak = self.reopen_streak.saturating_add(1);
2332        }
2333    }
2334
2335    /// Transition the circuit to half-open state.
2336    pub fn half_open(&mut self) {
2337        if self.state != CircuitState::HalfOpen {
2338            self.state = CircuitState::HalfOpen;
2339            self.last_state_change = Self::now_millis();
2340            self.consecutive_successes = 0;
2341            self.active_probes = 0;
2342        }
2343    }
2344
2345    /// Transition the circuit to closed state.
2346    pub fn close(&mut self) {
2347        if self.state != CircuitState::Closed {
2348            self.state = CircuitState::Closed;
2349            self.last_state_change = Self::now_millis();
2350            self.opened_at = None;
2351            self.consecutive_failures = 0;
2352            self.consecutive_successes = 0;
2353            self.active_probes = 0;
2354            self.consecutive_command_failures = 0;
2355            self.reopen_streak = 0;
2356            // Reset the window on close, INCLUDING the recent-results history
2357            // that error_rate() now reads. Otherwise a freshly-recovered circuit
2358            // would still compute a high error rate from the pre-failure samples
2359            // and immediately re-open (flapping). Matches the original intent of
2360            // resetting the error-rate inputs when the circuit recovers.
2361            self.window_successes = 0;
2362            self.window_failures = 0;
2363            self.recent_results.clear();
2364        }
2365    }
2366
2367    /// Reset the rolling window counters.
2368    ///
2369    /// Called periodically to ensure the window reflects recent activity.
2370    /// Also clears the recent-results history that `error_rate()` is computed
2371    /// from, so a reset truly zeroes the observed error rate.
2372    pub fn reset_window(&mut self) {
2373        self.window_successes = 0;
2374        self.window_failures = 0;
2375        self.recent_results.clear();
2376    }
2377
2378    /// Get recent health check results for history visualization.
2379    ///
2380    /// Returns a slice of recent results (true=success, false=failure),
2381    /// with the most recent result at the end.
2382    pub fn recent_results(&self) -> &[bool] {
2383        &self.recent_results
2384    }
2385
2386    /// Calculate seconds remaining until circuit auto-transitions to half-open.
2387    ///
2388    /// Returns None if circuit is not open or cooldown has already elapsed.
2389    pub fn recovery_remaining_secs(&self, config: &CircuitBreakerConfig) -> Option<u64> {
2390        if self.state != CircuitState::Open {
2391            return None;
2392        }
2393
2394        let now = Self::now_millis();
2395        let cooldown_ms = self.effective_cooldown_ms(config);
2396
2397        if let Some(opened_at) = self.opened_at {
2398            let elapsed_ms = now.saturating_sub(opened_at);
2399            if elapsed_ms < cooldown_ms {
2400                return Some((cooldown_ms - elapsed_ms) / 1000);
2401            }
2402        }
2403        None
2404    }
2405
2406    /// Push a result to the history, maintaining the maximum size.
2407    fn push_result(&mut self, success: bool) {
2408        if self.recent_results.len() >= CIRCUIT_HISTORY_SIZE {
2409            self.recent_results.remove(0);
2410        }
2411        self.recent_results.push(success);
2412    }
2413
2414    fn now_millis() -> u64 {
2415        std::time::SystemTime::now()
2416            .duration_since(std::time::UNIX_EPOCH)
2417            .map(|d| d.as_millis() as u64)
2418            .unwrap_or(0)
2419    }
2420
2421    /// Drive the circuit through a single health-check outcome and return the
2422    /// resulting [`CircuitState`].
2423    ///
2424    /// This is the single, self-contained transition engine shared by every
2425    /// circuit consumer (the daemon health monitor via
2426    /// `WorkerState::record_health_check`, and the diagnostic `WorkerHealth`
2427    /// panel via `WorkerHealth::observe`). Keeping the transition logic in ONE
2428    /// place is what makes `WorkerState.circuit` (which selection reads) and the
2429    /// diagnostic `WorkerHealth.circuit` agree.
2430    ///
2431    /// Behaviour:
2432    /// - On a **healthy** outcome: if the circuit is Open and its cooldown has
2433    ///   elapsed it first transitions Open → HalfOpen *before* crediting the
2434    ///   success, so recovery needs exactly `success_threshold` clean probes
2435    ///   (crediting a success against an Open circuit would be lost). Then the
2436    ///   success is recorded and the circuit closes once the threshold is met.
2437    /// - On an **unhealthy** outcome: the failure is recorded, the circuit opens
2438    ///   if the closed-state open condition is met, and a HalfOpen probe failure
2439    ///   reopens the circuit — UNLESS the failure was `transient` (a retryable
2440    ///   transport blip that survived the in-check retries). A transient
2441    ///   half-open failure neither reopens nor resets the cooldown, so a single
2442    ///   network hiccup during recovery cannot bounce a worker back to Open.
2443    /// - After either branch, an Open circuit past its cooldown is nudged to
2444    ///   HalfOpen so probing resumes on the next poll even for a still-failing
2445    ///   worker.
2446    pub fn apply_health_outcome(
2447        &mut self,
2448        healthy: bool,
2449        transient: bool,
2450        config: &CircuitBreakerConfig,
2451    ) -> CircuitState {
2452        if healthy {
2453            // Transition to half-open BEFORE crediting the success so the probe
2454            // counts toward recovery. record_success() only increments the
2455            // consecutive-success counter while in HalfOpen.
2456            if self.state == CircuitState::Open && self.should_half_open(config) {
2457                self.half_open();
2458            }
2459            self.record_success();
2460            if self.should_close(config) {
2461                self.close();
2462            }
2463        } else {
2464            let was_half_open = self.state == CircuitState::HalfOpen;
2465            self.record_failure();
2466            if self.should_open(config) {
2467                self.open();
2468            } else if was_half_open && !transient {
2469                // A real (non-transient) probe failure in half-open reopens the
2470                // circuit and restarts the cooldown. A transient failure is
2471                // ignored so a network blip can't undo an in-progress recovery.
2472                self.open();
2473            }
2474        }
2475
2476        // Even for a still-failing worker, resume probing once the cooldown
2477        // elapses so a recovered worker is retried promptly.
2478        if self.state == CircuitState::Open && self.should_half_open(config) {
2479            self.half_open();
2480        }
2481
2482        self.state
2483    }
2484
2485    /// Drive the circuit through a single *command-class* outcome and return
2486    /// the resulting [`CircuitState`].
2487    ///
2488    /// Command-class evidence comes from an operation of the same shape as
2489    /// dispatched work: a remote binary that must run to completion (the
2490    /// telemetry poll, or a health probe whose SSH session connected but whose
2491    /// command never finished). It is the signal the plain liveness probe
2492    /// cannot provide — a host under fork exhaustion or swap thrash keeps
2493    /// answering `echo` over a warm connection while every real command hangs,
2494    /// and on that evidence alone the breaker would happily re-admit it.
2495    ///
2496    /// Behaviour:
2497    /// - **Success**: clears the outstanding command-failure count and then
2498    ///   counts as an ordinary healthy outcome (may advance Open -> HalfOpen ->
2499    ///   Closed exactly like a liveness success).
2500    /// - **Failure**: increments the command-failure count and records a plain
2501    ///   failure. The circuit opens when either the ordinary open condition or
2502    ///   `command_failure_threshold` is met. In HalfOpen a command failure
2503    ///   ALWAYS reopens — there is no transient exemption, because "connected
2504    ///   but the command did not complete" is precisely the condition the
2505    ///   half-open trial exists to detect. While any command failure is
2506    ///   outstanding, [`Self::should_close`] refuses to close on liveness
2507    ///   successes alone.
2508    pub fn apply_command_outcome(
2509        &mut self,
2510        success: bool,
2511        config: &CircuitBreakerConfig,
2512    ) -> CircuitState {
2513        if success {
2514            self.consecutive_command_failures = 0;
2515            return self.apply_health_outcome(true, false, config);
2516        }
2517
2518        self.consecutive_command_failures = self.consecutive_command_failures.saturating_add(1);
2519        let was_half_open = self.state == CircuitState::HalfOpen;
2520        self.record_failure();
2521        let threshold = config.command_failure_threshold.max(1);
2522        if was_half_open
2523            || self.should_open(config)
2524            || (self.state == CircuitState::Closed
2525                && self.consecutive_command_failures >= threshold)
2526        {
2527            self.open();
2528        }
2529
2530        if self.state == CircuitState::Open && self.should_half_open(config) {
2531            self.half_open();
2532        }
2533
2534        self.state
2535    }
2536}
2537
2538#[derive(Debug, Clone, Serialize, Deserialize)]
2539pub struct CompilationConfig {
2540    /// Minimum confidence score to intercept (0.0-1.0).
2541    #[serde(default = "default_confidence")]
2542    pub confidence_threshold: f64,
2543    /// Run locally when this build's measured local median is below this (ms).
2544    /// Consulted only by the speed gate, i.e. when `local_slots > 0`.
2545    #[serde(default = "default_min_local_time")]
2546    pub min_local_time_ms: u64,
2547    /// Minimum measured speedup ratio (local_time / remote_time) to offload.
2548    /// If the measured speedup < threshold, run locally. Default: 1.2 (20% faster).
2549    /// Consulted only by the speed gate, i.e. when `local_slots > 0`.
2550    #[serde(default = "default_speedup_threshold")]
2551    pub remote_speedup_threshold: f64,
2552    /// Speed gate (bd-2spi4): how many offloadable builds may run concurrently
2553    /// on THIS machine because its own measured build history says remote is
2554    /// not worth it. Zero (default) disables the gate, so every offloadable
2555    /// build is offloaded as before. Set it only on a dispatcher that is faster
2556    /// than its workers; local runs also need free memory (see the hook).
2557    #[serde(default)]
2558    pub local_slots: u32,
2559    /// Default slot estimate for build commands (cargo build, gcc, etc.).
2560    #[serde(default = "default_build_slots")]
2561    pub build_slots: u32,
2562    /// Default slot estimate for test commands (cargo test, nextest).
2563    /// Tests typically use more parallelism than builds.
2564    #[serde(default = "default_test_slots")]
2565    pub test_slots: u32,
2566    /// Default slot estimate for check/lint commands (cargo check, clippy).
2567    /// These are typically faster and use fewer resources.
2568    #[serde(default = "default_check_slots")]
2569    pub check_slots: u32,
2570    /// Per-build additional free-space budget on the worker's reported build
2571    /// filesystem, in GiB. Include expected output growth and a safety margin.
2572    /// Zero (default) disables this explicit budget. Positive budgets require
2573    /// a fresh disk observation and are held in durable daemon ownership until
2574    /// completion. They are admission accounting, not filesystem quotas.
2575    #[serde(default)]
2576    pub disk_headroom_gib: u32,
2577    /// Timeout in seconds for build commands.
2578    /// Build commands (cargo build, gcc, etc.) typically complete faster than tests.
2579    #[serde(default = "default_build_timeout")]
2580    pub build_timeout_sec: u64,
2581    /// Timeout in seconds for test commands.
2582    /// Test commands often need longer timeouts due to test suite execution time.
2583    #[serde(default = "default_test_timeout")]
2584    pub test_timeout_sec: u64,
2585    /// Timeout in seconds for bun commands (bun test, bun typecheck).
2586    /// Bun has known issues where its internal --timeout flag doesn't work for
2587    /// CPU-bound hangs, so we use a shorter external timeout by default.
2588    #[serde(default = "default_bun_timeout")]
2589    pub bun_timeout_sec: u64,
2590    /// Whether to wrap remote commands with external timeout protection.
2591    /// This prevents runaway/stuck processes from consuming worker slots indefinitely.
2592    /// Default: true. Set to false to disable timeout wrapping entirely.
2593    #[serde(default = "default_external_timeout_enabled")]
2594    pub external_timeout_enabled: bool,
2595    /// Whether a remote build FAILURE may fall back to LOCAL execution on the
2596    /// orchestrator once every eligible remote worker has been exhausted.
2597    ///
2598    /// On a remote failure the hook first retries the build on a *different,
2599    /// higher-capacity* worker (see `run_exec`'s worker-fallback loop). Local
2600    /// execution runs the heavy compile on the orchestrator that coordinates the
2601    /// whole fleet, so it is a genuine last resort and can flood that box. This
2602    /// flag gates it. Default `true` preserves the historical fail-open behavior
2603    /// (local only when no worker can serve); set to `false` on an orchestrator
2604    /// to refuse local fallback and fail the build closed instead. `force_remote`
2605    /// / proof mode / `RCH_REQUIRE_REMOTE` still override this by refusing local
2606    /// unconditionally.
2607    #[serde(default = "default_allow_local_fallback")]
2608    pub allow_local_fallback: bool,
2609    /// Per-job rustc parallelism cap applied on the WORKER (issue #49).
2610    ///
2611    /// `total_slots` bounds concurrent *jobs* per worker, but cargo defaults
2612    /// `-j` to the worker's `nproc`, so one offloaded `cargo test --workspace`
2613    /// forks `nproc` rustc regardless of slots and can swap-thrash a
2614    /// RAM-light box. This policy injects `CARGO_BUILD_JOBS` into the remote
2615    /// build environment; see [`RemoteBuildJobs`] for the modes. It never
2616    /// overrides a `CARGO_BUILD_JOBS` the worker session already carries, an
2617    /// inline `CARGO_BUILD_JOBS=N` or `-j N` in the command, or a project
2618    /// `.cargo/config.toml` `[build] jobs` — explicit intent always wins.
2619    #[serde(default)]
2620    pub remote_build_jobs: RemoteBuildJobs,
2621}
2622
2623/// How rch derives the `CARGO_BUILD_JOBS` it injects into a remote build.
2624///
2625/// - `auto` (default): computed on the worker from its live `nproc` and
2626///   total RAM as `clamp(ram_gib / 8, 2, min(nproc, 8))` — budgeting roughly
2627///   8 GiB of RAM per concurrent rustc leaves headroom for two to three
2628///   concurrent jobs at a realistic per-rustc peak of ~3 GiB.
2629/// - `off`: inject nothing (historical behavior; cargo uses `nproc`).
2630/// - `<n>`: inject a fixed job count fleet-wide.
2631///
2632/// In every mode an explicitly set `CARGO_BUILD_JOBS` on the worker (for
2633/// example from `/etc/environment`) or in the command is left untouched.
2634#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
2635pub enum RemoteBuildJobs {
2636    /// Derive from the worker's cores and RAM at execution time.
2637    #[default]
2638    Auto,
2639    /// Do not inject `CARGO_BUILD_JOBS`.
2640    Off,
2641    /// Inject a fixed job count (must be >= 1).
2642    Fixed(u32),
2643}
2644
2645impl RemoteBuildJobs {
2646    /// Parse the config/CLI spelling: `auto`, `off`, or a positive integer.
2647    pub fn parse(value: &str) -> Result<Self, String> {
2648        let trimmed = value.trim();
2649        match trimmed.to_ascii_lowercase().as_str() {
2650            "auto" => Ok(Self::Auto),
2651            "off" | "none" | "disabled" => Ok(Self::Off),
2652            _ => match trimmed.parse::<u32>() {
2653                Ok(0) => Err(
2654                    "remote_build_jobs must be >= 1 (cargo rejects `-j 0`); use `off` to disable"
2655                        .to_string(),
2656                ),
2657                Ok(n) => Ok(Self::Fixed(n)),
2658                Err(_) => Err(format!(
2659                    "'{trimmed}' is not a valid remote_build_jobs value (expected `auto`, `off`, or a positive integer)"
2660                )),
2661            },
2662        }
2663    }
2664}
2665
2666impl std::fmt::Display for RemoteBuildJobs {
2667    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2668        match self {
2669            Self::Auto => f.write_str("auto"),
2670            Self::Off => f.write_str("off"),
2671            Self::Fixed(n) => write!(f, "{n}"),
2672        }
2673    }
2674}
2675
2676impl std::str::FromStr for RemoteBuildJobs {
2677    type Err = String;
2678
2679    fn from_str(value: &str) -> Result<Self, Self::Err> {
2680        Self::parse(value)
2681    }
2682}
2683
2684impl Serialize for RemoteBuildJobs {
2685    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
2686        match self {
2687            Self::Auto => serializer.serialize_str("auto"),
2688            Self::Off => serializer.serialize_str("off"),
2689            Self::Fixed(n) => serializer.serialize_u32(*n),
2690        }
2691    }
2692}
2693
2694impl<'de> Deserialize<'de> for RemoteBuildJobs {
2695    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
2696        #[derive(Deserialize)]
2697        #[serde(untagged)]
2698        enum Raw {
2699            Count(u64),
2700            Spelling(String),
2701        }
2702        match Raw::deserialize(deserializer)? {
2703            Raw::Count(n) => {
2704                let n = u32::try_from(n).map_err(|_| {
2705                    serde::de::Error::custom(format!("remote_build_jobs {n} is out of range"))
2706                })?;
2707                Self::parse(&n.to_string()).map_err(serde::de::Error::custom)
2708            }
2709            Raw::Spelling(s) => Self::parse(&s).map_err(serde::de::Error::custom),
2710        }
2711    }
2712}
2713
2714impl Default for CompilationConfig {
2715    fn default() -> Self {
2716        Self {
2717            confidence_threshold: 0.85,
2718            min_local_time_ms: 2000,
2719            remote_speedup_threshold: default_speedup_threshold(),
2720            local_slots: 0,
2721            build_slots: default_build_slots(),
2722            test_slots: default_test_slots(),
2723            check_slots: default_check_slots(),
2724            disk_headroom_gib: 0,
2725            build_timeout_sec: default_build_timeout(),
2726            test_timeout_sec: default_test_timeout(),
2727            bun_timeout_sec: default_bun_timeout(),
2728            external_timeout_enabled: default_external_timeout_enabled(),
2729            allow_local_fallback: default_allow_local_fallback(),
2730            remote_build_jobs: RemoteBuildJobs::default(),
2731        }
2732    }
2733}
2734
2735fn default_build_slots() -> u32 {
2736    4
2737}
2738
2739fn default_test_slots() -> u32 {
2740    8
2741}
2742
2743fn default_check_slots() -> u32 {
2744    2
2745}
2746
2747/// Default build timeout: 5 minutes (300 seconds).
2748/// Most builds complete well within this time.
2749fn default_build_timeout() -> u64 {
2750    300
2751}
2752
2753/// Default test timeout: 30 minutes (1800 seconds).
2754/// Test suites often take significantly longer than builds.
2755fn default_test_timeout() -> u64 {
2756    1800
2757}
2758
2759/// Default bun timeout: 10 minutes (600 seconds).
2760/// Bun's internal --timeout flag doesn't work for CPU-bound hangs,
2761/// so we use a shorter external timeout to prevent zombies.
2762fn default_bun_timeout() -> u64 {
2763    600
2764}
2765
2766/// Default: external timeout protection is enabled.
2767fn default_external_timeout_enabled() -> bool {
2768    true
2769}
2770
2771/// Default: permit last-resort local fallback after remote workers are
2772/// exhausted. Set `compilation.allow_local_fallback = false` to fail closed and
2773/// keep heavy compiles off the orchestrator entirely.
2774fn default_allow_local_fallback() -> bool {
2775    true
2776}
2777
2778impl CompilationConfig {
2779    /// Returns the appropriate external timeout for the given compilation kind.
2780    ///
2781    /// - Bun commands get the shorter bun_timeout (to protect against known hang issues)
2782    /// - Test commands get the longer test_timeout
2783    /// - All other commands (builds, checks, clippy) get the build_timeout
2784    pub fn timeout_for_kind(&self, kind: Option<CompilationKind>) -> std::time::Duration {
2785        let secs = match kind {
2786            // Bun has known issues where internal timeout doesn't work for CPU hangs
2787            Some(CompilationKind::BunTest) | Some(CompilationKind::BunTypecheck) => {
2788                self.bun_timeout_sec
2789            }
2790            // Other test commands use the standard test timeout
2791            Some(CompilationKind::CargoTest) | Some(CompilationKind::CargoNextest) => {
2792                self.test_timeout_sec
2793            }
2794            // Nix derivation builds can take a long time (fetch + compile large
2795            // closures); give them the generous test-timeout bucket rather than
2796            // the short build timeout so legitimate builds are not killed early.
2797            Some(CompilationKind::NixBuild) => self.test_timeout_sec,
2798            // Go test suites are long-running like cargo test; give them the
2799            // test bucket rather than the short build timeout.
2800            Some(CompilationKind::GoTest) => self.test_timeout_sec,
2801            // Builds, checks, clippy, and unknown use build timeout
2802            _ => self.build_timeout_sec,
2803        };
2804        std::time::Duration::from_secs(secs)
2805    }
2806
2807    /// Returns whether external timeout wrapping is enabled.
2808    pub fn external_timeout_enabled(&self) -> bool {
2809        self.external_timeout_enabled
2810    }
2811}
2812
2813/// Version of the Layer 0 configuration pack schema (bd-bqu38 / M-1 §149).
2814///
2815/// Bump when a knob is added, removed, or its meaning changes; consumers
2816/// record the version alongside emitted baselines so regressions are
2817/// attributable to a specific pack revision.
2818pub const LAYER0_PACK_VERSION: &str = "1";
2819
2820/// Layer 0 configuration pack knobs (bd-bqu38).
2821///
2822/// Every knob defaults to OFF and toggles independently — the kill/rollback
2823/// contract from the master plan (§149: removed from defaults if it regresses
2824/// representative p95, output equivalence, debugger behavior, or
2825/// compatibility). Knobs are injected into remote cargo builds as
2826/// `CARGO_PROFILE_*` environment variables, so enabling one never mutates any
2827/// user-owned `.cargo/config.toml`.
2828#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
2829#[serde(default)]
2830pub struct Layer0Config {
2831    /// Master switch: nothing is injected unless this is on.
2832    pub enabled: bool,
2833    /// `profile.release.lto = "thin"` via `CARGO_PROFILE_RELEASE_LTO=thin`.
2834    pub release_lto_thin: bool,
2835    /// `profile.release.codegen-units = 1` via
2836    /// `CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1`.
2837    pub release_codegen_units_1: bool,
2838}
2839
2840impl Layer0Config {
2841    /// Environment pairs to inject into a remote cargo build.
2842    ///
2843    /// Empty unless [`Self::enabled`] — the master switch gates every knob so
2844    /// an operator can disable the whole pack with one key. Order is stable
2845    /// (declaration order) for deterministic emission and test assertions.
2846    #[must_use]
2847    pub fn active_env(&self) -> Vec<(&'static str, &'static str)> {
2848        if !self.enabled {
2849            return Vec::new();
2850        }
2851        let mut env = Vec::new();
2852        if self.release_lto_thin {
2853            env.push(("CARGO_PROFILE_RELEASE_LTO", "thin"));
2854        }
2855        if self.release_codegen_units_1 {
2856            env.push(("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"));
2857        }
2858        env
2859    }
2860}
2861
2862#[derive(Debug, Clone, Serialize, Deserialize)]
2863pub struct TransferConfig {
2864    /// Durable controller retrieval staging base. Unset uses the job-lease
2865    /// directory's retrieval subtree. Must be absolute; TMPDIR is not used.
2866    #[serde(default, skip_serializing_if = "Option::is_none")]
2867    pub retrieval_staging_dir: Option<PathBuf>,
2868    /// Free bytes to retain on the staging filesystem in addition to selected
2869    /// uncompressed output bytes. The preflight never deletes retained data.
2870    #[serde(default)]
2871    pub retrieval_min_free_bytes: u64,
2872    /// zstd compression level (1-19).
2873    #[serde(default = "default_compression")]
2874    pub compression_level: u32,
2875    /// Patterns to exclude from transfer.
2876    #[serde(default = "default_excludes")]
2877    pub exclude_patterns: Vec<String>,
2878    /// SSH keepalive interval in seconds (`ssh -o ServerAliveInterval=<N>`).
2879    ///
2880    /// When unset, build transfers use 15 seconds. Zero defers to OpenSSH configuration.
2881    #[serde(default, skip_serializing_if = "Option::is_none")]
2882    pub ssh_server_alive_interval_secs: Option<u64>,
2883    /// SSH ControlPersist idle timeout in seconds (`ssh -o ControlPersist=<N>s`).
2884    ///
2885    /// - Unset preserves OpenSSH defaults (ControlPersist enabled by default via mux).
2886    /// - `0` disables persistence (`ControlPersist=no`).
2887    #[serde(default, skip_serializing_if = "Option::is_none")]
2888    pub ssh_control_persist_secs: Option<u64>,
2889    /// Staging base on the worker for the transfer paths that do NOT mirror
2890    /// the client's project layout: `--clean-overlay` roots, Windows workers,
2891    /// and `rch cache warm`. Must be an absolute path; defaults to
2892    /// [`default_remote_base`] (`/data/tmp/rch`).
2893    ///
2894    /// An ordinary `rch exec` does not use this: it mirrors the project under
2895    /// the worker's canonical root (`[path_topology] canonical_root`), and the
2896    /// pooled Cargo target store rides along inside that mirror. To move
2897    /// pooled stores onto another filesystem, set
2898    /// `[remediation.pooled_target] store_base` (issue #64).
2899    #[serde(default = "default_remote_base")]
2900    pub remote_base: String,
2901    /// Per-attempt timeout for source synchronization, in milliseconds.
2902    ///
2903    /// This is deliberately separate from remote compilation and artifact-return
2904    /// timeouts. When unset, RCH derives a conservative timeout from the source
2905    /// payload size (30 seconds plus one second per MiB, capped at one hour).
2906    #[serde(default, skip_serializing_if = "Option::is_none")]
2907    pub sync_timeout_ms: Option<u64>,
2908    /// Silence-based stall timeout for source synchronization, in seconds
2909    /// (issue #59).
2910    ///
2911    /// Aborts a source sync when rsync emits NO output at all — no progress
2912    /// refresh, no stats, no itemized line — for this long. This is deliberately
2913    /// far tighter than the wall-clock `sync_timeout_ms` cap (which scales with
2914    /// payload size up to an hour): a large first-time sync keeps refreshing its
2915    /// progress line and is never killed, while a dead channel or wedged rsync
2916    /// is detected within this window instead of holding the reservation until
2917    /// the wall-clock cap. On a stall the hook releases the worker and re-enters
2918    /// selection excluding it. `0` disables silence detection.
2919    #[serde(default = "default_source_sync_silence_timeout_secs")]
2920    pub source_sync_silence_timeout_secs: u64,
2921    /// Retry policy for transient network errors during transfer.
2922    #[serde(default)]
2923    pub retry: RetryConfig,
2924    /// Verify artifact integrity using blake3 hashes after transfer (bd-377q).
2925    ///
2926    /// When enabled, computes blake3 hashes of key artifacts (binaries, test outputs)
2927    /// on the worker after build, then verifies them locally after download.
2928    /// Disabled by default to avoid extra overhead.
2929    #[serde(default)]
2930    pub verify_artifacts: bool,
2931    /// Maximum file size (bytes) for artifact verification (bd-377q).
2932    ///
2933    /// Files larger than this limit are skipped during verification to avoid
2934    /// excessive I/O. Defaults to 100MB.
2935    #[serde(default = "default_verify_max_size")]
2936    pub verify_max_size_bytes: u64,
2937
2938    // =========================================================================
2939    // Transfer Optimization (bd-3hho)
2940    // =========================================================================
2941    /// Maximum transfer size in MB before skipping remote execution.
2942    ///
2943    /// When set, runs `rsync --dry-run --stats` to estimate transfer size.
2944    /// If the estimated size exceeds this threshold, remote offload is skipped
2945    /// and the command runs locally. Set to `None` (default) to disable.
2946    #[serde(default, skip_serializing_if = "Option::is_none")]
2947    pub max_transfer_mb: Option<u64>,
2948
2949    /// Maximum estimated transfer time in milliseconds before skipping remote.
2950    ///
2951    /// Uses `estimated_bandwidth_bps` (or measured link speed) to calculate
2952    /// expected transfer time. If it exceeds this threshold, remote offload
2953    /// is skipped. Set to `None` (default) to disable.
2954    #[serde(default, skip_serializing_if = "Option::is_none")]
2955    pub max_transfer_time_ms: Option<u64>,
2956
2957    /// Bandwidth limit for rsync in KB/s.
2958    ///
2959    /// Passed as `rsync --bwlimit=<N>` to limit transfer speed and prevent
2960    /// network saturation. Set to `None` or `0` (default) for unlimited.
2961    #[serde(default, skip_serializing_if = "Option::is_none")]
2962    pub bwlimit_kbps: Option<u64>,
2963
2964    /// Explicit rsync binary for every local transfer (issue #66).
2965    ///
2966    /// Unset (the default) lets rch pick: the PATH `rsync` when it is a
2967    /// modern (3.1+) rsync, otherwise a modern rsync from the well-known
2968    /// Homebrew/MacPorts locations, otherwise the PATH binary driven with the
2969    /// openrsync/2.6.9-compatible argv. Set this to force one binary
2970    /// (`~` is expanded; a bare name is looked up on PATH). The
2971    /// `RCH_RSYNC_BIN` environment variable overrides this setting.
2972    #[serde(default, skip_serializing_if = "Option::is_none")]
2973    pub rsync_bin: Option<String>,
2974
2975    /// Estimated link bandwidth in bytes per second.
2976    ///
2977    /// Used for transfer time estimation when `max_transfer_time_ms` is set.
2978    /// If not provided, defaults to 10 MB/s (10485760 bytes/sec).
2979    #[serde(default, skip_serializing_if = "Option::is_none")]
2980    pub estimated_bandwidth_bps: Option<u64>,
2981
2982    // =========================================================================
2983    // Adaptive Compression (bd-243w)
2984    // =========================================================================
2985    /// Enable adaptive zstd compression based on transfer size (bd-243w).
2986    ///
2987    /// When enabled, automatically selects compression level based on estimated
2988    /// payload size:
2989    /// - < 10MB: level 1 (fast, minimal compression)
2990    /// - 10-200MB: level 3 (balanced, default)
2991    /// - > 200MB: level 7 (slower, better compression)
2992    ///
2993    /// When disabled (default), uses the fixed `compression_level` setting.
2994    #[serde(default)]
2995    pub adaptive_compression: bool,
2996
2997    /// Minimum compression level for adaptive mode (bd-243w).
2998    ///
2999    /// Even in adaptive mode, compression level won't go below this.
3000    /// Defaults to 1.
3001    #[serde(default = "default_min_compression")]
3002    pub min_compression_level: u32,
3003
3004    /// Maximum compression level for adaptive mode (bd-243w).
3005    ///
3006    /// Even in adaptive mode, compression level won't exceed this.
3007    /// Defaults to 9 (avoids CPU-intensive levels 10+).
3008    #[serde(default = "default_max_compression")]
3009    pub max_compression_level: u32,
3010}
3011
3012impl Default for TransferConfig {
3013    fn default() -> Self {
3014        Self {
3015            retrieval_staging_dir: None,
3016            retrieval_min_free_bytes: 0,
3017            compression_level: 3,
3018            exclude_patterns: default_excludes(),
3019            ssh_server_alive_interval_secs: None,
3020            ssh_control_persist_secs: None,
3021            remote_base: default_remote_base(),
3022            sync_timeout_ms: None,
3023            source_sync_silence_timeout_secs: default_source_sync_silence_timeout_secs(),
3024            retry: RetryConfig::default(),
3025            verify_artifacts: false,
3026            verify_max_size_bytes: default_verify_max_size(),
3027            // Transfer optimization (bd-3hho)
3028            max_transfer_mb: None,
3029            max_transfer_time_ms: None,
3030            bwlimit_kbps: None,
3031            rsync_bin: None,
3032            estimated_bandwidth_bps: None,
3033            // Adaptive compression (bd-243w)
3034            adaptive_compression: false,
3035            min_compression_level: default_min_compression(),
3036            max_compression_level: default_max_compression(),
3037        }
3038    }
3039}
3040
3041impl TransferConfig {
3042    /// Smallest accepted explicit source-sync timeout (one second).
3043    pub const MIN_SYNC_TIMEOUT_MS: u64 = 1_000;
3044    /// Largest accepted explicit or derived source-sync timeout (one hour).
3045    pub const MAX_SYNC_TIMEOUT_MS: u64 = 3_600_000;
3046    /// Fixed floor used by the payload-aware default policy.
3047    pub const DEFAULT_SYNC_TIMEOUT_FLOOR_MS: u64 = 30_000;
3048    const DEFAULT_SYNC_BYTES_PER_SECOND: u64 = 1024 * 1024;
3049
3050    /// Whether an explicit source-sync timeout is within the supported range.
3051    #[must_use]
3052    pub fn valid_sync_timeout_ms(value: u64) -> bool {
3053        (Self::MIN_SYNC_TIMEOUT_MS..=Self::MAX_SYNC_TIMEOUT_MS).contains(&value)
3054    }
3055
3056    /// Resolve the per-attempt source-sync timeout for an exact payload size.
3057    ///
3058    /// Explicit configuration wins. Otherwise the default grants 30 seconds of
3059    /// setup time plus one second per MiB at a conservative 1 MiB/s floor. This
3060    /// prevents a cold, large pinned tree from inheriting the old fixed 30-second
3061    /// ceiling while still bounding every individual attempt.
3062    #[must_use]
3063    pub fn sync_timeout_for_payload(&self, payload_bytes: u64) -> std::time::Duration {
3064        let timeout_ms = self
3065            .sync_timeout_ms
3066            .filter(|value| Self::valid_sync_timeout_ms(*value))
3067            .unwrap_or_else(|| {
3068                let transfer_seconds = payload_bytes
3069                    .saturating_add(Self::DEFAULT_SYNC_BYTES_PER_SECOND - 1)
3070                    / Self::DEFAULT_SYNC_BYTES_PER_SECOND;
3071                Self::DEFAULT_SYNC_TIMEOUT_FLOOR_MS
3072                    .saturating_add(transfer_seconds.saturating_mul(1_000))
3073                    .min(Self::MAX_SYNC_TIMEOUT_MS)
3074            });
3075        std::time::Duration::from_millis(timeout_ms)
3076    }
3077
3078    /// Select compression level based on estimated transfer size (bd-243w).
3079    ///
3080    /// When adaptive compression is enabled, selects an appropriate level
3081    /// based on payload size:
3082    /// - < 10MB: level 1 (fast, minimal compression)
3083    /// - 10-200MB: level 3 (balanced)
3084    /// - > 200MB: level 7 (better compression)
3085    ///
3086    /// The result is clamped between `min_compression_level` and `max_compression_level`.
3087    ///
3088    /// When adaptive compression is disabled, returns the fixed `compression_level`.
3089    pub fn select_compression_level(&self, estimated_bytes: Option<u64>) -> u32 {
3090        if !self.adaptive_compression {
3091            return self.compression_level;
3092        }
3093
3094        let Some(bytes) = estimated_bytes else {
3095            // No estimate available, use default
3096            return self.compression_level;
3097        };
3098
3099        // Size thresholds
3100        const SMALL_THRESHOLD: u64 = 10_000_000; // 10 MB (decimal)
3101        const LARGE_THRESHOLD: u64 = 200_000_000; // 200 MB (decimal)
3102
3103        let level = if bytes < SMALL_THRESHOLD {
3104            1 // Fast compression for small payloads
3105        } else if bytes < LARGE_THRESHOLD {
3106            3 // Balanced for medium payloads
3107        } else {
3108            7 // Better compression for large payloads
3109        };
3110
3111        // Clamp to configured bounds
3112        level.clamp(self.min_compression_level, self.max_compression_level)
3113    }
3114}
3115
3116// =============================================================================
3117// Execution Configuration (bd-785w)
3118// =============================================================================
3119
3120/// Default allowlist of commands that can be executed remotely.
3121/// Aligned with the classifier's supported CompilationKind values.
3122/// Command bases of every compilation kind rch knows how to offload.
3123///
3124/// This is the built-in `execution.allowlist`. It is also what a persisted,
3125/// explicit allowlist is compared against: a config file written before a tool
3126/// family was added (Go, Nix, TypeScript, ...) silently keeps that family local,
3127/// and the only symptom is a `[RCH] local (... not in execution.allowlist)` line
3128/// (issue #51). `ExecutionConfig::missing_builtin_bases` surfaces the gap.
3129#[rustfmt::skip]
3130pub const BUILTIN_EXECUTION_BASES: &[&str] = &[
3131    // Rust
3132    "cargo",
3133    "rustc",
3134    // cargo-nextest
3135    "nextest",
3136    // C/C++
3137    "gcc",
3138    "g++",
3139    "clang",
3140    "clang++",
3141    "cc",
3142    "c++",
3143    // Build systems
3144    "make",
3145    "cmake",
3146    "ninja",
3147    "meson",
3148    // Bun
3149    "bun",
3150    // Nix (covers both `nix build` and the legacy `nix-build`)
3151    "nix",
3152    // Go
3153    "go",
3154    // TypeScript (covers both `tsc` and `npx tsc`; command_base is "tsc")
3155    "tsc",
3156];
3157
3158fn default_execution_allowlist() -> Vec<String> {
3159    BUILTIN_EXECUTION_BASES
3160        .iter()
3161        .map(|base| (*base).to_string())
3162        .collect()
3163}
3164
3165/// Execution allowlist configuration for remote builds (bd-785w).
3166///
3167/// Controls which command base names are permitted for remote execution.
3168/// Commands not in the allowlist will fail-open to local execution.
3169#[derive(Debug, Clone, Serialize, Deserialize)]
3170pub struct ExecutionConfig {
3171    /// Allowlist of command base names permitted for remote execution.
3172    ///
3173    /// Examples: "cargo", "rustc", "gcc", "bun"
3174    ///
3175    /// If a classified compilation command's base name is not in this list,
3176    /// RCH will fail-open to local execution with a clear reason.
3177    ///
3178    /// An empty allowlist disables all remote execution (local only).
3179    #[serde(default = "default_execution_allowlist")]
3180    pub allowlist: Vec<String>,
3181}
3182
3183impl Default for ExecutionConfig {
3184    fn default() -> Self {
3185        Self {
3186            allowlist: default_execution_allowlist(),
3187        }
3188    }
3189}
3190
3191impl ExecutionConfig {
3192    /// Check if a command base name is allowed for remote execution.
3193    pub fn is_allowed(&self, command_base: &str) -> bool {
3194        self.allowlist
3195            .iter()
3196            .any(|allowed| allowed.eq_ignore_ascii_case(command_base))
3197    }
3198
3199    /// Built-in compilation tool bases that this allowlist does NOT admit.
3200    ///
3201    /// Empty for the default allowlist. Non-empty means some command family rch
3202    /// classifies as compilation will always fall open to local execution on
3203    /// this box — usually because the config file predates that family.
3204    pub fn missing_builtin_bases(&self) -> Vec<&'static str> {
3205        BUILTIN_EXECUTION_BASES
3206            .iter()
3207            .copied()
3208            .filter(|base| !self.is_allowed(base))
3209            .collect()
3210    }
3211}
3212
3213// =============================================================================
3214// Retry Configuration (bd-x1ek)
3215// =============================================================================
3216
3217fn default_retry_max_attempts() -> u32 {
3218    3
3219}
3220
3221fn default_retry_base_delay_ms() -> u64 {
3222    100
3223}
3224
3225fn default_retry_max_delay_ms() -> u64 {
3226    5000
3227}
3228
3229fn default_retry_jitter_factor() -> f64 {
3230    0.1
3231}
3232
3233fn default_retry_total_timeout_ms() -> u64 {
3234    30000
3235}
3236
3237/// Configuration for retry behavior on transient errors.
3238///
3239/// Used by rsync and SSH operations to recover from transient network failures
3240/// without blocking too long or retrying non-recoverable errors.
3241#[derive(Debug, Clone, Serialize, Deserialize)]
3242pub struct RetryConfig {
3243    /// Maximum number of retry attempts (1 = no retries, just initial attempt).
3244    #[serde(default = "default_retry_max_attempts")]
3245    pub max_attempts: u32,
3246
3247    /// Initial delay between retries in milliseconds.
3248    #[serde(default = "default_retry_base_delay_ms")]
3249    pub base_delay_ms: u64,
3250
3251    /// Maximum delay between retries in milliseconds (caps exponential backoff).
3252    #[serde(default = "default_retry_max_delay_ms")]
3253    pub max_delay_ms: u64,
3254
3255    /// Jitter factor (0.0 - 1.0) to randomize delay.
3256    /// A value of 0.1 means ±10% randomization.
3257    #[serde(default = "default_retry_jitter_factor")]
3258    pub jitter_factor: f64,
3259
3260    /// Total timeout for all retry attempts in milliseconds.
3261    /// After this, retries stop even if max_attempts not reached.
3262    #[serde(default = "default_retry_total_timeout_ms")]
3263    pub total_timeout_ms: u64,
3264}
3265
3266impl Default for RetryConfig {
3267    fn default() -> Self {
3268        Self {
3269            max_attempts: default_retry_max_attempts(),
3270            base_delay_ms: default_retry_base_delay_ms(),
3271            max_delay_ms: default_retry_max_delay_ms(),
3272            jitter_factor: default_retry_jitter_factor(),
3273            total_timeout_ms: default_retry_total_timeout_ms(),
3274        }
3275    }
3276}
3277
3278impl RetryConfig {
3279    /// Create a config that disables retries (single attempt only).
3280    pub fn no_retry() -> Self {
3281        Self {
3282            max_attempts: 1,
3283            ..Default::default()
3284        }
3285    }
3286
3287    /// Calculate delay for a given attempt number (0-indexed).
3288    ///
3289    /// Uses exponential backoff with optional jitter.
3290    pub fn delay_for_attempt(&self, attempt: u32) -> std::time::Duration {
3291        if attempt == 0 {
3292            return std::time::Duration::ZERO;
3293        }
3294
3295        // Exponential backoff: base * 2^attempt
3296        let base = self.base_delay_ms as f64;
3297        let delay = base * (2.0_f64.powi(attempt as i32 - 1));
3298        let capped = delay.min(self.max_delay_ms as f64);
3299
3300        // Apply jitter: delay ± (delay * jitter_factor)
3301        let jittered = if self.jitter_factor > 0.0 {
3302            let jitter_range = capped * self.jitter_factor;
3303            let jitter = (rand::rng().random::<f64>() * 2.0 - 1.0) * jitter_range;
3304            (capped + jitter).max(0.0)
3305        } else {
3306            capped
3307        };
3308
3309        std::time::Duration::from_millis(jittered as u64)
3310    }
3311
3312    /// Check if we should retry given elapsed time since start.
3313    pub fn should_retry(&self, attempt: u32, elapsed: std::time::Duration) -> bool {
3314        if attempt >= self.max_attempts {
3315            return false;
3316        }
3317        elapsed.as_millis() < self.total_timeout_ms as u128
3318    }
3319}
3320
3321/// Default source-sync silence timeout (issue #59): two minutes without a
3322/// single byte of rsync output means a dead channel or wedged rsync, not a
3323/// large transfer (progress2 refreshes its line continuously while bytes
3324/// flow, and each refresh counts as forward progress).
3325fn default_source_sync_silence_timeout_secs() -> u64 {
3326    120
3327}
3328
3329/// Default remote base path for project transfers.
3330///
3331/// Rooted under `/data/tmp` (the fleet-standard managed ephemeral-build zone)
3332/// rather than `/tmp`: on the Linux workers `/tmp` is a RAM-backed tmpfs, so
3333/// dispatched build artifacts placed there consume RAM and evade the disk-GC
3334/// (sbh / datatmp-janitor) that watches `/data/tmp`. Keeping the remote base in
3335/// the managed zone lets those reapers reclaim orphaned build trees. Must agree
3336/// with the rchd cache-cleanup and reclaim `remote_base` defaults.
3337pub fn default_remote_base() -> String {
3338    "/data/tmp/rch".to_string()
3339}
3340
3341/// Validate and normalize a remote base path.
3342/// Returns an error if the path is not absolute or contains traversal sequences.
3343pub fn validate_remote_base(path: &str) -> Result<String, String> {
3344    // Expand tilde for user convenience
3345    let expanded = shellexpand::tilde(path).into_owned();
3346
3347    // Check for absolute path
3348    if !expanded.starts_with('/') {
3349        return Err(format!(
3350            "remote_base must be an absolute path, got: {}",
3351            path
3352        ));
3353    }
3354
3355    // Check for path traversal attempts
3356    if expanded.contains("..") {
3357        return Err(format!(
3358            "remote_base must not contain path traversal (..): {}",
3359            path
3360        ));
3361    }
3362
3363    // Normalize: remove trailing slashes
3364    let normalized = expanded.trim_end_matches('/').to_string();
3365
3366    // Safety checks
3367    if normalized.is_empty() || normalized == "/" {
3368        return Err("remote_base cannot be the root directory (safety restriction)".to_string());
3369    }
3370
3371    // Enforce at least 2 levels deep (e.g. /tmp/rch, /home/user/rch)
3372    // /tmp -> depth 1 (unsafe for recursive delete)
3373    // /tmp/rch -> depth 2 (safe)
3374    let components: Vec<&str> = normalized.split('/').filter(|c| !c.is_empty()).collect();
3375    if components.len() < 2 {
3376        return Err(format!(
3377            "remote_base must be at least 2 levels deep (e.g. /tmp/rch), got: {}",
3378            normalized
3379        ));
3380    }
3381
3382    Ok(normalized)
3383}
3384
3385fn default_circuit_failure_threshold() -> u32 {
3386    3
3387}
3388
3389fn default_circuit_success_threshold() -> u32 {
3390    2
3391}
3392
3393fn default_circuit_error_rate_threshold() -> f64 {
3394    0.5
3395}
3396
3397fn default_circuit_window_secs() -> u64 {
3398    60
3399}
3400
3401fn default_circuit_open_cooldown_secs() -> u64 {
3402    30
3403}
3404
3405fn default_circuit_half_open_max_probes() -> u32 {
3406    1
3407}
3408
3409fn default_circuit_command_failure_threshold() -> u32 {
3410    2
3411}
3412
3413fn default_circuit_open_cooldown_max_secs() -> u64 {
3414    // 30s base doubled four times: 30, 60, 120, 240, 480.
3415    480
3416}
3417
3418fn default_true() -> bool {
3419    true
3420}
3421
3422fn default_log_level() -> String {
3423    "info".to_string()
3424}
3425
3426pub fn default_socket_path() -> String {
3427    // Prefer XDG_RUNTIME_DIR if available (per-user runtime directory).
3428    if let Ok(runtime_dir) = std::env::var("XDG_RUNTIME_DIR")
3429        && !runtime_dir.trim().is_empty()
3430    {
3431        let path = PathBuf::from(runtime_dir).join("rch.sock");
3432        return path.to_string_lossy().to_string();
3433    }
3434
3435    // Fallback to ~/.cache/rch/rch.sock (persistent across reboots).
3436    if let Some(cache_dir) = dirs::cache_dir() {
3437        let rch_cache = cache_dir.join("rch");
3438        let _ = std::fs::create_dir_all(&rch_cache);
3439        return rch_cache.join("rch.sock").to_string_lossy().to_string();
3440    }
3441
3442    // Last resort: /tmp/rch.sock
3443    "/tmp/rch.sock".to_string()
3444}
3445
3446#[allow(dead_code)]
3447fn default_output_visibility() -> OutputVisibility {
3448    OutputVisibility::None
3449}
3450
3451fn default_confidence() -> f64 {
3452    0.85
3453}
3454
3455fn default_min_local_time() -> u64 {
3456    2000
3457}
3458
3459/// Default speedup threshold: 1.2 (require 20% speedup to offload).
3460/// This is conservative to avoid offloading builds that would be only
3461/// marginally faster remotely when accounting for transfer overhead.
3462fn default_speedup_threshold() -> f64 {
3463    1.2
3464}
3465
3466fn default_compression() -> u32 {
3467    3
3468}
3469
3470/// Default maximum file size for artifact verification (100MB).
3471fn default_verify_max_size() -> u64 {
3472    100 * 1024 * 1024 // 100MB
3473}
3474
3475/// Default minimum compression level for adaptive mode.
3476fn default_min_compression() -> u32 {
3477    1
3478}
3479
3480/// Default maximum compression level for adaptive mode.
3481fn default_max_compression() -> u32 {
3482    9
3483}
3484
3485fn default_excludes() -> Vec<String> {
3486    vec![
3487        // Rust build artifacts
3488        "target/".to_string(),
3489        "*.rlib".to_string(),
3490        "*.rmeta".to_string(),
3491        // Git metadata is large, often permission-sensitive on workers, and
3492        // a partial `.git/` tree is not reliable for build scripts anyway.
3493        ".git/".to_string(),
3494        // Node.js / Bun dependencies (massive, reinstalled on worker)
3495        "node_modules/".to_string(),
3496        // Bun cache and runtime files
3497        ".bun/".to_string(),
3498        // npm / pnpm caches
3499        ".npm/".to_string(),
3500        ".pnpm-store/".to_string(),
3501        // Common build output directories
3502        "dist/".to_string(),
3503        "build/".to_string(),
3504        // Framework-specific caches
3505        ".next/".to_string(),
3506        ".nuxt/".to_string(),
3507        ".turbo/".to_string(),
3508        ".parcel-cache/".to_string(),
3509        // Beads runtime state (local issue DB + lock/temp files); not needed remotely.
3510        ".beads/".to_string(),
3511        // Local diagnostic runs may contain host-specific symlinks and are not build inputs.
3512        ".doctor/".to_string(),
3513        // Coverage reports (generated during tests)
3514        "coverage/".to_string(),
3515        ".nyc_output/".to_string(),
3516        // Credentials and secrets (must never be transferred to workers)
3517        ".cargo/credentials".to_string(),
3518        ".cargo/credentials.toml".to_string(),
3519        ".env".to_string(),
3520        ".env.*".to_string(),
3521        "*.pem".to_string(),
3522        "*.key".to_string(),
3523        "credentials.json".to_string(),
3524        "secrets.json".to_string(),
3525        "secrets.yaml".to_string(),
3526        "secrets.yml".to_string(),
3527        "secrets.toml".to_string(),
3528        "secrets.env".to_string(),
3529        "secrets.txt".to_string(),
3530        "secrets.conf".to_string(),
3531        "secrets.cfg".to_string(),
3532        "secrets.properties".to_string(),
3533        "secrets.xml".to_string(),
3534        // The classes above were already covered. These are the ones
3535        // RABS's E027 source-capture seed set denies
3536        // (rabs-sandbox/src/source_capture.rs) that this list did not,
3537        // so the two halves of the product now agree about what a
3538        // secret is. Two components disagreeing on that is the kind of
3539        // gap nobody notices until one of them is the path that ran.
3540        //
3541        // rsync patterns without a `/` match a basename at any depth,
3542        // so these cover nested copies too — a key checked into a
3543        // fixtures directory is excluded exactly like one at the root.
3544        //
3545        // SSH private keys. The public halves (.pub) are deliberately
3546        // not excluded: they are not secret and some builds read them.
3547        "id_rsa".to_string(),
3548        "id_ed25519".to_string(),
3549        "id_ecdsa".to_string(),
3550        "id_dsa".to_string(),
3551        // Key/certificate container formats (*.pem and *.key are above).
3552        "*.p12".to_string(),
3553        "*.pfx".to_string(),
3554        "*.keystore".to_string(),
3555        "*.jks".to_string(),
3556        // Machine credentials and cloud/container config that carries
3557        // long-lived tokens.
3558        ".netrc".to_string(),
3559        "_netrc".to_string(),
3560        ".aws/credentials".to_string(),
3561        ".docker/config.json".to_string(),
3562        // Package-registry auth tokens. These sit in project roots far
3563        // more often than the key formats above and are the likeliest
3564        // real leak in a normal repo.
3565        ".npmrc".to_string(),
3566        ".pypirc".to_string(),
3567        // GnuPG private keyring.
3568        "secring.*".to_string(),
3569    ]
3570}
3571
3572// ============================================================================
3573// Build History Types
3574// ============================================================================
3575
3576/// Location where a build was executed.
3577#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
3578#[serde(rename_all = "snake_case")]
3579pub enum BuildLocation {
3580    /// Build executed locally.
3581    #[default]
3582    Local,
3583    /// Build executed on a remote worker.
3584    Remote,
3585}
3586
3587/// Worker health snapshot captured when a cancellation completes.
3588#[derive(Debug, Clone, Serialize, Deserialize)]
3589pub struct BuildCancellationWorkerHealth {
3590    /// Worker status at cancellation completion.
3591    pub status: String,
3592    /// Worker speed score at cancellation completion.
3593    pub speed_score: f64,
3594    /// Slots currently used on the worker.
3595    pub used_slots: u32,
3596    /// Slots currently available on the worker.
3597    pub available_slots: u32,
3598    /// Normalized pressure state.
3599    pub pressure_state: String,
3600    /// Stable pressure reason code.
3601    pub pressure_reason_code: String,
3602}
3603
3604/// Structured cancellation metadata attached to cancelled build records.
3605#[derive(Debug, Clone, Serialize, Deserialize)]
3606pub struct BuildCancellationMetadata {
3607    /// Stable cancellation operation identifier.
3608    pub operation_id: String,
3609    /// Cancellation origin (for example `user`, `timeout`).
3610    pub origin: String,
3611    /// Stable reason code for cancellation.
3612    pub reason_code: String,
3613    /// Ordered cancellation state path.
3614    #[serde(default)]
3615    pub decision_path: Vec<String>,
3616    /// Highest escalation stage reached (`term`, `remote_kill`, `sigkill`).
3617    pub escalation_stage: String,
3618    /// Number of escalation transitions taken.
3619    pub escalation_count: u32,
3620    /// Whether remote kill was attempted.
3621    pub remote_kill_attempted: bool,
3622    /// Whether cleanup completed successfully.
3623    pub cleanup_ok: bool,
3624    /// Whether active build history was successfully claimed/cancelled.
3625    pub history_cancelled: bool,
3626    /// Final cancellation state.
3627    pub final_state: String,
3628    /// Worker health snapshot at completion.
3629    #[serde(default, skip_serializing_if = "Option::is_none")]
3630    pub worker_health: Option<BuildCancellationWorkerHealth>,
3631}
3632
3633/// Record of a completed build.
3634#[derive(Debug, Clone, Serialize, Deserialize)]
3635pub struct BuildRecord {
3636    /// Unique build identifier.
3637    pub id: u64,
3638    /// When the build started (ISO 8601 timestamp).
3639    pub started_at: String,
3640    /// When the build completed (ISO 8601 timestamp).
3641    pub completed_at: String,
3642    /// Project identifier (usually directory name or hash).
3643    pub project_id: String,
3644    /// Worker that executed the build (None if local).
3645    pub worker_id: Option<String>,
3646    /// Full command executed.
3647    pub command: String,
3648    /// Exit code (0 = success).
3649    pub exit_code: i32,
3650    /// Duration in milliseconds.
3651    pub duration_ms: u64,
3652    /// Build location (local or remote).
3653    pub location: BuildLocation,
3654    /// Bytes transferred (if remote).
3655    pub bytes_transferred: Option<u64>,
3656    /// Optional timing breakdown for the pipeline.
3657    #[serde(default)]
3658    pub timing: Option<CommandTimingBreakdown>,
3659    /// Structured cancellation metadata for cancelled builds.
3660    #[serde(default, skip_serializing_if = "Option::is_none")]
3661    pub cancellation: Option<BuildCancellationMetadata>,
3662}
3663
3664/// Input payload for recording a completed build.
3665#[derive(Debug, Clone, Serialize, Deserialize)]
3666pub struct BuildRecordInput {
3667    /// When the build started (ISO 8601 timestamp).
3668    pub started_at: String,
3669    /// When the build completed (ISO 8601 timestamp).
3670    pub completed_at: String,
3671    /// Project identifier (usually directory name or hash).
3672    pub project_id: String,
3673    /// Worker that executed the build (None if local).
3674    pub worker_id: Option<String>,
3675    /// Full command executed.
3676    pub command: String,
3677    /// Exit code (0 = success).
3678    pub exit_code: i32,
3679    /// Duration in milliseconds.
3680    pub duration_ms: u64,
3681    /// Build location (local or remote).
3682    pub location: BuildLocation,
3683    /// Bytes transferred (if remote).
3684    pub bytes_transferred: Option<u64>,
3685    /// Optional timing breakdown for the pipeline.
3686    #[serde(default)]
3687    pub timing: Option<CommandTimingBreakdown>,
3688    /// Structured cancellation metadata for cancelled builds.
3689    #[serde(default, skip_serializing_if = "Option::is_none")]
3690    pub cancellation: Option<BuildCancellationMetadata>,
3691}
3692
3693impl BuildRecordInput {
3694    /// Convert input payload into a build record with a supplied ID.
3695    pub fn into_record(self, id: u64) -> BuildRecord {
3696        BuildRecord {
3697            id,
3698            started_at: self.started_at,
3699            completed_at: self.completed_at,
3700            project_id: self.project_id,
3701            worker_id: self.worker_id,
3702            command: self.command,
3703            exit_code: self.exit_code,
3704            duration_ms: self.duration_ms,
3705            location: self.location,
3706            bytes_transferred: self.bytes_transferred,
3707            timing: self.timing,
3708            cancellation: self.cancellation,
3709        }
3710    }
3711}
3712
3713/// Aggregate terminal-command statistics for build history.
3714///
3715/// These counts do not infer whether compilation completed before a failing
3716/// test or interrupted command. The denominator includes every stored record.
3717#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3718pub struct BuildStats {
3719    /// Total number of builds in history.
3720    pub total_builds: usize,
3721    /// Number of successfully completed commands (exit_code == 0).
3722    pub success_count: usize,
3723    /// Number of unsuccessful commands, including interruption (exit_code != 0).
3724    pub failure_count: usize,
3725    /// Number of remote builds.
3726    pub remote_count: usize,
3727    /// Number of local builds.
3728    pub local_count: usize,
3729    /// Average build duration in milliseconds.
3730    pub avg_duration_ms: u64,
3731}
3732
3733/// Saved time statistics from remote builds.
3734///
3735/// Tracks estimated time savings from offloading builds to remote workers.
3736/// Estimates are only produced from an observed local baseline; without one the
3737/// stats carry measured remote durations with `estimate_basis = "none"` and
3738/// zero estimated savings rather than a fabricated speedup factor.
3739#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3740pub struct SavedTimeStats {
3741    /// Total time spent on remote builds (milliseconds).
3742    pub total_remote_duration_ms: u64,
3743    /// Estimated total time if those builds ran locally (milliseconds).
3744    /// Zero when no local baseline exists.
3745    pub estimated_local_duration_ms: u64,
3746    /// Time saved by offloading (milliseconds).
3747    /// Computed as max(0, estimated_local - remote); zero without a baseline.
3748    pub time_saved_ms: u64,
3749    /// Number of remote builds included in calculation.
3750    pub builds_counted: usize,
3751    /// Average speedup factor (local_estimate / remote_duration).
3752    /// A value of 2.0 means remote builds are ~2x faster than local.
3753    /// Zero when the estimate basis is "none".
3754    pub avg_speedup: f64,
3755    /// Total time saved today (milliseconds).
3756    pub today_saved_ms: u64,
3757    /// Total time saved this week (milliseconds).
3758    pub week_saved_ms: u64,
3759    /// How the local-time estimate was derived: "observed_local_mean" when
3760    /// successful local builds back the baseline, "none" otherwise.
3761    #[serde(default)]
3762    pub estimate_basis: String,
3763    /// Number of successful local builds backing the baseline (0 for "none").
3764    #[serde(default)]
3765    pub local_baseline_builds: usize,
3766}
3767
3768// ============================================================================
3769// Compilation Timing and Metrics
3770// ============================================================================
3771
3772use std::collections::VecDeque;
3773use std::time::{Duration, Instant};
3774
3775/// Breakdown of timing for each compilation phase.
3776#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3777pub struct CompilationTimingBreakdown {
3778    /// Time to sync source files to worker.
3779    #[serde(with = "duration_millis")]
3780    pub rsync_up: Duration,
3781    /// Time for cargo build on worker.
3782    #[serde(with = "duration_millis")]
3783    pub remote_build: Duration,
3784    /// Time to sync artifacts back.
3785    #[serde(with = "duration_millis")]
3786    pub rsync_down: Duration,
3787    /// Total end-to-end latency.
3788    #[serde(with = "duration_millis")]
3789    pub total: Duration,
3790}
3791
3792/// Timing breakdown for a single command pipeline.
3793///
3794/// Uses optional durations so missing phases serialize as `null`.
3795#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3796pub struct CommandTimingBreakdown {
3797    /// Time spent classifying the command.
3798    #[serde(with = "option_duration_millis")]
3799    pub classify: Option<Duration>,
3800    /// Time spent selecting a worker.
3801    #[serde(with = "option_duration_millis")]
3802    pub select: Option<Duration>,
3803    /// Time spent syncing source files to the worker.
3804    #[serde(with = "option_duration_millis")]
3805    pub sync_up: Option<Duration>,
3806    /// Time spent executing the remote command.
3807    #[serde(with = "option_duration_millis")]
3808    pub exec: Option<Duration>,
3809    /// Time spent syncing artifacts back.
3810    #[serde(with = "option_duration_millis")]
3811    pub sync_down: Option<Duration>,
3812    /// Time spent on cleanup (slot release, bookkeeping).
3813    #[serde(with = "option_duration_millis")]
3814    pub cleanup: Option<Duration>,
3815    /// Total end-to-end time for the pipeline.
3816    #[serde(with = "option_duration_millis")]
3817    pub total: Option<Duration>,
3818}
3819
3820mod duration_millis {
3821    use serde::{Deserialize, Deserializer, Serialize, Serializer};
3822    use std::time::Duration;
3823
3824    pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
3825    where
3826        S: Serializer,
3827    {
3828        duration.as_millis().serialize(serializer)
3829    }
3830
3831    pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
3832    where
3833        D: Deserializer<'de>,
3834    {
3835        let millis = u64::deserialize(deserializer)?;
3836        Ok(Duration::from_millis(millis))
3837    }
3838}
3839
3840/// Comprehensive metrics for a single compilation.
3841#[derive(Debug, Clone, Serialize, Deserialize)]
3842pub struct CompilationMetrics {
3843    /// Project identifier (usually directory name or hash).
3844    pub project_id: String,
3845    /// Worker that performed the compilation.
3846    pub worker_id: String,
3847    /// When the compilation occurred.
3848    pub timestamp: chrono::DateTime<chrono::Utc>,
3849    /// Timing breakdown for each phase.
3850    pub timing: CompilationTimingBreakdown,
3851    /// Local build time for comparison (if available).
3852    #[serde(with = "option_duration_millis")]
3853    pub local_build_time: Option<Duration>,
3854    /// Speedup ratio (local_time / remote_time).
3855    pub speedup: Option<f64>,
3856    /// Number of files synced to worker.
3857    pub files_synced: u64,
3858    /// Total bytes transferred.
3859    pub bytes_transferred: u64,
3860    /// Exit code from compilation.
3861    pub exit_code: i32,
3862    /// Whether compilation succeeded.
3863    pub success: bool,
3864}
3865
3866mod option_duration_millis {
3867    use serde::{Deserialize, Deserializer, Serialize, Serializer};
3868    use std::time::Duration;
3869
3870    pub fn serialize<S>(duration: &Option<Duration>, serializer: S) -> Result<S::Ok, S::Error>
3871    where
3872        S: Serializer,
3873    {
3874        match duration {
3875            Some(d) => Some(d.as_millis() as u64).serialize(serializer),
3876            None => Option::<u64>::None.serialize(serializer),
3877        }
3878    }
3879
3880    pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Duration>, D::Error>
3881    where
3882        D: Deserializer<'de>,
3883    {
3884        let opt: Option<u64> = Option::deserialize(deserializer)?;
3885        Ok(opt.map(Duration::from_millis))
3886    }
3887}
3888
3889impl CompilationMetrics {
3890    /// Calculate speedup based on local vs remote build time.
3891    pub fn calculate_speedup(&mut self) {
3892        if let Some(local) = self.local_build_time
3893            && self.timing.total.as_millis() > 0
3894        {
3895            self.speedup = Some(local.as_secs_f64() / self.timing.total.as_secs_f64());
3896        }
3897    }
3898
3899    /// Check if remote compilation was beneficial (faster than local).
3900    pub fn is_beneficial(&self) -> bool {
3901        self.speedup.map(|s| s > 1.0).unwrap_or(false)
3902    }
3903}
3904
3905impl Default for CompilationMetrics {
3906    fn default() -> Self {
3907        Self {
3908            project_id: String::new(),
3909            worker_id: String::new(),
3910            timestamp: chrono::Utc::now(),
3911            timing: CompilationTimingBreakdown::default(),
3912            local_build_time: None,
3913            speedup: None,
3914            files_synced: 0,
3915            bytes_transferred: 0,
3916            exit_code: 0,
3917            success: true,
3918        }
3919    }
3920}
3921
3922/// Timer for tracking compilation phases.
3923///
3924/// Use this to measure the duration of each phase of remote compilation.
3925///
3926/// # Example
3927/// ```ignore
3928/// let mut timer = CompilationTimer::new("my-project", "worker-1");
3929/// // ... rsync source files ...
3930/// timer.end_rsync_up();
3931/// // ... run cargo build ...
3932/// timer.end_remote_build();
3933/// // ... rsync artifacts back ...
3934/// timer.end_rsync_down();
3935/// let metrics = timer.finish(0, 100, 1_000_000);
3936/// ```
3937#[derive(Debug)]
3938pub struct CompilationTimer {
3939    project_id: String,
3940    worker_id: String,
3941    start: Instant,
3942    phase_start: Instant,
3943    rsync_up: Option<Duration>,
3944    remote_build: Option<Duration>,
3945    rsync_down: Option<Duration>,
3946}
3947
3948impl CompilationTimer {
3949    /// Create a new timer for a compilation.
3950    pub fn new(project_id: &str, worker_id: &str) -> Self {
3951        let now = Instant::now();
3952        Self {
3953            project_id: project_id.to_string(),
3954            worker_id: worker_id.to_string(),
3955            start: now,
3956            phase_start: now,
3957            rsync_up: None,
3958            remote_build: None,
3959            rsync_down: None,
3960        }
3961    }
3962
3963    /// Mark the end of the rsync upload phase.
3964    pub fn end_rsync_up(&mut self) {
3965        self.rsync_up = Some(self.phase_start.elapsed());
3966        self.phase_start = Instant::now();
3967        tracing::info!(
3968            rsync_up_ms = %self.rsync_up.unwrap().as_millis(),
3969            "TIMING: rsync_up completed"
3970        );
3971    }
3972
3973    /// Mark the end of the remote build phase.
3974    pub fn end_remote_build(&mut self) {
3975        self.remote_build = Some(self.phase_start.elapsed());
3976        self.phase_start = Instant::now();
3977        tracing::info!(
3978            remote_build_ms = %self.remote_build.unwrap().as_millis(),
3979            "TIMING: remote_build completed"
3980        );
3981    }
3982
3983    /// Mark the end of the rsync download phase.
3984    pub fn end_rsync_down(&mut self) {
3985        self.rsync_down = Some(self.phase_start.elapsed());
3986        tracing::info!(
3987            rsync_down_ms = %self.rsync_down.unwrap().as_millis(),
3988            "TIMING: rsync_down completed"
3989        );
3990    }
3991
3992    /// Finish timing and produce metrics.
3993    pub fn finish(self, exit_code: i32, files: u64, bytes: u64) -> CompilationMetrics {
3994        let total = self.start.elapsed();
3995        tracing::info!(
3996            total_ms = %total.as_millis(),
3997            exit_code = %exit_code,
3998            "TIMING: compilation completed"
3999        );
4000
4001        CompilationMetrics {
4002            project_id: self.project_id,
4003            worker_id: self.worker_id,
4004            timestamp: chrono::Utc::now(),
4005            timing: CompilationTimingBreakdown {
4006                rsync_up: self.rsync_up.unwrap_or_default(),
4007                remote_build: self.remote_build.unwrap_or_default(),
4008                rsync_down: self.rsync_down.unwrap_or_default(),
4009                total,
4010            },
4011            local_build_time: None,
4012            speedup: None,
4013            files_synced: files,
4014            bytes_transferred: bytes,
4015            exit_code,
4016            success: exit_code == 0,
4017        }
4018    }
4019}
4020
4021/// Aggregator for compilation metrics with statistics.
4022#[derive(Debug)]
4023pub struct MetricsAggregator {
4024    history: VecDeque<CompilationMetrics>,
4025    max_history: usize,
4026}
4027
4028impl MetricsAggregator {
4029    /// Create a new aggregator with a maximum history size.
4030    pub fn new(max_history: usize) -> Self {
4031        Self {
4032            history: VecDeque::with_capacity(max_history),
4033            max_history,
4034        }
4035    }
4036
4037    /// Record a new compilation's metrics.
4038    pub fn record(&mut self, metrics: CompilationMetrics) {
4039        if self.history.len() >= self.max_history {
4040            self.history.pop_front();
4041        }
4042        self.history.push_back(metrics);
4043    }
4044
4045    /// Get the average speedup across all recorded compilations.
4046    pub fn average_speedup(&self) -> Option<f64> {
4047        let speedups: Vec<f64> = self.history.iter().filter_map(|m| m.speedup).collect();
4048
4049        if speedups.is_empty() {
4050            None
4051        } else {
4052            Some(speedups.iter().sum::<f64>() / speedups.len() as f64)
4053        }
4054    }
4055
4056    /// Get the p50 (median) total compilation time.
4057    pub fn p50_total_time(&self) -> Option<Duration> {
4058        self.percentile_total_time(0.50)
4059    }
4060
4061    /// Get the p95 total compilation time.
4062    pub fn p95_total_time(&self) -> Option<Duration> {
4063        self.percentile_total_time(0.95)
4064    }
4065
4066    /// Get the p99 total compilation time.
4067    pub fn p99_total_time(&self) -> Option<Duration> {
4068        self.percentile_total_time(0.99)
4069    }
4070
4071    /// Get a percentile of total compilation times.
4072    ///
4073    /// Uses the "inclusive" percentile method (matching numpy.percentile, Excel PERCENTILE.INC):
4074    /// index = (n - 1) * percentile, where n is the number of samples.
4075    fn percentile_total_time(&self, percentile: f64) -> Option<Duration> {
4076        let mut times: Vec<_> = self.history.iter().map(|m| m.timing.total).collect();
4077
4078        if times.is_empty() {
4079            return None;
4080        }
4081
4082        times.sort();
4083        // Use inclusive percentile: index = (n - 1) * p
4084        // For p50 with 100 items: (99 * 0.50) = 49.5 -> 49 (true median position)
4085        // For p99 with 100 items: (99 * 0.99) = 98.01 -> 98
4086        let idx = (((times.len() - 1) as f64 * percentile).round() as usize).min(times.len() - 1);
4087        Some(times[idx])
4088    }
4089
4090    /// Get the success rate as a percentage.
4091    pub fn success_rate(&self) -> f64 {
4092        if self.history.is_empty() {
4093            return 100.0;
4094        }
4095        let successes = self.history.iter().filter(|m| m.success).count();
4096        (successes as f64 / self.history.len() as f64) * 100.0
4097    }
4098
4099    /// Get the number of recorded compilations.
4100    pub fn count(&self) -> usize {
4101        self.history.len()
4102    }
4103
4104    /// Get all recorded metrics.
4105    pub fn metrics(&self) -> &VecDeque<CompilationMetrics> {
4106        &self.history
4107    }
4108
4109    /// Clear all recorded metrics.
4110    pub fn clear(&mut self) {
4111        self.history.clear();
4112    }
4113}
4114
4115impl Default for MetricsAggregator {
4116    fn default() -> Self {
4117        Self::new(1000)
4118    }
4119}
4120
4121#[cfg(test)]
4122mod tests {
4123    use super::*;
4124    use crate::test_guard;
4125
4126    #[test]
4127    fn test_declared_os_reads_reserved_tag() {
4128        assert_eq!(
4129            declared_os(&["rust".to_string(), "os:windows".to_string()]),
4130            Some("windows".to_string())
4131        );
4132        assert_eq!(
4133            declared_os(&["os:WINDOWS".to_string()]),
4134            Some("windows".to_string())
4135        );
4136    }
4137
4138    #[test]
4139    fn test_declared_os_absent_for_ordinary_tags() {
4140        // Every historical worker: no reserved tag, so no exclusivity and the
4141        // worker stays a candidate for any job.
4142        assert_eq!(declared_os(&[]), None);
4143        assert_eq!(
4144            declared_os(&["rust".to_string(), "bun".to_string(), "go".to_string()]),
4145            None
4146        );
4147        // A malformed declaration must not silently fence the worker off.
4148        assert_eq!(declared_os(&["os:".to_string()]), None);
4149        assert_eq!(declared_os(&["os:  ".to_string()]), None);
4150    }
4151
4152    #[test]
4153    fn test_os_tag_roundtrips_through_declared_os() {
4154        for os in ["windows", "Darwin", " linux "] {
4155            let tag = os_tag(os);
4156            assert_eq!(
4157                declared_os(std::slice::from_ref(&tag)).as_deref(),
4158                Some(os.trim().to_ascii_lowercase().as_str())
4159            );
4160        }
4161    }
4162
4163    /// Verdict-webhook config must NEVER carry an inline secret. Operators
4164    /// reference secrets by env-var name (`*_env`); the resolved value is
4165    /// read at dispatch time and never persisted to the cache or config TOML.
4166    /// This guards against a future careless `pub secret: String` addition.
4167    #[test]
4168    fn test_no_inline_secret_fields() {
4169        let endpoint = DoctorWebhookEndpoint {
4170            name: "ops".to_string(),
4171            url: "https://example.com/hook".to_string(),
4172            format: DoctorWebhookFormat::Pagerduty,
4173            on_transitions: vec!["any_to_failing".to_string()],
4174            retry_max: 3,
4175            retry_backoff_ms: 200,
4176            timeout_ms: 5000,
4177            enabled: true,
4178            routing_key_env: Some("RCH_PAGERDUTY_ROUTING_KEY".to_string()),
4179            bearer_token_env: Some("RCH_WEBHOOK_BEARER".to_string()),
4180            signing_secret_env: Some("RCH_WEBHOOK_HMAC".to_string()),
4181        };
4182        // Serialize to a JSON object and inspect the *keys*. The forbidden
4183        // field names must be entirely absent; only `*_env` references survive.
4184        let value = serde_json::to_value(&endpoint).unwrap();
4185        let obj = value.as_object().expect("endpoint serializes to an object");
4186        for forbidden in ["secret", "routing_key", "bearer_token", "password", "token"] {
4187            assert!(
4188                !obj.contains_key(forbidden),
4189                "endpoint config must not expose an inline `{forbidden}` field"
4190            );
4191        }
4192        assert!(obj.contains_key("routing_key_env"));
4193        assert!(obj.contains_key("bearer_token_env"));
4194        assert!(obj.contains_key("signing_secret_env"));
4195    }
4196
4197    #[test]
4198    fn test_webhook_format_toml_spelling() {
4199        // The config TOML uses snake_case spellings. TOML documents are tables
4200        // at the root, so a bare enum cannot be (de)serialized standalone —
4201        // verify the wire spelling through a `format = "..."` table field, the
4202        // shape it actually appears in within the config.
4203        #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)]
4204        struct FormatWrap {
4205            format: DoctorWebhookFormat,
4206        }
4207        let rendered = toml::to_string(&FormatWrap {
4208            format: DoctorWebhookFormat::GenericJson,
4209        })
4210        .unwrap();
4211        assert!(
4212            rendered.contains("format = \"generic_json\""),
4213            "unexpected TOML: {rendered}"
4214        );
4215        let slack: FormatWrap = toml::from_str("format = \"slack\"").unwrap();
4216        assert_eq!(slack.format, DoctorWebhookFormat::Slack);
4217        let pd: FormatWrap = toml::from_str("format = \"pagerduty\"").unwrap();
4218        assert_eq!(pd.format, DoctorWebhookFormat::Pagerduty);
4219    }
4220
4221    #[test]
4222    fn test_doctor_webhooks_defaults() {
4223        let cfg = DoctorWebhooksConfig::default();
4224        assert!(cfg.endpoints.is_empty());
4225        assert_eq!(cfg.max_inflight, 16);
4226        // RchConfig wires the section in with serde(default), so an empty
4227        // config still yields an empty (never-firing) webhook surface.
4228        let rch: RchConfig = toml::from_str("").unwrap();
4229        assert!(rch.doctor.webhooks.endpoints.is_empty());
4230    }
4231
4232    #[test]
4233    fn test_doctor_webhook_endpoint_parses_from_toml() {
4234        let toml_src = r#"
4235[doctor.webhooks]
4236max_inflight = 8
4237
4238[[doctor.webhooks.endpoints]]
4239name = "ops-slack"
4240url = "https://hooks.slack.com/services/T0/B0/x"
4241format = "slack"
4242on_transitions = ["healthy_to_degraded", "any_to_failing"]
4243retry_max = 2
4244"#;
4245        let cfg: RchConfig = toml::from_str(toml_src).unwrap();
4246        assert_eq!(cfg.doctor.webhooks.max_inflight, 8);
4247        assert_eq!(cfg.doctor.webhooks.endpoints.len(), 1);
4248        let ep = &cfg.doctor.webhooks.endpoints[0];
4249        assert_eq!(ep.name, "ops-slack");
4250        assert_eq!(ep.format, DoctorWebhookFormat::Slack);
4251        assert_eq!(ep.on_transitions.len(), 2);
4252        assert_eq!(ep.retry_max, 2);
4253        // Defaults filled in for omitted fields.
4254        assert_eq!(ep.retry_backoff_ms, 200);
4255        assert_eq!(ep.timeout_ms, 5000);
4256        assert!(ep.enabled);
4257    }
4258
4259    fn assert_golden_json(
4260        actual: serde_json::Value,
4261        expected: &serde_json::Value,
4262        fixture_path: &str,
4263    ) {
4264        if &actual == expected {
4265            return;
4266        }
4267
4268        let actual_json = serde_json::to_string_pretty(&actual).expect("actual JSON renders");
4269        let expected_json = serde_json::to_string_pretty(expected).expect("expected JSON renders");
4270        assert_eq!(
4271            &actual,
4272            expected,
4273            "golden mismatch in {fixture_path}\n\
4274             expected_blake3={}\n\
4275             actual_blake3={}\n\
4276             bless path: review the semantic diff, update the fixture expected_* field, \
4277             and record both hashes in the Beads closeout.\n\
4278             expected:\n{expected_json}\n\
4279             actual:\n{actual_json}",
4280            blake3::hash(expected_json.as_bytes()),
4281            blake3::hash(actual_json.as_bytes())
4282        );
4283    }
4284
4285    #[test]
4286    fn test_circuit_state_default() {
4287        let _guard = test_guard!();
4288        assert_eq!(CircuitState::default(), CircuitState::Closed);
4289    }
4290
4291    #[test]
4292    fn test_circuit_config_defaults() {
4293        let _guard = test_guard!();
4294        let config = CircuitBreakerConfig::default();
4295        assert_eq!(config.failure_threshold, 3);
4296        assert_eq!(config.success_threshold, 2);
4297        assert_eq!(config.error_rate_threshold, 0.5);
4298        assert_eq!(config.window_secs, 60);
4299        assert_eq!(config.open_cooldown_secs, 30);
4300        assert_eq!(config.half_open_max_probes, 1);
4301    }
4302
4303    #[test]
4304    fn test_rch_config_has_circuit_defaults() {
4305        let _guard = test_guard!();
4306        let config = RchConfig::default();
4307        assert_eq!(config.circuit.failure_threshold, 3);
4308    }
4309
4310    #[test]
4311    fn test_circuit_config_serde_roundtrip() {
4312        let _guard = test_guard!();
4313        let config = CircuitBreakerConfig {
4314            failure_threshold: 5,
4315            success_threshold: 3,
4316            error_rate_threshold: 0.75,
4317            window_secs: 120,
4318            open_cooldown_secs: 45,
4319            half_open_max_probes: 2,
4320            command_failure_threshold: 4,
4321            open_cooldown_max_secs: 900,
4322        };
4323        let json = serde_json::to_string(&config).unwrap();
4324        let parsed: CircuitBreakerConfig = serde_json::from_str(&json).unwrap();
4325        assert_eq!(parsed.command_failure_threshold, 4);
4326        assert_eq!(parsed.open_cooldown_max_secs, 900);
4327        assert_eq!(parsed.failure_threshold, 5);
4328        assert_eq!(parsed.success_threshold, 3);
4329        assert_eq!(parsed.error_rate_threshold, 0.75);
4330        assert_eq!(parsed.window_secs, 120);
4331        assert_eq!(parsed.open_cooldown_secs, 45);
4332        assert_eq!(parsed.half_open_max_probes, 2);
4333    }
4334
4335    // ========================================================================
4336    // WorkerCapabilities Tests
4337    // ========================================================================
4338
4339    #[test]
4340    fn test_worker_capabilities_default() {
4341        let _guard = test_guard!();
4342        let caps = WorkerCapabilities::default();
4343        assert!(caps.rustc_version.is_none());
4344        assert!(caps.rustup_toolchains.is_empty());
4345        assert!(caps.rustup_components.is_empty());
4346        assert!(caps.bun_version.is_none());
4347        assert!(caps.node_version.is_none());
4348        assert!(caps.npm_version.is_none());
4349        assert!(caps.projects_root_ok.is_none());
4350        assert!(caps.projects_root_issue.is_none());
4351        assert!(caps.projects_root_checked_at_unix_ms.is_none());
4352    }
4353
4354    #[test]
4355    fn test_worker_capabilities_new() {
4356        let _guard = test_guard!();
4357        let caps = WorkerCapabilities::new();
4358        assert!(caps.rustc_version.is_none());
4359        assert!(!caps.has_rust());
4360        assert!(!caps.has_bun());
4361        assert!(!caps.has_node());
4362        assert!(caps.is_topology_healthy().is_none());
4363    }
4364
4365    #[test]
4366    fn test_worker_capabilities_has_rust() {
4367        let _guard = test_guard!();
4368        let mut caps = WorkerCapabilities::new();
4369        assert!(!caps.has_rust());
4370
4371        caps.rustc_version = Some("rustc 1.76.0 (07dca489a 2024-02-04)".to_string());
4372        assert!(caps.has_rust());
4373    }
4374
4375    #[test]
4376    fn test_worker_capabilities_components_are_toolchain_scoped_and_exact() {
4377        let _guard = test_guard!();
4378        let caps = WorkerCapabilities {
4379            rustup_components: vec![
4380                "stable-x86_64-unknown-linux-gnu:clippy".to_string(),
4381                "nightly-2026-07-05-x86_64-unknown-linux-gnu:rustfmt".to_string(),
4382                "beta-x86_64-unknown-linux-gnu:clippy-preview".to_string(),
4383            ],
4384            ..Default::default()
4385        };
4386
4387        assert!(caps.has_rustup_component("stable", "clippy"));
4388        assert!(caps.has_rustup_component("nightly-2026-07-05", "rustfmt"));
4389        assert!(!caps.has_rustup_component("nightly-2026-07-05", "clippy"));
4390        assert!(!caps.has_rustup_component("beta", "clippy"));
4391        assert!(!caps.has_rustup_component("nightly-2026-07-0", "rustfmt"));
4392    }
4393
4394    #[test]
4395    fn test_worker_capabilities_has_bun() {
4396        let _guard = test_guard!();
4397        let mut caps = WorkerCapabilities::new();
4398        assert!(!caps.has_bun());
4399
4400        caps.bun_version = Some("1.0.25".to_string());
4401        assert!(caps.has_bun());
4402    }
4403
4404    #[test]
4405    fn test_worker_capabilities_has_nix() {
4406        let _guard = test_guard!();
4407        let mut caps = WorkerCapabilities::new();
4408        assert!(!caps.has_nix());
4409
4410        caps.nix_version = Some("nix (Nix) 2.24.9".to_string());
4411        assert!(caps.has_nix());
4412    }
4413
4414    #[test]
4415    fn test_worker_capabilities_has_zig_requires_both_halves() {
4416        let _guard = test_guard!();
4417        let mut caps = WorkerCapabilities::new();
4418        assert!(!caps.has_zig());
4419
4420        // `zig` alone cannot run `cargo zigbuild` — cargo would report
4421        // `no such command: 'zigbuild'`.
4422        caps.zig_version = Some("0.14.1".to_string());
4423        assert!(!caps.has_zig());
4424
4425        // cargo-zigbuild alone cannot link — it shells out to `zig cc`.
4426        caps.zig_version = None;
4427        caps.cargo_zigbuild_version = Some("cargo-zigbuild 0.23.0".to_string());
4428        assert!(!caps.has_zig());
4429
4430        caps.zig_version = Some("0.14.1".to_string());
4431        assert!(caps.has_zig());
4432    }
4433
4434    #[test]
4435    fn test_nix_build_uses_generous_timeout() {
4436        let _guard = test_guard!();
4437        let cfg = CompilationConfig::default();
4438        // Nix builds get the long test-timeout bucket, not the short build timeout.
4439        assert_eq!(
4440            cfg.timeout_for_kind(Some(CompilationKind::NixBuild)),
4441            std::time::Duration::from_secs(cfg.test_timeout_sec)
4442        );
4443    }
4444
4445    #[test]
4446    fn test_job_uses_build_timeout_bucket() {
4447        let _guard = test_guard!();
4448        // Jobs (explicit `rch exec --job`) reuse the standard build-timeout
4449        // bucket, so `RCH_BUILD_TIMEOUT_SEC` governs them without new config.
4450        let cfg = CompilationConfig::default();
4451        assert_eq!(
4452            cfg.timeout_for_kind(Some(CompilationKind::Job)),
4453            std::time::Duration::from_secs(cfg.build_timeout_sec)
4454        );
4455    }
4456
4457    #[test]
4458    fn test_layer0_disabled_by_default_and_env_gated() {
4459        let _guard = test_guard!();
4460        let cfg = Layer0Config::default();
4461        assert!(!cfg.enabled);
4462        // Master switch off ⇒ nothing injects even with knobs on.
4463        let knobs_on = Layer0Config {
4464            enabled: false,
4465            release_lto_thin: true,
4466            release_codegen_units_1: true,
4467        };
4468        assert!(cfg.active_env().is_empty());
4469        assert!(knobs_on.active_env().is_empty());
4470    }
4471
4472    #[test]
4473    fn test_layer0_knobs_emit_stable_env_pairs() {
4474        let _guard = test_guard!();
4475        let lto_only = Layer0Config {
4476            enabled: true,
4477            release_lto_thin: true,
4478            release_codegen_units_1: false,
4479        };
4480        assert_eq!(
4481            lto_only.active_env(),
4482            vec![("CARGO_PROFILE_RELEASE_LTO", "thin")]
4483        );
4484        let both = Layer0Config {
4485            enabled: true,
4486            release_lto_thin: true,
4487            release_codegen_units_1: true,
4488        };
4489        assert_eq!(
4490            both.active_env(),
4491            vec![
4492                ("CARGO_PROFILE_RELEASE_LTO", "thin"),
4493                ("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"),
4494            ],
4495            "emission order must stay stable for deterministic baselines"
4496        );
4497    }
4498
4499    #[test]
4500    fn test_layer0_pack_version_is_recorded() {
4501        let _guard = test_guard!();
4502        // Baselines record the pack version; it must exist and be non-empty.
4503        assert!(!LAYER0_PACK_VERSION.is_empty());
4504    }
4505    #[test]
4506    fn test_default_allowlist_includes_nix() {
4507        let _guard = test_guard!();
4508        let cfg = ExecutionConfig::default();
4509        assert!(cfg.is_allowed("nix"));
4510    }
4511
4512    #[test]
4513    fn test_worker_capabilities_has_node() {
4514        let _guard = test_guard!();
4515        let mut caps = WorkerCapabilities::new();
4516        assert!(!caps.has_node());
4517
4518        caps.node_version = Some("v20.11.0".to_string());
4519        assert!(caps.has_node());
4520    }
4521
4522    #[test]
4523    fn test_worker_capabilities_multiple_runtimes() {
4524        let _guard = test_guard!();
4525        let caps = WorkerCapabilities {
4526            rustc_version: Some("rustc 1.76.0".to_string()),
4527            bun_version: Some("1.0.25".to_string()),
4528            node_version: Some("v20.11.0".to_string()),
4529            npm_version: Some("10.2.4".to_string()),
4530            ..Default::default()
4531        };
4532
4533        assert!(caps.has_rust());
4534        assert!(caps.has_bun());
4535        assert!(caps.has_node());
4536    }
4537
4538    #[test]
4539    fn test_worker_capabilities_serialization_empty() {
4540        let _guard = test_guard!();
4541        let caps = WorkerCapabilities::new();
4542        let json = serde_json::to_string(&caps).unwrap();
4543        // Empty capabilities should serialize to {}
4544        assert_eq!(json, "{}");
4545
4546        // And deserialize back correctly
4547        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4548        assert!(!parsed.has_rust());
4549        assert!(!parsed.has_bun());
4550        assert!(!parsed.has_node());
4551    }
4552
4553    #[test]
4554    fn test_worker_capabilities_serialization_with_versions() {
4555        let _guard = test_guard!();
4556        let caps = WorkerCapabilities {
4557            rustc_version: Some("rustc 1.76.0-nightly".to_string()),
4558            bun_version: Some("1.0.25".to_string()),
4559            node_version: None,
4560            npm_version: None,
4561            ..Default::default()
4562        };
4563
4564        let json = serde_json::to_string(&caps).unwrap();
4565        assert!(json.contains("rustc_version"));
4566        assert!(json.contains("rustc 1.76.0-nightly"));
4567        assert!(json.contains("bun_version"));
4568        assert!(json.contains("1.0.25"));
4569        // None fields should be skipped
4570        assert!(!json.contains("node_version"));
4571        assert!(!json.contains("npm_version"));
4572
4573        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4574        assert!(parsed.has_rust());
4575        assert!(parsed.has_bun());
4576        assert!(!parsed.has_node());
4577    }
4578
4579    #[test]
4580    fn test_worker_capabilities_deserialization_partial() {
4581        let _guard = test_guard!();
4582        // Deserialize JSON with only some fields
4583        let json = r#"{"bun_version": "1.0.0"}"#;
4584        let caps: WorkerCapabilities = serde_json::from_str(json).unwrap();
4585
4586        assert!(!caps.has_rust());
4587        assert!(caps.has_bun());
4588        assert!(!caps.has_node());
4589        assert_eq!(caps.bun_version, Some("1.0.0".to_string()));
4590    }
4591
4592    #[test]
4593    fn test_worker_capabilities_clone() {
4594        let _guard = test_guard!();
4595        let caps = WorkerCapabilities {
4596            rustc_version: Some("1.76.0".to_string()),
4597            bun_version: None,
4598            node_version: Some("v20".to_string()),
4599            npm_version: None,
4600            projects_root_ok: Some(false),
4601            projects_root_issue: Some("alias_missing".to_string()),
4602            projects_root_checked_at_unix_ms: Some(123),
4603            ..Default::default()
4604        };
4605
4606        let cloned = caps.clone();
4607        assert_eq!(cloned.rustc_version, caps.rustc_version);
4608        assert_eq!(cloned.bun_version, caps.bun_version);
4609        assert_eq!(cloned.node_version, caps.node_version);
4610        assert_eq!(cloned.npm_version, caps.npm_version);
4611        assert_eq!(cloned.projects_root_ok, caps.projects_root_ok);
4612        assert_eq!(cloned.projects_root_issue, caps.projects_root_issue);
4613        assert_eq!(
4614            cloned.projects_root_checked_at_unix_ms,
4615            caps.projects_root_checked_at_unix_ms
4616        );
4617    }
4618
4619    #[test]
4620    fn test_worker_capabilities_topology_health_helper() {
4621        let _guard = test_guard!();
4622        let mut caps = WorkerCapabilities::new();
4623        assert_eq!(caps.is_topology_healthy(), None);
4624
4625        caps.projects_root_ok = Some(true);
4626        assert_eq!(caps.is_topology_healthy(), Some(true));
4627
4628        caps.projects_root_ok = Some(false);
4629        assert_eq!(caps.is_topology_healthy(), Some(false));
4630    }
4631
4632    #[test]
4633    fn test_selection_reason_serialization() {
4634        let _guard = test_guard!();
4635        // Test that all variants serialize to snake_case
4636        assert_eq!(
4637            serde_json::to_string(&SelectionReason::Success).unwrap(),
4638            "\"success\""
4639        );
4640        assert_eq!(
4641            serde_json::to_string(&SelectionReason::NoWorkersConfigured).unwrap(),
4642            "\"no_workers_configured\""
4643        );
4644        assert_eq!(
4645            serde_json::to_string(&SelectionReason::AllWorkersUnreachable).unwrap(),
4646            "\"all_workers_unreachable\""
4647        );
4648        assert_eq!(
4649            serde_json::to_string(&SelectionReason::AllCircuitsOpen).unwrap(),
4650            "\"all_circuits_open\""
4651        );
4652        assert_eq!(
4653            serde_json::to_string(&SelectionReason::AllWorkersBusy).unwrap(),
4654            "\"all_workers_busy\""
4655        );
4656        assert_eq!(
4657            serde_json::to_string(&SelectionReason::NoWorkersPassedHealth).unwrap(),
4658            "\"no_workers_passed_health\""
4659        );
4660        assert_eq!(
4661            serde_json::to_string(&SelectionReason::AllWorkersFailedPreflight).unwrap(),
4662            "\"all_workers_failed_preflight\""
4663        );
4664        assert_eq!(
4665            serde_json::to_string(&SelectionReason::AllWorkersFailedConvergence).unwrap(),
4666            "\"all_workers_failed_convergence\""
4667        );
4668        assert_eq!(
4669            serde_json::to_string(&SelectionReason::NoAdmissibleWorkers(
4670                "critical_pressure=1,insufficient_slots=1".to_string()
4671            ))
4672            .unwrap(),
4673            "{\"no_admissible_workers\":\"critical_pressure=1,insufficient_slots=1\"}"
4674        );
4675        assert_eq!(
4676            serde_json::to_string(&SelectionReason::NoMatchingWorkers).unwrap(),
4677            "\"no_matching_workers\""
4678        );
4679    }
4680
4681    #[test]
4682    fn test_selection_reason_with_error() {
4683        let _guard = test_guard!();
4684        let reason = SelectionReason::SelectionError("test error".to_string());
4685        let json = serde_json::to_string(&reason).unwrap();
4686        assert!(json.contains("selection_error"));
4687        assert!(json.contains("test error"));
4688    }
4689
4690    #[test]
4691    fn test_selection_reason_deserialization() {
4692        let _guard = test_guard!();
4693        assert_eq!(
4694            serde_json::from_str::<SelectionReason>("\"success\"").unwrap(),
4695            SelectionReason::Success
4696        );
4697        assert_eq!(
4698            serde_json::from_str::<SelectionReason>("\"all_workers_busy\"").unwrap(),
4699            SelectionReason::AllWorkersBusy
4700        );
4701        assert_eq!(
4702            serde_json::from_str::<SelectionReason>("\"no_workers_passed_health\"").unwrap(),
4703            SelectionReason::NoWorkersPassedHealth
4704        );
4705        assert_eq!(
4706            serde_json::from_str::<SelectionReason>("\"all_workers_failed_preflight\"").unwrap(),
4707            SelectionReason::AllWorkersFailedPreflight
4708        );
4709        assert_eq!(
4710            serde_json::from_str::<SelectionReason>("\"all_workers_failed_convergence\"").unwrap(),
4711            SelectionReason::AllWorkersFailedConvergence
4712        );
4713        assert_eq!(
4714            serde_json::from_str::<SelectionReason>(
4715                "{\"no_admissible_workers\":\"critical_pressure=1\"}"
4716            )
4717            .unwrap(),
4718            SelectionReason::NoAdmissibleWorkers("critical_pressure=1".to_string())
4719        );
4720    }
4721
4722    #[test]
4723    fn test_selection_reason_display() {
4724        let _guard = test_guard!();
4725        assert_eq!(
4726            SelectionReason::Success.to_string(),
4727            "worker assigned successfully"
4728        );
4729        assert_eq!(
4730            SelectionReason::NoWorkersConfigured.to_string(),
4731            "no workers configured"
4732        );
4733        assert_eq!(
4734            SelectionReason::AllWorkersUnreachable.to_string(),
4735            "all workers unreachable"
4736        );
4737        assert_eq!(
4738            SelectionReason::AllWorkersBusy.to_string(),
4739            "all workers at capacity"
4740        );
4741        assert_eq!(
4742            SelectionReason::NoWorkersPassedHealth.to_string(),
4743            "no workers passed health thresholds"
4744        );
4745        assert_eq!(
4746            SelectionReason::AllWorkersFailedPreflight.to_string(),
4747            "all workers failed preflight checks"
4748        );
4749        assert_eq!(
4750            SelectionReason::AllWorkersFailedConvergence.to_string(),
4751            "all workers failed repo convergence checks"
4752        );
4753        assert_eq!(
4754            SelectionReason::NoAdmissibleWorkers(
4755                "critical_pressure=1,insufficient_slots=1".to_string()
4756            )
4757            .to_string(),
4758            "no admissible workers: critical_pressure=1,insufficient_slots=1"
4759        );
4760        assert_eq!(
4761            SelectionReason::SelectionError("oops".to_string()).to_string(),
4762            "selection error: oops"
4763        );
4764    }
4765
4766    #[test]
4767    fn test_selection_response_with_worker() {
4768        let _guard = test_guard!();
4769        let response = SelectionResponse {
4770            worker: Some(SelectedWorker {
4771                id: WorkerId::new("test"),
4772                host: "localhost".to_string(),
4773                user: "user".to_string(),
4774                identity_file: "~/.ssh/id_rsa".to_string(),
4775                slots_available: 8,
4776                speed_score: 75.0,
4777                declared_os: None,
4778            }),
4779            reason: SelectionReason::Success,
4780            build_id: None,
4781            diagnostics: None,
4782        };
4783
4784        let json = serde_json::to_string(&response).unwrap();
4785        assert!(json.contains("\"reason\":\"success\""));
4786        assert!(json.contains("\"id\":\"test\""));
4787    }
4788
4789    #[test]
4790    fn test_selection_response_without_worker() {
4791        let _guard = test_guard!();
4792        let response = SelectionResponse {
4793            worker: None,
4794            reason: SelectionReason::AllWorkersBusy,
4795            build_id: None,
4796            diagnostics: None,
4797        };
4798
4799        let json = serde_json::to_string(&response).unwrap();
4800        assert!(json.contains("\"worker\":null"));
4801        assert!(json.contains("\"reason\":\"all_workers_busy\""));
4802    }
4803
4804    #[test]
4805    fn golden_selection_response_with_no_workers_passed_health_diagnostics() {
4806        let _guard = test_guard!();
4807        const FIXTURE_PATH: &str =
4808            "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json";
4809        let fixture: serde_json::Value = serde_json::from_str(include_str!(
4810            "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json"
4811        ))
4812        .expect("selection diagnostics golden fixture parses");
4813        assert_eq!(
4814            fixture["schema_version"],
4815            "rch.golden.selection_diagnostics.v1"
4816        );
4817
4818        let response = SelectionResponse {
4819            worker: None,
4820            reason: SelectionReason::NoWorkersPassedHealth,
4821            build_id: None,
4822            diagnostics: Some(SelectionDiagnostics {
4823                required_runtime: RequiredRuntime::Rust,
4824                estimated_cores: 2,
4825                min_success_rate: 0.8,
4826                fallback_min_success_rate: 0.5,
4827                active_project_exclusion_count: 0,
4828                workers: vec![WorkerSelectionDiagnostic {
4829                    worker_id: WorkerId::new("fixture-worker"),
4830                    status: "healthy".to_string(),
4831                    circuit_state: "closed".to_string(),
4832                    pressure_state: "healthy".to_string(),
4833                    pressure_reason_code: "pressure.ok".to_string(),
4834                    success_rate: Some(0.2),
4835                    min_success_rate: 0.8,
4836                    fallback_min_success_rate: 0.5,
4837                    required_runtime: RequiredRuntime::Rust,
4838                    runtime_available: true,
4839                    available_slots: 8,
4840                    total_slots: 8,
4841                    estimated_cores: 2,
4842                    active_project_excluded: false,
4843                    final_decision: WorkerSelectionDiagnosticDecision::Deny,
4844                    final_reason: "success_rate 0.20 < fallback 0.50".to_string(),
4845                    reason_codes: vec![
4846                        "health.below_min_success_rate".to_string(),
4847                        "health.below_fallback_min_success_rate".to_string(),
4848                    ],
4849                }],
4850            }),
4851        };
4852
4853        assert_golden_json(
4854            serde_json::to_value(&response).expect("response serializes"),
4855            &fixture["expected_response"],
4856            FIXTURE_PATH,
4857        );
4858    }
4859
4860    #[test]
4861    fn test_selection_response_roundtrip() {
4862        let _guard = test_guard!();
4863        let original = SelectionResponse {
4864            worker: Some(SelectedWorker {
4865                id: WorkerId::new("worker-1"),
4866                host: "192.168.1.100".to_string(),
4867                user: "ubuntu".to_string(),
4868                identity_file: "/path/to/key".to_string(),
4869                slots_available: 16,
4870                speed_score: 90.5,
4871                declared_os: None,
4872            }),
4873            reason: SelectionReason::Success,
4874            build_id: None,
4875            diagnostics: None,
4876        };
4877
4878        let json = serde_json::to_string(&original).unwrap();
4879        let parsed: SelectionResponse = serde_json::from_str(&json).unwrap();
4880
4881        assert!(parsed.worker.is_some());
4882        let worker = parsed.worker.unwrap();
4883        assert_eq!(worker.id.as_str(), "worker-1");
4884        assert_eq!(worker.host, "192.168.1.100");
4885        assert_eq!(worker.slots_available, 16);
4886        assert_eq!(parsed.reason, SelectionReason::Success);
4887    }
4888
4889    #[test]
4890    fn test_build_heartbeat_phase_roundtrip() {
4891        let _guard = test_guard!();
4892        let json = serde_json::to_string(&BuildHeartbeatPhase::SyncUp).unwrap();
4893        assert_eq!(json, "\"sync_up\"");
4894        let parsed: BuildHeartbeatPhase = serde_json::from_str(&json).unwrap();
4895        assert_eq!(parsed, BuildHeartbeatPhase::SyncUp);
4896    }
4897
4898    #[test]
4899    fn test_build_heartbeat_request_roundtrip() {
4900        let _guard = test_guard!();
4901        let request = BuildHeartbeatRequest {
4902            build_id: 42,
4903            worker_id: WorkerId::new("worker-a"),
4904            hook_pid: Some(12345),
4905            local_wrapper_id: Some("wrapper-abc123".to_string()),
4906            remote_pgid_file: Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string()),
4907            phase: BuildHeartbeatPhase::Execute,
4908            detail: Some("Compiling crates".to_string()),
4909            progress_counter: Some(7),
4910            progress_percent: Some(42.5),
4911        };
4912
4913        let json = serde_json::to_string(&request).unwrap();
4914        let parsed: BuildHeartbeatRequest = serde_json::from_str(&json).unwrap();
4915        assert_eq!(parsed.build_id, 42);
4916        assert_eq!(parsed.worker_id.as_str(), "worker-a");
4917        assert_eq!(
4918            parsed.remote_pgid_file,
4919            Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string())
4920        );
4921        assert_eq!(parsed.phase, BuildHeartbeatPhase::Execute);
4922        assert_eq!(parsed.progress_counter, Some(7));
4923        assert_eq!(parsed.progress_percent, Some(42.5));
4924    }
4925
4926    // CircuitStats tests
4927
4928    #[test]
4929    fn test_circuit_stats_new() {
4930        let _guard = test_guard!();
4931        let stats = CircuitStats::new();
4932        assert_eq!(stats.state(), CircuitState::Closed);
4933        assert_eq!(stats.consecutive_failures(), 0);
4934        assert_eq!(stats.error_rate(), 0.0);
4935        assert!(stats.opened_at().is_none());
4936    }
4937
4938    #[test]
4939    fn test_circuit_stats_record_success() {
4940        let _guard = test_guard!();
4941        let mut stats = CircuitStats::new();
4942        stats.record_success();
4943        stats.record_success();
4944        assert_eq!(stats.error_rate(), 0.0);
4945        assert_eq!(stats.consecutive_failures(), 0);
4946    }
4947
4948    #[test]
4949    fn test_circuit_stats_record_failure() {
4950        let _guard = test_guard!();
4951        let mut stats = CircuitStats::new();
4952        stats.record_failure();
4953        stats.record_failure();
4954        assert_eq!(stats.consecutive_failures(), 2);
4955        assert_eq!(stats.error_rate(), 1.0); // 2 failures, 0 successes
4956    }
4957
4958    #[test]
4959    fn test_circuit_stats_error_rate() {
4960        let _guard = test_guard!();
4961        let mut stats = CircuitStats::new();
4962        stats.record_success();
4963        stats.record_success();
4964        stats.record_failure();
4965        // 1 failure / 3 total = 0.333...
4966        assert!((stats.error_rate() - 0.333).abs() < 0.01);
4967    }
4968
4969    #[test]
4970    fn test_circuit_stats_should_open_consecutive_failures() {
4971        let _guard = test_guard!();
4972        let mut stats = CircuitStats::new();
4973        let config = CircuitBreakerConfig::default(); // threshold = 3
4974
4975        stats.record_failure();
4976        assert!(!stats.should_open(&config));
4977
4978        stats.record_failure();
4979        assert!(!stats.should_open(&config));
4980
4981        stats.record_failure();
4982        assert!(stats.should_open(&config)); // 3 consecutive failures
4983    }
4984
4985    #[test]
4986    fn test_circuit_stats_should_open_error_rate() {
4987        let _guard = test_guard!();
4988        let mut stats = CircuitStats::new();
4989        let config = CircuitBreakerConfig {
4990            error_rate_threshold: 0.5,
4991            ..Default::default()
4992        };
4993
4994        // Need at least 5 samples
4995        stats.record_success();
4996        stats.record_success();
4997        stats.record_failure();
4998        stats.record_failure();
4999        assert!(!stats.should_open(&config)); // Only 4 samples
5000
5001        stats.record_failure(); // 5 samples: 2 success, 3 failures = 60% error rate
5002        assert!(stats.should_open(&config));
5003    }
5004
5005    #[test]
5006    fn test_circuit_close_clears_recent_results_no_flap() {
5007        let _guard = test_guard!();
5008        // Regression: error_rate() is computed over recent_results, so close()
5009        // must clear that history or a recovered circuit immediately re-opens
5010        // on stale pre-failure samples (flapping).
5011        let mut stats = CircuitStats::new();
5012        let config = CircuitBreakerConfig {
5013            error_rate_threshold: 0.5,
5014            ..Default::default()
5015        };
5016        for _ in 0..6 {
5017            stats.record_failure();
5018        }
5019        assert!(stats.should_open(&config));
5020        stats.open();
5021        // Recover via half-open probes, then close.
5022        stats.half_open();
5023        stats.record_success();
5024        stats.record_success();
5025        stats.close();
5026        // Error rate must be reset and the circuit must NOT immediately re-open.
5027        assert_eq!(stats.error_rate(), 0.0);
5028        assert!(!stats.should_open(&config));
5029    }
5030
5031    #[test]
5032    fn test_apply_health_outcome_opens_on_failure_threshold() {
5033        let _guard = test_guard!();
5034        // failure_threshold=3: three unhealthy outcomes open the circuit.
5035        let config = CircuitBreakerConfig {
5036            failure_threshold: 3,
5037            open_cooldown_secs: 300,
5038            ..Default::default()
5039        };
5040        let mut stats = CircuitStats::new();
5041        assert_eq!(
5042            stats.apply_health_outcome(false, false, &config),
5043            CircuitState::Closed
5044        );
5045        assert_eq!(
5046            stats.apply_health_outcome(false, false, &config),
5047            CircuitState::Closed
5048        );
5049        assert_eq!(
5050            stats.apply_health_outcome(false, false, &config),
5051            CircuitState::Open
5052        );
5053    }
5054
5055    #[test]
5056    fn test_apply_health_outcome_half_open_two_probe_recovery() {
5057        let _guard = test_guard!();
5058        // success_threshold=2: recovery from Open needs exactly two clean probes.
5059        // open_cooldown_secs=0 makes should_half_open() true immediately, so the
5060        // engine's trailing nudge leaves an opened circuit already in HalfOpen —
5061        // exactly the state a real daemon would be in on the next poll once the
5062        // cooldown had elapsed. From HalfOpen the engine credits successes.
5063        let config = CircuitBreakerConfig {
5064            failure_threshold: 3,
5065            success_threshold: 2,
5066            open_cooldown_secs: 0,
5067            ..Default::default()
5068        };
5069        let mut stats = CircuitStats::new();
5070        for _ in 0..3 {
5071            stats.apply_health_outcome(false, false, &config);
5072        }
5073        // With a 0s cooldown the engine nudges Open -> HalfOpen immediately.
5074        assert_eq!(stats.state(), CircuitState::HalfOpen);
5075
5076        // First healthy probe in half-open: one success credited, not yet closed.
5077        let after_first = stats.apply_health_outcome(true, false, &config);
5078        assert_eq!(after_first, CircuitState::HalfOpen);
5079
5080        // Second healthy probe: threshold met -> Closed.
5081        let after_second = stats.apply_health_outcome(true, false, &config);
5082        assert_eq!(after_second, CircuitState::Closed);
5083    }
5084
5085    #[test]
5086    fn test_apply_health_outcome_transient_half_open_does_not_reopen() {
5087        let _guard = test_guard!();
5088        // A transient failure while half-open must NOT reset the recovery streak
5089        // — a network blip can't undo an in-progress recovery.
5090        let config = CircuitBreakerConfig {
5091            failure_threshold: 3,
5092            success_threshold: 2,
5093            open_cooldown_secs: 0,
5094            ..Default::default()
5095        };
5096        let mut stats = CircuitStats::new();
5097        for _ in 0..3 {
5098            stats.apply_health_outcome(false, false, &config);
5099        }
5100        // 0s cooldown -> already nudged to HalfOpen.
5101        assert_eq!(stats.state(), CircuitState::HalfOpen);
5102
5103        // One clean probe (1 of 2 needed).
5104        assert_eq!(
5105            stats.apply_health_outcome(true, false, &config),
5106            CircuitState::HalfOpen
5107        );
5108
5109        // Transient failure in half-open: stays HalfOpen (does NOT reopen the
5110        // circuit or restart the cooldown). The key contrast with a real failure
5111        // is that the circuit does not drop back to Open — probing continues in
5112        // place, so recovery resumes immediately on the next clean probes rather
5113        // than waiting out a fresh cooldown.
5114        let after_transient = stats.apply_health_outcome(false, true, &config);
5115        assert_eq!(after_transient, CircuitState::HalfOpen);
5116
5117        // Recovery still completes from half-open with the required clean probes.
5118        assert_eq!(
5119            stats.apply_health_outcome(true, false, &config),
5120            CircuitState::HalfOpen
5121        );
5122        assert_eq!(
5123            stats.apply_health_outcome(true, false, &config),
5124            CircuitState::Closed
5125        );
5126    }
5127
5128    #[test]
5129    fn test_apply_health_outcome_real_vs_transient_half_open_failure() {
5130        let _guard = test_guard!();
5131        // With a positive cooldown the difference is directly observable: a REAL
5132        // half-open failure reopens the circuit (state -> Open, must wait out the
5133        // cooldown again), whereas a TRANSIENT half-open failure keeps it in
5134        // HalfOpen so recovery continues without a fresh cooldown.
5135        //
5136        // Manually drive to HalfOpen (open() + half_open()) so the positive
5137        // cooldown doesn't force a real wallclock wait to reach half-open.
5138        let config = CircuitBreakerConfig {
5139            failure_threshold: 3,
5140            success_threshold: 2,
5141            open_cooldown_secs: 300,
5142            ..Default::default()
5143        };
5144
5145        // Real failure path.
5146        let mut real = CircuitStats::new();
5147        real.open();
5148        real.half_open();
5149        assert_eq!(real.state(), CircuitState::HalfOpen);
5150        let after_real = real.apply_health_outcome(false, false, &config);
5151        assert_eq!(
5152            after_real,
5153            CircuitState::Open,
5154            "a real half-open failure must reopen the circuit"
5155        );
5156
5157        // Transient failure path (same starting state).
5158        let mut transient = CircuitStats::new();
5159        transient.open();
5160        transient.half_open();
5161        assert_eq!(transient.state(), CircuitState::HalfOpen);
5162        let after_transient = transient.apply_health_outcome(false, true, &config);
5163        assert_eq!(
5164            after_transient,
5165            CircuitState::HalfOpen,
5166            "a transient half-open failure must NOT reopen the circuit"
5167        );
5168    }
5169
5170    /// Issue #48 regression: a worker whose liveness probe (`echo` over a warm
5171    /// session) keeps succeeding while every real command times out must NOT
5172    /// be re-admitted. Liveness successes alone cannot close a half-open
5173    /// circuit while command-class evidence is outstanding.
5174    #[test]
5175    fn test_command_failure_blocks_half_open_close_on_liveness_successes() {
5176        let _guard = test_guard!();
5177        let config = CircuitBreakerConfig {
5178            failure_threshold: 3,
5179            success_threshold: 2,
5180            open_cooldown_secs: 0,
5181            command_failure_threshold: 2,
5182            ..Default::default()
5183        };
5184        let mut stats = CircuitStats::new();
5185        stats.open();
5186        stats.half_open();
5187
5188        // The half-open trial: a real command hangs on the worker.
5189        assert_eq!(
5190            stats.apply_command_outcome(false, &config),
5191            CircuitState::HalfOpen,
5192            "cooldown 0: reopen then immediately half-open again"
5193        );
5194        assert_eq!(stats.consecutive_command_failures(), 1);
5195
5196        // Liveness keeps answering — many times over.
5197        for _ in 0..10 {
5198            assert_eq!(
5199                stats.apply_health_outcome(true, false, &config),
5200                CircuitState::HalfOpen,
5201                "liveness successes must not close the circuit while a command failure is outstanding"
5202            );
5203        }
5204
5205        // Only a command-class success clears the evidence and recovers.
5206        assert_eq!(
5207            stats.apply_command_outcome(true, &config),
5208            CircuitState::Closed
5209        );
5210        assert_eq!(stats.consecutive_command_failures(), 0);
5211        assert_eq!(stats.reopen_streak(), 0);
5212    }
5213
5214    /// Issue #48: interleaved liveness successes reset `failure_threshold`, so
5215    /// command failures need their own threshold to open a Closed circuit.
5216    #[test]
5217    fn test_command_failures_open_closed_circuit_despite_liveness_successes() {
5218        let _guard = test_guard!();
5219        let config = CircuitBreakerConfig {
5220            failure_threshold: 3,
5221            command_failure_threshold: 2,
5222            open_cooldown_secs: 300,
5223            ..Default::default()
5224        };
5225        let mut stats = CircuitStats::new();
5226
5227        assert_eq!(
5228            stats.apply_command_outcome(false, &config),
5229            CircuitState::Closed
5230        );
5231        // Two liveness successes in between: consecutive_failures is reset...
5232        stats.apply_health_outcome(true, false, &config);
5233        stats.apply_health_outcome(true, false, &config);
5234        assert_eq!(stats.consecutive_failures(), 0);
5235        // ...but the command-failure count is not, and the second one opens.
5236        assert_eq!(
5237            stats.apply_command_outcome(false, &config),
5238            CircuitState::Open
5239        );
5240        assert_eq!(stats.reopen_streak(), 1);
5241    }
5242
5243    /// A command failure in HalfOpen always reopens: "connected but the
5244    /// command never completed" is exactly what the trial exists to detect,
5245    /// so there is no transient exemption on this channel.
5246    #[test]
5247    fn test_command_failure_in_half_open_always_reopens() {
5248        let _guard = test_guard!();
5249        let config = CircuitBreakerConfig {
5250            open_cooldown_secs: 300,
5251            ..Default::default()
5252        };
5253        let mut stats = CircuitStats::new();
5254        stats.open();
5255        stats.half_open();
5256        assert_eq!(
5257            stats.apply_command_outcome(false, &config),
5258            CircuitState::Open
5259        );
5260        assert_eq!(stats.reopen_streak(), 2);
5261    }
5262
5263    /// Repeated Open -> HalfOpen -> Open cycles lengthen the cooldown
5264    /// (exponential, capped) so a flapping worker converges to Open instead of
5265    /// being re-admitted on every base cooldown.
5266    #[test]
5267    fn test_reopen_streak_backs_off_cooldown_until_cap() {
5268        let _guard = test_guard!();
5269        let config = CircuitBreakerConfig {
5270            open_cooldown_secs: 30,
5271            open_cooldown_max_secs: 100,
5272            ..Default::default()
5273        };
5274        assert_eq!(config.cooldown_ms_for_streak(0), 30_000);
5275        assert_eq!(config.cooldown_ms_for_streak(1), 30_000);
5276        assert_eq!(config.cooldown_ms_for_streak(2), 60_000);
5277        assert_eq!(config.cooldown_ms_for_streak(3), 100_000, "capped");
5278        assert_eq!(config.cooldown_ms_for_streak(40), 100_000, "no overflow");
5279
5280        // A cap below the base never shortens the base cooldown.
5281        let inverted = CircuitBreakerConfig {
5282            open_cooldown_secs: 30,
5283            open_cooldown_max_secs: 5,
5284            ..Default::default()
5285        };
5286        assert_eq!(inverted.cooldown_ms_for_streak(3), 30_000);
5287
5288        // The streak is observable through recovery_remaining_secs: after two
5289        // opens the remaining cooldown exceeds the base.
5290        let mut stats = CircuitStats::new();
5291        stats.open();
5292        stats.half_open();
5293        stats.open();
5294        assert_eq!(stats.reopen_streak(), 2);
5295        let remaining = stats.recovery_remaining_secs(&config).unwrap();
5296        assert!(
5297            remaining > 30 && remaining <= 60,
5298            "second open should wait ~60s, got {remaining}"
5299        );
5300
5301        // Closing resets the streak, so the next open starts from the base.
5302        stats.close();
5303        assert_eq!(stats.reopen_streak(), 0);
5304        stats.open();
5305        assert!(stats.recovery_remaining_secs(&config).unwrap() <= 30);
5306    }
5307
5308    #[test]
5309    fn test_circuit_stats_success_resets_consecutive_failures() {
5310        let _guard = test_guard!();
5311        let mut stats = CircuitStats::new();
5312        stats.record_failure();
5313        stats.record_failure();
5314        assert_eq!(stats.consecutive_failures(), 2);
5315
5316        stats.record_success();
5317        assert_eq!(stats.consecutive_failures(), 0);
5318    }
5319
5320    #[test]
5321    fn test_circuit_stats_open_transition() {
5322        let _guard = test_guard!();
5323        let mut stats = CircuitStats::new();
5324        stats.open();
5325
5326        assert_eq!(stats.state(), CircuitState::Open);
5327        assert!(stats.opened_at().is_some());
5328    }
5329
5330    #[test]
5331    fn test_circuit_stats_half_open_transition() {
5332        let _guard = test_guard!();
5333        let mut stats = CircuitStats::new();
5334        stats.open();
5335        stats.half_open();
5336
5337        assert_eq!(stats.state(), CircuitState::HalfOpen);
5338    }
5339
5340    #[test]
5341    fn test_circuit_stats_close_transition() {
5342        let _guard = test_guard!();
5343        let mut stats = CircuitStats::new();
5344        stats.open();
5345        stats.half_open();
5346        stats.close();
5347
5348        assert_eq!(stats.state(), CircuitState::Closed);
5349        assert!(stats.opened_at().is_none());
5350        assert_eq!(stats.consecutive_failures(), 0);
5351    }
5352
5353    #[test]
5354    fn test_circuit_stats_should_close() {
5355        let _guard = test_guard!();
5356        let mut stats = CircuitStats::new();
5357        let config = CircuitBreakerConfig {
5358            success_threshold: 2,
5359            ..Default::default()
5360        };
5361
5362        stats.open();
5363        stats.half_open();
5364
5365        assert!(!stats.should_close(&config)); // 0 successes
5366
5367        stats.record_success();
5368        assert!(!stats.should_close(&config)); // 1 success
5369
5370        stats.record_success();
5371        assert!(stats.should_close(&config)); // 2 successes
5372    }
5373
5374    #[test]
5375    fn test_circuit_stats_can_probe() {
5376        let _guard = test_guard!();
5377        let mut stats = CircuitStats::new();
5378        let config = CircuitBreakerConfig {
5379            half_open_max_probes: 1,
5380            ..Default::default()
5381        };
5382
5383        // Can't probe when closed
5384        assert!(!stats.can_probe(&config));
5385
5386        stats.open();
5387        // Can't probe when open
5388        assert!(!stats.can_probe(&config));
5389
5390        stats.half_open();
5391        // Can probe when half-open
5392        assert!(stats.can_probe(&config));
5393
5394        // Start a probe
5395        assert!(stats.start_probe(&config));
5396        // Can't start another probe
5397        assert!(!stats.can_probe(&config));
5398        assert!(!stats.start_probe(&config));
5399    }
5400
5401    #[test]
5402    fn test_circuit_stats_probe_completion() {
5403        let _guard = test_guard!();
5404        let mut stats = CircuitStats::new();
5405        let config = CircuitBreakerConfig {
5406            half_open_max_probes: 1,
5407            ..Default::default()
5408        };
5409
5410        stats.open();
5411        stats.half_open();
5412        stats.start_probe(&config);
5413
5414        // Probe completes with success
5415        stats.record_success();
5416
5417        // Can start another probe
5418        assert!(stats.can_probe(&config));
5419    }
5420
5421    #[test]
5422    fn test_circuit_stats_failure_in_half_open() {
5423        let _guard = test_guard!();
5424        let mut stats = CircuitStats::new();
5425        let config = CircuitBreakerConfig {
5426            success_threshold: 2,
5427            ..Default::default()
5428        };
5429
5430        stats.open();
5431        stats.half_open();
5432
5433        stats.record_success();
5434        assert_eq!(stats.consecutive_failures(), 0);
5435
5436        // Failure resets consecutive successes
5437        stats.record_failure();
5438        assert!(!stats.should_close(&config));
5439
5440        // Need 2 more successes
5441        stats.record_success();
5442        stats.record_success();
5443        assert!(stats.should_close(&config));
5444    }
5445
5446    #[test]
5447    fn test_circuit_stats_reset_window() {
5448        let _guard = test_guard!();
5449        let mut stats = CircuitStats::new();
5450        stats.record_success();
5451        stats.record_failure();
5452
5453        assert!(stats.error_rate() > 0.0);
5454
5455        stats.reset_window();
5456        assert_eq!(stats.error_rate(), 0.0);
5457    }
5458
5459    #[test]
5460    fn test_circuit_state_transitions_are_deterministic() {
5461        let _guard = test_guard!();
5462        let config = CircuitBreakerConfig::default();
5463        let mut stats = CircuitStats::new();
5464
5465        // Start closed
5466        assert_eq!(stats.state(), CircuitState::Closed);
5467
5468        // Cause failures to open
5469        for _ in 0..3 {
5470            stats.record_failure();
5471        }
5472        assert!(stats.should_open(&config));
5473        stats.open();
5474        assert_eq!(stats.state(), CircuitState::Open);
5475
5476        // Transition to half-open after cooldown would be checked
5477        stats.half_open();
5478        assert_eq!(stats.state(), CircuitState::HalfOpen);
5479
5480        // Successes to close
5481        for _ in 0..2 {
5482            stats.record_success();
5483        }
5484        assert!(stats.should_close(&config));
5485        stats.close();
5486        assert_eq!(stats.state(), CircuitState::Closed);
5487    }
5488
5489    // ========================================================================
5490    // Compilation Timing Tests
5491    // ========================================================================
5492
5493    fn make_test_metrics(
5494        speedup: Option<f64>,
5495        total_secs: u64,
5496        success: bool,
5497    ) -> CompilationMetrics {
5498        CompilationMetrics {
5499            project_id: "test-project".to_string(),
5500            worker_id: "worker-1".to_string(),
5501            timestamp: chrono::Utc::now(),
5502            timing: CompilationTimingBreakdown {
5503                rsync_up: Duration::from_millis(100),
5504                remote_build: Duration::from_millis(800),
5505                rsync_down: Duration::from_millis(100),
5506                total: Duration::from_secs(total_secs),
5507            },
5508            local_build_time: speedup.map(|s| Duration::from_secs_f64(total_secs as f64 * s)),
5509            speedup,
5510            files_synced: 100,
5511            bytes_transferred: 1_000_000,
5512            exit_code: if success { 0 } else { 1 },
5513            success,
5514        }
5515    }
5516
5517    #[test]
5518    fn test_compilation_timing_breakdown_default() {
5519        let _guard = test_guard!();
5520        let timing = CompilationTimingBreakdown::default();
5521        assert_eq!(timing.rsync_up, Duration::ZERO);
5522        assert_eq!(timing.remote_build, Duration::ZERO);
5523        assert_eq!(timing.rsync_down, Duration::ZERO);
5524        assert_eq!(timing.total, Duration::ZERO);
5525    }
5526
5527    #[test]
5528    fn test_compilation_timing_breakdown_serialization() {
5529        let _guard = test_guard!();
5530        let timing = CompilationTimingBreakdown {
5531            rsync_up: Duration::from_millis(100),
5532            remote_build: Duration::from_millis(2000),
5533            rsync_down: Duration::from_millis(50),
5534            total: Duration::from_millis(2150),
5535        };
5536
5537        let json = serde_json::to_string(&timing).unwrap();
5538        assert!(json.contains("100")); // rsync_up
5539        assert!(json.contains("2000")); // remote_build
5540        assert!(json.contains("50")); // rsync_down
5541        assert!(json.contains("2150")); // total
5542
5543        let parsed: CompilationTimingBreakdown = serde_json::from_str(&json).unwrap();
5544        assert_eq!(parsed.rsync_up, timing.rsync_up);
5545        assert_eq!(parsed.remote_build, timing.remote_build);
5546        assert_eq!(parsed.rsync_down, timing.rsync_down);
5547        assert_eq!(parsed.total, timing.total);
5548    }
5549
5550    #[test]
5551    fn test_command_timing_breakdown_serialization() {
5552        let _guard = test_guard!();
5553        let timing = CommandTimingBreakdown {
5554            classify: Some(Duration::from_millis(2)),
5555            select: None,
5556            sync_up: Some(Duration::from_millis(120)),
5557            exec: Some(Duration::from_millis(900)),
5558            sync_down: None,
5559            cleanup: Some(Duration::from_millis(5)),
5560            total: Some(Duration::from_millis(1027)),
5561        };
5562
5563        let json = serde_json::to_string(&timing).unwrap();
5564        assert!(json.contains("\"classify\":2"));
5565        assert!(json.contains("\"select\":null"));
5566        assert!(json.contains("\"sync_up\":120"));
5567        assert!(json.contains("\"exec\":900"));
5568        assert!(json.contains("\"sync_down\":null"));
5569        assert!(json.contains("\"cleanup\":5"));
5570        assert!(json.contains("\"total\":1027"));
5571
5572        let parsed: CommandTimingBreakdown = serde_json::from_str(&json).unwrap();
5573        assert_eq!(parsed.classify, timing.classify);
5574        assert_eq!(parsed.select, timing.select);
5575        assert_eq!(parsed.sync_up, timing.sync_up);
5576        assert_eq!(parsed.exec, timing.exec);
5577        assert_eq!(parsed.sync_down, timing.sync_down);
5578        assert_eq!(parsed.cleanup, timing.cleanup);
5579        assert_eq!(parsed.total, timing.total);
5580    }
5581
5582    #[test]
5583    fn test_compilation_metrics_calculate_speedup() {
5584        let _guard = test_guard!();
5585        let mut metrics = CompilationMetrics {
5586            timing: CompilationTimingBreakdown {
5587                total: Duration::from_secs(10),
5588                ..Default::default()
5589            },
5590            local_build_time: Some(Duration::from_secs(30)),
5591            ..Default::default()
5592        };
5593
5594        metrics.calculate_speedup();
5595
5596        assert!(metrics.speedup.is_some());
5597        let speedup = metrics.speedup.unwrap();
5598        assert!(
5599            (speedup - 3.0).abs() < 0.01,
5600            "Expected 3.0x speedup, got {}",
5601            speedup
5602        );
5603    }
5604
5605    #[test]
5606    fn test_compilation_metrics_is_beneficial() {
5607        let _guard = test_guard!();
5608        let mut metrics = CompilationMetrics::default();
5609
5610        // No speedup calculated
5611        assert!(!metrics.is_beneficial());
5612
5613        // Speedup > 1 is beneficial
5614        metrics.speedup = Some(1.5);
5615        assert!(metrics.is_beneficial());
5616
5617        // Speedup < 1 is not beneficial
5618        metrics.speedup = Some(0.8);
5619        assert!(!metrics.is_beneficial());
5620
5621        // Speedup = 1 is not beneficial (exactly same speed)
5622        metrics.speedup = Some(1.0);
5623        assert!(!metrics.is_beneficial());
5624    }
5625
5626    #[test]
5627    fn test_compilation_metrics_serialization() {
5628        let _guard = test_guard!();
5629        let metrics = make_test_metrics(Some(2.5), 10, true);
5630        let json = serde_json::to_string(&metrics).unwrap();
5631
5632        assert!(json.contains("test-project"));
5633        assert!(json.contains("worker-1"));
5634        assert!(json.contains("2.5")); // speedup
5635
5636        let parsed: CompilationMetrics = serde_json::from_str(&json).unwrap();
5637        assert_eq!(parsed.project_id, "test-project");
5638        assert_eq!(parsed.worker_id, "worker-1");
5639        assert!(parsed.success);
5640    }
5641
5642    #[test]
5643    fn test_compilation_timer_new() {
5644        let _guard = test_guard!();
5645        let timer = CompilationTimer::new("my-project", "my-worker");
5646        assert_eq!(timer.project_id, "my-project");
5647        assert_eq!(timer.worker_id, "my-worker");
5648        assert!(timer.rsync_up.is_none());
5649        assert!(timer.remote_build.is_none());
5650        assert!(timer.rsync_down.is_none());
5651    }
5652
5653    #[test]
5654    fn test_compilation_timer_phases() {
5655        let _guard = test_guard!();
5656        let mut timer = CompilationTimer::new("test", "worker");
5657
5658        // Simulate rsync up
5659        std::thread::sleep(Duration::from_millis(5));
5660        timer.end_rsync_up();
5661        assert!(timer.rsync_up.is_some());
5662        assert!(timer.rsync_up.unwrap() >= Duration::from_millis(5));
5663
5664        // Simulate remote build
5665        std::thread::sleep(Duration::from_millis(10));
5666        timer.end_remote_build();
5667        assert!(timer.remote_build.is_some());
5668        assert!(timer.remote_build.unwrap() >= Duration::from_millis(10));
5669
5670        // Simulate rsync down
5671        std::thread::sleep(Duration::from_millis(5));
5672        timer.end_rsync_down();
5673        assert!(timer.rsync_down.is_some());
5674        assert!(timer.rsync_down.unwrap() >= Duration::from_millis(5));
5675
5676        // Finish
5677        let metrics = timer.finish(0, 50, 500_000);
5678        assert_eq!(metrics.project_id, "test");
5679        assert_eq!(metrics.worker_id, "worker");
5680        assert!(metrics.success);
5681        assert!(metrics.timing.total >= Duration::from_millis(20));
5682        assert_eq!(metrics.files_synced, 50);
5683        assert_eq!(metrics.bytes_transferred, 500_000);
5684    }
5685
5686    #[test]
5687    fn test_metrics_aggregator_new() {
5688        let _guard = test_guard!();
5689        let agg = MetricsAggregator::new(100);
5690        assert_eq!(agg.count(), 0);
5691        assert!(agg.average_speedup().is_none());
5692    }
5693
5694    #[test]
5695    fn test_metrics_aggregator_record() {
5696        let _guard = test_guard!();
5697        let mut agg = MetricsAggregator::new(3);
5698
5699        // Record first metric
5700        agg.record(make_test_metrics(Some(2.0), 10, true));
5701        assert_eq!(agg.count(), 1);
5702
5703        // Record up to max
5704        agg.record(make_test_metrics(Some(3.0), 20, true));
5705        agg.record(make_test_metrics(Some(4.0), 30, true));
5706        assert_eq!(agg.count(), 3);
5707
5708        // Exceed max should drop oldest
5709        agg.record(make_test_metrics(Some(5.0), 40, true));
5710        assert_eq!(agg.count(), 3);
5711    }
5712
5713    #[test]
5714    fn test_metrics_aggregator_average_speedup() {
5715        let _guard = test_guard!();
5716        let mut agg = MetricsAggregator::new(100);
5717
5718        // Empty case
5719        assert!(agg.average_speedup().is_none());
5720
5721        // Add metrics with speedups 1, 2, 3, 4, 5 -> average = 3
5722        for i in 1..=5 {
5723            agg.record(make_test_metrics(Some(i as f64), 10, true));
5724        }
5725
5726        let avg = agg.average_speedup().unwrap();
5727        assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5728    }
5729
5730    #[test]
5731    fn test_metrics_aggregator_average_speedup_with_none() {
5732        let _guard = test_guard!();
5733        let mut agg = MetricsAggregator::new(100);
5734
5735        // Mix of Some and None speedups
5736        agg.record(make_test_metrics(Some(2.0), 10, true));
5737        agg.record(make_test_metrics(None, 10, true)); // No speedup
5738        agg.record(make_test_metrics(Some(4.0), 10, true));
5739
5740        // Average should only consider metrics with speedup
5741        let avg = agg.average_speedup().unwrap();
5742        assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5743    }
5744
5745    #[test]
5746    fn test_metrics_aggregator_percentiles() {
5747        let _guard = test_guard!();
5748        let mut agg = MetricsAggregator::new(100);
5749
5750        // Add metrics with total times 1s, 2s, 3s, ..., 10s
5751        for i in 1..=10 {
5752            agg.record(make_test_metrics(Some(1.0), i, true));
5753        }
5754
5755        // p50 (median) should be around 5s
5756        let p50 = agg.p50_total_time().unwrap();
5757        assert!(p50 >= Duration::from_secs(5) && p50 <= Duration::from_secs(6));
5758
5759        // p95 should be around 9s or 10s
5760        let p95 = agg.p95_total_time().unwrap();
5761        assert!(p95 >= Duration::from_secs(9) && p95 <= Duration::from_secs(10));
5762
5763        // p99 should be 10s
5764        let p99 = agg.p99_total_time().unwrap();
5765        assert!(p99 >= Duration::from_secs(9));
5766    }
5767
5768    #[test]
5769    fn test_metrics_aggregator_success_rate() {
5770        let _guard = test_guard!();
5771        let mut agg = MetricsAggregator::new(100);
5772
5773        // Empty case
5774        assert_eq!(agg.success_rate(), 100.0);
5775
5776        // All successful
5777        agg.record(make_test_metrics(Some(1.0), 10, true));
5778        agg.record(make_test_metrics(Some(1.0), 10, true));
5779        assert_eq!(agg.success_rate(), 100.0);
5780
5781        // Add a failure (2 success, 1 failure = 66.67%)
5782        agg.record(make_test_metrics(Some(1.0), 10, false));
5783        let rate = agg.success_rate();
5784        assert!((rate - 66.67).abs() < 1.0, "Expected ~66.67%, got {}", rate);
5785    }
5786
5787    #[test]
5788    fn test_metrics_aggregator_clear() {
5789        let _guard = test_guard!();
5790        let mut agg = MetricsAggregator::new(100);
5791        agg.record(make_test_metrics(Some(1.0), 10, true));
5792        agg.record(make_test_metrics(Some(2.0), 20, true));
5793
5794        assert_eq!(agg.count(), 2);
5795
5796        agg.clear();
5797        assert_eq!(agg.count(), 0);
5798        assert!(agg.average_speedup().is_none());
5799    }
5800
5801    // CompilationConfig timeout tests
5802
5803    #[test]
5804    fn test_remote_build_jobs_parse_display_and_serde() {
5805        let _guard = test_guard!();
5806        assert_eq!(RemoteBuildJobs::default(), RemoteBuildJobs::Auto);
5807        for (spelling, expected) in [
5808            ("auto", RemoteBuildJobs::Auto),
5809            (" AUTO ", RemoteBuildJobs::Auto),
5810            ("off", RemoteBuildJobs::Off),
5811            ("none", RemoteBuildJobs::Off),
5812            ("4", RemoteBuildJobs::Fixed(4)),
5813        ] {
5814            assert_eq!(
5815                RemoteBuildJobs::parse(spelling).unwrap(),
5816                expected,
5817                "{spelling}"
5818            );
5819        }
5820        assert!(RemoteBuildJobs::parse("0").is_err(), "cargo rejects -j 0");
5821        assert!(RemoteBuildJobs::parse("many").is_err());
5822        assert_eq!(RemoteBuildJobs::Fixed(4).to_string(), "4");
5823        assert_eq!(RemoteBuildJobs::Off.to_string(), "off");
5824
5825        // TOML accepts both the string spellings and a bare integer, and an
5826        // absent key keeps the default.
5827        #[derive(Deserialize)]
5828        struct Wrap {
5829            compilation: CompilationConfig,
5830        }
5831        let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = \"off\"\n").unwrap();
5832        assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Off);
5833        let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = 6\n").unwrap();
5834        assert_eq!(
5835            parsed.compilation.remote_build_jobs,
5836            RemoteBuildJobs::Fixed(6)
5837        );
5838        let parsed: Wrap = toml::from_str("[compilation]\nbuild_slots = 4\n").unwrap();
5839        assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Auto);
5840        assert!(toml::from_str::<Wrap>("[compilation]\nremote_build_jobs = 0\n").is_err());
5841
5842        // Serialization round-trips through the same spellings.
5843        let json = serde_json::to_string(&RemoteBuildJobs::Fixed(3)).unwrap();
5844        assert_eq!(json, "3");
5845        assert_eq!(
5846            serde_json::to_string(&RemoteBuildJobs::Auto).unwrap(),
5847            "\"auto\""
5848        );
5849        let back: RemoteBuildJobs = serde_json::from_str("\"auto\"").unwrap();
5850        assert_eq!(back, RemoteBuildJobs::Auto);
5851    }
5852
5853    #[test]
5854    fn test_compilation_config_default_timeouts() {
5855        let _guard = test_guard!();
5856        let config = CompilationConfig::default();
5857        // Default build timeout: 5 minutes
5858        assert_eq!(config.build_timeout_sec, 300);
5859        // Default test timeout: 30 minutes
5860        assert_eq!(config.test_timeout_sec, 1800);
5861    }
5862
5863    #[test]
5864    fn test_compilation_config_timeout_for_test_kinds() {
5865        let _guard = test_guard!();
5866        let config = CompilationConfig::default();
5867
5868        // Test commands should get test_timeout_sec
5869        assert_eq!(
5870            config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5871            std::time::Duration::from_secs(1800)
5872        );
5873        assert_eq!(
5874            config.timeout_for_kind(Some(crate::CompilationKind::CargoNextest)),
5875            std::time::Duration::from_secs(1800)
5876        );
5877        assert_eq!(
5878            config.timeout_for_kind(Some(crate::CompilationKind::BunTest)),
5879            std::time::Duration::from_secs(config.bun_timeout_sec)
5880        );
5881    }
5882
5883    #[test]
5884    fn test_compilation_config_timeout_for_build_kinds() {
5885        let _guard = test_guard!();
5886        let config = CompilationConfig::default();
5887
5888        // Build/check commands should get build_timeout_sec
5889        assert_eq!(
5890            config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5891            std::time::Duration::from_secs(300)
5892        );
5893        assert_eq!(
5894            config.timeout_for_kind(Some(crate::CompilationKind::CargoCheck)),
5895            std::time::Duration::from_secs(300)
5896        );
5897        assert_eq!(
5898            config.timeout_for_kind(Some(crate::CompilationKind::CargoClippy)),
5899            std::time::Duration::from_secs(300)
5900        );
5901        assert_eq!(
5902            config.timeout_for_kind(None),
5903            std::time::Duration::from_secs(300)
5904        );
5905    }
5906
5907    #[test]
5908    fn test_compilation_config_custom_timeouts() {
5909        let _guard = test_guard!();
5910        let config = CompilationConfig {
5911            build_timeout_sec: 600, // 10 minutes
5912            test_timeout_sec: 3600, // 1 hour
5913            ..Default::default()
5914        };
5915
5916        assert_eq!(
5917            config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5918            std::time::Duration::from_secs(600)
5919        );
5920        assert_eq!(
5921            config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5922            std::time::Duration::from_secs(3600)
5923        );
5924    }
5925
5926    #[test]
5927    fn test_compilation_config_speedup_threshold_default() {
5928        let _guard = test_guard!();
5929        let config = CompilationConfig::default();
5930        // Default speedup threshold: 1.2 (20% faster required)
5931        assert!((config.remote_speedup_threshold - 1.2).abs() < 0.001);
5932    }
5933
5934    #[test]
5935    fn test_compilation_config_speedup_threshold_custom() {
5936        let _guard = test_guard!();
5937        let config = CompilationConfig {
5938            remote_speedup_threshold: 1.5, // Require 50% faster
5939            ..Default::default()
5940        };
5941        assert!((config.remote_speedup_threshold - 1.5).abs() < 0.001);
5942    }
5943
5944    #[test]
5945    fn test_compilation_config_speedup_threshold_no_minimum() {
5946        let _guard = test_guard!();
5947        // Setting to 1.0 means "always offload when other criteria met"
5948        let config = CompilationConfig {
5949            remote_speedup_threshold: 1.0,
5950            ..Default::default()
5951        };
5952        assert!((config.remote_speedup_threshold - 1.0).abs() < 0.001);
5953    }
5954
5955    #[test]
5956    fn test_compilation_config_min_local_time_ms_default() {
5957        let _guard = test_guard!();
5958        let config = CompilationConfig::default();
5959        // Default min_local_time_ms: 2000ms
5960        assert_eq!(config.min_local_time_ms, 2000);
5961    }
5962
5963    #[test]
5964    fn test_compilation_config_serde_roundtrip_speedup_threshold() {
5965        let _guard = test_guard!();
5966        let config = CompilationConfig {
5967            remote_speedup_threshold: 2.5,
5968            min_local_time_ms: 5000,
5969            ..Default::default()
5970        };
5971        let json = serde_json::to_string(&config).unwrap();
5972        let parsed: CompilationConfig = serde_json::from_str(&json).unwrap();
5973        assert!((parsed.remote_speedup_threshold - 2.5).abs() < 0.001);
5974        assert_eq!(parsed.min_local_time_ms, 5000);
5975    }
5976
5977    // ========================================================================
5978    // validate_remote_base Tests
5979    // ========================================================================
5980
5981    #[test]
5982    fn test_validate_remote_base_absolute_path() {
5983        let _guard = test_guard!();
5984        assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5985        assert_eq!(
5986            validate_remote_base("/var/rch-builds").unwrap(),
5987            "/var/rch-builds"
5988        );
5989        assert_eq!(
5990            validate_remote_base("/home/builder/.rch").unwrap(),
5991            "/home/builder/.rch"
5992        );
5993    }
5994
5995    #[test]
5996    fn test_validate_remote_base_tilde_expansion() {
5997        let _guard = test_guard!();
5998        // Tilde expansion should work
5999        let result = validate_remote_base("~/rch");
6000        assert!(result.is_ok());
6001        let path = result.unwrap();
6002        assert!(
6003            path.starts_with('/'),
6004            "Path should be absolute after expansion: {}",
6005            path
6006        );
6007        assert!(!path.contains('~'), "Tilde should be expanded: {}", path);
6008    }
6009
6010    #[test]
6011    fn test_validate_remote_base_rejects_relative_path() {
6012        let _guard = test_guard!();
6013        let result = validate_remote_base("tmp/rch");
6014        assert!(result.is_err());
6015        assert!(result.unwrap_err().contains("absolute path"));
6016    }
6017
6018    #[test]
6019    fn test_validate_remote_base_rejects_path_traversal() {
6020        let _guard = test_guard!();
6021        let result = validate_remote_base("/tmp/../etc/rch");
6022        assert!(result.is_err());
6023        assert!(result.unwrap_err().contains("path traversal"));
6024
6025        let result = validate_remote_base("/tmp/rch/../other");
6026        assert!(result.is_err());
6027        assert!(result.unwrap_err().contains("path traversal"));
6028    }
6029
6030    #[test]
6031    fn test_validate_remote_base_normalizes_trailing_slash() {
6032        let _guard = test_guard!();
6033        assert_eq!(validate_remote_base("/tmp/rch/").unwrap(), "/tmp/rch");
6034        assert_eq!(validate_remote_base("/tmp/rch///").unwrap(), "/tmp/rch");
6035    }
6036
6037    #[test]
6038    fn test_validate_remote_base_root_path() {
6039        let _guard = test_guard!();
6040        // Root path should be rejected
6041        let result = validate_remote_base("/");
6042        assert!(result.is_err());
6043        assert!(result.unwrap_err().contains("root directory"));
6044    }
6045
6046    #[test]
6047    fn test_validate_remote_base_rejects_top_level() {
6048        let _guard = test_guard!();
6049        // Top-level directories should be rejected (depth 1)
6050        let result = validate_remote_base("/tmp");
6051        assert!(result.is_err());
6052        assert!(result.unwrap_err().contains("at least 2 levels deep"));
6053
6054        let result = validate_remote_base("/home");
6055        assert!(result.is_err());
6056
6057        // Depth 2 should be allowed
6058        assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
6059    }
6060
6061    #[test]
6062    fn test_default_remote_base() {
6063        let _guard = test_guard!();
6064        assert_eq!(default_remote_base(), "/data/tmp/rch");
6065    }
6066
6067    #[test]
6068    fn test_transfer_config_default_has_remote_base() {
6069        let _guard = test_guard!();
6070        let config = TransferConfig::default();
6071        assert_eq!(config.remote_base, "/data/tmp/rch");
6072    }
6073
6074    #[test]
6075    fn test_source_sync_timeout_validation_boundaries() {
6076        let _guard = test_guard!();
6077        assert!(!TransferConfig::valid_sync_timeout_ms(0));
6078        assert!(!TransferConfig::valid_sync_timeout_ms(999));
6079        assert!(TransferConfig::valid_sync_timeout_ms(1_000));
6080        assert!(TransferConfig::valid_sync_timeout_ms(3_600_000));
6081        assert!(!TransferConfig::valid_sync_timeout_ms(3_600_001));
6082    }
6083
6084    #[test]
6085    fn test_source_sync_timeout_is_explicit_or_payload_aware() {
6086        let _guard = test_guard!();
6087        let default = TransferConfig::default();
6088        assert_eq!(
6089            default.sync_timeout_for_payload(0),
6090            std::time::Duration::from_millis(30_000)
6091        );
6092        assert_eq!(
6093            default.sync_timeout_for_payload(10 * 1024 * 1024),
6094            std::time::Duration::from_millis(40_000)
6095        );
6096        assert_eq!(
6097            default.sync_timeout_for_payload(u64::MAX),
6098            std::time::Duration::from_millis(3_600_000)
6099        );
6100
6101        let explicit = TransferConfig {
6102            sync_timeout_ms: Some(120_000),
6103            ..TransferConfig::default()
6104        };
6105        assert_eq!(
6106            explicit.sync_timeout_for_payload(u64::MAX),
6107            std::time::Duration::from_millis(120_000)
6108        );
6109    }
6110
6111    // ========================================================================
6112    // RetryConfig Tests (bd-x1ek)
6113    // ========================================================================
6114
6115    #[test]
6116    fn test_retry_config_default() {
6117        let _guard = test_guard!();
6118        let config = RetryConfig::default();
6119        assert_eq!(config.max_attempts, 3);
6120        assert_eq!(config.base_delay_ms, 100);
6121        assert_eq!(config.max_delay_ms, 5000);
6122        assert_eq!(config.jitter_factor, 0.1);
6123        assert_eq!(config.total_timeout_ms, 30000);
6124    }
6125
6126    #[test]
6127    fn test_retry_config_no_retry() {
6128        let _guard = test_guard!();
6129        let config = RetryConfig::no_retry();
6130        assert_eq!(config.max_attempts, 1);
6131    }
6132
6133    #[test]
6134    fn test_retry_config_delay_for_attempt_zero() {
6135        let _guard = test_guard!();
6136        let config = RetryConfig::default();
6137        assert_eq!(config.delay_for_attempt(0), std::time::Duration::ZERO);
6138    }
6139
6140    #[test]
6141    fn test_retry_config_delay_for_attempt_exponential() {
6142        let _guard = test_guard!();
6143        let config = RetryConfig {
6144            base_delay_ms: 100,
6145            max_delay_ms: 10000,
6146            jitter_factor: 0.0, // No jitter for deterministic test
6147            ..Default::default()
6148        };
6149
6150        // attempt 1: 100 * 2^0 = 100ms
6151        assert_eq!(
6152            config.delay_for_attempt(1),
6153            std::time::Duration::from_millis(100)
6154        );
6155        // attempt 2: 100 * 2^1 = 200ms
6156        assert_eq!(
6157            config.delay_for_attempt(2),
6158            std::time::Duration::from_millis(200)
6159        );
6160        // attempt 3: 100 * 2^2 = 400ms
6161        assert_eq!(
6162            config.delay_for_attempt(3),
6163            std::time::Duration::from_millis(400)
6164        );
6165    }
6166
6167    #[test]
6168    fn test_retry_config_delay_capped_at_max() {
6169        let _guard = test_guard!();
6170        let config = RetryConfig {
6171            base_delay_ms: 1000,
6172            max_delay_ms: 2000,
6173            jitter_factor: 0.0,
6174            ..Default::default()
6175        };
6176
6177        // attempt 2: 1000 * 2^1 = 2000ms (at cap)
6178        assert_eq!(
6179            config.delay_for_attempt(2),
6180            std::time::Duration::from_millis(2000)
6181        );
6182        // attempt 3: 1000 * 2^2 = 4000ms, capped to 2000ms
6183        assert_eq!(
6184            config.delay_for_attempt(3),
6185            std::time::Duration::from_millis(2000)
6186        );
6187    }
6188
6189    #[test]
6190    fn test_retry_config_random_jitter_stays_within_bounds() {
6191        let _guard = test_guard!();
6192        let config = RetryConfig {
6193            base_delay_ms: 100,
6194            max_delay_ms: 200,
6195            jitter_factor: 0.25,
6196            ..Default::default()
6197        };
6198
6199        // Jitter applies after the exponential delay is capped. Exercise the
6200        // real RNG without requiring any particular random value or sequence.
6201        for (attempt, minimum_ms, maximum_ms) in [(1, 75, 125), (3, 150, 250)] {
6202            for _ in 0..128 {
6203                let delay = config.delay_for_attempt(attempt);
6204                assert!(delay >= std::time::Duration::from_millis(minimum_ms));
6205                assert!(delay <= std::time::Duration::from_millis(maximum_ms));
6206            }
6207        }
6208    }
6209
6210    #[test]
6211    fn test_retry_config_should_retry() {
6212        let _guard = test_guard!();
6213        let config = RetryConfig {
6214            max_attempts: 3,
6215            total_timeout_ms: 1000,
6216            ..Default::default()
6217        };
6218
6219        // Before max attempts and within timeout
6220        assert!(config.should_retry(0, std::time::Duration::from_millis(0)));
6221        assert!(config.should_retry(1, std::time::Duration::from_millis(500)));
6222        assert!(config.should_retry(2, std::time::Duration::from_millis(900)));
6223
6224        // At max attempts
6225        assert!(!config.should_retry(3, std::time::Duration::from_millis(0)));
6226
6227        // Past timeout
6228        assert!(!config.should_retry(1, std::time::Duration::from_millis(1001)));
6229    }
6230
6231    #[test]
6232    fn test_retry_config_serde_roundtrip() {
6233        let _guard = test_guard!();
6234        let config = RetryConfig {
6235            max_attempts: 5,
6236            base_delay_ms: 200,
6237            max_delay_ms: 8000,
6238            jitter_factor: 0.2,
6239            total_timeout_ms: 60000,
6240        };
6241        let json = serde_json::to_string(&config).unwrap();
6242        let parsed: RetryConfig = serde_json::from_str(&json).unwrap();
6243        assert_eq!(parsed.max_attempts, 5);
6244        assert_eq!(parsed.base_delay_ms, 200);
6245        assert_eq!(parsed.max_delay_ms, 8000);
6246        assert_eq!(parsed.jitter_factor, 0.2);
6247        assert_eq!(parsed.total_timeout_ms, 60000);
6248    }
6249
6250    #[test]
6251    fn test_transfer_config_includes_retry() {
6252        let _guard = test_guard!();
6253        let config = TransferConfig::default();
6254        assert_eq!(config.retry.max_attempts, 3);
6255    }
6256
6257    // =========================================================================
6258    // ExecutionConfig Tests (bd-785w)
6259    // =========================================================================
6260
6261    #[test]
6262    fn test_execution_config_default_allowlist() {
6263        let _guard = test_guard!();
6264        let config = ExecutionConfig::default();
6265        // All supported compilers should be in the default allowlist
6266        assert!(config.is_allowed("cargo"));
6267        assert!(config.is_allowed("rustc"));
6268        assert!(config.is_allowed("gcc"));
6269        assert!(config.is_allowed("g++"));
6270        assert!(config.is_allowed("clang"));
6271        assert!(config.is_allowed("clang++"));
6272        assert!(config.is_allowed("make"));
6273        assert!(config.is_allowed("cmake"));
6274        assert!(config.is_allowed("ninja"));
6275        assert!(config.is_allowed("meson"));
6276        assert!(config.is_allowed("bun"));
6277        assert!(config.is_allowed("nextest"));
6278        assert!(config.is_allowed("cc"));
6279        assert!(config.is_allowed("c++"));
6280    }
6281
6282    #[test]
6283    fn test_execution_config_is_allowed_case_insensitive() {
6284        let _guard = test_guard!();
6285        let config = ExecutionConfig::default();
6286        // Should be case-insensitive
6287        assert!(config.is_allowed("CARGO"));
6288        assert!(config.is_allowed("Cargo"));
6289        assert!(config.is_allowed("GCC"));
6290        assert!(config.is_allowed("Gcc"));
6291    }
6292
6293    #[test]
6294    fn test_execution_config_is_not_allowed() {
6295        let _guard = test_guard!();
6296        let config = ExecutionConfig::default();
6297        // Unknown commands should not be allowed
6298        assert!(!config.is_allowed("python"));
6299        assert!(!config.is_allowed("npm"));
6300        assert!(!config.is_allowed(""));
6301        // `go` and `tsc` ARE allowed now that Go/TypeScript are offload targets.
6302        // If they were missing from the allowlist the hook would fail open to
6303        // LOCAL execution — classification would look correct while every build
6304        // silently ran on the orchestrator.
6305        assert!(config.is_allowed("go"));
6306        assert!(config.is_allowed("tsc"));
6307    }
6308
6309    #[test]
6310    fn test_execution_config_empty_allowlist() {
6311        let _guard = test_guard!();
6312        let config = ExecutionConfig { allowlist: vec![] };
6313        // Empty allowlist should block everything
6314        assert!(!config.is_allowed("cargo"));
6315        assert!(!config.is_allowed("gcc"));
6316    }
6317
6318    #[test]
6319    fn test_execution_config_custom_allowlist() {
6320        let _guard = test_guard!();
6321        let config = ExecutionConfig {
6322            allowlist: vec!["cargo".to_string(), "custom_tool".to_string()],
6323        };
6324        assert!(config.is_allowed("cargo"));
6325        assert!(config.is_allowed("custom_tool"));
6326        assert!(!config.is_allowed("gcc"));
6327    }
6328
6329    #[test]
6330    fn test_execution_config_serde() {
6331        let _guard = test_guard!();
6332        let config = ExecutionConfig {
6333            allowlist: vec!["cargo".to_string(), "rustc".to_string()],
6334        };
6335        let json = serde_json::to_string(&config).unwrap();
6336        let parsed: ExecutionConfig = serde_json::from_str(&json).unwrap();
6337        assert_eq!(parsed.allowlist, config.allowlist);
6338    }
6339
6340    #[test]
6341    fn test_rch_config_includes_execution() {
6342        let _guard = test_guard!();
6343        let config = RchConfig::default();
6344        // RchConfig should include ExecutionConfig with default allowlist
6345        assert!(config.execution.is_allowed("cargo"));
6346    }
6347
6348    // ========================================================================
6349    // Adaptive Compression Tests (bd-243w)
6350    // ========================================================================
6351
6352    #[test]
6353    fn test_adaptive_compression_disabled_by_default() {
6354        let _guard = test_guard!();
6355        let config = TransferConfig::default();
6356        assert!(!config.adaptive_compression);
6357        // Should use fixed level when disabled
6358        assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6359        assert_eq!(config.select_compression_level(Some(500_000_000)), 3);
6360    }
6361
6362    #[test]
6363    fn test_adaptive_compression_small_payload() {
6364        let _guard = test_guard!();
6365        let config = TransferConfig {
6366            adaptive_compression: true,
6367            ..Default::default()
6368        };
6369        // < 10MB should use level 1
6370        assert_eq!(config.select_compression_level(Some(0)), 1);
6371        assert_eq!(config.select_compression_level(Some(1_000_000)), 1); // 1MB
6372        assert_eq!(config.select_compression_level(Some(9_999_999)), 1); // ~10MB
6373    }
6374
6375    #[test]
6376    fn test_adaptive_compression_medium_payload() {
6377        let _guard = test_guard!();
6378        let config = TransferConfig {
6379            adaptive_compression: true,
6380            ..Default::default()
6381        };
6382        // 10-200MB should use level 3
6383        assert_eq!(config.select_compression_level(Some(10_000_000)), 3); // 10MB
6384        assert_eq!(config.select_compression_level(Some(100_000_000)), 3); // 100MB
6385        assert_eq!(config.select_compression_level(Some(199_999_999)), 3); // ~200MB
6386    }
6387
6388    #[test]
6389    fn test_adaptive_compression_large_payload() {
6390        let _guard = test_guard!();
6391        let config = TransferConfig {
6392            adaptive_compression: true,
6393            ..Default::default()
6394        };
6395        // > 200MB should use level 7
6396        assert_eq!(config.select_compression_level(Some(200_000_000)), 7); // 200MB
6397        assert_eq!(config.select_compression_level(Some(500_000_000)), 7); // 500MB
6398        assert_eq!(config.select_compression_level(Some(1_000_000_000)), 7); // 1GB
6399    }
6400
6401    #[test]
6402    fn test_adaptive_compression_no_estimate() {
6403        let _guard = test_guard!();
6404        let config = TransferConfig {
6405            adaptive_compression: true,
6406            compression_level: 5,
6407            ..Default::default()
6408        };
6409        // No estimate should fallback to default level
6410        assert_eq!(config.select_compression_level(None), 5);
6411    }
6412
6413    #[test]
6414    fn test_adaptive_compression_respects_min_level() {
6415        let _guard = test_guard!();
6416        let config = TransferConfig {
6417            adaptive_compression: true,
6418            min_compression_level: 3,
6419            ..Default::default()
6420        };
6421        // Small payload would want level 1, but min is 3
6422        assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6423    }
6424
6425    #[test]
6426    fn test_adaptive_compression_respects_max_level() {
6427        let _guard = test_guard!();
6428        let config = TransferConfig {
6429            adaptive_compression: true,
6430            max_compression_level: 5,
6431            ..Default::default()
6432        };
6433        // Large payload would want level 7, but max is 5
6434        assert_eq!(config.select_compression_level(Some(500_000_000)), 5);
6435    }
6436
6437    #[test]
6438    fn test_adaptive_compression_serde_roundtrip() {
6439        let _guard = test_guard!();
6440        let config = TransferConfig {
6441            adaptive_compression: true,
6442            min_compression_level: 2,
6443            max_compression_level: 8,
6444            ..Default::default()
6445        };
6446        let json = serde_json::to_string(&config).unwrap();
6447        let parsed: TransferConfig = serde_json::from_str(&json).unwrap();
6448        assert!(parsed.adaptive_compression);
6449        assert_eq!(parsed.min_compression_level, 2);
6450        assert_eq!(parsed.max_compression_level, 8);
6451    }
6452
6453    // -------------------------------------------------------------------------
6454    // Self-Healing Config Tests (bd-59kg)
6455    // -------------------------------------------------------------------------
6456
6457    #[test]
6458    fn test_self_healing_config_defaults() {
6459        let _guard = test_guard!();
6460        // TEST START: SelfHealingConfig::default() returns expected values
6461        let config = SelfHealingConfig::default();
6462
6463        // Both self-healing behaviors enabled by default
6464        assert!(
6465            config.hook_starts_daemon,
6466            "hook_starts_daemon should be true by default"
6467        );
6468        assert!(
6469            config.daemon_installs_hooks,
6470            "daemon_installs_hooks should be true by default"
6471        );
6472
6473        // Timing defaults
6474        assert_eq!(
6475            config.auto_start_cooldown_secs, 30,
6476            "auto_start_cooldown_secs should default to 30"
6477        );
6478        assert_eq!(
6479            config.auto_start_timeout_secs, 3,
6480            "auto_start_timeout_secs should default to 3"
6481        );
6482        // TEST PASS: SelfHealingConfig defaults
6483    }
6484
6485    #[test]
6486    fn test_self_healing_config_serde_full() {
6487        let _guard = test_guard!();
6488        // TEST START: Full SelfHealingConfig serialization/deserialization
6489        let config = SelfHealingConfig {
6490            hook_starts_daemon: false,
6491            daemon_installs_hooks: false,
6492            auto_start_cooldown_secs: 60,
6493            auto_start_timeout_secs: 10,
6494            self_healing_log_level: SelfHealingLogLevel::Debug,
6495        };
6496
6497        let json = serde_json::to_string(&config).unwrap();
6498        assert!(json.contains("\"hook_starts_daemon\":false"));
6499        assert!(json.contains("\"daemon_installs_hooks\":false"));
6500        assert!(json.contains("\"auto_start_cooldown_secs\":60"));
6501        assert!(json.contains("\"auto_start_timeout_secs\":10"));
6502        assert!(json.contains("\"self_healing_log_level\":\"debug\""));
6503
6504        let parsed: SelfHealingConfig = serde_json::from_str(&json).unwrap();
6505        assert!(!parsed.hook_starts_daemon);
6506        assert!(!parsed.daemon_installs_hooks);
6507        assert_eq!(parsed.auto_start_cooldown_secs, 60);
6508        assert_eq!(parsed.auto_start_timeout_secs, 10);
6509        assert_eq!(parsed.self_healing_log_level, SelfHealingLogLevel::Debug);
6510        // TEST PASS: Full SelfHealingConfig serde
6511    }
6512
6513    // br-4zf3p: Tests for the self_healing_log_level field.
6514
6515    #[test]
6516    fn test_self_healing_log_level_default_is_info() {
6517        // TEST START: SelfHealingLogLevel::default() == Info
6518        let level = SelfHealingLogLevel::default();
6519        assert_eq!(level, SelfHealingLogLevel::Info);
6520        // Also verify the SelfHealingConfig default surfaces it.
6521        let config = SelfHealingConfig::default();
6522        assert_eq!(
6523            config.self_healing_log_level,
6524            SelfHealingLogLevel::Info,
6525            "config default should be Info"
6526        );
6527        // TEST PASS
6528    }
6529
6530    #[test]
6531    fn test_self_healing_log_level_serializes_lowercase() {
6532        // TEST START: serde uses lowercase variants
6533        let json = serde_json::to_string(&SelfHealingLogLevel::Warn).unwrap();
6534        assert_eq!(json, "\"warn\"");
6535        let parsed: SelfHealingLogLevel = serde_json::from_str("\"error\"").unwrap();
6536        assert_eq!(parsed, SelfHealingLogLevel::Error);
6537        // TEST PASS
6538    }
6539
6540    #[test]
6541    fn test_self_healing_log_level_from_env_str_accepts_known() {
6542        // TEST START: from_env_str parses each level (case-insensitive) and aliases
6543        assert_eq!(
6544            SelfHealingLogLevel::from_env_str("debug"),
6545            Some(SelfHealingLogLevel::Debug)
6546        );
6547        assert_eq!(
6548            SelfHealingLogLevel::from_env_str("DEBUG"),
6549            Some(SelfHealingLogLevel::Debug)
6550        );
6551        assert_eq!(
6552            SelfHealingLogLevel::from_env_str("info"),
6553            Some(SelfHealingLogLevel::Info)
6554        );
6555        assert_eq!(
6556            SelfHealingLogLevel::from_env_str("warn"),
6557            Some(SelfHealingLogLevel::Warn)
6558        );
6559        assert_eq!(
6560            SelfHealingLogLevel::from_env_str("warning"),
6561            Some(SelfHealingLogLevel::Warn),
6562            "warning should map to Warn"
6563        );
6564        assert_eq!(
6565            SelfHealingLogLevel::from_env_str("error"),
6566            Some(SelfHealingLogLevel::Error)
6567        );
6568        // Whitespace tolerance
6569        assert_eq!(
6570            SelfHealingLogLevel::from_env_str("  debug  "),
6571            Some(SelfHealingLogLevel::Debug)
6572        );
6573        // TEST PASS
6574    }
6575
6576    #[test]
6577    fn test_self_healing_log_level_from_env_str_rejects_unknown() {
6578        // TEST START: unknown values return None (caller falls back to default)
6579        assert!(SelfHealingLogLevel::from_env_str("banana").is_none());
6580        assert!(SelfHealingLogLevel::from_env_str("").is_none());
6581        assert!(SelfHealingLogLevel::from_env_str("trace").is_none());
6582        // TEST PASS
6583    }
6584
6585    #[test]
6586    fn test_self_healing_config_serde_partial_uses_defaults() {
6587        let _guard = test_guard!();
6588        // TEST START: Partial TOML/JSON uses defaults for missing fields
6589        let json = r#"{"hook_starts_daemon": false}"#;
6590        let config: SelfHealingConfig = serde_json::from_str(json).unwrap();
6591
6592        assert!(
6593            !config.hook_starts_daemon,
6594            "Explicit false should be parsed"
6595        );
6596        assert!(
6597            config.daemon_installs_hooks,
6598            "Missing field should use default (true)"
6599        );
6600        assert_eq!(
6601            config.auto_start_cooldown_secs, 30,
6602            "Missing field should use default (30)"
6603        );
6604        assert_eq!(
6605            config.auto_start_timeout_secs, 3,
6606            "Missing field should use default (3)"
6607        );
6608        // TEST PASS: Partial SelfHealingConfig uses defaults
6609    }
6610
6611    #[test]
6612    fn test_self_healing_config_toml_with_alias() {
6613        let _guard = test_guard!();
6614        // TEST START: daemon_start_timeout alias works for auto_start_timeout_secs
6615        let toml_str = r#"
6616            hook_starts_daemon = true
6617            daemon_installs_hooks = false
6618            auto_start_cooldown_secs = 45
6619            daemon_start_timeout = 7
6620        "#;
6621
6622        let config: SelfHealingConfig = toml::from_str(toml_str).unwrap();
6623        assert!(config.hook_starts_daemon);
6624        assert!(!config.daemon_installs_hooks);
6625        assert_eq!(config.auto_start_cooldown_secs, 45);
6626        assert_eq!(
6627            config.auto_start_timeout_secs, 7,
6628            "daemon_start_timeout alias should set auto_start_timeout_secs"
6629        );
6630        // TEST PASS: TOML alias works
6631    }
6632
6633    #[allow(unsafe_code)]
6634    mod self_healing_env_override_tests {
6635        use super::*;
6636        use crate::config::env_test_lock;
6637
6638        fn env_guard() -> std::sync::MutexGuard<'static, ()> {
6639            env_test_lock()
6640        }
6641
6642        fn set_env(key: &str, value: &str) {
6643            // SAFETY: Tests are serialized with env_guard().
6644            unsafe { std::env::set_var(key, value) };
6645        }
6646
6647        fn remove_env(key: &str) {
6648            // SAFETY: Tests are serialized with env_guard().
6649            unsafe { std::env::remove_var(key) };
6650        }
6651
6652        struct EnvVarGuard {
6653            key: &'static str,
6654            old: Option<String>,
6655        }
6656
6657        impl EnvVarGuard {
6658            fn set(key: &'static str, value: &str) -> Self {
6659                let old = std::env::var(key).ok();
6660                set_env(key, value);
6661                Self { key, old }
6662            }
6663        }
6664
6665        impl Drop for EnvVarGuard {
6666            fn drop(&mut self) {
6667                if let Some(old) = &self.old {
6668                    set_env(self.key, old);
6669                } else {
6670                    remove_env(self.key);
6671                }
6672            }
6673        }
6674
6675        #[test]
6676        fn test_self_healing_config_with_env_overrides_master_disable() {
6677            let _guard = test_guard!();
6678            let _guard = env_guard();
6679
6680            let _no_self_healing = EnvVarGuard::set("RCH_NO_SELF_HEALING", "1");
6681            let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "1");
6682            let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "1");
6683            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "99");
6684            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "99");
6685
6686            let config = SelfHealingConfig::default().with_env_overrides();
6687            assert!(
6688                !config.hook_starts_daemon,
6689                "RCH_NO_SELF_HEALING should disable hook auto-start"
6690            );
6691            assert!(
6692                !config.daemon_installs_hooks,
6693                "RCH_NO_SELF_HEALING should disable daemon hook installation"
6694            );
6695
6696            // Master disable returns early; numeric overrides should not apply.
6697            assert_eq!(config.auto_start_cooldown_secs, 30);
6698            assert_eq!(config.auto_start_timeout_secs, 3);
6699        }
6700
6701        #[test]
6702        fn test_self_healing_config_with_env_overrides_toggles_and_numbers() {
6703            let _guard = test_guard!();
6704            let _guard = env_guard();
6705
6706            let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "0");
6707            let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "false");
6708            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "45");
6709            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "7");
6710
6711            let config = SelfHealingConfig::default().with_env_overrides();
6712            assert!(!config.hook_starts_daemon);
6713            assert!(!config.daemon_installs_hooks);
6714            assert_eq!(config.auto_start_cooldown_secs, 45);
6715            assert_eq!(config.auto_start_timeout_secs, 7);
6716        }
6717
6718        #[test]
6719        fn test_self_healing_config_with_env_overrides_invalid_numbers_ignored() {
6720            let _guard = test_guard!();
6721            let _guard = env_guard();
6722
6723            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "not-a-number");
6724            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "nope");
6725
6726            let config = SelfHealingConfig::default().with_env_overrides();
6727            assert_eq!(config.auto_start_cooldown_secs, 30);
6728            assert_eq!(config.auto_start_timeout_secs, 3);
6729        }
6730    }
6731
6732    // -------------------------------------------------------------------------
6733    // Preflight Health Guard Tests (bd-3eaa)
6734    // -------------------------------------------------------------------------
6735
6736    #[test]
6737    fn test_load_per_core_calculation() {
6738        let _guard = test_guard!();
6739        let mut caps = WorkerCapabilities::new();
6740        // No metrics -> None
6741        assert!(caps.load_per_core().is_none());
6742
6743        // Only load, no cpus -> None
6744        caps.load_avg_1 = Some(4.0);
6745        assert!(caps.load_per_core().is_none());
6746
6747        // Both available
6748        caps.num_cpus = Some(4);
6749        assert_eq!(caps.load_per_core(), Some(1.0)); // 4.0 / 4 = 1.0
6750
6751        // High load
6752        caps.load_avg_1 = Some(16.0);
6753        assert_eq!(caps.load_per_core(), Some(4.0)); // 16.0 / 4 = 4.0
6754    }
6755
6756    #[test]
6757    fn test_is_high_load() {
6758        let _guard = test_guard!();
6759        let mut caps = WorkerCapabilities::new();
6760        caps.load_avg_1 = Some(8.0);
6761        caps.num_cpus = Some(4);
6762        // load_per_core = 2.0
6763
6764        // Below threshold
6765        assert_eq!(caps.is_high_load(3.0), Some(false));
6766        // At threshold
6767        assert_eq!(caps.is_high_load(2.0), Some(false)); // equal is OK
6768        // Above threshold
6769        assert_eq!(caps.is_high_load(1.5), Some(true));
6770
6771        // No metrics -> None (fail-open)
6772        caps.load_avg_1 = None;
6773        assert!(caps.is_high_load(1.0).is_none());
6774    }
6775
6776    #[test]
6777    fn test_is_low_disk() {
6778        let _guard = test_guard!();
6779        let mut caps = WorkerCapabilities::new();
6780        caps.disk_free_gb = Some(15.0);
6781
6782        // Above threshold
6783        assert_eq!(caps.is_low_disk(10.0), Some(false));
6784        // At threshold
6785        assert_eq!(caps.is_low_disk(15.0), Some(false)); // equal is OK
6786        // Below threshold
6787        assert_eq!(caps.is_low_disk(20.0), Some(true));
6788
6789        // No metrics -> None (fail-open)
6790        caps.disk_free_gb = None;
6791        assert!(caps.is_low_disk(10.0).is_none());
6792    }
6793
6794    #[test]
6795    fn low_disk_judges_the_build_disk_not_a_small_tmpfs() {
6796        let _guard = test_guard!();
6797        // GH #78: a 16 GB tmpfs /tmp is the tightest mount by ratio, but the
6798        // build trees live on a roomy data disk.
6799        let mut caps = WorkerCapabilities::new();
6800        caps.disk_free_gb = Some(8.0);
6801        caps.disk_total_gb = Some(15.5);
6802        caps.build_disk_free_gb = Some(1700.0);
6803        caps.build_disk_total_gb = Some(1900.0);
6804        assert_eq!(caps.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6805        assert_eq!(caps.is_low_disk(10.0), Some(false));
6806
6807        // A full build disk is low even when /tmp is roomy.
6808        caps.build_disk_free_gb = Some(4.0);
6809        assert_eq!(caps.is_low_disk(10.0), Some(true));
6810
6811        // An older rch-wkr without the build sample keeps the old behaviour.
6812        caps.build_disk_free_gb = None;
6813        caps.build_disk_total_gb = None;
6814        assert_eq!(caps.build_disk_gb(), (Some(8.0), Some(15.5)));
6815        assert_eq!(caps.is_low_disk(10.0), Some(true));
6816    }
6817
6818    #[test]
6819    fn build_disk_sample_is_optional_on_the_wire() {
6820        let _guard = test_guard!();
6821        let old: WorkerCapabilities =
6822            serde_json::from_str(r#"{"disk_free_gb":13.9,"disk_total_gb":15.5}"#).unwrap();
6823        assert_eq!(old.build_disk_free_gb, None);
6824        assert_eq!(old.build_disk_gb(), (Some(13.9), Some(15.5)));
6825
6826        let new: WorkerCapabilities = serde_json::from_str(
6827            r#"{"disk_free_gb":13.9,"disk_total_gb":15.5,"build_disk_free_gb":1700.0,"build_disk_total_gb":1900.0}"#,
6828        )
6829        .unwrap();
6830        assert_eq!(new.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6831        let json = serde_json::to_string(&new).unwrap();
6832        assert!(json.contains("\"build_disk_free_gb\":1700.0"), "{json}");
6833    }
6834
6835    #[test]
6836    fn test_selection_config_preflight_defaults() {
6837        let _guard = test_guard!();
6838        let config = SelectionConfig::default();
6839        // Default thresholds
6840        assert_eq!(config.max_load_per_core, Some(2.0));
6841        assert_eq!(config.min_free_gb, Some(10.0));
6842        assert_eq!(config.disk_gb_per_slot, 10.0);
6843    }
6844
6845    #[test]
6846    fn test_selection_config_preflight_serde() {
6847        let _guard = test_guard!();
6848        let config = SelectionConfig {
6849            max_load_per_core: Some(3.5),
6850            min_free_gb: Some(25.0),
6851            disk_gb_per_slot: 0.83,
6852            ..Default::default()
6853        };
6854        let json = serde_json::to_string(&config).unwrap();
6855        let parsed: SelectionConfig = serde_json::from_str(&json).unwrap();
6856        assert_eq!(parsed.max_load_per_core, Some(3.5));
6857        assert_eq!(parsed.min_free_gb, Some(25.0));
6858        assert_eq!(parsed.disk_gb_per_slot, 0.83);
6859    }
6860
6861    #[test]
6862    fn test_selection_disk_slot_budget_validation() {
6863        let _guard = test_guard!();
6864        for value in ["0.0", "-1.0", "nan", "inf", "-inf"] {
6865            let result = toml::from_str::<SelectionConfig>(&format!("disk_gb_per_slot = {value}"));
6866            assert!(result.is_err(), "invalid disk slot budget {value}");
6867        }
6868        let default: SelectionConfig = toml::from_str("").unwrap();
6869        assert_eq!(default.disk_gb_per_slot, 10.0);
6870    }
6871
6872    #[test]
6873    fn test_selection_disk_weight_validation_and_roundtrip() {
6874        for value in ["-0.1", "1.1", "nan", "inf", "-inf"] {
6875            assert!(toml::from_str::<SelectionWeightConfig>(&format!("disk = {value}")).is_err());
6876        }
6877        for value in [0.0, 0.2, 1.0] {
6878            let weights: SelectionWeightConfig =
6879                toml::from_str(&format!("disk = {value}")).unwrap();
6880            assert_eq!(weights.disk, value);
6881            let roundtrip: SelectionWeightConfig =
6882                serde_json::from_str(&serde_json::to_string(&weights).unwrap()).unwrap();
6883            assert_eq!(roundtrip.disk, value);
6884        }
6885        let defaults: SelectionWeightConfig = toml::from_str("").unwrap();
6886        assert_eq!(defaults.disk, 0.2);
6887    }
6888
6889    #[test]
6890    fn test_worker_capabilities_health_metrics_serde() {
6891        let _guard = test_guard!();
6892        let caps = WorkerCapabilities {
6893            num_cpus: Some(8),
6894            load_avg_1: Some(1.5),
6895            load_avg_5: Some(2.0),
6896            load_avg_15: Some(1.8),
6897            disk_free_gb: Some(50.5),
6898            disk_total_gb: Some(100.0),
6899            ..Default::default()
6900        };
6901        let json = serde_json::to_string(&caps).unwrap();
6902        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
6903        assert_eq!(parsed.num_cpus, Some(8));
6904        assert_eq!(parsed.load_avg_1, Some(1.5));
6905        assert_eq!(parsed.disk_free_gb, Some(50.5));
6906    }
6907
6908    #[test]
6909    fn microarch_level_classifies_ivy_bridge_as_v2() {
6910        // Abbreviated real flag set from ovh-b (Xeon E3-1245 V2, Ivy Bridge):
6911        // AVX present, but no avx2/bmi/fma — the bd-68hon SIGILL class.
6912        let ivy = "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov \
6913                   pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx rdtscp lm \
6914                   constant_tsc pni pclmulqdq ssse3 cx16 sse4_1 sse4_2 x2apic \
6915                   popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm";
6916        assert_eq!(x86_64_microarch_level_from_flags(ivy), 2);
6917    }
6918
6919    #[test]
6920    fn microarch_level_classifies_v3_and_v4() {
6921        let v3 = "cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 abm avx avx2 bmi1 bmi2 \
6922                  f16c fma movbe xsave";
6923        assert_eq!(x86_64_microarch_level_from_flags(v3), 3);
6924
6925        let v4 = format!("{v3} avx512f avx512bw avx512cd avx512dq avx512vl");
6926        assert_eq!(x86_64_microarch_level_from_flags(&v4), 4);
6927    }
6928
6929    #[test]
6930    fn microarch_level_v1_for_ancient_or_empty_flags() {
6931        assert_eq!(x86_64_microarch_level_from_flags(""), 1);
6932        assert_eq!(x86_64_microarch_level_from_flags("fpu mmx sse sse2"), 1);
6933    }
6934
6935    #[test]
6936    fn pre_v3_predicate_only_fires_on_affirmative_evidence() {
6937        let mut caps = WorkerCapabilities::new();
6938        assert!(!caps.is_pre_v3_x86(), "unknown must not deprioritize");
6939        caps.cpu_microarch_level = Some(2);
6940        assert!(caps.is_pre_v3_x86());
6941        caps.cpu_microarch_level = Some(3);
6942        assert!(!caps.is_pre_v3_x86());
6943    }
6944}