Skip to main content

rch_common/
types.rs

1//! Common types used across RCH components.
2
3use crate::{CompilationKind, toolchain::ToolchainInfo};
4use rand::RngExt;
5use schemars::JsonSchema;
6use serde::{Deserialize, Serialize};
7use std::path::PathBuf;
8
9/// Unique identifier for a worker in the fleet.
10#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
11pub struct WorkerId(pub String);
12
13impl WorkerId {
14    pub fn new(id: impl Into<String>) -> Self {
15        Self(id.into())
16    }
17
18    pub fn as_str(&self) -> &str {
19        &self.0
20    }
21}
22
23impl std::fmt::Display for WorkerId {
24    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
25        write!(f, "{}", self.0)
26    }
27}
28
29/// Status of a worker in the fleet.
30///
31/// State transitions:
32/// ```text
33/// HEALTHY ←→ DEGRADED (automatic based on response times)
34///    ↓           ↓
35///    ↓       UNREACHABLE (automatic on heartbeat failure)
36///    ↓
37/// DRAINING (via `rch workers drain`) - finishing current jobs
38///    ↓
39/// DRAINED (automatic when all jobs complete) - idle, ready to disable
40///    ↓
41/// DISABLED (via `rch workers disable`) - completely offline
42///
43/// Use `rch workers enable` to return from DRAINING/DRAINED/DISABLED → HEALTHY
44/// ```
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
46#[serde(rename_all = "snake_case")]
47pub enum WorkerStatus {
48    /// Worker is healthy and accepting jobs.
49    #[default]
50    Healthy,
51    /// Worker is responding slowly.
52    Degraded,
53    /// Worker failed to respond to heartbeat.
54    Unreachable,
55    /// Worker is not accepting new jobs (finishing current).
56    Draining,
57    /// Worker has finished draining (no active jobs, idle).
58    Drained,
59    /// Worker is manually disabled.
60    Disabled,
61}
62
63/// Circuit breaker state for a worker.
64#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
65#[serde(rename_all = "snake_case")]
66pub enum CircuitState {
67    /// Normal operation.
68    #[default]
69    Closed,
70    /// Circuit is open (short-circuit).
71    Open,
72    /// Circuit is half-open (probing).
73    HalfOpen,
74}
75
76/// Required runtime for command execution.
77#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
78#[serde(rename_all = "lowercase")]
79pub enum RequiredRuntime {
80    /// No specific runtime required (default).
81    #[default]
82    None,
83    /// Requires Rust toolchain.
84    Rust,
85    /// Requires Bun runtime.
86    Bun,
87    /// Requires Node.js runtime.
88    Node,
89    /// Requires the Nix package manager (nix binary + populated `/nix/store`).
90    Nix,
91    /// Requires the Go toolchain (`go` binary).
92    Go,
93    /// Requires the zig cross-compilation toolchain (`cargo-zigbuild` + `zig`).
94    Zig,
95}
96
97/// Per-command priority hint for worker selection.
98///
99/// This is an *input hint* from the caller (typically the hook) and should not
100/// change default behavior when omitted.
101#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
102#[serde(rename_all = "lowercase")]
103pub enum CommandPriority {
104    Low,
105    #[default]
106    Normal,
107    High,
108}
109
110impl std::fmt::Display for CommandPriority {
111    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112        let value = match self {
113            Self::Low => "low",
114            Self::Normal => "normal",
115            Self::High => "high",
116        };
117        write!(f, "{}", value)
118    }
119}
120
121impl std::str::FromStr for CommandPriority {
122    type Err = ();
123
124    fn from_str(s: &str) -> Result<Self, Self::Err> {
125        match s.trim().to_lowercase().as_str() {
126            "low" => Ok(Self::Low),
127            "normal" => Ok(Self::Normal),
128            "high" => Ok(Self::High),
129            _ => Err(()),
130        }
131    }
132}
133
134/// Worker selection request sent from hook to daemon.
135#[derive(Debug, Clone, Serialize, Deserialize)]
136pub struct SelectionRequest {
137    /// Project identifier (usually directory name or hash).
138    pub project: String,
139    /// Full command being executed (optional, for active build tracking).
140    #[serde(default, skip_serializing_if = "Option::is_none")]
141    pub command: Option<String>,
142    /// Priority hint for this request.
143    #[serde(default)]
144    pub command_priority: CommandPriority,
145    /// Estimated CPU cores needed for this compilation.
146    pub estimated_cores: u32,
147    /// Preferred worker IDs (e.g., from project config).
148    #[serde(default)]
149    pub preferred_workers: Vec<WorkerId>,
150    /// Rust toolchain information for the project.
151    #[serde(default)]
152    pub toolchain: Option<ToolchainInfo>,
153    /// Required runtime for command execution.
154    #[serde(default)]
155    pub required_runtime: RequiredRuntime,
156    /// Classification decision latency in microseconds (for AGENTS.md compliance).
157    /// This tracks how long the 5-tier classification took on the hook side.
158    #[serde(default, skip_serializing_if = "Option::is_none")]
159    pub classification_duration_us: Option<u64>,
160    /// Job-mode admission (bd-g7rpy): set by `rch exec --job` only. When the
161    /// one-active-job-per-project-per-worker guard excludes every otherwise-
162    /// eligible worker, a job request may QUEUE on that exclusion instead of
163    /// failing over to local execution. Compilation requests never set it,
164    /// preserving their immediate NoAdmissibleWorkers behavior.
165    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
166    pub job_mode: bool,
167    /// Operator-declared named tools this work requires VERIFIED on the worker
168    /// (`rch exec --job --require-tool NAME`, or a project's
169    /// `jobs.required_tools`).
170    ///
171    /// Job mode only — the compilation classifier never sets it, so ordinary
172    /// offloaded builds keep their existing admission behavior exactly. Empty
173    /// means no gate; a name no worker has verified excludes every worker,
174    /// because the caller asked for the gate precisely because the job cannot
175    /// run without the tool.
176    #[serde(default, skip_serializing_if = "Vec::is_empty")]
177    pub required_tools: Vec<String>,
178    /// Process ID of the hook (for active build tracking).
179    #[serde(default, skip_serializing_if = "Option::is_none")]
180    pub hook_pid: Option<u32>,
181}
182
183/// Reason for worker selection result.
184///
185/// Provides context when no worker is available, enabling informative
186/// fallback messages in the hook.
187#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
188#[serde(rename_all = "snake_case")]
189pub enum SelectionReason {
190    /// Worker assigned successfully.
191    Success,
192    /// No workers configured in workers.toml.
193    NoWorkersConfigured,
194    /// All workers are unreachable (failed health checks).
195    AllWorkersUnreachable,
196    /// All workers have circuits open (after repeated failures).
197    AllCircuitsOpen,
198    /// All workers are at capacity (no available slots).
199    AllWorkersBusy,
200    /// Workers were present but none passed selection health thresholds.
201    NoWorkersPassedHealth,
202    /// All candidate workers failed hard preflight checks.
203    AllWorkersFailedPreflight,
204    /// All candidate workers failed repo convergence checks (repos missing/stale/failed).
205    AllWorkersFailedConvergence,
206    /// Workers exist but admission was blocked by concrete capacity/preflight reasons.
207    NoAdmissibleWorkers(String),
208    /// No workers match required tags or preferences.
209    NoMatchingWorkers,
210    /// No workers have the required runtime (e.g., Bun, Node).
211    NoWorkersWithRuntime(String),
212    /// Internal error during selection.
213    SelectionError(String),
214    /// Worker assigned via affinity pinning (recent successful build).
215    AffinityPinned,
216    /// Worker assigned via last-success fallback (all others unavailable).
217    AffinityFallback,
218}
219
220/// Wire protocol version for daemon `/select-worker` responses.
221///
222/// This is intentionally separate from the broader CLI API envelope version:
223/// selection responses are emitted as a raw [`SelectionResponse`] on the hook
224/// hot path, so the client needs a small explicit compatibility marker.
225pub const SELECTION_RESPONSE_PROTOCOL_VERSION: u64 = 1;
226
227impl std::fmt::Display for SelectionReason {
228    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
229        match self {
230            Self::Success => write!(f, "worker assigned successfully"),
231            Self::NoWorkersConfigured => write!(f, "no workers configured"),
232            Self::AllWorkersUnreachable => write!(f, "all workers unreachable"),
233            Self::AllCircuitsOpen => write!(f, "all worker circuits open"),
234            Self::AllWorkersBusy => write!(f, "all workers at capacity"),
235            Self::NoWorkersPassedHealth => write!(f, "no workers passed health thresholds"),
236            Self::AllWorkersFailedPreflight => write!(f, "all workers failed preflight checks"),
237            Self::AllWorkersFailedConvergence => {
238                write!(f, "all workers failed repo convergence checks")
239            }
240            Self::NoAdmissibleWorkers(summary) => {
241                write!(f, "no admissible workers: {}", summary)
242            }
243            Self::NoMatchingWorkers => write!(f, "no matching workers found"),
244            Self::NoWorkersWithRuntime(rt) => write!(f, "no workers with {} installed", rt),
245            Self::SelectionError(e) => write!(f, "selection error: {}", e),
246            Self::AffinityPinned => write!(f, "worker assigned via affinity pinning"),
247            Self::AffinityFallback => write!(f, "worker assigned via last-success fallback"),
248        }
249    }
250}
251
252// ============================================================================
253// Worker Selection Strategy
254// ============================================================================
255
256/// Worker selection strategy determining how workers are chosen for jobs.
257#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
258#[serde(rename_all = "snake_case")]
259pub enum SelectionStrategy {
260    /// Legacy/manual control: prioritize worker.priority first, then break ties
261    /// using command-aware cache/speed hints.
262    Priority,
263    /// Select worker with highest SpeedScore.
264    /// Best for performance-critical builds with homogeneous workers.
265    Fastest,
266    /// Balance all factors: SpeedScore, load, health, cache affinity.
267    /// Default for general use across diverse worker pools.
268    #[default]
269    Balanced,
270    /// Prefer workers with warm caches for the project.
271    /// Best for incremental builds on large codebases.
272    CacheAffinity,
273    /// Weighted random selection favoring fast workers but ensuring fairness.
274    /// Prevents hot-spotting when many workers are available.
275    FairFastest,
276}
277
278impl std::fmt::Display for SelectionStrategy {
279    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
280        let name = match self {
281            Self::Priority => "priority",
282            Self::Fastest => "fastest",
283            Self::Balanced => "balanced",
284            Self::CacheAffinity => "cache_affinity",
285            Self::FairFastest => "fair_fastest",
286        };
287        write!(f, "{}", name)
288    }
289}
290
291impl std::str::FromStr for SelectionStrategy {
292    type Err = String;
293
294    fn from_str(s: &str) -> Result<Self, Self::Err> {
295        match s.trim().to_lowercase().as_str() {
296            "priority" => Ok(Self::Priority),
297            "fastest" => Ok(Self::Fastest),
298            "balanced" => Ok(Self::Balanced),
299            "cache_affinity" | "cache-affinity" | "cacheaffinity" => Ok(Self::CacheAffinity),
300            "fair_fastest" | "fair-fastest" | "fairfastest" => Ok(Self::FairFastest),
301            _ => Err(format!(
302                "unknown selection strategy '{}', expected one of: priority, fastest, balanced, cache_affinity, fair_fastest",
303                s
304            )),
305        }
306    }
307}
308
309/// Configuration for the worker selection algorithm.
310#[derive(Debug, Clone, Serialize, Deserialize)]
311pub struct SelectionConfig {
312    /// Selection strategy to use.
313    #[serde(default)]
314    pub strategy: SelectionStrategy,
315    /// Minimum success rate (0.0-1.0) for a worker to be considered.
316    #[serde(default = "default_min_success_rate")]
317    pub min_success_rate: f64,
318    /// Factor weights for the balanced strategy.
319    #[serde(default)]
320    pub weights: SelectionWeightConfig,
321    /// Fairness settings for fair_fastest strategy.
322    #[serde(default)]
323    pub fairness: FairnessConfig,
324    /// Cache affinity settings for project-to-worker pinning.
325    #[serde(default)]
326    pub affinity: AffinityConfig,
327
328    // Preflight health guards (bd-3eaa)
329    /// Maximum load-per-core threshold. Workers exceeding this are skipped.
330    /// Set to None to disable load-based filtering.
331    #[serde(default = "default_max_load_per_core")]
332    pub max_load_per_core: Option<f64>,
333    /// Minimum free disk space in GB. Workers below this are skipped.
334    /// Set to None to disable the free-space floor; per-slot disk derating still applies.
335    #[serde(default = "default_min_free_gb")]
336    pub min_free_gb: Option<f64>,
337    /// Disk budget in GiB per concurrent slot, after reserving `min_free_gb`.
338    /// Must be finite and positive. Unknown disk telemetry preserves configured slots.
339    #[serde(
340        default = "default_disk_gb_per_slot",
341        deserialize_with = "deserialize_disk_gb_per_slot"
342    )]
343    pub disk_gb_per_slot: f64,
344}
345
346impl Default for SelectionConfig {
347    fn default() -> Self {
348        Self {
349            strategy: SelectionStrategy::default(),
350            min_success_rate: default_min_success_rate(),
351            weights: SelectionWeightConfig::default(),
352            fairness: FairnessConfig::default(),
353            affinity: AffinityConfig::default(),
354            max_load_per_core: default_max_load_per_core(),
355            min_free_gb: default_min_free_gb(),
356            disk_gb_per_slot: default_disk_gb_per_slot(),
357        }
358    }
359}
360
361fn default_min_success_rate() -> f64 {
362    0.8
363}
364
365fn default_max_load_per_core() -> Option<f64> {
366    Some(2.0) // Skip workers with load/core > 2.0
367}
368
369fn default_min_free_gb() -> Option<f64> {
370    Some(10.0) // Skip workers with < 10 GB free
371}
372
373fn default_disk_gb_per_slot() -> f64 {
374    10.0
375}
376
377fn deserialize_disk_gb_per_slot<'de, D>(deserializer: D) -> Result<f64, D::Error>
378where
379    D: serde::Deserializer<'de>,
380{
381    let budget = f64::deserialize(deserializer)?;
382    if budget.is_finite() && budget > 0.0 {
383        Ok(budget)
384    } else {
385        Err(serde::de::Error::custom(
386            "disk_gb_per_slot must be finite and greater than zero",
387        ))
388    }
389}
390
391/// Weight configuration for the balanced selection strategy.
392#[derive(Debug, Clone, Serialize, Deserialize)]
393pub struct SelectionWeightConfig {
394    /// Weight for SpeedScore (0.0-1.0).
395    #[serde(default = "default_weight_speedscore")]
396    pub speedscore: f64,
397    /// Weight for available slots (0.0-1.0).
398    #[serde(default = "default_weight_slots")]
399    pub slots: f64,
400    /// Weight for health/success rate (0.0-1.0).
401    #[serde(default = "default_weight_health")]
402    pub health: f64,
403    /// Weight for cache affinity (0.0-1.0).
404    #[serde(default = "default_weight_cache")]
405    pub cache: f64,
406    /// Weight for network latency (0.0-1.0).
407    #[serde(default = "default_weight_network")]
408    pub network: f64,
409    /// Weight for worker priority (0.0-1.0).
410    #[serde(default = "default_weight_priority")]
411    pub priority: f64,
412    /// Weight for disk headroom (0.0-1.0); zero disables disk ranking.
413    #[serde(
414        default = "default_weight_disk",
415        deserialize_with = "deserialize_weight_disk"
416    )]
417    pub disk: f64,
418    /// Penalty multiplier for half-open circuit workers (0.0-1.0).
419    #[serde(default = "default_half_open_penalty")]
420    pub half_open_penalty: f64,
421}
422
423impl Default for SelectionWeightConfig {
424    fn default() -> Self {
425        Self {
426            speedscore: default_weight_speedscore(),
427            slots: default_weight_slots(),
428            health: default_weight_health(),
429            cache: default_weight_cache(),
430            network: default_weight_network(),
431            priority: default_weight_priority(),
432            disk: default_weight_disk(),
433            half_open_penalty: default_half_open_penalty(),
434        }
435    }
436}
437
438fn default_weight_speedscore() -> f64 {
439    0.5
440}
441fn default_weight_slots() -> f64 {
442    0.4
443}
444fn default_weight_health() -> f64 {
445    0.3
446}
447fn default_weight_cache() -> f64 {
448    0.2
449}
450fn default_weight_network() -> f64 {
451    0.1
452}
453fn default_weight_priority() -> f64 {
454    // Worker priority is a first-class selection factor: operators set high
455    // priority on preferred boxes (e.g. fast dedicated workers) and expect them
456    // to actually receive work. At the old 0.1 a top-priority worker gained at
457    // most +0.1 to its balanced score — too weak to overcome a warm-cache or
458    // lower-latency competitor, so the highest-priority workers could sit idle
459    // indefinitely. Load/slots still throttle a high-priority worker as it fills.
460    0.5
461}
462fn default_half_open_penalty() -> f64 {
463    0.5
464}
465
466fn default_weight_disk() -> f64 {
467    0.2
468}
469
470fn deserialize_weight_disk<'de, D>(deserializer: D) -> Result<f64, D::Error>
471where
472    D: serde::Deserializer<'de>,
473{
474    let weight = f64::deserialize(deserializer)?;
475    if weight.is_finite() && (0.0..=1.0).contains(&weight) {
476        Ok(weight)
477    } else {
478        Err(serde::de::Error::custom(
479            "disk weight must be finite and between zero and one",
480        ))
481    }
482}
483
484/// Fairness settings for the fair_fastest selection strategy.
485#[derive(Debug, Clone, Serialize, Deserialize)]
486pub struct FairnessConfig {
487    /// Lookback window in seconds for tracking recent selections.
488    #[serde(default = "default_fairness_lookback_secs")]
489    pub lookback_secs: u64,
490    /// Maximum consecutive selections for a single worker before penalty.
491    #[serde(default = "default_max_consecutive_selections")]
492    pub max_consecutive_selections: u32,
493}
494
495impl Default for FairnessConfig {
496    fn default() -> Self {
497        Self {
498            lookback_secs: default_fairness_lookback_secs(),
499            max_consecutive_selections: default_max_consecutive_selections(),
500        }
501    }
502}
503
504fn default_fairness_lookback_secs() -> u64 {
505    300 // 5 minutes
506}
507
508fn default_max_consecutive_selections() -> u32 {
509    3
510}
511
512/// Cache affinity settings for project-to-worker pinning.
513#[derive(Debug, Clone, Serialize, Deserialize)]
514pub struct AffinityConfig {
515    /// Enable affinity pinning (pin projects to last successful worker).
516    #[serde(default = "default_affinity_enabled")]
517    pub enabled: bool,
518    /// Time window in minutes to pin a project to its last successful worker.
519    /// After this window expires, normal selection resumes.
520    #[serde(default = "default_affinity_pin_minutes")]
521    pub pin_minutes: u64,
522    /// Enable last-success fallback when no workers match selection criteria.
523    /// If enabled, the daemon will attempt the last successful worker for a project
524    /// when all other workers are unavailable or fail selection.
525    #[serde(default = "default_last_success_fallback")]
526    pub enable_last_success_fallback: bool,
527    /// Minimum success rate for the last-success worker to be used as fallback.
528    /// Must be lower than min_success_rate to be meaningful as a fallback.
529    #[serde(default = "default_fallback_min_success_rate")]
530    pub fallback_min_success_rate: f64,
531}
532
533impl Default for AffinityConfig {
534    fn default() -> Self {
535        Self {
536            enabled: default_affinity_enabled(),
537            pin_minutes: default_affinity_pin_minutes(),
538            enable_last_success_fallback: default_last_success_fallback(),
539            fallback_min_success_rate: default_fallback_min_success_rate(),
540        }
541    }
542}
543
544fn default_affinity_enabled() -> bool {
545    true
546}
547
548fn default_affinity_pin_minutes() -> u64 {
549    60 // 1 hour default
550}
551
552fn default_last_success_fallback() -> bool {
553    true
554}
555
556fn default_fallback_min_success_rate() -> f64 {
557    0.5 // More lenient than normal min_success_rate (0.8)
558}
559
560/// Details about a selected worker for remote compilation.
561#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
562pub struct SelectedWorker {
563    /// Selected worker ID.
564    pub id: WorkerId,
565    /// Host address for SSH.
566    pub host: String,
567    /// SSH user.
568    pub user: String,
569    /// Path to SSH identity file.
570    pub identity_file: String,
571    /// Number of slots available on this worker after reservation.
572    pub slots_available: u32,
573    /// Worker's speed score (0-100).
574    pub speed_score: f64,
575    /// Operator-declared host OS (e.g. `windows`), if any. Propagated so the
576    /// hook can pick the right transport (Windows workers use `C:/rch` +
577    /// tar-over-ssh instead of the Unix rsync path). `None` for the historical
578    /// unlabelled workers. `#[serde(default)]` keeps old daemons/hooks
579    /// wire-compatible.
580    #[serde(default)]
581    pub declared_os: Option<String>,
582}
583
584/// Worker selection response from daemon to hook.
585#[derive(Debug, Clone, Serialize, Deserialize)]
586pub struct SelectionResponse {
587    /// Selected worker details, if available.
588    pub worker: Option<SelectedWorker>,
589    /// Reason for the selection result.
590    pub reason: SelectionReason,
591    /// Optional build ID assigned by the daemon (for active build tracking).
592    #[serde(default, skip_serializing_if = "Option::is_none")]
593    pub build_id: Option<u64>,
594    /// Optional per-worker selector diagnostics for machine-readable triage.
595    #[serde(default, skip_serializing_if = "Option::is_none")]
596    pub diagnostics: Option<SelectionDiagnostics>,
597}
598
599/// Final selector decision for a worker considered during routing.
600#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
601#[serde(rename_all = "snake_case")]
602pub enum WorkerSelectionDiagnosticDecision {
603    /// Worker passed hard gates and normal health threshold.
604    Allow,
605    /// Worker failed normal health threshold but passed fallback threshold.
606    FallbackCandidate,
607    /// Worker was denied by a hard gate or both health thresholds.
608    Deny,
609}
610
611/// Per-worker selector facts captured at worker-selection time.
612#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
613pub struct WorkerSelectionDiagnostic {
614    /// Worker ID.
615    pub worker_id: WorkerId,
616    /// Worker lifecycle/status label.
617    pub status: String,
618    /// Circuit breaker state label.
619    pub circuit_state: String,
620    /// Current normalized pressure state.
621    pub pressure_state: String,
622    /// Stable pressure reason code.
623    pub pressure_reason_code: String,
624    /// Rolling selector success rate, where available.
625    #[serde(skip_serializing_if = "Option::is_none")]
626    pub success_rate: Option<f64>,
627    /// Normal selection success-rate threshold.
628    pub min_success_rate: f64,
629    /// Last-success fallback success-rate threshold.
630    pub fallback_min_success_rate: f64,
631    /// Required runtime for this command.
632    pub required_runtime: RequiredRuntime,
633    /// Whether this worker has the required runtime.
634    pub runtime_available: bool,
635    /// Available slots at selection time.
636    pub available_slots: u32,
637    /// Total configured slots.
638    pub total_slots: u32,
639    /// Estimated slots requested by this command.
640    pub estimated_cores: u32,
641    /// Whether this worker was excluded because it already runs this project.
642    pub active_project_excluded: bool,
643    /// Final per-worker selector decision.
644    pub final_decision: WorkerSelectionDiagnosticDecision,
645    /// Human-readable final reason.
646    pub final_reason: String,
647    /// Stable machine-readable reasons collected for this worker.
648    pub reason_codes: Vec<String>,
649}
650
651/// Machine-readable worker selector diagnostics for a selection request.
652#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
653pub struct SelectionDiagnostics {
654    /// Required runtime for this command.
655    pub required_runtime: RequiredRuntime,
656    /// Estimated slots requested by this command.
657    pub estimated_cores: u32,
658    /// Normal success-rate threshold.
659    pub min_success_rate: f64,
660    /// Last-success fallback success-rate threshold.
661    pub fallback_min_success_rate: f64,
662    /// Count of workers excluded because they already run this project.
663    pub active_project_exclusion_count: usize,
664    /// One diagnostic entry per configured worker.
665    pub workers: Vec<WorkerSelectionDiagnostic>,
666}
667
668/// Request to release reserved worker slots.
669#[derive(Debug, Clone, Serialize, Deserialize)]
670pub struct ReleaseRequest {
671    /// ID of the worker to release slots on.
672    pub worker_id: WorkerId,
673    /// Number of slots to release.
674    pub slots: u32,
675    /// Optional build ID to mark complete.
676    #[serde(default, skip_serializing_if = "Option::is_none")]
677    pub build_id: Option<u64>,
678    /// Exact client wrapper identity associated with the admitted build.
679    #[serde(default, skip_serializing_if = "Option::is_none")]
680    pub local_wrapper_id: Option<String>,
681    /// Optional exit code for the build (used to finalize active build tracking).
682    #[serde(default, skip_serializing_if = "Option::is_none")]
683    pub exit_code: Option<i32>,
684    /// Optional total duration in milliseconds (pipeline or execution).
685    ///
686    /// If omitted, the daemon will compute duration from the active build's start time.
687    #[serde(default, skip_serializing_if = "Option::is_none")]
688    pub duration_ms: Option<u64>,
689    /// Optional bytes transferred during the pipeline (upload + artifact download).
690    #[serde(default, skip_serializing_if = "Option::is_none")]
691    pub bytes_transferred: Option<u64>,
692    /// Optional per-phase timing breakdown for the build pipeline.
693    #[serde(default, skip_serializing_if = "Option::is_none")]
694    pub timing: Option<CommandTimingBreakdown>,
695}
696
697/// Build execution phase for daemon heartbeat tracking.
698#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
699#[serde(rename_all = "snake_case")]
700pub enum BuildHeartbeatPhase {
701    /// Uploading sources/dependencies to the worker.
702    SyncUp,
703    /// Running the remote compilation/test command.
704    Execute,
705    /// Downloading build artifacts from the worker.
706    SyncDown,
707    /// Build is completing or finalizing.
708    Finalize,
709}
710
711/// Request to update liveness/progress for an active build.
712#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
713pub struct BuildHeartbeatRequest {
714    /// Active build identifier assigned by the daemon.
715    pub build_id: u64,
716    /// Worker executing the build.
717    pub worker_id: WorkerId,
718    /// Hook process ID sending the heartbeat.
719    #[serde(default, skip_serializing_if = "Option::is_none")]
720    pub hook_pid: Option<u32>,
721    /// Client-minted wrapper identity for local durable-lease correlation.
722    #[serde(default, skip_serializing_if = "Option::is_none")]
723    pub local_wrapper_id: Option<String>,
724    /// Remote file containing the process-group leader PID for cancellation.
725    #[serde(default, skip_serializing_if = "Option::is_none")]
726    pub remote_pgid_file: Option<String>,
727    /// Current build execution phase.
728    pub phase: BuildHeartbeatPhase,
729    /// Optional human-readable progress detail.
730    #[serde(default, skip_serializing_if = "Option::is_none")]
731    pub detail: Option<String>,
732    /// Optional monotonic progress counter.
733    ///
734    /// The hook increments this when it observes meaningful progress
735    /// (phase transitions, new output, or richer progress signals).
736    #[serde(default, skip_serializing_if = "Option::is_none")]
737    pub progress_counter: Option<u64>,
738    /// Optional progress estimate in [0,100].
739    #[serde(default, skip_serializing_if = "Option::is_none")]
740    pub progress_percent: Option<f64>,
741}
742
743/// Configuration for a remote worker.
744#[derive(Debug, Clone, Serialize, Deserialize)]
745pub struct WorkerConfig {
746    /// Unique identifier for this worker.
747    pub id: WorkerId,
748    /// SSH hostname or IP address.
749    pub host: String,
750    /// SSH username.
751    pub user: String,
752    /// Path to SSH private key.
753    pub identity_file: String,
754    /// Total CPU slots available on this worker.
755    pub total_slots: u32,
756    /// Priority for worker selection (higher = preferred).
757    #[serde(default = "default_priority")]
758    pub priority: u32,
759    /// Optional tags for filtering.
760    ///
761    /// Mostly descriptive, with one reserved form: `os:<name>` (see
762    /// [`crate::declared_os`]) is a hard admission gate, written in
763    /// `workers.toml` as `os = "<name>"`.
764    #[serde(default)]
765    pub tags: Vec<String>,
766    /// Operator-declared named tool probes for this worker, written in
767    /// `workers.toml` as
768    /// `tools = [{ name = "clang", command = ["clang", "--version"] }]`.
769    ///
770    /// Unlike a tag, which is an unverified naming convention, each entry is
771    /// VERIFIED by running its fixed argv on the worker as the configured user;
772    /// only a zero exit makes the tool present. That is what lets
773    /// `--require-tool` gate selection on evidence instead of on a promise.
774    #[serde(default, skip_serializing_if = "Vec::is_empty")]
775    pub tools: Vec<WorkerToolProbe>,
776}
777
778/// One operator-declared tool probe as it appears in `workers.toml`.
779///
780/// Kept as plain configuration data; [`crate::capability_probe::NamedToolProbe`]
781/// is the validated form, and the conversion is where a malformed name is
782/// refused.
783#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
784pub struct WorkerToolProbe {
785    /// The name `--require-tool` and `jobs.required_tools` refer to.
786    pub name: String,
787    /// The exact argv to run. Zero exit = present; anything else = absent.
788    pub command: Vec<String>,
789}
790
791fn default_priority() -> u32 {
792    100
793}
794
795/// Tag prefix reserved for the operator-declared host OS.
796pub const OS_TAG_PREFIX: &str = "os:";
797
798/// The host OS an operator declared for a worker, from its tags.
799///
800/// Written in `workers.toml` as `os = "windows"` and normalized into the
801/// reserved `os:windows` tag on load, so it rides the existing tag vector
802/// through every path that already carries a [`WorkerConfig`].
803///
804/// Declaring it makes the worker **exclusive**: selection admits it only for
805/// commands that require that OS, so a Windows box cannot quietly absorb a
806/// `cargo check` dispatched from Linux and return wrong-platform artifacts.
807/// Workers without the tag — every historical worker — keep taking anything.
808#[must_use]
809pub fn declared_os(tags: &[String]) -> Option<String> {
810    // `find` over the mapped values, not `find_map` over the tags: a malformed
811    // `os:` entry must not mask a valid declaration later in the list, because
812    // failing open here is precisely the "Windows worker silently takes Linux
813    // jobs" outcome the gate exists to prevent.
814    tags.iter()
815        .filter_map(|tag| tag.strip_prefix(OS_TAG_PREFIX))
816        .map(|os| os.trim().to_ascii_lowercase())
817        .find(|os| !os.is_empty())
818}
819
820/// Build the reserved tag for a declared OS.
821#[must_use]
822pub fn os_tag(os: &str) -> String {
823    format!("{OS_TAG_PREFIX}{}", os.trim().to_ascii_lowercase())
824}
825
826impl Default for WorkerConfig {
827    fn default() -> Self {
828        Self {
829            id: WorkerId::new("default-worker"),
830            host: "localhost".to_string(),
831            user: "user".to_string(),
832            identity_file: "~/.ssh/id_rsa".to_string(),
833            total_slots: 4,
834            priority: default_priority(),
835            tags: Vec::new(),
836            tools: Vec::new(),
837        }
838    }
839}
840
841/// Runtime capabilities detected on a worker.
842///
843/// These are probed during health checks and cached for routing decisions.
844/// Commands requiring specific runtimes (e.g., `bun test`) can be routed
845/// only to workers with the corresponding capability.
846#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)]
847pub struct WorkerCapabilities {
848    /// Rust compiler version (from `rustc --version`).
849    #[serde(default, skip_serializing_if = "Option::is_none")]
850    pub rustc_version: Option<String>,
851    /// Installed rustup toolchains, including their host triple when rustup
852    /// reports one (for example `nightly-2026-07-05-x86_64-unknown-linux-gnu`).
853    #[serde(default, skip_serializing_if = "Vec::is_empty")]
854    pub rustup_toolchains: Vec<String>,
855    /// Installed rustup components as `<toolchain>:<component>` facts. Component
856    /// names are normalized by removing only the probed host-triple suffix, so
857    /// `clippy-preview` can never satisfy a `clippy` requirement.
858    #[serde(default, skip_serializing_if = "Vec::is_empty")]
859    pub rustup_components: Vec<String>,
860    /// Non-fatal diagnostics gathered while probing (for example a tool binary
861    /// that had to be resolved from a well-known fallback location because the
862    /// caller's PATH was minimal). Empty in the common case; clients may
863    /// surface these so "no facts" is distinguishable from "facts show
864    /// absence" (bd-deft5).
865    #[serde(default, skip_serializing_if = "Vec::is_empty")]
866    pub probe_warnings: Vec<String>,
867    /// Bun runtime version (from `bun --version`).
868    #[serde(default, skip_serializing_if = "Option::is_none")]
869    pub bun_version: Option<String>,
870    /// Node.js version (from `node --version`).
871    #[serde(default, skip_serializing_if = "Option::is_none")]
872    pub node_version: Option<String>,
873    /// npm version (from `npm --version`).
874    #[serde(default, skip_serializing_if = "Option::is_none")]
875    pub npm_version: Option<String>,
876    /// Nix version (from `nix --version`). Presence implies a usable nix binary
877    /// with an accompanying `/nix/store`, gating `nix build`/`nix develop -c`
878    /// routing to this worker.
879    #[serde(default, skip_serializing_if = "Option::is_none")]
880    pub nix_version: Option<String>,
881    /// Go toolchain version (from `go version`). Presence gates `go build`/
882    /// `go test`/`go vet` routing to this worker.
883    #[serde(default, skip_serializing_if = "Option::is_none")]
884    pub go_version: Option<String>,
885    /// Zig version (from `zig version`). `cargo zigbuild` shells out to `zig`
886    /// as the linker/C toolchain, so the bare `zig` binary is necessary but not
887    /// sufficient — see [`WorkerCapabilities::has_zig`].
888    #[serde(default, skip_serializing_if = "Option::is_none")]
889    pub zig_version: Option<String>,
890    /// cargo-zigbuild version (from `cargo-zigbuild --version`). Probed as the
891    /// hyphenated binary rather than `cargo zigbuild --version`, which the
892    /// subcommand's own parser rejects.
893    #[serde(default, skip_serializing_if = "Option::is_none")]
894    pub cargo_zigbuild_version: Option<String>,
895    /// x86-64 microarchitecture level (1..=4) parsed from `/proc/cpuinfo`
896    /// flags on the worker (bd-6qchz / bd-68hon item 4). `Some(2)` marks a
897    /// pre-AVX2 box (e.g. ovh-b's Ivy Bridge) whose build scripts SIGILL on
898    /// x86-64-v3 codegen; selection deprioritizes such workers proactively
899    /// while the reactive SIGILL quarantine remains the primary mechanism.
900    /// `None` on non-x86 workers or when the probe cannot read cpuinfo.
901    #[serde(default, skip_serializing_if = "Option::is_none")]
902    pub cpu_microarch_level: Option<u8>,
903    /// Operator-declared named tools whose probe exited ZERO on this worker.
904    ///
905    /// The list is evidence, not configuration: a name appears here only
906    /// because the worker ran the declared argv and it succeeded. Selection
907    /// gates `--require-tool` on this, which is the difference between a
908    /// verified capability and a tag somebody wrote by hand.
909    #[serde(default, skip_serializing_if = "Vec::is_empty")]
910    pub tools_present: Vec<String>,
911    /// Operator-declared named tools whose probe RAN and did not exit zero.
912    ///
913    /// Kept separate from "never declared" so an operator can tell a worker
914    /// that is missing a tool from a fleet that never probed for it — the two
915    /// have different fixes.
916    #[serde(default, skip_serializing_if = "Vec::is_empty")]
917    pub tools_absent: Vec<String>,
918
919    // Health metrics (bd-3eaa)
920    /// Number of CPU cores on the worker.
921    #[serde(default, skip_serializing_if = "Option::is_none")]
922    pub num_cpus: Option<u32>,
923    /// 1-minute load average.
924    #[serde(default, skip_serializing_if = "Option::is_none")]
925    pub load_avg_1: Option<f64>,
926    /// 5-minute load average.
927    #[serde(default, skip_serializing_if = "Option::is_none")]
928    pub load_avg_5: Option<f64>,
929    /// 15-minute load average.
930    #[serde(default, skip_serializing_if = "Option::is_none")]
931    pub load_avg_15: Option<f64>,
932    /// Free disk space in GB (on /tmp or build directory).
933    #[serde(default, skip_serializing_if = "Option::is_none")]
934    pub disk_free_gb: Option<f64>,
935    /// Total disk space in GB.
936    #[serde(default, skip_serializing_if = "Option::is_none")]
937    pub disk_total_gb: Option<f64>,
938    /// Canonical path-topology preflight status (`/data/projects` + `/dp` alias).
939    ///
940    /// `Some(false)` indicates a hard preflight failure that should exclude this
941    /// worker from remote scheduling until a subsequent successful revalidation.
942    #[serde(default, skip_serializing_if = "Option::is_none")]
943    pub projects_root_ok: Option<bool>,
944    /// Machine-readable failure reason for path-topology preflight.
945    #[serde(default, skip_serializing_if = "Option::is_none")]
946    pub projects_root_issue: Option<String>,
947    /// Unix timestamp (ms) when topology preflight last executed.
948    #[serde(default, skip_serializing_if = "Option::is_none")]
949    pub projects_root_checked_at_unix_ms: Option<i64>,
950}
951
952impl WorkerCapabilities {
953    /// Create new empty capabilities.
954    pub fn new() -> Self {
955        Self::default()
956    }
957
958    /// Create mock capabilities with Rust installed.
959    ///
960    /// Used by mock transport to simulate a worker with Rust toolchain.
961    pub fn mock_with_rust() -> Self {
962        Self {
963            rustc_version: Some("1.85.0-nightly (mock)".to_string()),
964            projects_root_ok: Some(true),
965            ..Self::default()
966        }
967    }
968
969    /// Whether this is a known pre-x86-64-v3 (no AVX2) x86 worker — the class
970    /// whose build-script/proc-macro binaries SIGILL on v3 codegen (bd-68hon).
971    /// `None` (unknown/non-x86) is NOT pre-v3: only affirmative evidence
972    /// deprioritizes a worker.
973    pub fn is_pre_v3_x86(&self) -> bool {
974        matches!(self.cpu_microarch_level, Some(1 | 2))
975    }
976
977    /// Check if this worker has Bun installed.
978    pub fn has_bun(&self) -> bool {
979        self.bun_version.is_some()
980    }
981
982    /// Check if this worker has Node.js installed.
983    pub fn has_node(&self) -> bool {
984        self.node_version.is_some()
985    }
986
987    /// Check if this worker has Rust installed.
988    pub fn has_rust(&self) -> bool {
989        self.rustc_version.is_some()
990    }
991
992    /// Check whether `component` is installed for `toolchain`.
993    ///
994    /// Rustup reports pinned toolchains with a host-triple suffix. The request
995    /// carries the project pin without that suffix, so the owner half uses the
996    /// same exact-or-`-`-boundary prefix rule as toolchain admission. The
997    /// component half is exact.
998    pub fn has_rustup_component(&self, toolchain: &str, component: &str) -> bool {
999        let owner_prefix = format!("{toolchain}-");
1000        self.rustup_components.iter().any(|fact| {
1001            fact.split_once(':').is_some_and(|(owner, name)| {
1002                name == component && (owner == toolchain || owner.starts_with(&owner_prefix))
1003            })
1004        })
1005    }
1006
1007    /// Check if this worker has Nix installed (nix binary + `/nix/store`).
1008    pub fn has_nix(&self) -> bool {
1009        self.nix_version.is_some()
1010    }
1011
1012    /// Check if this worker has the Go toolchain installed.
1013    pub fn has_go(&self) -> bool {
1014        self.go_version.is_some()
1015    }
1016
1017    /// Check if this worker can run `cargo zigbuild` cross-compiles.
1018    ///
1019    /// Requires BOTH halves of the toolchain: the `cargo-zigbuild` cargo
1020    /// subcommand and the `zig` binary it drives as the linker. Either one
1021    /// alone fails at build time with an error the remote-result heuristics do
1022    /// not recognize as infrastructure (`error: no such command: 'zigbuild'` is
1023    /// not a rustup message), so the exit code would be surfaced verbatim
1024    /// instead of falling open to local. Gating both here keeps that failure
1025    /// out of the user's build.
1026    pub fn has_zig(&self) -> bool {
1027        self.zig_version.is_some() && self.cargo_zigbuild_version.is_some()
1028    }
1029
1030    /// Calculate load per core (1-minute load average / num_cpus).
1031    /// Returns None if metrics are unavailable.
1032    pub fn load_per_core(&self) -> Option<f64> {
1033        match (self.load_avg_1, self.num_cpus) {
1034            (Some(load), Some(cpus)) if cpus > 0 => Some(load / cpus as f64),
1035            _ => None,
1036        }
1037    }
1038
1039    /// Check if worker has high load (exceeds threshold).
1040    /// Returns None if metrics unavailable (fail-open).
1041    pub fn is_high_load(&self, max_load_per_core: f64) -> Option<bool> {
1042        self.load_per_core().map(|lpc| lpc > max_load_per_core)
1043    }
1044
1045    /// Check if worker has low disk space (below threshold).
1046    /// Returns None if metrics unavailable (fail-open).
1047    pub fn is_low_disk(&self, min_free_gb: f64) -> Option<bool> {
1048        self.disk_free_gb.map(|free| free < min_free_gb)
1049    }
1050
1051    /// Check if canonical path-topology preflight is healthy.
1052    ///
1053    /// Returns `None` when probe data is unavailable (fail-open behavior).
1054    pub fn is_topology_healthy(&self) -> Option<bool> {
1055        self.projects_root_ok
1056    }
1057}
1058
1059/// Classify an x86-64 microarchitecture level (1..=4) from a `/proc/cpuinfo`
1060/// `flags` line (bd-6qchz).
1061///
1062/// Levels follow the psABI x86-64-v2/v3/v4 feature baselines, restricted to
1063/// flags as they appear in Linux cpuinfo. A CPU missing any flag of a level is
1064/// classified at the level below — e.g. ovh-b's Ivy Bridge E3-1245 V2 has AVX
1065/// but no `avx2`/`bmi2`/`fma`, so it classifies as 2 and its v3-codegen build
1066/// scripts SIGILL (the bd-68hon incident class).
1067#[must_use]
1068pub fn x86_64_microarch_level_from_flags(flags: &str) -> u8 {
1069    let set: std::collections::HashSet<&str> = flags.split_whitespace().collect();
1070    let has_all = |required: &[&str]| required.iter().all(|flag| set.contains(flag));
1071    // x86-64-v2: CMPXCHG16B, LAHF/SAHF, POPCNT, SSE3/SSSE3/SSE4.x.
1072    const V2: &[&str] = &["cx16", "lahf_lm", "popcnt", "sse4_1", "sse4_2", "ssse3"];
1073    // x86-64-v3: AVX/AVX2, BMI1/2, F16C, FMA, MOVBE, XSAVE (LZCNT is `abm`).
1074    const V3: &[&str] = &[
1075        "abm", "avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "movbe", "xsave",
1076    ];
1077    // x86-64-v4: the AVX-512 F/BW/CD/DQ/VL baseline.
1078    const V4: &[&str] = &["avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl"];
1079    if !has_all(V2) {
1080        return 1;
1081    }
1082    if !has_all(V3) {
1083        return 2;
1084    }
1085    if !has_all(V4) {
1086        return 3;
1087    }
1088    4
1089}
1090
1091/// Path topology configuration.
1092///
1093/// Overrides the default canonical and alias project roots used by
1094/// [`crate::path_topology::PathTopologyPolicy`].  This is useful on platforms
1095/// such as macOS where SIP prevents using `/data/projects` and the user must
1096/// choose a different root (e.g. `~/Projects`).
1097///
1098/// ```toml
1099/// [path_topology]
1100/// canonical_root = "/Users/me/Projects"
1101/// alias_root = "/Users/me/p"
1102/// ```
1103#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1104pub struct PathTopologyConfig {
1105    /// Canonical project root directory.
1106    /// Defaults to `/data/projects` when absent.
1107    #[serde(default)]
1108    pub canonical_root: Option<String>,
1109
1110    /// Alias (symlink) root that points at the canonical root.
1111    /// Defaults to `/dp` when absent.
1112    #[serde(default)]
1113    pub alias_root: Option<String>,
1114}
1115
1116impl PathTopologyConfig {
1117    /// Build a [`crate::path_topology::PathTopologyPolicy`] from this config,
1118    /// falling back to the compiled-in defaults for any field left unset.
1119    ///
1120    /// Empty strings are treated as unset (fall back to defaults).
1121    /// Tilde prefixes (`~/...`) are expanded to the user's home directory.
1122    pub fn to_policy(&self) -> crate::path_topology::PathTopologyPolicy {
1123        let canonical = self
1124            .canonical_root
1125            .as_deref()
1126            .filter(|s| !s.is_empty())
1127            .map(|s| shellexpand::tilde(s).into_owned())
1128            .unwrap_or_else(|| crate::path_topology::DEFAULT_CANONICAL_PROJECT_ROOT.to_string());
1129        let alias = self
1130            .alias_root
1131            .as_deref()
1132            .filter(|s| !s.is_empty())
1133            .map(|s| shellexpand::tilde(s).into_owned())
1134            .unwrap_or_else(|| crate::path_topology::DEFAULT_ALIAS_PROJECT_ROOT.to_string());
1135        crate::path_topology::PathTopologyPolicy::new(
1136            PathBuf::from(canonical),
1137            PathBuf::from(alias),
1138        )
1139    }
1140}
1141
1142/// RCH configuration.
1143#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1144pub struct RchConfig {
1145    #[serde(default)]
1146    pub general: GeneralConfig,
1147    #[serde(default)]
1148    pub compilation: CompilationConfig,
1149    #[serde(default)]
1150    pub transfer: TransferConfig,
1151    #[serde(default)]
1152    pub environment: EnvironmentConfig,
1153    #[serde(default)]
1154    pub circuit: CircuitBreakerConfig,
1155    #[serde(default)]
1156    pub output: OutputConfig,
1157    #[serde(default)]
1158    pub self_healing: SelfHealingConfig,
1159    #[serde(default)]
1160    pub self_test: SelfTestConfig,
1161    /// Worker selection algorithm configuration.
1162    #[serde(default)]
1163    pub selection: SelectionConfig,
1164    /// Execution allowlist configuration (bd-785w).
1165    #[serde(default)]
1166    pub execution: ExecutionConfig,
1167    /// Worker health alerting configuration (daemon).
1168    #[serde(default)]
1169    pub alerts: AlertsConfig,
1170    /// Fleet operation configuration (bd-rs7w.2).
1171    #[serde(default)]
1172    pub fleet: FleetConfig,
1173    /// Path topology overrides for project root directories.
1174    #[serde(default)]
1175    pub path_topology: PathTopologyConfig,
1176    /// Doctor / reliability subsystem configuration (verdict webhooks).
1177    #[serde(default)]
1178    pub doctor: DoctorConfig,
1179    /// Session-history remediation knobs: the central schema and default policy
1180    /// for temporary bypass, auto-rejoin, reconciliation, proof, incident
1181    /// ledger, build-root, pooled targets, telemetry freshness, log retention,
1182    /// disk pressure, and smoke defaults (bd-...remediation-ocv9i.17.1).
1183    #[serde(default)]
1184    pub remediation: crate::remediation_config::RemediationConfig,
1185    /// Layer 0 configuration pack knobs (bd-bqu38 / M-1 §149). Defaults OFF;
1186    /// injected as CARGO_PROFILE_* env vars into remote cargo builds.
1187    #[serde(default)]
1188    pub layer0: Layer0Config,
1189    /// Daemon status API over TCP for other machines on the tailnet
1190    /// (`[api]`, bd-2f5ms). Off unless `bind` is set.
1191    #[serde(default)]
1192    pub api: ApiConfig,
1193    /// Where the fleet dashboard lives (`[dashboard]`), for `rch web`.
1194    #[serde(default)]
1195    pub dashboard: DashboardConfig,
1196    /// Job-mode defaults (`[jobs]`), for `rch exec --job`.
1197    #[serde(default)]
1198    pub jobs: JobsConfig,
1199}
1200
1201/// Job-mode configuration (`[jobs]`).
1202#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1203pub struct JobsConfig {
1204    /// Named tools every job from this project requires a worker to have
1205    /// VERIFIED, as though each were passed with `--require-tool`.
1206    ///
1207    /// This is the project saying "my jobs cannot run on a worker without
1208    /// these", so it applies on top of any `--require-tool` given on the
1209    /// command line rather than being replaced by it. It affects job mode
1210    /// ONLY: ordinary offloaded compilation keeps its existing admission
1211    /// behavior, because a project-wide requirement that silently narrowed
1212    /// every build's worker pool would be a surprising way to lose the fleet.
1213    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1214    pub required_tools: Vec<String>,
1215}
1216
1217/// Daemon status API over TCP (`[api]`).
1218///
1219/// rchd's rich state (`/status`, `/workers/capabilities`) lives on a `0600`
1220/// Unix socket, which only the local user can read. This section opens the
1221/// same JSON on a TCP listener so agents and the fleet dashboard on OTHER
1222/// machines can ask a dispatcher "why are you local-only right now?" without
1223/// ssh. It is meant for a tailnet: the bind address must be loopback or a
1224/// Tailscale address (100.64.0.0/10, fd7a:115c:a1e0::/48) unless
1225/// `allow_any_addr` is set, and every status route requires the bearer token
1226/// unless `token` is unset AND `no_token` is explicitly true.
1227///
1228/// ```toml
1229/// [api]
1230/// bind = "tailscale"       # or "tailscale:9101", "100.68.51.94:9101", "127.0.0.1:9101"
1231/// token = "…"              # or token_file = "~/.config/rch/api.token"
1232/// ```
1233#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1234#[serde(default)]
1235pub struct ApiConfig {
1236    /// Where to listen. `"tailscale"` resolves this machine's Tailscale IPv4
1237    /// at startup (default port 9101); `"tailscale:PORT"`, `"IP:PORT"` or
1238    /// `"[v6]:PORT"` are taken literally. Unset (empty) = API off.
1239    pub bind: String,
1240    /// Bearer token required on `/status` and `/workers/capabilities`.
1241    pub token: Option<String>,
1242    /// File holding the bearer token (trimmed). Takes precedence over `token`.
1243    pub token_file: Option<String>,
1244    /// Serve the status routes with NO token. Only honoured when no token is
1245    /// configured; must be set explicitly so an empty token is never an
1246    /// accidental open door.
1247    pub no_token: bool,
1248    /// Permit a bind address outside loopback and the Tailscale ranges.
1249    pub allow_any_addr: bool,
1250}
1251
1252/// Where the fleet dashboard is served (`[dashboard]`).
1253#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1254#[serde(default)]
1255pub struct DashboardConfig {
1256    /// URL `rch web` opens, e.g. `https://rch-fleet.vercel.app`.
1257    pub url: Option<String>,
1258}
1259
1260/// Doctor reliability subsystem configuration.
1261///
1262/// Currently carries the verdict-transition webhook surface. Kept as its own
1263/// section (`[doctor]`) so future doctor knobs land here rather than bloating
1264/// the top-level config.
1265#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1266pub struct DoctorConfig {
1267    /// Verdict-transition webhook destinations.
1268    #[serde(default)]
1269    pub webhooks: DoctorWebhooksConfig,
1270}
1271
1272/// Webhook notification surface for reliability verdict transitions.
1273///
1274/// Fires when `rch doctor --reliability --watch` observes a verdict change
1275/// (`Healthy → Degraded`, `Degraded → Failing`, etc.). Each endpoint selects
1276/// which transitions it cares about and how the payload is formatted.
1277#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1278pub struct DoctorWebhooksConfig {
1279    /// Named webhook destinations. Empty = no webhook dispatch.
1280    #[serde(default)]
1281    pub endpoints: Vec<DoctorWebhookEndpoint>,
1282    /// Maximum number of in-flight webhook deliveries. Bounds memory and
1283    /// outbound connection pressure: when the queue is full, a transition's
1284    /// delivery is dropped (and journaled) rather than growing unboundedly.
1285    /// Per-process, shared across all endpoints.
1286    #[serde(default = "default_webhook_max_inflight")]
1287    pub max_inflight: usize,
1288}
1289
1290// Manual `Default` so the in-code default matches the serde default
1291// (`#[derive(Default)]` would give `max_inflight = 0`, diverging from a
1292// from-TOML load that fills in 16).
1293impl Default for DoctorWebhooksConfig {
1294    fn default() -> Self {
1295        Self {
1296            endpoints: Vec::new(),
1297            max_inflight: default_webhook_max_inflight(),
1298        }
1299    }
1300}
1301
1302/// A single webhook destination for verdict transitions.
1303///
1304/// SECRETS ARE NEVER STORED HERE. Routing keys, bearer tokens, and signing
1305/// secrets are referenced by the *name* of an environment variable
1306/// (`*_env` fields) read at dispatch time. There is deliberately no
1307/// `secret` / `routing_key` / `bearer_token` / `password` field on this
1308/// struct (enforced by `test_no_inline_secret_fields`).
1309#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1310pub struct DoctorWebhookEndpoint {
1311    /// Operator-facing name (appears in logs and the audit trail).
1312    pub name: String,
1313    /// Destination URL.
1314    pub url: String,
1315    /// Payload format.
1316    #[serde(default)]
1317    pub format: DoctorWebhookFormat,
1318    /// Transition predicates that arm this endpoint. Empty = never fire.
1319    /// Supported: `healthy_to_degraded`, `degraded_to_failing`,
1320    /// `any_to_failing`, `any_to_degraded`, `failing_to_healthy`,
1321    /// `degraded_to_healthy`, `any_to_healthy`, `any_to_any`.
1322    #[serde(default)]
1323    pub on_transitions: Vec<String>,
1324    /// Maximum retry attempts after the first try fails.
1325    #[serde(default = "default_webhook_retry_max")]
1326    pub retry_max: u32,
1327    /// Base backoff between retries (milliseconds); doubles each attempt.
1328    #[serde(default = "default_webhook_retry_backoff_ms")]
1329    pub retry_backoff_ms: u64,
1330    /// Per-attempt request timeout (milliseconds).
1331    #[serde(default = "default_webhook_timeout_ms")]
1332    pub timeout_ms: u64,
1333    /// Per-endpoint kill switch.
1334    #[serde(default = "default_true")]
1335    pub enabled: bool,
1336    /// Name of the env var holding the PagerDuty Events API routing key.
1337    /// Required for `format = "pagerduty"`. Read at runtime, never stored.
1338    #[serde(default)]
1339    pub routing_key_env: Option<String>,
1340    /// Name of the env var holding a bearer token sent as
1341    /// `Authorization: Bearer <token>`. Read at runtime, never stored.
1342    #[serde(default)]
1343    pub bearer_token_env: Option<String>,
1344    /// Name of the env var holding an HMAC-SHA256 signing secret. When set,
1345    /// the body is signed and the signature sent as `X-RCH-Signature:
1346    /// sha256=<hex>`. Read at runtime, never stored.
1347    #[serde(default)]
1348    pub signing_secret_env: Option<String>,
1349}
1350
1351/// Built-in webhook payload formats.
1352#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
1353#[serde(rename_all = "snake_case")]
1354pub enum DoctorWebhookFormat {
1355    /// Opinionated RCH JSON schema (default).
1356    #[default]
1357    GenericJson,
1358    /// Slack Block Kit payload.
1359    Slack,
1360    /// PagerDuty Events API v2 payload.
1361    Pagerduty,
1362}
1363
1364fn default_webhook_max_inflight() -> usize {
1365    16
1366}
1367
1368fn default_webhook_retry_max() -> u32 {
1369    3
1370}
1371
1372fn default_webhook_retry_backoff_ms() -> u64 {
1373    200
1374}
1375
1376fn default_webhook_timeout_ms() -> u64 {
1377    5000
1378}
1379
1380/// Worker health alerting configuration.
1381#[derive(Debug, Clone, Serialize, Deserialize)]
1382pub struct AlertsConfig {
1383    /// Enable or disable alert generation in the daemon.
1384    #[serde(default = "default_true")]
1385    pub enabled: bool,
1386    /// Suppress duplicate alerts for this many seconds.
1387    #[serde(default = "default_alert_suppress_duplicates_secs")]
1388    pub suppress_duplicates_secs: u64,
1389    /// How long to keep a cleared alert visible (with state
1390    /// `cleared_pending_clean`) before removing it entirely. Prevents
1391    /// transient, self-healing circuit-breaker warnings from lingering on
1392    /// `rch status` long after the fleet returned to healthy (bd-3ogaz).
1393    #[serde(default = "default_alert_cleared_retention_secs")]
1394    pub cleared_retention_secs: u64,
1395    /// Webhook configuration for external notifications.
1396    #[serde(default)]
1397    pub webhook: Option<WebhookConfig>,
1398}
1399
1400/// Webhook configuration for alert dispatch.
1401#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1402pub struct WebhookConfig {
1403    /// Webhook endpoint URL (e.g., Slack/Discord webhook URL).
1404    pub url: Option<String>,
1405    /// Secret for HMAC-SHA256 signing (optional).
1406    #[serde(default)]
1407    pub secret: Option<String>,
1408    /// Timeout in seconds for webhook requests.
1409    #[serde(default = "default_webhook_timeout_secs")]
1410    pub timeout_secs: u64,
1411    /// Number of retries on failure.
1412    #[serde(default = "default_webhook_retry_count")]
1413    pub retry_count: u32,
1414    /// Events to send (empty = all events).
1415    /// Supported: worker_offline, worker_degraded, circuit_open, all_workers_offline
1416    #[serde(default)]
1417    pub events: Vec<String>,
1418}
1419
1420fn default_webhook_timeout_secs() -> u64 {
1421    5
1422}
1423
1424fn default_webhook_retry_count() -> u32 {
1425    3
1426}
1427
1428fn default_alert_suppress_duplicates_secs() -> u64 {
1429    300
1430}
1431
1432fn default_alert_cleared_retention_secs() -> u64 {
1433    300
1434}
1435
1436impl Default for AlertsConfig {
1437    fn default() -> Self {
1438        Self {
1439            enabled: true,
1440            suppress_duplicates_secs: default_alert_suppress_duplicates_secs(),
1441            cleared_retention_secs: default_alert_cleared_retention_secs(),
1442            webhook: None,
1443        }
1444    }
1445}
1446
1447/// Fleet operation configuration.
1448///
1449/// Controls timeouts, thresholds, and behavior for fleet management commands
1450/// like `rch fleet preflight`, `rch fleet status`, and `rch fleet deploy`.
1451#[derive(Debug, Clone, Serialize, Deserialize)]
1452pub struct FleetConfig {
1453    /// SSH connection timeout in seconds.
1454    #[serde(default = "default_fleet_ssh_connect_timeout_secs")]
1455    pub ssh_connect_timeout_secs: u64,
1456    /// SSH command execution timeout in seconds.
1457    #[serde(default = "default_fleet_ssh_command_timeout_secs")]
1458    pub ssh_command_timeout_secs: u64,
1459    /// Minimum required disk space in MB for worker health.
1460    #[serde(default = "default_fleet_min_disk_space_mb")]
1461    pub min_disk_space_mb: u64,
1462    /// Maximum acceptable load average for worker health.
1463    #[serde(default = "default_fleet_max_load_average")]
1464    pub max_load_average: f64,
1465    /// Maximum number of concurrent SSH connections.
1466    #[serde(default = "default_fleet_max_concurrent_workers")]
1467    pub max_concurrent_workers: usize,
1468    /// Number of retry attempts for transient failures.
1469    #[serde(default = "default_fleet_retry_count")]
1470    pub retry_count: u32,
1471    /// Delay between retries in milliseconds.
1472    #[serde(default = "default_fleet_retry_delay_ms")]
1473    pub retry_delay_ms: u64,
1474}
1475
1476fn default_fleet_ssh_connect_timeout_secs() -> u64 {
1477    10
1478}
1479
1480fn default_fleet_ssh_command_timeout_secs() -> u64 {
1481    30
1482}
1483
1484fn default_fleet_min_disk_space_mb() -> u64 {
1485    500
1486}
1487
1488fn default_fleet_max_load_average() -> f64 {
1489    10.0
1490}
1491
1492fn default_fleet_max_concurrent_workers() -> usize {
1493    10
1494}
1495
1496fn default_fleet_retry_count() -> u32 {
1497    2
1498}
1499
1500fn default_fleet_retry_delay_ms() -> u64 {
1501    1000
1502}
1503
1504impl Default for FleetConfig {
1505    fn default() -> Self {
1506        Self {
1507            ssh_connect_timeout_secs: default_fleet_ssh_connect_timeout_secs(),
1508            ssh_command_timeout_secs: default_fleet_ssh_command_timeout_secs(),
1509            min_disk_space_mb: default_fleet_min_disk_space_mb(),
1510            max_load_average: default_fleet_max_load_average(),
1511            max_concurrent_workers: default_fleet_max_concurrent_workers(),
1512            retry_count: default_fleet_retry_count(),
1513            retry_delay_ms: default_fleet_retry_delay_ms(),
1514        }
1515    }
1516}
1517
1518impl FleetConfig {
1519    /// Get SSH connect timeout as Duration.
1520    pub fn ssh_connect_timeout(&self) -> std::time::Duration {
1521        std::time::Duration::from_secs(self.ssh_connect_timeout_secs)
1522    }
1523
1524    /// Get SSH command timeout as Duration.
1525    pub fn ssh_command_timeout(&self) -> std::time::Duration {
1526        std::time::Duration::from_secs(self.ssh_command_timeout_secs)
1527    }
1528
1529    /// Get retry delay as Duration.
1530    pub fn retry_delay(&self) -> std::time::Duration {
1531        std::time::Duration::from_millis(self.retry_delay_ms)
1532    }
1533}
1534
1535/// Box-level role (bd-wywsj): what this MACHINE is for, so "never
1536/// build locally" is configuration rather than a per-call env var.
1537#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
1538#[serde(rename_all = "lowercase")]
1539pub enum BoxRole {
1540    /// Orchestrator box: offloadable builds default FAIL-CLOSED +
1541    /// queue (RCH_REQUIRE_REMOTE semantics) — a saturated fleet waits
1542    /// for a worker instead of repopulating local rustc. Explicit
1543    /// per-call env still overrides.
1544    Dispatcher,
1545    /// The box IS the compute: fail-open/local; `rch shim install`
1546    /// refuses here.
1547    Worker,
1548    /// Today's behavior (fully backward compatible).
1549    #[default]
1550    Hybrid,
1551}
1552
1553impl BoxRole {
1554    /// Stable lowercase token.
1555    #[must_use]
1556    pub fn as_str(self) -> &'static str {
1557        match self {
1558            BoxRole::Dispatcher => "dispatcher",
1559            BoxRole::Worker => "worker",
1560            BoxRole::Hybrid => "hybrid",
1561        }
1562    }
1563}
1564
1565#[derive(Debug, Clone, Serialize, Deserialize)]
1566pub struct GeneralConfig {
1567    /// Whether RCH is enabled.
1568    #[serde(default = "default_true")]
1569    pub enabled: bool,
1570    /// Box-level role (bd-wywsj): dispatcher boxes default fail-closed
1571    /// + queue for offloadable builds; worker boxes are the compute.
1572    #[serde(default)]
1573    pub role: BoxRole,
1574    /// Force local execution for this project (never offload), even if the command is classified.
1575    ///
1576    /// Intended for sensitive projects or cases where local state must be authoritative.
1577    #[serde(default)]
1578    pub force_local: bool,
1579    /// Force remote execution for this project (always attempt offload) when safe.
1580    ///
1581    /// This bypasses heuristic gating (e.g. confidence threshold overrides) but still respects
1582    /// structural safety checks and NEVER_INTERCEPT patterns.
1583    #[serde(default)]
1584    pub force_remote: bool,
1585    /// Log level (trace, debug, info, warn, error).
1586    #[serde(default = "default_log_level")]
1587    pub log_level: String,
1588    /// Path to Unix socket for daemon communication.
1589    #[serde(default = "default_socket_path")]
1590    pub socket_path: String,
1591}
1592
1593/// Environment variable passthrough configuration.
1594#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1595pub struct EnvironmentConfig {
1596    /// Allowlist of environment variables to forward to remote workers.
1597    #[serde(default)]
1598    pub allowlist: Vec<String>,
1599}
1600
1601impl Default for GeneralConfig {
1602    fn default() -> Self {
1603        Self {
1604            enabled: true,
1605            role: BoxRole::default(),
1606            force_local: false,
1607            force_remote: false,
1608            log_level: "info".to_string(),
1609            socket_path: default_socket_path(),
1610        }
1611    }
1612}
1613
1614/// Visibility level for hook output.
1615#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
1616#[serde(rename_all = "lowercase")]
1617pub enum OutputVisibility {
1618    #[default]
1619    None,
1620    Summary,
1621    Verbose,
1622}
1623
1624impl std::fmt::Display for OutputVisibility {
1625    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1626        let value = match self {
1627            OutputVisibility::None => "none",
1628            OutputVisibility::Summary => "summary",
1629            OutputVisibility::Verbose => "verbose",
1630        };
1631        write!(f, "{}", value)
1632    }
1633}
1634
1635impl std::str::FromStr for OutputVisibility {
1636    type Err = ();
1637
1638    fn from_str(s: &str) -> Result<Self, Self::Err> {
1639        match s.trim().to_lowercase().as_str() {
1640            "none" | "silent" | "quiet" => Ok(Self::None),
1641            "summary" | "short" => Ok(Self::Summary),
1642            "verbose" | "debug" => Ok(Self::Verbose),
1643            _ => Err(()),
1644        }
1645    }
1646}
1647
1648/// Color mode for remote command output.
1649///
1650/// Controls whether ANSI color codes are preserved when streaming
1651/// output from remote compilation commands.
1652#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1653#[serde(rename_all = "lowercase")]
1654pub enum ColorMode {
1655    /// Force color output regardless of terminal detection.
1656    /// Sets CARGO_TERM_COLOR=always and similar environment variables.
1657    #[default]
1658    Always,
1659    /// Let the remote command detect terminal capabilities (may lose colors).
1660    Auto,
1661    /// Disable color output entirely.
1662    Never,
1663}
1664
1665impl std::fmt::Display for ColorMode {
1666    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1667        let value = match self {
1668            ColorMode::Always => "always",
1669            ColorMode::Auto => "auto",
1670            ColorMode::Never => "never",
1671        };
1672        write!(f, "{}", value)
1673    }
1674}
1675
1676impl std::str::FromStr for ColorMode {
1677    type Err = ();
1678
1679    fn from_str(s: &str) -> Result<Self, Self::Err> {
1680        match s.trim().to_lowercase().as_str() {
1681            "always" | "force" | "yes" | "true" => Ok(Self::Always),
1682            "auto" | "detect" => Ok(Self::Auto),
1683            "never" | "none" | "no" | "false" => Ok(Self::Never),
1684            _ => Err(()),
1685        }
1686    }
1687}
1688
1689fn default_color_mode() -> ColorMode {
1690    ColorMode::default()
1691}
1692
1693/// Hook output configuration.
1694#[derive(Debug, Clone, Serialize, Deserialize)]
1695pub struct OutputConfig {
1696    /// Output visibility level for hook messages.
1697    #[serde(default = "default_output_visibility")]
1698    pub visibility: OutputVisibility,
1699    /// Whether the first-run success message has been shown.
1700    #[serde(default)]
1701    pub first_run_complete: bool,
1702    /// Color mode for remote command output.
1703    /// Controls whether ANSI color codes are preserved in remote output.
1704    #[serde(default = "default_color_mode")]
1705    pub color_mode: ColorMode,
1706}
1707
1708impl Default for OutputConfig {
1709    fn default() -> Self {
1710        Self {
1711            visibility: OutputVisibility::None,
1712            first_run_complete: false,
1713            color_mode: ColorMode::default(),
1714        }
1715    }
1716}
1717
1718// ============================================================================
1719// Self-Healing Configuration
1720// ============================================================================
1721
1722fn default_autostart_cooldown_secs() -> u64 {
1723    30
1724}
1725
1726fn default_autostart_timeout_secs() -> u64 {
1727    3
1728}
1729
1730/// Log level for self-healing diagnostic output.
1731///
1732/// Used to filter `tracing` events emitted by the self-healing subsystem
1733/// (hook auto-start, daemon hook-install). Read from
1734/// `RCH_SELF_HEALING_LOG_LEVEL=debug|info|warn|error` (case-insensitive).
1735#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
1736#[serde(rename_all = "lowercase")]
1737pub enum SelfHealingLogLevel {
1738    Debug,
1739    #[default]
1740    Info,
1741    Warn,
1742    Error,
1743}
1744
1745impl SelfHealingLogLevel {
1746    /// Parse a human-friendly level name. Returns `None` for unknown input.
1747    pub fn from_env_str(s: &str) -> Option<Self> {
1748        match s.trim().to_ascii_lowercase().as_str() {
1749            "debug" => Some(Self::Debug),
1750            "info" => Some(Self::Info),
1751            "warn" | "warning" => Some(Self::Warn),
1752            "error" => Some(Self::Error),
1753            _ => None,
1754        }
1755    }
1756}
1757
1758/// Self-healing behaviors for the hook and daemon.
1759#[derive(Debug, Clone, Serialize, Deserialize)]
1760pub struct SelfHealingConfig {
1761    /// Whether the hook should attempt to auto-start the daemon.
1762    #[serde(default = "default_true")]
1763    pub hook_starts_daemon: bool,
1764    /// Whether the daemon should auto-install Claude Code hooks on startup.
1765    #[serde(default = "default_true")]
1766    pub daemon_installs_hooks: bool,
1767    /// Cooldown between auto-start attempts in seconds.
1768    #[serde(default = "default_autostart_cooldown_secs")]
1769    pub auto_start_cooldown_secs: u64,
1770    /// Time to wait for the daemon socket after spawning in seconds.
1771    /// Also aliased as `daemon_start_timeout` for backwards compatibility.
1772    #[serde(
1773        default = "default_autostart_timeout_secs",
1774        alias = "daemon_start_timeout"
1775    )]
1776    pub auto_start_timeout_secs: u64,
1777    /// Log level for self-healing diagnostic events. Default: info.
1778    /// Override via `RCH_SELF_HEALING_LOG_LEVEL=debug|info|warn|error`.
1779    #[serde(default)]
1780    pub self_healing_log_level: SelfHealingLogLevel,
1781}
1782
1783impl Default for SelfHealingConfig {
1784    fn default() -> Self {
1785        Self {
1786            hook_starts_daemon: default_true(),
1787            daemon_installs_hooks: default_true(),
1788            auto_start_cooldown_secs: default_autostart_cooldown_secs(),
1789            auto_start_timeout_secs: default_autostart_timeout_secs(),
1790            self_healing_log_level: SelfHealingLogLevel::default(),
1791        }
1792    }
1793}
1794
1795impl SelfHealingConfig {
1796    /// Apply environment variable overrides to this config.
1797    ///
1798    /// Supported environment variables:
1799    /// - `RCH_NO_SELF_HEALING=1` - Master switch to disable all self-healing
1800    /// - `RCH_HOOK_STARTS_DAEMON=0|1` - Control hook auto-starting daemon
1801    /// - `RCH_DAEMON_INSTALLS_HOOKS=0|1` - Control daemon auto-installing hooks
1802    /// - `RCH_AUTO_START_TIMEOUT_SECS=<seconds>` - Max wait for daemon start
1803    /// - `RCH_AUTO_START_COOLDOWN_SECS=<seconds>` - Min time between auto-starts
1804    pub fn with_env_overrides(mut self) -> Self {
1805        // Master disable switch
1806        if let Ok(val) = std::env::var("RCH_NO_SELF_HEALING")
1807            && (val == "1" || val.eq_ignore_ascii_case("true"))
1808        {
1809            self.hook_starts_daemon = false;
1810            self.daemon_installs_hooks = false;
1811            return self;
1812        }
1813
1814        // Individual toggles
1815        if let Ok(val) = std::env::var("RCH_HOOK_STARTS_DAEMON") {
1816            self.hook_starts_daemon = val != "0" && !val.eq_ignore_ascii_case("false");
1817        }
1818        if let Ok(val) = std::env::var("RCH_DAEMON_INSTALLS_HOOKS") {
1819            self.daemon_installs_hooks = val != "0" && !val.eq_ignore_ascii_case("false");
1820        }
1821
1822        // Numeric settings
1823        if let Ok(val) = std::env::var("RCH_AUTO_START_TIMEOUT_SECS")
1824            && let Ok(secs) = val.parse()
1825        {
1826            self.auto_start_timeout_secs = secs;
1827        }
1828        if let Ok(val) = std::env::var("RCH_AUTO_START_COOLDOWN_SECS")
1829            && let Ok(secs) = val.parse()
1830        {
1831            self.auto_start_cooldown_secs = secs;
1832        }
1833        // br-4zf3p: log-level override.
1834        if let Ok(val) = std::env::var("RCH_SELF_HEALING_LOG_LEVEL") {
1835            if let Some(level) = SelfHealingLogLevel::from_env_str(&val) {
1836                self.self_healing_log_level = level;
1837            } else {
1838                tracing::warn!(
1839                    target: "rch::self_healing",
1840                    invalid = %val,
1841                    "RCH_SELF_HEALING_LOG_LEVEL has invalid value; using default=info"
1842                );
1843            }
1844        }
1845
1846        self
1847    }
1848}
1849
1850// ============================================================================
1851// Self-Test Configuration
1852// ============================================================================
1853
1854/// Action to take when a scheduled self-test fails.
1855#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1856#[serde(rename_all = "snake_case")]
1857pub enum SelfTestFailureAction {
1858    /// Only emit an alert/log entry.
1859    #[default]
1860    Alert,
1861    /// Disable the failing worker automatically.
1862    DisableWorker,
1863    /// Alert and disable the worker.
1864    AlertAndDisable,
1865}
1866
1867/// Which workers to include in self-test runs.
1868#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1869#[serde(untagged)]
1870pub enum SelfTestWorkers {
1871    /// Special value "all" (or a single worker id string).
1872    All(String),
1873    /// Explicit worker list.
1874    List(Vec<WorkerId>),
1875}
1876
1877impl Default for SelfTestWorkers {
1878    fn default() -> Self {
1879        Self::All("all".to_string())
1880    }
1881}
1882
1883impl SelfTestWorkers {
1884    /// Resolve to an explicit list if configured, or None to indicate "all".
1885    pub fn resolve(&self) -> Option<Vec<WorkerId>> {
1886        match self {
1887            SelfTestWorkers::All(value) => {
1888                if value.eq_ignore_ascii_case("all") {
1889                    None
1890                } else {
1891                    Some(vec![WorkerId::new(value.clone())])
1892                }
1893            }
1894            SelfTestWorkers::List(list) => Some(list.clone()),
1895        }
1896    }
1897}
1898
1899fn default_self_test_retry_count() -> u32 {
1900    3
1901}
1902
1903fn default_self_test_retry_delay() -> String {
1904    "5m".to_string()
1905}
1906
1907fn default_self_test_enabled() -> bool {
1908    false
1909}
1910
1911/// Self-test scheduling and behavior configuration.
1912#[derive(Debug, Clone, Serialize, Deserialize)]
1913pub struct SelfTestConfig {
1914    /// Enable scheduled self-tests.
1915    #[serde(default = "default_self_test_enabled")]
1916    pub enabled: bool,
1917    /// Cron schedule (e.g., "0 3 * * *").
1918    #[serde(default)]
1919    pub schedule: Option<String>,
1920    /// Interval duration (e.g., "24h").
1921    #[serde(default)]
1922    pub interval: Option<String>,
1923    /// Which workers to test (default: "all").
1924    #[serde(default)]
1925    pub workers: SelfTestWorkers,
1926    /// Action on failure.
1927    #[serde(default)]
1928    pub on_failure: SelfTestFailureAction,
1929    /// Retry count for failed tests.
1930    #[serde(default = "default_self_test_retry_count")]
1931    pub retry_count: u32,
1932    /// Delay between retries.
1933    #[serde(default = "default_self_test_retry_delay")]
1934    pub retry_delay: String,
1935}
1936
1937impl Default for SelfTestConfig {
1938    fn default() -> Self {
1939        Self {
1940            enabled: default_self_test_enabled(),
1941            schedule: None,
1942            interval: None,
1943            workers: SelfTestWorkers::default(),
1944            on_failure: SelfTestFailureAction::default(),
1945            retry_count: default_self_test_retry_count(),
1946            retry_delay: default_self_test_retry_delay(),
1947        }
1948    }
1949}
1950
1951/// Circuit breaker configuration.
1952#[derive(Debug, Clone, Serialize, Deserialize)]
1953pub struct CircuitBreakerConfig {
1954    /// Consecutive failures to open circuit.
1955    #[serde(default = "default_circuit_failure_threshold")]
1956    pub failure_threshold: u32,
1957    /// Consecutive successes to close circuit.
1958    #[serde(default = "default_circuit_success_threshold")]
1959    pub success_threshold: u32,
1960    /// Error rate threshold (0.0-1.0) to open circuit.
1961    #[serde(default = "default_circuit_error_rate_threshold")]
1962    pub error_rate_threshold: f64,
1963    /// Rolling window size in seconds.
1964    #[serde(default = "default_circuit_window_secs")]
1965    pub window_secs: u64,
1966    /// Cooldown duration before half-open (seconds).
1967    #[serde(default = "default_circuit_open_cooldown_secs")]
1968    pub open_cooldown_secs: u64,
1969    /// Max concurrent probes in half-open.
1970    #[serde(default = "default_circuit_half_open_max_probes")]
1971    pub half_open_max_probes: u32,
1972    /// Consecutive *command-class* failures to open the circuit.
1973    ///
1974    /// Command-class evidence is an operation of the same shape as dispatched
1975    /// work — a remote binary that must run to completion (the telemetry poll,
1976    /// or a health probe whose session connected but whose command never
1977    /// finished). It is tracked separately from the plain liveness probe
1978    /// because a host under fork exhaustion or swap thrash keeps answering
1979    /// `echo` over a warm connection while every real command times out;
1980    /// interleaved liveness successes would otherwise keep resetting
1981    /// `failure_threshold` and the breaker would never trip.
1982    #[serde(default = "default_circuit_command_failure_threshold")]
1983    pub command_failure_threshold: u32,
1984    /// Upper bound (seconds) for the open cooldown after repeated reopens.
1985    ///
1986    /// Each Open -> HalfOpen -> Open cycle without an intervening Closed
1987    /// doubles the cooldown (`open_cooldown_secs`, x2, x4, ...) up to this
1988    /// cap, so a flapping worker converges to Open instead of being
1989    /// re-admitted every cooldown. Set equal to `open_cooldown_secs` to
1990    /// disable the backoff.
1991    #[serde(default = "default_circuit_open_cooldown_max_secs")]
1992    pub open_cooldown_max_secs: u64,
1993}
1994
1995impl Default for CircuitBreakerConfig {
1996    fn default() -> Self {
1997        Self {
1998            failure_threshold: default_circuit_failure_threshold(),
1999            success_threshold: default_circuit_success_threshold(),
2000            error_rate_threshold: default_circuit_error_rate_threshold(),
2001            window_secs: default_circuit_window_secs(),
2002            open_cooldown_secs: default_circuit_open_cooldown_secs(),
2003            half_open_max_probes: default_circuit_half_open_max_probes(),
2004            command_failure_threshold: default_circuit_command_failure_threshold(),
2005            open_cooldown_max_secs: default_circuit_open_cooldown_max_secs(),
2006        }
2007    }
2008}
2009
2010impl CircuitBreakerConfig {
2011    /// Effective cooldown (ms) for the given reopen streak, with exponential
2012    /// backoff capped at `open_cooldown_max_secs`. `reopen_streak` counts
2013    /// Open transitions since the circuit last closed (1 = first open).
2014    pub fn cooldown_ms_for_streak(&self, reopen_streak: u32) -> u64 {
2015        let base_ms = self.open_cooldown_secs.saturating_mul(1000);
2016        let cap_ms = self
2017            .open_cooldown_max_secs
2018            .saturating_mul(1000)
2019            .max(base_ms);
2020        // Exponent bounded so the shift cannot overflow; the cap clamps anyway.
2021        let exponent = reopen_streak.saturating_sub(1).min(16);
2022        base_ms.saturating_mul(1u64 << exponent).min(cap_ms)
2023    }
2024}
2025
2026/// Circuit breaker statistics for tracking state transitions.
2027///
2028/// Maintains a rolling window of successes and failures to determine
2029/// when the circuit should open, close, or enter half-open state.
2030#[derive(Debug, Clone, Default)]
2031pub struct CircuitStats {
2032    /// Current circuit state.
2033    state: CircuitState,
2034    /// Consecutive failure count (reset on success).
2035    consecutive_failures: u32,
2036    /// Consecutive success count in half-open state (reset on failure).
2037    consecutive_successes: u32,
2038    /// Total successes in the current rolling window.
2039    window_successes: u32,
2040    /// Total failures in the current rolling window.
2041    window_failures: u32,
2042    /// Timestamp when circuit entered Open state (epoch millis).
2043    opened_at: Option<u64>,
2044    /// Timestamp of last state change (epoch millis).
2045    last_state_change: u64,
2046    /// Current active probes in half-open state.
2047    active_probes: u32,
2048    /// Consecutive command-class failures (telemetry poll exhaustion, probe
2049    /// sessions that connected but never completed). Reset only by a
2050    /// command-class success or a circuit close — NOT by a liveness success,
2051    /// which is the whole point (see `CircuitBreakerConfig::command_failure_threshold`).
2052    consecutive_command_failures: u32,
2053    /// Number of Open transitions since the circuit last closed. Drives the
2054    /// cooldown backoff; 0 while Closed-since-recovery.
2055    reopen_streak: u32,
2056    /// Recent health check results (true=success, false=failure).
2057    /// Used for history visualization in status display.
2058    recent_results: Vec<bool>,
2059}
2060
2061/// Maximum number of recent results to keep for history visualization.
2062const CIRCUIT_HISTORY_SIZE: usize = 10;
2063
2064impl CircuitStats {
2065    /// Create new circuit stats in the closed state.
2066    pub fn new() -> Self {
2067        Self {
2068            state: CircuitState::Closed,
2069            consecutive_failures: 0,
2070            consecutive_successes: 0,
2071            window_successes: 0,
2072            window_failures: 0,
2073            opened_at: None,
2074            last_state_change: Self::now_millis(),
2075            active_probes: 0,
2076            consecutive_command_failures: 0,
2077            reopen_streak: 0,
2078            recent_results: Vec::with_capacity(CIRCUIT_HISTORY_SIZE),
2079        }
2080    }
2081
2082    /// Consecutive command-class failures observed since the last
2083    /// command-class success (or circuit close).
2084    pub fn consecutive_command_failures(&self) -> u32 {
2085        self.consecutive_command_failures
2086    }
2087
2088    /// Open transitions since the circuit last closed (0 when it has not
2089    /// opened since recovery). Exposed for diagnostics.
2090    pub fn reopen_streak(&self) -> u32 {
2091        self.reopen_streak
2092    }
2093
2094    /// Cooldown currently in force (ms), including reopen backoff.
2095    fn effective_cooldown_ms(&self, config: &CircuitBreakerConfig) -> u64 {
2096        config.cooldown_ms_for_streak(self.reopen_streak.max(1))
2097    }
2098
2099    /// Get the current circuit state.
2100    pub fn state(&self) -> CircuitState {
2101        self.state
2102    }
2103
2104    /// Get the timestamp when the circuit opened (if open).
2105    pub fn opened_at(&self) -> Option<u64> {
2106        self.opened_at
2107    }
2108
2109    /// Get the timestamp of the last state change.
2110    pub fn last_state_change(&self) -> u64 {
2111        self.last_state_change
2112    }
2113
2114    /// Get the consecutive failure count.
2115    pub fn consecutive_failures(&self) -> u32 {
2116        self.consecutive_failures
2117    }
2118
2119    /// Get the error rate in the current window (0.0-1.0).
2120    ///
2121    /// Computed over the bounded recent-results window (the last
2122    /// `CIRCUIT_HISTORY_SIZE` health checks) rather than the unbounded lifetime
2123    /// `window_successes`/`window_failures` counters. A long-lived daemon polling
2124    /// distant workers accumulates occasional transient failures over days; with a
2125    /// lifetime ratio a currently-healthy worker could stay gated (circuit kept
2126    /// open, depressed health score) until the daemon was restarted. A recency
2127    /// window lets the rate self-heal as fresh successes age out old failures.
2128    pub fn error_rate(&self) -> f64 {
2129        if self.recent_results.is_empty() {
2130            return 0.0;
2131        }
2132        let failures = self.recent_results.iter().filter(|&&ok| !ok).count();
2133        failures as f64 / self.recent_results.len() as f64
2134    }
2135
2136    /// Record a successful operation.
2137    ///
2138    /// In closed state: resets consecutive failures, increments window successes.
2139    /// In half-open state: increments consecutive successes.
2140    pub fn record_success(&mut self) {
2141        self.window_successes += 1;
2142        self.consecutive_failures = 0;
2143        self.push_result(true);
2144
2145        if self.state == CircuitState::HalfOpen {
2146            self.consecutive_successes += 1;
2147            if self.active_probes > 0 {
2148                self.active_probes -= 1;
2149            }
2150        }
2151    }
2152
2153    /// Record a failed operation.
2154    ///
2155    /// Increments consecutive failures and window failures.
2156    /// In half-open state, resets consecutive successes.
2157    pub fn record_failure(&mut self) {
2158        self.window_failures += 1;
2159        self.consecutive_failures += 1;
2160        self.push_result(false);
2161
2162        if self.state == CircuitState::HalfOpen {
2163            self.consecutive_successes = 0;
2164            if self.active_probes > 0 {
2165                self.active_probes -= 1;
2166            }
2167        }
2168    }
2169
2170    /// Check if the circuit should open based on config thresholds.
2171    ///
2172    /// Returns true if:
2173    /// - Consecutive failures exceed threshold, OR
2174    /// - Error rate exceeds threshold (with minimum sample size)
2175    pub fn should_open(&self, config: &CircuitBreakerConfig) -> bool {
2176        if self.state != CircuitState::Closed {
2177            return false;
2178        }
2179
2180        // Open if consecutive failures exceed threshold
2181        if self.consecutive_failures >= config.failure_threshold {
2182            return true;
2183        }
2184
2185        // Open if error rate exceeds threshold (with a minimum sample size).
2186        // Counts samples in the bounded recent-results window so the gate
2187        // reflects recent behavior, matching error_rate() above.
2188        if self.recent_results.len() >= 5 && self.error_rate() >= config.error_rate_threshold {
2189            return true;
2190        }
2191
2192        false
2193    }
2194
2195    /// Check if the circuit should transition to half-open.
2196    ///
2197    /// Returns true if the circuit is open and the cooldown period has elapsed.
2198    pub fn should_half_open(&self, config: &CircuitBreakerConfig) -> bool {
2199        if self.state != CircuitState::Open {
2200            return false;
2201        }
2202
2203        let now = Self::now_millis();
2204        let cooldown_ms = self.effective_cooldown_ms(config);
2205
2206        if let Some(opened_at) = self.opened_at {
2207            now.saturating_sub(opened_at) >= cooldown_ms
2208        } else {
2209            false
2210        }
2211    }
2212
2213    /// Check if the circuit should close.
2214    ///
2215    /// Returns true if in half-open state, consecutive successes meet or
2216    /// exceed the success threshold, AND no command-class failure is
2217    /// outstanding. A worker whose last real command timed out is not
2218    /// recovered no matter how many liveness probes it answers; it stays
2219    /// HalfOpen (degraded, probe-limited) until a command-class success
2220    /// clears the evidence.
2221    pub fn should_close(&self, config: &CircuitBreakerConfig) -> bool {
2222        if self.state != CircuitState::HalfOpen {
2223            return false;
2224        }
2225
2226        self.consecutive_successes >= config.success_threshold
2227            && self.consecutive_command_failures == 0
2228    }
2229
2230    /// Check if a probe request can be made in half-open state.
2231    ///
2232    /// Returns true if active probes are below the maximum allowed.
2233    pub fn can_probe(&self, config: &CircuitBreakerConfig) -> bool {
2234        if self.state != CircuitState::HalfOpen {
2235            return false;
2236        }
2237
2238        self.active_probes < config.half_open_max_probes
2239    }
2240
2241    /// Start a probe request in half-open state.
2242    ///
2243    /// Returns true if the probe was started, false if already at max probes.
2244    pub fn start_probe(&mut self, config: &CircuitBreakerConfig) -> bool {
2245        if !self.can_probe(config) {
2246            return false;
2247        }
2248        self.active_probes += 1;
2249        true
2250    }
2251
2252    /// Transition the circuit to open state.
2253    pub fn open(&mut self) {
2254        if self.state != CircuitState::Open {
2255            self.state = CircuitState::Open;
2256            self.opened_at = Some(Self::now_millis());
2257            self.last_state_change = Self::now_millis();
2258            self.consecutive_successes = 0;
2259            self.active_probes = 0;
2260            // Every open without an intervening close lengthens the next
2261            // cooldown (see `CircuitBreakerConfig::cooldown_ms_for_streak`).
2262            self.reopen_streak = self.reopen_streak.saturating_add(1);
2263        }
2264    }
2265
2266    /// Transition the circuit to half-open state.
2267    pub fn half_open(&mut self) {
2268        if self.state != CircuitState::HalfOpen {
2269            self.state = CircuitState::HalfOpen;
2270            self.last_state_change = Self::now_millis();
2271            self.consecutive_successes = 0;
2272            self.active_probes = 0;
2273        }
2274    }
2275
2276    /// Transition the circuit to closed state.
2277    pub fn close(&mut self) {
2278        if self.state != CircuitState::Closed {
2279            self.state = CircuitState::Closed;
2280            self.last_state_change = Self::now_millis();
2281            self.opened_at = None;
2282            self.consecutive_failures = 0;
2283            self.consecutive_successes = 0;
2284            self.active_probes = 0;
2285            self.consecutive_command_failures = 0;
2286            self.reopen_streak = 0;
2287            // Reset the window on close, INCLUDING the recent-results history
2288            // that error_rate() now reads. Otherwise a freshly-recovered circuit
2289            // would still compute a high error rate from the pre-failure samples
2290            // and immediately re-open (flapping). Matches the original intent of
2291            // resetting the error-rate inputs when the circuit recovers.
2292            self.window_successes = 0;
2293            self.window_failures = 0;
2294            self.recent_results.clear();
2295        }
2296    }
2297
2298    /// Reset the rolling window counters.
2299    ///
2300    /// Called periodically to ensure the window reflects recent activity.
2301    /// Also clears the recent-results history that `error_rate()` is computed
2302    /// from, so a reset truly zeroes the observed error rate.
2303    pub fn reset_window(&mut self) {
2304        self.window_successes = 0;
2305        self.window_failures = 0;
2306        self.recent_results.clear();
2307    }
2308
2309    /// Get recent health check results for history visualization.
2310    ///
2311    /// Returns a slice of recent results (true=success, false=failure),
2312    /// with the most recent result at the end.
2313    pub fn recent_results(&self) -> &[bool] {
2314        &self.recent_results
2315    }
2316
2317    /// Calculate seconds remaining until circuit auto-transitions to half-open.
2318    ///
2319    /// Returns None if circuit is not open or cooldown has already elapsed.
2320    pub fn recovery_remaining_secs(&self, config: &CircuitBreakerConfig) -> Option<u64> {
2321        if self.state != CircuitState::Open {
2322            return None;
2323        }
2324
2325        let now = Self::now_millis();
2326        let cooldown_ms = self.effective_cooldown_ms(config);
2327
2328        if let Some(opened_at) = self.opened_at {
2329            let elapsed_ms = now.saturating_sub(opened_at);
2330            if elapsed_ms < cooldown_ms {
2331                return Some((cooldown_ms - elapsed_ms) / 1000);
2332            }
2333        }
2334        None
2335    }
2336
2337    /// Push a result to the history, maintaining the maximum size.
2338    fn push_result(&mut self, success: bool) {
2339        if self.recent_results.len() >= CIRCUIT_HISTORY_SIZE {
2340            self.recent_results.remove(0);
2341        }
2342        self.recent_results.push(success);
2343    }
2344
2345    fn now_millis() -> u64 {
2346        std::time::SystemTime::now()
2347            .duration_since(std::time::UNIX_EPOCH)
2348            .map(|d| d.as_millis() as u64)
2349            .unwrap_or(0)
2350    }
2351
2352    /// Drive the circuit through a single health-check outcome and return the
2353    /// resulting [`CircuitState`].
2354    ///
2355    /// This is the single, self-contained transition engine shared by every
2356    /// circuit consumer (the daemon health monitor via
2357    /// `WorkerState::record_health_check`, and the diagnostic `WorkerHealth`
2358    /// panel via `WorkerHealth::observe`). Keeping the transition logic in ONE
2359    /// place is what makes `WorkerState.circuit` (which selection reads) and the
2360    /// diagnostic `WorkerHealth.circuit` agree.
2361    ///
2362    /// Behaviour:
2363    /// - On a **healthy** outcome: if the circuit is Open and its cooldown has
2364    ///   elapsed it first transitions Open → HalfOpen *before* crediting the
2365    ///   success, so recovery needs exactly `success_threshold` clean probes
2366    ///   (crediting a success against an Open circuit would be lost). Then the
2367    ///   success is recorded and the circuit closes once the threshold is met.
2368    /// - On an **unhealthy** outcome: the failure is recorded, the circuit opens
2369    ///   if the closed-state open condition is met, and a HalfOpen probe failure
2370    ///   reopens the circuit — UNLESS the failure was `transient` (a retryable
2371    ///   transport blip that survived the in-check retries). A transient
2372    ///   half-open failure neither reopens nor resets the cooldown, so a single
2373    ///   network hiccup during recovery cannot bounce a worker back to Open.
2374    /// - After either branch, an Open circuit past its cooldown is nudged to
2375    ///   HalfOpen so probing resumes on the next poll even for a still-failing
2376    ///   worker.
2377    pub fn apply_health_outcome(
2378        &mut self,
2379        healthy: bool,
2380        transient: bool,
2381        config: &CircuitBreakerConfig,
2382    ) -> CircuitState {
2383        if healthy {
2384            // Transition to half-open BEFORE crediting the success so the probe
2385            // counts toward recovery. record_success() only increments the
2386            // consecutive-success counter while in HalfOpen.
2387            if self.state == CircuitState::Open && self.should_half_open(config) {
2388                self.half_open();
2389            }
2390            self.record_success();
2391            if self.should_close(config) {
2392                self.close();
2393            }
2394        } else {
2395            let was_half_open = self.state == CircuitState::HalfOpen;
2396            self.record_failure();
2397            if self.should_open(config) {
2398                self.open();
2399            } else if was_half_open && !transient {
2400                // A real (non-transient) probe failure in half-open reopens the
2401                // circuit and restarts the cooldown. A transient failure is
2402                // ignored so a network blip can't undo an in-progress recovery.
2403                self.open();
2404            }
2405        }
2406
2407        // Even for a still-failing worker, resume probing once the cooldown
2408        // elapses so a recovered worker is retried promptly.
2409        if self.state == CircuitState::Open && self.should_half_open(config) {
2410            self.half_open();
2411        }
2412
2413        self.state
2414    }
2415
2416    /// Drive the circuit through a single *command-class* outcome and return
2417    /// the resulting [`CircuitState`].
2418    ///
2419    /// Command-class evidence comes from an operation of the same shape as
2420    /// dispatched work: a remote binary that must run to completion (the
2421    /// telemetry poll, or a health probe whose SSH session connected but whose
2422    /// command never finished). It is the signal the plain liveness probe
2423    /// cannot provide — a host under fork exhaustion or swap thrash keeps
2424    /// answering `echo` over a warm connection while every real command hangs,
2425    /// and on that evidence alone the breaker would happily re-admit it.
2426    ///
2427    /// Behaviour:
2428    /// - **Success**: clears the outstanding command-failure count and then
2429    ///   counts as an ordinary healthy outcome (may advance Open -> HalfOpen ->
2430    ///   Closed exactly like a liveness success).
2431    /// - **Failure**: increments the command-failure count and records a plain
2432    ///   failure. The circuit opens when either the ordinary open condition or
2433    ///   `command_failure_threshold` is met. In HalfOpen a command failure
2434    ///   ALWAYS reopens — there is no transient exemption, because "connected
2435    ///   but the command did not complete" is precisely the condition the
2436    ///   half-open trial exists to detect. While any command failure is
2437    ///   outstanding, [`Self::should_close`] refuses to close on liveness
2438    ///   successes alone.
2439    pub fn apply_command_outcome(
2440        &mut self,
2441        success: bool,
2442        config: &CircuitBreakerConfig,
2443    ) -> CircuitState {
2444        if success {
2445            self.consecutive_command_failures = 0;
2446            return self.apply_health_outcome(true, false, config);
2447        }
2448
2449        self.consecutive_command_failures = self.consecutive_command_failures.saturating_add(1);
2450        let was_half_open = self.state == CircuitState::HalfOpen;
2451        self.record_failure();
2452        let threshold = config.command_failure_threshold.max(1);
2453        if was_half_open
2454            || self.should_open(config)
2455            || (self.state == CircuitState::Closed
2456                && self.consecutive_command_failures >= threshold)
2457        {
2458            self.open();
2459        }
2460
2461        if self.state == CircuitState::Open && self.should_half_open(config) {
2462            self.half_open();
2463        }
2464
2465        self.state
2466    }
2467}
2468
2469#[derive(Debug, Clone, Serialize, Deserialize)]
2470pub struct CompilationConfig {
2471    /// Minimum confidence score to intercept (0.0-1.0).
2472    #[serde(default = "default_confidence")]
2473    pub confidence_threshold: f64,
2474    /// Skip interception if estimated local time < this (ms).
2475    #[serde(default = "default_min_local_time")]
2476    pub min_local_time_ms: u64,
2477    /// Minimum expected speedup ratio (local_time / remote_time) to offload.
2478    /// If predicted speedup < threshold, run locally. Default: 1.2 (20% faster).
2479    /// Set to 1.0 to always offload when other criteria are met.
2480    #[serde(default = "default_speedup_threshold")]
2481    pub remote_speedup_threshold: f64,
2482    /// Default slot estimate for build commands (cargo build, gcc, etc.).
2483    #[serde(default = "default_build_slots")]
2484    pub build_slots: u32,
2485    /// Default slot estimate for test commands (cargo test, nextest).
2486    /// Tests typically use more parallelism than builds.
2487    #[serde(default = "default_test_slots")]
2488    pub test_slots: u32,
2489    /// Default slot estimate for check/lint commands (cargo check, clippy).
2490    /// These are typically faster and use fewer resources.
2491    #[serde(default = "default_check_slots")]
2492    pub check_slots: u32,
2493    /// Timeout in seconds for build commands.
2494    /// Build commands (cargo build, gcc, etc.) typically complete faster than tests.
2495    #[serde(default = "default_build_timeout")]
2496    pub build_timeout_sec: u64,
2497    /// Timeout in seconds for test commands.
2498    /// Test commands often need longer timeouts due to test suite execution time.
2499    #[serde(default = "default_test_timeout")]
2500    pub test_timeout_sec: u64,
2501    /// Timeout in seconds for bun commands (bun test, bun typecheck).
2502    /// Bun has known issues where its internal --timeout flag doesn't work for
2503    /// CPU-bound hangs, so we use a shorter external timeout by default.
2504    #[serde(default = "default_bun_timeout")]
2505    pub bun_timeout_sec: u64,
2506    /// Whether to wrap remote commands with external timeout protection.
2507    /// This prevents runaway/stuck processes from consuming worker slots indefinitely.
2508    /// Default: true. Set to false to disable timeout wrapping entirely.
2509    #[serde(default = "default_external_timeout_enabled")]
2510    pub external_timeout_enabled: bool,
2511    /// Whether a remote build FAILURE may fall back to LOCAL execution on the
2512    /// orchestrator once every eligible remote worker has been exhausted.
2513    ///
2514    /// On a remote failure the hook first retries the build on a *different,
2515    /// higher-capacity* worker (see `run_exec`'s worker-fallback loop). Local
2516    /// execution runs the heavy compile on the orchestrator that coordinates the
2517    /// whole fleet, so it is a genuine last resort and can flood that box. This
2518    /// flag gates it. Default `true` preserves the historical fail-open behavior
2519    /// (local only when no worker can serve); set to `false` on an orchestrator
2520    /// to refuse local fallback and fail the build closed instead. `force_remote`
2521    /// / proof mode / `RCH_REQUIRE_REMOTE` still override this by refusing local
2522    /// unconditionally.
2523    #[serde(default = "default_allow_local_fallback")]
2524    pub allow_local_fallback: bool,
2525    /// Per-job rustc parallelism cap applied on the WORKER (issue #49).
2526    ///
2527    /// `total_slots` bounds concurrent *jobs* per worker, but cargo defaults
2528    /// `-j` to the worker's `nproc`, so one offloaded `cargo test --workspace`
2529    /// forks `nproc` rustc regardless of slots and can swap-thrash a
2530    /// RAM-light box. This policy injects `CARGO_BUILD_JOBS` into the remote
2531    /// build environment; see [`RemoteBuildJobs`] for the modes. It never
2532    /// overrides a `CARGO_BUILD_JOBS` the worker session already carries, an
2533    /// inline `CARGO_BUILD_JOBS=N` or `-j N` in the command, or a project
2534    /// `.cargo/config.toml` `[build] jobs` — explicit intent always wins.
2535    #[serde(default)]
2536    pub remote_build_jobs: RemoteBuildJobs,
2537}
2538
2539/// How rch derives the `CARGO_BUILD_JOBS` it injects into a remote build.
2540///
2541/// - `auto` (default): computed on the worker from its live `nproc` and
2542///   total RAM as `clamp(ram_gib / 8, 2, min(nproc, 8))` — budgeting roughly
2543///   8 GiB of RAM per concurrent rustc leaves headroom for two to three
2544///   concurrent jobs at a realistic per-rustc peak of ~3 GiB.
2545/// - `off`: inject nothing (historical behavior; cargo uses `nproc`).
2546/// - `<n>`: inject a fixed job count fleet-wide.
2547///
2548/// In every mode an explicitly set `CARGO_BUILD_JOBS` on the worker (for
2549/// example from `/etc/environment`) or in the command is left untouched.
2550#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
2551pub enum RemoteBuildJobs {
2552    /// Derive from the worker's cores and RAM at execution time.
2553    #[default]
2554    Auto,
2555    /// Do not inject `CARGO_BUILD_JOBS`.
2556    Off,
2557    /// Inject a fixed job count (must be >= 1).
2558    Fixed(u32),
2559}
2560
2561impl RemoteBuildJobs {
2562    /// Parse the config/CLI spelling: `auto`, `off`, or a positive integer.
2563    pub fn parse(value: &str) -> Result<Self, String> {
2564        let trimmed = value.trim();
2565        match trimmed.to_ascii_lowercase().as_str() {
2566            "auto" => Ok(Self::Auto),
2567            "off" | "none" | "disabled" => Ok(Self::Off),
2568            _ => match trimmed.parse::<u32>() {
2569                Ok(0) => Err(
2570                    "remote_build_jobs must be >= 1 (cargo rejects `-j 0`); use `off` to disable"
2571                        .to_string(),
2572                ),
2573                Ok(n) => Ok(Self::Fixed(n)),
2574                Err(_) => Err(format!(
2575                    "'{trimmed}' is not a valid remote_build_jobs value (expected `auto`, `off`, or a positive integer)"
2576                )),
2577            },
2578        }
2579    }
2580}
2581
2582impl std::fmt::Display for RemoteBuildJobs {
2583    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2584        match self {
2585            Self::Auto => f.write_str("auto"),
2586            Self::Off => f.write_str("off"),
2587            Self::Fixed(n) => write!(f, "{n}"),
2588        }
2589    }
2590}
2591
2592impl std::str::FromStr for RemoteBuildJobs {
2593    type Err = String;
2594
2595    fn from_str(value: &str) -> Result<Self, Self::Err> {
2596        Self::parse(value)
2597    }
2598}
2599
2600impl Serialize for RemoteBuildJobs {
2601    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
2602        match self {
2603            Self::Auto => serializer.serialize_str("auto"),
2604            Self::Off => serializer.serialize_str("off"),
2605            Self::Fixed(n) => serializer.serialize_u32(*n),
2606        }
2607    }
2608}
2609
2610impl<'de> Deserialize<'de> for RemoteBuildJobs {
2611    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
2612        #[derive(Deserialize)]
2613        #[serde(untagged)]
2614        enum Raw {
2615            Count(u64),
2616            Spelling(String),
2617        }
2618        match Raw::deserialize(deserializer)? {
2619            Raw::Count(n) => {
2620                let n = u32::try_from(n).map_err(|_| {
2621                    serde::de::Error::custom(format!("remote_build_jobs {n} is out of range"))
2622                })?;
2623                Self::parse(&n.to_string()).map_err(serde::de::Error::custom)
2624            }
2625            Raw::Spelling(s) => Self::parse(&s).map_err(serde::de::Error::custom),
2626        }
2627    }
2628}
2629
2630impl Default for CompilationConfig {
2631    fn default() -> Self {
2632        Self {
2633            confidence_threshold: 0.85,
2634            min_local_time_ms: 2000,
2635            remote_speedup_threshold: default_speedup_threshold(),
2636            build_slots: default_build_slots(),
2637            test_slots: default_test_slots(),
2638            check_slots: default_check_slots(),
2639            build_timeout_sec: default_build_timeout(),
2640            test_timeout_sec: default_test_timeout(),
2641            bun_timeout_sec: default_bun_timeout(),
2642            external_timeout_enabled: default_external_timeout_enabled(),
2643            allow_local_fallback: default_allow_local_fallback(),
2644            remote_build_jobs: RemoteBuildJobs::default(),
2645        }
2646    }
2647}
2648
2649fn default_build_slots() -> u32 {
2650    4
2651}
2652
2653fn default_test_slots() -> u32 {
2654    8
2655}
2656
2657fn default_check_slots() -> u32 {
2658    2
2659}
2660
2661/// Default build timeout: 5 minutes (300 seconds).
2662/// Most builds complete well within this time.
2663fn default_build_timeout() -> u64 {
2664    300
2665}
2666
2667/// Default test timeout: 30 minutes (1800 seconds).
2668/// Test suites often take significantly longer than builds.
2669fn default_test_timeout() -> u64 {
2670    1800
2671}
2672
2673/// Default bun timeout: 10 minutes (600 seconds).
2674/// Bun's internal --timeout flag doesn't work for CPU-bound hangs,
2675/// so we use a shorter external timeout to prevent zombies.
2676fn default_bun_timeout() -> u64 {
2677    600
2678}
2679
2680/// Default: external timeout protection is enabled.
2681fn default_external_timeout_enabled() -> bool {
2682    true
2683}
2684
2685/// Default: permit last-resort local fallback after remote workers are
2686/// exhausted. Set `compilation.allow_local_fallback = false` to fail closed and
2687/// keep heavy compiles off the orchestrator entirely.
2688fn default_allow_local_fallback() -> bool {
2689    true
2690}
2691
2692impl CompilationConfig {
2693    /// Returns the appropriate external timeout for the given compilation kind.
2694    ///
2695    /// - Bun commands get the shorter bun_timeout (to protect against known hang issues)
2696    /// - Test commands get the longer test_timeout
2697    /// - All other commands (builds, checks, clippy) get the build_timeout
2698    pub fn timeout_for_kind(&self, kind: Option<CompilationKind>) -> std::time::Duration {
2699        let secs = match kind {
2700            // Bun has known issues where internal timeout doesn't work for CPU hangs
2701            Some(CompilationKind::BunTest) | Some(CompilationKind::BunTypecheck) => {
2702                self.bun_timeout_sec
2703            }
2704            // Other test commands use the standard test timeout
2705            Some(CompilationKind::CargoTest) | Some(CompilationKind::CargoNextest) => {
2706                self.test_timeout_sec
2707            }
2708            // Nix derivation builds can take a long time (fetch + compile large
2709            // closures); give them the generous test-timeout bucket rather than
2710            // the short build timeout so legitimate builds are not killed early.
2711            Some(CompilationKind::NixBuild) => self.test_timeout_sec,
2712            // Go test suites are long-running like cargo test; give them the
2713            // test bucket rather than the short build timeout.
2714            Some(CompilationKind::GoTest) => self.test_timeout_sec,
2715            // Builds, checks, clippy, and unknown use build timeout
2716            _ => self.build_timeout_sec,
2717        };
2718        std::time::Duration::from_secs(secs)
2719    }
2720
2721    /// Returns whether external timeout wrapping is enabled.
2722    pub fn external_timeout_enabled(&self) -> bool {
2723        self.external_timeout_enabled
2724    }
2725}
2726
2727/// Version of the Layer 0 configuration pack schema (bd-bqu38 / M-1 §149).
2728///
2729/// Bump when a knob is added, removed, or its meaning changes; consumers
2730/// record the version alongside emitted baselines so regressions are
2731/// attributable to a specific pack revision.
2732pub const LAYER0_PACK_VERSION: &str = "1";
2733
2734/// Layer 0 configuration pack knobs (bd-bqu38).
2735///
2736/// Every knob defaults to OFF and toggles independently — the kill/rollback
2737/// contract from the master plan (§149: removed from defaults if it regresses
2738/// representative p95, output equivalence, debugger behavior, or
2739/// compatibility). Knobs are injected into remote cargo builds as
2740/// `CARGO_PROFILE_*` environment variables, so enabling one never mutates any
2741/// user-owned `.cargo/config.toml`.
2742#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
2743#[serde(default)]
2744pub struct Layer0Config {
2745    /// Master switch: nothing is injected unless this is on.
2746    pub enabled: bool,
2747    /// `profile.release.lto = "thin"` via `CARGO_PROFILE_RELEASE_LTO=thin`.
2748    pub release_lto_thin: bool,
2749    /// `profile.release.codegen-units = 1` via
2750    /// `CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1`.
2751    pub release_codegen_units_1: bool,
2752}
2753
2754impl Layer0Config {
2755    /// Environment pairs to inject into a remote cargo build.
2756    ///
2757    /// Empty unless [`Self::enabled`] — the master switch gates every knob so
2758    /// an operator can disable the whole pack with one key. Order is stable
2759    /// (declaration order) for deterministic emission and test assertions.
2760    #[must_use]
2761    pub fn active_env(&self) -> Vec<(&'static str, &'static str)> {
2762        if !self.enabled {
2763            return Vec::new();
2764        }
2765        let mut env = Vec::new();
2766        if self.release_lto_thin {
2767            env.push(("CARGO_PROFILE_RELEASE_LTO", "thin"));
2768        }
2769        if self.release_codegen_units_1 {
2770            env.push(("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"));
2771        }
2772        env
2773    }
2774}
2775
2776#[derive(Debug, Clone, Serialize, Deserialize)]
2777pub struct TransferConfig {
2778    /// zstd compression level (1-19).
2779    #[serde(default = "default_compression")]
2780    pub compression_level: u32,
2781    /// Patterns to exclude from transfer.
2782    #[serde(default = "default_excludes")]
2783    pub exclude_patterns: Vec<String>,
2784    /// SSH keepalive interval in seconds (`ssh -o ServerAliveInterval=<N>`).
2785    ///
2786    /// When unset, build transfers use 15 seconds. Zero defers to OpenSSH configuration.
2787    #[serde(default, skip_serializing_if = "Option::is_none")]
2788    pub ssh_server_alive_interval_secs: Option<u64>,
2789    /// SSH ControlPersist idle timeout in seconds (`ssh -o ControlPersist=<N>s`).
2790    ///
2791    /// - Unset preserves OpenSSH defaults (ControlPersist enabled by default via mux).
2792    /// - `0` disables persistence (`ControlPersist=no`).
2793    #[serde(default, skip_serializing_if = "Option::is_none")]
2794    pub ssh_control_persist_secs: Option<u64>,
2795    /// Staging base on the worker for the transfer paths that do NOT mirror
2796    /// the client's project layout: `--clean-overlay` roots, Windows workers,
2797    /// and `rch cache warm`. Must be an absolute path; defaults to
2798    /// [`default_remote_base`] (`/data/tmp/rch`).
2799    ///
2800    /// An ordinary `rch exec` does not use this: it mirrors the project under
2801    /// the worker's canonical root (`[path_topology] canonical_root`), and the
2802    /// pooled Cargo target store rides along inside that mirror. To move
2803    /// pooled stores onto another filesystem, set
2804    /// `[remediation.pooled_target] store_base` (issue #64).
2805    #[serde(default = "default_remote_base")]
2806    pub remote_base: String,
2807    /// Per-attempt timeout for source synchronization, in milliseconds.
2808    ///
2809    /// This is deliberately separate from remote compilation and artifact-return
2810    /// timeouts. When unset, RCH derives a conservative timeout from the source
2811    /// payload size (30 seconds plus one second per MiB, capped at one hour).
2812    #[serde(default, skip_serializing_if = "Option::is_none")]
2813    pub sync_timeout_ms: Option<u64>,
2814    /// Silence-based stall timeout for source synchronization, in seconds
2815    /// (issue #59).
2816    ///
2817    /// Aborts a source sync when rsync emits NO output at all — no progress
2818    /// refresh, no stats, no itemized line — for this long. This is deliberately
2819    /// far tighter than the wall-clock `sync_timeout_ms` cap (which scales with
2820    /// payload size up to an hour): a large first-time sync keeps refreshing its
2821    /// progress line and is never killed, while a dead channel or wedged rsync
2822    /// is detected within this window instead of holding the reservation until
2823    /// the wall-clock cap. On a stall the hook releases the worker and re-enters
2824    /// selection excluding it. `0` disables silence detection.
2825    #[serde(default = "default_source_sync_silence_timeout_secs")]
2826    pub source_sync_silence_timeout_secs: u64,
2827    /// Retry policy for transient network errors during transfer.
2828    #[serde(default)]
2829    pub retry: RetryConfig,
2830    /// Verify artifact integrity using blake3 hashes after transfer (bd-377q).
2831    ///
2832    /// When enabled, computes blake3 hashes of key artifacts (binaries, test outputs)
2833    /// on the worker after build, then verifies them locally after download.
2834    /// Disabled by default to avoid extra overhead.
2835    #[serde(default)]
2836    pub verify_artifacts: bool,
2837    /// Maximum file size (bytes) for artifact verification (bd-377q).
2838    ///
2839    /// Files larger than this limit are skipped during verification to avoid
2840    /// excessive I/O. Defaults to 100MB.
2841    #[serde(default = "default_verify_max_size")]
2842    pub verify_max_size_bytes: u64,
2843
2844    // =========================================================================
2845    // Transfer Optimization (bd-3hho)
2846    // =========================================================================
2847    /// Maximum transfer size in MB before skipping remote execution.
2848    ///
2849    /// When set, runs `rsync --dry-run --stats` to estimate transfer size.
2850    /// If the estimated size exceeds this threshold, remote offload is skipped
2851    /// and the command runs locally. Set to `None` (default) to disable.
2852    #[serde(default, skip_serializing_if = "Option::is_none")]
2853    pub max_transfer_mb: Option<u64>,
2854
2855    /// Maximum estimated transfer time in milliseconds before skipping remote.
2856    ///
2857    /// Uses `estimated_bandwidth_bps` (or measured link speed) to calculate
2858    /// expected transfer time. If it exceeds this threshold, remote offload
2859    /// is skipped. Set to `None` (default) to disable.
2860    #[serde(default, skip_serializing_if = "Option::is_none")]
2861    pub max_transfer_time_ms: Option<u64>,
2862
2863    /// Bandwidth limit for rsync in KB/s.
2864    ///
2865    /// Passed as `rsync --bwlimit=<N>` to limit transfer speed and prevent
2866    /// network saturation. Set to `None` or `0` (default) for unlimited.
2867    #[serde(default, skip_serializing_if = "Option::is_none")]
2868    pub bwlimit_kbps: Option<u64>,
2869
2870    /// Explicit rsync binary for every local transfer (issue #66).
2871    ///
2872    /// Unset (the default) lets rch pick: the PATH `rsync` when it is a
2873    /// modern (3.1+) rsync, otherwise a modern rsync from the well-known
2874    /// Homebrew/MacPorts locations, otherwise the PATH binary driven with the
2875    /// openrsync/2.6.9-compatible argv. Set this to force one binary
2876    /// (`~` is expanded; a bare name is looked up on PATH). The
2877    /// `RCH_RSYNC_BIN` environment variable overrides this setting.
2878    #[serde(default, skip_serializing_if = "Option::is_none")]
2879    pub rsync_bin: Option<String>,
2880
2881    /// Estimated link bandwidth in bytes per second.
2882    ///
2883    /// Used for transfer time estimation when `max_transfer_time_ms` is set.
2884    /// If not provided, defaults to 10 MB/s (10485760 bytes/sec).
2885    #[serde(default, skip_serializing_if = "Option::is_none")]
2886    pub estimated_bandwidth_bps: Option<u64>,
2887
2888    // =========================================================================
2889    // Adaptive Compression (bd-243w)
2890    // =========================================================================
2891    /// Enable adaptive zstd compression based on transfer size (bd-243w).
2892    ///
2893    /// When enabled, automatically selects compression level based on estimated
2894    /// payload size:
2895    /// - < 10MB: level 1 (fast, minimal compression)
2896    /// - 10-200MB: level 3 (balanced, default)
2897    /// - > 200MB: level 7 (slower, better compression)
2898    ///
2899    /// When disabled (default), uses the fixed `compression_level` setting.
2900    #[serde(default)]
2901    pub adaptive_compression: bool,
2902
2903    /// Minimum compression level for adaptive mode (bd-243w).
2904    ///
2905    /// Even in adaptive mode, compression level won't go below this.
2906    /// Defaults to 1.
2907    #[serde(default = "default_min_compression")]
2908    pub min_compression_level: u32,
2909
2910    /// Maximum compression level for adaptive mode (bd-243w).
2911    ///
2912    /// Even in adaptive mode, compression level won't exceed this.
2913    /// Defaults to 9 (avoids CPU-intensive levels 10+).
2914    #[serde(default = "default_max_compression")]
2915    pub max_compression_level: u32,
2916}
2917
2918impl Default for TransferConfig {
2919    fn default() -> Self {
2920        Self {
2921            compression_level: 3,
2922            exclude_patterns: default_excludes(),
2923            ssh_server_alive_interval_secs: None,
2924            ssh_control_persist_secs: None,
2925            remote_base: default_remote_base(),
2926            sync_timeout_ms: None,
2927            source_sync_silence_timeout_secs: default_source_sync_silence_timeout_secs(),
2928            retry: RetryConfig::default(),
2929            verify_artifacts: false,
2930            verify_max_size_bytes: default_verify_max_size(),
2931            // Transfer optimization (bd-3hho)
2932            max_transfer_mb: None,
2933            max_transfer_time_ms: None,
2934            bwlimit_kbps: None,
2935            rsync_bin: None,
2936            estimated_bandwidth_bps: None,
2937            // Adaptive compression (bd-243w)
2938            adaptive_compression: false,
2939            min_compression_level: default_min_compression(),
2940            max_compression_level: default_max_compression(),
2941        }
2942    }
2943}
2944
2945impl TransferConfig {
2946    /// Smallest accepted explicit source-sync timeout (one second).
2947    pub const MIN_SYNC_TIMEOUT_MS: u64 = 1_000;
2948    /// Largest accepted explicit or derived source-sync timeout (one hour).
2949    pub const MAX_SYNC_TIMEOUT_MS: u64 = 3_600_000;
2950    /// Fixed floor used by the payload-aware default policy.
2951    pub const DEFAULT_SYNC_TIMEOUT_FLOOR_MS: u64 = 30_000;
2952    const DEFAULT_SYNC_BYTES_PER_SECOND: u64 = 1024 * 1024;
2953
2954    /// Whether an explicit source-sync timeout is within the supported range.
2955    #[must_use]
2956    pub fn valid_sync_timeout_ms(value: u64) -> bool {
2957        (Self::MIN_SYNC_TIMEOUT_MS..=Self::MAX_SYNC_TIMEOUT_MS).contains(&value)
2958    }
2959
2960    /// Resolve the per-attempt source-sync timeout for an exact payload size.
2961    ///
2962    /// Explicit configuration wins. Otherwise the default grants 30 seconds of
2963    /// setup time plus one second per MiB at a conservative 1 MiB/s floor. This
2964    /// prevents a cold, large pinned tree from inheriting the old fixed 30-second
2965    /// ceiling while still bounding every individual attempt.
2966    #[must_use]
2967    pub fn sync_timeout_for_payload(&self, payload_bytes: u64) -> std::time::Duration {
2968        let timeout_ms = self
2969            .sync_timeout_ms
2970            .filter(|value| Self::valid_sync_timeout_ms(*value))
2971            .unwrap_or_else(|| {
2972                let transfer_seconds = payload_bytes
2973                    .saturating_add(Self::DEFAULT_SYNC_BYTES_PER_SECOND - 1)
2974                    / Self::DEFAULT_SYNC_BYTES_PER_SECOND;
2975                Self::DEFAULT_SYNC_TIMEOUT_FLOOR_MS
2976                    .saturating_add(transfer_seconds.saturating_mul(1_000))
2977                    .min(Self::MAX_SYNC_TIMEOUT_MS)
2978            });
2979        std::time::Duration::from_millis(timeout_ms)
2980    }
2981
2982    /// Select compression level based on estimated transfer size (bd-243w).
2983    ///
2984    /// When adaptive compression is enabled, selects an appropriate level
2985    /// based on payload size:
2986    /// - < 10MB: level 1 (fast, minimal compression)
2987    /// - 10-200MB: level 3 (balanced)
2988    /// - > 200MB: level 7 (better compression)
2989    ///
2990    /// The result is clamped between `min_compression_level` and `max_compression_level`.
2991    ///
2992    /// When adaptive compression is disabled, returns the fixed `compression_level`.
2993    pub fn select_compression_level(&self, estimated_bytes: Option<u64>) -> u32 {
2994        if !self.adaptive_compression {
2995            return self.compression_level;
2996        }
2997
2998        let Some(bytes) = estimated_bytes else {
2999            // No estimate available, use default
3000            return self.compression_level;
3001        };
3002
3003        // Size thresholds
3004        const SMALL_THRESHOLD: u64 = 10_000_000; // 10 MB (decimal)
3005        const LARGE_THRESHOLD: u64 = 200_000_000; // 200 MB (decimal)
3006
3007        let level = if bytes < SMALL_THRESHOLD {
3008            1 // Fast compression for small payloads
3009        } else if bytes < LARGE_THRESHOLD {
3010            3 // Balanced for medium payloads
3011        } else {
3012            7 // Better compression for large payloads
3013        };
3014
3015        // Clamp to configured bounds
3016        level.clamp(self.min_compression_level, self.max_compression_level)
3017    }
3018}
3019
3020// =============================================================================
3021// Execution Configuration (bd-785w)
3022// =============================================================================
3023
3024/// Default allowlist of commands that can be executed remotely.
3025/// Aligned with the classifier's supported CompilationKind values.
3026/// Command bases of every compilation kind rch knows how to offload.
3027///
3028/// This is the built-in `execution.allowlist`. It is also what a persisted,
3029/// explicit allowlist is compared against: a config file written before a tool
3030/// family was added (Go, Nix, TypeScript, ...) silently keeps that family local,
3031/// and the only symptom is a `[RCH] local (... not in execution.allowlist)` line
3032/// (issue #51). `ExecutionConfig::missing_builtin_bases` surfaces the gap.
3033#[rustfmt::skip]
3034pub const BUILTIN_EXECUTION_BASES: &[&str] = &[
3035    // Rust
3036    "cargo",
3037    "rustc",
3038    // cargo-nextest
3039    "nextest",
3040    // C/C++
3041    "gcc",
3042    "g++",
3043    "clang",
3044    "clang++",
3045    "cc",
3046    "c++",
3047    // Build systems
3048    "make",
3049    "cmake",
3050    "ninja",
3051    "meson",
3052    // Bun
3053    "bun",
3054    // Nix (covers both `nix build` and the legacy `nix-build`)
3055    "nix",
3056    // Go
3057    "go",
3058    // TypeScript (covers both `tsc` and `npx tsc`; command_base is "tsc")
3059    "tsc",
3060];
3061
3062fn default_execution_allowlist() -> Vec<String> {
3063    BUILTIN_EXECUTION_BASES
3064        .iter()
3065        .map(|base| (*base).to_string())
3066        .collect()
3067}
3068
3069/// Execution allowlist configuration for remote builds (bd-785w).
3070///
3071/// Controls which command base names are permitted for remote execution.
3072/// Commands not in the allowlist will fail-open to local execution.
3073#[derive(Debug, Clone, Serialize, Deserialize)]
3074pub struct ExecutionConfig {
3075    /// Allowlist of command base names permitted for remote execution.
3076    ///
3077    /// Examples: "cargo", "rustc", "gcc", "bun"
3078    ///
3079    /// If a classified compilation command's base name is not in this list,
3080    /// RCH will fail-open to local execution with a clear reason.
3081    ///
3082    /// An empty allowlist disables all remote execution (local only).
3083    #[serde(default = "default_execution_allowlist")]
3084    pub allowlist: Vec<String>,
3085}
3086
3087impl Default for ExecutionConfig {
3088    fn default() -> Self {
3089        Self {
3090            allowlist: default_execution_allowlist(),
3091        }
3092    }
3093}
3094
3095impl ExecutionConfig {
3096    /// Check if a command base name is allowed for remote execution.
3097    pub fn is_allowed(&self, command_base: &str) -> bool {
3098        self.allowlist
3099            .iter()
3100            .any(|allowed| allowed.eq_ignore_ascii_case(command_base))
3101    }
3102
3103    /// Built-in compilation tool bases that this allowlist does NOT admit.
3104    ///
3105    /// Empty for the default allowlist. Non-empty means some command family rch
3106    /// classifies as compilation will always fall open to local execution on
3107    /// this box — usually because the config file predates that family.
3108    pub fn missing_builtin_bases(&self) -> Vec<&'static str> {
3109        BUILTIN_EXECUTION_BASES
3110            .iter()
3111            .copied()
3112            .filter(|base| !self.is_allowed(base))
3113            .collect()
3114    }
3115}
3116
3117// =============================================================================
3118// Retry Configuration (bd-x1ek)
3119// =============================================================================
3120
3121fn default_retry_max_attempts() -> u32 {
3122    3
3123}
3124
3125fn default_retry_base_delay_ms() -> u64 {
3126    100
3127}
3128
3129fn default_retry_max_delay_ms() -> u64 {
3130    5000
3131}
3132
3133fn default_retry_jitter_factor() -> f64 {
3134    0.1
3135}
3136
3137fn default_retry_total_timeout_ms() -> u64 {
3138    30000
3139}
3140
3141/// Configuration for retry behavior on transient errors.
3142///
3143/// Used by rsync and SSH operations to recover from transient network failures
3144/// without blocking too long or retrying non-recoverable errors.
3145#[derive(Debug, Clone, Serialize, Deserialize)]
3146pub struct RetryConfig {
3147    /// Maximum number of retry attempts (1 = no retries, just initial attempt).
3148    #[serde(default = "default_retry_max_attempts")]
3149    pub max_attempts: u32,
3150
3151    /// Initial delay between retries in milliseconds.
3152    #[serde(default = "default_retry_base_delay_ms")]
3153    pub base_delay_ms: u64,
3154
3155    /// Maximum delay between retries in milliseconds (caps exponential backoff).
3156    #[serde(default = "default_retry_max_delay_ms")]
3157    pub max_delay_ms: u64,
3158
3159    /// Jitter factor (0.0 - 1.0) to randomize delay.
3160    /// A value of 0.1 means ±10% randomization.
3161    #[serde(default = "default_retry_jitter_factor")]
3162    pub jitter_factor: f64,
3163
3164    /// Total timeout for all retry attempts in milliseconds.
3165    /// After this, retries stop even if max_attempts not reached.
3166    #[serde(default = "default_retry_total_timeout_ms")]
3167    pub total_timeout_ms: u64,
3168}
3169
3170impl Default for RetryConfig {
3171    fn default() -> Self {
3172        Self {
3173            max_attempts: default_retry_max_attempts(),
3174            base_delay_ms: default_retry_base_delay_ms(),
3175            max_delay_ms: default_retry_max_delay_ms(),
3176            jitter_factor: default_retry_jitter_factor(),
3177            total_timeout_ms: default_retry_total_timeout_ms(),
3178        }
3179    }
3180}
3181
3182impl RetryConfig {
3183    /// Create a config that disables retries (single attempt only).
3184    pub fn no_retry() -> Self {
3185        Self {
3186            max_attempts: 1,
3187            ..Default::default()
3188        }
3189    }
3190
3191    /// Calculate delay for a given attempt number (0-indexed).
3192    ///
3193    /// Uses exponential backoff with optional jitter.
3194    pub fn delay_for_attempt(&self, attempt: u32) -> std::time::Duration {
3195        if attempt == 0 {
3196            return std::time::Duration::ZERO;
3197        }
3198
3199        // Exponential backoff: base * 2^attempt
3200        let base = self.base_delay_ms as f64;
3201        let delay = base * (2.0_f64.powi(attempt as i32 - 1));
3202        let capped = delay.min(self.max_delay_ms as f64);
3203
3204        // Apply jitter: delay ± (delay * jitter_factor)
3205        let jittered = if self.jitter_factor > 0.0 {
3206            let jitter_range = capped * self.jitter_factor;
3207            let jitter = (rand::rng().random::<f64>() * 2.0 - 1.0) * jitter_range;
3208            (capped + jitter).max(0.0)
3209        } else {
3210            capped
3211        };
3212
3213        std::time::Duration::from_millis(jittered as u64)
3214    }
3215
3216    /// Check if we should retry given elapsed time since start.
3217    pub fn should_retry(&self, attempt: u32, elapsed: std::time::Duration) -> bool {
3218        if attempt >= self.max_attempts {
3219            return false;
3220        }
3221        elapsed.as_millis() < self.total_timeout_ms as u128
3222    }
3223}
3224
3225/// Default source-sync silence timeout (issue #59): two minutes without a
3226/// single byte of rsync output means a dead channel or wedged rsync, not a
3227/// large transfer (progress2 refreshes its line continuously while bytes
3228/// flow, and each refresh counts as forward progress).
3229fn default_source_sync_silence_timeout_secs() -> u64 {
3230    120
3231}
3232
3233/// Default remote base path for project transfers.
3234///
3235/// Rooted under `/data/tmp` (the fleet-standard managed ephemeral-build zone)
3236/// rather than `/tmp`: on the Linux workers `/tmp` is a RAM-backed tmpfs, so
3237/// dispatched build artifacts placed there consume RAM and evade the disk-GC
3238/// (sbh / datatmp-janitor) that watches `/data/tmp`. Keeping the remote base in
3239/// the managed zone lets those reapers reclaim orphaned build trees. Must agree
3240/// with the rchd cache-cleanup and reclaim `remote_base` defaults.
3241pub fn default_remote_base() -> String {
3242    "/data/tmp/rch".to_string()
3243}
3244
3245/// Validate and normalize a remote base path.
3246/// Returns an error if the path is not absolute or contains traversal sequences.
3247pub fn validate_remote_base(path: &str) -> Result<String, String> {
3248    // Expand tilde for user convenience
3249    let expanded = shellexpand::tilde(path).into_owned();
3250
3251    // Check for absolute path
3252    if !expanded.starts_with('/') {
3253        return Err(format!(
3254            "remote_base must be an absolute path, got: {}",
3255            path
3256        ));
3257    }
3258
3259    // Check for path traversal attempts
3260    if expanded.contains("..") {
3261        return Err(format!(
3262            "remote_base must not contain path traversal (..): {}",
3263            path
3264        ));
3265    }
3266
3267    // Normalize: remove trailing slashes
3268    let normalized = expanded.trim_end_matches('/').to_string();
3269
3270    // Safety checks
3271    if normalized.is_empty() || normalized == "/" {
3272        return Err("remote_base cannot be the root directory (safety restriction)".to_string());
3273    }
3274
3275    // Enforce at least 2 levels deep (e.g. /tmp/rch, /home/user/rch)
3276    // /tmp -> depth 1 (unsafe for recursive delete)
3277    // /tmp/rch -> depth 2 (safe)
3278    let components: Vec<&str> = normalized.split('/').filter(|c| !c.is_empty()).collect();
3279    if components.len() < 2 {
3280        return Err(format!(
3281            "remote_base must be at least 2 levels deep (e.g. /tmp/rch), got: {}",
3282            normalized
3283        ));
3284    }
3285
3286    Ok(normalized)
3287}
3288
3289fn default_circuit_failure_threshold() -> u32 {
3290    3
3291}
3292
3293fn default_circuit_success_threshold() -> u32 {
3294    2
3295}
3296
3297fn default_circuit_error_rate_threshold() -> f64 {
3298    0.5
3299}
3300
3301fn default_circuit_window_secs() -> u64 {
3302    60
3303}
3304
3305fn default_circuit_open_cooldown_secs() -> u64 {
3306    30
3307}
3308
3309fn default_circuit_half_open_max_probes() -> u32 {
3310    1
3311}
3312
3313fn default_circuit_command_failure_threshold() -> u32 {
3314    2
3315}
3316
3317fn default_circuit_open_cooldown_max_secs() -> u64 {
3318    // 30s base doubled four times: 30, 60, 120, 240, 480.
3319    480
3320}
3321
3322fn default_true() -> bool {
3323    true
3324}
3325
3326fn default_log_level() -> String {
3327    "info".to_string()
3328}
3329
3330pub fn default_socket_path() -> String {
3331    // Prefer XDG_RUNTIME_DIR if available (per-user runtime directory).
3332    if let Ok(runtime_dir) = std::env::var("XDG_RUNTIME_DIR")
3333        && !runtime_dir.trim().is_empty()
3334    {
3335        let path = PathBuf::from(runtime_dir).join("rch.sock");
3336        return path.to_string_lossy().to_string();
3337    }
3338
3339    // Fallback to ~/.cache/rch/rch.sock (persistent across reboots).
3340    if let Some(cache_dir) = dirs::cache_dir() {
3341        let rch_cache = cache_dir.join("rch");
3342        let _ = std::fs::create_dir_all(&rch_cache);
3343        return rch_cache.join("rch.sock").to_string_lossy().to_string();
3344    }
3345
3346    // Last resort: /tmp/rch.sock
3347    "/tmp/rch.sock".to_string()
3348}
3349
3350#[allow(dead_code)]
3351fn default_output_visibility() -> OutputVisibility {
3352    OutputVisibility::None
3353}
3354
3355fn default_confidence() -> f64 {
3356    0.85
3357}
3358
3359fn default_min_local_time() -> u64 {
3360    2000
3361}
3362
3363/// Default speedup threshold: 1.2 (require 20% speedup to offload).
3364/// This is conservative to avoid offloading builds that would be only
3365/// marginally faster remotely when accounting for transfer overhead.
3366fn default_speedup_threshold() -> f64 {
3367    1.2
3368}
3369
3370fn default_compression() -> u32 {
3371    3
3372}
3373
3374/// Default maximum file size for artifact verification (100MB).
3375fn default_verify_max_size() -> u64 {
3376    100 * 1024 * 1024 // 100MB
3377}
3378
3379/// Default minimum compression level for adaptive mode.
3380fn default_min_compression() -> u32 {
3381    1
3382}
3383
3384/// Default maximum compression level for adaptive mode.
3385fn default_max_compression() -> u32 {
3386    9
3387}
3388
3389fn default_excludes() -> Vec<String> {
3390    vec![
3391        // Rust build artifacts
3392        "target/".to_string(),
3393        "*.rlib".to_string(),
3394        "*.rmeta".to_string(),
3395        // Git metadata is large, often permission-sensitive on workers, and
3396        // a partial `.git/` tree is not reliable for build scripts anyway.
3397        ".git/".to_string(),
3398        // Node.js / Bun dependencies (massive, reinstalled on worker)
3399        "node_modules/".to_string(),
3400        // Bun cache and runtime files
3401        ".bun/".to_string(),
3402        // npm / pnpm caches
3403        ".npm/".to_string(),
3404        ".pnpm-store/".to_string(),
3405        // Common build output directories
3406        "dist/".to_string(),
3407        "build/".to_string(),
3408        // Framework-specific caches
3409        ".next/".to_string(),
3410        ".nuxt/".to_string(),
3411        ".turbo/".to_string(),
3412        ".parcel-cache/".to_string(),
3413        // Beads runtime state (local issue DB + lock/temp files); not needed remotely.
3414        ".beads/".to_string(),
3415        // Local diagnostic runs may contain host-specific symlinks and are not build inputs.
3416        ".doctor/".to_string(),
3417        // Coverage reports (generated during tests)
3418        "coverage/".to_string(),
3419        ".nyc_output/".to_string(),
3420        // Credentials and secrets (must never be transferred to workers)
3421        ".cargo/credentials".to_string(),
3422        ".cargo/credentials.toml".to_string(),
3423        ".env".to_string(),
3424        ".env.*".to_string(),
3425        "*.pem".to_string(),
3426        "*.key".to_string(),
3427        "credentials.json".to_string(),
3428        "secrets.json".to_string(),
3429        "secrets.yaml".to_string(),
3430        "secrets.yml".to_string(),
3431        "secrets.toml".to_string(),
3432        "secrets.env".to_string(),
3433        "secrets.txt".to_string(),
3434        "secrets.conf".to_string(),
3435        "secrets.cfg".to_string(),
3436        "secrets.properties".to_string(),
3437        "secrets.xml".to_string(),
3438        // The classes above were already covered. These are the ones
3439        // RABS's E027 source-capture seed set denies
3440        // (rabs-sandbox/src/source_capture.rs) that this list did not,
3441        // so the two halves of the product now agree about what a
3442        // secret is. Two components disagreeing on that is the kind of
3443        // gap nobody notices until one of them is the path that ran.
3444        //
3445        // rsync patterns without a `/` match a basename at any depth,
3446        // so these cover nested copies too — a key checked into a
3447        // fixtures directory is excluded exactly like one at the root.
3448        //
3449        // SSH private keys. The public halves (.pub) are deliberately
3450        // not excluded: they are not secret and some builds read them.
3451        "id_rsa".to_string(),
3452        "id_ed25519".to_string(),
3453        "id_ecdsa".to_string(),
3454        "id_dsa".to_string(),
3455        // Key/certificate container formats (*.pem and *.key are above).
3456        "*.p12".to_string(),
3457        "*.pfx".to_string(),
3458        "*.keystore".to_string(),
3459        "*.jks".to_string(),
3460        // Machine credentials and cloud/container config that carries
3461        // long-lived tokens.
3462        ".netrc".to_string(),
3463        "_netrc".to_string(),
3464        ".aws/credentials".to_string(),
3465        ".docker/config.json".to_string(),
3466        // Package-registry auth tokens. These sit in project roots far
3467        // more often than the key formats above and are the likeliest
3468        // real leak in a normal repo.
3469        ".npmrc".to_string(),
3470        ".pypirc".to_string(),
3471        // GnuPG private keyring.
3472        "secring.*".to_string(),
3473    ]
3474}
3475
3476// ============================================================================
3477// Build History Types
3478// ============================================================================
3479
3480/// Location where a build was executed.
3481#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
3482#[serde(rename_all = "snake_case")]
3483pub enum BuildLocation {
3484    /// Build executed locally.
3485    #[default]
3486    Local,
3487    /// Build executed on a remote worker.
3488    Remote,
3489}
3490
3491/// Worker health snapshot captured when a cancellation completes.
3492#[derive(Debug, Clone, Serialize, Deserialize)]
3493pub struct BuildCancellationWorkerHealth {
3494    /// Worker status at cancellation completion.
3495    pub status: String,
3496    /// Worker speed score at cancellation completion.
3497    pub speed_score: f64,
3498    /// Slots currently used on the worker.
3499    pub used_slots: u32,
3500    /// Slots currently available on the worker.
3501    pub available_slots: u32,
3502    /// Normalized pressure state.
3503    pub pressure_state: String,
3504    /// Stable pressure reason code.
3505    pub pressure_reason_code: String,
3506}
3507
3508/// Structured cancellation metadata attached to cancelled build records.
3509#[derive(Debug, Clone, Serialize, Deserialize)]
3510pub struct BuildCancellationMetadata {
3511    /// Stable cancellation operation identifier.
3512    pub operation_id: String,
3513    /// Cancellation origin (for example `user`, `timeout`).
3514    pub origin: String,
3515    /// Stable reason code for cancellation.
3516    pub reason_code: String,
3517    /// Ordered cancellation state path.
3518    #[serde(default)]
3519    pub decision_path: Vec<String>,
3520    /// Highest escalation stage reached (`term`, `remote_kill`, `sigkill`).
3521    pub escalation_stage: String,
3522    /// Number of escalation transitions taken.
3523    pub escalation_count: u32,
3524    /// Whether remote kill was attempted.
3525    pub remote_kill_attempted: bool,
3526    /// Whether cleanup completed successfully.
3527    pub cleanup_ok: bool,
3528    /// Whether active build history was successfully claimed/cancelled.
3529    pub history_cancelled: bool,
3530    /// Final cancellation state.
3531    pub final_state: String,
3532    /// Worker health snapshot at completion.
3533    #[serde(default, skip_serializing_if = "Option::is_none")]
3534    pub worker_health: Option<BuildCancellationWorkerHealth>,
3535}
3536
3537/// Record of a completed build.
3538#[derive(Debug, Clone, Serialize, Deserialize)]
3539pub struct BuildRecord {
3540    /// Unique build identifier.
3541    pub id: u64,
3542    /// When the build started (ISO 8601 timestamp).
3543    pub started_at: String,
3544    /// When the build completed (ISO 8601 timestamp).
3545    pub completed_at: String,
3546    /// Project identifier (usually directory name or hash).
3547    pub project_id: String,
3548    /// Worker that executed the build (None if local).
3549    pub worker_id: Option<String>,
3550    /// Full command executed.
3551    pub command: String,
3552    /// Exit code (0 = success).
3553    pub exit_code: i32,
3554    /// Duration in milliseconds.
3555    pub duration_ms: u64,
3556    /// Build location (local or remote).
3557    pub location: BuildLocation,
3558    /// Bytes transferred (if remote).
3559    pub bytes_transferred: Option<u64>,
3560    /// Optional timing breakdown for the pipeline.
3561    #[serde(default)]
3562    pub timing: Option<CommandTimingBreakdown>,
3563    /// Structured cancellation metadata for cancelled builds.
3564    #[serde(default, skip_serializing_if = "Option::is_none")]
3565    pub cancellation: Option<BuildCancellationMetadata>,
3566}
3567
3568/// Input payload for recording a completed build.
3569#[derive(Debug, Clone, Serialize, Deserialize)]
3570pub struct BuildRecordInput {
3571    /// When the build started (ISO 8601 timestamp).
3572    pub started_at: String,
3573    /// When the build completed (ISO 8601 timestamp).
3574    pub completed_at: String,
3575    /// Project identifier (usually directory name or hash).
3576    pub project_id: String,
3577    /// Worker that executed the build (None if local).
3578    pub worker_id: Option<String>,
3579    /// Full command executed.
3580    pub command: String,
3581    /// Exit code (0 = success).
3582    pub exit_code: i32,
3583    /// Duration in milliseconds.
3584    pub duration_ms: u64,
3585    /// Build location (local or remote).
3586    pub location: BuildLocation,
3587    /// Bytes transferred (if remote).
3588    pub bytes_transferred: Option<u64>,
3589    /// Optional timing breakdown for the pipeline.
3590    #[serde(default)]
3591    pub timing: Option<CommandTimingBreakdown>,
3592    /// Structured cancellation metadata for cancelled builds.
3593    #[serde(default, skip_serializing_if = "Option::is_none")]
3594    pub cancellation: Option<BuildCancellationMetadata>,
3595}
3596
3597impl BuildRecordInput {
3598    /// Convert input payload into a build record with a supplied ID.
3599    pub fn into_record(self, id: u64) -> BuildRecord {
3600        BuildRecord {
3601            id,
3602            started_at: self.started_at,
3603            completed_at: self.completed_at,
3604            project_id: self.project_id,
3605            worker_id: self.worker_id,
3606            command: self.command,
3607            exit_code: self.exit_code,
3608            duration_ms: self.duration_ms,
3609            location: self.location,
3610            bytes_transferred: self.bytes_transferred,
3611            timing: self.timing,
3612            cancellation: self.cancellation,
3613        }
3614    }
3615}
3616
3617/// Aggregate terminal-command statistics for build history.
3618///
3619/// These counts do not infer whether compilation completed before a failing
3620/// test or interrupted command. The denominator includes every stored record.
3621#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3622pub struct BuildStats {
3623    /// Total number of builds in history.
3624    pub total_builds: usize,
3625    /// Number of successfully completed commands (exit_code == 0).
3626    pub success_count: usize,
3627    /// Number of unsuccessful commands, including interruption (exit_code != 0).
3628    pub failure_count: usize,
3629    /// Number of remote builds.
3630    pub remote_count: usize,
3631    /// Number of local builds.
3632    pub local_count: usize,
3633    /// Average build duration in milliseconds.
3634    pub avg_duration_ms: u64,
3635}
3636
3637/// Saved time statistics from remote builds.
3638///
3639/// Tracks estimated time savings from offloading builds to remote workers.
3640/// Estimates are only produced from an observed local baseline; without one the
3641/// stats carry measured remote durations with `estimate_basis = "none"` and
3642/// zero estimated savings rather than a fabricated speedup factor.
3643#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3644pub struct SavedTimeStats {
3645    /// Total time spent on remote builds (milliseconds).
3646    pub total_remote_duration_ms: u64,
3647    /// Estimated total time if those builds ran locally (milliseconds).
3648    /// Zero when no local baseline exists.
3649    pub estimated_local_duration_ms: u64,
3650    /// Time saved by offloading (milliseconds).
3651    /// Computed as max(0, estimated_local - remote); zero without a baseline.
3652    pub time_saved_ms: u64,
3653    /// Number of remote builds included in calculation.
3654    pub builds_counted: usize,
3655    /// Average speedup factor (local_estimate / remote_duration).
3656    /// A value of 2.0 means remote builds are ~2x faster than local.
3657    /// Zero when the estimate basis is "none".
3658    pub avg_speedup: f64,
3659    /// Total time saved today (milliseconds).
3660    pub today_saved_ms: u64,
3661    /// Total time saved this week (milliseconds).
3662    pub week_saved_ms: u64,
3663    /// How the local-time estimate was derived: "observed_local_mean" when
3664    /// successful local builds back the baseline, "none" otherwise.
3665    #[serde(default)]
3666    pub estimate_basis: String,
3667    /// Number of successful local builds backing the baseline (0 for "none").
3668    #[serde(default)]
3669    pub local_baseline_builds: usize,
3670}
3671
3672// ============================================================================
3673// Compilation Timing and Metrics
3674// ============================================================================
3675
3676use std::collections::VecDeque;
3677use std::time::{Duration, Instant};
3678
3679/// Breakdown of timing for each compilation phase.
3680#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3681pub struct CompilationTimingBreakdown {
3682    /// Time to sync source files to worker.
3683    #[serde(with = "duration_millis")]
3684    pub rsync_up: Duration,
3685    /// Time for cargo build on worker.
3686    #[serde(with = "duration_millis")]
3687    pub remote_build: Duration,
3688    /// Time to sync artifacts back.
3689    #[serde(with = "duration_millis")]
3690    pub rsync_down: Duration,
3691    /// Total end-to-end latency.
3692    #[serde(with = "duration_millis")]
3693    pub total: Duration,
3694}
3695
3696/// Timing breakdown for a single command pipeline.
3697///
3698/// Uses optional durations so missing phases serialize as `null`.
3699#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3700pub struct CommandTimingBreakdown {
3701    /// Time spent classifying the command.
3702    #[serde(with = "option_duration_millis")]
3703    pub classify: Option<Duration>,
3704    /// Time spent selecting a worker.
3705    #[serde(with = "option_duration_millis")]
3706    pub select: Option<Duration>,
3707    /// Time spent syncing source files to the worker.
3708    #[serde(with = "option_duration_millis")]
3709    pub sync_up: Option<Duration>,
3710    /// Time spent executing the remote command.
3711    #[serde(with = "option_duration_millis")]
3712    pub exec: Option<Duration>,
3713    /// Time spent syncing artifacts back.
3714    #[serde(with = "option_duration_millis")]
3715    pub sync_down: Option<Duration>,
3716    /// Time spent on cleanup (slot release, bookkeeping).
3717    #[serde(with = "option_duration_millis")]
3718    pub cleanup: Option<Duration>,
3719    /// Total end-to-end time for the pipeline.
3720    #[serde(with = "option_duration_millis")]
3721    pub total: Option<Duration>,
3722}
3723
3724mod duration_millis {
3725    use serde::{Deserialize, Deserializer, Serialize, Serializer};
3726    use std::time::Duration;
3727
3728    pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
3729    where
3730        S: Serializer,
3731    {
3732        duration.as_millis().serialize(serializer)
3733    }
3734
3735    pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
3736    where
3737        D: Deserializer<'de>,
3738    {
3739        let millis = u64::deserialize(deserializer)?;
3740        Ok(Duration::from_millis(millis))
3741    }
3742}
3743
3744/// Comprehensive metrics for a single compilation.
3745#[derive(Debug, Clone, Serialize, Deserialize)]
3746pub struct CompilationMetrics {
3747    /// Project identifier (usually directory name or hash).
3748    pub project_id: String,
3749    /// Worker that performed the compilation.
3750    pub worker_id: String,
3751    /// When the compilation occurred.
3752    pub timestamp: chrono::DateTime<chrono::Utc>,
3753    /// Timing breakdown for each phase.
3754    pub timing: CompilationTimingBreakdown,
3755    /// Local build time for comparison (if available).
3756    #[serde(with = "option_duration_millis")]
3757    pub local_build_time: Option<Duration>,
3758    /// Speedup ratio (local_time / remote_time).
3759    pub speedup: Option<f64>,
3760    /// Number of files synced to worker.
3761    pub files_synced: u64,
3762    /// Total bytes transferred.
3763    pub bytes_transferred: u64,
3764    /// Exit code from compilation.
3765    pub exit_code: i32,
3766    /// Whether compilation succeeded.
3767    pub success: bool,
3768}
3769
3770mod option_duration_millis {
3771    use serde::{Deserialize, Deserializer, Serialize, Serializer};
3772    use std::time::Duration;
3773
3774    pub fn serialize<S>(duration: &Option<Duration>, serializer: S) -> Result<S::Ok, S::Error>
3775    where
3776        S: Serializer,
3777    {
3778        match duration {
3779            Some(d) => Some(d.as_millis() as u64).serialize(serializer),
3780            None => Option::<u64>::None.serialize(serializer),
3781        }
3782    }
3783
3784    pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Duration>, D::Error>
3785    where
3786        D: Deserializer<'de>,
3787    {
3788        let opt: Option<u64> = Option::deserialize(deserializer)?;
3789        Ok(opt.map(Duration::from_millis))
3790    }
3791}
3792
3793impl CompilationMetrics {
3794    /// Calculate speedup based on local vs remote build time.
3795    pub fn calculate_speedup(&mut self) {
3796        if let Some(local) = self.local_build_time
3797            && self.timing.total.as_millis() > 0
3798        {
3799            self.speedup = Some(local.as_secs_f64() / self.timing.total.as_secs_f64());
3800        }
3801    }
3802
3803    /// Check if remote compilation was beneficial (faster than local).
3804    pub fn is_beneficial(&self) -> bool {
3805        self.speedup.map(|s| s > 1.0).unwrap_or(false)
3806    }
3807}
3808
3809impl Default for CompilationMetrics {
3810    fn default() -> Self {
3811        Self {
3812            project_id: String::new(),
3813            worker_id: String::new(),
3814            timestamp: chrono::Utc::now(),
3815            timing: CompilationTimingBreakdown::default(),
3816            local_build_time: None,
3817            speedup: None,
3818            files_synced: 0,
3819            bytes_transferred: 0,
3820            exit_code: 0,
3821            success: true,
3822        }
3823    }
3824}
3825
3826/// Timer for tracking compilation phases.
3827///
3828/// Use this to measure the duration of each phase of remote compilation.
3829///
3830/// # Example
3831/// ```ignore
3832/// let mut timer = CompilationTimer::new("my-project", "worker-1");
3833/// // ... rsync source files ...
3834/// timer.end_rsync_up();
3835/// // ... run cargo build ...
3836/// timer.end_remote_build();
3837/// // ... rsync artifacts back ...
3838/// timer.end_rsync_down();
3839/// let metrics = timer.finish(0, 100, 1_000_000);
3840/// ```
3841#[derive(Debug)]
3842pub struct CompilationTimer {
3843    project_id: String,
3844    worker_id: String,
3845    start: Instant,
3846    phase_start: Instant,
3847    rsync_up: Option<Duration>,
3848    remote_build: Option<Duration>,
3849    rsync_down: Option<Duration>,
3850}
3851
3852impl CompilationTimer {
3853    /// Create a new timer for a compilation.
3854    pub fn new(project_id: &str, worker_id: &str) -> Self {
3855        let now = Instant::now();
3856        Self {
3857            project_id: project_id.to_string(),
3858            worker_id: worker_id.to_string(),
3859            start: now,
3860            phase_start: now,
3861            rsync_up: None,
3862            remote_build: None,
3863            rsync_down: None,
3864        }
3865    }
3866
3867    /// Mark the end of the rsync upload phase.
3868    pub fn end_rsync_up(&mut self) {
3869        self.rsync_up = Some(self.phase_start.elapsed());
3870        self.phase_start = Instant::now();
3871        tracing::info!(
3872            rsync_up_ms = %self.rsync_up.unwrap().as_millis(),
3873            "TIMING: rsync_up completed"
3874        );
3875    }
3876
3877    /// Mark the end of the remote build phase.
3878    pub fn end_remote_build(&mut self) {
3879        self.remote_build = Some(self.phase_start.elapsed());
3880        self.phase_start = Instant::now();
3881        tracing::info!(
3882            remote_build_ms = %self.remote_build.unwrap().as_millis(),
3883            "TIMING: remote_build completed"
3884        );
3885    }
3886
3887    /// Mark the end of the rsync download phase.
3888    pub fn end_rsync_down(&mut self) {
3889        self.rsync_down = Some(self.phase_start.elapsed());
3890        tracing::info!(
3891            rsync_down_ms = %self.rsync_down.unwrap().as_millis(),
3892            "TIMING: rsync_down completed"
3893        );
3894    }
3895
3896    /// Finish timing and produce metrics.
3897    pub fn finish(self, exit_code: i32, files: u64, bytes: u64) -> CompilationMetrics {
3898        let total = self.start.elapsed();
3899        tracing::info!(
3900            total_ms = %total.as_millis(),
3901            exit_code = %exit_code,
3902            "TIMING: compilation completed"
3903        );
3904
3905        CompilationMetrics {
3906            project_id: self.project_id,
3907            worker_id: self.worker_id,
3908            timestamp: chrono::Utc::now(),
3909            timing: CompilationTimingBreakdown {
3910                rsync_up: self.rsync_up.unwrap_or_default(),
3911                remote_build: self.remote_build.unwrap_or_default(),
3912                rsync_down: self.rsync_down.unwrap_or_default(),
3913                total,
3914            },
3915            local_build_time: None,
3916            speedup: None,
3917            files_synced: files,
3918            bytes_transferred: bytes,
3919            exit_code,
3920            success: exit_code == 0,
3921        }
3922    }
3923}
3924
3925/// Aggregator for compilation metrics with statistics.
3926#[derive(Debug)]
3927pub struct MetricsAggregator {
3928    history: VecDeque<CompilationMetrics>,
3929    max_history: usize,
3930}
3931
3932impl MetricsAggregator {
3933    /// Create a new aggregator with a maximum history size.
3934    pub fn new(max_history: usize) -> Self {
3935        Self {
3936            history: VecDeque::with_capacity(max_history),
3937            max_history,
3938        }
3939    }
3940
3941    /// Record a new compilation's metrics.
3942    pub fn record(&mut self, metrics: CompilationMetrics) {
3943        if self.history.len() >= self.max_history {
3944            self.history.pop_front();
3945        }
3946        self.history.push_back(metrics);
3947    }
3948
3949    /// Get the average speedup across all recorded compilations.
3950    pub fn average_speedup(&self) -> Option<f64> {
3951        let speedups: Vec<f64> = self.history.iter().filter_map(|m| m.speedup).collect();
3952
3953        if speedups.is_empty() {
3954            None
3955        } else {
3956            Some(speedups.iter().sum::<f64>() / speedups.len() as f64)
3957        }
3958    }
3959
3960    /// Get the p50 (median) total compilation time.
3961    pub fn p50_total_time(&self) -> Option<Duration> {
3962        self.percentile_total_time(0.50)
3963    }
3964
3965    /// Get the p95 total compilation time.
3966    pub fn p95_total_time(&self) -> Option<Duration> {
3967        self.percentile_total_time(0.95)
3968    }
3969
3970    /// Get the p99 total compilation time.
3971    pub fn p99_total_time(&self) -> Option<Duration> {
3972        self.percentile_total_time(0.99)
3973    }
3974
3975    /// Get a percentile of total compilation times.
3976    ///
3977    /// Uses the "inclusive" percentile method (matching numpy.percentile, Excel PERCENTILE.INC):
3978    /// index = (n - 1) * percentile, where n is the number of samples.
3979    fn percentile_total_time(&self, percentile: f64) -> Option<Duration> {
3980        let mut times: Vec<_> = self.history.iter().map(|m| m.timing.total).collect();
3981
3982        if times.is_empty() {
3983            return None;
3984        }
3985
3986        times.sort();
3987        // Use inclusive percentile: index = (n - 1) * p
3988        // For p50 with 100 items: (99 * 0.50) = 49.5 -> 49 (true median position)
3989        // For p99 with 100 items: (99 * 0.99) = 98.01 -> 98
3990        let idx = (((times.len() - 1) as f64 * percentile).round() as usize).min(times.len() - 1);
3991        Some(times[idx])
3992    }
3993
3994    /// Get the success rate as a percentage.
3995    pub fn success_rate(&self) -> f64 {
3996        if self.history.is_empty() {
3997            return 100.0;
3998        }
3999        let successes = self.history.iter().filter(|m| m.success).count();
4000        (successes as f64 / self.history.len() as f64) * 100.0
4001    }
4002
4003    /// Get the number of recorded compilations.
4004    pub fn count(&self) -> usize {
4005        self.history.len()
4006    }
4007
4008    /// Get all recorded metrics.
4009    pub fn metrics(&self) -> &VecDeque<CompilationMetrics> {
4010        &self.history
4011    }
4012
4013    /// Clear all recorded metrics.
4014    pub fn clear(&mut self) {
4015        self.history.clear();
4016    }
4017}
4018
4019impl Default for MetricsAggregator {
4020    fn default() -> Self {
4021        Self::new(1000)
4022    }
4023}
4024
4025#[cfg(test)]
4026mod tests {
4027    use super::*;
4028    use crate::test_guard;
4029
4030    #[test]
4031    fn test_declared_os_reads_reserved_tag() {
4032        assert_eq!(
4033            declared_os(&["rust".to_string(), "os:windows".to_string()]),
4034            Some("windows".to_string())
4035        );
4036        assert_eq!(
4037            declared_os(&["os:WINDOWS".to_string()]),
4038            Some("windows".to_string())
4039        );
4040    }
4041
4042    #[test]
4043    fn test_declared_os_absent_for_ordinary_tags() {
4044        // Every historical worker: no reserved tag, so no exclusivity and the
4045        // worker stays a candidate for any job.
4046        assert_eq!(declared_os(&[]), None);
4047        assert_eq!(
4048            declared_os(&["rust".to_string(), "bun".to_string(), "go".to_string()]),
4049            None
4050        );
4051        // A malformed declaration must not silently fence the worker off.
4052        assert_eq!(declared_os(&["os:".to_string()]), None);
4053        assert_eq!(declared_os(&["os:  ".to_string()]), None);
4054    }
4055
4056    #[test]
4057    fn test_os_tag_roundtrips_through_declared_os() {
4058        for os in ["windows", "Darwin", " linux "] {
4059            let tag = os_tag(os);
4060            assert_eq!(
4061                declared_os(std::slice::from_ref(&tag)).as_deref(),
4062                Some(os.trim().to_ascii_lowercase().as_str())
4063            );
4064        }
4065    }
4066
4067    /// Verdict-webhook config must NEVER carry an inline secret. Operators
4068    /// reference secrets by env-var name (`*_env`); the resolved value is
4069    /// read at dispatch time and never persisted to the cache or config TOML.
4070    /// This guards against a future careless `pub secret: String` addition.
4071    #[test]
4072    fn test_no_inline_secret_fields() {
4073        let endpoint = DoctorWebhookEndpoint {
4074            name: "ops".to_string(),
4075            url: "https://example.com/hook".to_string(),
4076            format: DoctorWebhookFormat::Pagerduty,
4077            on_transitions: vec!["any_to_failing".to_string()],
4078            retry_max: 3,
4079            retry_backoff_ms: 200,
4080            timeout_ms: 5000,
4081            enabled: true,
4082            routing_key_env: Some("RCH_PAGERDUTY_ROUTING_KEY".to_string()),
4083            bearer_token_env: Some("RCH_WEBHOOK_BEARER".to_string()),
4084            signing_secret_env: Some("RCH_WEBHOOK_HMAC".to_string()),
4085        };
4086        // Serialize to a JSON object and inspect the *keys*. The forbidden
4087        // field names must be entirely absent; only `*_env` references survive.
4088        let value = serde_json::to_value(&endpoint).unwrap();
4089        let obj = value.as_object().expect("endpoint serializes to an object");
4090        for forbidden in ["secret", "routing_key", "bearer_token", "password", "token"] {
4091            assert!(
4092                !obj.contains_key(forbidden),
4093                "endpoint config must not expose an inline `{forbidden}` field"
4094            );
4095        }
4096        assert!(obj.contains_key("routing_key_env"));
4097        assert!(obj.contains_key("bearer_token_env"));
4098        assert!(obj.contains_key("signing_secret_env"));
4099    }
4100
4101    #[test]
4102    fn test_webhook_format_toml_spelling() {
4103        // The config TOML uses snake_case spellings. TOML documents are tables
4104        // at the root, so a bare enum cannot be (de)serialized standalone —
4105        // verify the wire spelling through a `format = "..."` table field, the
4106        // shape it actually appears in within the config.
4107        #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)]
4108        struct FormatWrap {
4109            format: DoctorWebhookFormat,
4110        }
4111        let rendered = toml::to_string(&FormatWrap {
4112            format: DoctorWebhookFormat::GenericJson,
4113        })
4114        .unwrap();
4115        assert!(
4116            rendered.contains("format = \"generic_json\""),
4117            "unexpected TOML: {rendered}"
4118        );
4119        let slack: FormatWrap = toml::from_str("format = \"slack\"").unwrap();
4120        assert_eq!(slack.format, DoctorWebhookFormat::Slack);
4121        let pd: FormatWrap = toml::from_str("format = \"pagerduty\"").unwrap();
4122        assert_eq!(pd.format, DoctorWebhookFormat::Pagerduty);
4123    }
4124
4125    #[test]
4126    fn test_doctor_webhooks_defaults() {
4127        let cfg = DoctorWebhooksConfig::default();
4128        assert!(cfg.endpoints.is_empty());
4129        assert_eq!(cfg.max_inflight, 16);
4130        // RchConfig wires the section in with serde(default), so an empty
4131        // config still yields an empty (never-firing) webhook surface.
4132        let rch: RchConfig = toml::from_str("").unwrap();
4133        assert!(rch.doctor.webhooks.endpoints.is_empty());
4134    }
4135
4136    #[test]
4137    fn test_doctor_webhook_endpoint_parses_from_toml() {
4138        let toml_src = r#"
4139[doctor.webhooks]
4140max_inflight = 8
4141
4142[[doctor.webhooks.endpoints]]
4143name = "ops-slack"
4144url = "https://hooks.slack.com/services/T0/B0/x"
4145format = "slack"
4146on_transitions = ["healthy_to_degraded", "any_to_failing"]
4147retry_max = 2
4148"#;
4149        let cfg: RchConfig = toml::from_str(toml_src).unwrap();
4150        assert_eq!(cfg.doctor.webhooks.max_inflight, 8);
4151        assert_eq!(cfg.doctor.webhooks.endpoints.len(), 1);
4152        let ep = &cfg.doctor.webhooks.endpoints[0];
4153        assert_eq!(ep.name, "ops-slack");
4154        assert_eq!(ep.format, DoctorWebhookFormat::Slack);
4155        assert_eq!(ep.on_transitions.len(), 2);
4156        assert_eq!(ep.retry_max, 2);
4157        // Defaults filled in for omitted fields.
4158        assert_eq!(ep.retry_backoff_ms, 200);
4159        assert_eq!(ep.timeout_ms, 5000);
4160        assert!(ep.enabled);
4161    }
4162
4163    fn assert_golden_json(
4164        actual: serde_json::Value,
4165        expected: &serde_json::Value,
4166        fixture_path: &str,
4167    ) {
4168        if &actual == expected {
4169            return;
4170        }
4171
4172        let actual_json = serde_json::to_string_pretty(&actual).expect("actual JSON renders");
4173        let expected_json = serde_json::to_string_pretty(expected).expect("expected JSON renders");
4174        assert_eq!(
4175            &actual,
4176            expected,
4177            "golden mismatch in {fixture_path}\n\
4178             expected_blake3={}\n\
4179             actual_blake3={}\n\
4180             bless path: review the semantic diff, update the fixture expected_* field, \
4181             and record both hashes in the Beads closeout.\n\
4182             expected:\n{expected_json}\n\
4183             actual:\n{actual_json}",
4184            blake3::hash(expected_json.as_bytes()),
4185            blake3::hash(actual_json.as_bytes())
4186        );
4187    }
4188
4189    #[test]
4190    fn test_circuit_state_default() {
4191        let _guard = test_guard!();
4192        assert_eq!(CircuitState::default(), CircuitState::Closed);
4193    }
4194
4195    #[test]
4196    fn test_circuit_config_defaults() {
4197        let _guard = test_guard!();
4198        let config = CircuitBreakerConfig::default();
4199        assert_eq!(config.failure_threshold, 3);
4200        assert_eq!(config.success_threshold, 2);
4201        assert_eq!(config.error_rate_threshold, 0.5);
4202        assert_eq!(config.window_secs, 60);
4203        assert_eq!(config.open_cooldown_secs, 30);
4204        assert_eq!(config.half_open_max_probes, 1);
4205    }
4206
4207    #[test]
4208    fn test_rch_config_has_circuit_defaults() {
4209        let _guard = test_guard!();
4210        let config = RchConfig::default();
4211        assert_eq!(config.circuit.failure_threshold, 3);
4212    }
4213
4214    #[test]
4215    fn test_circuit_config_serde_roundtrip() {
4216        let _guard = test_guard!();
4217        let config = CircuitBreakerConfig {
4218            failure_threshold: 5,
4219            success_threshold: 3,
4220            error_rate_threshold: 0.75,
4221            window_secs: 120,
4222            open_cooldown_secs: 45,
4223            half_open_max_probes: 2,
4224            command_failure_threshold: 4,
4225            open_cooldown_max_secs: 900,
4226        };
4227        let json = serde_json::to_string(&config).unwrap();
4228        let parsed: CircuitBreakerConfig = serde_json::from_str(&json).unwrap();
4229        assert_eq!(parsed.command_failure_threshold, 4);
4230        assert_eq!(parsed.open_cooldown_max_secs, 900);
4231        assert_eq!(parsed.failure_threshold, 5);
4232        assert_eq!(parsed.success_threshold, 3);
4233        assert_eq!(parsed.error_rate_threshold, 0.75);
4234        assert_eq!(parsed.window_secs, 120);
4235        assert_eq!(parsed.open_cooldown_secs, 45);
4236        assert_eq!(parsed.half_open_max_probes, 2);
4237    }
4238
4239    // ========================================================================
4240    // WorkerCapabilities Tests
4241    // ========================================================================
4242
4243    #[test]
4244    fn test_worker_capabilities_default() {
4245        let _guard = test_guard!();
4246        let caps = WorkerCapabilities::default();
4247        assert!(caps.rustc_version.is_none());
4248        assert!(caps.rustup_toolchains.is_empty());
4249        assert!(caps.rustup_components.is_empty());
4250        assert!(caps.bun_version.is_none());
4251        assert!(caps.node_version.is_none());
4252        assert!(caps.npm_version.is_none());
4253        assert!(caps.projects_root_ok.is_none());
4254        assert!(caps.projects_root_issue.is_none());
4255        assert!(caps.projects_root_checked_at_unix_ms.is_none());
4256    }
4257
4258    #[test]
4259    fn test_worker_capabilities_new() {
4260        let _guard = test_guard!();
4261        let caps = WorkerCapabilities::new();
4262        assert!(caps.rustc_version.is_none());
4263        assert!(!caps.has_rust());
4264        assert!(!caps.has_bun());
4265        assert!(!caps.has_node());
4266        assert!(caps.is_topology_healthy().is_none());
4267    }
4268
4269    #[test]
4270    fn test_worker_capabilities_has_rust() {
4271        let _guard = test_guard!();
4272        let mut caps = WorkerCapabilities::new();
4273        assert!(!caps.has_rust());
4274
4275        caps.rustc_version = Some("rustc 1.76.0 (07dca489a 2024-02-04)".to_string());
4276        assert!(caps.has_rust());
4277    }
4278
4279    #[test]
4280    fn test_worker_capabilities_components_are_toolchain_scoped_and_exact() {
4281        let _guard = test_guard!();
4282        let caps = WorkerCapabilities {
4283            rustup_components: vec![
4284                "stable-x86_64-unknown-linux-gnu:clippy".to_string(),
4285                "nightly-2026-07-05-x86_64-unknown-linux-gnu:rustfmt".to_string(),
4286                "beta-x86_64-unknown-linux-gnu:clippy-preview".to_string(),
4287            ],
4288            ..Default::default()
4289        };
4290
4291        assert!(caps.has_rustup_component("stable", "clippy"));
4292        assert!(caps.has_rustup_component("nightly-2026-07-05", "rustfmt"));
4293        assert!(!caps.has_rustup_component("nightly-2026-07-05", "clippy"));
4294        assert!(!caps.has_rustup_component("beta", "clippy"));
4295        assert!(!caps.has_rustup_component("nightly-2026-07-0", "rustfmt"));
4296    }
4297
4298    #[test]
4299    fn test_worker_capabilities_has_bun() {
4300        let _guard = test_guard!();
4301        let mut caps = WorkerCapabilities::new();
4302        assert!(!caps.has_bun());
4303
4304        caps.bun_version = Some("1.0.25".to_string());
4305        assert!(caps.has_bun());
4306    }
4307
4308    #[test]
4309    fn test_worker_capabilities_has_nix() {
4310        let _guard = test_guard!();
4311        let mut caps = WorkerCapabilities::new();
4312        assert!(!caps.has_nix());
4313
4314        caps.nix_version = Some("nix (Nix) 2.24.9".to_string());
4315        assert!(caps.has_nix());
4316    }
4317
4318    #[test]
4319    fn test_worker_capabilities_has_zig_requires_both_halves() {
4320        let _guard = test_guard!();
4321        let mut caps = WorkerCapabilities::new();
4322        assert!(!caps.has_zig());
4323
4324        // `zig` alone cannot run `cargo zigbuild` — cargo would report
4325        // `no such command: 'zigbuild'`.
4326        caps.zig_version = Some("0.14.1".to_string());
4327        assert!(!caps.has_zig());
4328
4329        // cargo-zigbuild alone cannot link — it shells out to `zig cc`.
4330        caps.zig_version = None;
4331        caps.cargo_zigbuild_version = Some("cargo-zigbuild 0.23.0".to_string());
4332        assert!(!caps.has_zig());
4333
4334        caps.zig_version = Some("0.14.1".to_string());
4335        assert!(caps.has_zig());
4336    }
4337
4338    #[test]
4339    fn test_nix_build_uses_generous_timeout() {
4340        let _guard = test_guard!();
4341        let cfg = CompilationConfig::default();
4342        // Nix builds get the long test-timeout bucket, not the short build timeout.
4343        assert_eq!(
4344            cfg.timeout_for_kind(Some(CompilationKind::NixBuild)),
4345            std::time::Duration::from_secs(cfg.test_timeout_sec)
4346        );
4347    }
4348
4349    #[test]
4350    fn test_job_uses_build_timeout_bucket() {
4351        let _guard = test_guard!();
4352        // Jobs (explicit `rch exec --job`) reuse the standard build-timeout
4353        // bucket, so `RCH_BUILD_TIMEOUT_SEC` governs them without new config.
4354        let cfg = CompilationConfig::default();
4355        assert_eq!(
4356            cfg.timeout_for_kind(Some(CompilationKind::Job)),
4357            std::time::Duration::from_secs(cfg.build_timeout_sec)
4358        );
4359    }
4360
4361    #[test]
4362    fn test_layer0_disabled_by_default_and_env_gated() {
4363        let _guard = test_guard!();
4364        let cfg = Layer0Config::default();
4365        assert!(!cfg.enabled);
4366        // Master switch off ⇒ nothing injects even with knobs on.
4367        let knobs_on = Layer0Config {
4368            enabled: false,
4369            release_lto_thin: true,
4370            release_codegen_units_1: true,
4371        };
4372        assert!(cfg.active_env().is_empty());
4373        assert!(knobs_on.active_env().is_empty());
4374    }
4375
4376    #[test]
4377    fn test_layer0_knobs_emit_stable_env_pairs() {
4378        let _guard = test_guard!();
4379        let lto_only = Layer0Config {
4380            enabled: true,
4381            release_lto_thin: true,
4382            release_codegen_units_1: false,
4383        };
4384        assert_eq!(
4385            lto_only.active_env(),
4386            vec![("CARGO_PROFILE_RELEASE_LTO", "thin")]
4387        );
4388        let both = Layer0Config {
4389            enabled: true,
4390            release_lto_thin: true,
4391            release_codegen_units_1: true,
4392        };
4393        assert_eq!(
4394            both.active_env(),
4395            vec![
4396                ("CARGO_PROFILE_RELEASE_LTO", "thin"),
4397                ("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"),
4398            ],
4399            "emission order must stay stable for deterministic baselines"
4400        );
4401    }
4402
4403    #[test]
4404    fn test_layer0_pack_version_is_recorded() {
4405        let _guard = test_guard!();
4406        // Baselines record the pack version; it must exist and be non-empty.
4407        assert!(!LAYER0_PACK_VERSION.is_empty());
4408    }
4409    #[test]
4410    fn test_default_allowlist_includes_nix() {
4411        let _guard = test_guard!();
4412        let cfg = ExecutionConfig::default();
4413        assert!(cfg.is_allowed("nix"));
4414    }
4415
4416    #[test]
4417    fn test_worker_capabilities_has_node() {
4418        let _guard = test_guard!();
4419        let mut caps = WorkerCapabilities::new();
4420        assert!(!caps.has_node());
4421
4422        caps.node_version = Some("v20.11.0".to_string());
4423        assert!(caps.has_node());
4424    }
4425
4426    #[test]
4427    fn test_worker_capabilities_multiple_runtimes() {
4428        let _guard = test_guard!();
4429        let caps = WorkerCapabilities {
4430            rustc_version: Some("rustc 1.76.0".to_string()),
4431            bun_version: Some("1.0.25".to_string()),
4432            node_version: Some("v20.11.0".to_string()),
4433            npm_version: Some("10.2.4".to_string()),
4434            ..Default::default()
4435        };
4436
4437        assert!(caps.has_rust());
4438        assert!(caps.has_bun());
4439        assert!(caps.has_node());
4440    }
4441
4442    #[test]
4443    fn test_worker_capabilities_serialization_empty() {
4444        let _guard = test_guard!();
4445        let caps = WorkerCapabilities::new();
4446        let json = serde_json::to_string(&caps).unwrap();
4447        // Empty capabilities should serialize to {}
4448        assert_eq!(json, "{}");
4449
4450        // And deserialize back correctly
4451        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4452        assert!(!parsed.has_rust());
4453        assert!(!parsed.has_bun());
4454        assert!(!parsed.has_node());
4455    }
4456
4457    #[test]
4458    fn test_worker_capabilities_serialization_with_versions() {
4459        let _guard = test_guard!();
4460        let caps = WorkerCapabilities {
4461            rustc_version: Some("rustc 1.76.0-nightly".to_string()),
4462            bun_version: Some("1.0.25".to_string()),
4463            node_version: None,
4464            npm_version: None,
4465            ..Default::default()
4466        };
4467
4468        let json = serde_json::to_string(&caps).unwrap();
4469        assert!(json.contains("rustc_version"));
4470        assert!(json.contains("rustc 1.76.0-nightly"));
4471        assert!(json.contains("bun_version"));
4472        assert!(json.contains("1.0.25"));
4473        // None fields should be skipped
4474        assert!(!json.contains("node_version"));
4475        assert!(!json.contains("npm_version"));
4476
4477        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4478        assert!(parsed.has_rust());
4479        assert!(parsed.has_bun());
4480        assert!(!parsed.has_node());
4481    }
4482
4483    #[test]
4484    fn test_worker_capabilities_deserialization_partial() {
4485        let _guard = test_guard!();
4486        // Deserialize JSON with only some fields
4487        let json = r#"{"bun_version": "1.0.0"}"#;
4488        let caps: WorkerCapabilities = serde_json::from_str(json).unwrap();
4489
4490        assert!(!caps.has_rust());
4491        assert!(caps.has_bun());
4492        assert!(!caps.has_node());
4493        assert_eq!(caps.bun_version, Some("1.0.0".to_string()));
4494    }
4495
4496    #[test]
4497    fn test_worker_capabilities_clone() {
4498        let _guard = test_guard!();
4499        let caps = WorkerCapabilities {
4500            rustc_version: Some("1.76.0".to_string()),
4501            bun_version: None,
4502            node_version: Some("v20".to_string()),
4503            npm_version: None,
4504            projects_root_ok: Some(false),
4505            projects_root_issue: Some("alias_missing".to_string()),
4506            projects_root_checked_at_unix_ms: Some(123),
4507            ..Default::default()
4508        };
4509
4510        let cloned = caps.clone();
4511        assert_eq!(cloned.rustc_version, caps.rustc_version);
4512        assert_eq!(cloned.bun_version, caps.bun_version);
4513        assert_eq!(cloned.node_version, caps.node_version);
4514        assert_eq!(cloned.npm_version, caps.npm_version);
4515        assert_eq!(cloned.projects_root_ok, caps.projects_root_ok);
4516        assert_eq!(cloned.projects_root_issue, caps.projects_root_issue);
4517        assert_eq!(
4518            cloned.projects_root_checked_at_unix_ms,
4519            caps.projects_root_checked_at_unix_ms
4520        );
4521    }
4522
4523    #[test]
4524    fn test_worker_capabilities_topology_health_helper() {
4525        let _guard = test_guard!();
4526        let mut caps = WorkerCapabilities::new();
4527        assert_eq!(caps.is_topology_healthy(), None);
4528
4529        caps.projects_root_ok = Some(true);
4530        assert_eq!(caps.is_topology_healthy(), Some(true));
4531
4532        caps.projects_root_ok = Some(false);
4533        assert_eq!(caps.is_topology_healthy(), Some(false));
4534    }
4535
4536    #[test]
4537    fn test_selection_reason_serialization() {
4538        let _guard = test_guard!();
4539        // Test that all variants serialize to snake_case
4540        assert_eq!(
4541            serde_json::to_string(&SelectionReason::Success).unwrap(),
4542            "\"success\""
4543        );
4544        assert_eq!(
4545            serde_json::to_string(&SelectionReason::NoWorkersConfigured).unwrap(),
4546            "\"no_workers_configured\""
4547        );
4548        assert_eq!(
4549            serde_json::to_string(&SelectionReason::AllWorkersUnreachable).unwrap(),
4550            "\"all_workers_unreachable\""
4551        );
4552        assert_eq!(
4553            serde_json::to_string(&SelectionReason::AllCircuitsOpen).unwrap(),
4554            "\"all_circuits_open\""
4555        );
4556        assert_eq!(
4557            serde_json::to_string(&SelectionReason::AllWorkersBusy).unwrap(),
4558            "\"all_workers_busy\""
4559        );
4560        assert_eq!(
4561            serde_json::to_string(&SelectionReason::NoWorkersPassedHealth).unwrap(),
4562            "\"no_workers_passed_health\""
4563        );
4564        assert_eq!(
4565            serde_json::to_string(&SelectionReason::AllWorkersFailedPreflight).unwrap(),
4566            "\"all_workers_failed_preflight\""
4567        );
4568        assert_eq!(
4569            serde_json::to_string(&SelectionReason::AllWorkersFailedConvergence).unwrap(),
4570            "\"all_workers_failed_convergence\""
4571        );
4572        assert_eq!(
4573            serde_json::to_string(&SelectionReason::NoAdmissibleWorkers(
4574                "critical_pressure=1,insufficient_slots=1".to_string()
4575            ))
4576            .unwrap(),
4577            "{\"no_admissible_workers\":\"critical_pressure=1,insufficient_slots=1\"}"
4578        );
4579        assert_eq!(
4580            serde_json::to_string(&SelectionReason::NoMatchingWorkers).unwrap(),
4581            "\"no_matching_workers\""
4582        );
4583    }
4584
4585    #[test]
4586    fn test_selection_reason_with_error() {
4587        let _guard = test_guard!();
4588        let reason = SelectionReason::SelectionError("test error".to_string());
4589        let json = serde_json::to_string(&reason).unwrap();
4590        assert!(json.contains("selection_error"));
4591        assert!(json.contains("test error"));
4592    }
4593
4594    #[test]
4595    fn test_selection_reason_deserialization() {
4596        let _guard = test_guard!();
4597        assert_eq!(
4598            serde_json::from_str::<SelectionReason>("\"success\"").unwrap(),
4599            SelectionReason::Success
4600        );
4601        assert_eq!(
4602            serde_json::from_str::<SelectionReason>("\"all_workers_busy\"").unwrap(),
4603            SelectionReason::AllWorkersBusy
4604        );
4605        assert_eq!(
4606            serde_json::from_str::<SelectionReason>("\"no_workers_passed_health\"").unwrap(),
4607            SelectionReason::NoWorkersPassedHealth
4608        );
4609        assert_eq!(
4610            serde_json::from_str::<SelectionReason>("\"all_workers_failed_preflight\"").unwrap(),
4611            SelectionReason::AllWorkersFailedPreflight
4612        );
4613        assert_eq!(
4614            serde_json::from_str::<SelectionReason>("\"all_workers_failed_convergence\"").unwrap(),
4615            SelectionReason::AllWorkersFailedConvergence
4616        );
4617        assert_eq!(
4618            serde_json::from_str::<SelectionReason>(
4619                "{\"no_admissible_workers\":\"critical_pressure=1\"}"
4620            )
4621            .unwrap(),
4622            SelectionReason::NoAdmissibleWorkers("critical_pressure=1".to_string())
4623        );
4624    }
4625
4626    #[test]
4627    fn test_selection_reason_display() {
4628        let _guard = test_guard!();
4629        assert_eq!(
4630            SelectionReason::Success.to_string(),
4631            "worker assigned successfully"
4632        );
4633        assert_eq!(
4634            SelectionReason::NoWorkersConfigured.to_string(),
4635            "no workers configured"
4636        );
4637        assert_eq!(
4638            SelectionReason::AllWorkersUnreachable.to_string(),
4639            "all workers unreachable"
4640        );
4641        assert_eq!(
4642            SelectionReason::AllWorkersBusy.to_string(),
4643            "all workers at capacity"
4644        );
4645        assert_eq!(
4646            SelectionReason::NoWorkersPassedHealth.to_string(),
4647            "no workers passed health thresholds"
4648        );
4649        assert_eq!(
4650            SelectionReason::AllWorkersFailedPreflight.to_string(),
4651            "all workers failed preflight checks"
4652        );
4653        assert_eq!(
4654            SelectionReason::AllWorkersFailedConvergence.to_string(),
4655            "all workers failed repo convergence checks"
4656        );
4657        assert_eq!(
4658            SelectionReason::NoAdmissibleWorkers(
4659                "critical_pressure=1,insufficient_slots=1".to_string()
4660            )
4661            .to_string(),
4662            "no admissible workers: critical_pressure=1,insufficient_slots=1"
4663        );
4664        assert_eq!(
4665            SelectionReason::SelectionError("oops".to_string()).to_string(),
4666            "selection error: oops"
4667        );
4668    }
4669
4670    #[test]
4671    fn test_selection_response_with_worker() {
4672        let _guard = test_guard!();
4673        let response = SelectionResponse {
4674            worker: Some(SelectedWorker {
4675                id: WorkerId::new("test"),
4676                host: "localhost".to_string(),
4677                user: "user".to_string(),
4678                identity_file: "~/.ssh/id_rsa".to_string(),
4679                slots_available: 8,
4680                speed_score: 75.0,
4681                declared_os: None,
4682            }),
4683            reason: SelectionReason::Success,
4684            build_id: None,
4685            diagnostics: None,
4686        };
4687
4688        let json = serde_json::to_string(&response).unwrap();
4689        assert!(json.contains("\"reason\":\"success\""));
4690        assert!(json.contains("\"id\":\"test\""));
4691    }
4692
4693    #[test]
4694    fn test_selection_response_without_worker() {
4695        let _guard = test_guard!();
4696        let response = SelectionResponse {
4697            worker: None,
4698            reason: SelectionReason::AllWorkersBusy,
4699            build_id: None,
4700            diagnostics: None,
4701        };
4702
4703        let json = serde_json::to_string(&response).unwrap();
4704        assert!(json.contains("\"worker\":null"));
4705        assert!(json.contains("\"reason\":\"all_workers_busy\""));
4706    }
4707
4708    #[test]
4709    fn golden_selection_response_with_no_workers_passed_health_diagnostics() {
4710        let _guard = test_guard!();
4711        const FIXTURE_PATH: &str =
4712            "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json";
4713        let fixture: serde_json::Value = serde_json::from_str(include_str!(
4714            "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json"
4715        ))
4716        .expect("selection diagnostics golden fixture parses");
4717        assert_eq!(
4718            fixture["schema_version"],
4719            "rch.golden.selection_diagnostics.v1"
4720        );
4721
4722        let response = SelectionResponse {
4723            worker: None,
4724            reason: SelectionReason::NoWorkersPassedHealth,
4725            build_id: None,
4726            diagnostics: Some(SelectionDiagnostics {
4727                required_runtime: RequiredRuntime::Rust,
4728                estimated_cores: 2,
4729                min_success_rate: 0.8,
4730                fallback_min_success_rate: 0.5,
4731                active_project_exclusion_count: 0,
4732                workers: vec![WorkerSelectionDiagnostic {
4733                    worker_id: WorkerId::new("fixture-worker"),
4734                    status: "healthy".to_string(),
4735                    circuit_state: "closed".to_string(),
4736                    pressure_state: "healthy".to_string(),
4737                    pressure_reason_code: "pressure.ok".to_string(),
4738                    success_rate: Some(0.2),
4739                    min_success_rate: 0.8,
4740                    fallback_min_success_rate: 0.5,
4741                    required_runtime: RequiredRuntime::Rust,
4742                    runtime_available: true,
4743                    available_slots: 8,
4744                    total_slots: 8,
4745                    estimated_cores: 2,
4746                    active_project_excluded: false,
4747                    final_decision: WorkerSelectionDiagnosticDecision::Deny,
4748                    final_reason: "success_rate 0.20 < fallback 0.50".to_string(),
4749                    reason_codes: vec![
4750                        "health.below_min_success_rate".to_string(),
4751                        "health.below_fallback_min_success_rate".to_string(),
4752                    ],
4753                }],
4754            }),
4755        };
4756
4757        assert_golden_json(
4758            serde_json::to_value(&response).expect("response serializes"),
4759            &fixture["expected_response"],
4760            FIXTURE_PATH,
4761        );
4762    }
4763
4764    #[test]
4765    fn test_selection_response_roundtrip() {
4766        let _guard = test_guard!();
4767        let original = SelectionResponse {
4768            worker: Some(SelectedWorker {
4769                id: WorkerId::new("worker-1"),
4770                host: "192.168.1.100".to_string(),
4771                user: "ubuntu".to_string(),
4772                identity_file: "/path/to/key".to_string(),
4773                slots_available: 16,
4774                speed_score: 90.5,
4775                declared_os: None,
4776            }),
4777            reason: SelectionReason::Success,
4778            build_id: None,
4779            diagnostics: None,
4780        };
4781
4782        let json = serde_json::to_string(&original).unwrap();
4783        let parsed: SelectionResponse = serde_json::from_str(&json).unwrap();
4784
4785        assert!(parsed.worker.is_some());
4786        let worker = parsed.worker.unwrap();
4787        assert_eq!(worker.id.as_str(), "worker-1");
4788        assert_eq!(worker.host, "192.168.1.100");
4789        assert_eq!(worker.slots_available, 16);
4790        assert_eq!(parsed.reason, SelectionReason::Success);
4791    }
4792
4793    #[test]
4794    fn test_build_heartbeat_phase_roundtrip() {
4795        let _guard = test_guard!();
4796        let json = serde_json::to_string(&BuildHeartbeatPhase::SyncUp).unwrap();
4797        assert_eq!(json, "\"sync_up\"");
4798        let parsed: BuildHeartbeatPhase = serde_json::from_str(&json).unwrap();
4799        assert_eq!(parsed, BuildHeartbeatPhase::SyncUp);
4800    }
4801
4802    #[test]
4803    fn test_build_heartbeat_request_roundtrip() {
4804        let _guard = test_guard!();
4805        let request = BuildHeartbeatRequest {
4806            build_id: 42,
4807            worker_id: WorkerId::new("worker-a"),
4808            hook_pid: Some(12345),
4809            local_wrapper_id: Some("wrapper-abc123".to_string()),
4810            remote_pgid_file: Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string()),
4811            phase: BuildHeartbeatPhase::Execute,
4812            detail: Some("Compiling crates".to_string()),
4813            progress_counter: Some(7),
4814            progress_percent: Some(42.5),
4815        };
4816
4817        let json = serde_json::to_string(&request).unwrap();
4818        let parsed: BuildHeartbeatRequest = serde_json::from_str(&json).unwrap();
4819        assert_eq!(parsed.build_id, 42);
4820        assert_eq!(parsed.worker_id.as_str(), "worker-a");
4821        assert_eq!(
4822            parsed.remote_pgid_file,
4823            Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string())
4824        );
4825        assert_eq!(parsed.phase, BuildHeartbeatPhase::Execute);
4826        assert_eq!(parsed.progress_counter, Some(7));
4827        assert_eq!(parsed.progress_percent, Some(42.5));
4828    }
4829
4830    // CircuitStats tests
4831
4832    #[test]
4833    fn test_circuit_stats_new() {
4834        let _guard = test_guard!();
4835        let stats = CircuitStats::new();
4836        assert_eq!(stats.state(), CircuitState::Closed);
4837        assert_eq!(stats.consecutive_failures(), 0);
4838        assert_eq!(stats.error_rate(), 0.0);
4839        assert!(stats.opened_at().is_none());
4840    }
4841
4842    #[test]
4843    fn test_circuit_stats_record_success() {
4844        let _guard = test_guard!();
4845        let mut stats = CircuitStats::new();
4846        stats.record_success();
4847        stats.record_success();
4848        assert_eq!(stats.error_rate(), 0.0);
4849        assert_eq!(stats.consecutive_failures(), 0);
4850    }
4851
4852    #[test]
4853    fn test_circuit_stats_record_failure() {
4854        let _guard = test_guard!();
4855        let mut stats = CircuitStats::new();
4856        stats.record_failure();
4857        stats.record_failure();
4858        assert_eq!(stats.consecutive_failures(), 2);
4859        assert_eq!(stats.error_rate(), 1.0); // 2 failures, 0 successes
4860    }
4861
4862    #[test]
4863    fn test_circuit_stats_error_rate() {
4864        let _guard = test_guard!();
4865        let mut stats = CircuitStats::new();
4866        stats.record_success();
4867        stats.record_success();
4868        stats.record_failure();
4869        // 1 failure / 3 total = 0.333...
4870        assert!((stats.error_rate() - 0.333).abs() < 0.01);
4871    }
4872
4873    #[test]
4874    fn test_circuit_stats_should_open_consecutive_failures() {
4875        let _guard = test_guard!();
4876        let mut stats = CircuitStats::new();
4877        let config = CircuitBreakerConfig::default(); // threshold = 3
4878
4879        stats.record_failure();
4880        assert!(!stats.should_open(&config));
4881
4882        stats.record_failure();
4883        assert!(!stats.should_open(&config));
4884
4885        stats.record_failure();
4886        assert!(stats.should_open(&config)); // 3 consecutive failures
4887    }
4888
4889    #[test]
4890    fn test_circuit_stats_should_open_error_rate() {
4891        let _guard = test_guard!();
4892        let mut stats = CircuitStats::new();
4893        let config = CircuitBreakerConfig {
4894            error_rate_threshold: 0.5,
4895            ..Default::default()
4896        };
4897
4898        // Need at least 5 samples
4899        stats.record_success();
4900        stats.record_success();
4901        stats.record_failure();
4902        stats.record_failure();
4903        assert!(!stats.should_open(&config)); // Only 4 samples
4904
4905        stats.record_failure(); // 5 samples: 2 success, 3 failures = 60% error rate
4906        assert!(stats.should_open(&config));
4907    }
4908
4909    #[test]
4910    fn test_circuit_close_clears_recent_results_no_flap() {
4911        let _guard = test_guard!();
4912        // Regression: error_rate() is computed over recent_results, so close()
4913        // must clear that history or a recovered circuit immediately re-opens
4914        // on stale pre-failure samples (flapping).
4915        let mut stats = CircuitStats::new();
4916        let config = CircuitBreakerConfig {
4917            error_rate_threshold: 0.5,
4918            ..Default::default()
4919        };
4920        for _ in 0..6 {
4921            stats.record_failure();
4922        }
4923        assert!(stats.should_open(&config));
4924        stats.open();
4925        // Recover via half-open probes, then close.
4926        stats.half_open();
4927        stats.record_success();
4928        stats.record_success();
4929        stats.close();
4930        // Error rate must be reset and the circuit must NOT immediately re-open.
4931        assert_eq!(stats.error_rate(), 0.0);
4932        assert!(!stats.should_open(&config));
4933    }
4934
4935    #[test]
4936    fn test_apply_health_outcome_opens_on_failure_threshold() {
4937        let _guard = test_guard!();
4938        // failure_threshold=3: three unhealthy outcomes open the circuit.
4939        let config = CircuitBreakerConfig {
4940            failure_threshold: 3,
4941            open_cooldown_secs: 300,
4942            ..Default::default()
4943        };
4944        let mut stats = CircuitStats::new();
4945        assert_eq!(
4946            stats.apply_health_outcome(false, false, &config),
4947            CircuitState::Closed
4948        );
4949        assert_eq!(
4950            stats.apply_health_outcome(false, false, &config),
4951            CircuitState::Closed
4952        );
4953        assert_eq!(
4954            stats.apply_health_outcome(false, false, &config),
4955            CircuitState::Open
4956        );
4957    }
4958
4959    #[test]
4960    fn test_apply_health_outcome_half_open_two_probe_recovery() {
4961        let _guard = test_guard!();
4962        // success_threshold=2: recovery from Open needs exactly two clean probes.
4963        // open_cooldown_secs=0 makes should_half_open() true immediately, so the
4964        // engine's trailing nudge leaves an opened circuit already in HalfOpen —
4965        // exactly the state a real daemon would be in on the next poll once the
4966        // cooldown had elapsed. From HalfOpen the engine credits successes.
4967        let config = CircuitBreakerConfig {
4968            failure_threshold: 3,
4969            success_threshold: 2,
4970            open_cooldown_secs: 0,
4971            ..Default::default()
4972        };
4973        let mut stats = CircuitStats::new();
4974        for _ in 0..3 {
4975            stats.apply_health_outcome(false, false, &config);
4976        }
4977        // With a 0s cooldown the engine nudges Open -> HalfOpen immediately.
4978        assert_eq!(stats.state(), CircuitState::HalfOpen);
4979
4980        // First healthy probe in half-open: one success credited, not yet closed.
4981        let after_first = stats.apply_health_outcome(true, false, &config);
4982        assert_eq!(after_first, CircuitState::HalfOpen);
4983
4984        // Second healthy probe: threshold met -> Closed.
4985        let after_second = stats.apply_health_outcome(true, false, &config);
4986        assert_eq!(after_second, CircuitState::Closed);
4987    }
4988
4989    #[test]
4990    fn test_apply_health_outcome_transient_half_open_does_not_reopen() {
4991        let _guard = test_guard!();
4992        // A transient failure while half-open must NOT reset the recovery streak
4993        // — a network blip can't undo an in-progress recovery.
4994        let config = CircuitBreakerConfig {
4995            failure_threshold: 3,
4996            success_threshold: 2,
4997            open_cooldown_secs: 0,
4998            ..Default::default()
4999        };
5000        let mut stats = CircuitStats::new();
5001        for _ in 0..3 {
5002            stats.apply_health_outcome(false, false, &config);
5003        }
5004        // 0s cooldown -> already nudged to HalfOpen.
5005        assert_eq!(stats.state(), CircuitState::HalfOpen);
5006
5007        // One clean probe (1 of 2 needed).
5008        assert_eq!(
5009            stats.apply_health_outcome(true, false, &config),
5010            CircuitState::HalfOpen
5011        );
5012
5013        // Transient failure in half-open: stays HalfOpen (does NOT reopen the
5014        // circuit or restart the cooldown). The key contrast with a real failure
5015        // is that the circuit does not drop back to Open — probing continues in
5016        // place, so recovery resumes immediately on the next clean probes rather
5017        // than waiting out a fresh cooldown.
5018        let after_transient = stats.apply_health_outcome(false, true, &config);
5019        assert_eq!(after_transient, CircuitState::HalfOpen);
5020
5021        // Recovery still completes from half-open with the required clean probes.
5022        assert_eq!(
5023            stats.apply_health_outcome(true, false, &config),
5024            CircuitState::HalfOpen
5025        );
5026        assert_eq!(
5027            stats.apply_health_outcome(true, false, &config),
5028            CircuitState::Closed
5029        );
5030    }
5031
5032    #[test]
5033    fn test_apply_health_outcome_real_vs_transient_half_open_failure() {
5034        let _guard = test_guard!();
5035        // With a positive cooldown the difference is directly observable: a REAL
5036        // half-open failure reopens the circuit (state -> Open, must wait out the
5037        // cooldown again), whereas a TRANSIENT half-open failure keeps it in
5038        // HalfOpen so recovery continues without a fresh cooldown.
5039        //
5040        // Manually drive to HalfOpen (open() + half_open()) so the positive
5041        // cooldown doesn't force a real wallclock wait to reach half-open.
5042        let config = CircuitBreakerConfig {
5043            failure_threshold: 3,
5044            success_threshold: 2,
5045            open_cooldown_secs: 300,
5046            ..Default::default()
5047        };
5048
5049        // Real failure path.
5050        let mut real = CircuitStats::new();
5051        real.open();
5052        real.half_open();
5053        assert_eq!(real.state(), CircuitState::HalfOpen);
5054        let after_real = real.apply_health_outcome(false, false, &config);
5055        assert_eq!(
5056            after_real,
5057            CircuitState::Open,
5058            "a real half-open failure must reopen the circuit"
5059        );
5060
5061        // Transient failure path (same starting state).
5062        let mut transient = CircuitStats::new();
5063        transient.open();
5064        transient.half_open();
5065        assert_eq!(transient.state(), CircuitState::HalfOpen);
5066        let after_transient = transient.apply_health_outcome(false, true, &config);
5067        assert_eq!(
5068            after_transient,
5069            CircuitState::HalfOpen,
5070            "a transient half-open failure must NOT reopen the circuit"
5071        );
5072    }
5073
5074    /// Issue #48 regression: a worker whose liveness probe (`echo` over a warm
5075    /// session) keeps succeeding while every real command times out must NOT
5076    /// be re-admitted. Liveness successes alone cannot close a half-open
5077    /// circuit while command-class evidence is outstanding.
5078    #[test]
5079    fn test_command_failure_blocks_half_open_close_on_liveness_successes() {
5080        let _guard = test_guard!();
5081        let config = CircuitBreakerConfig {
5082            failure_threshold: 3,
5083            success_threshold: 2,
5084            open_cooldown_secs: 0,
5085            command_failure_threshold: 2,
5086            ..Default::default()
5087        };
5088        let mut stats = CircuitStats::new();
5089        stats.open();
5090        stats.half_open();
5091
5092        // The half-open trial: a real command hangs on the worker.
5093        assert_eq!(
5094            stats.apply_command_outcome(false, &config),
5095            CircuitState::HalfOpen,
5096            "cooldown 0: reopen then immediately half-open again"
5097        );
5098        assert_eq!(stats.consecutive_command_failures(), 1);
5099
5100        // Liveness keeps answering — many times over.
5101        for _ in 0..10 {
5102            assert_eq!(
5103                stats.apply_health_outcome(true, false, &config),
5104                CircuitState::HalfOpen,
5105                "liveness successes must not close the circuit while a command failure is outstanding"
5106            );
5107        }
5108
5109        // Only a command-class success clears the evidence and recovers.
5110        assert_eq!(
5111            stats.apply_command_outcome(true, &config),
5112            CircuitState::Closed
5113        );
5114        assert_eq!(stats.consecutive_command_failures(), 0);
5115        assert_eq!(stats.reopen_streak(), 0);
5116    }
5117
5118    /// Issue #48: interleaved liveness successes reset `failure_threshold`, so
5119    /// command failures need their own threshold to open a Closed circuit.
5120    #[test]
5121    fn test_command_failures_open_closed_circuit_despite_liveness_successes() {
5122        let _guard = test_guard!();
5123        let config = CircuitBreakerConfig {
5124            failure_threshold: 3,
5125            command_failure_threshold: 2,
5126            open_cooldown_secs: 300,
5127            ..Default::default()
5128        };
5129        let mut stats = CircuitStats::new();
5130
5131        assert_eq!(
5132            stats.apply_command_outcome(false, &config),
5133            CircuitState::Closed
5134        );
5135        // Two liveness successes in between: consecutive_failures is reset...
5136        stats.apply_health_outcome(true, false, &config);
5137        stats.apply_health_outcome(true, false, &config);
5138        assert_eq!(stats.consecutive_failures(), 0);
5139        // ...but the command-failure count is not, and the second one opens.
5140        assert_eq!(
5141            stats.apply_command_outcome(false, &config),
5142            CircuitState::Open
5143        );
5144        assert_eq!(stats.reopen_streak(), 1);
5145    }
5146
5147    /// A command failure in HalfOpen always reopens: "connected but the
5148    /// command never completed" is exactly what the trial exists to detect,
5149    /// so there is no transient exemption on this channel.
5150    #[test]
5151    fn test_command_failure_in_half_open_always_reopens() {
5152        let _guard = test_guard!();
5153        let config = CircuitBreakerConfig {
5154            open_cooldown_secs: 300,
5155            ..Default::default()
5156        };
5157        let mut stats = CircuitStats::new();
5158        stats.open();
5159        stats.half_open();
5160        assert_eq!(
5161            stats.apply_command_outcome(false, &config),
5162            CircuitState::Open
5163        );
5164        assert_eq!(stats.reopen_streak(), 2);
5165    }
5166
5167    /// Repeated Open -> HalfOpen -> Open cycles lengthen the cooldown
5168    /// (exponential, capped) so a flapping worker converges to Open instead of
5169    /// being re-admitted on every base cooldown.
5170    #[test]
5171    fn test_reopen_streak_backs_off_cooldown_until_cap() {
5172        let _guard = test_guard!();
5173        let config = CircuitBreakerConfig {
5174            open_cooldown_secs: 30,
5175            open_cooldown_max_secs: 100,
5176            ..Default::default()
5177        };
5178        assert_eq!(config.cooldown_ms_for_streak(0), 30_000);
5179        assert_eq!(config.cooldown_ms_for_streak(1), 30_000);
5180        assert_eq!(config.cooldown_ms_for_streak(2), 60_000);
5181        assert_eq!(config.cooldown_ms_for_streak(3), 100_000, "capped");
5182        assert_eq!(config.cooldown_ms_for_streak(40), 100_000, "no overflow");
5183
5184        // A cap below the base never shortens the base cooldown.
5185        let inverted = CircuitBreakerConfig {
5186            open_cooldown_secs: 30,
5187            open_cooldown_max_secs: 5,
5188            ..Default::default()
5189        };
5190        assert_eq!(inverted.cooldown_ms_for_streak(3), 30_000);
5191
5192        // The streak is observable through recovery_remaining_secs: after two
5193        // opens the remaining cooldown exceeds the base.
5194        let mut stats = CircuitStats::new();
5195        stats.open();
5196        stats.half_open();
5197        stats.open();
5198        assert_eq!(stats.reopen_streak(), 2);
5199        let remaining = stats.recovery_remaining_secs(&config).unwrap();
5200        assert!(
5201            remaining > 30 && remaining <= 60,
5202            "second open should wait ~60s, got {remaining}"
5203        );
5204
5205        // Closing resets the streak, so the next open starts from the base.
5206        stats.close();
5207        assert_eq!(stats.reopen_streak(), 0);
5208        stats.open();
5209        assert!(stats.recovery_remaining_secs(&config).unwrap() <= 30);
5210    }
5211
5212    #[test]
5213    fn test_circuit_stats_success_resets_consecutive_failures() {
5214        let _guard = test_guard!();
5215        let mut stats = CircuitStats::new();
5216        stats.record_failure();
5217        stats.record_failure();
5218        assert_eq!(stats.consecutive_failures(), 2);
5219
5220        stats.record_success();
5221        assert_eq!(stats.consecutive_failures(), 0);
5222    }
5223
5224    #[test]
5225    fn test_circuit_stats_open_transition() {
5226        let _guard = test_guard!();
5227        let mut stats = CircuitStats::new();
5228        stats.open();
5229
5230        assert_eq!(stats.state(), CircuitState::Open);
5231        assert!(stats.opened_at().is_some());
5232    }
5233
5234    #[test]
5235    fn test_circuit_stats_half_open_transition() {
5236        let _guard = test_guard!();
5237        let mut stats = CircuitStats::new();
5238        stats.open();
5239        stats.half_open();
5240
5241        assert_eq!(stats.state(), CircuitState::HalfOpen);
5242    }
5243
5244    #[test]
5245    fn test_circuit_stats_close_transition() {
5246        let _guard = test_guard!();
5247        let mut stats = CircuitStats::new();
5248        stats.open();
5249        stats.half_open();
5250        stats.close();
5251
5252        assert_eq!(stats.state(), CircuitState::Closed);
5253        assert!(stats.opened_at().is_none());
5254        assert_eq!(stats.consecutive_failures(), 0);
5255    }
5256
5257    #[test]
5258    fn test_circuit_stats_should_close() {
5259        let _guard = test_guard!();
5260        let mut stats = CircuitStats::new();
5261        let config = CircuitBreakerConfig {
5262            success_threshold: 2,
5263            ..Default::default()
5264        };
5265
5266        stats.open();
5267        stats.half_open();
5268
5269        assert!(!stats.should_close(&config)); // 0 successes
5270
5271        stats.record_success();
5272        assert!(!stats.should_close(&config)); // 1 success
5273
5274        stats.record_success();
5275        assert!(stats.should_close(&config)); // 2 successes
5276    }
5277
5278    #[test]
5279    fn test_circuit_stats_can_probe() {
5280        let _guard = test_guard!();
5281        let mut stats = CircuitStats::new();
5282        let config = CircuitBreakerConfig {
5283            half_open_max_probes: 1,
5284            ..Default::default()
5285        };
5286
5287        // Can't probe when closed
5288        assert!(!stats.can_probe(&config));
5289
5290        stats.open();
5291        // Can't probe when open
5292        assert!(!stats.can_probe(&config));
5293
5294        stats.half_open();
5295        // Can probe when half-open
5296        assert!(stats.can_probe(&config));
5297
5298        // Start a probe
5299        assert!(stats.start_probe(&config));
5300        // Can't start another probe
5301        assert!(!stats.can_probe(&config));
5302        assert!(!stats.start_probe(&config));
5303    }
5304
5305    #[test]
5306    fn test_circuit_stats_probe_completion() {
5307        let _guard = test_guard!();
5308        let mut stats = CircuitStats::new();
5309        let config = CircuitBreakerConfig {
5310            half_open_max_probes: 1,
5311            ..Default::default()
5312        };
5313
5314        stats.open();
5315        stats.half_open();
5316        stats.start_probe(&config);
5317
5318        // Probe completes with success
5319        stats.record_success();
5320
5321        // Can start another probe
5322        assert!(stats.can_probe(&config));
5323    }
5324
5325    #[test]
5326    fn test_circuit_stats_failure_in_half_open() {
5327        let _guard = test_guard!();
5328        let mut stats = CircuitStats::new();
5329        let config = CircuitBreakerConfig {
5330            success_threshold: 2,
5331            ..Default::default()
5332        };
5333
5334        stats.open();
5335        stats.half_open();
5336
5337        stats.record_success();
5338        assert_eq!(stats.consecutive_failures(), 0);
5339
5340        // Failure resets consecutive successes
5341        stats.record_failure();
5342        assert!(!stats.should_close(&config));
5343
5344        // Need 2 more successes
5345        stats.record_success();
5346        stats.record_success();
5347        assert!(stats.should_close(&config));
5348    }
5349
5350    #[test]
5351    fn test_circuit_stats_reset_window() {
5352        let _guard = test_guard!();
5353        let mut stats = CircuitStats::new();
5354        stats.record_success();
5355        stats.record_failure();
5356
5357        assert!(stats.error_rate() > 0.0);
5358
5359        stats.reset_window();
5360        assert_eq!(stats.error_rate(), 0.0);
5361    }
5362
5363    #[test]
5364    fn test_circuit_state_transitions_are_deterministic() {
5365        let _guard = test_guard!();
5366        let config = CircuitBreakerConfig::default();
5367        let mut stats = CircuitStats::new();
5368
5369        // Start closed
5370        assert_eq!(stats.state(), CircuitState::Closed);
5371
5372        // Cause failures to open
5373        for _ in 0..3 {
5374            stats.record_failure();
5375        }
5376        assert!(stats.should_open(&config));
5377        stats.open();
5378        assert_eq!(stats.state(), CircuitState::Open);
5379
5380        // Transition to half-open after cooldown would be checked
5381        stats.half_open();
5382        assert_eq!(stats.state(), CircuitState::HalfOpen);
5383
5384        // Successes to close
5385        for _ in 0..2 {
5386            stats.record_success();
5387        }
5388        assert!(stats.should_close(&config));
5389        stats.close();
5390        assert_eq!(stats.state(), CircuitState::Closed);
5391    }
5392
5393    // ========================================================================
5394    // Compilation Timing Tests
5395    // ========================================================================
5396
5397    fn make_test_metrics(
5398        speedup: Option<f64>,
5399        total_secs: u64,
5400        success: bool,
5401    ) -> CompilationMetrics {
5402        CompilationMetrics {
5403            project_id: "test-project".to_string(),
5404            worker_id: "worker-1".to_string(),
5405            timestamp: chrono::Utc::now(),
5406            timing: CompilationTimingBreakdown {
5407                rsync_up: Duration::from_millis(100),
5408                remote_build: Duration::from_millis(800),
5409                rsync_down: Duration::from_millis(100),
5410                total: Duration::from_secs(total_secs),
5411            },
5412            local_build_time: speedup.map(|s| Duration::from_secs_f64(total_secs as f64 * s)),
5413            speedup,
5414            files_synced: 100,
5415            bytes_transferred: 1_000_000,
5416            exit_code: if success { 0 } else { 1 },
5417            success,
5418        }
5419    }
5420
5421    #[test]
5422    fn test_compilation_timing_breakdown_default() {
5423        let _guard = test_guard!();
5424        let timing = CompilationTimingBreakdown::default();
5425        assert_eq!(timing.rsync_up, Duration::ZERO);
5426        assert_eq!(timing.remote_build, Duration::ZERO);
5427        assert_eq!(timing.rsync_down, Duration::ZERO);
5428        assert_eq!(timing.total, Duration::ZERO);
5429    }
5430
5431    #[test]
5432    fn test_compilation_timing_breakdown_serialization() {
5433        let _guard = test_guard!();
5434        let timing = CompilationTimingBreakdown {
5435            rsync_up: Duration::from_millis(100),
5436            remote_build: Duration::from_millis(2000),
5437            rsync_down: Duration::from_millis(50),
5438            total: Duration::from_millis(2150),
5439        };
5440
5441        let json = serde_json::to_string(&timing).unwrap();
5442        assert!(json.contains("100")); // rsync_up
5443        assert!(json.contains("2000")); // remote_build
5444        assert!(json.contains("50")); // rsync_down
5445        assert!(json.contains("2150")); // total
5446
5447        let parsed: CompilationTimingBreakdown = serde_json::from_str(&json).unwrap();
5448        assert_eq!(parsed.rsync_up, timing.rsync_up);
5449        assert_eq!(parsed.remote_build, timing.remote_build);
5450        assert_eq!(parsed.rsync_down, timing.rsync_down);
5451        assert_eq!(parsed.total, timing.total);
5452    }
5453
5454    #[test]
5455    fn test_command_timing_breakdown_serialization() {
5456        let _guard = test_guard!();
5457        let timing = CommandTimingBreakdown {
5458            classify: Some(Duration::from_millis(2)),
5459            select: None,
5460            sync_up: Some(Duration::from_millis(120)),
5461            exec: Some(Duration::from_millis(900)),
5462            sync_down: None,
5463            cleanup: Some(Duration::from_millis(5)),
5464            total: Some(Duration::from_millis(1027)),
5465        };
5466
5467        let json = serde_json::to_string(&timing).unwrap();
5468        assert!(json.contains("\"classify\":2"));
5469        assert!(json.contains("\"select\":null"));
5470        assert!(json.contains("\"sync_up\":120"));
5471        assert!(json.contains("\"exec\":900"));
5472        assert!(json.contains("\"sync_down\":null"));
5473        assert!(json.contains("\"cleanup\":5"));
5474        assert!(json.contains("\"total\":1027"));
5475
5476        let parsed: CommandTimingBreakdown = serde_json::from_str(&json).unwrap();
5477        assert_eq!(parsed.classify, timing.classify);
5478        assert_eq!(parsed.select, timing.select);
5479        assert_eq!(parsed.sync_up, timing.sync_up);
5480        assert_eq!(parsed.exec, timing.exec);
5481        assert_eq!(parsed.sync_down, timing.sync_down);
5482        assert_eq!(parsed.cleanup, timing.cleanup);
5483        assert_eq!(parsed.total, timing.total);
5484    }
5485
5486    #[test]
5487    fn test_compilation_metrics_calculate_speedup() {
5488        let _guard = test_guard!();
5489        let mut metrics = CompilationMetrics {
5490            timing: CompilationTimingBreakdown {
5491                total: Duration::from_secs(10),
5492                ..Default::default()
5493            },
5494            local_build_time: Some(Duration::from_secs(30)),
5495            ..Default::default()
5496        };
5497
5498        metrics.calculate_speedup();
5499
5500        assert!(metrics.speedup.is_some());
5501        let speedup = metrics.speedup.unwrap();
5502        assert!(
5503            (speedup - 3.0).abs() < 0.01,
5504            "Expected 3.0x speedup, got {}",
5505            speedup
5506        );
5507    }
5508
5509    #[test]
5510    fn test_compilation_metrics_is_beneficial() {
5511        let _guard = test_guard!();
5512        let mut metrics = CompilationMetrics::default();
5513
5514        // No speedup calculated
5515        assert!(!metrics.is_beneficial());
5516
5517        // Speedup > 1 is beneficial
5518        metrics.speedup = Some(1.5);
5519        assert!(metrics.is_beneficial());
5520
5521        // Speedup < 1 is not beneficial
5522        metrics.speedup = Some(0.8);
5523        assert!(!metrics.is_beneficial());
5524
5525        // Speedup = 1 is not beneficial (exactly same speed)
5526        metrics.speedup = Some(1.0);
5527        assert!(!metrics.is_beneficial());
5528    }
5529
5530    #[test]
5531    fn test_compilation_metrics_serialization() {
5532        let _guard = test_guard!();
5533        let metrics = make_test_metrics(Some(2.5), 10, true);
5534        let json = serde_json::to_string(&metrics).unwrap();
5535
5536        assert!(json.contains("test-project"));
5537        assert!(json.contains("worker-1"));
5538        assert!(json.contains("2.5")); // speedup
5539
5540        let parsed: CompilationMetrics = serde_json::from_str(&json).unwrap();
5541        assert_eq!(parsed.project_id, "test-project");
5542        assert_eq!(parsed.worker_id, "worker-1");
5543        assert!(parsed.success);
5544    }
5545
5546    #[test]
5547    fn test_compilation_timer_new() {
5548        let _guard = test_guard!();
5549        let timer = CompilationTimer::new("my-project", "my-worker");
5550        assert_eq!(timer.project_id, "my-project");
5551        assert_eq!(timer.worker_id, "my-worker");
5552        assert!(timer.rsync_up.is_none());
5553        assert!(timer.remote_build.is_none());
5554        assert!(timer.rsync_down.is_none());
5555    }
5556
5557    #[test]
5558    fn test_compilation_timer_phases() {
5559        let _guard = test_guard!();
5560        let mut timer = CompilationTimer::new("test", "worker");
5561
5562        // Simulate rsync up
5563        std::thread::sleep(Duration::from_millis(5));
5564        timer.end_rsync_up();
5565        assert!(timer.rsync_up.is_some());
5566        assert!(timer.rsync_up.unwrap() >= Duration::from_millis(5));
5567
5568        // Simulate remote build
5569        std::thread::sleep(Duration::from_millis(10));
5570        timer.end_remote_build();
5571        assert!(timer.remote_build.is_some());
5572        assert!(timer.remote_build.unwrap() >= Duration::from_millis(10));
5573
5574        // Simulate rsync down
5575        std::thread::sleep(Duration::from_millis(5));
5576        timer.end_rsync_down();
5577        assert!(timer.rsync_down.is_some());
5578        assert!(timer.rsync_down.unwrap() >= Duration::from_millis(5));
5579
5580        // Finish
5581        let metrics = timer.finish(0, 50, 500_000);
5582        assert_eq!(metrics.project_id, "test");
5583        assert_eq!(metrics.worker_id, "worker");
5584        assert!(metrics.success);
5585        assert!(metrics.timing.total >= Duration::from_millis(20));
5586        assert_eq!(metrics.files_synced, 50);
5587        assert_eq!(metrics.bytes_transferred, 500_000);
5588    }
5589
5590    #[test]
5591    fn test_metrics_aggregator_new() {
5592        let _guard = test_guard!();
5593        let agg = MetricsAggregator::new(100);
5594        assert_eq!(agg.count(), 0);
5595        assert!(agg.average_speedup().is_none());
5596    }
5597
5598    #[test]
5599    fn test_metrics_aggregator_record() {
5600        let _guard = test_guard!();
5601        let mut agg = MetricsAggregator::new(3);
5602
5603        // Record first metric
5604        agg.record(make_test_metrics(Some(2.0), 10, true));
5605        assert_eq!(agg.count(), 1);
5606
5607        // Record up to max
5608        agg.record(make_test_metrics(Some(3.0), 20, true));
5609        agg.record(make_test_metrics(Some(4.0), 30, true));
5610        assert_eq!(agg.count(), 3);
5611
5612        // Exceed max should drop oldest
5613        agg.record(make_test_metrics(Some(5.0), 40, true));
5614        assert_eq!(agg.count(), 3);
5615    }
5616
5617    #[test]
5618    fn test_metrics_aggregator_average_speedup() {
5619        let _guard = test_guard!();
5620        let mut agg = MetricsAggregator::new(100);
5621
5622        // Empty case
5623        assert!(agg.average_speedup().is_none());
5624
5625        // Add metrics with speedups 1, 2, 3, 4, 5 -> average = 3
5626        for i in 1..=5 {
5627            agg.record(make_test_metrics(Some(i as f64), 10, true));
5628        }
5629
5630        let avg = agg.average_speedup().unwrap();
5631        assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5632    }
5633
5634    #[test]
5635    fn test_metrics_aggregator_average_speedup_with_none() {
5636        let _guard = test_guard!();
5637        let mut agg = MetricsAggregator::new(100);
5638
5639        // Mix of Some and None speedups
5640        agg.record(make_test_metrics(Some(2.0), 10, true));
5641        agg.record(make_test_metrics(None, 10, true)); // No speedup
5642        agg.record(make_test_metrics(Some(4.0), 10, true));
5643
5644        // Average should only consider metrics with speedup
5645        let avg = agg.average_speedup().unwrap();
5646        assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5647    }
5648
5649    #[test]
5650    fn test_metrics_aggregator_percentiles() {
5651        let _guard = test_guard!();
5652        let mut agg = MetricsAggregator::new(100);
5653
5654        // Add metrics with total times 1s, 2s, 3s, ..., 10s
5655        for i in 1..=10 {
5656            agg.record(make_test_metrics(Some(1.0), i, true));
5657        }
5658
5659        // p50 (median) should be around 5s
5660        let p50 = agg.p50_total_time().unwrap();
5661        assert!(p50 >= Duration::from_secs(5) && p50 <= Duration::from_secs(6));
5662
5663        // p95 should be around 9s or 10s
5664        let p95 = agg.p95_total_time().unwrap();
5665        assert!(p95 >= Duration::from_secs(9) && p95 <= Duration::from_secs(10));
5666
5667        // p99 should be 10s
5668        let p99 = agg.p99_total_time().unwrap();
5669        assert!(p99 >= Duration::from_secs(9));
5670    }
5671
5672    #[test]
5673    fn test_metrics_aggregator_success_rate() {
5674        let _guard = test_guard!();
5675        let mut agg = MetricsAggregator::new(100);
5676
5677        // Empty case
5678        assert_eq!(agg.success_rate(), 100.0);
5679
5680        // All successful
5681        agg.record(make_test_metrics(Some(1.0), 10, true));
5682        agg.record(make_test_metrics(Some(1.0), 10, true));
5683        assert_eq!(agg.success_rate(), 100.0);
5684
5685        // Add a failure (2 success, 1 failure = 66.67%)
5686        agg.record(make_test_metrics(Some(1.0), 10, false));
5687        let rate = agg.success_rate();
5688        assert!((rate - 66.67).abs() < 1.0, "Expected ~66.67%, got {}", rate);
5689    }
5690
5691    #[test]
5692    fn test_metrics_aggregator_clear() {
5693        let _guard = test_guard!();
5694        let mut agg = MetricsAggregator::new(100);
5695        agg.record(make_test_metrics(Some(1.0), 10, true));
5696        agg.record(make_test_metrics(Some(2.0), 20, true));
5697
5698        assert_eq!(agg.count(), 2);
5699
5700        agg.clear();
5701        assert_eq!(agg.count(), 0);
5702        assert!(agg.average_speedup().is_none());
5703    }
5704
5705    // CompilationConfig timeout tests
5706
5707    #[test]
5708    fn test_remote_build_jobs_parse_display_and_serde() {
5709        let _guard = test_guard!();
5710        assert_eq!(RemoteBuildJobs::default(), RemoteBuildJobs::Auto);
5711        for (spelling, expected) in [
5712            ("auto", RemoteBuildJobs::Auto),
5713            (" AUTO ", RemoteBuildJobs::Auto),
5714            ("off", RemoteBuildJobs::Off),
5715            ("none", RemoteBuildJobs::Off),
5716            ("4", RemoteBuildJobs::Fixed(4)),
5717        ] {
5718            assert_eq!(
5719                RemoteBuildJobs::parse(spelling).unwrap(),
5720                expected,
5721                "{spelling}"
5722            );
5723        }
5724        assert!(RemoteBuildJobs::parse("0").is_err(), "cargo rejects -j 0");
5725        assert!(RemoteBuildJobs::parse("many").is_err());
5726        assert_eq!(RemoteBuildJobs::Fixed(4).to_string(), "4");
5727        assert_eq!(RemoteBuildJobs::Off.to_string(), "off");
5728
5729        // TOML accepts both the string spellings and a bare integer, and an
5730        // absent key keeps the default.
5731        #[derive(Deserialize)]
5732        struct Wrap {
5733            compilation: CompilationConfig,
5734        }
5735        let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = \"off\"\n").unwrap();
5736        assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Off);
5737        let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = 6\n").unwrap();
5738        assert_eq!(
5739            parsed.compilation.remote_build_jobs,
5740            RemoteBuildJobs::Fixed(6)
5741        );
5742        let parsed: Wrap = toml::from_str("[compilation]\nbuild_slots = 4\n").unwrap();
5743        assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Auto);
5744        assert!(toml::from_str::<Wrap>("[compilation]\nremote_build_jobs = 0\n").is_err());
5745
5746        // Serialization round-trips through the same spellings.
5747        let json = serde_json::to_string(&RemoteBuildJobs::Fixed(3)).unwrap();
5748        assert_eq!(json, "3");
5749        assert_eq!(
5750            serde_json::to_string(&RemoteBuildJobs::Auto).unwrap(),
5751            "\"auto\""
5752        );
5753        let back: RemoteBuildJobs = serde_json::from_str("\"auto\"").unwrap();
5754        assert_eq!(back, RemoteBuildJobs::Auto);
5755    }
5756
5757    #[test]
5758    fn test_compilation_config_default_timeouts() {
5759        let _guard = test_guard!();
5760        let config = CompilationConfig::default();
5761        // Default build timeout: 5 minutes
5762        assert_eq!(config.build_timeout_sec, 300);
5763        // Default test timeout: 30 minutes
5764        assert_eq!(config.test_timeout_sec, 1800);
5765    }
5766
5767    #[test]
5768    fn test_compilation_config_timeout_for_test_kinds() {
5769        let _guard = test_guard!();
5770        let config = CompilationConfig::default();
5771
5772        // Test commands should get test_timeout_sec
5773        assert_eq!(
5774            config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5775            std::time::Duration::from_secs(1800)
5776        );
5777        assert_eq!(
5778            config.timeout_for_kind(Some(crate::CompilationKind::CargoNextest)),
5779            std::time::Duration::from_secs(1800)
5780        );
5781        assert_eq!(
5782            config.timeout_for_kind(Some(crate::CompilationKind::BunTest)),
5783            std::time::Duration::from_secs(config.bun_timeout_sec)
5784        );
5785    }
5786
5787    #[test]
5788    fn test_compilation_config_timeout_for_build_kinds() {
5789        let _guard = test_guard!();
5790        let config = CompilationConfig::default();
5791
5792        // Build/check commands should get build_timeout_sec
5793        assert_eq!(
5794            config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5795            std::time::Duration::from_secs(300)
5796        );
5797        assert_eq!(
5798            config.timeout_for_kind(Some(crate::CompilationKind::CargoCheck)),
5799            std::time::Duration::from_secs(300)
5800        );
5801        assert_eq!(
5802            config.timeout_for_kind(Some(crate::CompilationKind::CargoClippy)),
5803            std::time::Duration::from_secs(300)
5804        );
5805        assert_eq!(
5806            config.timeout_for_kind(None),
5807            std::time::Duration::from_secs(300)
5808        );
5809    }
5810
5811    #[test]
5812    fn test_compilation_config_custom_timeouts() {
5813        let _guard = test_guard!();
5814        let config = CompilationConfig {
5815            build_timeout_sec: 600, // 10 minutes
5816            test_timeout_sec: 3600, // 1 hour
5817            ..Default::default()
5818        };
5819
5820        assert_eq!(
5821            config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5822            std::time::Duration::from_secs(600)
5823        );
5824        assert_eq!(
5825            config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5826            std::time::Duration::from_secs(3600)
5827        );
5828    }
5829
5830    #[test]
5831    fn test_compilation_config_speedup_threshold_default() {
5832        let _guard = test_guard!();
5833        let config = CompilationConfig::default();
5834        // Default speedup threshold: 1.2 (20% faster required)
5835        assert!((config.remote_speedup_threshold - 1.2).abs() < 0.001);
5836    }
5837
5838    #[test]
5839    fn test_compilation_config_speedup_threshold_custom() {
5840        let _guard = test_guard!();
5841        let config = CompilationConfig {
5842            remote_speedup_threshold: 1.5, // Require 50% faster
5843            ..Default::default()
5844        };
5845        assert!((config.remote_speedup_threshold - 1.5).abs() < 0.001);
5846    }
5847
5848    #[test]
5849    fn test_compilation_config_speedup_threshold_no_minimum() {
5850        let _guard = test_guard!();
5851        // Setting to 1.0 means "always offload when other criteria met"
5852        let config = CompilationConfig {
5853            remote_speedup_threshold: 1.0,
5854            ..Default::default()
5855        };
5856        assert!((config.remote_speedup_threshold - 1.0).abs() < 0.001);
5857    }
5858
5859    #[test]
5860    fn test_compilation_config_min_local_time_ms_default() {
5861        let _guard = test_guard!();
5862        let config = CompilationConfig::default();
5863        // Default min_local_time_ms: 2000ms
5864        assert_eq!(config.min_local_time_ms, 2000);
5865    }
5866
5867    #[test]
5868    fn test_compilation_config_serde_roundtrip_speedup_threshold() {
5869        let _guard = test_guard!();
5870        let config = CompilationConfig {
5871            remote_speedup_threshold: 2.5,
5872            min_local_time_ms: 5000,
5873            ..Default::default()
5874        };
5875        let json = serde_json::to_string(&config).unwrap();
5876        let parsed: CompilationConfig = serde_json::from_str(&json).unwrap();
5877        assert!((parsed.remote_speedup_threshold - 2.5).abs() < 0.001);
5878        assert_eq!(parsed.min_local_time_ms, 5000);
5879    }
5880
5881    // ========================================================================
5882    // validate_remote_base Tests
5883    // ========================================================================
5884
5885    #[test]
5886    fn test_validate_remote_base_absolute_path() {
5887        let _guard = test_guard!();
5888        assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5889        assert_eq!(
5890            validate_remote_base("/var/rch-builds").unwrap(),
5891            "/var/rch-builds"
5892        );
5893        assert_eq!(
5894            validate_remote_base("/home/builder/.rch").unwrap(),
5895            "/home/builder/.rch"
5896        );
5897    }
5898
5899    #[test]
5900    fn test_validate_remote_base_tilde_expansion() {
5901        let _guard = test_guard!();
5902        // Tilde expansion should work
5903        let result = validate_remote_base("~/rch");
5904        assert!(result.is_ok());
5905        let path = result.unwrap();
5906        assert!(
5907            path.starts_with('/'),
5908            "Path should be absolute after expansion: {}",
5909            path
5910        );
5911        assert!(!path.contains('~'), "Tilde should be expanded: {}", path);
5912    }
5913
5914    #[test]
5915    fn test_validate_remote_base_rejects_relative_path() {
5916        let _guard = test_guard!();
5917        let result = validate_remote_base("tmp/rch");
5918        assert!(result.is_err());
5919        assert!(result.unwrap_err().contains("absolute path"));
5920    }
5921
5922    #[test]
5923    fn test_validate_remote_base_rejects_path_traversal() {
5924        let _guard = test_guard!();
5925        let result = validate_remote_base("/tmp/../etc/rch");
5926        assert!(result.is_err());
5927        assert!(result.unwrap_err().contains("path traversal"));
5928
5929        let result = validate_remote_base("/tmp/rch/../other");
5930        assert!(result.is_err());
5931        assert!(result.unwrap_err().contains("path traversal"));
5932    }
5933
5934    #[test]
5935    fn test_validate_remote_base_normalizes_trailing_slash() {
5936        let _guard = test_guard!();
5937        assert_eq!(validate_remote_base("/tmp/rch/").unwrap(), "/tmp/rch");
5938        assert_eq!(validate_remote_base("/tmp/rch///").unwrap(), "/tmp/rch");
5939    }
5940
5941    #[test]
5942    fn test_validate_remote_base_root_path() {
5943        let _guard = test_guard!();
5944        // Root path should be rejected
5945        let result = validate_remote_base("/");
5946        assert!(result.is_err());
5947        assert!(result.unwrap_err().contains("root directory"));
5948    }
5949
5950    #[test]
5951    fn test_validate_remote_base_rejects_top_level() {
5952        let _guard = test_guard!();
5953        // Top-level directories should be rejected (depth 1)
5954        let result = validate_remote_base("/tmp");
5955        assert!(result.is_err());
5956        assert!(result.unwrap_err().contains("at least 2 levels deep"));
5957
5958        let result = validate_remote_base("/home");
5959        assert!(result.is_err());
5960
5961        // Depth 2 should be allowed
5962        assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5963    }
5964
5965    #[test]
5966    fn test_default_remote_base() {
5967        let _guard = test_guard!();
5968        assert_eq!(default_remote_base(), "/data/tmp/rch");
5969    }
5970
5971    #[test]
5972    fn test_transfer_config_default_has_remote_base() {
5973        let _guard = test_guard!();
5974        let config = TransferConfig::default();
5975        assert_eq!(config.remote_base, "/data/tmp/rch");
5976    }
5977
5978    #[test]
5979    fn test_source_sync_timeout_validation_boundaries() {
5980        let _guard = test_guard!();
5981        assert!(!TransferConfig::valid_sync_timeout_ms(0));
5982        assert!(!TransferConfig::valid_sync_timeout_ms(999));
5983        assert!(TransferConfig::valid_sync_timeout_ms(1_000));
5984        assert!(TransferConfig::valid_sync_timeout_ms(3_600_000));
5985        assert!(!TransferConfig::valid_sync_timeout_ms(3_600_001));
5986    }
5987
5988    #[test]
5989    fn test_source_sync_timeout_is_explicit_or_payload_aware() {
5990        let _guard = test_guard!();
5991        let default = TransferConfig::default();
5992        assert_eq!(
5993            default.sync_timeout_for_payload(0),
5994            std::time::Duration::from_millis(30_000)
5995        );
5996        assert_eq!(
5997            default.sync_timeout_for_payload(10 * 1024 * 1024),
5998            std::time::Duration::from_millis(40_000)
5999        );
6000        assert_eq!(
6001            default.sync_timeout_for_payload(u64::MAX),
6002            std::time::Duration::from_millis(3_600_000)
6003        );
6004
6005        let explicit = TransferConfig {
6006            sync_timeout_ms: Some(120_000),
6007            ..TransferConfig::default()
6008        };
6009        assert_eq!(
6010            explicit.sync_timeout_for_payload(u64::MAX),
6011            std::time::Duration::from_millis(120_000)
6012        );
6013    }
6014
6015    // ========================================================================
6016    // RetryConfig Tests (bd-x1ek)
6017    // ========================================================================
6018
6019    #[test]
6020    fn test_retry_config_default() {
6021        let _guard = test_guard!();
6022        let config = RetryConfig::default();
6023        assert_eq!(config.max_attempts, 3);
6024        assert_eq!(config.base_delay_ms, 100);
6025        assert_eq!(config.max_delay_ms, 5000);
6026        assert_eq!(config.jitter_factor, 0.1);
6027        assert_eq!(config.total_timeout_ms, 30000);
6028    }
6029
6030    #[test]
6031    fn test_retry_config_no_retry() {
6032        let _guard = test_guard!();
6033        let config = RetryConfig::no_retry();
6034        assert_eq!(config.max_attempts, 1);
6035    }
6036
6037    #[test]
6038    fn test_retry_config_delay_for_attempt_zero() {
6039        let _guard = test_guard!();
6040        let config = RetryConfig::default();
6041        assert_eq!(config.delay_for_attempt(0), std::time::Duration::ZERO);
6042    }
6043
6044    #[test]
6045    fn test_retry_config_delay_for_attempt_exponential() {
6046        let _guard = test_guard!();
6047        let config = RetryConfig {
6048            base_delay_ms: 100,
6049            max_delay_ms: 10000,
6050            jitter_factor: 0.0, // No jitter for deterministic test
6051            ..Default::default()
6052        };
6053
6054        // attempt 1: 100 * 2^0 = 100ms
6055        assert_eq!(
6056            config.delay_for_attempt(1),
6057            std::time::Duration::from_millis(100)
6058        );
6059        // attempt 2: 100 * 2^1 = 200ms
6060        assert_eq!(
6061            config.delay_for_attempt(2),
6062            std::time::Duration::from_millis(200)
6063        );
6064        // attempt 3: 100 * 2^2 = 400ms
6065        assert_eq!(
6066            config.delay_for_attempt(3),
6067            std::time::Duration::from_millis(400)
6068        );
6069    }
6070
6071    #[test]
6072    fn test_retry_config_delay_capped_at_max() {
6073        let _guard = test_guard!();
6074        let config = RetryConfig {
6075            base_delay_ms: 1000,
6076            max_delay_ms: 2000,
6077            jitter_factor: 0.0,
6078            ..Default::default()
6079        };
6080
6081        // attempt 2: 1000 * 2^1 = 2000ms (at cap)
6082        assert_eq!(
6083            config.delay_for_attempt(2),
6084            std::time::Duration::from_millis(2000)
6085        );
6086        // attempt 3: 1000 * 2^2 = 4000ms, capped to 2000ms
6087        assert_eq!(
6088            config.delay_for_attempt(3),
6089            std::time::Duration::from_millis(2000)
6090        );
6091    }
6092
6093    #[test]
6094    fn test_retry_config_random_jitter_stays_within_bounds() {
6095        let _guard = test_guard!();
6096        let config = RetryConfig {
6097            base_delay_ms: 100,
6098            max_delay_ms: 200,
6099            jitter_factor: 0.25,
6100            ..Default::default()
6101        };
6102
6103        // Jitter applies after the exponential delay is capped. Exercise the
6104        // real RNG without requiring any particular random value or sequence.
6105        for (attempt, minimum_ms, maximum_ms) in [(1, 75, 125), (3, 150, 250)] {
6106            for _ in 0..128 {
6107                let delay = config.delay_for_attempt(attempt);
6108                assert!(delay >= std::time::Duration::from_millis(minimum_ms));
6109                assert!(delay <= std::time::Duration::from_millis(maximum_ms));
6110            }
6111        }
6112    }
6113
6114    #[test]
6115    fn test_retry_config_should_retry() {
6116        let _guard = test_guard!();
6117        let config = RetryConfig {
6118            max_attempts: 3,
6119            total_timeout_ms: 1000,
6120            ..Default::default()
6121        };
6122
6123        // Before max attempts and within timeout
6124        assert!(config.should_retry(0, std::time::Duration::from_millis(0)));
6125        assert!(config.should_retry(1, std::time::Duration::from_millis(500)));
6126        assert!(config.should_retry(2, std::time::Duration::from_millis(900)));
6127
6128        // At max attempts
6129        assert!(!config.should_retry(3, std::time::Duration::from_millis(0)));
6130
6131        // Past timeout
6132        assert!(!config.should_retry(1, std::time::Duration::from_millis(1001)));
6133    }
6134
6135    #[test]
6136    fn test_retry_config_serde_roundtrip() {
6137        let _guard = test_guard!();
6138        let config = RetryConfig {
6139            max_attempts: 5,
6140            base_delay_ms: 200,
6141            max_delay_ms: 8000,
6142            jitter_factor: 0.2,
6143            total_timeout_ms: 60000,
6144        };
6145        let json = serde_json::to_string(&config).unwrap();
6146        let parsed: RetryConfig = serde_json::from_str(&json).unwrap();
6147        assert_eq!(parsed.max_attempts, 5);
6148        assert_eq!(parsed.base_delay_ms, 200);
6149        assert_eq!(parsed.max_delay_ms, 8000);
6150        assert_eq!(parsed.jitter_factor, 0.2);
6151        assert_eq!(parsed.total_timeout_ms, 60000);
6152    }
6153
6154    #[test]
6155    fn test_transfer_config_includes_retry() {
6156        let _guard = test_guard!();
6157        let config = TransferConfig::default();
6158        assert_eq!(config.retry.max_attempts, 3);
6159    }
6160
6161    // =========================================================================
6162    // ExecutionConfig Tests (bd-785w)
6163    // =========================================================================
6164
6165    #[test]
6166    fn test_execution_config_default_allowlist() {
6167        let _guard = test_guard!();
6168        let config = ExecutionConfig::default();
6169        // All supported compilers should be in the default allowlist
6170        assert!(config.is_allowed("cargo"));
6171        assert!(config.is_allowed("rustc"));
6172        assert!(config.is_allowed("gcc"));
6173        assert!(config.is_allowed("g++"));
6174        assert!(config.is_allowed("clang"));
6175        assert!(config.is_allowed("clang++"));
6176        assert!(config.is_allowed("make"));
6177        assert!(config.is_allowed("cmake"));
6178        assert!(config.is_allowed("ninja"));
6179        assert!(config.is_allowed("meson"));
6180        assert!(config.is_allowed("bun"));
6181        assert!(config.is_allowed("nextest"));
6182        assert!(config.is_allowed("cc"));
6183        assert!(config.is_allowed("c++"));
6184    }
6185
6186    #[test]
6187    fn test_execution_config_is_allowed_case_insensitive() {
6188        let _guard = test_guard!();
6189        let config = ExecutionConfig::default();
6190        // Should be case-insensitive
6191        assert!(config.is_allowed("CARGO"));
6192        assert!(config.is_allowed("Cargo"));
6193        assert!(config.is_allowed("GCC"));
6194        assert!(config.is_allowed("Gcc"));
6195    }
6196
6197    #[test]
6198    fn test_execution_config_is_not_allowed() {
6199        let _guard = test_guard!();
6200        let config = ExecutionConfig::default();
6201        // Unknown commands should not be allowed
6202        assert!(!config.is_allowed("python"));
6203        assert!(!config.is_allowed("npm"));
6204        assert!(!config.is_allowed(""));
6205        // `go` and `tsc` ARE allowed now that Go/TypeScript are offload targets.
6206        // If they were missing from the allowlist the hook would fail open to
6207        // LOCAL execution — classification would look correct while every build
6208        // silently ran on the orchestrator.
6209        assert!(config.is_allowed("go"));
6210        assert!(config.is_allowed("tsc"));
6211    }
6212
6213    #[test]
6214    fn test_execution_config_empty_allowlist() {
6215        let _guard = test_guard!();
6216        let config = ExecutionConfig { allowlist: vec![] };
6217        // Empty allowlist should block everything
6218        assert!(!config.is_allowed("cargo"));
6219        assert!(!config.is_allowed("gcc"));
6220    }
6221
6222    #[test]
6223    fn test_execution_config_custom_allowlist() {
6224        let _guard = test_guard!();
6225        let config = ExecutionConfig {
6226            allowlist: vec!["cargo".to_string(), "custom_tool".to_string()],
6227        };
6228        assert!(config.is_allowed("cargo"));
6229        assert!(config.is_allowed("custom_tool"));
6230        assert!(!config.is_allowed("gcc"));
6231    }
6232
6233    #[test]
6234    fn test_execution_config_serde() {
6235        let _guard = test_guard!();
6236        let config = ExecutionConfig {
6237            allowlist: vec!["cargo".to_string(), "rustc".to_string()],
6238        };
6239        let json = serde_json::to_string(&config).unwrap();
6240        let parsed: ExecutionConfig = serde_json::from_str(&json).unwrap();
6241        assert_eq!(parsed.allowlist, config.allowlist);
6242    }
6243
6244    #[test]
6245    fn test_rch_config_includes_execution() {
6246        let _guard = test_guard!();
6247        let config = RchConfig::default();
6248        // RchConfig should include ExecutionConfig with default allowlist
6249        assert!(config.execution.is_allowed("cargo"));
6250    }
6251
6252    // ========================================================================
6253    // Adaptive Compression Tests (bd-243w)
6254    // ========================================================================
6255
6256    #[test]
6257    fn test_adaptive_compression_disabled_by_default() {
6258        let _guard = test_guard!();
6259        let config = TransferConfig::default();
6260        assert!(!config.adaptive_compression);
6261        // Should use fixed level when disabled
6262        assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6263        assert_eq!(config.select_compression_level(Some(500_000_000)), 3);
6264    }
6265
6266    #[test]
6267    fn test_adaptive_compression_small_payload() {
6268        let _guard = test_guard!();
6269        let config = TransferConfig {
6270            adaptive_compression: true,
6271            ..Default::default()
6272        };
6273        // < 10MB should use level 1
6274        assert_eq!(config.select_compression_level(Some(0)), 1);
6275        assert_eq!(config.select_compression_level(Some(1_000_000)), 1); // 1MB
6276        assert_eq!(config.select_compression_level(Some(9_999_999)), 1); // ~10MB
6277    }
6278
6279    #[test]
6280    fn test_adaptive_compression_medium_payload() {
6281        let _guard = test_guard!();
6282        let config = TransferConfig {
6283            adaptive_compression: true,
6284            ..Default::default()
6285        };
6286        // 10-200MB should use level 3
6287        assert_eq!(config.select_compression_level(Some(10_000_000)), 3); // 10MB
6288        assert_eq!(config.select_compression_level(Some(100_000_000)), 3); // 100MB
6289        assert_eq!(config.select_compression_level(Some(199_999_999)), 3); // ~200MB
6290    }
6291
6292    #[test]
6293    fn test_adaptive_compression_large_payload() {
6294        let _guard = test_guard!();
6295        let config = TransferConfig {
6296            adaptive_compression: true,
6297            ..Default::default()
6298        };
6299        // > 200MB should use level 7
6300        assert_eq!(config.select_compression_level(Some(200_000_000)), 7); // 200MB
6301        assert_eq!(config.select_compression_level(Some(500_000_000)), 7); // 500MB
6302        assert_eq!(config.select_compression_level(Some(1_000_000_000)), 7); // 1GB
6303    }
6304
6305    #[test]
6306    fn test_adaptive_compression_no_estimate() {
6307        let _guard = test_guard!();
6308        let config = TransferConfig {
6309            adaptive_compression: true,
6310            compression_level: 5,
6311            ..Default::default()
6312        };
6313        // No estimate should fallback to default level
6314        assert_eq!(config.select_compression_level(None), 5);
6315    }
6316
6317    #[test]
6318    fn test_adaptive_compression_respects_min_level() {
6319        let _guard = test_guard!();
6320        let config = TransferConfig {
6321            adaptive_compression: true,
6322            min_compression_level: 3,
6323            ..Default::default()
6324        };
6325        // Small payload would want level 1, but min is 3
6326        assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6327    }
6328
6329    #[test]
6330    fn test_adaptive_compression_respects_max_level() {
6331        let _guard = test_guard!();
6332        let config = TransferConfig {
6333            adaptive_compression: true,
6334            max_compression_level: 5,
6335            ..Default::default()
6336        };
6337        // Large payload would want level 7, but max is 5
6338        assert_eq!(config.select_compression_level(Some(500_000_000)), 5);
6339    }
6340
6341    #[test]
6342    fn test_adaptive_compression_serde_roundtrip() {
6343        let _guard = test_guard!();
6344        let config = TransferConfig {
6345            adaptive_compression: true,
6346            min_compression_level: 2,
6347            max_compression_level: 8,
6348            ..Default::default()
6349        };
6350        let json = serde_json::to_string(&config).unwrap();
6351        let parsed: TransferConfig = serde_json::from_str(&json).unwrap();
6352        assert!(parsed.adaptive_compression);
6353        assert_eq!(parsed.min_compression_level, 2);
6354        assert_eq!(parsed.max_compression_level, 8);
6355    }
6356
6357    // -------------------------------------------------------------------------
6358    // Self-Healing Config Tests (bd-59kg)
6359    // -------------------------------------------------------------------------
6360
6361    #[test]
6362    fn test_self_healing_config_defaults() {
6363        let _guard = test_guard!();
6364        // TEST START: SelfHealingConfig::default() returns expected values
6365        let config = SelfHealingConfig::default();
6366
6367        // Both self-healing behaviors enabled by default
6368        assert!(
6369            config.hook_starts_daemon,
6370            "hook_starts_daemon should be true by default"
6371        );
6372        assert!(
6373            config.daemon_installs_hooks,
6374            "daemon_installs_hooks should be true by default"
6375        );
6376
6377        // Timing defaults
6378        assert_eq!(
6379            config.auto_start_cooldown_secs, 30,
6380            "auto_start_cooldown_secs should default to 30"
6381        );
6382        assert_eq!(
6383            config.auto_start_timeout_secs, 3,
6384            "auto_start_timeout_secs should default to 3"
6385        );
6386        // TEST PASS: SelfHealingConfig defaults
6387    }
6388
6389    #[test]
6390    fn test_self_healing_config_serde_full() {
6391        let _guard = test_guard!();
6392        // TEST START: Full SelfHealingConfig serialization/deserialization
6393        let config = SelfHealingConfig {
6394            hook_starts_daemon: false,
6395            daemon_installs_hooks: false,
6396            auto_start_cooldown_secs: 60,
6397            auto_start_timeout_secs: 10,
6398            self_healing_log_level: SelfHealingLogLevel::Debug,
6399        };
6400
6401        let json = serde_json::to_string(&config).unwrap();
6402        assert!(json.contains("\"hook_starts_daemon\":false"));
6403        assert!(json.contains("\"daemon_installs_hooks\":false"));
6404        assert!(json.contains("\"auto_start_cooldown_secs\":60"));
6405        assert!(json.contains("\"auto_start_timeout_secs\":10"));
6406        assert!(json.contains("\"self_healing_log_level\":\"debug\""));
6407
6408        let parsed: SelfHealingConfig = serde_json::from_str(&json).unwrap();
6409        assert!(!parsed.hook_starts_daemon);
6410        assert!(!parsed.daemon_installs_hooks);
6411        assert_eq!(parsed.auto_start_cooldown_secs, 60);
6412        assert_eq!(parsed.auto_start_timeout_secs, 10);
6413        assert_eq!(parsed.self_healing_log_level, SelfHealingLogLevel::Debug);
6414        // TEST PASS: Full SelfHealingConfig serde
6415    }
6416
6417    // br-4zf3p: Tests for the self_healing_log_level field.
6418
6419    #[test]
6420    fn test_self_healing_log_level_default_is_info() {
6421        // TEST START: SelfHealingLogLevel::default() == Info
6422        let level = SelfHealingLogLevel::default();
6423        assert_eq!(level, SelfHealingLogLevel::Info);
6424        // Also verify the SelfHealingConfig default surfaces it.
6425        let config = SelfHealingConfig::default();
6426        assert_eq!(
6427            config.self_healing_log_level,
6428            SelfHealingLogLevel::Info,
6429            "config default should be Info"
6430        );
6431        // TEST PASS
6432    }
6433
6434    #[test]
6435    fn test_self_healing_log_level_serializes_lowercase() {
6436        // TEST START: serde uses lowercase variants
6437        let json = serde_json::to_string(&SelfHealingLogLevel::Warn).unwrap();
6438        assert_eq!(json, "\"warn\"");
6439        let parsed: SelfHealingLogLevel = serde_json::from_str("\"error\"").unwrap();
6440        assert_eq!(parsed, SelfHealingLogLevel::Error);
6441        // TEST PASS
6442    }
6443
6444    #[test]
6445    fn test_self_healing_log_level_from_env_str_accepts_known() {
6446        // TEST START: from_env_str parses each level (case-insensitive) and aliases
6447        assert_eq!(
6448            SelfHealingLogLevel::from_env_str("debug"),
6449            Some(SelfHealingLogLevel::Debug)
6450        );
6451        assert_eq!(
6452            SelfHealingLogLevel::from_env_str("DEBUG"),
6453            Some(SelfHealingLogLevel::Debug)
6454        );
6455        assert_eq!(
6456            SelfHealingLogLevel::from_env_str("info"),
6457            Some(SelfHealingLogLevel::Info)
6458        );
6459        assert_eq!(
6460            SelfHealingLogLevel::from_env_str("warn"),
6461            Some(SelfHealingLogLevel::Warn)
6462        );
6463        assert_eq!(
6464            SelfHealingLogLevel::from_env_str("warning"),
6465            Some(SelfHealingLogLevel::Warn),
6466            "warning should map to Warn"
6467        );
6468        assert_eq!(
6469            SelfHealingLogLevel::from_env_str("error"),
6470            Some(SelfHealingLogLevel::Error)
6471        );
6472        // Whitespace tolerance
6473        assert_eq!(
6474            SelfHealingLogLevel::from_env_str("  debug  "),
6475            Some(SelfHealingLogLevel::Debug)
6476        );
6477        // TEST PASS
6478    }
6479
6480    #[test]
6481    fn test_self_healing_log_level_from_env_str_rejects_unknown() {
6482        // TEST START: unknown values return None (caller falls back to default)
6483        assert!(SelfHealingLogLevel::from_env_str("banana").is_none());
6484        assert!(SelfHealingLogLevel::from_env_str("").is_none());
6485        assert!(SelfHealingLogLevel::from_env_str("trace").is_none());
6486        // TEST PASS
6487    }
6488
6489    #[test]
6490    fn test_self_healing_config_serde_partial_uses_defaults() {
6491        let _guard = test_guard!();
6492        // TEST START: Partial TOML/JSON uses defaults for missing fields
6493        let json = r#"{"hook_starts_daemon": false}"#;
6494        let config: SelfHealingConfig = serde_json::from_str(json).unwrap();
6495
6496        assert!(
6497            !config.hook_starts_daemon,
6498            "Explicit false should be parsed"
6499        );
6500        assert!(
6501            config.daemon_installs_hooks,
6502            "Missing field should use default (true)"
6503        );
6504        assert_eq!(
6505            config.auto_start_cooldown_secs, 30,
6506            "Missing field should use default (30)"
6507        );
6508        assert_eq!(
6509            config.auto_start_timeout_secs, 3,
6510            "Missing field should use default (3)"
6511        );
6512        // TEST PASS: Partial SelfHealingConfig uses defaults
6513    }
6514
6515    #[test]
6516    fn test_self_healing_config_toml_with_alias() {
6517        let _guard = test_guard!();
6518        // TEST START: daemon_start_timeout alias works for auto_start_timeout_secs
6519        let toml_str = r#"
6520            hook_starts_daemon = true
6521            daemon_installs_hooks = false
6522            auto_start_cooldown_secs = 45
6523            daemon_start_timeout = 7
6524        "#;
6525
6526        let config: SelfHealingConfig = toml::from_str(toml_str).unwrap();
6527        assert!(config.hook_starts_daemon);
6528        assert!(!config.daemon_installs_hooks);
6529        assert_eq!(config.auto_start_cooldown_secs, 45);
6530        assert_eq!(
6531            config.auto_start_timeout_secs, 7,
6532            "daemon_start_timeout alias should set auto_start_timeout_secs"
6533        );
6534        // TEST PASS: TOML alias works
6535    }
6536
6537    #[allow(unsafe_code)]
6538    mod self_healing_env_override_tests {
6539        use super::*;
6540        use crate::config::env_test_lock;
6541
6542        fn env_guard() -> std::sync::MutexGuard<'static, ()> {
6543            env_test_lock()
6544        }
6545
6546        fn set_env(key: &str, value: &str) {
6547            // SAFETY: Tests are serialized with env_guard().
6548            unsafe { std::env::set_var(key, value) };
6549        }
6550
6551        fn remove_env(key: &str) {
6552            // SAFETY: Tests are serialized with env_guard().
6553            unsafe { std::env::remove_var(key) };
6554        }
6555
6556        struct EnvVarGuard {
6557            key: &'static str,
6558            old: Option<String>,
6559        }
6560
6561        impl EnvVarGuard {
6562            fn set(key: &'static str, value: &str) -> Self {
6563                let old = std::env::var(key).ok();
6564                set_env(key, value);
6565                Self { key, old }
6566            }
6567        }
6568
6569        impl Drop for EnvVarGuard {
6570            fn drop(&mut self) {
6571                if let Some(old) = &self.old {
6572                    set_env(self.key, old);
6573                } else {
6574                    remove_env(self.key);
6575                }
6576            }
6577        }
6578
6579        #[test]
6580        fn test_self_healing_config_with_env_overrides_master_disable() {
6581            let _guard = test_guard!();
6582            let _guard = env_guard();
6583
6584            let _no_self_healing = EnvVarGuard::set("RCH_NO_SELF_HEALING", "1");
6585            let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "1");
6586            let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "1");
6587            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "99");
6588            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "99");
6589
6590            let config = SelfHealingConfig::default().with_env_overrides();
6591            assert!(
6592                !config.hook_starts_daemon,
6593                "RCH_NO_SELF_HEALING should disable hook auto-start"
6594            );
6595            assert!(
6596                !config.daemon_installs_hooks,
6597                "RCH_NO_SELF_HEALING should disable daemon hook installation"
6598            );
6599
6600            // Master disable returns early; numeric overrides should not apply.
6601            assert_eq!(config.auto_start_cooldown_secs, 30);
6602            assert_eq!(config.auto_start_timeout_secs, 3);
6603        }
6604
6605        #[test]
6606        fn test_self_healing_config_with_env_overrides_toggles_and_numbers() {
6607            let _guard = test_guard!();
6608            let _guard = env_guard();
6609
6610            let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "0");
6611            let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "false");
6612            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "45");
6613            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "7");
6614
6615            let config = SelfHealingConfig::default().with_env_overrides();
6616            assert!(!config.hook_starts_daemon);
6617            assert!(!config.daemon_installs_hooks);
6618            assert_eq!(config.auto_start_cooldown_secs, 45);
6619            assert_eq!(config.auto_start_timeout_secs, 7);
6620        }
6621
6622        #[test]
6623        fn test_self_healing_config_with_env_overrides_invalid_numbers_ignored() {
6624            let _guard = test_guard!();
6625            let _guard = env_guard();
6626
6627            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "not-a-number");
6628            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "nope");
6629
6630            let config = SelfHealingConfig::default().with_env_overrides();
6631            assert_eq!(config.auto_start_cooldown_secs, 30);
6632            assert_eq!(config.auto_start_timeout_secs, 3);
6633        }
6634    }
6635
6636    // -------------------------------------------------------------------------
6637    // Preflight Health Guard Tests (bd-3eaa)
6638    // -------------------------------------------------------------------------
6639
6640    #[test]
6641    fn test_load_per_core_calculation() {
6642        let _guard = test_guard!();
6643        let mut caps = WorkerCapabilities::new();
6644        // No metrics -> None
6645        assert!(caps.load_per_core().is_none());
6646
6647        // Only load, no cpus -> None
6648        caps.load_avg_1 = Some(4.0);
6649        assert!(caps.load_per_core().is_none());
6650
6651        // Both available
6652        caps.num_cpus = Some(4);
6653        assert_eq!(caps.load_per_core(), Some(1.0)); // 4.0 / 4 = 1.0
6654
6655        // High load
6656        caps.load_avg_1 = Some(16.0);
6657        assert_eq!(caps.load_per_core(), Some(4.0)); // 16.0 / 4 = 4.0
6658    }
6659
6660    #[test]
6661    fn test_is_high_load() {
6662        let _guard = test_guard!();
6663        let mut caps = WorkerCapabilities::new();
6664        caps.load_avg_1 = Some(8.0);
6665        caps.num_cpus = Some(4);
6666        // load_per_core = 2.0
6667
6668        // Below threshold
6669        assert_eq!(caps.is_high_load(3.0), Some(false));
6670        // At threshold
6671        assert_eq!(caps.is_high_load(2.0), Some(false)); // equal is OK
6672        // Above threshold
6673        assert_eq!(caps.is_high_load(1.5), Some(true));
6674
6675        // No metrics -> None (fail-open)
6676        caps.load_avg_1 = None;
6677        assert!(caps.is_high_load(1.0).is_none());
6678    }
6679
6680    #[test]
6681    fn test_is_low_disk() {
6682        let _guard = test_guard!();
6683        let mut caps = WorkerCapabilities::new();
6684        caps.disk_free_gb = Some(15.0);
6685
6686        // Above threshold
6687        assert_eq!(caps.is_low_disk(10.0), Some(false));
6688        // At threshold
6689        assert_eq!(caps.is_low_disk(15.0), Some(false)); // equal is OK
6690        // Below threshold
6691        assert_eq!(caps.is_low_disk(20.0), Some(true));
6692
6693        // No metrics -> None (fail-open)
6694        caps.disk_free_gb = None;
6695        assert!(caps.is_low_disk(10.0).is_none());
6696    }
6697
6698    #[test]
6699    fn test_selection_config_preflight_defaults() {
6700        let _guard = test_guard!();
6701        let config = SelectionConfig::default();
6702        // Default thresholds
6703        assert_eq!(config.max_load_per_core, Some(2.0));
6704        assert_eq!(config.min_free_gb, Some(10.0));
6705        assert_eq!(config.disk_gb_per_slot, 10.0);
6706    }
6707
6708    #[test]
6709    fn test_selection_config_preflight_serde() {
6710        let _guard = test_guard!();
6711        let config = SelectionConfig {
6712            max_load_per_core: Some(3.5),
6713            min_free_gb: Some(25.0),
6714            disk_gb_per_slot: 0.83,
6715            ..Default::default()
6716        };
6717        let json = serde_json::to_string(&config).unwrap();
6718        let parsed: SelectionConfig = serde_json::from_str(&json).unwrap();
6719        assert_eq!(parsed.max_load_per_core, Some(3.5));
6720        assert_eq!(parsed.min_free_gb, Some(25.0));
6721        assert_eq!(parsed.disk_gb_per_slot, 0.83);
6722    }
6723
6724    #[test]
6725    fn test_selection_disk_slot_budget_validation() {
6726        let _guard = test_guard!();
6727        for value in ["0.0", "-1.0", "nan", "inf", "-inf"] {
6728            let result = toml::from_str::<SelectionConfig>(&format!("disk_gb_per_slot = {value}"));
6729            assert!(result.is_err(), "invalid disk slot budget {value}");
6730        }
6731        let default: SelectionConfig = toml::from_str("").unwrap();
6732        assert_eq!(default.disk_gb_per_slot, 10.0);
6733    }
6734
6735    #[test]
6736    fn test_selection_disk_weight_validation_and_roundtrip() {
6737        for value in ["-0.1", "1.1", "nan", "inf", "-inf"] {
6738            assert!(toml::from_str::<SelectionWeightConfig>(&format!("disk = {value}")).is_err());
6739        }
6740        for value in [0.0, 0.2, 1.0] {
6741            let weights: SelectionWeightConfig =
6742                toml::from_str(&format!("disk = {value}")).unwrap();
6743            assert_eq!(weights.disk, value);
6744            let roundtrip: SelectionWeightConfig =
6745                serde_json::from_str(&serde_json::to_string(&weights).unwrap()).unwrap();
6746            assert_eq!(roundtrip.disk, value);
6747        }
6748        let defaults: SelectionWeightConfig = toml::from_str("").unwrap();
6749        assert_eq!(defaults.disk, 0.2);
6750    }
6751
6752    #[test]
6753    fn test_worker_capabilities_health_metrics_serde() {
6754        let _guard = test_guard!();
6755        let caps = WorkerCapabilities {
6756            num_cpus: Some(8),
6757            load_avg_1: Some(1.5),
6758            load_avg_5: Some(2.0),
6759            load_avg_15: Some(1.8),
6760            disk_free_gb: Some(50.5),
6761            disk_total_gb: Some(100.0),
6762            ..Default::default()
6763        };
6764        let json = serde_json::to_string(&caps).unwrap();
6765        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
6766        assert_eq!(parsed.num_cpus, Some(8));
6767        assert_eq!(parsed.load_avg_1, Some(1.5));
6768        assert_eq!(parsed.disk_free_gb, Some(50.5));
6769    }
6770
6771    #[test]
6772    fn microarch_level_classifies_ivy_bridge_as_v2() {
6773        // Abbreviated real flag set from ovh-b (Xeon E3-1245 V2, Ivy Bridge):
6774        // AVX present, but no avx2/bmi/fma — the bd-68hon SIGILL class.
6775        let ivy = "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov \
6776                   pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx rdtscp lm \
6777                   constant_tsc pni pclmulqdq ssse3 cx16 sse4_1 sse4_2 x2apic \
6778                   popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm";
6779        assert_eq!(x86_64_microarch_level_from_flags(ivy), 2);
6780    }
6781
6782    #[test]
6783    fn microarch_level_classifies_v3_and_v4() {
6784        let v3 = "cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 abm avx avx2 bmi1 bmi2 \
6785                  f16c fma movbe xsave";
6786        assert_eq!(x86_64_microarch_level_from_flags(v3), 3);
6787
6788        let v4 = format!("{v3} avx512f avx512bw avx512cd avx512dq avx512vl");
6789        assert_eq!(x86_64_microarch_level_from_flags(&v4), 4);
6790    }
6791
6792    #[test]
6793    fn microarch_level_v1_for_ancient_or_empty_flags() {
6794        assert_eq!(x86_64_microarch_level_from_flags(""), 1);
6795        assert_eq!(x86_64_microarch_level_from_flags("fpu mmx sse sse2"), 1);
6796    }
6797
6798    #[test]
6799    fn pre_v3_predicate_only_fires_on_affirmative_evidence() {
6800        let mut caps = WorkerCapabilities::new();
6801        assert!(!caps.is_pre_v3_x86(), "unknown must not deprioritize");
6802        caps.cpu_microarch_level = Some(2);
6803        assert!(caps.is_pre_v3_x86());
6804        caps.cpu_microarch_level = Some(3);
6805        assert!(!caps.is_pre_v3_x86());
6806    }
6807}