Skip to main content

rch_common/
types.rs

1//! Common types used across RCH components.
2
3use crate::{CompilationKind, toolchain::ToolchainInfo};
4use rand::RngExt;
5use schemars::JsonSchema;
6use serde::{Deserialize, Serialize};
7use std::path::PathBuf;
8
9/// Unique identifier for a worker in the fleet.
10#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
11pub struct WorkerId(pub String);
12
13impl WorkerId {
14    pub fn new(id: impl Into<String>) -> Self {
15        Self(id.into())
16    }
17
18    pub fn as_str(&self) -> &str {
19        &self.0
20    }
21}
22
23impl std::fmt::Display for WorkerId {
24    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
25        write!(f, "{}", self.0)
26    }
27}
28
29/// Status of a worker in the fleet.
30///
31/// State transitions:
32/// ```text
33/// HEALTHY ←→ DEGRADED (automatic based on response times)
34///    ↓           ↓
35///    ↓       UNREACHABLE (automatic on heartbeat failure)
36///    ↓
37/// DRAINING (via `rch workers drain`) - finishing current jobs
38///    ↓
39/// DRAINED (automatic when all jobs complete) - idle, ready to disable
40///    ↓
41/// DISABLED (via `rch workers disable`) - completely offline
42///
43/// Use `rch workers enable` to return from DRAINING/DRAINED/DISABLED → HEALTHY
44/// ```
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
46#[serde(rename_all = "snake_case")]
47pub enum WorkerStatus {
48    /// Worker is healthy and accepting jobs.
49    #[default]
50    Healthy,
51    /// Worker is responding slowly.
52    Degraded,
53    /// Worker failed to respond to heartbeat.
54    Unreachable,
55    /// Worker is not accepting new jobs (finishing current).
56    Draining,
57    /// Worker has finished draining (no active jobs, idle).
58    Drained,
59    /// Worker is manually disabled.
60    Disabled,
61}
62
63/// Circuit breaker state for a worker.
64#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
65#[serde(rename_all = "snake_case")]
66pub enum CircuitState {
67    /// Normal operation.
68    #[default]
69    Closed,
70    /// Circuit is open (short-circuit).
71    Open,
72    /// Circuit is half-open (probing).
73    HalfOpen,
74}
75
76/// Required runtime for command execution.
77#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
78#[serde(rename_all = "lowercase")]
79pub enum RequiredRuntime {
80    /// No specific runtime required (default).
81    #[default]
82    None,
83    /// Requires Rust toolchain.
84    Rust,
85    /// Requires Bun runtime.
86    Bun,
87    /// Requires Node.js runtime.
88    Node,
89    /// Requires the Nix package manager (nix binary + populated `/nix/store`).
90    Nix,
91    /// Requires the Go toolchain (`go` binary).
92    Go,
93    /// Requires the zig cross-compilation toolchain (`cargo-zigbuild` + `zig`).
94    Zig,
95}
96
97/// Per-command priority hint for worker selection.
98///
99/// This is an *input hint* from the caller (typically the hook) and should not
100/// change default behavior when omitted.
101#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
102#[serde(rename_all = "lowercase")]
103pub enum CommandPriority {
104    Low,
105    #[default]
106    Normal,
107    High,
108}
109
110impl std::fmt::Display for CommandPriority {
111    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112        let value = match self {
113            Self::Low => "low",
114            Self::Normal => "normal",
115            Self::High => "high",
116        };
117        write!(f, "{}", value)
118    }
119}
120
121impl std::str::FromStr for CommandPriority {
122    type Err = ();
123
124    fn from_str(s: &str) -> Result<Self, Self::Err> {
125        match s.trim().to_lowercase().as_str() {
126            "low" => Ok(Self::Low),
127            "normal" => Ok(Self::Normal),
128            "high" => Ok(Self::High),
129            _ => Err(()),
130        }
131    }
132}
133
134/// Worker selection request sent from hook to daemon.
135#[derive(Debug, Clone, Serialize, Deserialize)]
136pub struct SelectionRequest {
137    /// Project identifier (usually directory name or hash).
138    pub project: String,
139    /// Full command being executed (optional, for active build tracking).
140    #[serde(default, skip_serializing_if = "Option::is_none")]
141    pub command: Option<String>,
142    /// Priority hint for this request.
143    #[serde(default)]
144    pub command_priority: CommandPriority,
145    /// Estimated CPU cores needed for this compilation.
146    pub estimated_cores: u32,
147    /// Preferred worker IDs (e.g., from project config).
148    #[serde(default)]
149    pub preferred_workers: Vec<WorkerId>,
150    /// Rust toolchain information for the project.
151    #[serde(default)]
152    pub toolchain: Option<ToolchainInfo>,
153    /// Required runtime for command execution.
154    #[serde(default)]
155    pub required_runtime: RequiredRuntime,
156    /// Classification decision latency in microseconds (for AGENTS.md compliance).
157    /// This tracks how long the 5-tier classification took on the hook side.
158    #[serde(default, skip_serializing_if = "Option::is_none")]
159    pub classification_duration_us: Option<u64>,
160    /// Job-mode admission (bd-g7rpy): set by `rch exec --job` only. When the
161    /// one-active-job-per-project-per-worker guard excludes every otherwise-
162    /// eligible worker, a job request may QUEUE on that exclusion instead of
163    /// failing over to local execution. Compilation requests never set it,
164    /// preserving their immediate NoAdmissibleWorkers behavior.
165    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
166    pub job_mode: bool,
167    /// Operator-declared named tools this work requires VERIFIED on the worker
168    /// (`rch exec --job --require-tool NAME`, or a project's
169    /// `jobs.required_tools`).
170    ///
171    /// Job mode only — the compilation classifier never sets it, so ordinary
172    /// offloaded builds keep their existing admission behavior exactly. Empty
173    /// means no gate; a name no worker has verified excludes every worker,
174    /// because the caller asked for the gate precisely because the job cannot
175    /// run without the tool.
176    #[serde(default, skip_serializing_if = "Vec::is_empty")]
177    pub required_tools: Vec<String>,
178    /// Process ID of the hook (for active build tracking).
179    #[serde(default, skip_serializing_if = "Option::is_none")]
180    pub hook_pid: Option<u32>,
181}
182
183/// Reason for worker selection result.
184///
185/// Provides context when no worker is available, enabling informative
186/// fallback messages in the hook.
187#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
188#[serde(rename_all = "snake_case")]
189pub enum SelectionReason {
190    /// Worker assigned successfully.
191    Success,
192    /// No workers configured in workers.toml.
193    NoWorkersConfigured,
194    /// All workers are unreachable (failed health checks).
195    AllWorkersUnreachable,
196    /// All workers have circuits open (after repeated failures).
197    AllCircuitsOpen,
198    /// All workers are at capacity (no available slots).
199    AllWorkersBusy,
200    /// Workers were present but none passed selection health thresholds.
201    NoWorkersPassedHealth,
202    /// All candidate workers failed hard preflight checks.
203    AllWorkersFailedPreflight,
204    /// All candidate workers failed repo convergence checks (repos missing/stale/failed).
205    AllWorkersFailedConvergence,
206    /// Workers exist but admission was blocked by concrete capacity/preflight reasons.
207    NoAdmissibleWorkers(String),
208    /// No workers match required tags or preferences.
209    NoMatchingWorkers,
210    /// No workers have the required runtime (e.g., Bun, Node).
211    NoWorkersWithRuntime(String),
212    /// Internal error during selection.
213    SelectionError(String),
214    /// Worker assigned via affinity pinning (recent successful build).
215    AffinityPinned,
216    /// Worker assigned via last-success fallback (all others unavailable).
217    AffinityFallback,
218}
219
220/// Wire protocol version for daemon `/select-worker` responses.
221///
222/// This is intentionally separate from the broader CLI API envelope version:
223/// selection responses are emitted as a raw [`SelectionResponse`] on the hook
224/// hot path, so the client needs a small explicit compatibility marker.
225pub const SELECTION_RESPONSE_PROTOCOL_VERSION: u64 = 1;
226
227impl std::fmt::Display for SelectionReason {
228    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
229        match self {
230            Self::Success => write!(f, "worker assigned successfully"),
231            Self::NoWorkersConfigured => write!(f, "no workers configured"),
232            Self::AllWorkersUnreachable => write!(f, "all workers unreachable"),
233            Self::AllCircuitsOpen => write!(f, "all worker circuits open"),
234            Self::AllWorkersBusy => write!(f, "all workers at capacity"),
235            Self::NoWorkersPassedHealth => write!(f, "no workers passed health thresholds"),
236            Self::AllWorkersFailedPreflight => write!(f, "all workers failed preflight checks"),
237            Self::AllWorkersFailedConvergence => {
238                write!(f, "all workers failed repo convergence checks")
239            }
240            Self::NoAdmissibleWorkers(summary) => {
241                write!(f, "no admissible workers: {}", summary)
242            }
243            Self::NoMatchingWorkers => write!(f, "no matching workers found"),
244            Self::NoWorkersWithRuntime(rt) => write!(f, "no workers with {} installed", rt),
245            Self::SelectionError(e) => write!(f, "selection error: {}", e),
246            Self::AffinityPinned => write!(f, "worker assigned via affinity pinning"),
247            Self::AffinityFallback => write!(f, "worker assigned via last-success fallback"),
248        }
249    }
250}
251
252// ============================================================================
253// Worker Selection Strategy
254// ============================================================================
255
256/// Worker selection strategy determining how workers are chosen for jobs.
257#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
258#[serde(rename_all = "snake_case")]
259pub enum SelectionStrategy {
260    /// Legacy/manual control: prioritize worker.priority first, then break ties
261    /// using command-aware cache/speed hints.
262    Priority,
263    /// Select worker with highest SpeedScore.
264    /// Best for performance-critical builds with homogeneous workers.
265    Fastest,
266    /// Balance all factors: SpeedScore, load, health, cache affinity.
267    /// Default for general use across diverse worker pools.
268    #[default]
269    Balanced,
270    /// Prefer workers with warm caches for the project.
271    /// Best for incremental builds on large codebases.
272    CacheAffinity,
273    /// Weighted random selection favoring fast workers but ensuring fairness.
274    /// Prevents hot-spotting when many workers are available.
275    FairFastest,
276}
277
278impl std::fmt::Display for SelectionStrategy {
279    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
280        let name = match self {
281            Self::Priority => "priority",
282            Self::Fastest => "fastest",
283            Self::Balanced => "balanced",
284            Self::CacheAffinity => "cache_affinity",
285            Self::FairFastest => "fair_fastest",
286        };
287        write!(f, "{}", name)
288    }
289}
290
291impl std::str::FromStr for SelectionStrategy {
292    type Err = String;
293
294    fn from_str(s: &str) -> Result<Self, Self::Err> {
295        match s.trim().to_lowercase().as_str() {
296            "priority" => Ok(Self::Priority),
297            "fastest" => Ok(Self::Fastest),
298            "balanced" => Ok(Self::Balanced),
299            "cache_affinity" | "cache-affinity" | "cacheaffinity" => Ok(Self::CacheAffinity),
300            "fair_fastest" | "fair-fastest" | "fairfastest" => Ok(Self::FairFastest),
301            _ => Err(format!(
302                "unknown selection strategy '{}', expected one of: priority, fastest, balanced, cache_affinity, fair_fastest",
303                s
304            )),
305        }
306    }
307}
308
309/// Configuration for the worker selection algorithm.
310#[derive(Debug, Clone, Serialize, Deserialize)]
311pub struct SelectionConfig {
312    /// Selection strategy to use.
313    #[serde(default)]
314    pub strategy: SelectionStrategy,
315    /// Minimum success rate (0.0-1.0) for a worker to be considered.
316    #[serde(default = "default_min_success_rate")]
317    pub min_success_rate: f64,
318    /// Factor weights for the balanced strategy.
319    #[serde(default)]
320    pub weights: SelectionWeightConfig,
321    /// Fairness settings for fair_fastest strategy.
322    #[serde(default)]
323    pub fairness: FairnessConfig,
324    /// Cache affinity settings for project-to-worker pinning.
325    #[serde(default)]
326    pub affinity: AffinityConfig,
327
328    // Preflight health guards (bd-3eaa)
329    /// Maximum load-per-core threshold. Workers exceeding this are skipped.
330    /// Set to None to disable load-based filtering.
331    #[serde(default = "default_max_load_per_core")]
332    pub max_load_per_core: Option<f64>,
333    /// Minimum free disk space in GB. Workers below this are skipped.
334    /// Set to None to disable the free-space floor; per-slot disk derating still applies.
335    #[serde(default = "default_min_free_gb")]
336    pub min_free_gb: Option<f64>,
337    /// Disk budget in GiB per concurrent slot, after reserving `min_free_gb`.
338    /// Must be finite and positive. Unknown disk telemetry preserves configured slots.
339    #[serde(
340        default = "default_disk_gb_per_slot",
341        deserialize_with = "deserialize_disk_gb_per_slot"
342    )]
343    pub disk_gb_per_slot: f64,
344}
345
346impl Default for SelectionConfig {
347    fn default() -> Self {
348        Self {
349            strategy: SelectionStrategy::default(),
350            min_success_rate: default_min_success_rate(),
351            weights: SelectionWeightConfig::default(),
352            fairness: FairnessConfig::default(),
353            affinity: AffinityConfig::default(),
354            max_load_per_core: default_max_load_per_core(),
355            min_free_gb: default_min_free_gb(),
356            disk_gb_per_slot: default_disk_gb_per_slot(),
357        }
358    }
359}
360
361fn default_min_success_rate() -> f64 {
362    0.8
363}
364
365fn default_max_load_per_core() -> Option<f64> {
366    Some(2.0) // Skip workers with load/core > 2.0
367}
368
369fn default_min_free_gb() -> Option<f64> {
370    Some(10.0) // Skip workers with < 10 GB free
371}
372
373fn default_disk_gb_per_slot() -> f64 {
374    10.0
375}
376
377fn deserialize_disk_gb_per_slot<'de, D>(deserializer: D) -> Result<f64, D::Error>
378where
379    D: serde::Deserializer<'de>,
380{
381    let budget = f64::deserialize(deserializer)?;
382    if budget.is_finite() && budget > 0.0 {
383        Ok(budget)
384    } else {
385        Err(serde::de::Error::custom(
386            "disk_gb_per_slot must be finite and greater than zero",
387        ))
388    }
389}
390
391/// Weight configuration for the balanced selection strategy.
392#[derive(Debug, Clone, Serialize, Deserialize)]
393pub struct SelectionWeightConfig {
394    /// Weight for SpeedScore (0.0-1.0).
395    #[serde(default = "default_weight_speedscore")]
396    pub speedscore: f64,
397    /// Weight for available slots (0.0-1.0).
398    #[serde(default = "default_weight_slots")]
399    pub slots: f64,
400    /// Weight for health/success rate (0.0-1.0).
401    #[serde(default = "default_weight_health")]
402    pub health: f64,
403    /// Weight for cache affinity (0.0-1.0).
404    #[serde(default = "default_weight_cache")]
405    pub cache: f64,
406    /// Weight for network latency (0.0-1.0).
407    #[serde(default = "default_weight_network")]
408    pub network: f64,
409    /// Weight for worker priority (0.0-1.0).
410    #[serde(default = "default_weight_priority")]
411    pub priority: f64,
412    /// Weight for disk headroom (0.0-1.0); zero disables disk ranking.
413    #[serde(
414        default = "default_weight_disk",
415        deserialize_with = "deserialize_weight_disk"
416    )]
417    pub disk: f64,
418    /// Penalty multiplier for half-open circuit workers (0.0-1.0).
419    #[serde(default = "default_half_open_penalty")]
420    pub half_open_penalty: f64,
421}
422
423impl Default for SelectionWeightConfig {
424    fn default() -> Self {
425        Self {
426            speedscore: default_weight_speedscore(),
427            slots: default_weight_slots(),
428            health: default_weight_health(),
429            cache: default_weight_cache(),
430            network: default_weight_network(),
431            priority: default_weight_priority(),
432            disk: default_weight_disk(),
433            half_open_penalty: default_half_open_penalty(),
434        }
435    }
436}
437
438fn default_weight_speedscore() -> f64 {
439    0.5
440}
441fn default_weight_slots() -> f64 {
442    0.4
443}
444fn default_weight_health() -> f64 {
445    0.3
446}
447fn default_weight_cache() -> f64 {
448    0.2
449}
450fn default_weight_network() -> f64 {
451    0.1
452}
453fn default_weight_priority() -> f64 {
454    // Worker priority is a first-class selection factor: operators set high
455    // priority on preferred boxes (e.g. fast dedicated workers) and expect them
456    // to actually receive work. At the old 0.1 a top-priority worker gained at
457    // most +0.1 to its balanced score — too weak to overcome a warm-cache or
458    // lower-latency competitor, so the highest-priority workers could sit idle
459    // indefinitely. Load/slots still throttle a high-priority worker as it fills.
460    0.5
461}
462fn default_half_open_penalty() -> f64 {
463    0.5
464}
465
466fn default_weight_disk() -> f64 {
467    0.2
468}
469
470fn deserialize_weight_disk<'de, D>(deserializer: D) -> Result<f64, D::Error>
471where
472    D: serde::Deserializer<'de>,
473{
474    let weight = f64::deserialize(deserializer)?;
475    if weight.is_finite() && (0.0..=1.0).contains(&weight) {
476        Ok(weight)
477    } else {
478        Err(serde::de::Error::custom(
479            "disk weight must be finite and between zero and one",
480        ))
481    }
482}
483
484/// Fairness settings for the fair_fastest selection strategy.
485#[derive(Debug, Clone, Serialize, Deserialize)]
486pub struct FairnessConfig {
487    /// Lookback window in seconds for tracking recent selections.
488    #[serde(default = "default_fairness_lookback_secs")]
489    pub lookback_secs: u64,
490    /// Maximum consecutive selections for a single worker before penalty.
491    #[serde(default = "default_max_consecutive_selections")]
492    pub max_consecutive_selections: u32,
493}
494
495impl Default for FairnessConfig {
496    fn default() -> Self {
497        Self {
498            lookback_secs: default_fairness_lookback_secs(),
499            max_consecutive_selections: default_max_consecutive_selections(),
500        }
501    }
502}
503
504fn default_fairness_lookback_secs() -> u64 {
505    300 // 5 minutes
506}
507
508fn default_max_consecutive_selections() -> u32 {
509    3
510}
511
512/// Cache affinity settings for project-to-worker pinning.
513#[derive(Debug, Clone, Serialize, Deserialize)]
514pub struct AffinityConfig {
515    /// Enable affinity pinning (pin projects to last successful worker).
516    #[serde(default = "default_affinity_enabled")]
517    pub enabled: bool,
518    /// Time window in minutes to pin a project to its last successful worker.
519    /// After this window expires, normal selection resumes.
520    #[serde(default = "default_affinity_pin_minutes")]
521    pub pin_minutes: u64,
522    /// Enable last-success fallback when no workers match selection criteria.
523    /// If enabled, the daemon will attempt the last successful worker for a project
524    /// when all other workers are unavailable or fail selection.
525    #[serde(default = "default_last_success_fallback")]
526    pub enable_last_success_fallback: bool,
527    /// Minimum success rate for the last-success worker to be used as fallback.
528    /// Must be lower than min_success_rate to be meaningful as a fallback.
529    #[serde(default = "default_fallback_min_success_rate")]
530    pub fallback_min_success_rate: f64,
531}
532
533impl Default for AffinityConfig {
534    fn default() -> Self {
535        Self {
536            enabled: default_affinity_enabled(),
537            pin_minutes: default_affinity_pin_minutes(),
538            enable_last_success_fallback: default_last_success_fallback(),
539            fallback_min_success_rate: default_fallback_min_success_rate(),
540        }
541    }
542}
543
544fn default_affinity_enabled() -> bool {
545    true
546}
547
548fn default_affinity_pin_minutes() -> u64 {
549    60 // 1 hour default
550}
551
552fn default_last_success_fallback() -> bool {
553    true
554}
555
556fn default_fallback_min_success_rate() -> f64 {
557    0.5 // More lenient than normal min_success_rate (0.8)
558}
559
560/// Details about a selected worker for remote compilation.
561#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
562pub struct SelectedWorker {
563    /// Selected worker ID.
564    pub id: WorkerId,
565    /// Host address for SSH.
566    pub host: String,
567    /// SSH user.
568    pub user: String,
569    /// Path to SSH identity file.
570    pub identity_file: String,
571    /// Number of slots available on this worker after reservation.
572    pub slots_available: u32,
573    /// Worker's speed score (0-100).
574    pub speed_score: f64,
575    /// Operator-declared host OS (e.g. `windows`), if any. Propagated so the
576    /// hook can pick the right transport (Windows workers use `C:/rch` +
577    /// tar-over-ssh instead of the Unix rsync path). `None` for the historical
578    /// unlabelled workers. `#[serde(default)]` keeps old daemons/hooks
579    /// wire-compatible.
580    #[serde(default)]
581    pub declared_os: Option<String>,
582}
583
584/// Worker selection response from daemon to hook.
585#[derive(Debug, Clone, Serialize, Deserialize)]
586pub struct SelectionResponse {
587    /// Selected worker details, if available.
588    pub worker: Option<SelectedWorker>,
589    /// Reason for the selection result.
590    pub reason: SelectionReason,
591    /// Optional build ID assigned by the daemon (for active build tracking).
592    #[serde(default, skip_serializing_if = "Option::is_none")]
593    pub build_id: Option<u64>,
594    /// Optional per-worker selector diagnostics for machine-readable triage.
595    #[serde(default, skip_serializing_if = "Option::is_none")]
596    pub diagnostics: Option<SelectionDiagnostics>,
597}
598
599/// Final selector decision for a worker considered during routing.
600#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
601#[serde(rename_all = "snake_case")]
602pub enum WorkerSelectionDiagnosticDecision {
603    /// Worker passed hard gates and normal health threshold.
604    Allow,
605    /// Worker failed normal health threshold but passed fallback threshold.
606    FallbackCandidate,
607    /// Worker was denied by a hard gate or both health thresholds.
608    Deny,
609}
610
611/// Per-worker selector facts captured at worker-selection time.
612#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
613pub struct WorkerSelectionDiagnostic {
614    /// Worker ID.
615    pub worker_id: WorkerId,
616    /// Worker lifecycle/status label.
617    pub status: String,
618    /// Circuit breaker state label.
619    pub circuit_state: String,
620    /// Current normalized pressure state.
621    pub pressure_state: String,
622    /// Stable pressure reason code.
623    pub pressure_reason_code: String,
624    /// Rolling selector success rate, where available.
625    #[serde(skip_serializing_if = "Option::is_none")]
626    pub success_rate: Option<f64>,
627    /// Normal selection success-rate threshold.
628    pub min_success_rate: f64,
629    /// Last-success fallback success-rate threshold.
630    pub fallback_min_success_rate: f64,
631    /// Required runtime for this command.
632    pub required_runtime: RequiredRuntime,
633    /// Whether this worker has the required runtime.
634    pub runtime_available: bool,
635    /// Available slots at selection time.
636    pub available_slots: u32,
637    /// Total configured slots.
638    pub total_slots: u32,
639    /// Estimated slots requested by this command.
640    pub estimated_cores: u32,
641    /// Whether this worker was excluded because it already runs this project.
642    pub active_project_excluded: bool,
643    /// Final per-worker selector decision.
644    pub final_decision: WorkerSelectionDiagnosticDecision,
645    /// Human-readable final reason.
646    pub final_reason: String,
647    /// Stable machine-readable reasons collected for this worker.
648    pub reason_codes: Vec<String>,
649}
650
651/// Machine-readable worker selector diagnostics for a selection request.
652#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
653pub struct SelectionDiagnostics {
654    /// Required runtime for this command.
655    pub required_runtime: RequiredRuntime,
656    /// Estimated slots requested by this command.
657    pub estimated_cores: u32,
658    /// Normal success-rate threshold.
659    pub min_success_rate: f64,
660    /// Last-success fallback success-rate threshold.
661    pub fallback_min_success_rate: f64,
662    /// Count of workers excluded because they already run this project.
663    pub active_project_exclusion_count: usize,
664    /// One diagnostic entry per configured worker.
665    pub workers: Vec<WorkerSelectionDiagnostic>,
666}
667
668/// Request to release reserved worker slots.
669#[derive(Debug, Clone, Serialize, Deserialize)]
670pub struct ReleaseRequest {
671    /// ID of the worker to release slots on.
672    pub worker_id: WorkerId,
673    /// Number of slots to release.
674    pub slots: u32,
675    /// Optional build ID to mark complete.
676    #[serde(default, skip_serializing_if = "Option::is_none")]
677    pub build_id: Option<u64>,
678    /// Exact client wrapper identity associated with the admitted build.
679    #[serde(default, skip_serializing_if = "Option::is_none")]
680    pub local_wrapper_id: Option<String>,
681    /// Optional exit code for the build (used to finalize active build tracking).
682    #[serde(default, skip_serializing_if = "Option::is_none")]
683    pub exit_code: Option<i32>,
684    /// Optional total duration in milliseconds (pipeline or execution).
685    ///
686    /// If omitted, the daemon will compute duration from the active build's start time.
687    #[serde(default, skip_serializing_if = "Option::is_none")]
688    pub duration_ms: Option<u64>,
689    /// Optional bytes transferred during the pipeline (upload + artifact download).
690    #[serde(default, skip_serializing_if = "Option::is_none")]
691    pub bytes_transferred: Option<u64>,
692    /// Optional per-phase timing breakdown for the build pipeline.
693    #[serde(default, skip_serializing_if = "Option::is_none")]
694    pub timing: Option<CommandTimingBreakdown>,
695}
696
697/// Build execution phase for daemon heartbeat tracking.
698#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
699#[serde(rename_all = "snake_case")]
700pub enum BuildHeartbeatPhase {
701    /// Uploading sources/dependencies to the worker.
702    SyncUp,
703    /// Running the remote compilation/test command.
704    Execute,
705    /// Downloading build artifacts from the worker.
706    SyncDown,
707    /// Build is completing or finalizing.
708    Finalize,
709}
710
711/// Request to update liveness/progress for an active build.
712#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
713pub struct BuildHeartbeatRequest {
714    /// Active build identifier assigned by the daemon.
715    pub build_id: u64,
716    /// Worker executing the build.
717    pub worker_id: WorkerId,
718    /// Hook process ID sending the heartbeat.
719    #[serde(default, skip_serializing_if = "Option::is_none")]
720    pub hook_pid: Option<u32>,
721    /// Client-minted wrapper identity for local durable-lease correlation.
722    #[serde(default, skip_serializing_if = "Option::is_none")]
723    pub local_wrapper_id: Option<String>,
724    /// Remote file containing the process-group leader PID for cancellation.
725    #[serde(default, skip_serializing_if = "Option::is_none")]
726    pub remote_pgid_file: Option<String>,
727    /// Current build execution phase.
728    pub phase: BuildHeartbeatPhase,
729    /// Optional human-readable progress detail.
730    #[serde(default, skip_serializing_if = "Option::is_none")]
731    pub detail: Option<String>,
732    /// Optional monotonic progress counter.
733    ///
734    /// The hook increments this when it observes meaningful progress
735    /// (phase transitions, new output, or richer progress signals).
736    #[serde(default, skip_serializing_if = "Option::is_none")]
737    pub progress_counter: Option<u64>,
738    /// Optional progress estimate in [0,100].
739    #[serde(default, skip_serializing_if = "Option::is_none")]
740    pub progress_percent: Option<f64>,
741}
742
743/// Configuration for a remote worker.
744#[derive(Debug, Clone, Serialize, Deserialize)]
745pub struct WorkerConfig {
746    /// Unique identifier for this worker.
747    pub id: WorkerId,
748    /// SSH hostname or IP address.
749    pub host: String,
750    /// SSH username.
751    pub user: String,
752    /// Path to SSH private key.
753    pub identity_file: String,
754    /// Total CPU slots available on this worker.
755    pub total_slots: u32,
756    /// Priority for worker selection (higher = preferred).
757    #[serde(default = "default_priority")]
758    pub priority: u32,
759    /// Optional tags for filtering.
760    ///
761    /// Mostly descriptive, with one reserved form: `os:<name>` (see
762    /// [`crate::declared_os`]) is a hard admission gate, written in
763    /// `workers.toml` as `os = "<name>"`.
764    #[serde(default)]
765    pub tags: Vec<String>,
766    /// Operator-declared named tool probes for this worker, written in
767    /// `workers.toml` as
768    /// `tools = [{ name = "clang", command = ["clang", "--version"] }]`.
769    ///
770    /// Unlike a tag, which is an unverified naming convention, each entry is
771    /// VERIFIED by running its fixed argv on the worker as the configured user;
772    /// only a zero exit makes the tool present. That is what lets
773    /// `--require-tool` gate selection on evidence instead of on a promise.
774    #[serde(default, skip_serializing_if = "Vec::is_empty")]
775    pub tools: Vec<WorkerToolProbe>,
776}
777
778/// One operator-declared tool probe as it appears in `workers.toml`.
779///
780/// Kept as plain configuration data; [`crate::capability_probe::NamedToolProbe`]
781/// is the validated form, and the conversion is where a malformed name is
782/// refused.
783#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
784pub struct WorkerToolProbe {
785    /// The name `--require-tool` and `jobs.required_tools` refer to.
786    pub name: String,
787    /// The exact argv to run. Zero exit = present; anything else = absent.
788    pub command: Vec<String>,
789}
790
791fn default_priority() -> u32 {
792    100
793}
794
795/// Tag prefix reserved for the operator-declared host OS.
796pub const OS_TAG_PREFIX: &str = "os:";
797
798/// The host OS an operator declared for a worker, from its tags.
799///
800/// Written in `workers.toml` as `os = "windows"` and normalized into the
801/// reserved `os:windows` tag on load, so it rides the existing tag vector
802/// through every path that already carries a [`WorkerConfig`].
803///
804/// Declaring it makes the worker **exclusive**: selection admits it only for
805/// commands that require that OS, so a Windows box cannot quietly absorb a
806/// `cargo check` dispatched from Linux and return wrong-platform artifacts.
807/// Workers without the tag — every historical worker — keep taking anything.
808#[must_use]
809pub fn declared_os(tags: &[String]) -> Option<String> {
810    // `find` over the mapped values, not `find_map` over the tags: a malformed
811    // `os:` entry must not mask a valid declaration later in the list, because
812    // failing open here is precisely the "Windows worker silently takes Linux
813    // jobs" outcome the gate exists to prevent.
814    tags.iter()
815        .filter_map(|tag| tag.strip_prefix(OS_TAG_PREFIX))
816        .map(|os| os.trim().to_ascii_lowercase())
817        .find(|os| !os.is_empty())
818}
819
820/// Build the reserved tag for a declared OS.
821#[must_use]
822pub fn os_tag(os: &str) -> String {
823    format!("{OS_TAG_PREFIX}{}", os.trim().to_ascii_lowercase())
824}
825
826impl Default for WorkerConfig {
827    fn default() -> Self {
828        Self {
829            id: WorkerId::new("default-worker"),
830            host: "localhost".to_string(),
831            user: "user".to_string(),
832            identity_file: "~/.ssh/id_rsa".to_string(),
833            total_slots: 4,
834            priority: default_priority(),
835            tags: Vec::new(),
836            tools: Vec::new(),
837        }
838    }
839}
840
841/// Runtime capabilities detected on a worker.
842///
843/// These are probed during health checks and cached for routing decisions.
844/// Commands requiring specific runtimes (e.g., `bun test`) can be routed
845/// only to workers with the corresponding capability.
846#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)]
847pub struct WorkerCapabilities {
848    /// Rust compiler version (from `rustc --version`).
849    #[serde(default, skip_serializing_if = "Option::is_none")]
850    pub rustc_version: Option<String>,
851    /// Installed rustup toolchains, including their host triple when rustup
852    /// reports one (for example `nightly-2026-07-05-x86_64-unknown-linux-gnu`).
853    #[serde(default, skip_serializing_if = "Vec::is_empty")]
854    pub rustup_toolchains: Vec<String>,
855    /// Installed rustup components as `<toolchain>:<component>` facts. Component
856    /// names are normalized by removing only the probed host-triple suffix, so
857    /// `clippy-preview` can never satisfy a `clippy` requirement.
858    #[serde(default, skip_serializing_if = "Vec::is_empty")]
859    pub rustup_components: Vec<String>,
860    /// Non-fatal diagnostics gathered while probing (for example a tool binary
861    /// that had to be resolved from a well-known fallback location because the
862    /// caller's PATH was minimal). Empty in the common case; clients may
863    /// surface these so "no facts" is distinguishable from "facts show
864    /// absence" (bd-deft5).
865    #[serde(default, skip_serializing_if = "Vec::is_empty")]
866    pub probe_warnings: Vec<String>,
867    /// Bun runtime version (from `bun --version`).
868    #[serde(default, skip_serializing_if = "Option::is_none")]
869    pub bun_version: Option<String>,
870    /// Node.js version (from `node --version`).
871    #[serde(default, skip_serializing_if = "Option::is_none")]
872    pub node_version: Option<String>,
873    /// npm version (from `npm --version`).
874    #[serde(default, skip_serializing_if = "Option::is_none")]
875    pub npm_version: Option<String>,
876    /// Nix version (from `nix --version`). Presence implies a usable nix binary
877    /// with an accompanying `/nix/store`, gating `nix build`/`nix develop -c`
878    /// routing to this worker.
879    #[serde(default, skip_serializing_if = "Option::is_none")]
880    pub nix_version: Option<String>,
881    /// Go toolchain version (from `go version`). Presence gates `go build`/
882    /// `go test`/`go vet` routing to this worker.
883    #[serde(default, skip_serializing_if = "Option::is_none")]
884    pub go_version: Option<String>,
885    /// Zig version (from `zig version`). `cargo zigbuild` shells out to `zig`
886    /// as the linker/C toolchain, so the bare `zig` binary is necessary but not
887    /// sufficient — see [`WorkerCapabilities::has_zig`].
888    #[serde(default, skip_serializing_if = "Option::is_none")]
889    pub zig_version: Option<String>,
890    /// cargo-zigbuild version (from `cargo-zigbuild --version`). Probed as the
891    /// hyphenated binary rather than `cargo zigbuild --version`, which the
892    /// subcommand's own parser rejects.
893    #[serde(default, skip_serializing_if = "Option::is_none")]
894    pub cargo_zigbuild_version: Option<String>,
895    /// x86-64 microarchitecture level (1..=4) parsed from `/proc/cpuinfo`
896    /// flags on the worker (bd-6qchz / bd-68hon item 4). `Some(2)` marks a
897    /// pre-AVX2 box (e.g. ovh-b's Ivy Bridge) whose build scripts SIGILL on
898    /// x86-64-v3 codegen; selection deprioritizes such workers proactively
899    /// while the reactive SIGILL quarantine remains the primary mechanism.
900    /// `None` on non-x86 workers or when the probe cannot read cpuinfo.
901    #[serde(default, skip_serializing_if = "Option::is_none")]
902    pub cpu_microarch_level: Option<u8>,
903    /// Operator-declared named tools whose probe exited ZERO on this worker.
904    ///
905    /// The list is evidence, not configuration: a name appears here only
906    /// because the worker ran the declared argv and it succeeded. Selection
907    /// gates `--require-tool` on this, which is the difference between a
908    /// verified capability and a tag somebody wrote by hand.
909    #[serde(default, skip_serializing_if = "Vec::is_empty")]
910    pub tools_present: Vec<String>,
911    /// Operator-declared named tools whose probe RAN and did not exit zero.
912    ///
913    /// Kept separate from "never declared" so an operator can tell a worker
914    /// that is missing a tool from a fleet that never probed for it — the two
915    /// have different fixes.
916    #[serde(default, skip_serializing_if = "Vec::is_empty")]
917    pub tools_absent: Vec<String>,
918
919    // Health metrics (bd-3eaa)
920    /// Number of CPU cores on the worker.
921    #[serde(default, skip_serializing_if = "Option::is_none")]
922    pub num_cpus: Option<u32>,
923    /// 1-minute load average.
924    #[serde(default, skip_serializing_if = "Option::is_none")]
925    pub load_avg_1: Option<f64>,
926    /// 5-minute load average.
927    #[serde(default, skip_serializing_if = "Option::is_none")]
928    pub load_avg_5: Option<f64>,
929    /// 15-minute load average.
930    #[serde(default, skip_serializing_if = "Option::is_none")]
931    pub load_avg_15: Option<f64>,
932    /// Free space in GB on the tightest mount a build touches: whichever of
933    /// the projects root, its alias and `/tmp` has the lowest free ratio.
934    /// Drives pressure classification (a full `/tmp` breaks scp/mktemp).
935    #[serde(default, skip_serializing_if = "Option::is_none")]
936    pub disk_free_gb: Option<f64>,
937    /// Total size in GB of the mount reported in `disk_free_gb`.
938    #[serde(default, skip_serializing_if = "Option::is_none")]
939    pub disk_total_gb: Option<f64>,
940    /// Free space in GB on the filesystem that holds build trees (the fuller
941    /// of the projects root and its alias; `/tmp` excluded). Sizes capacity:
942    /// slot derating, disk headroom and the free-space floor (GH #78).
943    /// `None` from an `rch-wkr` that predates the field.
944    #[serde(default, skip_serializing_if = "Option::is_none")]
945    pub build_disk_free_gb: Option<f64>,
946    /// Total size in GB of the mount reported in `build_disk_free_gb`.
947    #[serde(default, skip_serializing_if = "Option::is_none")]
948    pub build_disk_total_gb: Option<f64>,
949    /// Canonical path-topology preflight status (`/data/projects` + `/dp` alias).
950    ///
951    /// `Some(false)` indicates a hard preflight failure that should exclude this
952    /// worker from remote scheduling until a subsequent successful revalidation.
953    #[serde(default, skip_serializing_if = "Option::is_none")]
954    pub projects_root_ok: Option<bool>,
955    /// Machine-readable failure reason for path-topology preflight.
956    #[serde(default, skip_serializing_if = "Option::is_none")]
957    pub projects_root_issue: Option<String>,
958    /// Unix timestamp (ms) when topology preflight last executed.
959    #[serde(default, skip_serializing_if = "Option::is_none")]
960    pub projects_root_checked_at_unix_ms: Option<i64>,
961}
962
963impl WorkerCapabilities {
964    /// Create new empty capabilities.
965    pub fn new() -> Self {
966        Self::default()
967    }
968
969    /// Create mock capabilities with Rust installed.
970    ///
971    /// Used by mock transport to simulate a worker with Rust toolchain.
972    pub fn mock_with_rust() -> Self {
973        Self {
974            rustc_version: Some("1.85.0-nightly (mock)".to_string()),
975            projects_root_ok: Some(true),
976            ..Self::default()
977        }
978    }
979
980    /// Whether this is a known pre-x86-64-v3 (no AVX2) x86 worker — the class
981    /// whose build-script/proc-macro binaries SIGILL on v3 codegen (bd-68hon).
982    /// `None` (unknown/non-x86) is NOT pre-v3: only affirmative evidence
983    /// deprioritizes a worker.
984    pub fn is_pre_v3_x86(&self) -> bool {
985        matches!(self.cpu_microarch_level, Some(1 | 2))
986    }
987
988    /// Check if this worker has Bun installed.
989    pub fn has_bun(&self) -> bool {
990        self.bun_version.is_some()
991    }
992
993    /// Check if this worker has Node.js installed.
994    pub fn has_node(&self) -> bool {
995        self.node_version.is_some()
996    }
997
998    /// Check if this worker has Rust installed.
999    pub fn has_rust(&self) -> bool {
1000        self.rustc_version.is_some()
1001    }
1002
1003    /// Check whether `component` is installed for `toolchain`.
1004    ///
1005    /// Rustup reports pinned toolchains with a host-triple suffix. The request
1006    /// carries the project pin without that suffix, so the owner half uses the
1007    /// same exact-or-`-`-boundary prefix rule as toolchain admission. The
1008    /// component half is exact.
1009    pub fn has_rustup_component(&self, toolchain: &str, component: &str) -> bool {
1010        let owner_prefix = format!("{toolchain}-");
1011        self.rustup_components.iter().any(|fact| {
1012            fact.split_once(':').is_some_and(|(owner, name)| {
1013                name == component && (owner == toolchain || owner.starts_with(&owner_prefix))
1014            })
1015        })
1016    }
1017
1018    /// Check if this worker has Nix installed (nix binary + `/nix/store`).
1019    pub fn has_nix(&self) -> bool {
1020        self.nix_version.is_some()
1021    }
1022
1023    /// Check if this worker has the Go toolchain installed.
1024    pub fn has_go(&self) -> bool {
1025        self.go_version.is_some()
1026    }
1027
1028    /// Check if this worker can run `cargo zigbuild` cross-compiles.
1029    ///
1030    /// Requires BOTH halves of the toolchain: the `cargo-zigbuild` cargo
1031    /// subcommand and the `zig` binary it drives as the linker. Either one
1032    /// alone fails at build time with an error the remote-result heuristics do
1033    /// not recognize as infrastructure (`error: no such command: 'zigbuild'` is
1034    /// not a rustup message), so the exit code would be surfaced verbatim
1035    /// instead of falling open to local. Gating both here keeps that failure
1036    /// out of the user's build.
1037    pub fn has_zig(&self) -> bool {
1038        self.zig_version.is_some() && self.cargo_zigbuild_version.is_some()
1039    }
1040
1041    /// Calculate load per core (1-minute load average / num_cpus).
1042    /// Returns None if metrics are unavailable.
1043    pub fn load_per_core(&self) -> Option<f64> {
1044        match (self.load_avg_1, self.num_cpus) {
1045            (Some(load), Some(cpus)) if cpus > 0 => Some(load / cpus as f64),
1046            _ => None,
1047        }
1048    }
1049
1050    /// Check if worker has high load (exceeds threshold).
1051    /// Returns None if metrics unavailable (fail-open).
1052    pub fn is_high_load(&self, max_load_per_core: f64) -> Option<bool> {
1053        self.load_per_core().map(|lpc| lpc > max_load_per_core)
1054    }
1055
1056    /// `(free_gb, total_gb)` of the filesystem that holds build trees.
1057    ///
1058    /// Capacity questions (how many builds fit, is the free-space floor met)
1059    /// are about that filesystem, not about a small tmpfs `/tmp` that happens
1060    /// to be the tightest mount. Falls back to the tightest-mount sample when
1061    /// the worker does not report a build-disk sample (older `rch-wkr`), which
1062    /// is the conservative pre-GH-#78 behaviour.
1063    pub fn build_disk_gb(&self) -> (Option<f64>, Option<f64>) {
1064        if self.build_disk_free_gb.is_some() {
1065            (self.build_disk_free_gb, self.build_disk_total_gb)
1066        } else {
1067            (self.disk_free_gb, self.disk_total_gb)
1068        }
1069    }
1070
1071    /// Check if the build disk is below the free-space threshold.
1072    /// Returns None if metrics unavailable (fail-open).
1073    pub fn is_low_disk(&self, min_free_gb: f64) -> Option<bool> {
1074        self.build_disk_gb().0.map(|free| free < min_free_gb)
1075    }
1076
1077    /// Check if canonical path-topology preflight is healthy.
1078    ///
1079    /// Returns `None` when probe data is unavailable (fail-open behavior).
1080    pub fn is_topology_healthy(&self) -> Option<bool> {
1081        self.projects_root_ok
1082    }
1083}
1084
1085/// Classify an x86-64 microarchitecture level (1..=4) from a `/proc/cpuinfo`
1086/// `flags` line (bd-6qchz).
1087///
1088/// Levels follow the psABI x86-64-v2/v3/v4 feature baselines, restricted to
1089/// flags as they appear in Linux cpuinfo. A CPU missing any flag of a level is
1090/// classified at the level below — e.g. ovh-b's Ivy Bridge E3-1245 V2 has AVX
1091/// but no `avx2`/`bmi2`/`fma`, so it classifies as 2 and its v3-codegen build
1092/// scripts SIGILL (the bd-68hon incident class).
1093#[must_use]
1094pub fn x86_64_microarch_level_from_flags(flags: &str) -> u8 {
1095    let set: std::collections::HashSet<&str> = flags.split_whitespace().collect();
1096    let has_all = |required: &[&str]| required.iter().all(|flag| set.contains(flag));
1097    // x86-64-v2: CMPXCHG16B, LAHF/SAHF, POPCNT, SSE3/SSSE3/SSE4.x.
1098    const V2: &[&str] = &["cx16", "lahf_lm", "popcnt", "sse4_1", "sse4_2", "ssse3"];
1099    // x86-64-v3: AVX/AVX2, BMI1/2, F16C, FMA, MOVBE, XSAVE (LZCNT is `abm`).
1100    const V3: &[&str] = &[
1101        "abm", "avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "movbe", "xsave",
1102    ];
1103    // x86-64-v4: the AVX-512 F/BW/CD/DQ/VL baseline.
1104    const V4: &[&str] = &["avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl"];
1105    if !has_all(V2) {
1106        return 1;
1107    }
1108    if !has_all(V3) {
1109        return 2;
1110    }
1111    if !has_all(V4) {
1112        return 3;
1113    }
1114    4
1115}
1116
1117/// Path topology configuration.
1118///
1119/// Overrides the default canonical and alias project roots used by
1120/// [`crate::path_topology::PathTopologyPolicy`].  This is useful on platforms
1121/// such as macOS where SIP prevents using `/data/projects` and the user must
1122/// choose a different root (e.g. `~/Projects`).
1123///
1124/// ```toml
1125/// [path_topology]
1126/// canonical_root = "/Users/me/Projects"
1127/// alias_root = "/Users/me/p"
1128/// ```
1129#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1130pub struct PathTopologyConfig {
1131    /// Canonical project root directory.
1132    /// Defaults to `/data/projects` when absent.
1133    #[serde(default)]
1134    pub canonical_root: Option<String>,
1135
1136    /// Alias (symlink) root that points at the canonical root.
1137    /// Defaults to `/dp` when absent.
1138    #[serde(default)]
1139    pub alias_root: Option<String>,
1140}
1141
1142impl PathTopologyConfig {
1143    /// Build a [`crate::path_topology::PathTopologyPolicy`] from this config,
1144    /// falling back to the compiled-in defaults for any field left unset.
1145    ///
1146    /// Empty strings are treated as unset (fall back to defaults).
1147    /// Tilde prefixes (`~/...`) are expanded to the user's home directory.
1148    pub fn to_policy(&self) -> crate::path_topology::PathTopologyPolicy {
1149        let canonical = self
1150            .canonical_root
1151            .as_deref()
1152            .filter(|s| !s.is_empty())
1153            .map(|s| shellexpand::tilde(s).into_owned())
1154            .unwrap_or_else(|| crate::path_topology::DEFAULT_CANONICAL_PROJECT_ROOT.to_string());
1155        let alias = self
1156            .alias_root
1157            .as_deref()
1158            .filter(|s| !s.is_empty())
1159            .map(|s| shellexpand::tilde(s).into_owned())
1160            .unwrap_or_else(|| crate::path_topology::DEFAULT_ALIAS_PROJECT_ROOT.to_string());
1161        crate::path_topology::PathTopologyPolicy::new(
1162            PathBuf::from(canonical),
1163            PathBuf::from(alias),
1164        )
1165    }
1166}
1167
1168/// RCH configuration.
1169#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1170pub struct RchConfig {
1171    #[serde(default)]
1172    pub general: GeneralConfig,
1173    #[serde(default)]
1174    pub compilation: CompilationConfig,
1175    #[serde(default)]
1176    pub transfer: TransferConfig,
1177    #[serde(default)]
1178    pub environment: EnvironmentConfig,
1179    #[serde(default)]
1180    pub circuit: CircuitBreakerConfig,
1181    #[serde(default)]
1182    pub output: OutputConfig,
1183    #[serde(default)]
1184    pub self_healing: SelfHealingConfig,
1185    #[serde(default)]
1186    pub self_test: SelfTestConfig,
1187    /// Worker selection algorithm configuration.
1188    #[serde(default)]
1189    pub selection: SelectionConfig,
1190    /// Execution allowlist configuration (bd-785w).
1191    #[serde(default)]
1192    pub execution: ExecutionConfig,
1193    /// Worker health alerting configuration (daemon).
1194    #[serde(default)]
1195    pub alerts: AlertsConfig,
1196    /// Fleet operation configuration (bd-rs7w.2).
1197    #[serde(default)]
1198    pub fleet: FleetConfig,
1199    /// Path topology overrides for project root directories.
1200    #[serde(default)]
1201    pub path_topology: PathTopologyConfig,
1202    /// Doctor / reliability subsystem configuration (verdict webhooks).
1203    #[serde(default)]
1204    pub doctor: DoctorConfig,
1205    /// Session-history remediation knobs: the central schema and default policy
1206    /// for temporary bypass, auto-rejoin, reconciliation, proof, incident
1207    /// ledger, build-root, pooled targets, telemetry freshness, log retention,
1208    /// disk pressure, and smoke defaults (bd-...remediation-ocv9i.17.1).
1209    #[serde(default)]
1210    pub remediation: crate::remediation_config::RemediationConfig,
1211    /// Layer 0 configuration pack knobs (bd-bqu38 / M-1 §149). Defaults OFF;
1212    /// injected as CARGO_PROFILE_* env vars into remote cargo builds.
1213    #[serde(default)]
1214    pub layer0: Layer0Config,
1215    /// Daemon status API over TCP for other machines on the tailnet
1216    /// (`[api]`, bd-2f5ms). Off unless `bind` is set.
1217    #[serde(default)]
1218    pub api: ApiConfig,
1219    /// Where the fleet dashboard lives (`[dashboard]`), for `rch web`.
1220    #[serde(default)]
1221    pub dashboard: DashboardConfig,
1222    /// Job-mode defaults (`[jobs]`), for `rch exec --job`.
1223    #[serde(default)]
1224    pub jobs: JobsConfig,
1225}
1226
1227/// Job-mode configuration (`[jobs]`).
1228#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1229pub struct JobsConfig {
1230    /// Named tools every job from this project requires a worker to have
1231    /// VERIFIED, as though each were passed with `--require-tool`.
1232    ///
1233    /// This is the project saying "my jobs cannot run on a worker without
1234    /// these", so it applies on top of any `--require-tool` given on the
1235    /// command line rather than being replaced by it. It affects job mode
1236    /// ONLY: ordinary offloaded compilation keeps its existing admission
1237    /// behavior, because a project-wide requirement that silently narrowed
1238    /// every build's worker pool would be a surprising way to lose the fleet.
1239    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1240    pub required_tools: Vec<String>,
1241}
1242
1243/// Daemon status API over TCP (`[api]`).
1244///
1245/// rchd's rich state (`/status`, `/workers/capabilities`) lives on a `0600`
1246/// Unix socket, which only the local user can read. This section opens the
1247/// same JSON on a TCP listener so agents and the fleet dashboard on OTHER
1248/// machines can ask a dispatcher "why are you local-only right now?" without
1249/// ssh. It is meant for a tailnet: the bind address must be loopback or a
1250/// Tailscale address (100.64.0.0/10, fd7a:115c:a1e0::/48) unless
1251/// `allow_any_addr` is set, and every status route requires the bearer token
1252/// unless `token` is unset AND `no_token` is explicitly true.
1253///
1254/// ```toml
1255/// [api]
1256/// bind = "tailscale"       # or "tailscale:9101", "100.68.51.94:9101", "127.0.0.1:9101"
1257/// token = "…"              # or token_file = "~/.config/rch/api.token"
1258/// ```
1259#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1260#[serde(default)]
1261pub struct ApiConfig {
1262    /// Where to listen. `"tailscale"` resolves this machine's Tailscale IPv4
1263    /// at startup (default port 9101); `"tailscale:PORT"`, `"IP:PORT"` or
1264    /// `"[v6]:PORT"` are taken literally. Unset (empty) = API off.
1265    pub bind: String,
1266    /// Bearer token required on `/status` and `/workers/capabilities`.
1267    pub token: Option<String>,
1268    /// File holding the bearer token (trimmed). Takes precedence over `token`.
1269    pub token_file: Option<String>,
1270    /// Serve the status routes with NO token. Only honoured when no token is
1271    /// configured; must be set explicitly so an empty token is never an
1272    /// accidental open door.
1273    pub no_token: bool,
1274    /// Permit a bind address outside loopback and the Tailscale ranges.
1275    pub allow_any_addr: bool,
1276}
1277
1278/// Where the fleet dashboard is served (`[dashboard]`).
1279#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1280#[serde(default)]
1281pub struct DashboardConfig {
1282    /// URL `rch web` opens, e.g. `https://rch-fleet.vercel.app`.
1283    pub url: Option<String>,
1284}
1285
1286/// Doctor reliability subsystem configuration.
1287///
1288/// Currently carries the verdict-transition webhook surface. Kept as its own
1289/// section (`[doctor]`) so future doctor knobs land here rather than bloating
1290/// the top-level config.
1291#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1292pub struct DoctorConfig {
1293    /// Verdict-transition webhook destinations.
1294    #[serde(default)]
1295    pub webhooks: DoctorWebhooksConfig,
1296}
1297
1298/// Webhook notification surface for reliability verdict transitions.
1299///
1300/// Fires when `rch doctor --reliability --watch` observes a verdict change
1301/// (`Healthy → Degraded`, `Degraded → Failing`, etc.). Each endpoint selects
1302/// which transitions it cares about and how the payload is formatted.
1303#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1304pub struct DoctorWebhooksConfig {
1305    /// Named webhook destinations. Empty = no webhook dispatch.
1306    #[serde(default)]
1307    pub endpoints: Vec<DoctorWebhookEndpoint>,
1308    /// Maximum number of in-flight webhook deliveries. Bounds memory and
1309    /// outbound connection pressure: when the queue is full, a transition's
1310    /// delivery is dropped (and journaled) rather than growing unboundedly.
1311    /// Per-process, shared across all endpoints.
1312    #[serde(default = "default_webhook_max_inflight")]
1313    pub max_inflight: usize,
1314}
1315
1316// Manual `Default` so the in-code default matches the serde default
1317// (`#[derive(Default)]` would give `max_inflight = 0`, diverging from a
1318// from-TOML load that fills in 16).
1319impl Default for DoctorWebhooksConfig {
1320    fn default() -> Self {
1321        Self {
1322            endpoints: Vec::new(),
1323            max_inflight: default_webhook_max_inflight(),
1324        }
1325    }
1326}
1327
1328/// A single webhook destination for verdict transitions.
1329///
1330/// SECRETS ARE NEVER STORED HERE. Routing keys, bearer tokens, and signing
1331/// secrets are referenced by the *name* of an environment variable
1332/// (`*_env` fields) read at dispatch time. There is deliberately no
1333/// `secret` / `routing_key` / `bearer_token` / `password` field on this
1334/// struct (enforced by `test_no_inline_secret_fields`).
1335#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1336pub struct DoctorWebhookEndpoint {
1337    /// Operator-facing name (appears in logs and the audit trail).
1338    pub name: String,
1339    /// Destination URL.
1340    pub url: String,
1341    /// Payload format.
1342    #[serde(default)]
1343    pub format: DoctorWebhookFormat,
1344    /// Transition predicates that arm this endpoint. Empty = never fire.
1345    /// Supported: `healthy_to_degraded`, `degraded_to_failing`,
1346    /// `any_to_failing`, `any_to_degraded`, `failing_to_healthy`,
1347    /// `degraded_to_healthy`, `any_to_healthy`, `any_to_any`.
1348    #[serde(default)]
1349    pub on_transitions: Vec<String>,
1350    /// Maximum retry attempts after the first try fails.
1351    #[serde(default = "default_webhook_retry_max")]
1352    pub retry_max: u32,
1353    /// Base backoff between retries (milliseconds); doubles each attempt.
1354    #[serde(default = "default_webhook_retry_backoff_ms")]
1355    pub retry_backoff_ms: u64,
1356    /// Per-attempt request timeout (milliseconds).
1357    #[serde(default = "default_webhook_timeout_ms")]
1358    pub timeout_ms: u64,
1359    /// Per-endpoint kill switch.
1360    #[serde(default = "default_true")]
1361    pub enabled: bool,
1362    /// Name of the env var holding the PagerDuty Events API routing key.
1363    /// Required for `format = "pagerduty"`. Read at runtime, never stored.
1364    #[serde(default)]
1365    pub routing_key_env: Option<String>,
1366    /// Name of the env var holding a bearer token sent as
1367    /// `Authorization: Bearer <token>`. Read at runtime, never stored.
1368    #[serde(default)]
1369    pub bearer_token_env: Option<String>,
1370    /// Name of the env var holding an HMAC-SHA256 signing secret. When set,
1371    /// the body is signed and the signature sent as `X-RCH-Signature:
1372    /// sha256=<hex>`. Read at runtime, never stored.
1373    #[serde(default)]
1374    pub signing_secret_env: Option<String>,
1375}
1376
1377/// Built-in webhook payload formats.
1378#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
1379#[serde(rename_all = "snake_case")]
1380pub enum DoctorWebhookFormat {
1381    /// Opinionated RCH JSON schema (default).
1382    #[default]
1383    GenericJson,
1384    /// Slack Block Kit payload.
1385    Slack,
1386    /// PagerDuty Events API v2 payload.
1387    Pagerduty,
1388}
1389
1390fn default_webhook_max_inflight() -> usize {
1391    16
1392}
1393
1394fn default_webhook_retry_max() -> u32 {
1395    3
1396}
1397
1398fn default_webhook_retry_backoff_ms() -> u64 {
1399    200
1400}
1401
1402fn default_webhook_timeout_ms() -> u64 {
1403    5000
1404}
1405
1406/// Worker health alerting configuration.
1407#[derive(Debug, Clone, Serialize, Deserialize)]
1408pub struct AlertsConfig {
1409    /// Enable or disable alert generation in the daemon.
1410    #[serde(default = "default_true")]
1411    pub enabled: bool,
1412    /// Suppress duplicate alerts for this many seconds.
1413    #[serde(default = "default_alert_suppress_duplicates_secs")]
1414    pub suppress_duplicates_secs: u64,
1415    /// How long to keep a cleared alert visible (with state
1416    /// `cleared_pending_clean`) before removing it entirely. Prevents
1417    /// transient, self-healing circuit-breaker warnings from lingering on
1418    /// `rch status` long after the fleet returned to healthy (bd-3ogaz).
1419    #[serde(default = "default_alert_cleared_retention_secs")]
1420    pub cleared_retention_secs: u64,
1421    /// Webhook configuration for external notifications.
1422    #[serde(default)]
1423    pub webhook: Option<WebhookConfig>,
1424}
1425
1426/// Webhook configuration for alert dispatch.
1427#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1428pub struct WebhookConfig {
1429    /// Webhook endpoint URL (e.g., Slack/Discord webhook URL).
1430    pub url: Option<String>,
1431    /// Secret for HMAC-SHA256 signing (optional).
1432    #[serde(default)]
1433    pub secret: Option<String>,
1434    /// Timeout in seconds for webhook requests.
1435    #[serde(default = "default_webhook_timeout_secs")]
1436    pub timeout_secs: u64,
1437    /// Number of retries on failure.
1438    #[serde(default = "default_webhook_retry_count")]
1439    pub retry_count: u32,
1440    /// Events to send (empty = all events).
1441    /// Supported: worker_offline, worker_degraded, circuit_open, all_workers_offline
1442    #[serde(default)]
1443    pub events: Vec<String>,
1444}
1445
1446fn default_webhook_timeout_secs() -> u64 {
1447    5
1448}
1449
1450fn default_webhook_retry_count() -> u32 {
1451    3
1452}
1453
1454fn default_alert_suppress_duplicates_secs() -> u64 {
1455    300
1456}
1457
1458fn default_alert_cleared_retention_secs() -> u64 {
1459    300
1460}
1461
1462impl Default for AlertsConfig {
1463    fn default() -> Self {
1464        Self {
1465            enabled: true,
1466            suppress_duplicates_secs: default_alert_suppress_duplicates_secs(),
1467            cleared_retention_secs: default_alert_cleared_retention_secs(),
1468            webhook: None,
1469        }
1470    }
1471}
1472
1473/// Fleet operation configuration.
1474///
1475/// Controls timeouts, thresholds, and behavior for fleet management commands
1476/// like `rch fleet preflight`, `rch fleet status`, and `rch fleet deploy`.
1477#[derive(Debug, Clone, Serialize, Deserialize)]
1478pub struct FleetConfig {
1479    /// SSH connection timeout in seconds.
1480    #[serde(default = "default_fleet_ssh_connect_timeout_secs")]
1481    pub ssh_connect_timeout_secs: u64,
1482    /// SSH command execution timeout in seconds.
1483    #[serde(default = "default_fleet_ssh_command_timeout_secs")]
1484    pub ssh_command_timeout_secs: u64,
1485    /// Minimum required disk space in MB for worker health.
1486    #[serde(default = "default_fleet_min_disk_space_mb")]
1487    pub min_disk_space_mb: u64,
1488    /// Maximum acceptable load average for worker health.
1489    #[serde(default = "default_fleet_max_load_average")]
1490    pub max_load_average: f64,
1491    /// Maximum number of concurrent SSH connections.
1492    #[serde(default = "default_fleet_max_concurrent_workers")]
1493    pub max_concurrent_workers: usize,
1494    /// Number of retry attempts for transient failures.
1495    #[serde(default = "default_fleet_retry_count")]
1496    pub retry_count: u32,
1497    /// Delay between retries in milliseconds.
1498    #[serde(default = "default_fleet_retry_delay_ms")]
1499    pub retry_delay_ms: u64,
1500}
1501
1502fn default_fleet_ssh_connect_timeout_secs() -> u64 {
1503    10
1504}
1505
1506fn default_fleet_ssh_command_timeout_secs() -> u64 {
1507    30
1508}
1509
1510fn default_fleet_min_disk_space_mb() -> u64 {
1511    500
1512}
1513
1514fn default_fleet_max_load_average() -> f64 {
1515    10.0
1516}
1517
1518fn default_fleet_max_concurrent_workers() -> usize {
1519    10
1520}
1521
1522fn default_fleet_retry_count() -> u32 {
1523    2
1524}
1525
1526fn default_fleet_retry_delay_ms() -> u64 {
1527    1000
1528}
1529
1530impl Default for FleetConfig {
1531    fn default() -> Self {
1532        Self {
1533            ssh_connect_timeout_secs: default_fleet_ssh_connect_timeout_secs(),
1534            ssh_command_timeout_secs: default_fleet_ssh_command_timeout_secs(),
1535            min_disk_space_mb: default_fleet_min_disk_space_mb(),
1536            max_load_average: default_fleet_max_load_average(),
1537            max_concurrent_workers: default_fleet_max_concurrent_workers(),
1538            retry_count: default_fleet_retry_count(),
1539            retry_delay_ms: default_fleet_retry_delay_ms(),
1540        }
1541    }
1542}
1543
1544impl FleetConfig {
1545    /// Get SSH connect timeout as Duration.
1546    pub fn ssh_connect_timeout(&self) -> std::time::Duration {
1547        std::time::Duration::from_secs(self.ssh_connect_timeout_secs)
1548    }
1549
1550    /// Get SSH command timeout as Duration.
1551    pub fn ssh_command_timeout(&self) -> std::time::Duration {
1552        std::time::Duration::from_secs(self.ssh_command_timeout_secs)
1553    }
1554
1555    /// Get retry delay as Duration.
1556    pub fn retry_delay(&self) -> std::time::Duration {
1557        std::time::Duration::from_millis(self.retry_delay_ms)
1558    }
1559}
1560
1561/// Box-level role (bd-wywsj): what this MACHINE is for, so "never
1562/// build locally" is configuration rather than a per-call env var.
1563#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
1564#[serde(rename_all = "lowercase")]
1565pub enum BoxRole {
1566    /// Orchestrator box: offloadable builds default FAIL-CLOSED +
1567    /// queue (RCH_REQUIRE_REMOTE semantics) — a saturated fleet waits
1568    /// for a worker instead of repopulating local rustc. Explicit
1569    /// per-call env still overrides.
1570    Dispatcher,
1571    /// The box IS the compute: fail-open/local; `rch shim install`
1572    /// refuses here.
1573    Worker,
1574    /// Today's behavior (fully backward compatible).
1575    #[default]
1576    Hybrid,
1577}
1578
1579impl BoxRole {
1580    /// Stable lowercase token.
1581    #[must_use]
1582    pub fn as_str(self) -> &'static str {
1583        match self {
1584            BoxRole::Dispatcher => "dispatcher",
1585            BoxRole::Worker => "worker",
1586            BoxRole::Hybrid => "hybrid",
1587        }
1588    }
1589}
1590
1591#[derive(Debug, Clone, Serialize, Deserialize)]
1592pub struct GeneralConfig {
1593    /// Whether RCH is enabled.
1594    #[serde(default = "default_true")]
1595    pub enabled: bool,
1596    /// Box-level role (bd-wywsj): dispatcher boxes default fail-closed
1597    /// + queue for offloadable builds; worker boxes are the compute.
1598    #[serde(default)]
1599    pub role: BoxRole,
1600    /// Force local execution for this project (never offload), even if the command is classified.
1601    ///
1602    /// Intended for sensitive projects or cases where local state must be authoritative.
1603    #[serde(default)]
1604    pub force_local: bool,
1605    /// Force remote execution for this project (always attempt offload) when safe.
1606    ///
1607    /// This bypasses heuristic gating (e.g. confidence threshold overrides) but still respects
1608    /// structural safety checks and NEVER_INTERCEPT patterns.
1609    #[serde(default)]
1610    pub force_remote: bool,
1611    /// Log level (trace, debug, info, warn, error).
1612    #[serde(default = "default_log_level")]
1613    pub log_level: String,
1614    /// Path to Unix socket for daemon communication.
1615    #[serde(default = "default_socket_path")]
1616    pub socket_path: String,
1617}
1618
1619/// Environment variable passthrough configuration.
1620#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1621pub struct EnvironmentConfig {
1622    /// Allowlist of environment variables to forward to remote workers.
1623    #[serde(default)]
1624    pub allowlist: Vec<String>,
1625}
1626
1627impl Default for GeneralConfig {
1628    fn default() -> Self {
1629        Self {
1630            enabled: true,
1631            role: BoxRole::default(),
1632            force_local: false,
1633            force_remote: false,
1634            log_level: "info".to_string(),
1635            socket_path: default_socket_path(),
1636        }
1637    }
1638}
1639
1640/// Visibility level for hook output.
1641#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
1642#[serde(rename_all = "lowercase")]
1643pub enum OutputVisibility {
1644    #[default]
1645    None,
1646    Summary,
1647    Verbose,
1648}
1649
1650impl std::fmt::Display for OutputVisibility {
1651    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1652        let value = match self {
1653            OutputVisibility::None => "none",
1654            OutputVisibility::Summary => "summary",
1655            OutputVisibility::Verbose => "verbose",
1656        };
1657        write!(f, "{}", value)
1658    }
1659}
1660
1661impl std::str::FromStr for OutputVisibility {
1662    type Err = ();
1663
1664    fn from_str(s: &str) -> Result<Self, Self::Err> {
1665        match s.trim().to_lowercase().as_str() {
1666            "none" | "silent" | "quiet" => Ok(Self::None),
1667            "summary" | "short" => Ok(Self::Summary),
1668            "verbose" | "debug" => Ok(Self::Verbose),
1669            _ => Err(()),
1670        }
1671    }
1672}
1673
1674/// Color mode for remote command output.
1675///
1676/// Controls whether ANSI color codes are preserved when streaming
1677/// output from remote compilation commands.
1678#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1679#[serde(rename_all = "lowercase")]
1680pub enum ColorMode {
1681    /// Force color output regardless of terminal detection.
1682    /// Sets CARGO_TERM_COLOR=always and similar environment variables.
1683    #[default]
1684    Always,
1685    /// Let the remote command detect terminal capabilities (may lose colors).
1686    Auto,
1687    /// Disable color output entirely.
1688    Never,
1689}
1690
1691impl std::fmt::Display for ColorMode {
1692    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1693        let value = match self {
1694            ColorMode::Always => "always",
1695            ColorMode::Auto => "auto",
1696            ColorMode::Never => "never",
1697        };
1698        write!(f, "{}", value)
1699    }
1700}
1701
1702impl std::str::FromStr for ColorMode {
1703    type Err = ();
1704
1705    fn from_str(s: &str) -> Result<Self, Self::Err> {
1706        match s.trim().to_lowercase().as_str() {
1707            "always" | "force" | "yes" | "true" => Ok(Self::Always),
1708            "auto" | "detect" => Ok(Self::Auto),
1709            "never" | "none" | "no" | "false" => Ok(Self::Never),
1710            _ => Err(()),
1711        }
1712    }
1713}
1714
1715fn default_color_mode() -> ColorMode {
1716    ColorMode::default()
1717}
1718
1719/// Hook output configuration.
1720#[derive(Debug, Clone, Serialize, Deserialize)]
1721pub struct OutputConfig {
1722    /// Output visibility level for hook messages.
1723    #[serde(default = "default_output_visibility")]
1724    pub visibility: OutputVisibility,
1725    /// Whether the first-run success message has been shown.
1726    #[serde(default)]
1727    pub first_run_complete: bool,
1728    /// Color mode for remote command output.
1729    /// Controls whether ANSI color codes are preserved in remote output.
1730    #[serde(default = "default_color_mode")]
1731    pub color_mode: ColorMode,
1732}
1733
1734impl Default for OutputConfig {
1735    fn default() -> Self {
1736        Self {
1737            visibility: OutputVisibility::None,
1738            first_run_complete: false,
1739            color_mode: ColorMode::default(),
1740        }
1741    }
1742}
1743
1744// ============================================================================
1745// Self-Healing Configuration
1746// ============================================================================
1747
1748fn default_autostart_cooldown_secs() -> u64 {
1749    30
1750}
1751
1752fn default_autostart_timeout_secs() -> u64 {
1753    3
1754}
1755
1756/// Log level for self-healing diagnostic output.
1757///
1758/// Used to filter `tracing` events emitted by the self-healing subsystem
1759/// (hook auto-start, daemon hook-install). Read from
1760/// `RCH_SELF_HEALING_LOG_LEVEL=debug|info|warn|error` (case-insensitive).
1761#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
1762#[serde(rename_all = "lowercase")]
1763pub enum SelfHealingLogLevel {
1764    Debug,
1765    #[default]
1766    Info,
1767    Warn,
1768    Error,
1769}
1770
1771impl SelfHealingLogLevel {
1772    /// Parse a human-friendly level name. Returns `None` for unknown input.
1773    pub fn from_env_str(s: &str) -> Option<Self> {
1774        match s.trim().to_ascii_lowercase().as_str() {
1775            "debug" => Some(Self::Debug),
1776            "info" => Some(Self::Info),
1777            "warn" | "warning" => Some(Self::Warn),
1778            "error" => Some(Self::Error),
1779            _ => None,
1780        }
1781    }
1782}
1783
1784/// Self-healing behaviors for the hook and daemon.
1785#[derive(Debug, Clone, Serialize, Deserialize)]
1786pub struct SelfHealingConfig {
1787    /// Whether the hook should attempt to auto-start the daemon.
1788    #[serde(default = "default_true")]
1789    pub hook_starts_daemon: bool,
1790    /// Whether the daemon should auto-install Claude Code hooks on startup.
1791    #[serde(default = "default_true")]
1792    pub daemon_installs_hooks: bool,
1793    /// Cooldown between auto-start attempts in seconds.
1794    #[serde(default = "default_autostart_cooldown_secs")]
1795    pub auto_start_cooldown_secs: u64,
1796    /// Time to wait for the daemon socket after spawning in seconds.
1797    /// Also aliased as `daemon_start_timeout` for backwards compatibility.
1798    #[serde(
1799        default = "default_autostart_timeout_secs",
1800        alias = "daemon_start_timeout"
1801    )]
1802    pub auto_start_timeout_secs: u64,
1803    /// Log level for self-healing diagnostic events. Default: info.
1804    /// Override via `RCH_SELF_HEALING_LOG_LEVEL=debug|info|warn|error`.
1805    #[serde(default)]
1806    pub self_healing_log_level: SelfHealingLogLevel,
1807}
1808
1809impl Default for SelfHealingConfig {
1810    fn default() -> Self {
1811        Self {
1812            hook_starts_daemon: default_true(),
1813            daemon_installs_hooks: default_true(),
1814            auto_start_cooldown_secs: default_autostart_cooldown_secs(),
1815            auto_start_timeout_secs: default_autostart_timeout_secs(),
1816            self_healing_log_level: SelfHealingLogLevel::default(),
1817        }
1818    }
1819}
1820
1821impl SelfHealingConfig {
1822    /// Apply environment variable overrides to this config.
1823    ///
1824    /// Supported environment variables:
1825    /// - `RCH_NO_SELF_HEALING=1` - Master switch to disable all self-healing
1826    /// - `RCH_HOOK_STARTS_DAEMON=0|1` - Control hook auto-starting daemon
1827    /// - `RCH_DAEMON_INSTALLS_HOOKS=0|1` - Control daemon auto-installing hooks
1828    /// - `RCH_AUTO_START_TIMEOUT_SECS=<seconds>` - Max wait for daemon start
1829    /// - `RCH_AUTO_START_COOLDOWN_SECS=<seconds>` - Min time between auto-starts
1830    pub fn with_env_overrides(mut self) -> Self {
1831        // Master disable switch
1832        if let Ok(val) = std::env::var("RCH_NO_SELF_HEALING")
1833            && (val == "1" || val.eq_ignore_ascii_case("true"))
1834        {
1835            self.hook_starts_daemon = false;
1836            self.daemon_installs_hooks = false;
1837            return self;
1838        }
1839
1840        // Individual toggles
1841        if let Ok(val) = std::env::var("RCH_HOOK_STARTS_DAEMON") {
1842            self.hook_starts_daemon = val != "0" && !val.eq_ignore_ascii_case("false");
1843        }
1844        if let Ok(val) = std::env::var("RCH_DAEMON_INSTALLS_HOOKS") {
1845            self.daemon_installs_hooks = val != "0" && !val.eq_ignore_ascii_case("false");
1846        }
1847
1848        // Numeric settings
1849        if let Ok(val) = std::env::var("RCH_AUTO_START_TIMEOUT_SECS")
1850            && let Ok(secs) = val.parse()
1851        {
1852            self.auto_start_timeout_secs = secs;
1853        }
1854        if let Ok(val) = std::env::var("RCH_AUTO_START_COOLDOWN_SECS")
1855            && let Ok(secs) = val.parse()
1856        {
1857            self.auto_start_cooldown_secs = secs;
1858        }
1859        // br-4zf3p: log-level override.
1860        if let Ok(val) = std::env::var("RCH_SELF_HEALING_LOG_LEVEL") {
1861            if let Some(level) = SelfHealingLogLevel::from_env_str(&val) {
1862                self.self_healing_log_level = level;
1863            } else {
1864                tracing::warn!(
1865                    target: "rch::self_healing",
1866                    invalid = %val,
1867                    "RCH_SELF_HEALING_LOG_LEVEL has invalid value; using default=info"
1868                );
1869            }
1870        }
1871
1872        self
1873    }
1874}
1875
1876// ============================================================================
1877// Self-Test Configuration
1878// ============================================================================
1879
1880/// Action to take when a scheduled self-test fails.
1881#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1882#[serde(rename_all = "snake_case")]
1883pub enum SelfTestFailureAction {
1884    /// Only emit an alert/log entry.
1885    #[default]
1886    Alert,
1887    /// Disable the failing worker automatically.
1888    DisableWorker,
1889    /// Alert and disable the worker.
1890    AlertAndDisable,
1891}
1892
1893/// Which workers to include in self-test runs.
1894#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1895#[serde(untagged)]
1896pub enum SelfTestWorkers {
1897    /// Special value "all" (or a single worker id string).
1898    All(String),
1899    /// Explicit worker list.
1900    List(Vec<WorkerId>),
1901}
1902
1903impl Default for SelfTestWorkers {
1904    fn default() -> Self {
1905        Self::All("all".to_string())
1906    }
1907}
1908
1909impl SelfTestWorkers {
1910    /// Resolve to an explicit list if configured, or None to indicate "all".
1911    pub fn resolve(&self) -> Option<Vec<WorkerId>> {
1912        match self {
1913            SelfTestWorkers::All(value) => {
1914                if value.eq_ignore_ascii_case("all") {
1915                    None
1916                } else {
1917                    Some(vec![WorkerId::new(value.clone())])
1918                }
1919            }
1920            SelfTestWorkers::List(list) => Some(list.clone()),
1921        }
1922    }
1923}
1924
1925fn default_self_test_retry_count() -> u32 {
1926    3
1927}
1928
1929fn default_self_test_retry_delay() -> String {
1930    "5m".to_string()
1931}
1932
1933fn default_self_test_enabled() -> bool {
1934    false
1935}
1936
1937/// Self-test scheduling and behavior configuration.
1938#[derive(Debug, Clone, Serialize, Deserialize)]
1939pub struct SelfTestConfig {
1940    /// Enable scheduled self-tests.
1941    #[serde(default = "default_self_test_enabled")]
1942    pub enabled: bool,
1943    /// Cron schedule (e.g., "0 3 * * *").
1944    #[serde(default)]
1945    pub schedule: Option<String>,
1946    /// Interval duration (e.g., "24h").
1947    #[serde(default)]
1948    pub interval: Option<String>,
1949    /// Which workers to test (default: "all").
1950    #[serde(default)]
1951    pub workers: SelfTestWorkers,
1952    /// Action on failure.
1953    #[serde(default)]
1954    pub on_failure: SelfTestFailureAction,
1955    /// Retry count for failed tests.
1956    #[serde(default = "default_self_test_retry_count")]
1957    pub retry_count: u32,
1958    /// Delay between retries.
1959    #[serde(default = "default_self_test_retry_delay")]
1960    pub retry_delay: String,
1961}
1962
1963impl Default for SelfTestConfig {
1964    fn default() -> Self {
1965        Self {
1966            enabled: default_self_test_enabled(),
1967            schedule: None,
1968            interval: None,
1969            workers: SelfTestWorkers::default(),
1970            on_failure: SelfTestFailureAction::default(),
1971            retry_count: default_self_test_retry_count(),
1972            retry_delay: default_self_test_retry_delay(),
1973        }
1974    }
1975}
1976
1977/// Circuit breaker configuration.
1978#[derive(Debug, Clone, Serialize, Deserialize)]
1979pub struct CircuitBreakerConfig {
1980    /// Consecutive failures to open circuit.
1981    #[serde(default = "default_circuit_failure_threshold")]
1982    pub failure_threshold: u32,
1983    /// Consecutive successes to close circuit.
1984    #[serde(default = "default_circuit_success_threshold")]
1985    pub success_threshold: u32,
1986    /// Error rate threshold (0.0-1.0) to open circuit.
1987    #[serde(default = "default_circuit_error_rate_threshold")]
1988    pub error_rate_threshold: f64,
1989    /// Rolling window size in seconds.
1990    #[serde(default = "default_circuit_window_secs")]
1991    pub window_secs: u64,
1992    /// Cooldown duration before half-open (seconds).
1993    #[serde(default = "default_circuit_open_cooldown_secs")]
1994    pub open_cooldown_secs: u64,
1995    /// Max concurrent probes in half-open.
1996    #[serde(default = "default_circuit_half_open_max_probes")]
1997    pub half_open_max_probes: u32,
1998    /// Consecutive *command-class* failures to open the circuit.
1999    ///
2000    /// Command-class evidence is an operation of the same shape as dispatched
2001    /// work — a remote binary that must run to completion (the telemetry poll,
2002    /// or a health probe whose session connected but whose command never
2003    /// finished). It is tracked separately from the plain liveness probe
2004    /// because a host under fork exhaustion or swap thrash keeps answering
2005    /// `echo` over a warm connection while every real command times out;
2006    /// interleaved liveness successes would otherwise keep resetting
2007    /// `failure_threshold` and the breaker would never trip.
2008    #[serde(default = "default_circuit_command_failure_threshold")]
2009    pub command_failure_threshold: u32,
2010    /// Upper bound (seconds) for the open cooldown after repeated reopens.
2011    ///
2012    /// Each Open -> HalfOpen -> Open cycle without an intervening Closed
2013    /// doubles the cooldown (`open_cooldown_secs`, x2, x4, ...) up to this
2014    /// cap, so a flapping worker converges to Open instead of being
2015    /// re-admitted every cooldown. Set equal to `open_cooldown_secs` to
2016    /// disable the backoff.
2017    #[serde(default = "default_circuit_open_cooldown_max_secs")]
2018    pub open_cooldown_max_secs: u64,
2019}
2020
2021impl Default for CircuitBreakerConfig {
2022    fn default() -> Self {
2023        Self {
2024            failure_threshold: default_circuit_failure_threshold(),
2025            success_threshold: default_circuit_success_threshold(),
2026            error_rate_threshold: default_circuit_error_rate_threshold(),
2027            window_secs: default_circuit_window_secs(),
2028            open_cooldown_secs: default_circuit_open_cooldown_secs(),
2029            half_open_max_probes: default_circuit_half_open_max_probes(),
2030            command_failure_threshold: default_circuit_command_failure_threshold(),
2031            open_cooldown_max_secs: default_circuit_open_cooldown_max_secs(),
2032        }
2033    }
2034}
2035
2036impl CircuitBreakerConfig {
2037    /// Effective cooldown (ms) for the given reopen streak, with exponential
2038    /// backoff capped at `open_cooldown_max_secs`. `reopen_streak` counts
2039    /// Open transitions since the circuit last closed (1 = first open).
2040    pub fn cooldown_ms_for_streak(&self, reopen_streak: u32) -> u64 {
2041        let base_ms = self.open_cooldown_secs.saturating_mul(1000);
2042        let cap_ms = self
2043            .open_cooldown_max_secs
2044            .saturating_mul(1000)
2045            .max(base_ms);
2046        // Exponent bounded so the shift cannot overflow; the cap clamps anyway.
2047        let exponent = reopen_streak.saturating_sub(1).min(16);
2048        base_ms.saturating_mul(1u64 << exponent).min(cap_ms)
2049    }
2050}
2051
2052/// Circuit breaker statistics for tracking state transitions.
2053///
2054/// Maintains a rolling window of successes and failures to determine
2055/// when the circuit should open, close, or enter half-open state.
2056#[derive(Debug, Clone, Default)]
2057pub struct CircuitStats {
2058    /// Current circuit state.
2059    state: CircuitState,
2060    /// Consecutive failure count (reset on success).
2061    consecutive_failures: u32,
2062    /// Consecutive success count in half-open state (reset on failure).
2063    consecutive_successes: u32,
2064    /// Total successes in the current rolling window.
2065    window_successes: u32,
2066    /// Total failures in the current rolling window.
2067    window_failures: u32,
2068    /// Timestamp when circuit entered Open state (epoch millis).
2069    opened_at: Option<u64>,
2070    /// Timestamp of last state change (epoch millis).
2071    last_state_change: u64,
2072    /// Current active probes in half-open state.
2073    active_probes: u32,
2074    /// Consecutive command-class failures (telemetry poll exhaustion, probe
2075    /// sessions that connected but never completed). Reset only by a
2076    /// command-class success or a circuit close — NOT by a liveness success,
2077    /// which is the whole point (see `CircuitBreakerConfig::command_failure_threshold`).
2078    consecutive_command_failures: u32,
2079    /// Number of Open transitions since the circuit last closed. Drives the
2080    /// cooldown backoff; 0 while Closed-since-recovery.
2081    reopen_streak: u32,
2082    /// Recent health check results (true=success, false=failure).
2083    /// Used for history visualization in status display.
2084    recent_results: Vec<bool>,
2085}
2086
2087/// Maximum number of recent results to keep for history visualization.
2088const CIRCUIT_HISTORY_SIZE: usize = 10;
2089
2090impl CircuitStats {
2091    /// Create new circuit stats in the closed state.
2092    pub fn new() -> Self {
2093        Self {
2094            state: CircuitState::Closed,
2095            consecutive_failures: 0,
2096            consecutive_successes: 0,
2097            window_successes: 0,
2098            window_failures: 0,
2099            opened_at: None,
2100            last_state_change: Self::now_millis(),
2101            active_probes: 0,
2102            consecutive_command_failures: 0,
2103            reopen_streak: 0,
2104            recent_results: Vec::with_capacity(CIRCUIT_HISTORY_SIZE),
2105        }
2106    }
2107
2108    /// Consecutive command-class failures observed since the last
2109    /// command-class success (or circuit close).
2110    pub fn consecutive_command_failures(&self) -> u32 {
2111        self.consecutive_command_failures
2112    }
2113
2114    /// Open transitions since the circuit last closed (0 when it has not
2115    /// opened since recovery). Exposed for diagnostics.
2116    pub fn reopen_streak(&self) -> u32 {
2117        self.reopen_streak
2118    }
2119
2120    /// Cooldown currently in force (ms), including reopen backoff.
2121    fn effective_cooldown_ms(&self, config: &CircuitBreakerConfig) -> u64 {
2122        config.cooldown_ms_for_streak(self.reopen_streak.max(1))
2123    }
2124
2125    /// Get the current circuit state.
2126    pub fn state(&self) -> CircuitState {
2127        self.state
2128    }
2129
2130    /// Get the timestamp when the circuit opened (if open).
2131    pub fn opened_at(&self) -> Option<u64> {
2132        self.opened_at
2133    }
2134
2135    /// Get the timestamp of the last state change.
2136    pub fn last_state_change(&self) -> u64 {
2137        self.last_state_change
2138    }
2139
2140    /// Get the consecutive failure count.
2141    pub fn consecutive_failures(&self) -> u32 {
2142        self.consecutive_failures
2143    }
2144
2145    /// Get the error rate in the current window (0.0-1.0).
2146    ///
2147    /// Computed over the bounded recent-results window (the last
2148    /// `CIRCUIT_HISTORY_SIZE` health checks) rather than the unbounded lifetime
2149    /// `window_successes`/`window_failures` counters. A long-lived daemon polling
2150    /// distant workers accumulates occasional transient failures over days; with a
2151    /// lifetime ratio a currently-healthy worker could stay gated (circuit kept
2152    /// open, depressed health score) until the daemon was restarted. A recency
2153    /// window lets the rate self-heal as fresh successes age out old failures.
2154    pub fn error_rate(&self) -> f64 {
2155        if self.recent_results.is_empty() {
2156            return 0.0;
2157        }
2158        let failures = self.recent_results.iter().filter(|&&ok| !ok).count();
2159        failures as f64 / self.recent_results.len() as f64
2160    }
2161
2162    /// Record a successful operation.
2163    ///
2164    /// In closed state: resets consecutive failures, increments window successes.
2165    /// In half-open state: increments consecutive successes.
2166    pub fn record_success(&mut self) {
2167        self.window_successes += 1;
2168        self.consecutive_failures = 0;
2169        self.push_result(true);
2170
2171        if self.state == CircuitState::HalfOpen {
2172            self.consecutive_successes += 1;
2173            if self.active_probes > 0 {
2174                self.active_probes -= 1;
2175            }
2176        }
2177    }
2178
2179    /// Record a failed operation.
2180    ///
2181    /// Increments consecutive failures and window failures.
2182    /// In half-open state, resets consecutive successes.
2183    pub fn record_failure(&mut self) {
2184        self.window_failures += 1;
2185        self.consecutive_failures += 1;
2186        self.push_result(false);
2187
2188        if self.state == CircuitState::HalfOpen {
2189            self.consecutive_successes = 0;
2190            if self.active_probes > 0 {
2191                self.active_probes -= 1;
2192            }
2193        }
2194    }
2195
2196    /// Check if the circuit should open based on config thresholds.
2197    ///
2198    /// Returns true if:
2199    /// - Consecutive failures exceed threshold, OR
2200    /// - Error rate exceeds threshold (with minimum sample size)
2201    pub fn should_open(&self, config: &CircuitBreakerConfig) -> bool {
2202        if self.state != CircuitState::Closed {
2203            return false;
2204        }
2205
2206        // Open if consecutive failures exceed threshold
2207        if self.consecutive_failures >= config.failure_threshold {
2208            return true;
2209        }
2210
2211        // Open if error rate exceeds threshold (with a minimum sample size).
2212        // Counts samples in the bounded recent-results window so the gate
2213        // reflects recent behavior, matching error_rate() above.
2214        if self.recent_results.len() >= 5 && self.error_rate() >= config.error_rate_threshold {
2215            return true;
2216        }
2217
2218        false
2219    }
2220
2221    /// Check if the circuit should transition to half-open.
2222    ///
2223    /// Returns true if the circuit is open and the cooldown period has elapsed.
2224    pub fn should_half_open(&self, config: &CircuitBreakerConfig) -> bool {
2225        if self.state != CircuitState::Open {
2226            return false;
2227        }
2228
2229        let now = Self::now_millis();
2230        let cooldown_ms = self.effective_cooldown_ms(config);
2231
2232        if let Some(opened_at) = self.opened_at {
2233            now.saturating_sub(opened_at) >= cooldown_ms
2234        } else {
2235            false
2236        }
2237    }
2238
2239    /// Check if the circuit should close.
2240    ///
2241    /// Returns true if in half-open state, consecutive successes meet or
2242    /// exceed the success threshold, AND no command-class failure is
2243    /// outstanding. A worker whose last real command timed out is not
2244    /// recovered no matter how many liveness probes it answers; it stays
2245    /// HalfOpen (degraded, probe-limited) until a command-class success
2246    /// clears the evidence.
2247    pub fn should_close(&self, config: &CircuitBreakerConfig) -> bool {
2248        if self.state != CircuitState::HalfOpen {
2249            return false;
2250        }
2251
2252        self.consecutive_successes >= config.success_threshold
2253            && self.consecutive_command_failures == 0
2254    }
2255
2256    /// Check if a probe request can be made in half-open state.
2257    ///
2258    /// Returns true if active probes are below the maximum allowed.
2259    pub fn can_probe(&self, config: &CircuitBreakerConfig) -> bool {
2260        if self.state != CircuitState::HalfOpen {
2261            return false;
2262        }
2263
2264        self.active_probes < config.half_open_max_probes
2265    }
2266
2267    /// Start a probe request in half-open state.
2268    ///
2269    /// Returns true if the probe was started, false if already at max probes.
2270    pub fn start_probe(&mut self, config: &CircuitBreakerConfig) -> bool {
2271        if !self.can_probe(config) {
2272            return false;
2273        }
2274        self.active_probes += 1;
2275        true
2276    }
2277
2278    /// Transition the circuit to open state.
2279    pub fn open(&mut self) {
2280        if self.state != CircuitState::Open {
2281            self.state = CircuitState::Open;
2282            self.opened_at = Some(Self::now_millis());
2283            self.last_state_change = Self::now_millis();
2284            self.consecutive_successes = 0;
2285            self.active_probes = 0;
2286            // Every open without an intervening close lengthens the next
2287            // cooldown (see `CircuitBreakerConfig::cooldown_ms_for_streak`).
2288            self.reopen_streak = self.reopen_streak.saturating_add(1);
2289        }
2290    }
2291
2292    /// Transition the circuit to half-open state.
2293    pub fn half_open(&mut self) {
2294        if self.state != CircuitState::HalfOpen {
2295            self.state = CircuitState::HalfOpen;
2296            self.last_state_change = Self::now_millis();
2297            self.consecutive_successes = 0;
2298            self.active_probes = 0;
2299        }
2300    }
2301
2302    /// Transition the circuit to closed state.
2303    pub fn close(&mut self) {
2304        if self.state != CircuitState::Closed {
2305            self.state = CircuitState::Closed;
2306            self.last_state_change = Self::now_millis();
2307            self.opened_at = None;
2308            self.consecutive_failures = 0;
2309            self.consecutive_successes = 0;
2310            self.active_probes = 0;
2311            self.consecutive_command_failures = 0;
2312            self.reopen_streak = 0;
2313            // Reset the window on close, INCLUDING the recent-results history
2314            // that error_rate() now reads. Otherwise a freshly-recovered circuit
2315            // would still compute a high error rate from the pre-failure samples
2316            // and immediately re-open (flapping). Matches the original intent of
2317            // resetting the error-rate inputs when the circuit recovers.
2318            self.window_successes = 0;
2319            self.window_failures = 0;
2320            self.recent_results.clear();
2321        }
2322    }
2323
2324    /// Reset the rolling window counters.
2325    ///
2326    /// Called periodically to ensure the window reflects recent activity.
2327    /// Also clears the recent-results history that `error_rate()` is computed
2328    /// from, so a reset truly zeroes the observed error rate.
2329    pub fn reset_window(&mut self) {
2330        self.window_successes = 0;
2331        self.window_failures = 0;
2332        self.recent_results.clear();
2333    }
2334
2335    /// Get recent health check results for history visualization.
2336    ///
2337    /// Returns a slice of recent results (true=success, false=failure),
2338    /// with the most recent result at the end.
2339    pub fn recent_results(&self) -> &[bool] {
2340        &self.recent_results
2341    }
2342
2343    /// Calculate seconds remaining until circuit auto-transitions to half-open.
2344    ///
2345    /// Returns None if circuit is not open or cooldown has already elapsed.
2346    pub fn recovery_remaining_secs(&self, config: &CircuitBreakerConfig) -> Option<u64> {
2347        if self.state != CircuitState::Open {
2348            return None;
2349        }
2350
2351        let now = Self::now_millis();
2352        let cooldown_ms = self.effective_cooldown_ms(config);
2353
2354        if let Some(opened_at) = self.opened_at {
2355            let elapsed_ms = now.saturating_sub(opened_at);
2356            if elapsed_ms < cooldown_ms {
2357                return Some((cooldown_ms - elapsed_ms) / 1000);
2358            }
2359        }
2360        None
2361    }
2362
2363    /// Push a result to the history, maintaining the maximum size.
2364    fn push_result(&mut self, success: bool) {
2365        if self.recent_results.len() >= CIRCUIT_HISTORY_SIZE {
2366            self.recent_results.remove(0);
2367        }
2368        self.recent_results.push(success);
2369    }
2370
2371    fn now_millis() -> u64 {
2372        std::time::SystemTime::now()
2373            .duration_since(std::time::UNIX_EPOCH)
2374            .map(|d| d.as_millis() as u64)
2375            .unwrap_or(0)
2376    }
2377
2378    /// Drive the circuit through a single health-check outcome and return the
2379    /// resulting [`CircuitState`].
2380    ///
2381    /// This is the single, self-contained transition engine shared by every
2382    /// circuit consumer (the daemon health monitor via
2383    /// `WorkerState::record_health_check`, and the diagnostic `WorkerHealth`
2384    /// panel via `WorkerHealth::observe`). Keeping the transition logic in ONE
2385    /// place is what makes `WorkerState.circuit` (which selection reads) and the
2386    /// diagnostic `WorkerHealth.circuit` agree.
2387    ///
2388    /// Behaviour:
2389    /// - On a **healthy** outcome: if the circuit is Open and its cooldown has
2390    ///   elapsed it first transitions Open → HalfOpen *before* crediting the
2391    ///   success, so recovery needs exactly `success_threshold` clean probes
2392    ///   (crediting a success against an Open circuit would be lost). Then the
2393    ///   success is recorded and the circuit closes once the threshold is met.
2394    /// - On an **unhealthy** outcome: the failure is recorded, the circuit opens
2395    ///   if the closed-state open condition is met, and a HalfOpen probe failure
2396    ///   reopens the circuit — UNLESS the failure was `transient` (a retryable
2397    ///   transport blip that survived the in-check retries). A transient
2398    ///   half-open failure neither reopens nor resets the cooldown, so a single
2399    ///   network hiccup during recovery cannot bounce a worker back to Open.
2400    /// - After either branch, an Open circuit past its cooldown is nudged to
2401    ///   HalfOpen so probing resumes on the next poll even for a still-failing
2402    ///   worker.
2403    pub fn apply_health_outcome(
2404        &mut self,
2405        healthy: bool,
2406        transient: bool,
2407        config: &CircuitBreakerConfig,
2408    ) -> CircuitState {
2409        if healthy {
2410            // Transition to half-open BEFORE crediting the success so the probe
2411            // counts toward recovery. record_success() only increments the
2412            // consecutive-success counter while in HalfOpen.
2413            if self.state == CircuitState::Open && self.should_half_open(config) {
2414                self.half_open();
2415            }
2416            self.record_success();
2417            if self.should_close(config) {
2418                self.close();
2419            }
2420        } else {
2421            let was_half_open = self.state == CircuitState::HalfOpen;
2422            self.record_failure();
2423            if self.should_open(config) {
2424                self.open();
2425            } else if was_half_open && !transient {
2426                // A real (non-transient) probe failure in half-open reopens the
2427                // circuit and restarts the cooldown. A transient failure is
2428                // ignored so a network blip can't undo an in-progress recovery.
2429                self.open();
2430            }
2431        }
2432
2433        // Even for a still-failing worker, resume probing once the cooldown
2434        // elapses so a recovered worker is retried promptly.
2435        if self.state == CircuitState::Open && self.should_half_open(config) {
2436            self.half_open();
2437        }
2438
2439        self.state
2440    }
2441
2442    /// Drive the circuit through a single *command-class* outcome and return
2443    /// the resulting [`CircuitState`].
2444    ///
2445    /// Command-class evidence comes from an operation of the same shape as
2446    /// dispatched work: a remote binary that must run to completion (the
2447    /// telemetry poll, or a health probe whose SSH session connected but whose
2448    /// command never finished). It is the signal the plain liveness probe
2449    /// cannot provide — a host under fork exhaustion or swap thrash keeps
2450    /// answering `echo` over a warm connection while every real command hangs,
2451    /// and on that evidence alone the breaker would happily re-admit it.
2452    ///
2453    /// Behaviour:
2454    /// - **Success**: clears the outstanding command-failure count and then
2455    ///   counts as an ordinary healthy outcome (may advance Open -> HalfOpen ->
2456    ///   Closed exactly like a liveness success).
2457    /// - **Failure**: increments the command-failure count and records a plain
2458    ///   failure. The circuit opens when either the ordinary open condition or
2459    ///   `command_failure_threshold` is met. In HalfOpen a command failure
2460    ///   ALWAYS reopens — there is no transient exemption, because "connected
2461    ///   but the command did not complete" is precisely the condition the
2462    ///   half-open trial exists to detect. While any command failure is
2463    ///   outstanding, [`Self::should_close`] refuses to close on liveness
2464    ///   successes alone.
2465    pub fn apply_command_outcome(
2466        &mut self,
2467        success: bool,
2468        config: &CircuitBreakerConfig,
2469    ) -> CircuitState {
2470        if success {
2471            self.consecutive_command_failures = 0;
2472            return self.apply_health_outcome(true, false, config);
2473        }
2474
2475        self.consecutive_command_failures = self.consecutive_command_failures.saturating_add(1);
2476        let was_half_open = self.state == CircuitState::HalfOpen;
2477        self.record_failure();
2478        let threshold = config.command_failure_threshold.max(1);
2479        if was_half_open
2480            || self.should_open(config)
2481            || (self.state == CircuitState::Closed
2482                && self.consecutive_command_failures >= threshold)
2483        {
2484            self.open();
2485        }
2486
2487        if self.state == CircuitState::Open && self.should_half_open(config) {
2488            self.half_open();
2489        }
2490
2491        self.state
2492    }
2493}
2494
2495#[derive(Debug, Clone, Serialize, Deserialize)]
2496pub struct CompilationConfig {
2497    /// Minimum confidence score to intercept (0.0-1.0).
2498    #[serde(default = "default_confidence")]
2499    pub confidence_threshold: f64,
2500    /// Skip interception if estimated local time < this (ms).
2501    #[serde(default = "default_min_local_time")]
2502    pub min_local_time_ms: u64,
2503    /// Minimum expected speedup ratio (local_time / remote_time) to offload.
2504    /// If predicted speedup < threshold, run locally. Default: 1.2 (20% faster).
2505    /// Set to 1.0 to always offload when other criteria are met.
2506    #[serde(default = "default_speedup_threshold")]
2507    pub remote_speedup_threshold: f64,
2508    /// Default slot estimate for build commands (cargo build, gcc, etc.).
2509    #[serde(default = "default_build_slots")]
2510    pub build_slots: u32,
2511    /// Default slot estimate for test commands (cargo test, nextest).
2512    /// Tests typically use more parallelism than builds.
2513    #[serde(default = "default_test_slots")]
2514    pub test_slots: u32,
2515    /// Default slot estimate for check/lint commands (cargo check, clippy).
2516    /// These are typically faster and use fewer resources.
2517    #[serde(default = "default_check_slots")]
2518    pub check_slots: u32,
2519    /// Timeout in seconds for build commands.
2520    /// Build commands (cargo build, gcc, etc.) typically complete faster than tests.
2521    #[serde(default = "default_build_timeout")]
2522    pub build_timeout_sec: u64,
2523    /// Timeout in seconds for test commands.
2524    /// Test commands often need longer timeouts due to test suite execution time.
2525    #[serde(default = "default_test_timeout")]
2526    pub test_timeout_sec: u64,
2527    /// Timeout in seconds for bun commands (bun test, bun typecheck).
2528    /// Bun has known issues where its internal --timeout flag doesn't work for
2529    /// CPU-bound hangs, so we use a shorter external timeout by default.
2530    #[serde(default = "default_bun_timeout")]
2531    pub bun_timeout_sec: u64,
2532    /// Whether to wrap remote commands with external timeout protection.
2533    /// This prevents runaway/stuck processes from consuming worker slots indefinitely.
2534    /// Default: true. Set to false to disable timeout wrapping entirely.
2535    #[serde(default = "default_external_timeout_enabled")]
2536    pub external_timeout_enabled: bool,
2537    /// Whether a remote build FAILURE may fall back to LOCAL execution on the
2538    /// orchestrator once every eligible remote worker has been exhausted.
2539    ///
2540    /// On a remote failure the hook first retries the build on a *different,
2541    /// higher-capacity* worker (see `run_exec`'s worker-fallback loop). Local
2542    /// execution runs the heavy compile on the orchestrator that coordinates the
2543    /// whole fleet, so it is a genuine last resort and can flood that box. This
2544    /// flag gates it. Default `true` preserves the historical fail-open behavior
2545    /// (local only when no worker can serve); set to `false` on an orchestrator
2546    /// to refuse local fallback and fail the build closed instead. `force_remote`
2547    /// / proof mode / `RCH_REQUIRE_REMOTE` still override this by refusing local
2548    /// unconditionally.
2549    #[serde(default = "default_allow_local_fallback")]
2550    pub allow_local_fallback: bool,
2551    /// Per-job rustc parallelism cap applied on the WORKER (issue #49).
2552    ///
2553    /// `total_slots` bounds concurrent *jobs* per worker, but cargo defaults
2554    /// `-j` to the worker's `nproc`, so one offloaded `cargo test --workspace`
2555    /// forks `nproc` rustc regardless of slots and can swap-thrash a
2556    /// RAM-light box. This policy injects `CARGO_BUILD_JOBS` into the remote
2557    /// build environment; see [`RemoteBuildJobs`] for the modes. It never
2558    /// overrides a `CARGO_BUILD_JOBS` the worker session already carries, an
2559    /// inline `CARGO_BUILD_JOBS=N` or `-j N` in the command, or a project
2560    /// `.cargo/config.toml` `[build] jobs` — explicit intent always wins.
2561    #[serde(default)]
2562    pub remote_build_jobs: RemoteBuildJobs,
2563}
2564
2565/// How rch derives the `CARGO_BUILD_JOBS` it injects into a remote build.
2566///
2567/// - `auto` (default): computed on the worker from its live `nproc` and
2568///   total RAM as `clamp(ram_gib / 8, 2, min(nproc, 8))` — budgeting roughly
2569///   8 GiB of RAM per concurrent rustc leaves headroom for two to three
2570///   concurrent jobs at a realistic per-rustc peak of ~3 GiB.
2571/// - `off`: inject nothing (historical behavior; cargo uses `nproc`).
2572/// - `<n>`: inject a fixed job count fleet-wide.
2573///
2574/// In every mode an explicitly set `CARGO_BUILD_JOBS` on the worker (for
2575/// example from `/etc/environment`) or in the command is left untouched.
2576#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
2577pub enum RemoteBuildJobs {
2578    /// Derive from the worker's cores and RAM at execution time.
2579    #[default]
2580    Auto,
2581    /// Do not inject `CARGO_BUILD_JOBS`.
2582    Off,
2583    /// Inject a fixed job count (must be >= 1).
2584    Fixed(u32),
2585}
2586
2587impl RemoteBuildJobs {
2588    /// Parse the config/CLI spelling: `auto`, `off`, or a positive integer.
2589    pub fn parse(value: &str) -> Result<Self, String> {
2590        let trimmed = value.trim();
2591        match trimmed.to_ascii_lowercase().as_str() {
2592            "auto" => Ok(Self::Auto),
2593            "off" | "none" | "disabled" => Ok(Self::Off),
2594            _ => match trimmed.parse::<u32>() {
2595                Ok(0) => Err(
2596                    "remote_build_jobs must be >= 1 (cargo rejects `-j 0`); use `off` to disable"
2597                        .to_string(),
2598                ),
2599                Ok(n) => Ok(Self::Fixed(n)),
2600                Err(_) => Err(format!(
2601                    "'{trimmed}' is not a valid remote_build_jobs value (expected `auto`, `off`, or a positive integer)"
2602                )),
2603            },
2604        }
2605    }
2606}
2607
2608impl std::fmt::Display for RemoteBuildJobs {
2609    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2610        match self {
2611            Self::Auto => f.write_str("auto"),
2612            Self::Off => f.write_str("off"),
2613            Self::Fixed(n) => write!(f, "{n}"),
2614        }
2615    }
2616}
2617
2618impl std::str::FromStr for RemoteBuildJobs {
2619    type Err = String;
2620
2621    fn from_str(value: &str) -> Result<Self, Self::Err> {
2622        Self::parse(value)
2623    }
2624}
2625
2626impl Serialize for RemoteBuildJobs {
2627    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
2628        match self {
2629            Self::Auto => serializer.serialize_str("auto"),
2630            Self::Off => serializer.serialize_str("off"),
2631            Self::Fixed(n) => serializer.serialize_u32(*n),
2632        }
2633    }
2634}
2635
2636impl<'de> Deserialize<'de> for RemoteBuildJobs {
2637    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
2638        #[derive(Deserialize)]
2639        #[serde(untagged)]
2640        enum Raw {
2641            Count(u64),
2642            Spelling(String),
2643        }
2644        match Raw::deserialize(deserializer)? {
2645            Raw::Count(n) => {
2646                let n = u32::try_from(n).map_err(|_| {
2647                    serde::de::Error::custom(format!("remote_build_jobs {n} is out of range"))
2648                })?;
2649                Self::parse(&n.to_string()).map_err(serde::de::Error::custom)
2650            }
2651            Raw::Spelling(s) => Self::parse(&s).map_err(serde::de::Error::custom),
2652        }
2653    }
2654}
2655
2656impl Default for CompilationConfig {
2657    fn default() -> Self {
2658        Self {
2659            confidence_threshold: 0.85,
2660            min_local_time_ms: 2000,
2661            remote_speedup_threshold: default_speedup_threshold(),
2662            build_slots: default_build_slots(),
2663            test_slots: default_test_slots(),
2664            check_slots: default_check_slots(),
2665            build_timeout_sec: default_build_timeout(),
2666            test_timeout_sec: default_test_timeout(),
2667            bun_timeout_sec: default_bun_timeout(),
2668            external_timeout_enabled: default_external_timeout_enabled(),
2669            allow_local_fallback: default_allow_local_fallback(),
2670            remote_build_jobs: RemoteBuildJobs::default(),
2671        }
2672    }
2673}
2674
2675fn default_build_slots() -> u32 {
2676    4
2677}
2678
2679fn default_test_slots() -> u32 {
2680    8
2681}
2682
2683fn default_check_slots() -> u32 {
2684    2
2685}
2686
2687/// Default build timeout: 5 minutes (300 seconds).
2688/// Most builds complete well within this time.
2689fn default_build_timeout() -> u64 {
2690    300
2691}
2692
2693/// Default test timeout: 30 minutes (1800 seconds).
2694/// Test suites often take significantly longer than builds.
2695fn default_test_timeout() -> u64 {
2696    1800
2697}
2698
2699/// Default bun timeout: 10 minutes (600 seconds).
2700/// Bun's internal --timeout flag doesn't work for CPU-bound hangs,
2701/// so we use a shorter external timeout to prevent zombies.
2702fn default_bun_timeout() -> u64 {
2703    600
2704}
2705
2706/// Default: external timeout protection is enabled.
2707fn default_external_timeout_enabled() -> bool {
2708    true
2709}
2710
2711/// Default: permit last-resort local fallback after remote workers are
2712/// exhausted. Set `compilation.allow_local_fallback = false` to fail closed and
2713/// keep heavy compiles off the orchestrator entirely.
2714fn default_allow_local_fallback() -> bool {
2715    true
2716}
2717
2718impl CompilationConfig {
2719    /// Returns the appropriate external timeout for the given compilation kind.
2720    ///
2721    /// - Bun commands get the shorter bun_timeout (to protect against known hang issues)
2722    /// - Test commands get the longer test_timeout
2723    /// - All other commands (builds, checks, clippy) get the build_timeout
2724    pub fn timeout_for_kind(&self, kind: Option<CompilationKind>) -> std::time::Duration {
2725        let secs = match kind {
2726            // Bun has known issues where internal timeout doesn't work for CPU hangs
2727            Some(CompilationKind::BunTest) | Some(CompilationKind::BunTypecheck) => {
2728                self.bun_timeout_sec
2729            }
2730            // Other test commands use the standard test timeout
2731            Some(CompilationKind::CargoTest) | Some(CompilationKind::CargoNextest) => {
2732                self.test_timeout_sec
2733            }
2734            // Nix derivation builds can take a long time (fetch + compile large
2735            // closures); give them the generous test-timeout bucket rather than
2736            // the short build timeout so legitimate builds are not killed early.
2737            Some(CompilationKind::NixBuild) => self.test_timeout_sec,
2738            // Go test suites are long-running like cargo test; give them the
2739            // test bucket rather than the short build timeout.
2740            Some(CompilationKind::GoTest) => self.test_timeout_sec,
2741            // Builds, checks, clippy, and unknown use build timeout
2742            _ => self.build_timeout_sec,
2743        };
2744        std::time::Duration::from_secs(secs)
2745    }
2746
2747    /// Returns whether external timeout wrapping is enabled.
2748    pub fn external_timeout_enabled(&self) -> bool {
2749        self.external_timeout_enabled
2750    }
2751}
2752
2753/// Version of the Layer 0 configuration pack schema (bd-bqu38 / M-1 §149).
2754///
2755/// Bump when a knob is added, removed, or its meaning changes; consumers
2756/// record the version alongside emitted baselines so regressions are
2757/// attributable to a specific pack revision.
2758pub const LAYER0_PACK_VERSION: &str = "1";
2759
2760/// Layer 0 configuration pack knobs (bd-bqu38).
2761///
2762/// Every knob defaults to OFF and toggles independently — the kill/rollback
2763/// contract from the master plan (§149: removed from defaults if it regresses
2764/// representative p95, output equivalence, debugger behavior, or
2765/// compatibility). Knobs are injected into remote cargo builds as
2766/// `CARGO_PROFILE_*` environment variables, so enabling one never mutates any
2767/// user-owned `.cargo/config.toml`.
2768#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
2769#[serde(default)]
2770pub struct Layer0Config {
2771    /// Master switch: nothing is injected unless this is on.
2772    pub enabled: bool,
2773    /// `profile.release.lto = "thin"` via `CARGO_PROFILE_RELEASE_LTO=thin`.
2774    pub release_lto_thin: bool,
2775    /// `profile.release.codegen-units = 1` via
2776    /// `CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1`.
2777    pub release_codegen_units_1: bool,
2778}
2779
2780impl Layer0Config {
2781    /// Environment pairs to inject into a remote cargo build.
2782    ///
2783    /// Empty unless [`Self::enabled`] — the master switch gates every knob so
2784    /// an operator can disable the whole pack with one key. Order is stable
2785    /// (declaration order) for deterministic emission and test assertions.
2786    #[must_use]
2787    pub fn active_env(&self) -> Vec<(&'static str, &'static str)> {
2788        if !self.enabled {
2789            return Vec::new();
2790        }
2791        let mut env = Vec::new();
2792        if self.release_lto_thin {
2793            env.push(("CARGO_PROFILE_RELEASE_LTO", "thin"));
2794        }
2795        if self.release_codegen_units_1 {
2796            env.push(("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"));
2797        }
2798        env
2799    }
2800}
2801
2802#[derive(Debug, Clone, Serialize, Deserialize)]
2803pub struct TransferConfig {
2804    /// zstd compression level (1-19).
2805    #[serde(default = "default_compression")]
2806    pub compression_level: u32,
2807    /// Patterns to exclude from transfer.
2808    #[serde(default = "default_excludes")]
2809    pub exclude_patterns: Vec<String>,
2810    /// SSH keepalive interval in seconds (`ssh -o ServerAliveInterval=<N>`).
2811    ///
2812    /// When unset, build transfers use 15 seconds. Zero defers to OpenSSH configuration.
2813    #[serde(default, skip_serializing_if = "Option::is_none")]
2814    pub ssh_server_alive_interval_secs: Option<u64>,
2815    /// SSH ControlPersist idle timeout in seconds (`ssh -o ControlPersist=<N>s`).
2816    ///
2817    /// - Unset preserves OpenSSH defaults (ControlPersist enabled by default via mux).
2818    /// - `0` disables persistence (`ControlPersist=no`).
2819    #[serde(default, skip_serializing_if = "Option::is_none")]
2820    pub ssh_control_persist_secs: Option<u64>,
2821    /// Staging base on the worker for the transfer paths that do NOT mirror
2822    /// the client's project layout: `--clean-overlay` roots, Windows workers,
2823    /// and `rch cache warm`. Must be an absolute path; defaults to
2824    /// [`default_remote_base`] (`/data/tmp/rch`).
2825    ///
2826    /// An ordinary `rch exec` does not use this: it mirrors the project under
2827    /// the worker's canonical root (`[path_topology] canonical_root`), and the
2828    /// pooled Cargo target store rides along inside that mirror. To move
2829    /// pooled stores onto another filesystem, set
2830    /// `[remediation.pooled_target] store_base` (issue #64).
2831    #[serde(default = "default_remote_base")]
2832    pub remote_base: String,
2833    /// Per-attempt timeout for source synchronization, in milliseconds.
2834    ///
2835    /// This is deliberately separate from remote compilation and artifact-return
2836    /// timeouts. When unset, RCH derives a conservative timeout from the source
2837    /// payload size (30 seconds plus one second per MiB, capped at one hour).
2838    #[serde(default, skip_serializing_if = "Option::is_none")]
2839    pub sync_timeout_ms: Option<u64>,
2840    /// Silence-based stall timeout for source synchronization, in seconds
2841    /// (issue #59).
2842    ///
2843    /// Aborts a source sync when rsync emits NO output at all — no progress
2844    /// refresh, no stats, no itemized line — for this long. This is deliberately
2845    /// far tighter than the wall-clock `sync_timeout_ms` cap (which scales with
2846    /// payload size up to an hour): a large first-time sync keeps refreshing its
2847    /// progress line and is never killed, while a dead channel or wedged rsync
2848    /// is detected within this window instead of holding the reservation until
2849    /// the wall-clock cap. On a stall the hook releases the worker and re-enters
2850    /// selection excluding it. `0` disables silence detection.
2851    #[serde(default = "default_source_sync_silence_timeout_secs")]
2852    pub source_sync_silence_timeout_secs: u64,
2853    /// Retry policy for transient network errors during transfer.
2854    #[serde(default)]
2855    pub retry: RetryConfig,
2856    /// Verify artifact integrity using blake3 hashes after transfer (bd-377q).
2857    ///
2858    /// When enabled, computes blake3 hashes of key artifacts (binaries, test outputs)
2859    /// on the worker after build, then verifies them locally after download.
2860    /// Disabled by default to avoid extra overhead.
2861    #[serde(default)]
2862    pub verify_artifacts: bool,
2863    /// Maximum file size (bytes) for artifact verification (bd-377q).
2864    ///
2865    /// Files larger than this limit are skipped during verification to avoid
2866    /// excessive I/O. Defaults to 100MB.
2867    #[serde(default = "default_verify_max_size")]
2868    pub verify_max_size_bytes: u64,
2869
2870    // =========================================================================
2871    // Transfer Optimization (bd-3hho)
2872    // =========================================================================
2873    /// Maximum transfer size in MB before skipping remote execution.
2874    ///
2875    /// When set, runs `rsync --dry-run --stats` to estimate transfer size.
2876    /// If the estimated size exceeds this threshold, remote offload is skipped
2877    /// and the command runs locally. Set to `None` (default) to disable.
2878    #[serde(default, skip_serializing_if = "Option::is_none")]
2879    pub max_transfer_mb: Option<u64>,
2880
2881    /// Maximum estimated transfer time in milliseconds before skipping remote.
2882    ///
2883    /// Uses `estimated_bandwidth_bps` (or measured link speed) to calculate
2884    /// expected transfer time. If it exceeds this threshold, remote offload
2885    /// is skipped. Set to `None` (default) to disable.
2886    #[serde(default, skip_serializing_if = "Option::is_none")]
2887    pub max_transfer_time_ms: Option<u64>,
2888
2889    /// Bandwidth limit for rsync in KB/s.
2890    ///
2891    /// Passed as `rsync --bwlimit=<N>` to limit transfer speed and prevent
2892    /// network saturation. Set to `None` or `0` (default) for unlimited.
2893    #[serde(default, skip_serializing_if = "Option::is_none")]
2894    pub bwlimit_kbps: Option<u64>,
2895
2896    /// Explicit rsync binary for every local transfer (issue #66).
2897    ///
2898    /// Unset (the default) lets rch pick: the PATH `rsync` when it is a
2899    /// modern (3.1+) rsync, otherwise a modern rsync from the well-known
2900    /// Homebrew/MacPorts locations, otherwise the PATH binary driven with the
2901    /// openrsync/2.6.9-compatible argv. Set this to force one binary
2902    /// (`~` is expanded; a bare name is looked up on PATH). The
2903    /// `RCH_RSYNC_BIN` environment variable overrides this setting.
2904    #[serde(default, skip_serializing_if = "Option::is_none")]
2905    pub rsync_bin: Option<String>,
2906
2907    /// Estimated link bandwidth in bytes per second.
2908    ///
2909    /// Used for transfer time estimation when `max_transfer_time_ms` is set.
2910    /// If not provided, defaults to 10 MB/s (10485760 bytes/sec).
2911    #[serde(default, skip_serializing_if = "Option::is_none")]
2912    pub estimated_bandwidth_bps: Option<u64>,
2913
2914    // =========================================================================
2915    // Adaptive Compression (bd-243w)
2916    // =========================================================================
2917    /// Enable adaptive zstd compression based on transfer size (bd-243w).
2918    ///
2919    /// When enabled, automatically selects compression level based on estimated
2920    /// payload size:
2921    /// - < 10MB: level 1 (fast, minimal compression)
2922    /// - 10-200MB: level 3 (balanced, default)
2923    /// - > 200MB: level 7 (slower, better compression)
2924    ///
2925    /// When disabled (default), uses the fixed `compression_level` setting.
2926    #[serde(default)]
2927    pub adaptive_compression: bool,
2928
2929    /// Minimum compression level for adaptive mode (bd-243w).
2930    ///
2931    /// Even in adaptive mode, compression level won't go below this.
2932    /// Defaults to 1.
2933    #[serde(default = "default_min_compression")]
2934    pub min_compression_level: u32,
2935
2936    /// Maximum compression level for adaptive mode (bd-243w).
2937    ///
2938    /// Even in adaptive mode, compression level won't exceed this.
2939    /// Defaults to 9 (avoids CPU-intensive levels 10+).
2940    #[serde(default = "default_max_compression")]
2941    pub max_compression_level: u32,
2942}
2943
2944impl Default for TransferConfig {
2945    fn default() -> Self {
2946        Self {
2947            compression_level: 3,
2948            exclude_patterns: default_excludes(),
2949            ssh_server_alive_interval_secs: None,
2950            ssh_control_persist_secs: None,
2951            remote_base: default_remote_base(),
2952            sync_timeout_ms: None,
2953            source_sync_silence_timeout_secs: default_source_sync_silence_timeout_secs(),
2954            retry: RetryConfig::default(),
2955            verify_artifacts: false,
2956            verify_max_size_bytes: default_verify_max_size(),
2957            // Transfer optimization (bd-3hho)
2958            max_transfer_mb: None,
2959            max_transfer_time_ms: None,
2960            bwlimit_kbps: None,
2961            rsync_bin: None,
2962            estimated_bandwidth_bps: None,
2963            // Adaptive compression (bd-243w)
2964            adaptive_compression: false,
2965            min_compression_level: default_min_compression(),
2966            max_compression_level: default_max_compression(),
2967        }
2968    }
2969}
2970
2971impl TransferConfig {
2972    /// Smallest accepted explicit source-sync timeout (one second).
2973    pub const MIN_SYNC_TIMEOUT_MS: u64 = 1_000;
2974    /// Largest accepted explicit or derived source-sync timeout (one hour).
2975    pub const MAX_SYNC_TIMEOUT_MS: u64 = 3_600_000;
2976    /// Fixed floor used by the payload-aware default policy.
2977    pub const DEFAULT_SYNC_TIMEOUT_FLOOR_MS: u64 = 30_000;
2978    const DEFAULT_SYNC_BYTES_PER_SECOND: u64 = 1024 * 1024;
2979
2980    /// Whether an explicit source-sync timeout is within the supported range.
2981    #[must_use]
2982    pub fn valid_sync_timeout_ms(value: u64) -> bool {
2983        (Self::MIN_SYNC_TIMEOUT_MS..=Self::MAX_SYNC_TIMEOUT_MS).contains(&value)
2984    }
2985
2986    /// Resolve the per-attempt source-sync timeout for an exact payload size.
2987    ///
2988    /// Explicit configuration wins. Otherwise the default grants 30 seconds of
2989    /// setup time plus one second per MiB at a conservative 1 MiB/s floor. This
2990    /// prevents a cold, large pinned tree from inheriting the old fixed 30-second
2991    /// ceiling while still bounding every individual attempt.
2992    #[must_use]
2993    pub fn sync_timeout_for_payload(&self, payload_bytes: u64) -> std::time::Duration {
2994        let timeout_ms = self
2995            .sync_timeout_ms
2996            .filter(|value| Self::valid_sync_timeout_ms(*value))
2997            .unwrap_or_else(|| {
2998                let transfer_seconds = payload_bytes
2999                    .saturating_add(Self::DEFAULT_SYNC_BYTES_PER_SECOND - 1)
3000                    / Self::DEFAULT_SYNC_BYTES_PER_SECOND;
3001                Self::DEFAULT_SYNC_TIMEOUT_FLOOR_MS
3002                    .saturating_add(transfer_seconds.saturating_mul(1_000))
3003                    .min(Self::MAX_SYNC_TIMEOUT_MS)
3004            });
3005        std::time::Duration::from_millis(timeout_ms)
3006    }
3007
3008    /// Select compression level based on estimated transfer size (bd-243w).
3009    ///
3010    /// When adaptive compression is enabled, selects an appropriate level
3011    /// based on payload size:
3012    /// - < 10MB: level 1 (fast, minimal compression)
3013    /// - 10-200MB: level 3 (balanced)
3014    /// - > 200MB: level 7 (better compression)
3015    ///
3016    /// The result is clamped between `min_compression_level` and `max_compression_level`.
3017    ///
3018    /// When adaptive compression is disabled, returns the fixed `compression_level`.
3019    pub fn select_compression_level(&self, estimated_bytes: Option<u64>) -> u32 {
3020        if !self.adaptive_compression {
3021            return self.compression_level;
3022        }
3023
3024        let Some(bytes) = estimated_bytes else {
3025            // No estimate available, use default
3026            return self.compression_level;
3027        };
3028
3029        // Size thresholds
3030        const SMALL_THRESHOLD: u64 = 10_000_000; // 10 MB (decimal)
3031        const LARGE_THRESHOLD: u64 = 200_000_000; // 200 MB (decimal)
3032
3033        let level = if bytes < SMALL_THRESHOLD {
3034            1 // Fast compression for small payloads
3035        } else if bytes < LARGE_THRESHOLD {
3036            3 // Balanced for medium payloads
3037        } else {
3038            7 // Better compression for large payloads
3039        };
3040
3041        // Clamp to configured bounds
3042        level.clamp(self.min_compression_level, self.max_compression_level)
3043    }
3044}
3045
3046// =============================================================================
3047// Execution Configuration (bd-785w)
3048// =============================================================================
3049
3050/// Default allowlist of commands that can be executed remotely.
3051/// Aligned with the classifier's supported CompilationKind values.
3052/// Command bases of every compilation kind rch knows how to offload.
3053///
3054/// This is the built-in `execution.allowlist`. It is also what a persisted,
3055/// explicit allowlist is compared against: a config file written before a tool
3056/// family was added (Go, Nix, TypeScript, ...) silently keeps that family local,
3057/// and the only symptom is a `[RCH] local (... not in execution.allowlist)` line
3058/// (issue #51). `ExecutionConfig::missing_builtin_bases` surfaces the gap.
3059#[rustfmt::skip]
3060pub const BUILTIN_EXECUTION_BASES: &[&str] = &[
3061    // Rust
3062    "cargo",
3063    "rustc",
3064    // cargo-nextest
3065    "nextest",
3066    // C/C++
3067    "gcc",
3068    "g++",
3069    "clang",
3070    "clang++",
3071    "cc",
3072    "c++",
3073    // Build systems
3074    "make",
3075    "cmake",
3076    "ninja",
3077    "meson",
3078    // Bun
3079    "bun",
3080    // Nix (covers both `nix build` and the legacy `nix-build`)
3081    "nix",
3082    // Go
3083    "go",
3084    // TypeScript (covers both `tsc` and `npx tsc`; command_base is "tsc")
3085    "tsc",
3086];
3087
3088fn default_execution_allowlist() -> Vec<String> {
3089    BUILTIN_EXECUTION_BASES
3090        .iter()
3091        .map(|base| (*base).to_string())
3092        .collect()
3093}
3094
3095/// Execution allowlist configuration for remote builds (bd-785w).
3096///
3097/// Controls which command base names are permitted for remote execution.
3098/// Commands not in the allowlist will fail-open to local execution.
3099#[derive(Debug, Clone, Serialize, Deserialize)]
3100pub struct ExecutionConfig {
3101    /// Allowlist of command base names permitted for remote execution.
3102    ///
3103    /// Examples: "cargo", "rustc", "gcc", "bun"
3104    ///
3105    /// If a classified compilation command's base name is not in this list,
3106    /// RCH will fail-open to local execution with a clear reason.
3107    ///
3108    /// An empty allowlist disables all remote execution (local only).
3109    #[serde(default = "default_execution_allowlist")]
3110    pub allowlist: Vec<String>,
3111}
3112
3113impl Default for ExecutionConfig {
3114    fn default() -> Self {
3115        Self {
3116            allowlist: default_execution_allowlist(),
3117        }
3118    }
3119}
3120
3121impl ExecutionConfig {
3122    /// Check if a command base name is allowed for remote execution.
3123    pub fn is_allowed(&self, command_base: &str) -> bool {
3124        self.allowlist
3125            .iter()
3126            .any(|allowed| allowed.eq_ignore_ascii_case(command_base))
3127    }
3128
3129    /// Built-in compilation tool bases that this allowlist does NOT admit.
3130    ///
3131    /// Empty for the default allowlist. Non-empty means some command family rch
3132    /// classifies as compilation will always fall open to local execution on
3133    /// this box — usually because the config file predates that family.
3134    pub fn missing_builtin_bases(&self) -> Vec<&'static str> {
3135        BUILTIN_EXECUTION_BASES
3136            .iter()
3137            .copied()
3138            .filter(|base| !self.is_allowed(base))
3139            .collect()
3140    }
3141}
3142
3143// =============================================================================
3144// Retry Configuration (bd-x1ek)
3145// =============================================================================
3146
3147fn default_retry_max_attempts() -> u32 {
3148    3
3149}
3150
3151fn default_retry_base_delay_ms() -> u64 {
3152    100
3153}
3154
3155fn default_retry_max_delay_ms() -> u64 {
3156    5000
3157}
3158
3159fn default_retry_jitter_factor() -> f64 {
3160    0.1
3161}
3162
3163fn default_retry_total_timeout_ms() -> u64 {
3164    30000
3165}
3166
3167/// Configuration for retry behavior on transient errors.
3168///
3169/// Used by rsync and SSH operations to recover from transient network failures
3170/// without blocking too long or retrying non-recoverable errors.
3171#[derive(Debug, Clone, Serialize, Deserialize)]
3172pub struct RetryConfig {
3173    /// Maximum number of retry attempts (1 = no retries, just initial attempt).
3174    #[serde(default = "default_retry_max_attempts")]
3175    pub max_attempts: u32,
3176
3177    /// Initial delay between retries in milliseconds.
3178    #[serde(default = "default_retry_base_delay_ms")]
3179    pub base_delay_ms: u64,
3180
3181    /// Maximum delay between retries in milliseconds (caps exponential backoff).
3182    #[serde(default = "default_retry_max_delay_ms")]
3183    pub max_delay_ms: u64,
3184
3185    /// Jitter factor (0.0 - 1.0) to randomize delay.
3186    /// A value of 0.1 means ±10% randomization.
3187    #[serde(default = "default_retry_jitter_factor")]
3188    pub jitter_factor: f64,
3189
3190    /// Total timeout for all retry attempts in milliseconds.
3191    /// After this, retries stop even if max_attempts not reached.
3192    #[serde(default = "default_retry_total_timeout_ms")]
3193    pub total_timeout_ms: u64,
3194}
3195
3196impl Default for RetryConfig {
3197    fn default() -> Self {
3198        Self {
3199            max_attempts: default_retry_max_attempts(),
3200            base_delay_ms: default_retry_base_delay_ms(),
3201            max_delay_ms: default_retry_max_delay_ms(),
3202            jitter_factor: default_retry_jitter_factor(),
3203            total_timeout_ms: default_retry_total_timeout_ms(),
3204        }
3205    }
3206}
3207
3208impl RetryConfig {
3209    /// Create a config that disables retries (single attempt only).
3210    pub fn no_retry() -> Self {
3211        Self {
3212            max_attempts: 1,
3213            ..Default::default()
3214        }
3215    }
3216
3217    /// Calculate delay for a given attempt number (0-indexed).
3218    ///
3219    /// Uses exponential backoff with optional jitter.
3220    pub fn delay_for_attempt(&self, attempt: u32) -> std::time::Duration {
3221        if attempt == 0 {
3222            return std::time::Duration::ZERO;
3223        }
3224
3225        // Exponential backoff: base * 2^attempt
3226        let base = self.base_delay_ms as f64;
3227        let delay = base * (2.0_f64.powi(attempt as i32 - 1));
3228        let capped = delay.min(self.max_delay_ms as f64);
3229
3230        // Apply jitter: delay ± (delay * jitter_factor)
3231        let jittered = if self.jitter_factor > 0.0 {
3232            let jitter_range = capped * self.jitter_factor;
3233            let jitter = (rand::rng().random::<f64>() * 2.0 - 1.0) * jitter_range;
3234            (capped + jitter).max(0.0)
3235        } else {
3236            capped
3237        };
3238
3239        std::time::Duration::from_millis(jittered as u64)
3240    }
3241
3242    /// Check if we should retry given elapsed time since start.
3243    pub fn should_retry(&self, attempt: u32, elapsed: std::time::Duration) -> bool {
3244        if attempt >= self.max_attempts {
3245            return false;
3246        }
3247        elapsed.as_millis() < self.total_timeout_ms as u128
3248    }
3249}
3250
3251/// Default source-sync silence timeout (issue #59): two minutes without a
3252/// single byte of rsync output means a dead channel or wedged rsync, not a
3253/// large transfer (progress2 refreshes its line continuously while bytes
3254/// flow, and each refresh counts as forward progress).
3255fn default_source_sync_silence_timeout_secs() -> u64 {
3256    120
3257}
3258
3259/// Default remote base path for project transfers.
3260///
3261/// Rooted under `/data/tmp` (the fleet-standard managed ephemeral-build zone)
3262/// rather than `/tmp`: on the Linux workers `/tmp` is a RAM-backed tmpfs, so
3263/// dispatched build artifacts placed there consume RAM and evade the disk-GC
3264/// (sbh / datatmp-janitor) that watches `/data/tmp`. Keeping the remote base in
3265/// the managed zone lets those reapers reclaim orphaned build trees. Must agree
3266/// with the rchd cache-cleanup and reclaim `remote_base` defaults.
3267pub fn default_remote_base() -> String {
3268    "/data/tmp/rch".to_string()
3269}
3270
3271/// Validate and normalize a remote base path.
3272/// Returns an error if the path is not absolute or contains traversal sequences.
3273pub fn validate_remote_base(path: &str) -> Result<String, String> {
3274    // Expand tilde for user convenience
3275    let expanded = shellexpand::tilde(path).into_owned();
3276
3277    // Check for absolute path
3278    if !expanded.starts_with('/') {
3279        return Err(format!(
3280            "remote_base must be an absolute path, got: {}",
3281            path
3282        ));
3283    }
3284
3285    // Check for path traversal attempts
3286    if expanded.contains("..") {
3287        return Err(format!(
3288            "remote_base must not contain path traversal (..): {}",
3289            path
3290        ));
3291    }
3292
3293    // Normalize: remove trailing slashes
3294    let normalized = expanded.trim_end_matches('/').to_string();
3295
3296    // Safety checks
3297    if normalized.is_empty() || normalized == "/" {
3298        return Err("remote_base cannot be the root directory (safety restriction)".to_string());
3299    }
3300
3301    // Enforce at least 2 levels deep (e.g. /tmp/rch, /home/user/rch)
3302    // /tmp -> depth 1 (unsafe for recursive delete)
3303    // /tmp/rch -> depth 2 (safe)
3304    let components: Vec<&str> = normalized.split('/').filter(|c| !c.is_empty()).collect();
3305    if components.len() < 2 {
3306        return Err(format!(
3307            "remote_base must be at least 2 levels deep (e.g. /tmp/rch), got: {}",
3308            normalized
3309        ));
3310    }
3311
3312    Ok(normalized)
3313}
3314
3315fn default_circuit_failure_threshold() -> u32 {
3316    3
3317}
3318
3319fn default_circuit_success_threshold() -> u32 {
3320    2
3321}
3322
3323fn default_circuit_error_rate_threshold() -> f64 {
3324    0.5
3325}
3326
3327fn default_circuit_window_secs() -> u64 {
3328    60
3329}
3330
3331fn default_circuit_open_cooldown_secs() -> u64 {
3332    30
3333}
3334
3335fn default_circuit_half_open_max_probes() -> u32 {
3336    1
3337}
3338
3339fn default_circuit_command_failure_threshold() -> u32 {
3340    2
3341}
3342
3343fn default_circuit_open_cooldown_max_secs() -> u64 {
3344    // 30s base doubled four times: 30, 60, 120, 240, 480.
3345    480
3346}
3347
3348fn default_true() -> bool {
3349    true
3350}
3351
3352fn default_log_level() -> String {
3353    "info".to_string()
3354}
3355
3356pub fn default_socket_path() -> String {
3357    // Prefer XDG_RUNTIME_DIR if available (per-user runtime directory).
3358    if let Ok(runtime_dir) = std::env::var("XDG_RUNTIME_DIR")
3359        && !runtime_dir.trim().is_empty()
3360    {
3361        let path = PathBuf::from(runtime_dir).join("rch.sock");
3362        return path.to_string_lossy().to_string();
3363    }
3364
3365    // Fallback to ~/.cache/rch/rch.sock (persistent across reboots).
3366    if let Some(cache_dir) = dirs::cache_dir() {
3367        let rch_cache = cache_dir.join("rch");
3368        let _ = std::fs::create_dir_all(&rch_cache);
3369        return rch_cache.join("rch.sock").to_string_lossy().to_string();
3370    }
3371
3372    // Last resort: /tmp/rch.sock
3373    "/tmp/rch.sock".to_string()
3374}
3375
3376#[allow(dead_code)]
3377fn default_output_visibility() -> OutputVisibility {
3378    OutputVisibility::None
3379}
3380
3381fn default_confidence() -> f64 {
3382    0.85
3383}
3384
3385fn default_min_local_time() -> u64 {
3386    2000
3387}
3388
3389/// Default speedup threshold: 1.2 (require 20% speedup to offload).
3390/// This is conservative to avoid offloading builds that would be only
3391/// marginally faster remotely when accounting for transfer overhead.
3392fn default_speedup_threshold() -> f64 {
3393    1.2
3394}
3395
3396fn default_compression() -> u32 {
3397    3
3398}
3399
3400/// Default maximum file size for artifact verification (100MB).
3401fn default_verify_max_size() -> u64 {
3402    100 * 1024 * 1024 // 100MB
3403}
3404
3405/// Default minimum compression level for adaptive mode.
3406fn default_min_compression() -> u32 {
3407    1
3408}
3409
3410/// Default maximum compression level for adaptive mode.
3411fn default_max_compression() -> u32 {
3412    9
3413}
3414
3415fn default_excludes() -> Vec<String> {
3416    vec![
3417        // Rust build artifacts
3418        "target/".to_string(),
3419        "*.rlib".to_string(),
3420        "*.rmeta".to_string(),
3421        // Git metadata is large, often permission-sensitive on workers, and
3422        // a partial `.git/` tree is not reliable for build scripts anyway.
3423        ".git/".to_string(),
3424        // Node.js / Bun dependencies (massive, reinstalled on worker)
3425        "node_modules/".to_string(),
3426        // Bun cache and runtime files
3427        ".bun/".to_string(),
3428        // npm / pnpm caches
3429        ".npm/".to_string(),
3430        ".pnpm-store/".to_string(),
3431        // Common build output directories
3432        "dist/".to_string(),
3433        "build/".to_string(),
3434        // Framework-specific caches
3435        ".next/".to_string(),
3436        ".nuxt/".to_string(),
3437        ".turbo/".to_string(),
3438        ".parcel-cache/".to_string(),
3439        // Beads runtime state (local issue DB + lock/temp files); not needed remotely.
3440        ".beads/".to_string(),
3441        // Local diagnostic runs may contain host-specific symlinks and are not build inputs.
3442        ".doctor/".to_string(),
3443        // Coverage reports (generated during tests)
3444        "coverage/".to_string(),
3445        ".nyc_output/".to_string(),
3446        // Credentials and secrets (must never be transferred to workers)
3447        ".cargo/credentials".to_string(),
3448        ".cargo/credentials.toml".to_string(),
3449        ".env".to_string(),
3450        ".env.*".to_string(),
3451        "*.pem".to_string(),
3452        "*.key".to_string(),
3453        "credentials.json".to_string(),
3454        "secrets.json".to_string(),
3455        "secrets.yaml".to_string(),
3456        "secrets.yml".to_string(),
3457        "secrets.toml".to_string(),
3458        "secrets.env".to_string(),
3459        "secrets.txt".to_string(),
3460        "secrets.conf".to_string(),
3461        "secrets.cfg".to_string(),
3462        "secrets.properties".to_string(),
3463        "secrets.xml".to_string(),
3464        // The classes above were already covered. These are the ones
3465        // RABS's E027 source-capture seed set denies
3466        // (rabs-sandbox/src/source_capture.rs) that this list did not,
3467        // so the two halves of the product now agree about what a
3468        // secret is. Two components disagreeing on that is the kind of
3469        // gap nobody notices until one of them is the path that ran.
3470        //
3471        // rsync patterns without a `/` match a basename at any depth,
3472        // so these cover nested copies too — a key checked into a
3473        // fixtures directory is excluded exactly like one at the root.
3474        //
3475        // SSH private keys. The public halves (.pub) are deliberately
3476        // not excluded: they are not secret and some builds read them.
3477        "id_rsa".to_string(),
3478        "id_ed25519".to_string(),
3479        "id_ecdsa".to_string(),
3480        "id_dsa".to_string(),
3481        // Key/certificate container formats (*.pem and *.key are above).
3482        "*.p12".to_string(),
3483        "*.pfx".to_string(),
3484        "*.keystore".to_string(),
3485        "*.jks".to_string(),
3486        // Machine credentials and cloud/container config that carries
3487        // long-lived tokens.
3488        ".netrc".to_string(),
3489        "_netrc".to_string(),
3490        ".aws/credentials".to_string(),
3491        ".docker/config.json".to_string(),
3492        // Package-registry auth tokens. These sit in project roots far
3493        // more often than the key formats above and are the likeliest
3494        // real leak in a normal repo.
3495        ".npmrc".to_string(),
3496        ".pypirc".to_string(),
3497        // GnuPG private keyring.
3498        "secring.*".to_string(),
3499    ]
3500}
3501
3502// ============================================================================
3503// Build History Types
3504// ============================================================================
3505
3506/// Location where a build was executed.
3507#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
3508#[serde(rename_all = "snake_case")]
3509pub enum BuildLocation {
3510    /// Build executed locally.
3511    #[default]
3512    Local,
3513    /// Build executed on a remote worker.
3514    Remote,
3515}
3516
3517/// Worker health snapshot captured when a cancellation completes.
3518#[derive(Debug, Clone, Serialize, Deserialize)]
3519pub struct BuildCancellationWorkerHealth {
3520    /// Worker status at cancellation completion.
3521    pub status: String,
3522    /// Worker speed score at cancellation completion.
3523    pub speed_score: f64,
3524    /// Slots currently used on the worker.
3525    pub used_slots: u32,
3526    /// Slots currently available on the worker.
3527    pub available_slots: u32,
3528    /// Normalized pressure state.
3529    pub pressure_state: String,
3530    /// Stable pressure reason code.
3531    pub pressure_reason_code: String,
3532}
3533
3534/// Structured cancellation metadata attached to cancelled build records.
3535#[derive(Debug, Clone, Serialize, Deserialize)]
3536pub struct BuildCancellationMetadata {
3537    /// Stable cancellation operation identifier.
3538    pub operation_id: String,
3539    /// Cancellation origin (for example `user`, `timeout`).
3540    pub origin: String,
3541    /// Stable reason code for cancellation.
3542    pub reason_code: String,
3543    /// Ordered cancellation state path.
3544    #[serde(default)]
3545    pub decision_path: Vec<String>,
3546    /// Highest escalation stage reached (`term`, `remote_kill`, `sigkill`).
3547    pub escalation_stage: String,
3548    /// Number of escalation transitions taken.
3549    pub escalation_count: u32,
3550    /// Whether remote kill was attempted.
3551    pub remote_kill_attempted: bool,
3552    /// Whether cleanup completed successfully.
3553    pub cleanup_ok: bool,
3554    /// Whether active build history was successfully claimed/cancelled.
3555    pub history_cancelled: bool,
3556    /// Final cancellation state.
3557    pub final_state: String,
3558    /// Worker health snapshot at completion.
3559    #[serde(default, skip_serializing_if = "Option::is_none")]
3560    pub worker_health: Option<BuildCancellationWorkerHealth>,
3561}
3562
3563/// Record of a completed build.
3564#[derive(Debug, Clone, Serialize, Deserialize)]
3565pub struct BuildRecord {
3566    /// Unique build identifier.
3567    pub id: u64,
3568    /// When the build started (ISO 8601 timestamp).
3569    pub started_at: String,
3570    /// When the build completed (ISO 8601 timestamp).
3571    pub completed_at: String,
3572    /// Project identifier (usually directory name or hash).
3573    pub project_id: String,
3574    /// Worker that executed the build (None if local).
3575    pub worker_id: Option<String>,
3576    /// Full command executed.
3577    pub command: String,
3578    /// Exit code (0 = success).
3579    pub exit_code: i32,
3580    /// Duration in milliseconds.
3581    pub duration_ms: u64,
3582    /// Build location (local or remote).
3583    pub location: BuildLocation,
3584    /// Bytes transferred (if remote).
3585    pub bytes_transferred: Option<u64>,
3586    /// Optional timing breakdown for the pipeline.
3587    #[serde(default)]
3588    pub timing: Option<CommandTimingBreakdown>,
3589    /// Structured cancellation metadata for cancelled builds.
3590    #[serde(default, skip_serializing_if = "Option::is_none")]
3591    pub cancellation: Option<BuildCancellationMetadata>,
3592}
3593
3594/// Input payload for recording a completed build.
3595#[derive(Debug, Clone, Serialize, Deserialize)]
3596pub struct BuildRecordInput {
3597    /// When the build started (ISO 8601 timestamp).
3598    pub started_at: String,
3599    /// When the build completed (ISO 8601 timestamp).
3600    pub completed_at: String,
3601    /// Project identifier (usually directory name or hash).
3602    pub project_id: String,
3603    /// Worker that executed the build (None if local).
3604    pub worker_id: Option<String>,
3605    /// Full command executed.
3606    pub command: String,
3607    /// Exit code (0 = success).
3608    pub exit_code: i32,
3609    /// Duration in milliseconds.
3610    pub duration_ms: u64,
3611    /// Build location (local or remote).
3612    pub location: BuildLocation,
3613    /// Bytes transferred (if remote).
3614    pub bytes_transferred: Option<u64>,
3615    /// Optional timing breakdown for the pipeline.
3616    #[serde(default)]
3617    pub timing: Option<CommandTimingBreakdown>,
3618    /// Structured cancellation metadata for cancelled builds.
3619    #[serde(default, skip_serializing_if = "Option::is_none")]
3620    pub cancellation: Option<BuildCancellationMetadata>,
3621}
3622
3623impl BuildRecordInput {
3624    /// Convert input payload into a build record with a supplied ID.
3625    pub fn into_record(self, id: u64) -> BuildRecord {
3626        BuildRecord {
3627            id,
3628            started_at: self.started_at,
3629            completed_at: self.completed_at,
3630            project_id: self.project_id,
3631            worker_id: self.worker_id,
3632            command: self.command,
3633            exit_code: self.exit_code,
3634            duration_ms: self.duration_ms,
3635            location: self.location,
3636            bytes_transferred: self.bytes_transferred,
3637            timing: self.timing,
3638            cancellation: self.cancellation,
3639        }
3640    }
3641}
3642
3643/// Aggregate terminal-command statistics for build history.
3644///
3645/// These counts do not infer whether compilation completed before a failing
3646/// test or interrupted command. The denominator includes every stored record.
3647#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3648pub struct BuildStats {
3649    /// Total number of builds in history.
3650    pub total_builds: usize,
3651    /// Number of successfully completed commands (exit_code == 0).
3652    pub success_count: usize,
3653    /// Number of unsuccessful commands, including interruption (exit_code != 0).
3654    pub failure_count: usize,
3655    /// Number of remote builds.
3656    pub remote_count: usize,
3657    /// Number of local builds.
3658    pub local_count: usize,
3659    /// Average build duration in milliseconds.
3660    pub avg_duration_ms: u64,
3661}
3662
3663/// Saved time statistics from remote builds.
3664///
3665/// Tracks estimated time savings from offloading builds to remote workers.
3666/// Estimates are only produced from an observed local baseline; without one the
3667/// stats carry measured remote durations with `estimate_basis = "none"` and
3668/// zero estimated savings rather than a fabricated speedup factor.
3669#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3670pub struct SavedTimeStats {
3671    /// Total time spent on remote builds (milliseconds).
3672    pub total_remote_duration_ms: u64,
3673    /// Estimated total time if those builds ran locally (milliseconds).
3674    /// Zero when no local baseline exists.
3675    pub estimated_local_duration_ms: u64,
3676    /// Time saved by offloading (milliseconds).
3677    /// Computed as max(0, estimated_local - remote); zero without a baseline.
3678    pub time_saved_ms: u64,
3679    /// Number of remote builds included in calculation.
3680    pub builds_counted: usize,
3681    /// Average speedup factor (local_estimate / remote_duration).
3682    /// A value of 2.0 means remote builds are ~2x faster than local.
3683    /// Zero when the estimate basis is "none".
3684    pub avg_speedup: f64,
3685    /// Total time saved today (milliseconds).
3686    pub today_saved_ms: u64,
3687    /// Total time saved this week (milliseconds).
3688    pub week_saved_ms: u64,
3689    /// How the local-time estimate was derived: "observed_local_mean" when
3690    /// successful local builds back the baseline, "none" otherwise.
3691    #[serde(default)]
3692    pub estimate_basis: String,
3693    /// Number of successful local builds backing the baseline (0 for "none").
3694    #[serde(default)]
3695    pub local_baseline_builds: usize,
3696}
3697
3698// ============================================================================
3699// Compilation Timing and Metrics
3700// ============================================================================
3701
3702use std::collections::VecDeque;
3703use std::time::{Duration, Instant};
3704
3705/// Breakdown of timing for each compilation phase.
3706#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3707pub struct CompilationTimingBreakdown {
3708    /// Time to sync source files to worker.
3709    #[serde(with = "duration_millis")]
3710    pub rsync_up: Duration,
3711    /// Time for cargo build on worker.
3712    #[serde(with = "duration_millis")]
3713    pub remote_build: Duration,
3714    /// Time to sync artifacts back.
3715    #[serde(with = "duration_millis")]
3716    pub rsync_down: Duration,
3717    /// Total end-to-end latency.
3718    #[serde(with = "duration_millis")]
3719    pub total: Duration,
3720}
3721
3722/// Timing breakdown for a single command pipeline.
3723///
3724/// Uses optional durations so missing phases serialize as `null`.
3725#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3726pub struct CommandTimingBreakdown {
3727    /// Time spent classifying the command.
3728    #[serde(with = "option_duration_millis")]
3729    pub classify: Option<Duration>,
3730    /// Time spent selecting a worker.
3731    #[serde(with = "option_duration_millis")]
3732    pub select: Option<Duration>,
3733    /// Time spent syncing source files to the worker.
3734    #[serde(with = "option_duration_millis")]
3735    pub sync_up: Option<Duration>,
3736    /// Time spent executing the remote command.
3737    #[serde(with = "option_duration_millis")]
3738    pub exec: Option<Duration>,
3739    /// Time spent syncing artifacts back.
3740    #[serde(with = "option_duration_millis")]
3741    pub sync_down: Option<Duration>,
3742    /// Time spent on cleanup (slot release, bookkeeping).
3743    #[serde(with = "option_duration_millis")]
3744    pub cleanup: Option<Duration>,
3745    /// Total end-to-end time for the pipeline.
3746    #[serde(with = "option_duration_millis")]
3747    pub total: Option<Duration>,
3748}
3749
3750mod duration_millis {
3751    use serde::{Deserialize, Deserializer, Serialize, Serializer};
3752    use std::time::Duration;
3753
3754    pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
3755    where
3756        S: Serializer,
3757    {
3758        duration.as_millis().serialize(serializer)
3759    }
3760
3761    pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
3762    where
3763        D: Deserializer<'de>,
3764    {
3765        let millis = u64::deserialize(deserializer)?;
3766        Ok(Duration::from_millis(millis))
3767    }
3768}
3769
3770/// Comprehensive metrics for a single compilation.
3771#[derive(Debug, Clone, Serialize, Deserialize)]
3772pub struct CompilationMetrics {
3773    /// Project identifier (usually directory name or hash).
3774    pub project_id: String,
3775    /// Worker that performed the compilation.
3776    pub worker_id: String,
3777    /// When the compilation occurred.
3778    pub timestamp: chrono::DateTime<chrono::Utc>,
3779    /// Timing breakdown for each phase.
3780    pub timing: CompilationTimingBreakdown,
3781    /// Local build time for comparison (if available).
3782    #[serde(with = "option_duration_millis")]
3783    pub local_build_time: Option<Duration>,
3784    /// Speedup ratio (local_time / remote_time).
3785    pub speedup: Option<f64>,
3786    /// Number of files synced to worker.
3787    pub files_synced: u64,
3788    /// Total bytes transferred.
3789    pub bytes_transferred: u64,
3790    /// Exit code from compilation.
3791    pub exit_code: i32,
3792    /// Whether compilation succeeded.
3793    pub success: bool,
3794}
3795
3796mod option_duration_millis {
3797    use serde::{Deserialize, Deserializer, Serialize, Serializer};
3798    use std::time::Duration;
3799
3800    pub fn serialize<S>(duration: &Option<Duration>, serializer: S) -> Result<S::Ok, S::Error>
3801    where
3802        S: Serializer,
3803    {
3804        match duration {
3805            Some(d) => Some(d.as_millis() as u64).serialize(serializer),
3806            None => Option::<u64>::None.serialize(serializer),
3807        }
3808    }
3809
3810    pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Duration>, D::Error>
3811    where
3812        D: Deserializer<'de>,
3813    {
3814        let opt: Option<u64> = Option::deserialize(deserializer)?;
3815        Ok(opt.map(Duration::from_millis))
3816    }
3817}
3818
3819impl CompilationMetrics {
3820    /// Calculate speedup based on local vs remote build time.
3821    pub fn calculate_speedup(&mut self) {
3822        if let Some(local) = self.local_build_time
3823            && self.timing.total.as_millis() > 0
3824        {
3825            self.speedup = Some(local.as_secs_f64() / self.timing.total.as_secs_f64());
3826        }
3827    }
3828
3829    /// Check if remote compilation was beneficial (faster than local).
3830    pub fn is_beneficial(&self) -> bool {
3831        self.speedup.map(|s| s > 1.0).unwrap_or(false)
3832    }
3833}
3834
3835impl Default for CompilationMetrics {
3836    fn default() -> Self {
3837        Self {
3838            project_id: String::new(),
3839            worker_id: String::new(),
3840            timestamp: chrono::Utc::now(),
3841            timing: CompilationTimingBreakdown::default(),
3842            local_build_time: None,
3843            speedup: None,
3844            files_synced: 0,
3845            bytes_transferred: 0,
3846            exit_code: 0,
3847            success: true,
3848        }
3849    }
3850}
3851
3852/// Timer for tracking compilation phases.
3853///
3854/// Use this to measure the duration of each phase of remote compilation.
3855///
3856/// # Example
3857/// ```ignore
3858/// let mut timer = CompilationTimer::new("my-project", "worker-1");
3859/// // ... rsync source files ...
3860/// timer.end_rsync_up();
3861/// // ... run cargo build ...
3862/// timer.end_remote_build();
3863/// // ... rsync artifacts back ...
3864/// timer.end_rsync_down();
3865/// let metrics = timer.finish(0, 100, 1_000_000);
3866/// ```
3867#[derive(Debug)]
3868pub struct CompilationTimer {
3869    project_id: String,
3870    worker_id: String,
3871    start: Instant,
3872    phase_start: Instant,
3873    rsync_up: Option<Duration>,
3874    remote_build: Option<Duration>,
3875    rsync_down: Option<Duration>,
3876}
3877
3878impl CompilationTimer {
3879    /// Create a new timer for a compilation.
3880    pub fn new(project_id: &str, worker_id: &str) -> Self {
3881        let now = Instant::now();
3882        Self {
3883            project_id: project_id.to_string(),
3884            worker_id: worker_id.to_string(),
3885            start: now,
3886            phase_start: now,
3887            rsync_up: None,
3888            remote_build: None,
3889            rsync_down: None,
3890        }
3891    }
3892
3893    /// Mark the end of the rsync upload phase.
3894    pub fn end_rsync_up(&mut self) {
3895        self.rsync_up = Some(self.phase_start.elapsed());
3896        self.phase_start = Instant::now();
3897        tracing::info!(
3898            rsync_up_ms = %self.rsync_up.unwrap().as_millis(),
3899            "TIMING: rsync_up completed"
3900        );
3901    }
3902
3903    /// Mark the end of the remote build phase.
3904    pub fn end_remote_build(&mut self) {
3905        self.remote_build = Some(self.phase_start.elapsed());
3906        self.phase_start = Instant::now();
3907        tracing::info!(
3908            remote_build_ms = %self.remote_build.unwrap().as_millis(),
3909            "TIMING: remote_build completed"
3910        );
3911    }
3912
3913    /// Mark the end of the rsync download phase.
3914    pub fn end_rsync_down(&mut self) {
3915        self.rsync_down = Some(self.phase_start.elapsed());
3916        tracing::info!(
3917            rsync_down_ms = %self.rsync_down.unwrap().as_millis(),
3918            "TIMING: rsync_down completed"
3919        );
3920    }
3921
3922    /// Finish timing and produce metrics.
3923    pub fn finish(self, exit_code: i32, files: u64, bytes: u64) -> CompilationMetrics {
3924        let total = self.start.elapsed();
3925        tracing::info!(
3926            total_ms = %total.as_millis(),
3927            exit_code = %exit_code,
3928            "TIMING: compilation completed"
3929        );
3930
3931        CompilationMetrics {
3932            project_id: self.project_id,
3933            worker_id: self.worker_id,
3934            timestamp: chrono::Utc::now(),
3935            timing: CompilationTimingBreakdown {
3936                rsync_up: self.rsync_up.unwrap_or_default(),
3937                remote_build: self.remote_build.unwrap_or_default(),
3938                rsync_down: self.rsync_down.unwrap_or_default(),
3939                total,
3940            },
3941            local_build_time: None,
3942            speedup: None,
3943            files_synced: files,
3944            bytes_transferred: bytes,
3945            exit_code,
3946            success: exit_code == 0,
3947        }
3948    }
3949}
3950
3951/// Aggregator for compilation metrics with statistics.
3952#[derive(Debug)]
3953pub struct MetricsAggregator {
3954    history: VecDeque<CompilationMetrics>,
3955    max_history: usize,
3956}
3957
3958impl MetricsAggregator {
3959    /// Create a new aggregator with a maximum history size.
3960    pub fn new(max_history: usize) -> Self {
3961        Self {
3962            history: VecDeque::with_capacity(max_history),
3963            max_history,
3964        }
3965    }
3966
3967    /// Record a new compilation's metrics.
3968    pub fn record(&mut self, metrics: CompilationMetrics) {
3969        if self.history.len() >= self.max_history {
3970            self.history.pop_front();
3971        }
3972        self.history.push_back(metrics);
3973    }
3974
3975    /// Get the average speedup across all recorded compilations.
3976    pub fn average_speedup(&self) -> Option<f64> {
3977        let speedups: Vec<f64> = self.history.iter().filter_map(|m| m.speedup).collect();
3978
3979        if speedups.is_empty() {
3980            None
3981        } else {
3982            Some(speedups.iter().sum::<f64>() / speedups.len() as f64)
3983        }
3984    }
3985
3986    /// Get the p50 (median) total compilation time.
3987    pub fn p50_total_time(&self) -> Option<Duration> {
3988        self.percentile_total_time(0.50)
3989    }
3990
3991    /// Get the p95 total compilation time.
3992    pub fn p95_total_time(&self) -> Option<Duration> {
3993        self.percentile_total_time(0.95)
3994    }
3995
3996    /// Get the p99 total compilation time.
3997    pub fn p99_total_time(&self) -> Option<Duration> {
3998        self.percentile_total_time(0.99)
3999    }
4000
4001    /// Get a percentile of total compilation times.
4002    ///
4003    /// Uses the "inclusive" percentile method (matching numpy.percentile, Excel PERCENTILE.INC):
4004    /// index = (n - 1) * percentile, where n is the number of samples.
4005    fn percentile_total_time(&self, percentile: f64) -> Option<Duration> {
4006        let mut times: Vec<_> = self.history.iter().map(|m| m.timing.total).collect();
4007
4008        if times.is_empty() {
4009            return None;
4010        }
4011
4012        times.sort();
4013        // Use inclusive percentile: index = (n - 1) * p
4014        // For p50 with 100 items: (99 * 0.50) = 49.5 -> 49 (true median position)
4015        // For p99 with 100 items: (99 * 0.99) = 98.01 -> 98
4016        let idx = (((times.len() - 1) as f64 * percentile).round() as usize).min(times.len() - 1);
4017        Some(times[idx])
4018    }
4019
4020    /// Get the success rate as a percentage.
4021    pub fn success_rate(&self) -> f64 {
4022        if self.history.is_empty() {
4023            return 100.0;
4024        }
4025        let successes = self.history.iter().filter(|m| m.success).count();
4026        (successes as f64 / self.history.len() as f64) * 100.0
4027    }
4028
4029    /// Get the number of recorded compilations.
4030    pub fn count(&self) -> usize {
4031        self.history.len()
4032    }
4033
4034    /// Get all recorded metrics.
4035    pub fn metrics(&self) -> &VecDeque<CompilationMetrics> {
4036        &self.history
4037    }
4038
4039    /// Clear all recorded metrics.
4040    pub fn clear(&mut self) {
4041        self.history.clear();
4042    }
4043}
4044
4045impl Default for MetricsAggregator {
4046    fn default() -> Self {
4047        Self::new(1000)
4048    }
4049}
4050
4051#[cfg(test)]
4052mod tests {
4053    use super::*;
4054    use crate::test_guard;
4055
4056    #[test]
4057    fn test_declared_os_reads_reserved_tag() {
4058        assert_eq!(
4059            declared_os(&["rust".to_string(), "os:windows".to_string()]),
4060            Some("windows".to_string())
4061        );
4062        assert_eq!(
4063            declared_os(&["os:WINDOWS".to_string()]),
4064            Some("windows".to_string())
4065        );
4066    }
4067
4068    #[test]
4069    fn test_declared_os_absent_for_ordinary_tags() {
4070        // Every historical worker: no reserved tag, so no exclusivity and the
4071        // worker stays a candidate for any job.
4072        assert_eq!(declared_os(&[]), None);
4073        assert_eq!(
4074            declared_os(&["rust".to_string(), "bun".to_string(), "go".to_string()]),
4075            None
4076        );
4077        // A malformed declaration must not silently fence the worker off.
4078        assert_eq!(declared_os(&["os:".to_string()]), None);
4079        assert_eq!(declared_os(&["os:  ".to_string()]), None);
4080    }
4081
4082    #[test]
4083    fn test_os_tag_roundtrips_through_declared_os() {
4084        for os in ["windows", "Darwin", " linux "] {
4085            let tag = os_tag(os);
4086            assert_eq!(
4087                declared_os(std::slice::from_ref(&tag)).as_deref(),
4088                Some(os.trim().to_ascii_lowercase().as_str())
4089            );
4090        }
4091    }
4092
4093    /// Verdict-webhook config must NEVER carry an inline secret. Operators
4094    /// reference secrets by env-var name (`*_env`); the resolved value is
4095    /// read at dispatch time and never persisted to the cache or config TOML.
4096    /// This guards against a future careless `pub secret: String` addition.
4097    #[test]
4098    fn test_no_inline_secret_fields() {
4099        let endpoint = DoctorWebhookEndpoint {
4100            name: "ops".to_string(),
4101            url: "https://example.com/hook".to_string(),
4102            format: DoctorWebhookFormat::Pagerduty,
4103            on_transitions: vec!["any_to_failing".to_string()],
4104            retry_max: 3,
4105            retry_backoff_ms: 200,
4106            timeout_ms: 5000,
4107            enabled: true,
4108            routing_key_env: Some("RCH_PAGERDUTY_ROUTING_KEY".to_string()),
4109            bearer_token_env: Some("RCH_WEBHOOK_BEARER".to_string()),
4110            signing_secret_env: Some("RCH_WEBHOOK_HMAC".to_string()),
4111        };
4112        // Serialize to a JSON object and inspect the *keys*. The forbidden
4113        // field names must be entirely absent; only `*_env` references survive.
4114        let value = serde_json::to_value(&endpoint).unwrap();
4115        let obj = value.as_object().expect("endpoint serializes to an object");
4116        for forbidden in ["secret", "routing_key", "bearer_token", "password", "token"] {
4117            assert!(
4118                !obj.contains_key(forbidden),
4119                "endpoint config must not expose an inline `{forbidden}` field"
4120            );
4121        }
4122        assert!(obj.contains_key("routing_key_env"));
4123        assert!(obj.contains_key("bearer_token_env"));
4124        assert!(obj.contains_key("signing_secret_env"));
4125    }
4126
4127    #[test]
4128    fn test_webhook_format_toml_spelling() {
4129        // The config TOML uses snake_case spellings. TOML documents are tables
4130        // at the root, so a bare enum cannot be (de)serialized standalone —
4131        // verify the wire spelling through a `format = "..."` table field, the
4132        // shape it actually appears in within the config.
4133        #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)]
4134        struct FormatWrap {
4135            format: DoctorWebhookFormat,
4136        }
4137        let rendered = toml::to_string(&FormatWrap {
4138            format: DoctorWebhookFormat::GenericJson,
4139        })
4140        .unwrap();
4141        assert!(
4142            rendered.contains("format = \"generic_json\""),
4143            "unexpected TOML: {rendered}"
4144        );
4145        let slack: FormatWrap = toml::from_str("format = \"slack\"").unwrap();
4146        assert_eq!(slack.format, DoctorWebhookFormat::Slack);
4147        let pd: FormatWrap = toml::from_str("format = \"pagerduty\"").unwrap();
4148        assert_eq!(pd.format, DoctorWebhookFormat::Pagerduty);
4149    }
4150
4151    #[test]
4152    fn test_doctor_webhooks_defaults() {
4153        let cfg = DoctorWebhooksConfig::default();
4154        assert!(cfg.endpoints.is_empty());
4155        assert_eq!(cfg.max_inflight, 16);
4156        // RchConfig wires the section in with serde(default), so an empty
4157        // config still yields an empty (never-firing) webhook surface.
4158        let rch: RchConfig = toml::from_str("").unwrap();
4159        assert!(rch.doctor.webhooks.endpoints.is_empty());
4160    }
4161
4162    #[test]
4163    fn test_doctor_webhook_endpoint_parses_from_toml() {
4164        let toml_src = r#"
4165[doctor.webhooks]
4166max_inflight = 8
4167
4168[[doctor.webhooks.endpoints]]
4169name = "ops-slack"
4170url = "https://hooks.slack.com/services/T0/B0/x"
4171format = "slack"
4172on_transitions = ["healthy_to_degraded", "any_to_failing"]
4173retry_max = 2
4174"#;
4175        let cfg: RchConfig = toml::from_str(toml_src).unwrap();
4176        assert_eq!(cfg.doctor.webhooks.max_inflight, 8);
4177        assert_eq!(cfg.doctor.webhooks.endpoints.len(), 1);
4178        let ep = &cfg.doctor.webhooks.endpoints[0];
4179        assert_eq!(ep.name, "ops-slack");
4180        assert_eq!(ep.format, DoctorWebhookFormat::Slack);
4181        assert_eq!(ep.on_transitions.len(), 2);
4182        assert_eq!(ep.retry_max, 2);
4183        // Defaults filled in for omitted fields.
4184        assert_eq!(ep.retry_backoff_ms, 200);
4185        assert_eq!(ep.timeout_ms, 5000);
4186        assert!(ep.enabled);
4187    }
4188
4189    fn assert_golden_json(
4190        actual: serde_json::Value,
4191        expected: &serde_json::Value,
4192        fixture_path: &str,
4193    ) {
4194        if &actual == expected {
4195            return;
4196        }
4197
4198        let actual_json = serde_json::to_string_pretty(&actual).expect("actual JSON renders");
4199        let expected_json = serde_json::to_string_pretty(expected).expect("expected JSON renders");
4200        assert_eq!(
4201            &actual,
4202            expected,
4203            "golden mismatch in {fixture_path}\n\
4204             expected_blake3={}\n\
4205             actual_blake3={}\n\
4206             bless path: review the semantic diff, update the fixture expected_* field, \
4207             and record both hashes in the Beads closeout.\n\
4208             expected:\n{expected_json}\n\
4209             actual:\n{actual_json}",
4210            blake3::hash(expected_json.as_bytes()),
4211            blake3::hash(actual_json.as_bytes())
4212        );
4213    }
4214
4215    #[test]
4216    fn test_circuit_state_default() {
4217        let _guard = test_guard!();
4218        assert_eq!(CircuitState::default(), CircuitState::Closed);
4219    }
4220
4221    #[test]
4222    fn test_circuit_config_defaults() {
4223        let _guard = test_guard!();
4224        let config = CircuitBreakerConfig::default();
4225        assert_eq!(config.failure_threshold, 3);
4226        assert_eq!(config.success_threshold, 2);
4227        assert_eq!(config.error_rate_threshold, 0.5);
4228        assert_eq!(config.window_secs, 60);
4229        assert_eq!(config.open_cooldown_secs, 30);
4230        assert_eq!(config.half_open_max_probes, 1);
4231    }
4232
4233    #[test]
4234    fn test_rch_config_has_circuit_defaults() {
4235        let _guard = test_guard!();
4236        let config = RchConfig::default();
4237        assert_eq!(config.circuit.failure_threshold, 3);
4238    }
4239
4240    #[test]
4241    fn test_circuit_config_serde_roundtrip() {
4242        let _guard = test_guard!();
4243        let config = CircuitBreakerConfig {
4244            failure_threshold: 5,
4245            success_threshold: 3,
4246            error_rate_threshold: 0.75,
4247            window_secs: 120,
4248            open_cooldown_secs: 45,
4249            half_open_max_probes: 2,
4250            command_failure_threshold: 4,
4251            open_cooldown_max_secs: 900,
4252        };
4253        let json = serde_json::to_string(&config).unwrap();
4254        let parsed: CircuitBreakerConfig = serde_json::from_str(&json).unwrap();
4255        assert_eq!(parsed.command_failure_threshold, 4);
4256        assert_eq!(parsed.open_cooldown_max_secs, 900);
4257        assert_eq!(parsed.failure_threshold, 5);
4258        assert_eq!(parsed.success_threshold, 3);
4259        assert_eq!(parsed.error_rate_threshold, 0.75);
4260        assert_eq!(parsed.window_secs, 120);
4261        assert_eq!(parsed.open_cooldown_secs, 45);
4262        assert_eq!(parsed.half_open_max_probes, 2);
4263    }
4264
4265    // ========================================================================
4266    // WorkerCapabilities Tests
4267    // ========================================================================
4268
4269    #[test]
4270    fn test_worker_capabilities_default() {
4271        let _guard = test_guard!();
4272        let caps = WorkerCapabilities::default();
4273        assert!(caps.rustc_version.is_none());
4274        assert!(caps.rustup_toolchains.is_empty());
4275        assert!(caps.rustup_components.is_empty());
4276        assert!(caps.bun_version.is_none());
4277        assert!(caps.node_version.is_none());
4278        assert!(caps.npm_version.is_none());
4279        assert!(caps.projects_root_ok.is_none());
4280        assert!(caps.projects_root_issue.is_none());
4281        assert!(caps.projects_root_checked_at_unix_ms.is_none());
4282    }
4283
4284    #[test]
4285    fn test_worker_capabilities_new() {
4286        let _guard = test_guard!();
4287        let caps = WorkerCapabilities::new();
4288        assert!(caps.rustc_version.is_none());
4289        assert!(!caps.has_rust());
4290        assert!(!caps.has_bun());
4291        assert!(!caps.has_node());
4292        assert!(caps.is_topology_healthy().is_none());
4293    }
4294
4295    #[test]
4296    fn test_worker_capabilities_has_rust() {
4297        let _guard = test_guard!();
4298        let mut caps = WorkerCapabilities::new();
4299        assert!(!caps.has_rust());
4300
4301        caps.rustc_version = Some("rustc 1.76.0 (07dca489a 2024-02-04)".to_string());
4302        assert!(caps.has_rust());
4303    }
4304
4305    #[test]
4306    fn test_worker_capabilities_components_are_toolchain_scoped_and_exact() {
4307        let _guard = test_guard!();
4308        let caps = WorkerCapabilities {
4309            rustup_components: vec![
4310                "stable-x86_64-unknown-linux-gnu:clippy".to_string(),
4311                "nightly-2026-07-05-x86_64-unknown-linux-gnu:rustfmt".to_string(),
4312                "beta-x86_64-unknown-linux-gnu:clippy-preview".to_string(),
4313            ],
4314            ..Default::default()
4315        };
4316
4317        assert!(caps.has_rustup_component("stable", "clippy"));
4318        assert!(caps.has_rustup_component("nightly-2026-07-05", "rustfmt"));
4319        assert!(!caps.has_rustup_component("nightly-2026-07-05", "clippy"));
4320        assert!(!caps.has_rustup_component("beta", "clippy"));
4321        assert!(!caps.has_rustup_component("nightly-2026-07-0", "rustfmt"));
4322    }
4323
4324    #[test]
4325    fn test_worker_capabilities_has_bun() {
4326        let _guard = test_guard!();
4327        let mut caps = WorkerCapabilities::new();
4328        assert!(!caps.has_bun());
4329
4330        caps.bun_version = Some("1.0.25".to_string());
4331        assert!(caps.has_bun());
4332    }
4333
4334    #[test]
4335    fn test_worker_capabilities_has_nix() {
4336        let _guard = test_guard!();
4337        let mut caps = WorkerCapabilities::new();
4338        assert!(!caps.has_nix());
4339
4340        caps.nix_version = Some("nix (Nix) 2.24.9".to_string());
4341        assert!(caps.has_nix());
4342    }
4343
4344    #[test]
4345    fn test_worker_capabilities_has_zig_requires_both_halves() {
4346        let _guard = test_guard!();
4347        let mut caps = WorkerCapabilities::new();
4348        assert!(!caps.has_zig());
4349
4350        // `zig` alone cannot run `cargo zigbuild` — cargo would report
4351        // `no such command: 'zigbuild'`.
4352        caps.zig_version = Some("0.14.1".to_string());
4353        assert!(!caps.has_zig());
4354
4355        // cargo-zigbuild alone cannot link — it shells out to `zig cc`.
4356        caps.zig_version = None;
4357        caps.cargo_zigbuild_version = Some("cargo-zigbuild 0.23.0".to_string());
4358        assert!(!caps.has_zig());
4359
4360        caps.zig_version = Some("0.14.1".to_string());
4361        assert!(caps.has_zig());
4362    }
4363
4364    #[test]
4365    fn test_nix_build_uses_generous_timeout() {
4366        let _guard = test_guard!();
4367        let cfg = CompilationConfig::default();
4368        // Nix builds get the long test-timeout bucket, not the short build timeout.
4369        assert_eq!(
4370            cfg.timeout_for_kind(Some(CompilationKind::NixBuild)),
4371            std::time::Duration::from_secs(cfg.test_timeout_sec)
4372        );
4373    }
4374
4375    #[test]
4376    fn test_job_uses_build_timeout_bucket() {
4377        let _guard = test_guard!();
4378        // Jobs (explicit `rch exec --job`) reuse the standard build-timeout
4379        // bucket, so `RCH_BUILD_TIMEOUT_SEC` governs them without new config.
4380        let cfg = CompilationConfig::default();
4381        assert_eq!(
4382            cfg.timeout_for_kind(Some(CompilationKind::Job)),
4383            std::time::Duration::from_secs(cfg.build_timeout_sec)
4384        );
4385    }
4386
4387    #[test]
4388    fn test_layer0_disabled_by_default_and_env_gated() {
4389        let _guard = test_guard!();
4390        let cfg = Layer0Config::default();
4391        assert!(!cfg.enabled);
4392        // Master switch off ⇒ nothing injects even with knobs on.
4393        let knobs_on = Layer0Config {
4394            enabled: false,
4395            release_lto_thin: true,
4396            release_codegen_units_1: true,
4397        };
4398        assert!(cfg.active_env().is_empty());
4399        assert!(knobs_on.active_env().is_empty());
4400    }
4401
4402    #[test]
4403    fn test_layer0_knobs_emit_stable_env_pairs() {
4404        let _guard = test_guard!();
4405        let lto_only = Layer0Config {
4406            enabled: true,
4407            release_lto_thin: true,
4408            release_codegen_units_1: false,
4409        };
4410        assert_eq!(
4411            lto_only.active_env(),
4412            vec![("CARGO_PROFILE_RELEASE_LTO", "thin")]
4413        );
4414        let both = Layer0Config {
4415            enabled: true,
4416            release_lto_thin: true,
4417            release_codegen_units_1: true,
4418        };
4419        assert_eq!(
4420            both.active_env(),
4421            vec![
4422                ("CARGO_PROFILE_RELEASE_LTO", "thin"),
4423                ("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"),
4424            ],
4425            "emission order must stay stable for deterministic baselines"
4426        );
4427    }
4428
4429    #[test]
4430    fn test_layer0_pack_version_is_recorded() {
4431        let _guard = test_guard!();
4432        // Baselines record the pack version; it must exist and be non-empty.
4433        assert!(!LAYER0_PACK_VERSION.is_empty());
4434    }
4435    #[test]
4436    fn test_default_allowlist_includes_nix() {
4437        let _guard = test_guard!();
4438        let cfg = ExecutionConfig::default();
4439        assert!(cfg.is_allowed("nix"));
4440    }
4441
4442    #[test]
4443    fn test_worker_capabilities_has_node() {
4444        let _guard = test_guard!();
4445        let mut caps = WorkerCapabilities::new();
4446        assert!(!caps.has_node());
4447
4448        caps.node_version = Some("v20.11.0".to_string());
4449        assert!(caps.has_node());
4450    }
4451
4452    #[test]
4453    fn test_worker_capabilities_multiple_runtimes() {
4454        let _guard = test_guard!();
4455        let caps = WorkerCapabilities {
4456            rustc_version: Some("rustc 1.76.0".to_string()),
4457            bun_version: Some("1.0.25".to_string()),
4458            node_version: Some("v20.11.0".to_string()),
4459            npm_version: Some("10.2.4".to_string()),
4460            ..Default::default()
4461        };
4462
4463        assert!(caps.has_rust());
4464        assert!(caps.has_bun());
4465        assert!(caps.has_node());
4466    }
4467
4468    #[test]
4469    fn test_worker_capabilities_serialization_empty() {
4470        let _guard = test_guard!();
4471        let caps = WorkerCapabilities::new();
4472        let json = serde_json::to_string(&caps).unwrap();
4473        // Empty capabilities should serialize to {}
4474        assert_eq!(json, "{}");
4475
4476        // And deserialize back correctly
4477        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4478        assert!(!parsed.has_rust());
4479        assert!(!parsed.has_bun());
4480        assert!(!parsed.has_node());
4481    }
4482
4483    #[test]
4484    fn test_worker_capabilities_serialization_with_versions() {
4485        let _guard = test_guard!();
4486        let caps = WorkerCapabilities {
4487            rustc_version: Some("rustc 1.76.0-nightly".to_string()),
4488            bun_version: Some("1.0.25".to_string()),
4489            node_version: None,
4490            npm_version: None,
4491            ..Default::default()
4492        };
4493
4494        let json = serde_json::to_string(&caps).unwrap();
4495        assert!(json.contains("rustc_version"));
4496        assert!(json.contains("rustc 1.76.0-nightly"));
4497        assert!(json.contains("bun_version"));
4498        assert!(json.contains("1.0.25"));
4499        // None fields should be skipped
4500        assert!(!json.contains("node_version"));
4501        assert!(!json.contains("npm_version"));
4502
4503        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4504        assert!(parsed.has_rust());
4505        assert!(parsed.has_bun());
4506        assert!(!parsed.has_node());
4507    }
4508
4509    #[test]
4510    fn test_worker_capabilities_deserialization_partial() {
4511        let _guard = test_guard!();
4512        // Deserialize JSON with only some fields
4513        let json = r#"{"bun_version": "1.0.0"}"#;
4514        let caps: WorkerCapabilities = serde_json::from_str(json).unwrap();
4515
4516        assert!(!caps.has_rust());
4517        assert!(caps.has_bun());
4518        assert!(!caps.has_node());
4519        assert_eq!(caps.bun_version, Some("1.0.0".to_string()));
4520    }
4521
4522    #[test]
4523    fn test_worker_capabilities_clone() {
4524        let _guard = test_guard!();
4525        let caps = WorkerCapabilities {
4526            rustc_version: Some("1.76.0".to_string()),
4527            bun_version: None,
4528            node_version: Some("v20".to_string()),
4529            npm_version: None,
4530            projects_root_ok: Some(false),
4531            projects_root_issue: Some("alias_missing".to_string()),
4532            projects_root_checked_at_unix_ms: Some(123),
4533            ..Default::default()
4534        };
4535
4536        let cloned = caps.clone();
4537        assert_eq!(cloned.rustc_version, caps.rustc_version);
4538        assert_eq!(cloned.bun_version, caps.bun_version);
4539        assert_eq!(cloned.node_version, caps.node_version);
4540        assert_eq!(cloned.npm_version, caps.npm_version);
4541        assert_eq!(cloned.projects_root_ok, caps.projects_root_ok);
4542        assert_eq!(cloned.projects_root_issue, caps.projects_root_issue);
4543        assert_eq!(
4544            cloned.projects_root_checked_at_unix_ms,
4545            caps.projects_root_checked_at_unix_ms
4546        );
4547    }
4548
4549    #[test]
4550    fn test_worker_capabilities_topology_health_helper() {
4551        let _guard = test_guard!();
4552        let mut caps = WorkerCapabilities::new();
4553        assert_eq!(caps.is_topology_healthy(), None);
4554
4555        caps.projects_root_ok = Some(true);
4556        assert_eq!(caps.is_topology_healthy(), Some(true));
4557
4558        caps.projects_root_ok = Some(false);
4559        assert_eq!(caps.is_topology_healthy(), Some(false));
4560    }
4561
4562    #[test]
4563    fn test_selection_reason_serialization() {
4564        let _guard = test_guard!();
4565        // Test that all variants serialize to snake_case
4566        assert_eq!(
4567            serde_json::to_string(&SelectionReason::Success).unwrap(),
4568            "\"success\""
4569        );
4570        assert_eq!(
4571            serde_json::to_string(&SelectionReason::NoWorkersConfigured).unwrap(),
4572            "\"no_workers_configured\""
4573        );
4574        assert_eq!(
4575            serde_json::to_string(&SelectionReason::AllWorkersUnreachable).unwrap(),
4576            "\"all_workers_unreachable\""
4577        );
4578        assert_eq!(
4579            serde_json::to_string(&SelectionReason::AllCircuitsOpen).unwrap(),
4580            "\"all_circuits_open\""
4581        );
4582        assert_eq!(
4583            serde_json::to_string(&SelectionReason::AllWorkersBusy).unwrap(),
4584            "\"all_workers_busy\""
4585        );
4586        assert_eq!(
4587            serde_json::to_string(&SelectionReason::NoWorkersPassedHealth).unwrap(),
4588            "\"no_workers_passed_health\""
4589        );
4590        assert_eq!(
4591            serde_json::to_string(&SelectionReason::AllWorkersFailedPreflight).unwrap(),
4592            "\"all_workers_failed_preflight\""
4593        );
4594        assert_eq!(
4595            serde_json::to_string(&SelectionReason::AllWorkersFailedConvergence).unwrap(),
4596            "\"all_workers_failed_convergence\""
4597        );
4598        assert_eq!(
4599            serde_json::to_string(&SelectionReason::NoAdmissibleWorkers(
4600                "critical_pressure=1,insufficient_slots=1".to_string()
4601            ))
4602            .unwrap(),
4603            "{\"no_admissible_workers\":\"critical_pressure=1,insufficient_slots=1\"}"
4604        );
4605        assert_eq!(
4606            serde_json::to_string(&SelectionReason::NoMatchingWorkers).unwrap(),
4607            "\"no_matching_workers\""
4608        );
4609    }
4610
4611    #[test]
4612    fn test_selection_reason_with_error() {
4613        let _guard = test_guard!();
4614        let reason = SelectionReason::SelectionError("test error".to_string());
4615        let json = serde_json::to_string(&reason).unwrap();
4616        assert!(json.contains("selection_error"));
4617        assert!(json.contains("test error"));
4618    }
4619
4620    #[test]
4621    fn test_selection_reason_deserialization() {
4622        let _guard = test_guard!();
4623        assert_eq!(
4624            serde_json::from_str::<SelectionReason>("\"success\"").unwrap(),
4625            SelectionReason::Success
4626        );
4627        assert_eq!(
4628            serde_json::from_str::<SelectionReason>("\"all_workers_busy\"").unwrap(),
4629            SelectionReason::AllWorkersBusy
4630        );
4631        assert_eq!(
4632            serde_json::from_str::<SelectionReason>("\"no_workers_passed_health\"").unwrap(),
4633            SelectionReason::NoWorkersPassedHealth
4634        );
4635        assert_eq!(
4636            serde_json::from_str::<SelectionReason>("\"all_workers_failed_preflight\"").unwrap(),
4637            SelectionReason::AllWorkersFailedPreflight
4638        );
4639        assert_eq!(
4640            serde_json::from_str::<SelectionReason>("\"all_workers_failed_convergence\"").unwrap(),
4641            SelectionReason::AllWorkersFailedConvergence
4642        );
4643        assert_eq!(
4644            serde_json::from_str::<SelectionReason>(
4645                "{\"no_admissible_workers\":\"critical_pressure=1\"}"
4646            )
4647            .unwrap(),
4648            SelectionReason::NoAdmissibleWorkers("critical_pressure=1".to_string())
4649        );
4650    }
4651
4652    #[test]
4653    fn test_selection_reason_display() {
4654        let _guard = test_guard!();
4655        assert_eq!(
4656            SelectionReason::Success.to_string(),
4657            "worker assigned successfully"
4658        );
4659        assert_eq!(
4660            SelectionReason::NoWorkersConfigured.to_string(),
4661            "no workers configured"
4662        );
4663        assert_eq!(
4664            SelectionReason::AllWorkersUnreachable.to_string(),
4665            "all workers unreachable"
4666        );
4667        assert_eq!(
4668            SelectionReason::AllWorkersBusy.to_string(),
4669            "all workers at capacity"
4670        );
4671        assert_eq!(
4672            SelectionReason::NoWorkersPassedHealth.to_string(),
4673            "no workers passed health thresholds"
4674        );
4675        assert_eq!(
4676            SelectionReason::AllWorkersFailedPreflight.to_string(),
4677            "all workers failed preflight checks"
4678        );
4679        assert_eq!(
4680            SelectionReason::AllWorkersFailedConvergence.to_string(),
4681            "all workers failed repo convergence checks"
4682        );
4683        assert_eq!(
4684            SelectionReason::NoAdmissibleWorkers(
4685                "critical_pressure=1,insufficient_slots=1".to_string()
4686            )
4687            .to_string(),
4688            "no admissible workers: critical_pressure=1,insufficient_slots=1"
4689        );
4690        assert_eq!(
4691            SelectionReason::SelectionError("oops".to_string()).to_string(),
4692            "selection error: oops"
4693        );
4694    }
4695
4696    #[test]
4697    fn test_selection_response_with_worker() {
4698        let _guard = test_guard!();
4699        let response = SelectionResponse {
4700            worker: Some(SelectedWorker {
4701                id: WorkerId::new("test"),
4702                host: "localhost".to_string(),
4703                user: "user".to_string(),
4704                identity_file: "~/.ssh/id_rsa".to_string(),
4705                slots_available: 8,
4706                speed_score: 75.0,
4707                declared_os: None,
4708            }),
4709            reason: SelectionReason::Success,
4710            build_id: None,
4711            diagnostics: None,
4712        };
4713
4714        let json = serde_json::to_string(&response).unwrap();
4715        assert!(json.contains("\"reason\":\"success\""));
4716        assert!(json.contains("\"id\":\"test\""));
4717    }
4718
4719    #[test]
4720    fn test_selection_response_without_worker() {
4721        let _guard = test_guard!();
4722        let response = SelectionResponse {
4723            worker: None,
4724            reason: SelectionReason::AllWorkersBusy,
4725            build_id: None,
4726            diagnostics: None,
4727        };
4728
4729        let json = serde_json::to_string(&response).unwrap();
4730        assert!(json.contains("\"worker\":null"));
4731        assert!(json.contains("\"reason\":\"all_workers_busy\""));
4732    }
4733
4734    #[test]
4735    fn golden_selection_response_with_no_workers_passed_health_diagnostics() {
4736        let _guard = test_guard!();
4737        const FIXTURE_PATH: &str =
4738            "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json";
4739        let fixture: serde_json::Value = serde_json::from_str(include_str!(
4740            "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json"
4741        ))
4742        .expect("selection diagnostics golden fixture parses");
4743        assert_eq!(
4744            fixture["schema_version"],
4745            "rch.golden.selection_diagnostics.v1"
4746        );
4747
4748        let response = SelectionResponse {
4749            worker: None,
4750            reason: SelectionReason::NoWorkersPassedHealth,
4751            build_id: None,
4752            diagnostics: Some(SelectionDiagnostics {
4753                required_runtime: RequiredRuntime::Rust,
4754                estimated_cores: 2,
4755                min_success_rate: 0.8,
4756                fallback_min_success_rate: 0.5,
4757                active_project_exclusion_count: 0,
4758                workers: vec![WorkerSelectionDiagnostic {
4759                    worker_id: WorkerId::new("fixture-worker"),
4760                    status: "healthy".to_string(),
4761                    circuit_state: "closed".to_string(),
4762                    pressure_state: "healthy".to_string(),
4763                    pressure_reason_code: "pressure.ok".to_string(),
4764                    success_rate: Some(0.2),
4765                    min_success_rate: 0.8,
4766                    fallback_min_success_rate: 0.5,
4767                    required_runtime: RequiredRuntime::Rust,
4768                    runtime_available: true,
4769                    available_slots: 8,
4770                    total_slots: 8,
4771                    estimated_cores: 2,
4772                    active_project_excluded: false,
4773                    final_decision: WorkerSelectionDiagnosticDecision::Deny,
4774                    final_reason: "success_rate 0.20 < fallback 0.50".to_string(),
4775                    reason_codes: vec![
4776                        "health.below_min_success_rate".to_string(),
4777                        "health.below_fallback_min_success_rate".to_string(),
4778                    ],
4779                }],
4780            }),
4781        };
4782
4783        assert_golden_json(
4784            serde_json::to_value(&response).expect("response serializes"),
4785            &fixture["expected_response"],
4786            FIXTURE_PATH,
4787        );
4788    }
4789
4790    #[test]
4791    fn test_selection_response_roundtrip() {
4792        let _guard = test_guard!();
4793        let original = SelectionResponse {
4794            worker: Some(SelectedWorker {
4795                id: WorkerId::new("worker-1"),
4796                host: "192.168.1.100".to_string(),
4797                user: "ubuntu".to_string(),
4798                identity_file: "/path/to/key".to_string(),
4799                slots_available: 16,
4800                speed_score: 90.5,
4801                declared_os: None,
4802            }),
4803            reason: SelectionReason::Success,
4804            build_id: None,
4805            diagnostics: None,
4806        };
4807
4808        let json = serde_json::to_string(&original).unwrap();
4809        let parsed: SelectionResponse = serde_json::from_str(&json).unwrap();
4810
4811        assert!(parsed.worker.is_some());
4812        let worker = parsed.worker.unwrap();
4813        assert_eq!(worker.id.as_str(), "worker-1");
4814        assert_eq!(worker.host, "192.168.1.100");
4815        assert_eq!(worker.slots_available, 16);
4816        assert_eq!(parsed.reason, SelectionReason::Success);
4817    }
4818
4819    #[test]
4820    fn test_build_heartbeat_phase_roundtrip() {
4821        let _guard = test_guard!();
4822        let json = serde_json::to_string(&BuildHeartbeatPhase::SyncUp).unwrap();
4823        assert_eq!(json, "\"sync_up\"");
4824        let parsed: BuildHeartbeatPhase = serde_json::from_str(&json).unwrap();
4825        assert_eq!(parsed, BuildHeartbeatPhase::SyncUp);
4826    }
4827
4828    #[test]
4829    fn test_build_heartbeat_request_roundtrip() {
4830        let _guard = test_guard!();
4831        let request = BuildHeartbeatRequest {
4832            build_id: 42,
4833            worker_id: WorkerId::new("worker-a"),
4834            hook_pid: Some(12345),
4835            local_wrapper_id: Some("wrapper-abc123".to_string()),
4836            remote_pgid_file: Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string()),
4837            phase: BuildHeartbeatPhase::Execute,
4838            detail: Some("Compiling crates".to_string()),
4839            progress_counter: Some(7),
4840            progress_percent: Some(42.5),
4841        };
4842
4843        let json = serde_json::to_string(&request).unwrap();
4844        let parsed: BuildHeartbeatRequest = serde_json::from_str(&json).unwrap();
4845        assert_eq!(parsed.build_id, 42);
4846        assert_eq!(parsed.worker_id.as_str(), "worker-a");
4847        assert_eq!(
4848            parsed.remote_pgid_file,
4849            Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string())
4850        );
4851        assert_eq!(parsed.phase, BuildHeartbeatPhase::Execute);
4852        assert_eq!(parsed.progress_counter, Some(7));
4853        assert_eq!(parsed.progress_percent, Some(42.5));
4854    }
4855
4856    // CircuitStats tests
4857
4858    #[test]
4859    fn test_circuit_stats_new() {
4860        let _guard = test_guard!();
4861        let stats = CircuitStats::new();
4862        assert_eq!(stats.state(), CircuitState::Closed);
4863        assert_eq!(stats.consecutive_failures(), 0);
4864        assert_eq!(stats.error_rate(), 0.0);
4865        assert!(stats.opened_at().is_none());
4866    }
4867
4868    #[test]
4869    fn test_circuit_stats_record_success() {
4870        let _guard = test_guard!();
4871        let mut stats = CircuitStats::new();
4872        stats.record_success();
4873        stats.record_success();
4874        assert_eq!(stats.error_rate(), 0.0);
4875        assert_eq!(stats.consecutive_failures(), 0);
4876    }
4877
4878    #[test]
4879    fn test_circuit_stats_record_failure() {
4880        let _guard = test_guard!();
4881        let mut stats = CircuitStats::new();
4882        stats.record_failure();
4883        stats.record_failure();
4884        assert_eq!(stats.consecutive_failures(), 2);
4885        assert_eq!(stats.error_rate(), 1.0); // 2 failures, 0 successes
4886    }
4887
4888    #[test]
4889    fn test_circuit_stats_error_rate() {
4890        let _guard = test_guard!();
4891        let mut stats = CircuitStats::new();
4892        stats.record_success();
4893        stats.record_success();
4894        stats.record_failure();
4895        // 1 failure / 3 total = 0.333...
4896        assert!((stats.error_rate() - 0.333).abs() < 0.01);
4897    }
4898
4899    #[test]
4900    fn test_circuit_stats_should_open_consecutive_failures() {
4901        let _guard = test_guard!();
4902        let mut stats = CircuitStats::new();
4903        let config = CircuitBreakerConfig::default(); // threshold = 3
4904
4905        stats.record_failure();
4906        assert!(!stats.should_open(&config));
4907
4908        stats.record_failure();
4909        assert!(!stats.should_open(&config));
4910
4911        stats.record_failure();
4912        assert!(stats.should_open(&config)); // 3 consecutive failures
4913    }
4914
4915    #[test]
4916    fn test_circuit_stats_should_open_error_rate() {
4917        let _guard = test_guard!();
4918        let mut stats = CircuitStats::new();
4919        let config = CircuitBreakerConfig {
4920            error_rate_threshold: 0.5,
4921            ..Default::default()
4922        };
4923
4924        // Need at least 5 samples
4925        stats.record_success();
4926        stats.record_success();
4927        stats.record_failure();
4928        stats.record_failure();
4929        assert!(!stats.should_open(&config)); // Only 4 samples
4930
4931        stats.record_failure(); // 5 samples: 2 success, 3 failures = 60% error rate
4932        assert!(stats.should_open(&config));
4933    }
4934
4935    #[test]
4936    fn test_circuit_close_clears_recent_results_no_flap() {
4937        let _guard = test_guard!();
4938        // Regression: error_rate() is computed over recent_results, so close()
4939        // must clear that history or a recovered circuit immediately re-opens
4940        // on stale pre-failure samples (flapping).
4941        let mut stats = CircuitStats::new();
4942        let config = CircuitBreakerConfig {
4943            error_rate_threshold: 0.5,
4944            ..Default::default()
4945        };
4946        for _ in 0..6 {
4947            stats.record_failure();
4948        }
4949        assert!(stats.should_open(&config));
4950        stats.open();
4951        // Recover via half-open probes, then close.
4952        stats.half_open();
4953        stats.record_success();
4954        stats.record_success();
4955        stats.close();
4956        // Error rate must be reset and the circuit must NOT immediately re-open.
4957        assert_eq!(stats.error_rate(), 0.0);
4958        assert!(!stats.should_open(&config));
4959    }
4960
4961    #[test]
4962    fn test_apply_health_outcome_opens_on_failure_threshold() {
4963        let _guard = test_guard!();
4964        // failure_threshold=3: three unhealthy outcomes open the circuit.
4965        let config = CircuitBreakerConfig {
4966            failure_threshold: 3,
4967            open_cooldown_secs: 300,
4968            ..Default::default()
4969        };
4970        let mut stats = CircuitStats::new();
4971        assert_eq!(
4972            stats.apply_health_outcome(false, false, &config),
4973            CircuitState::Closed
4974        );
4975        assert_eq!(
4976            stats.apply_health_outcome(false, false, &config),
4977            CircuitState::Closed
4978        );
4979        assert_eq!(
4980            stats.apply_health_outcome(false, false, &config),
4981            CircuitState::Open
4982        );
4983    }
4984
4985    #[test]
4986    fn test_apply_health_outcome_half_open_two_probe_recovery() {
4987        let _guard = test_guard!();
4988        // success_threshold=2: recovery from Open needs exactly two clean probes.
4989        // open_cooldown_secs=0 makes should_half_open() true immediately, so the
4990        // engine's trailing nudge leaves an opened circuit already in HalfOpen —
4991        // exactly the state a real daemon would be in on the next poll once the
4992        // cooldown had elapsed. From HalfOpen the engine credits successes.
4993        let config = CircuitBreakerConfig {
4994            failure_threshold: 3,
4995            success_threshold: 2,
4996            open_cooldown_secs: 0,
4997            ..Default::default()
4998        };
4999        let mut stats = CircuitStats::new();
5000        for _ in 0..3 {
5001            stats.apply_health_outcome(false, false, &config);
5002        }
5003        // With a 0s cooldown the engine nudges Open -> HalfOpen immediately.
5004        assert_eq!(stats.state(), CircuitState::HalfOpen);
5005
5006        // First healthy probe in half-open: one success credited, not yet closed.
5007        let after_first = stats.apply_health_outcome(true, false, &config);
5008        assert_eq!(after_first, CircuitState::HalfOpen);
5009
5010        // Second healthy probe: threshold met -> Closed.
5011        let after_second = stats.apply_health_outcome(true, false, &config);
5012        assert_eq!(after_second, CircuitState::Closed);
5013    }
5014
5015    #[test]
5016    fn test_apply_health_outcome_transient_half_open_does_not_reopen() {
5017        let _guard = test_guard!();
5018        // A transient failure while half-open must NOT reset the recovery streak
5019        // — a network blip can't undo an in-progress recovery.
5020        let config = CircuitBreakerConfig {
5021            failure_threshold: 3,
5022            success_threshold: 2,
5023            open_cooldown_secs: 0,
5024            ..Default::default()
5025        };
5026        let mut stats = CircuitStats::new();
5027        for _ in 0..3 {
5028            stats.apply_health_outcome(false, false, &config);
5029        }
5030        // 0s cooldown -> already nudged to HalfOpen.
5031        assert_eq!(stats.state(), CircuitState::HalfOpen);
5032
5033        // One clean probe (1 of 2 needed).
5034        assert_eq!(
5035            stats.apply_health_outcome(true, false, &config),
5036            CircuitState::HalfOpen
5037        );
5038
5039        // Transient failure in half-open: stays HalfOpen (does NOT reopen the
5040        // circuit or restart the cooldown). The key contrast with a real failure
5041        // is that the circuit does not drop back to Open — probing continues in
5042        // place, so recovery resumes immediately on the next clean probes rather
5043        // than waiting out a fresh cooldown.
5044        let after_transient = stats.apply_health_outcome(false, true, &config);
5045        assert_eq!(after_transient, CircuitState::HalfOpen);
5046
5047        // Recovery still completes from half-open with the required clean probes.
5048        assert_eq!(
5049            stats.apply_health_outcome(true, false, &config),
5050            CircuitState::HalfOpen
5051        );
5052        assert_eq!(
5053            stats.apply_health_outcome(true, false, &config),
5054            CircuitState::Closed
5055        );
5056    }
5057
5058    #[test]
5059    fn test_apply_health_outcome_real_vs_transient_half_open_failure() {
5060        let _guard = test_guard!();
5061        // With a positive cooldown the difference is directly observable: a REAL
5062        // half-open failure reopens the circuit (state -> Open, must wait out the
5063        // cooldown again), whereas a TRANSIENT half-open failure keeps it in
5064        // HalfOpen so recovery continues without a fresh cooldown.
5065        //
5066        // Manually drive to HalfOpen (open() + half_open()) so the positive
5067        // cooldown doesn't force a real wallclock wait to reach half-open.
5068        let config = CircuitBreakerConfig {
5069            failure_threshold: 3,
5070            success_threshold: 2,
5071            open_cooldown_secs: 300,
5072            ..Default::default()
5073        };
5074
5075        // Real failure path.
5076        let mut real = CircuitStats::new();
5077        real.open();
5078        real.half_open();
5079        assert_eq!(real.state(), CircuitState::HalfOpen);
5080        let after_real = real.apply_health_outcome(false, false, &config);
5081        assert_eq!(
5082            after_real,
5083            CircuitState::Open,
5084            "a real half-open failure must reopen the circuit"
5085        );
5086
5087        // Transient failure path (same starting state).
5088        let mut transient = CircuitStats::new();
5089        transient.open();
5090        transient.half_open();
5091        assert_eq!(transient.state(), CircuitState::HalfOpen);
5092        let after_transient = transient.apply_health_outcome(false, true, &config);
5093        assert_eq!(
5094            after_transient,
5095            CircuitState::HalfOpen,
5096            "a transient half-open failure must NOT reopen the circuit"
5097        );
5098    }
5099
5100    /// Issue #48 regression: a worker whose liveness probe (`echo` over a warm
5101    /// session) keeps succeeding while every real command times out must NOT
5102    /// be re-admitted. Liveness successes alone cannot close a half-open
5103    /// circuit while command-class evidence is outstanding.
5104    #[test]
5105    fn test_command_failure_blocks_half_open_close_on_liveness_successes() {
5106        let _guard = test_guard!();
5107        let config = CircuitBreakerConfig {
5108            failure_threshold: 3,
5109            success_threshold: 2,
5110            open_cooldown_secs: 0,
5111            command_failure_threshold: 2,
5112            ..Default::default()
5113        };
5114        let mut stats = CircuitStats::new();
5115        stats.open();
5116        stats.half_open();
5117
5118        // The half-open trial: a real command hangs on the worker.
5119        assert_eq!(
5120            stats.apply_command_outcome(false, &config),
5121            CircuitState::HalfOpen,
5122            "cooldown 0: reopen then immediately half-open again"
5123        );
5124        assert_eq!(stats.consecutive_command_failures(), 1);
5125
5126        // Liveness keeps answering — many times over.
5127        for _ in 0..10 {
5128            assert_eq!(
5129                stats.apply_health_outcome(true, false, &config),
5130                CircuitState::HalfOpen,
5131                "liveness successes must not close the circuit while a command failure is outstanding"
5132            );
5133        }
5134
5135        // Only a command-class success clears the evidence and recovers.
5136        assert_eq!(
5137            stats.apply_command_outcome(true, &config),
5138            CircuitState::Closed
5139        );
5140        assert_eq!(stats.consecutive_command_failures(), 0);
5141        assert_eq!(stats.reopen_streak(), 0);
5142    }
5143
5144    /// Issue #48: interleaved liveness successes reset `failure_threshold`, so
5145    /// command failures need their own threshold to open a Closed circuit.
5146    #[test]
5147    fn test_command_failures_open_closed_circuit_despite_liveness_successes() {
5148        let _guard = test_guard!();
5149        let config = CircuitBreakerConfig {
5150            failure_threshold: 3,
5151            command_failure_threshold: 2,
5152            open_cooldown_secs: 300,
5153            ..Default::default()
5154        };
5155        let mut stats = CircuitStats::new();
5156
5157        assert_eq!(
5158            stats.apply_command_outcome(false, &config),
5159            CircuitState::Closed
5160        );
5161        // Two liveness successes in between: consecutive_failures is reset...
5162        stats.apply_health_outcome(true, false, &config);
5163        stats.apply_health_outcome(true, false, &config);
5164        assert_eq!(stats.consecutive_failures(), 0);
5165        // ...but the command-failure count is not, and the second one opens.
5166        assert_eq!(
5167            stats.apply_command_outcome(false, &config),
5168            CircuitState::Open
5169        );
5170        assert_eq!(stats.reopen_streak(), 1);
5171    }
5172
5173    /// A command failure in HalfOpen always reopens: "connected but the
5174    /// command never completed" is exactly what the trial exists to detect,
5175    /// so there is no transient exemption on this channel.
5176    #[test]
5177    fn test_command_failure_in_half_open_always_reopens() {
5178        let _guard = test_guard!();
5179        let config = CircuitBreakerConfig {
5180            open_cooldown_secs: 300,
5181            ..Default::default()
5182        };
5183        let mut stats = CircuitStats::new();
5184        stats.open();
5185        stats.half_open();
5186        assert_eq!(
5187            stats.apply_command_outcome(false, &config),
5188            CircuitState::Open
5189        );
5190        assert_eq!(stats.reopen_streak(), 2);
5191    }
5192
5193    /// Repeated Open -> HalfOpen -> Open cycles lengthen the cooldown
5194    /// (exponential, capped) so a flapping worker converges to Open instead of
5195    /// being re-admitted on every base cooldown.
5196    #[test]
5197    fn test_reopen_streak_backs_off_cooldown_until_cap() {
5198        let _guard = test_guard!();
5199        let config = CircuitBreakerConfig {
5200            open_cooldown_secs: 30,
5201            open_cooldown_max_secs: 100,
5202            ..Default::default()
5203        };
5204        assert_eq!(config.cooldown_ms_for_streak(0), 30_000);
5205        assert_eq!(config.cooldown_ms_for_streak(1), 30_000);
5206        assert_eq!(config.cooldown_ms_for_streak(2), 60_000);
5207        assert_eq!(config.cooldown_ms_for_streak(3), 100_000, "capped");
5208        assert_eq!(config.cooldown_ms_for_streak(40), 100_000, "no overflow");
5209
5210        // A cap below the base never shortens the base cooldown.
5211        let inverted = CircuitBreakerConfig {
5212            open_cooldown_secs: 30,
5213            open_cooldown_max_secs: 5,
5214            ..Default::default()
5215        };
5216        assert_eq!(inverted.cooldown_ms_for_streak(3), 30_000);
5217
5218        // The streak is observable through recovery_remaining_secs: after two
5219        // opens the remaining cooldown exceeds the base.
5220        let mut stats = CircuitStats::new();
5221        stats.open();
5222        stats.half_open();
5223        stats.open();
5224        assert_eq!(stats.reopen_streak(), 2);
5225        let remaining = stats.recovery_remaining_secs(&config).unwrap();
5226        assert!(
5227            remaining > 30 && remaining <= 60,
5228            "second open should wait ~60s, got {remaining}"
5229        );
5230
5231        // Closing resets the streak, so the next open starts from the base.
5232        stats.close();
5233        assert_eq!(stats.reopen_streak(), 0);
5234        stats.open();
5235        assert!(stats.recovery_remaining_secs(&config).unwrap() <= 30);
5236    }
5237
5238    #[test]
5239    fn test_circuit_stats_success_resets_consecutive_failures() {
5240        let _guard = test_guard!();
5241        let mut stats = CircuitStats::new();
5242        stats.record_failure();
5243        stats.record_failure();
5244        assert_eq!(stats.consecutive_failures(), 2);
5245
5246        stats.record_success();
5247        assert_eq!(stats.consecutive_failures(), 0);
5248    }
5249
5250    #[test]
5251    fn test_circuit_stats_open_transition() {
5252        let _guard = test_guard!();
5253        let mut stats = CircuitStats::new();
5254        stats.open();
5255
5256        assert_eq!(stats.state(), CircuitState::Open);
5257        assert!(stats.opened_at().is_some());
5258    }
5259
5260    #[test]
5261    fn test_circuit_stats_half_open_transition() {
5262        let _guard = test_guard!();
5263        let mut stats = CircuitStats::new();
5264        stats.open();
5265        stats.half_open();
5266
5267        assert_eq!(stats.state(), CircuitState::HalfOpen);
5268    }
5269
5270    #[test]
5271    fn test_circuit_stats_close_transition() {
5272        let _guard = test_guard!();
5273        let mut stats = CircuitStats::new();
5274        stats.open();
5275        stats.half_open();
5276        stats.close();
5277
5278        assert_eq!(stats.state(), CircuitState::Closed);
5279        assert!(stats.opened_at().is_none());
5280        assert_eq!(stats.consecutive_failures(), 0);
5281    }
5282
5283    #[test]
5284    fn test_circuit_stats_should_close() {
5285        let _guard = test_guard!();
5286        let mut stats = CircuitStats::new();
5287        let config = CircuitBreakerConfig {
5288            success_threshold: 2,
5289            ..Default::default()
5290        };
5291
5292        stats.open();
5293        stats.half_open();
5294
5295        assert!(!stats.should_close(&config)); // 0 successes
5296
5297        stats.record_success();
5298        assert!(!stats.should_close(&config)); // 1 success
5299
5300        stats.record_success();
5301        assert!(stats.should_close(&config)); // 2 successes
5302    }
5303
5304    #[test]
5305    fn test_circuit_stats_can_probe() {
5306        let _guard = test_guard!();
5307        let mut stats = CircuitStats::new();
5308        let config = CircuitBreakerConfig {
5309            half_open_max_probes: 1,
5310            ..Default::default()
5311        };
5312
5313        // Can't probe when closed
5314        assert!(!stats.can_probe(&config));
5315
5316        stats.open();
5317        // Can't probe when open
5318        assert!(!stats.can_probe(&config));
5319
5320        stats.half_open();
5321        // Can probe when half-open
5322        assert!(stats.can_probe(&config));
5323
5324        // Start a probe
5325        assert!(stats.start_probe(&config));
5326        // Can't start another probe
5327        assert!(!stats.can_probe(&config));
5328        assert!(!stats.start_probe(&config));
5329    }
5330
5331    #[test]
5332    fn test_circuit_stats_probe_completion() {
5333        let _guard = test_guard!();
5334        let mut stats = CircuitStats::new();
5335        let config = CircuitBreakerConfig {
5336            half_open_max_probes: 1,
5337            ..Default::default()
5338        };
5339
5340        stats.open();
5341        stats.half_open();
5342        stats.start_probe(&config);
5343
5344        // Probe completes with success
5345        stats.record_success();
5346
5347        // Can start another probe
5348        assert!(stats.can_probe(&config));
5349    }
5350
5351    #[test]
5352    fn test_circuit_stats_failure_in_half_open() {
5353        let _guard = test_guard!();
5354        let mut stats = CircuitStats::new();
5355        let config = CircuitBreakerConfig {
5356            success_threshold: 2,
5357            ..Default::default()
5358        };
5359
5360        stats.open();
5361        stats.half_open();
5362
5363        stats.record_success();
5364        assert_eq!(stats.consecutive_failures(), 0);
5365
5366        // Failure resets consecutive successes
5367        stats.record_failure();
5368        assert!(!stats.should_close(&config));
5369
5370        // Need 2 more successes
5371        stats.record_success();
5372        stats.record_success();
5373        assert!(stats.should_close(&config));
5374    }
5375
5376    #[test]
5377    fn test_circuit_stats_reset_window() {
5378        let _guard = test_guard!();
5379        let mut stats = CircuitStats::new();
5380        stats.record_success();
5381        stats.record_failure();
5382
5383        assert!(stats.error_rate() > 0.0);
5384
5385        stats.reset_window();
5386        assert_eq!(stats.error_rate(), 0.0);
5387    }
5388
5389    #[test]
5390    fn test_circuit_state_transitions_are_deterministic() {
5391        let _guard = test_guard!();
5392        let config = CircuitBreakerConfig::default();
5393        let mut stats = CircuitStats::new();
5394
5395        // Start closed
5396        assert_eq!(stats.state(), CircuitState::Closed);
5397
5398        // Cause failures to open
5399        for _ in 0..3 {
5400            stats.record_failure();
5401        }
5402        assert!(stats.should_open(&config));
5403        stats.open();
5404        assert_eq!(stats.state(), CircuitState::Open);
5405
5406        // Transition to half-open after cooldown would be checked
5407        stats.half_open();
5408        assert_eq!(stats.state(), CircuitState::HalfOpen);
5409
5410        // Successes to close
5411        for _ in 0..2 {
5412            stats.record_success();
5413        }
5414        assert!(stats.should_close(&config));
5415        stats.close();
5416        assert_eq!(stats.state(), CircuitState::Closed);
5417    }
5418
5419    // ========================================================================
5420    // Compilation Timing Tests
5421    // ========================================================================
5422
5423    fn make_test_metrics(
5424        speedup: Option<f64>,
5425        total_secs: u64,
5426        success: bool,
5427    ) -> CompilationMetrics {
5428        CompilationMetrics {
5429            project_id: "test-project".to_string(),
5430            worker_id: "worker-1".to_string(),
5431            timestamp: chrono::Utc::now(),
5432            timing: CompilationTimingBreakdown {
5433                rsync_up: Duration::from_millis(100),
5434                remote_build: Duration::from_millis(800),
5435                rsync_down: Duration::from_millis(100),
5436                total: Duration::from_secs(total_secs),
5437            },
5438            local_build_time: speedup.map(|s| Duration::from_secs_f64(total_secs as f64 * s)),
5439            speedup,
5440            files_synced: 100,
5441            bytes_transferred: 1_000_000,
5442            exit_code: if success { 0 } else { 1 },
5443            success,
5444        }
5445    }
5446
5447    #[test]
5448    fn test_compilation_timing_breakdown_default() {
5449        let _guard = test_guard!();
5450        let timing = CompilationTimingBreakdown::default();
5451        assert_eq!(timing.rsync_up, Duration::ZERO);
5452        assert_eq!(timing.remote_build, Duration::ZERO);
5453        assert_eq!(timing.rsync_down, Duration::ZERO);
5454        assert_eq!(timing.total, Duration::ZERO);
5455    }
5456
5457    #[test]
5458    fn test_compilation_timing_breakdown_serialization() {
5459        let _guard = test_guard!();
5460        let timing = CompilationTimingBreakdown {
5461            rsync_up: Duration::from_millis(100),
5462            remote_build: Duration::from_millis(2000),
5463            rsync_down: Duration::from_millis(50),
5464            total: Duration::from_millis(2150),
5465        };
5466
5467        let json = serde_json::to_string(&timing).unwrap();
5468        assert!(json.contains("100")); // rsync_up
5469        assert!(json.contains("2000")); // remote_build
5470        assert!(json.contains("50")); // rsync_down
5471        assert!(json.contains("2150")); // total
5472
5473        let parsed: CompilationTimingBreakdown = serde_json::from_str(&json).unwrap();
5474        assert_eq!(parsed.rsync_up, timing.rsync_up);
5475        assert_eq!(parsed.remote_build, timing.remote_build);
5476        assert_eq!(parsed.rsync_down, timing.rsync_down);
5477        assert_eq!(parsed.total, timing.total);
5478    }
5479
5480    #[test]
5481    fn test_command_timing_breakdown_serialization() {
5482        let _guard = test_guard!();
5483        let timing = CommandTimingBreakdown {
5484            classify: Some(Duration::from_millis(2)),
5485            select: None,
5486            sync_up: Some(Duration::from_millis(120)),
5487            exec: Some(Duration::from_millis(900)),
5488            sync_down: None,
5489            cleanup: Some(Duration::from_millis(5)),
5490            total: Some(Duration::from_millis(1027)),
5491        };
5492
5493        let json = serde_json::to_string(&timing).unwrap();
5494        assert!(json.contains("\"classify\":2"));
5495        assert!(json.contains("\"select\":null"));
5496        assert!(json.contains("\"sync_up\":120"));
5497        assert!(json.contains("\"exec\":900"));
5498        assert!(json.contains("\"sync_down\":null"));
5499        assert!(json.contains("\"cleanup\":5"));
5500        assert!(json.contains("\"total\":1027"));
5501
5502        let parsed: CommandTimingBreakdown = serde_json::from_str(&json).unwrap();
5503        assert_eq!(parsed.classify, timing.classify);
5504        assert_eq!(parsed.select, timing.select);
5505        assert_eq!(parsed.sync_up, timing.sync_up);
5506        assert_eq!(parsed.exec, timing.exec);
5507        assert_eq!(parsed.sync_down, timing.sync_down);
5508        assert_eq!(parsed.cleanup, timing.cleanup);
5509        assert_eq!(parsed.total, timing.total);
5510    }
5511
5512    #[test]
5513    fn test_compilation_metrics_calculate_speedup() {
5514        let _guard = test_guard!();
5515        let mut metrics = CompilationMetrics {
5516            timing: CompilationTimingBreakdown {
5517                total: Duration::from_secs(10),
5518                ..Default::default()
5519            },
5520            local_build_time: Some(Duration::from_secs(30)),
5521            ..Default::default()
5522        };
5523
5524        metrics.calculate_speedup();
5525
5526        assert!(metrics.speedup.is_some());
5527        let speedup = metrics.speedup.unwrap();
5528        assert!(
5529            (speedup - 3.0).abs() < 0.01,
5530            "Expected 3.0x speedup, got {}",
5531            speedup
5532        );
5533    }
5534
5535    #[test]
5536    fn test_compilation_metrics_is_beneficial() {
5537        let _guard = test_guard!();
5538        let mut metrics = CompilationMetrics::default();
5539
5540        // No speedup calculated
5541        assert!(!metrics.is_beneficial());
5542
5543        // Speedup > 1 is beneficial
5544        metrics.speedup = Some(1.5);
5545        assert!(metrics.is_beneficial());
5546
5547        // Speedup < 1 is not beneficial
5548        metrics.speedup = Some(0.8);
5549        assert!(!metrics.is_beneficial());
5550
5551        // Speedup = 1 is not beneficial (exactly same speed)
5552        metrics.speedup = Some(1.0);
5553        assert!(!metrics.is_beneficial());
5554    }
5555
5556    #[test]
5557    fn test_compilation_metrics_serialization() {
5558        let _guard = test_guard!();
5559        let metrics = make_test_metrics(Some(2.5), 10, true);
5560        let json = serde_json::to_string(&metrics).unwrap();
5561
5562        assert!(json.contains("test-project"));
5563        assert!(json.contains("worker-1"));
5564        assert!(json.contains("2.5")); // speedup
5565
5566        let parsed: CompilationMetrics = serde_json::from_str(&json).unwrap();
5567        assert_eq!(parsed.project_id, "test-project");
5568        assert_eq!(parsed.worker_id, "worker-1");
5569        assert!(parsed.success);
5570    }
5571
5572    #[test]
5573    fn test_compilation_timer_new() {
5574        let _guard = test_guard!();
5575        let timer = CompilationTimer::new("my-project", "my-worker");
5576        assert_eq!(timer.project_id, "my-project");
5577        assert_eq!(timer.worker_id, "my-worker");
5578        assert!(timer.rsync_up.is_none());
5579        assert!(timer.remote_build.is_none());
5580        assert!(timer.rsync_down.is_none());
5581    }
5582
5583    #[test]
5584    fn test_compilation_timer_phases() {
5585        let _guard = test_guard!();
5586        let mut timer = CompilationTimer::new("test", "worker");
5587
5588        // Simulate rsync up
5589        std::thread::sleep(Duration::from_millis(5));
5590        timer.end_rsync_up();
5591        assert!(timer.rsync_up.is_some());
5592        assert!(timer.rsync_up.unwrap() >= Duration::from_millis(5));
5593
5594        // Simulate remote build
5595        std::thread::sleep(Duration::from_millis(10));
5596        timer.end_remote_build();
5597        assert!(timer.remote_build.is_some());
5598        assert!(timer.remote_build.unwrap() >= Duration::from_millis(10));
5599
5600        // Simulate rsync down
5601        std::thread::sleep(Duration::from_millis(5));
5602        timer.end_rsync_down();
5603        assert!(timer.rsync_down.is_some());
5604        assert!(timer.rsync_down.unwrap() >= Duration::from_millis(5));
5605
5606        // Finish
5607        let metrics = timer.finish(0, 50, 500_000);
5608        assert_eq!(metrics.project_id, "test");
5609        assert_eq!(metrics.worker_id, "worker");
5610        assert!(metrics.success);
5611        assert!(metrics.timing.total >= Duration::from_millis(20));
5612        assert_eq!(metrics.files_synced, 50);
5613        assert_eq!(metrics.bytes_transferred, 500_000);
5614    }
5615
5616    #[test]
5617    fn test_metrics_aggregator_new() {
5618        let _guard = test_guard!();
5619        let agg = MetricsAggregator::new(100);
5620        assert_eq!(agg.count(), 0);
5621        assert!(agg.average_speedup().is_none());
5622    }
5623
5624    #[test]
5625    fn test_metrics_aggregator_record() {
5626        let _guard = test_guard!();
5627        let mut agg = MetricsAggregator::new(3);
5628
5629        // Record first metric
5630        agg.record(make_test_metrics(Some(2.0), 10, true));
5631        assert_eq!(agg.count(), 1);
5632
5633        // Record up to max
5634        agg.record(make_test_metrics(Some(3.0), 20, true));
5635        agg.record(make_test_metrics(Some(4.0), 30, true));
5636        assert_eq!(agg.count(), 3);
5637
5638        // Exceed max should drop oldest
5639        agg.record(make_test_metrics(Some(5.0), 40, true));
5640        assert_eq!(agg.count(), 3);
5641    }
5642
5643    #[test]
5644    fn test_metrics_aggregator_average_speedup() {
5645        let _guard = test_guard!();
5646        let mut agg = MetricsAggregator::new(100);
5647
5648        // Empty case
5649        assert!(agg.average_speedup().is_none());
5650
5651        // Add metrics with speedups 1, 2, 3, 4, 5 -> average = 3
5652        for i in 1..=5 {
5653            agg.record(make_test_metrics(Some(i as f64), 10, true));
5654        }
5655
5656        let avg = agg.average_speedup().unwrap();
5657        assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5658    }
5659
5660    #[test]
5661    fn test_metrics_aggregator_average_speedup_with_none() {
5662        let _guard = test_guard!();
5663        let mut agg = MetricsAggregator::new(100);
5664
5665        // Mix of Some and None speedups
5666        agg.record(make_test_metrics(Some(2.0), 10, true));
5667        agg.record(make_test_metrics(None, 10, true)); // No speedup
5668        agg.record(make_test_metrics(Some(4.0), 10, true));
5669
5670        // Average should only consider metrics with speedup
5671        let avg = agg.average_speedup().unwrap();
5672        assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5673    }
5674
5675    #[test]
5676    fn test_metrics_aggregator_percentiles() {
5677        let _guard = test_guard!();
5678        let mut agg = MetricsAggregator::new(100);
5679
5680        // Add metrics with total times 1s, 2s, 3s, ..., 10s
5681        for i in 1..=10 {
5682            agg.record(make_test_metrics(Some(1.0), i, true));
5683        }
5684
5685        // p50 (median) should be around 5s
5686        let p50 = agg.p50_total_time().unwrap();
5687        assert!(p50 >= Duration::from_secs(5) && p50 <= Duration::from_secs(6));
5688
5689        // p95 should be around 9s or 10s
5690        let p95 = agg.p95_total_time().unwrap();
5691        assert!(p95 >= Duration::from_secs(9) && p95 <= Duration::from_secs(10));
5692
5693        // p99 should be 10s
5694        let p99 = agg.p99_total_time().unwrap();
5695        assert!(p99 >= Duration::from_secs(9));
5696    }
5697
5698    #[test]
5699    fn test_metrics_aggregator_success_rate() {
5700        let _guard = test_guard!();
5701        let mut agg = MetricsAggregator::new(100);
5702
5703        // Empty case
5704        assert_eq!(agg.success_rate(), 100.0);
5705
5706        // All successful
5707        agg.record(make_test_metrics(Some(1.0), 10, true));
5708        agg.record(make_test_metrics(Some(1.0), 10, true));
5709        assert_eq!(agg.success_rate(), 100.0);
5710
5711        // Add a failure (2 success, 1 failure = 66.67%)
5712        agg.record(make_test_metrics(Some(1.0), 10, false));
5713        let rate = agg.success_rate();
5714        assert!((rate - 66.67).abs() < 1.0, "Expected ~66.67%, got {}", rate);
5715    }
5716
5717    #[test]
5718    fn test_metrics_aggregator_clear() {
5719        let _guard = test_guard!();
5720        let mut agg = MetricsAggregator::new(100);
5721        agg.record(make_test_metrics(Some(1.0), 10, true));
5722        agg.record(make_test_metrics(Some(2.0), 20, true));
5723
5724        assert_eq!(agg.count(), 2);
5725
5726        agg.clear();
5727        assert_eq!(agg.count(), 0);
5728        assert!(agg.average_speedup().is_none());
5729    }
5730
5731    // CompilationConfig timeout tests
5732
5733    #[test]
5734    fn test_remote_build_jobs_parse_display_and_serde() {
5735        let _guard = test_guard!();
5736        assert_eq!(RemoteBuildJobs::default(), RemoteBuildJobs::Auto);
5737        for (spelling, expected) in [
5738            ("auto", RemoteBuildJobs::Auto),
5739            (" AUTO ", RemoteBuildJobs::Auto),
5740            ("off", RemoteBuildJobs::Off),
5741            ("none", RemoteBuildJobs::Off),
5742            ("4", RemoteBuildJobs::Fixed(4)),
5743        ] {
5744            assert_eq!(
5745                RemoteBuildJobs::parse(spelling).unwrap(),
5746                expected,
5747                "{spelling}"
5748            );
5749        }
5750        assert!(RemoteBuildJobs::parse("0").is_err(), "cargo rejects -j 0");
5751        assert!(RemoteBuildJobs::parse("many").is_err());
5752        assert_eq!(RemoteBuildJobs::Fixed(4).to_string(), "4");
5753        assert_eq!(RemoteBuildJobs::Off.to_string(), "off");
5754
5755        // TOML accepts both the string spellings and a bare integer, and an
5756        // absent key keeps the default.
5757        #[derive(Deserialize)]
5758        struct Wrap {
5759            compilation: CompilationConfig,
5760        }
5761        let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = \"off\"\n").unwrap();
5762        assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Off);
5763        let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = 6\n").unwrap();
5764        assert_eq!(
5765            parsed.compilation.remote_build_jobs,
5766            RemoteBuildJobs::Fixed(6)
5767        );
5768        let parsed: Wrap = toml::from_str("[compilation]\nbuild_slots = 4\n").unwrap();
5769        assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Auto);
5770        assert!(toml::from_str::<Wrap>("[compilation]\nremote_build_jobs = 0\n").is_err());
5771
5772        // Serialization round-trips through the same spellings.
5773        let json = serde_json::to_string(&RemoteBuildJobs::Fixed(3)).unwrap();
5774        assert_eq!(json, "3");
5775        assert_eq!(
5776            serde_json::to_string(&RemoteBuildJobs::Auto).unwrap(),
5777            "\"auto\""
5778        );
5779        let back: RemoteBuildJobs = serde_json::from_str("\"auto\"").unwrap();
5780        assert_eq!(back, RemoteBuildJobs::Auto);
5781    }
5782
5783    #[test]
5784    fn test_compilation_config_default_timeouts() {
5785        let _guard = test_guard!();
5786        let config = CompilationConfig::default();
5787        // Default build timeout: 5 minutes
5788        assert_eq!(config.build_timeout_sec, 300);
5789        // Default test timeout: 30 minutes
5790        assert_eq!(config.test_timeout_sec, 1800);
5791    }
5792
5793    #[test]
5794    fn test_compilation_config_timeout_for_test_kinds() {
5795        let _guard = test_guard!();
5796        let config = CompilationConfig::default();
5797
5798        // Test commands should get test_timeout_sec
5799        assert_eq!(
5800            config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5801            std::time::Duration::from_secs(1800)
5802        );
5803        assert_eq!(
5804            config.timeout_for_kind(Some(crate::CompilationKind::CargoNextest)),
5805            std::time::Duration::from_secs(1800)
5806        );
5807        assert_eq!(
5808            config.timeout_for_kind(Some(crate::CompilationKind::BunTest)),
5809            std::time::Duration::from_secs(config.bun_timeout_sec)
5810        );
5811    }
5812
5813    #[test]
5814    fn test_compilation_config_timeout_for_build_kinds() {
5815        let _guard = test_guard!();
5816        let config = CompilationConfig::default();
5817
5818        // Build/check commands should get build_timeout_sec
5819        assert_eq!(
5820            config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5821            std::time::Duration::from_secs(300)
5822        );
5823        assert_eq!(
5824            config.timeout_for_kind(Some(crate::CompilationKind::CargoCheck)),
5825            std::time::Duration::from_secs(300)
5826        );
5827        assert_eq!(
5828            config.timeout_for_kind(Some(crate::CompilationKind::CargoClippy)),
5829            std::time::Duration::from_secs(300)
5830        );
5831        assert_eq!(
5832            config.timeout_for_kind(None),
5833            std::time::Duration::from_secs(300)
5834        );
5835    }
5836
5837    #[test]
5838    fn test_compilation_config_custom_timeouts() {
5839        let _guard = test_guard!();
5840        let config = CompilationConfig {
5841            build_timeout_sec: 600, // 10 minutes
5842            test_timeout_sec: 3600, // 1 hour
5843            ..Default::default()
5844        };
5845
5846        assert_eq!(
5847            config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5848            std::time::Duration::from_secs(600)
5849        );
5850        assert_eq!(
5851            config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5852            std::time::Duration::from_secs(3600)
5853        );
5854    }
5855
5856    #[test]
5857    fn test_compilation_config_speedup_threshold_default() {
5858        let _guard = test_guard!();
5859        let config = CompilationConfig::default();
5860        // Default speedup threshold: 1.2 (20% faster required)
5861        assert!((config.remote_speedup_threshold - 1.2).abs() < 0.001);
5862    }
5863
5864    #[test]
5865    fn test_compilation_config_speedup_threshold_custom() {
5866        let _guard = test_guard!();
5867        let config = CompilationConfig {
5868            remote_speedup_threshold: 1.5, // Require 50% faster
5869            ..Default::default()
5870        };
5871        assert!((config.remote_speedup_threshold - 1.5).abs() < 0.001);
5872    }
5873
5874    #[test]
5875    fn test_compilation_config_speedup_threshold_no_minimum() {
5876        let _guard = test_guard!();
5877        // Setting to 1.0 means "always offload when other criteria met"
5878        let config = CompilationConfig {
5879            remote_speedup_threshold: 1.0,
5880            ..Default::default()
5881        };
5882        assert!((config.remote_speedup_threshold - 1.0).abs() < 0.001);
5883    }
5884
5885    #[test]
5886    fn test_compilation_config_min_local_time_ms_default() {
5887        let _guard = test_guard!();
5888        let config = CompilationConfig::default();
5889        // Default min_local_time_ms: 2000ms
5890        assert_eq!(config.min_local_time_ms, 2000);
5891    }
5892
5893    #[test]
5894    fn test_compilation_config_serde_roundtrip_speedup_threshold() {
5895        let _guard = test_guard!();
5896        let config = CompilationConfig {
5897            remote_speedup_threshold: 2.5,
5898            min_local_time_ms: 5000,
5899            ..Default::default()
5900        };
5901        let json = serde_json::to_string(&config).unwrap();
5902        let parsed: CompilationConfig = serde_json::from_str(&json).unwrap();
5903        assert!((parsed.remote_speedup_threshold - 2.5).abs() < 0.001);
5904        assert_eq!(parsed.min_local_time_ms, 5000);
5905    }
5906
5907    // ========================================================================
5908    // validate_remote_base Tests
5909    // ========================================================================
5910
5911    #[test]
5912    fn test_validate_remote_base_absolute_path() {
5913        let _guard = test_guard!();
5914        assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5915        assert_eq!(
5916            validate_remote_base("/var/rch-builds").unwrap(),
5917            "/var/rch-builds"
5918        );
5919        assert_eq!(
5920            validate_remote_base("/home/builder/.rch").unwrap(),
5921            "/home/builder/.rch"
5922        );
5923    }
5924
5925    #[test]
5926    fn test_validate_remote_base_tilde_expansion() {
5927        let _guard = test_guard!();
5928        // Tilde expansion should work
5929        let result = validate_remote_base("~/rch");
5930        assert!(result.is_ok());
5931        let path = result.unwrap();
5932        assert!(
5933            path.starts_with('/'),
5934            "Path should be absolute after expansion: {}",
5935            path
5936        );
5937        assert!(!path.contains('~'), "Tilde should be expanded: {}", path);
5938    }
5939
5940    #[test]
5941    fn test_validate_remote_base_rejects_relative_path() {
5942        let _guard = test_guard!();
5943        let result = validate_remote_base("tmp/rch");
5944        assert!(result.is_err());
5945        assert!(result.unwrap_err().contains("absolute path"));
5946    }
5947
5948    #[test]
5949    fn test_validate_remote_base_rejects_path_traversal() {
5950        let _guard = test_guard!();
5951        let result = validate_remote_base("/tmp/../etc/rch");
5952        assert!(result.is_err());
5953        assert!(result.unwrap_err().contains("path traversal"));
5954
5955        let result = validate_remote_base("/tmp/rch/../other");
5956        assert!(result.is_err());
5957        assert!(result.unwrap_err().contains("path traversal"));
5958    }
5959
5960    #[test]
5961    fn test_validate_remote_base_normalizes_trailing_slash() {
5962        let _guard = test_guard!();
5963        assert_eq!(validate_remote_base("/tmp/rch/").unwrap(), "/tmp/rch");
5964        assert_eq!(validate_remote_base("/tmp/rch///").unwrap(), "/tmp/rch");
5965    }
5966
5967    #[test]
5968    fn test_validate_remote_base_root_path() {
5969        let _guard = test_guard!();
5970        // Root path should be rejected
5971        let result = validate_remote_base("/");
5972        assert!(result.is_err());
5973        assert!(result.unwrap_err().contains("root directory"));
5974    }
5975
5976    #[test]
5977    fn test_validate_remote_base_rejects_top_level() {
5978        let _guard = test_guard!();
5979        // Top-level directories should be rejected (depth 1)
5980        let result = validate_remote_base("/tmp");
5981        assert!(result.is_err());
5982        assert!(result.unwrap_err().contains("at least 2 levels deep"));
5983
5984        let result = validate_remote_base("/home");
5985        assert!(result.is_err());
5986
5987        // Depth 2 should be allowed
5988        assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5989    }
5990
5991    #[test]
5992    fn test_default_remote_base() {
5993        let _guard = test_guard!();
5994        assert_eq!(default_remote_base(), "/data/tmp/rch");
5995    }
5996
5997    #[test]
5998    fn test_transfer_config_default_has_remote_base() {
5999        let _guard = test_guard!();
6000        let config = TransferConfig::default();
6001        assert_eq!(config.remote_base, "/data/tmp/rch");
6002    }
6003
6004    #[test]
6005    fn test_source_sync_timeout_validation_boundaries() {
6006        let _guard = test_guard!();
6007        assert!(!TransferConfig::valid_sync_timeout_ms(0));
6008        assert!(!TransferConfig::valid_sync_timeout_ms(999));
6009        assert!(TransferConfig::valid_sync_timeout_ms(1_000));
6010        assert!(TransferConfig::valid_sync_timeout_ms(3_600_000));
6011        assert!(!TransferConfig::valid_sync_timeout_ms(3_600_001));
6012    }
6013
6014    #[test]
6015    fn test_source_sync_timeout_is_explicit_or_payload_aware() {
6016        let _guard = test_guard!();
6017        let default = TransferConfig::default();
6018        assert_eq!(
6019            default.sync_timeout_for_payload(0),
6020            std::time::Duration::from_millis(30_000)
6021        );
6022        assert_eq!(
6023            default.sync_timeout_for_payload(10 * 1024 * 1024),
6024            std::time::Duration::from_millis(40_000)
6025        );
6026        assert_eq!(
6027            default.sync_timeout_for_payload(u64::MAX),
6028            std::time::Duration::from_millis(3_600_000)
6029        );
6030
6031        let explicit = TransferConfig {
6032            sync_timeout_ms: Some(120_000),
6033            ..TransferConfig::default()
6034        };
6035        assert_eq!(
6036            explicit.sync_timeout_for_payload(u64::MAX),
6037            std::time::Duration::from_millis(120_000)
6038        );
6039    }
6040
6041    // ========================================================================
6042    // RetryConfig Tests (bd-x1ek)
6043    // ========================================================================
6044
6045    #[test]
6046    fn test_retry_config_default() {
6047        let _guard = test_guard!();
6048        let config = RetryConfig::default();
6049        assert_eq!(config.max_attempts, 3);
6050        assert_eq!(config.base_delay_ms, 100);
6051        assert_eq!(config.max_delay_ms, 5000);
6052        assert_eq!(config.jitter_factor, 0.1);
6053        assert_eq!(config.total_timeout_ms, 30000);
6054    }
6055
6056    #[test]
6057    fn test_retry_config_no_retry() {
6058        let _guard = test_guard!();
6059        let config = RetryConfig::no_retry();
6060        assert_eq!(config.max_attempts, 1);
6061    }
6062
6063    #[test]
6064    fn test_retry_config_delay_for_attempt_zero() {
6065        let _guard = test_guard!();
6066        let config = RetryConfig::default();
6067        assert_eq!(config.delay_for_attempt(0), std::time::Duration::ZERO);
6068    }
6069
6070    #[test]
6071    fn test_retry_config_delay_for_attempt_exponential() {
6072        let _guard = test_guard!();
6073        let config = RetryConfig {
6074            base_delay_ms: 100,
6075            max_delay_ms: 10000,
6076            jitter_factor: 0.0, // No jitter for deterministic test
6077            ..Default::default()
6078        };
6079
6080        // attempt 1: 100 * 2^0 = 100ms
6081        assert_eq!(
6082            config.delay_for_attempt(1),
6083            std::time::Duration::from_millis(100)
6084        );
6085        // attempt 2: 100 * 2^1 = 200ms
6086        assert_eq!(
6087            config.delay_for_attempt(2),
6088            std::time::Duration::from_millis(200)
6089        );
6090        // attempt 3: 100 * 2^2 = 400ms
6091        assert_eq!(
6092            config.delay_for_attempt(3),
6093            std::time::Duration::from_millis(400)
6094        );
6095    }
6096
6097    #[test]
6098    fn test_retry_config_delay_capped_at_max() {
6099        let _guard = test_guard!();
6100        let config = RetryConfig {
6101            base_delay_ms: 1000,
6102            max_delay_ms: 2000,
6103            jitter_factor: 0.0,
6104            ..Default::default()
6105        };
6106
6107        // attempt 2: 1000 * 2^1 = 2000ms (at cap)
6108        assert_eq!(
6109            config.delay_for_attempt(2),
6110            std::time::Duration::from_millis(2000)
6111        );
6112        // attempt 3: 1000 * 2^2 = 4000ms, capped to 2000ms
6113        assert_eq!(
6114            config.delay_for_attempt(3),
6115            std::time::Duration::from_millis(2000)
6116        );
6117    }
6118
6119    #[test]
6120    fn test_retry_config_random_jitter_stays_within_bounds() {
6121        let _guard = test_guard!();
6122        let config = RetryConfig {
6123            base_delay_ms: 100,
6124            max_delay_ms: 200,
6125            jitter_factor: 0.25,
6126            ..Default::default()
6127        };
6128
6129        // Jitter applies after the exponential delay is capped. Exercise the
6130        // real RNG without requiring any particular random value or sequence.
6131        for (attempt, minimum_ms, maximum_ms) in [(1, 75, 125), (3, 150, 250)] {
6132            for _ in 0..128 {
6133                let delay = config.delay_for_attempt(attempt);
6134                assert!(delay >= std::time::Duration::from_millis(minimum_ms));
6135                assert!(delay <= std::time::Duration::from_millis(maximum_ms));
6136            }
6137        }
6138    }
6139
6140    #[test]
6141    fn test_retry_config_should_retry() {
6142        let _guard = test_guard!();
6143        let config = RetryConfig {
6144            max_attempts: 3,
6145            total_timeout_ms: 1000,
6146            ..Default::default()
6147        };
6148
6149        // Before max attempts and within timeout
6150        assert!(config.should_retry(0, std::time::Duration::from_millis(0)));
6151        assert!(config.should_retry(1, std::time::Duration::from_millis(500)));
6152        assert!(config.should_retry(2, std::time::Duration::from_millis(900)));
6153
6154        // At max attempts
6155        assert!(!config.should_retry(3, std::time::Duration::from_millis(0)));
6156
6157        // Past timeout
6158        assert!(!config.should_retry(1, std::time::Duration::from_millis(1001)));
6159    }
6160
6161    #[test]
6162    fn test_retry_config_serde_roundtrip() {
6163        let _guard = test_guard!();
6164        let config = RetryConfig {
6165            max_attempts: 5,
6166            base_delay_ms: 200,
6167            max_delay_ms: 8000,
6168            jitter_factor: 0.2,
6169            total_timeout_ms: 60000,
6170        };
6171        let json = serde_json::to_string(&config).unwrap();
6172        let parsed: RetryConfig = serde_json::from_str(&json).unwrap();
6173        assert_eq!(parsed.max_attempts, 5);
6174        assert_eq!(parsed.base_delay_ms, 200);
6175        assert_eq!(parsed.max_delay_ms, 8000);
6176        assert_eq!(parsed.jitter_factor, 0.2);
6177        assert_eq!(parsed.total_timeout_ms, 60000);
6178    }
6179
6180    #[test]
6181    fn test_transfer_config_includes_retry() {
6182        let _guard = test_guard!();
6183        let config = TransferConfig::default();
6184        assert_eq!(config.retry.max_attempts, 3);
6185    }
6186
6187    // =========================================================================
6188    // ExecutionConfig Tests (bd-785w)
6189    // =========================================================================
6190
6191    #[test]
6192    fn test_execution_config_default_allowlist() {
6193        let _guard = test_guard!();
6194        let config = ExecutionConfig::default();
6195        // All supported compilers should be in the default allowlist
6196        assert!(config.is_allowed("cargo"));
6197        assert!(config.is_allowed("rustc"));
6198        assert!(config.is_allowed("gcc"));
6199        assert!(config.is_allowed("g++"));
6200        assert!(config.is_allowed("clang"));
6201        assert!(config.is_allowed("clang++"));
6202        assert!(config.is_allowed("make"));
6203        assert!(config.is_allowed("cmake"));
6204        assert!(config.is_allowed("ninja"));
6205        assert!(config.is_allowed("meson"));
6206        assert!(config.is_allowed("bun"));
6207        assert!(config.is_allowed("nextest"));
6208        assert!(config.is_allowed("cc"));
6209        assert!(config.is_allowed("c++"));
6210    }
6211
6212    #[test]
6213    fn test_execution_config_is_allowed_case_insensitive() {
6214        let _guard = test_guard!();
6215        let config = ExecutionConfig::default();
6216        // Should be case-insensitive
6217        assert!(config.is_allowed("CARGO"));
6218        assert!(config.is_allowed("Cargo"));
6219        assert!(config.is_allowed("GCC"));
6220        assert!(config.is_allowed("Gcc"));
6221    }
6222
6223    #[test]
6224    fn test_execution_config_is_not_allowed() {
6225        let _guard = test_guard!();
6226        let config = ExecutionConfig::default();
6227        // Unknown commands should not be allowed
6228        assert!(!config.is_allowed("python"));
6229        assert!(!config.is_allowed("npm"));
6230        assert!(!config.is_allowed(""));
6231        // `go` and `tsc` ARE allowed now that Go/TypeScript are offload targets.
6232        // If they were missing from the allowlist the hook would fail open to
6233        // LOCAL execution — classification would look correct while every build
6234        // silently ran on the orchestrator.
6235        assert!(config.is_allowed("go"));
6236        assert!(config.is_allowed("tsc"));
6237    }
6238
6239    #[test]
6240    fn test_execution_config_empty_allowlist() {
6241        let _guard = test_guard!();
6242        let config = ExecutionConfig { allowlist: vec![] };
6243        // Empty allowlist should block everything
6244        assert!(!config.is_allowed("cargo"));
6245        assert!(!config.is_allowed("gcc"));
6246    }
6247
6248    #[test]
6249    fn test_execution_config_custom_allowlist() {
6250        let _guard = test_guard!();
6251        let config = ExecutionConfig {
6252            allowlist: vec!["cargo".to_string(), "custom_tool".to_string()],
6253        };
6254        assert!(config.is_allowed("cargo"));
6255        assert!(config.is_allowed("custom_tool"));
6256        assert!(!config.is_allowed("gcc"));
6257    }
6258
6259    #[test]
6260    fn test_execution_config_serde() {
6261        let _guard = test_guard!();
6262        let config = ExecutionConfig {
6263            allowlist: vec!["cargo".to_string(), "rustc".to_string()],
6264        };
6265        let json = serde_json::to_string(&config).unwrap();
6266        let parsed: ExecutionConfig = serde_json::from_str(&json).unwrap();
6267        assert_eq!(parsed.allowlist, config.allowlist);
6268    }
6269
6270    #[test]
6271    fn test_rch_config_includes_execution() {
6272        let _guard = test_guard!();
6273        let config = RchConfig::default();
6274        // RchConfig should include ExecutionConfig with default allowlist
6275        assert!(config.execution.is_allowed("cargo"));
6276    }
6277
6278    // ========================================================================
6279    // Adaptive Compression Tests (bd-243w)
6280    // ========================================================================
6281
6282    #[test]
6283    fn test_adaptive_compression_disabled_by_default() {
6284        let _guard = test_guard!();
6285        let config = TransferConfig::default();
6286        assert!(!config.adaptive_compression);
6287        // Should use fixed level when disabled
6288        assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6289        assert_eq!(config.select_compression_level(Some(500_000_000)), 3);
6290    }
6291
6292    #[test]
6293    fn test_adaptive_compression_small_payload() {
6294        let _guard = test_guard!();
6295        let config = TransferConfig {
6296            adaptive_compression: true,
6297            ..Default::default()
6298        };
6299        // < 10MB should use level 1
6300        assert_eq!(config.select_compression_level(Some(0)), 1);
6301        assert_eq!(config.select_compression_level(Some(1_000_000)), 1); // 1MB
6302        assert_eq!(config.select_compression_level(Some(9_999_999)), 1); // ~10MB
6303    }
6304
6305    #[test]
6306    fn test_adaptive_compression_medium_payload() {
6307        let _guard = test_guard!();
6308        let config = TransferConfig {
6309            adaptive_compression: true,
6310            ..Default::default()
6311        };
6312        // 10-200MB should use level 3
6313        assert_eq!(config.select_compression_level(Some(10_000_000)), 3); // 10MB
6314        assert_eq!(config.select_compression_level(Some(100_000_000)), 3); // 100MB
6315        assert_eq!(config.select_compression_level(Some(199_999_999)), 3); // ~200MB
6316    }
6317
6318    #[test]
6319    fn test_adaptive_compression_large_payload() {
6320        let _guard = test_guard!();
6321        let config = TransferConfig {
6322            adaptive_compression: true,
6323            ..Default::default()
6324        };
6325        // > 200MB should use level 7
6326        assert_eq!(config.select_compression_level(Some(200_000_000)), 7); // 200MB
6327        assert_eq!(config.select_compression_level(Some(500_000_000)), 7); // 500MB
6328        assert_eq!(config.select_compression_level(Some(1_000_000_000)), 7); // 1GB
6329    }
6330
6331    #[test]
6332    fn test_adaptive_compression_no_estimate() {
6333        let _guard = test_guard!();
6334        let config = TransferConfig {
6335            adaptive_compression: true,
6336            compression_level: 5,
6337            ..Default::default()
6338        };
6339        // No estimate should fallback to default level
6340        assert_eq!(config.select_compression_level(None), 5);
6341    }
6342
6343    #[test]
6344    fn test_adaptive_compression_respects_min_level() {
6345        let _guard = test_guard!();
6346        let config = TransferConfig {
6347            adaptive_compression: true,
6348            min_compression_level: 3,
6349            ..Default::default()
6350        };
6351        // Small payload would want level 1, but min is 3
6352        assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6353    }
6354
6355    #[test]
6356    fn test_adaptive_compression_respects_max_level() {
6357        let _guard = test_guard!();
6358        let config = TransferConfig {
6359            adaptive_compression: true,
6360            max_compression_level: 5,
6361            ..Default::default()
6362        };
6363        // Large payload would want level 7, but max is 5
6364        assert_eq!(config.select_compression_level(Some(500_000_000)), 5);
6365    }
6366
6367    #[test]
6368    fn test_adaptive_compression_serde_roundtrip() {
6369        let _guard = test_guard!();
6370        let config = TransferConfig {
6371            adaptive_compression: true,
6372            min_compression_level: 2,
6373            max_compression_level: 8,
6374            ..Default::default()
6375        };
6376        let json = serde_json::to_string(&config).unwrap();
6377        let parsed: TransferConfig = serde_json::from_str(&json).unwrap();
6378        assert!(parsed.adaptive_compression);
6379        assert_eq!(parsed.min_compression_level, 2);
6380        assert_eq!(parsed.max_compression_level, 8);
6381    }
6382
6383    // -------------------------------------------------------------------------
6384    // Self-Healing Config Tests (bd-59kg)
6385    // -------------------------------------------------------------------------
6386
6387    #[test]
6388    fn test_self_healing_config_defaults() {
6389        let _guard = test_guard!();
6390        // TEST START: SelfHealingConfig::default() returns expected values
6391        let config = SelfHealingConfig::default();
6392
6393        // Both self-healing behaviors enabled by default
6394        assert!(
6395            config.hook_starts_daemon,
6396            "hook_starts_daemon should be true by default"
6397        );
6398        assert!(
6399            config.daemon_installs_hooks,
6400            "daemon_installs_hooks should be true by default"
6401        );
6402
6403        // Timing defaults
6404        assert_eq!(
6405            config.auto_start_cooldown_secs, 30,
6406            "auto_start_cooldown_secs should default to 30"
6407        );
6408        assert_eq!(
6409            config.auto_start_timeout_secs, 3,
6410            "auto_start_timeout_secs should default to 3"
6411        );
6412        // TEST PASS: SelfHealingConfig defaults
6413    }
6414
6415    #[test]
6416    fn test_self_healing_config_serde_full() {
6417        let _guard = test_guard!();
6418        // TEST START: Full SelfHealingConfig serialization/deserialization
6419        let config = SelfHealingConfig {
6420            hook_starts_daemon: false,
6421            daemon_installs_hooks: false,
6422            auto_start_cooldown_secs: 60,
6423            auto_start_timeout_secs: 10,
6424            self_healing_log_level: SelfHealingLogLevel::Debug,
6425        };
6426
6427        let json = serde_json::to_string(&config).unwrap();
6428        assert!(json.contains("\"hook_starts_daemon\":false"));
6429        assert!(json.contains("\"daemon_installs_hooks\":false"));
6430        assert!(json.contains("\"auto_start_cooldown_secs\":60"));
6431        assert!(json.contains("\"auto_start_timeout_secs\":10"));
6432        assert!(json.contains("\"self_healing_log_level\":\"debug\""));
6433
6434        let parsed: SelfHealingConfig = serde_json::from_str(&json).unwrap();
6435        assert!(!parsed.hook_starts_daemon);
6436        assert!(!parsed.daemon_installs_hooks);
6437        assert_eq!(parsed.auto_start_cooldown_secs, 60);
6438        assert_eq!(parsed.auto_start_timeout_secs, 10);
6439        assert_eq!(parsed.self_healing_log_level, SelfHealingLogLevel::Debug);
6440        // TEST PASS: Full SelfHealingConfig serde
6441    }
6442
6443    // br-4zf3p: Tests for the self_healing_log_level field.
6444
6445    #[test]
6446    fn test_self_healing_log_level_default_is_info() {
6447        // TEST START: SelfHealingLogLevel::default() == Info
6448        let level = SelfHealingLogLevel::default();
6449        assert_eq!(level, SelfHealingLogLevel::Info);
6450        // Also verify the SelfHealingConfig default surfaces it.
6451        let config = SelfHealingConfig::default();
6452        assert_eq!(
6453            config.self_healing_log_level,
6454            SelfHealingLogLevel::Info,
6455            "config default should be Info"
6456        );
6457        // TEST PASS
6458    }
6459
6460    #[test]
6461    fn test_self_healing_log_level_serializes_lowercase() {
6462        // TEST START: serde uses lowercase variants
6463        let json = serde_json::to_string(&SelfHealingLogLevel::Warn).unwrap();
6464        assert_eq!(json, "\"warn\"");
6465        let parsed: SelfHealingLogLevel = serde_json::from_str("\"error\"").unwrap();
6466        assert_eq!(parsed, SelfHealingLogLevel::Error);
6467        // TEST PASS
6468    }
6469
6470    #[test]
6471    fn test_self_healing_log_level_from_env_str_accepts_known() {
6472        // TEST START: from_env_str parses each level (case-insensitive) and aliases
6473        assert_eq!(
6474            SelfHealingLogLevel::from_env_str("debug"),
6475            Some(SelfHealingLogLevel::Debug)
6476        );
6477        assert_eq!(
6478            SelfHealingLogLevel::from_env_str("DEBUG"),
6479            Some(SelfHealingLogLevel::Debug)
6480        );
6481        assert_eq!(
6482            SelfHealingLogLevel::from_env_str("info"),
6483            Some(SelfHealingLogLevel::Info)
6484        );
6485        assert_eq!(
6486            SelfHealingLogLevel::from_env_str("warn"),
6487            Some(SelfHealingLogLevel::Warn)
6488        );
6489        assert_eq!(
6490            SelfHealingLogLevel::from_env_str("warning"),
6491            Some(SelfHealingLogLevel::Warn),
6492            "warning should map to Warn"
6493        );
6494        assert_eq!(
6495            SelfHealingLogLevel::from_env_str("error"),
6496            Some(SelfHealingLogLevel::Error)
6497        );
6498        // Whitespace tolerance
6499        assert_eq!(
6500            SelfHealingLogLevel::from_env_str("  debug  "),
6501            Some(SelfHealingLogLevel::Debug)
6502        );
6503        // TEST PASS
6504    }
6505
6506    #[test]
6507    fn test_self_healing_log_level_from_env_str_rejects_unknown() {
6508        // TEST START: unknown values return None (caller falls back to default)
6509        assert!(SelfHealingLogLevel::from_env_str("banana").is_none());
6510        assert!(SelfHealingLogLevel::from_env_str("").is_none());
6511        assert!(SelfHealingLogLevel::from_env_str("trace").is_none());
6512        // TEST PASS
6513    }
6514
6515    #[test]
6516    fn test_self_healing_config_serde_partial_uses_defaults() {
6517        let _guard = test_guard!();
6518        // TEST START: Partial TOML/JSON uses defaults for missing fields
6519        let json = r#"{"hook_starts_daemon": false}"#;
6520        let config: SelfHealingConfig = serde_json::from_str(json).unwrap();
6521
6522        assert!(
6523            !config.hook_starts_daemon,
6524            "Explicit false should be parsed"
6525        );
6526        assert!(
6527            config.daemon_installs_hooks,
6528            "Missing field should use default (true)"
6529        );
6530        assert_eq!(
6531            config.auto_start_cooldown_secs, 30,
6532            "Missing field should use default (30)"
6533        );
6534        assert_eq!(
6535            config.auto_start_timeout_secs, 3,
6536            "Missing field should use default (3)"
6537        );
6538        // TEST PASS: Partial SelfHealingConfig uses defaults
6539    }
6540
6541    #[test]
6542    fn test_self_healing_config_toml_with_alias() {
6543        let _guard = test_guard!();
6544        // TEST START: daemon_start_timeout alias works for auto_start_timeout_secs
6545        let toml_str = r#"
6546            hook_starts_daemon = true
6547            daemon_installs_hooks = false
6548            auto_start_cooldown_secs = 45
6549            daemon_start_timeout = 7
6550        "#;
6551
6552        let config: SelfHealingConfig = toml::from_str(toml_str).unwrap();
6553        assert!(config.hook_starts_daemon);
6554        assert!(!config.daemon_installs_hooks);
6555        assert_eq!(config.auto_start_cooldown_secs, 45);
6556        assert_eq!(
6557            config.auto_start_timeout_secs, 7,
6558            "daemon_start_timeout alias should set auto_start_timeout_secs"
6559        );
6560        // TEST PASS: TOML alias works
6561    }
6562
6563    #[allow(unsafe_code)]
6564    mod self_healing_env_override_tests {
6565        use super::*;
6566        use crate::config::env_test_lock;
6567
6568        fn env_guard() -> std::sync::MutexGuard<'static, ()> {
6569            env_test_lock()
6570        }
6571
6572        fn set_env(key: &str, value: &str) {
6573            // SAFETY: Tests are serialized with env_guard().
6574            unsafe { std::env::set_var(key, value) };
6575        }
6576
6577        fn remove_env(key: &str) {
6578            // SAFETY: Tests are serialized with env_guard().
6579            unsafe { std::env::remove_var(key) };
6580        }
6581
6582        struct EnvVarGuard {
6583            key: &'static str,
6584            old: Option<String>,
6585        }
6586
6587        impl EnvVarGuard {
6588            fn set(key: &'static str, value: &str) -> Self {
6589                let old = std::env::var(key).ok();
6590                set_env(key, value);
6591                Self { key, old }
6592            }
6593        }
6594
6595        impl Drop for EnvVarGuard {
6596            fn drop(&mut self) {
6597                if let Some(old) = &self.old {
6598                    set_env(self.key, old);
6599                } else {
6600                    remove_env(self.key);
6601                }
6602            }
6603        }
6604
6605        #[test]
6606        fn test_self_healing_config_with_env_overrides_master_disable() {
6607            let _guard = test_guard!();
6608            let _guard = env_guard();
6609
6610            let _no_self_healing = EnvVarGuard::set("RCH_NO_SELF_HEALING", "1");
6611            let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "1");
6612            let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "1");
6613            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "99");
6614            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "99");
6615
6616            let config = SelfHealingConfig::default().with_env_overrides();
6617            assert!(
6618                !config.hook_starts_daemon,
6619                "RCH_NO_SELF_HEALING should disable hook auto-start"
6620            );
6621            assert!(
6622                !config.daemon_installs_hooks,
6623                "RCH_NO_SELF_HEALING should disable daemon hook installation"
6624            );
6625
6626            // Master disable returns early; numeric overrides should not apply.
6627            assert_eq!(config.auto_start_cooldown_secs, 30);
6628            assert_eq!(config.auto_start_timeout_secs, 3);
6629        }
6630
6631        #[test]
6632        fn test_self_healing_config_with_env_overrides_toggles_and_numbers() {
6633            let _guard = test_guard!();
6634            let _guard = env_guard();
6635
6636            let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "0");
6637            let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "false");
6638            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "45");
6639            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "7");
6640
6641            let config = SelfHealingConfig::default().with_env_overrides();
6642            assert!(!config.hook_starts_daemon);
6643            assert!(!config.daemon_installs_hooks);
6644            assert_eq!(config.auto_start_cooldown_secs, 45);
6645            assert_eq!(config.auto_start_timeout_secs, 7);
6646        }
6647
6648        #[test]
6649        fn test_self_healing_config_with_env_overrides_invalid_numbers_ignored() {
6650            let _guard = test_guard!();
6651            let _guard = env_guard();
6652
6653            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "not-a-number");
6654            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "nope");
6655
6656            let config = SelfHealingConfig::default().with_env_overrides();
6657            assert_eq!(config.auto_start_cooldown_secs, 30);
6658            assert_eq!(config.auto_start_timeout_secs, 3);
6659        }
6660    }
6661
6662    // -------------------------------------------------------------------------
6663    // Preflight Health Guard Tests (bd-3eaa)
6664    // -------------------------------------------------------------------------
6665
6666    #[test]
6667    fn test_load_per_core_calculation() {
6668        let _guard = test_guard!();
6669        let mut caps = WorkerCapabilities::new();
6670        // No metrics -> None
6671        assert!(caps.load_per_core().is_none());
6672
6673        // Only load, no cpus -> None
6674        caps.load_avg_1 = Some(4.0);
6675        assert!(caps.load_per_core().is_none());
6676
6677        // Both available
6678        caps.num_cpus = Some(4);
6679        assert_eq!(caps.load_per_core(), Some(1.0)); // 4.0 / 4 = 1.0
6680
6681        // High load
6682        caps.load_avg_1 = Some(16.0);
6683        assert_eq!(caps.load_per_core(), Some(4.0)); // 16.0 / 4 = 4.0
6684    }
6685
6686    #[test]
6687    fn test_is_high_load() {
6688        let _guard = test_guard!();
6689        let mut caps = WorkerCapabilities::new();
6690        caps.load_avg_1 = Some(8.0);
6691        caps.num_cpus = Some(4);
6692        // load_per_core = 2.0
6693
6694        // Below threshold
6695        assert_eq!(caps.is_high_load(3.0), Some(false));
6696        // At threshold
6697        assert_eq!(caps.is_high_load(2.0), Some(false)); // equal is OK
6698        // Above threshold
6699        assert_eq!(caps.is_high_load(1.5), Some(true));
6700
6701        // No metrics -> None (fail-open)
6702        caps.load_avg_1 = None;
6703        assert!(caps.is_high_load(1.0).is_none());
6704    }
6705
6706    #[test]
6707    fn test_is_low_disk() {
6708        let _guard = test_guard!();
6709        let mut caps = WorkerCapabilities::new();
6710        caps.disk_free_gb = Some(15.0);
6711
6712        // Above threshold
6713        assert_eq!(caps.is_low_disk(10.0), Some(false));
6714        // At threshold
6715        assert_eq!(caps.is_low_disk(15.0), Some(false)); // equal is OK
6716        // Below threshold
6717        assert_eq!(caps.is_low_disk(20.0), Some(true));
6718
6719        // No metrics -> None (fail-open)
6720        caps.disk_free_gb = None;
6721        assert!(caps.is_low_disk(10.0).is_none());
6722    }
6723
6724    #[test]
6725    fn low_disk_judges_the_build_disk_not_a_small_tmpfs() {
6726        let _guard = test_guard!();
6727        // GH #78: a 16 GB tmpfs /tmp is the tightest mount by ratio, but the
6728        // build trees live on a roomy data disk.
6729        let mut caps = WorkerCapabilities::new();
6730        caps.disk_free_gb = Some(8.0);
6731        caps.disk_total_gb = Some(15.5);
6732        caps.build_disk_free_gb = Some(1700.0);
6733        caps.build_disk_total_gb = Some(1900.0);
6734        assert_eq!(caps.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6735        assert_eq!(caps.is_low_disk(10.0), Some(false));
6736
6737        // A full build disk is low even when /tmp is roomy.
6738        caps.build_disk_free_gb = Some(4.0);
6739        assert_eq!(caps.is_low_disk(10.0), Some(true));
6740
6741        // An older rch-wkr without the build sample keeps the old behaviour.
6742        caps.build_disk_free_gb = None;
6743        caps.build_disk_total_gb = None;
6744        assert_eq!(caps.build_disk_gb(), (Some(8.0), Some(15.5)));
6745        assert_eq!(caps.is_low_disk(10.0), Some(true));
6746    }
6747
6748    #[test]
6749    fn build_disk_sample_is_optional_on_the_wire() {
6750        let _guard = test_guard!();
6751        let old: WorkerCapabilities =
6752            serde_json::from_str(r#"{"disk_free_gb":13.9,"disk_total_gb":15.5}"#).unwrap();
6753        assert_eq!(old.build_disk_free_gb, None);
6754        assert_eq!(old.build_disk_gb(), (Some(13.9), Some(15.5)));
6755
6756        let new: WorkerCapabilities = serde_json::from_str(
6757            r#"{"disk_free_gb":13.9,"disk_total_gb":15.5,"build_disk_free_gb":1700.0,"build_disk_total_gb":1900.0}"#,
6758        )
6759        .unwrap();
6760        assert_eq!(new.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6761        let json = serde_json::to_string(&new).unwrap();
6762        assert!(json.contains("\"build_disk_free_gb\":1700.0"), "{json}");
6763    }
6764
6765    #[test]
6766    fn test_selection_config_preflight_defaults() {
6767        let _guard = test_guard!();
6768        let config = SelectionConfig::default();
6769        // Default thresholds
6770        assert_eq!(config.max_load_per_core, Some(2.0));
6771        assert_eq!(config.min_free_gb, Some(10.0));
6772        assert_eq!(config.disk_gb_per_slot, 10.0);
6773    }
6774
6775    #[test]
6776    fn test_selection_config_preflight_serde() {
6777        let _guard = test_guard!();
6778        let config = SelectionConfig {
6779            max_load_per_core: Some(3.5),
6780            min_free_gb: Some(25.0),
6781            disk_gb_per_slot: 0.83,
6782            ..Default::default()
6783        };
6784        let json = serde_json::to_string(&config).unwrap();
6785        let parsed: SelectionConfig = serde_json::from_str(&json).unwrap();
6786        assert_eq!(parsed.max_load_per_core, Some(3.5));
6787        assert_eq!(parsed.min_free_gb, Some(25.0));
6788        assert_eq!(parsed.disk_gb_per_slot, 0.83);
6789    }
6790
6791    #[test]
6792    fn test_selection_disk_slot_budget_validation() {
6793        let _guard = test_guard!();
6794        for value in ["0.0", "-1.0", "nan", "inf", "-inf"] {
6795            let result = toml::from_str::<SelectionConfig>(&format!("disk_gb_per_slot = {value}"));
6796            assert!(result.is_err(), "invalid disk slot budget {value}");
6797        }
6798        let default: SelectionConfig = toml::from_str("").unwrap();
6799        assert_eq!(default.disk_gb_per_slot, 10.0);
6800    }
6801
6802    #[test]
6803    fn test_selection_disk_weight_validation_and_roundtrip() {
6804        for value in ["-0.1", "1.1", "nan", "inf", "-inf"] {
6805            assert!(toml::from_str::<SelectionWeightConfig>(&format!("disk = {value}")).is_err());
6806        }
6807        for value in [0.0, 0.2, 1.0] {
6808            let weights: SelectionWeightConfig =
6809                toml::from_str(&format!("disk = {value}")).unwrap();
6810            assert_eq!(weights.disk, value);
6811            let roundtrip: SelectionWeightConfig =
6812                serde_json::from_str(&serde_json::to_string(&weights).unwrap()).unwrap();
6813            assert_eq!(roundtrip.disk, value);
6814        }
6815        let defaults: SelectionWeightConfig = toml::from_str("").unwrap();
6816        assert_eq!(defaults.disk, 0.2);
6817    }
6818
6819    #[test]
6820    fn test_worker_capabilities_health_metrics_serde() {
6821        let _guard = test_guard!();
6822        let caps = WorkerCapabilities {
6823            num_cpus: Some(8),
6824            load_avg_1: Some(1.5),
6825            load_avg_5: Some(2.0),
6826            load_avg_15: Some(1.8),
6827            disk_free_gb: Some(50.5),
6828            disk_total_gb: Some(100.0),
6829            ..Default::default()
6830        };
6831        let json = serde_json::to_string(&caps).unwrap();
6832        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
6833        assert_eq!(parsed.num_cpus, Some(8));
6834        assert_eq!(parsed.load_avg_1, Some(1.5));
6835        assert_eq!(parsed.disk_free_gb, Some(50.5));
6836    }
6837
6838    #[test]
6839    fn microarch_level_classifies_ivy_bridge_as_v2() {
6840        // Abbreviated real flag set from ovh-b (Xeon E3-1245 V2, Ivy Bridge):
6841        // AVX present, but no avx2/bmi/fma — the bd-68hon SIGILL class.
6842        let ivy = "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov \
6843                   pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx rdtscp lm \
6844                   constant_tsc pni pclmulqdq ssse3 cx16 sse4_1 sse4_2 x2apic \
6845                   popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm";
6846        assert_eq!(x86_64_microarch_level_from_flags(ivy), 2);
6847    }
6848
6849    #[test]
6850    fn microarch_level_classifies_v3_and_v4() {
6851        let v3 = "cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 abm avx avx2 bmi1 bmi2 \
6852                  f16c fma movbe xsave";
6853        assert_eq!(x86_64_microarch_level_from_flags(v3), 3);
6854
6855        let v4 = format!("{v3} avx512f avx512bw avx512cd avx512dq avx512vl");
6856        assert_eq!(x86_64_microarch_level_from_flags(&v4), 4);
6857    }
6858
6859    #[test]
6860    fn microarch_level_v1_for_ancient_or_empty_flags() {
6861        assert_eq!(x86_64_microarch_level_from_flags(""), 1);
6862        assert_eq!(x86_64_microarch_level_from_flags("fpu mmx sse sse2"), 1);
6863    }
6864
6865    #[test]
6866    fn pre_v3_predicate_only_fires_on_affirmative_evidence() {
6867        let mut caps = WorkerCapabilities::new();
6868        assert!(!caps.is_pre_v3_x86(), "unknown must not deprioritize");
6869        caps.cpu_microarch_level = Some(2);
6870        assert!(caps.is_pre_v3_x86());
6871        caps.cpu_microarch_level = Some(3);
6872        assert!(!caps.is_pre_v3_x86());
6873    }
6874}