Skip to main content

rch_common/
types.rs

1//! Common types used across RCH components.
2
3use crate::{CompilationKind, toolchain::ToolchainInfo};
4use rand::RngExt;
5use schemars::JsonSchema;
6use serde::{Deserialize, Serialize};
7use std::path::PathBuf;
8
9/// Unique identifier for a worker in the fleet.
10#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
11pub struct WorkerId(pub String);
12
13impl WorkerId {
14    pub fn new(id: impl Into<String>) -> Self {
15        Self(id.into())
16    }
17
18    pub fn as_str(&self) -> &str {
19        &self.0
20    }
21}
22
23impl std::fmt::Display for WorkerId {
24    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
25        write!(f, "{}", self.0)
26    }
27}
28
29/// Status of a worker in the fleet.
30///
31/// State transitions:
32/// ```text
33/// HEALTHY ←→ DEGRADED (automatic based on response times)
34///    ↓           ↓
35///    ↓       UNREACHABLE (automatic on heartbeat failure)
36///    ↓
37/// DRAINING (via `rch workers drain`) - finishing current jobs
38///    ↓
39/// DRAINED (automatic when all jobs complete) - idle, ready to disable
40///    ↓
41/// DISABLED (via `rch workers disable`) - completely offline
42///
43/// Use `rch workers enable` to return from DRAINING/DRAINED/DISABLED → HEALTHY
44/// ```
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
46#[serde(rename_all = "snake_case")]
47pub enum WorkerStatus {
48    /// Worker is healthy and accepting jobs.
49    #[default]
50    Healthy,
51    /// Worker is responding slowly.
52    Degraded,
53    /// Worker failed to respond to heartbeat.
54    Unreachable,
55    /// Worker is not accepting new jobs (finishing current).
56    Draining,
57    /// Worker has finished draining (no active jobs, idle).
58    Drained,
59    /// Worker is manually disabled.
60    Disabled,
61}
62
63/// Circuit breaker state for a worker.
64#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
65#[serde(rename_all = "snake_case")]
66pub enum CircuitState {
67    /// Normal operation.
68    #[default]
69    Closed,
70    /// Circuit is open (short-circuit).
71    Open,
72    /// Circuit is half-open (probing).
73    HalfOpen,
74}
75
76/// Required runtime for command execution.
77#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
78#[serde(rename_all = "lowercase")]
79pub enum RequiredRuntime {
80    /// No specific runtime required (default).
81    #[default]
82    None,
83    /// Requires Rust toolchain.
84    Rust,
85    /// Requires Bun runtime.
86    Bun,
87    /// Requires Node.js runtime.
88    Node,
89    /// Requires the Nix package manager (nix binary + populated `/nix/store`).
90    Nix,
91    /// Requires the Go toolchain (`go` binary).
92    Go,
93    /// Requires the zig cross-compilation toolchain (`cargo-zigbuild` + `zig`).
94    Zig,
95}
96
97/// Per-command priority hint for worker selection.
98///
99/// This is an *input hint* from the caller (typically the hook) and should not
100/// change default behavior when omitted.
101#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
102#[serde(rename_all = "lowercase")]
103pub enum CommandPriority {
104    Low,
105    #[default]
106    Normal,
107    High,
108}
109
110impl std::fmt::Display for CommandPriority {
111    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112        let value = match self {
113            Self::Low => "low",
114            Self::Normal => "normal",
115            Self::High => "high",
116        };
117        write!(f, "{}", value)
118    }
119}
120
121impl std::str::FromStr for CommandPriority {
122    type Err = ();
123
124    fn from_str(s: &str) -> Result<Self, Self::Err> {
125        match s.trim().to_lowercase().as_str() {
126            "low" => Ok(Self::Low),
127            "normal" => Ok(Self::Normal),
128            "high" => Ok(Self::High),
129            _ => Err(()),
130        }
131    }
132}
133
134/// Worker selection request sent from hook to daemon.
135#[derive(Debug, Clone, Serialize, Deserialize)]
136pub struct SelectionRequest {
137    /// Project identifier (usually directory name or hash).
138    pub project: String,
139    /// Full command being executed (optional, for active build tracking).
140    #[serde(default, skip_serializing_if = "Option::is_none")]
141    pub command: Option<String>,
142    /// Priority hint for this request.
143    #[serde(default)]
144    pub command_priority: CommandPriority,
145    /// Estimated CPU cores needed for this compilation.
146    pub estimated_cores: u32,
147    /// Additional free GiB required on the worker's reported build filesystem.
148    /// A positive value is a hard admission budget, retained by the daemon for
149    /// this build's lifetime; zero leaves disk admission to the ordinary policy.
150    #[serde(default, skip_serializing_if = "disk_headroom_disabled")]
151    pub disk_headroom_gib: u32,
152    /// Preferred worker IDs (e.g., from project config).
153    #[serde(default)]
154    pub preferred_workers: Vec<WorkerId>,
155    /// Rust toolchain information for the project.
156    #[serde(default)]
157    pub toolchain: Option<ToolchainInfo>,
158    /// Required runtime for command execution.
159    #[serde(default)]
160    pub required_runtime: RequiredRuntime,
161    /// Classification decision latency in microseconds (for AGENTS.md compliance).
162    /// This tracks how long the 5-tier classification took on the hook side.
163    #[serde(default, skip_serializing_if = "Option::is_none")]
164    pub classification_duration_us: Option<u64>,
165    /// Job-mode admission (bd-g7rpy): set by `rch exec --job` only. When the
166    /// one-active-job-per-project-per-worker guard excludes every otherwise-
167    /// eligible worker, a job request may QUEUE on that exclusion instead of
168    /// failing over to local execution. Compilation requests never set it,
169    /// preserving their immediate NoAdmissibleWorkers behavior.
170    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
171    pub job_mode: bool,
172    /// Operator-declared named tools this work requires VERIFIED on the worker
173    /// (`rch exec --job --require-tool NAME`, or a project's
174    /// `jobs.required_tools`).
175    ///
176    /// Job mode only — the compilation classifier never sets it, so ordinary
177    /// offloaded builds keep their existing admission behavior exactly. Empty
178    /// means no gate; a name no worker has verified excludes every worker,
179    /// because the caller asked for the gate precisely because the job cannot
180    /// run without the tool.
181    #[serde(default, skip_serializing_if = "Vec::is_empty")]
182    pub required_tools: Vec<String>,
183    /// Process ID of the hook (for active build tracking).
184    #[serde(default, skip_serializing_if = "Option::is_none")]
185    pub hook_pid: Option<u32>,
186}
187
188fn disk_headroom_disabled(gib: &u32) -> bool {
189    *gib == 0
190}
191
192/// Reason for worker selection result.
193///
194/// Provides context when no worker is available, enabling informative
195/// fallback messages in the hook.
196#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
197#[serde(rename_all = "snake_case")]
198pub enum SelectionReason {
199    /// Worker assigned successfully.
200    Success,
201    /// No workers configured in workers.toml.
202    NoWorkersConfigured,
203    /// All workers are unreachable (failed health checks).
204    AllWorkersUnreachable,
205    /// All workers have circuits open (after repeated failures).
206    AllCircuitsOpen,
207    /// All workers are at capacity (no available slots).
208    AllWorkersBusy,
209    /// Workers were present but none passed selection health thresholds.
210    NoWorkersPassedHealth,
211    /// All candidate workers failed hard preflight checks.
212    AllWorkersFailedPreflight,
213    /// All candidate workers failed repo convergence checks (repos missing/stale/failed).
214    AllWorkersFailedConvergence,
215    /// Workers exist but admission was blocked by concrete capacity/preflight reasons.
216    NoAdmissibleWorkers(String),
217    /// No workers match required tags or preferences.
218    NoMatchingWorkers,
219    /// No workers have the required runtime (e.g., Bun, Node).
220    NoWorkersWithRuntime(String),
221    /// Internal error during selection.
222    SelectionError(String),
223    /// Worker assigned via affinity pinning (recent successful build).
224    AffinityPinned,
225    /// Worker assigned via last-success fallback (all others unavailable).
226    AffinityFallback,
227}
228
229/// Wire protocol version for daemon `/select-worker` responses.
230///
231/// This is intentionally separate from the broader CLI API envelope version:
232/// selection responses are emitted as a raw [`SelectionResponse`] on the hook
233/// hot path, so the client needs a small explicit compatibility marker.
234pub const SELECTION_RESPONSE_PROTOCOL_VERSION: u64 = 1;
235
236impl std::fmt::Display for SelectionReason {
237    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
238        match self {
239            Self::Success => write!(f, "worker assigned successfully"),
240            Self::NoWorkersConfigured => write!(f, "no workers configured"),
241            Self::AllWorkersUnreachable => write!(f, "all workers unreachable"),
242            Self::AllCircuitsOpen => write!(f, "all worker circuits open"),
243            Self::AllWorkersBusy => write!(f, "all workers at capacity"),
244            Self::NoWorkersPassedHealth => write!(f, "no workers passed health thresholds"),
245            Self::AllWorkersFailedPreflight => write!(f, "all workers failed preflight checks"),
246            Self::AllWorkersFailedConvergence => {
247                write!(f, "all workers failed repo convergence checks")
248            }
249            Self::NoAdmissibleWorkers(summary) => {
250                write!(f, "no admissible workers: {}", summary)
251            }
252            Self::NoMatchingWorkers => write!(f, "no matching workers found"),
253            Self::NoWorkersWithRuntime(rt) => write!(f, "no workers with {} installed", rt),
254            Self::SelectionError(e) => write!(f, "selection error: {}", e),
255            Self::AffinityPinned => write!(f, "worker assigned via affinity pinning"),
256            Self::AffinityFallback => write!(f, "worker assigned via last-success fallback"),
257        }
258    }
259}
260
261// ============================================================================
262// Worker Selection Strategy
263// ============================================================================
264
265/// Worker selection strategy determining how workers are chosen for jobs.
266#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
267#[serde(rename_all = "snake_case")]
268pub enum SelectionStrategy {
269    /// Legacy/manual control: prioritize worker.priority first, then break ties
270    /// using command-aware cache/speed hints.
271    Priority,
272    /// Select worker with highest SpeedScore.
273    /// Best for performance-critical builds with homogeneous workers.
274    Fastest,
275    /// Balance all factors: SpeedScore, load, health, cache affinity.
276    /// Default for general use across diverse worker pools.
277    #[default]
278    Balanced,
279    /// Prefer workers with warm caches for the project.
280    /// Best for incremental builds on large codebases.
281    CacheAffinity,
282    /// Weighted random selection favoring fast workers but ensuring fairness.
283    /// Prevents hot-spotting when many workers are available.
284    FairFastest,
285}
286
287impl std::fmt::Display for SelectionStrategy {
288    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
289        let name = match self {
290            Self::Priority => "priority",
291            Self::Fastest => "fastest",
292            Self::Balanced => "balanced",
293            Self::CacheAffinity => "cache_affinity",
294            Self::FairFastest => "fair_fastest",
295        };
296        write!(f, "{}", name)
297    }
298}
299
300impl std::str::FromStr for SelectionStrategy {
301    type Err = String;
302
303    fn from_str(s: &str) -> Result<Self, Self::Err> {
304        match s.trim().to_lowercase().as_str() {
305            "priority" => Ok(Self::Priority),
306            "fastest" => Ok(Self::Fastest),
307            "balanced" => Ok(Self::Balanced),
308            "cache_affinity" | "cache-affinity" | "cacheaffinity" => Ok(Self::CacheAffinity),
309            "fair_fastest" | "fair-fastest" | "fairfastest" => Ok(Self::FairFastest),
310            _ => Err(format!(
311                "unknown selection strategy '{}', expected one of: priority, fastest, balanced, cache_affinity, fair_fastest",
312                s
313            )),
314        }
315    }
316}
317
318/// Configuration for the worker selection algorithm.
319#[derive(Debug, Clone, Serialize, Deserialize)]
320pub struct SelectionConfig {
321    /// Selection strategy to use.
322    #[serde(default)]
323    pub strategy: SelectionStrategy,
324    /// Minimum success rate (0.0-1.0) for a worker to be considered.
325    #[serde(default = "default_min_success_rate")]
326    pub min_success_rate: f64,
327    /// Factor weights for the balanced strategy.
328    #[serde(default)]
329    pub weights: SelectionWeightConfig,
330    /// Fairness settings for fair_fastest strategy.
331    #[serde(default)]
332    pub fairness: FairnessConfig,
333    /// Cache affinity settings for project-to-worker pinning.
334    #[serde(default)]
335    pub affinity: AffinityConfig,
336
337    // Preflight health guards (bd-3eaa)
338    /// Maximum load-per-core threshold. Workers exceeding this are skipped.
339    /// Set to None to disable load-based filtering.
340    #[serde(default = "default_max_load_per_core")]
341    pub max_load_per_core: Option<f64>,
342    /// Minimum free disk space in GB. Workers below this are skipped.
343    /// Set to None to disable the free-space floor; per-slot disk derating still applies.
344    #[serde(default = "default_min_free_gb")]
345    pub min_free_gb: Option<f64>,
346    /// Disk budget in GiB per concurrent slot, after reserving `min_free_gb`.
347    /// Must be finite and positive. Unknown disk telemetry preserves configured slots.
348    #[serde(
349        default = "default_disk_gb_per_slot",
350        deserialize_with = "deserialize_disk_gb_per_slot"
351    )]
352    pub disk_gb_per_slot: f64,
353}
354
355impl Default for SelectionConfig {
356    fn default() -> Self {
357        Self {
358            strategy: SelectionStrategy::default(),
359            min_success_rate: default_min_success_rate(),
360            weights: SelectionWeightConfig::default(),
361            fairness: FairnessConfig::default(),
362            affinity: AffinityConfig::default(),
363            max_load_per_core: default_max_load_per_core(),
364            min_free_gb: default_min_free_gb(),
365            disk_gb_per_slot: default_disk_gb_per_slot(),
366        }
367    }
368}
369
370fn default_min_success_rate() -> f64 {
371    0.8
372}
373
374fn default_max_load_per_core() -> Option<f64> {
375    Some(2.0) // Skip workers with load/core > 2.0
376}
377
378fn default_min_free_gb() -> Option<f64> {
379    Some(10.0) // Skip workers with < 10 GB free
380}
381
382fn default_disk_gb_per_slot() -> f64 {
383    10.0
384}
385
386fn deserialize_disk_gb_per_slot<'de, D>(deserializer: D) -> Result<f64, D::Error>
387where
388    D: serde::Deserializer<'de>,
389{
390    let budget = f64::deserialize(deserializer)?;
391    if budget.is_finite() && budget > 0.0 {
392        Ok(budget)
393    } else {
394        Err(serde::de::Error::custom(
395            "disk_gb_per_slot must be finite and greater than zero",
396        ))
397    }
398}
399
400/// Weight configuration for the balanced selection strategy.
401#[derive(Debug, Clone, Serialize, Deserialize)]
402pub struct SelectionWeightConfig {
403    /// Weight for SpeedScore (0.0-1.0).
404    #[serde(default = "default_weight_speedscore")]
405    pub speedscore: f64,
406    /// Weight for available slots (0.0-1.0).
407    #[serde(default = "default_weight_slots")]
408    pub slots: f64,
409    /// Weight for health/success rate (0.0-1.0).
410    #[serde(default = "default_weight_health")]
411    pub health: f64,
412    /// Weight for cache affinity (0.0-1.0).
413    #[serde(default = "default_weight_cache")]
414    pub cache: f64,
415    /// Weight for network latency (0.0-1.0).
416    #[serde(default = "default_weight_network")]
417    pub network: f64,
418    /// Weight for worker priority (0.0-1.0).
419    #[serde(default = "default_weight_priority")]
420    pub priority: f64,
421    /// Weight for disk headroom (0.0-1.0); zero disables disk ranking.
422    #[serde(
423        default = "default_weight_disk",
424        deserialize_with = "deserialize_weight_disk"
425    )]
426    pub disk: f64,
427    /// Penalty multiplier for half-open circuit workers (0.0-1.0).
428    #[serde(default = "default_half_open_penalty")]
429    pub half_open_penalty: f64,
430}
431
432impl Default for SelectionWeightConfig {
433    fn default() -> Self {
434        Self {
435            speedscore: default_weight_speedscore(),
436            slots: default_weight_slots(),
437            health: default_weight_health(),
438            cache: default_weight_cache(),
439            network: default_weight_network(),
440            priority: default_weight_priority(),
441            disk: default_weight_disk(),
442            half_open_penalty: default_half_open_penalty(),
443        }
444    }
445}
446
447fn default_weight_speedscore() -> f64 {
448    0.5
449}
450fn default_weight_slots() -> f64 {
451    0.4
452}
453fn default_weight_health() -> f64 {
454    0.3
455}
456fn default_weight_cache() -> f64 {
457    0.2
458}
459fn default_weight_network() -> f64 {
460    0.1
461}
462fn default_weight_priority() -> f64 {
463    // Worker priority is a first-class selection factor: operators set high
464    // priority on preferred boxes (e.g. fast dedicated workers) and expect them
465    // to actually receive work. At the old 0.1 a top-priority worker gained at
466    // most +0.1 to its balanced score — too weak to overcome a warm-cache or
467    // lower-latency competitor, so the highest-priority workers could sit idle
468    // indefinitely. Load/slots still throttle a high-priority worker as it fills.
469    0.5
470}
471fn default_half_open_penalty() -> f64 {
472    0.5
473}
474
475fn default_weight_disk() -> f64 {
476    0.2
477}
478
479fn deserialize_weight_disk<'de, D>(deserializer: D) -> Result<f64, D::Error>
480where
481    D: serde::Deserializer<'de>,
482{
483    let weight = f64::deserialize(deserializer)?;
484    if weight.is_finite() && (0.0..=1.0).contains(&weight) {
485        Ok(weight)
486    } else {
487        Err(serde::de::Error::custom(
488            "disk weight must be finite and between zero and one",
489        ))
490    }
491}
492
493/// Fairness settings for the fair_fastest selection strategy.
494#[derive(Debug, Clone, Serialize, Deserialize)]
495pub struct FairnessConfig {
496    /// Lookback window in seconds for tracking recent selections.
497    #[serde(default = "default_fairness_lookback_secs")]
498    pub lookback_secs: u64,
499    /// Maximum consecutive selections for a single worker before penalty.
500    #[serde(default = "default_max_consecutive_selections")]
501    pub max_consecutive_selections: u32,
502}
503
504impl Default for FairnessConfig {
505    fn default() -> Self {
506        Self {
507            lookback_secs: default_fairness_lookback_secs(),
508            max_consecutive_selections: default_max_consecutive_selections(),
509        }
510    }
511}
512
513fn default_fairness_lookback_secs() -> u64 {
514    300 // 5 minutes
515}
516
517fn default_max_consecutive_selections() -> u32 {
518    3
519}
520
521/// Cache affinity settings for project-to-worker pinning.
522#[derive(Debug, Clone, Serialize, Deserialize)]
523pub struct AffinityConfig {
524    /// Enable affinity pinning (pin projects to last successful worker).
525    #[serde(default = "default_affinity_enabled")]
526    pub enabled: bool,
527    /// Time window in minutes to pin a project to its last successful worker.
528    /// After this window expires, normal selection resumes.
529    #[serde(default = "default_affinity_pin_minutes")]
530    pub pin_minutes: u64,
531    /// Enable last-success fallback when no workers match selection criteria.
532    /// If enabled, the daemon will attempt the last successful worker for a project
533    /// when all other workers are unavailable or fail selection.
534    #[serde(default = "default_last_success_fallback")]
535    pub enable_last_success_fallback: bool,
536    /// Minimum success rate for the last-success worker to be used as fallback.
537    /// Must be lower than min_success_rate to be meaningful as a fallback.
538    #[serde(default = "default_fallback_min_success_rate")]
539    pub fallback_min_success_rate: f64,
540}
541
542impl Default for AffinityConfig {
543    fn default() -> Self {
544        Self {
545            enabled: default_affinity_enabled(),
546            pin_minutes: default_affinity_pin_minutes(),
547            enable_last_success_fallback: default_last_success_fallback(),
548            fallback_min_success_rate: default_fallback_min_success_rate(),
549        }
550    }
551}
552
553fn default_affinity_enabled() -> bool {
554    true
555}
556
557fn default_affinity_pin_minutes() -> u64 {
558    60 // 1 hour default
559}
560
561fn default_last_success_fallback() -> bool {
562    true
563}
564
565fn default_fallback_min_success_rate() -> f64 {
566    0.5 // More lenient than normal min_success_rate (0.8)
567}
568
569/// Details about a selected worker for remote compilation.
570#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
571pub struct SelectedWorker {
572    /// Selected worker ID.
573    pub id: WorkerId,
574    /// Host address for SSH.
575    pub host: String,
576    /// SSH user.
577    pub user: String,
578    /// Path to SSH identity file.
579    pub identity_file: String,
580    /// Number of slots available on this worker after reservation.
581    pub slots_available: u32,
582    /// Worker's speed score (0-100).
583    pub speed_score: f64,
584    /// Operator-declared host OS (e.g. `windows`), if any. Propagated so the
585    /// hook can pick the right transport (Windows workers use `C:/rch` +
586    /// tar-over-ssh instead of the Unix rsync path). `None` for the historical
587    /// unlabelled workers. `#[serde(default)]` keeps old daemons/hooks
588    /// wire-compatible.
589    #[serde(default)]
590    pub declared_os: Option<String>,
591}
592
593/// Worker selection response from daemon to hook.
594#[derive(Debug, Clone, Serialize, Deserialize)]
595pub struct SelectionResponse {
596    /// Selected worker details, if available.
597    pub worker: Option<SelectedWorker>,
598    /// Reason for the selection result.
599    pub reason: SelectionReason,
600    /// Optional build ID assigned by the daemon (for active build tracking).
601    #[serde(default, skip_serializing_if = "Option::is_none")]
602    pub build_id: Option<u64>,
603    /// Optional per-worker selector diagnostics for machine-readable triage.
604    #[serde(default, skip_serializing_if = "Option::is_none")]
605    pub diagnostics: Option<SelectionDiagnostics>,
606}
607
608/// Final selector decision for a worker considered during routing.
609#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
610#[serde(rename_all = "snake_case")]
611pub enum WorkerSelectionDiagnosticDecision {
612    /// Worker passed hard gates and normal health threshold.
613    Allow,
614    /// Worker failed normal health threshold but passed fallback threshold.
615    FallbackCandidate,
616    /// Worker was denied by a hard gate or both health thresholds.
617    Deny,
618}
619
620/// Per-worker selector facts captured at worker-selection time.
621#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
622pub struct WorkerSelectionDiagnostic {
623    /// Worker ID.
624    pub worker_id: WorkerId,
625    /// Worker lifecycle/status label.
626    pub status: String,
627    /// Circuit breaker state label.
628    pub circuit_state: String,
629    /// Current normalized pressure state.
630    pub pressure_state: String,
631    /// Stable pressure reason code.
632    pub pressure_reason_code: String,
633    /// Rolling selector success rate, where available.
634    #[serde(skip_serializing_if = "Option::is_none")]
635    pub success_rate: Option<f64>,
636    /// Normal selection success-rate threshold.
637    pub min_success_rate: f64,
638    /// Last-success fallback success-rate threshold.
639    pub fallback_min_success_rate: f64,
640    /// Required runtime for this command.
641    pub required_runtime: RequiredRuntime,
642    /// Whether this worker has the required runtime.
643    pub runtime_available: bool,
644    /// Available slots at selection time.
645    pub available_slots: u32,
646    /// Total configured slots.
647    pub total_slots: u32,
648    /// Estimated slots requested by this command.
649    pub estimated_cores: u32,
650    /// Whether this worker was excluded because it already runs this project.
651    pub active_project_excluded: bool,
652    /// Final per-worker selector decision.
653    pub final_decision: WorkerSelectionDiagnosticDecision,
654    /// Human-readable final reason.
655    pub final_reason: String,
656    /// Stable machine-readable reasons collected for this worker.
657    pub reason_codes: Vec<String>,
658}
659
660/// Machine-readable worker selector diagnostics for a selection request.
661#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
662pub struct SelectionDiagnostics {
663    /// Required runtime for this command.
664    pub required_runtime: RequiredRuntime,
665    /// Estimated slots requested by this command.
666    pub estimated_cores: u32,
667    /// Normal success-rate threshold.
668    pub min_success_rate: f64,
669    /// Last-success fallback success-rate threshold.
670    pub fallback_min_success_rate: f64,
671    /// Count of workers excluded because they already run this project.
672    pub active_project_exclusion_count: usize,
673    /// One diagnostic entry per configured worker.
674    pub workers: Vec<WorkerSelectionDiagnostic>,
675}
676
677/// Request to release reserved worker slots.
678#[derive(Debug, Clone, Serialize, Deserialize)]
679pub struct ReleaseRequest {
680    /// ID of the worker to release slots on.
681    pub worker_id: WorkerId,
682    /// Number of slots to release.
683    pub slots: u32,
684    /// Optional build ID to mark complete.
685    #[serde(default, skip_serializing_if = "Option::is_none")]
686    pub build_id: Option<u64>,
687    /// Exact client wrapper identity associated with the admitted build.
688    #[serde(default, skip_serializing_if = "Option::is_none")]
689    pub local_wrapper_id: Option<String>,
690    /// Optional exit code for the build (used to finalize active build tracking).
691    #[serde(default, skip_serializing_if = "Option::is_none")]
692    pub exit_code: Option<i32>,
693    /// Optional total duration in milliseconds (pipeline or execution).
694    ///
695    /// If omitted, the daemon will compute duration from the active build's start time.
696    #[serde(default, skip_serializing_if = "Option::is_none")]
697    pub duration_ms: Option<u64>,
698    /// Optional bytes transferred during the pipeline (upload + artifact download).
699    #[serde(default, skip_serializing_if = "Option::is_none")]
700    pub bytes_transferred: Option<u64>,
701    /// Optional per-phase timing breakdown for the build pipeline.
702    #[serde(default, skip_serializing_if = "Option::is_none")]
703    pub timing: Option<CommandTimingBreakdown>,
704    /// The hook classified this non-zero exit as a worker fault (missing
705    /// toolchain or system library, CPU-capability signal, full disk), so the
706    /// daemon records no cache warmth for it (review of GH #81).
707    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
708    pub worker_fault: bool,
709    /// The failed remote command reported exhausted disk space or quota.
710    /// After matching the durable build owner, the daemon temporarily bypasses
711    /// this worker until fresh recovery probes establish disk headroom.
712    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
713    pub worker_disk_full: bool,
714    /// Actual remote source and output roots used by the failed attempt.
715    /// Recovery must measure these filesystems as well as configured roots.
716    #[serde(default, skip_serializing_if = "Vec::is_empty")]
717    pub worker_disk_roots: Vec<String>,
718}
719
720/// Build execution phase for daemon heartbeat tracking.
721#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
722#[serde(rename_all = "snake_case")]
723pub enum BuildHeartbeatPhase {
724    /// Uploading sources/dependencies to the worker.
725    SyncUp,
726    /// Running the remote compilation/test command.
727    Execute,
728    /// Downloading build artifacts from the worker.
729    SyncDown,
730    /// Build is completing or finalizing.
731    Finalize,
732}
733
734/// Request to update liveness/progress for an active build.
735#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
736pub struct BuildHeartbeatRequest {
737    /// Active build identifier assigned by the daemon.
738    pub build_id: u64,
739    /// Worker executing the build.
740    pub worker_id: WorkerId,
741    /// Hook process ID sending the heartbeat.
742    #[serde(default, skip_serializing_if = "Option::is_none")]
743    pub hook_pid: Option<u32>,
744    /// Client-minted wrapper identity for local durable-lease correlation.
745    #[serde(default, skip_serializing_if = "Option::is_none")]
746    pub local_wrapper_id: Option<String>,
747    /// Remote file containing the process-group leader PID for cancellation.
748    #[serde(default, skip_serializing_if = "Option::is_none")]
749    pub remote_pgid_file: Option<String>,
750    /// Current build execution phase.
751    pub phase: BuildHeartbeatPhase,
752    /// Optional human-readable progress detail.
753    #[serde(default, skip_serializing_if = "Option::is_none")]
754    pub detail: Option<String>,
755    /// Optional monotonic progress counter.
756    ///
757    /// The hook increments this when it observes meaningful progress
758    /// (phase transitions, new output, or richer progress signals).
759    #[serde(default, skip_serializing_if = "Option::is_none")]
760    pub progress_counter: Option<u64>,
761    /// Optional progress estimate in [0,100].
762    #[serde(default, skip_serializing_if = "Option::is_none")]
763    pub progress_percent: Option<f64>,
764}
765
766/// Configuration for a remote worker.
767#[derive(Debug, Clone, Serialize, Deserialize)]
768pub struct WorkerConfig {
769    /// Unique identifier for this worker.
770    pub id: WorkerId,
771    /// SSH hostname or IP address.
772    pub host: String,
773    /// SSH username.
774    pub user: String,
775    /// Path to SSH private key.
776    pub identity_file: String,
777    /// Total CPU slots available on this worker.
778    pub total_slots: u32,
779    /// Priority for worker selection (higher = preferred).
780    #[serde(default = "default_priority")]
781    pub priority: u32,
782    /// Optional tags for filtering.
783    ///
784    /// Mostly descriptive, with one reserved form: `os:<name>` (see
785    /// [`crate::declared_os`]) is a hard admission gate, written in
786    /// `workers.toml` as `os = "<name>"`.
787    #[serde(default)]
788    pub tags: Vec<String>,
789    /// Operator-declared named tool probes for this worker, written in
790    /// `workers.toml` as
791    /// `tools = [{ name = "clang", command = ["clang", "--version"] }]`.
792    ///
793    /// Unlike a tag, which is an unverified naming convention, each entry is
794    /// VERIFIED by running its fixed argv on the worker as the configured user;
795    /// only a zero exit makes the tool present. That is what lets
796    /// `--require-tool` gate selection on evidence instead of on a promise.
797    #[serde(default, skip_serializing_if = "Vec::is_empty")]
798    pub tools: Vec<WorkerToolProbe>,
799}
800
801/// One operator-declared tool probe as it appears in `workers.toml`.
802///
803/// Kept as plain configuration data; [`crate::capability_probe::NamedToolProbe`]
804/// is the validated form, and the conversion is where a malformed name is
805/// refused.
806#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
807pub struct WorkerToolProbe {
808    /// The name `--require-tool` and `jobs.required_tools` refer to.
809    pub name: String,
810    /// The exact argv to run. Zero exit = present; anything else = absent.
811    pub command: Vec<String>,
812}
813
814fn default_priority() -> u32 {
815    100
816}
817
818/// Tag prefix reserved for the operator-declared host OS.
819pub const OS_TAG_PREFIX: &str = "os:";
820
821/// The host OS an operator declared for a worker, from its tags.
822///
823/// Written in `workers.toml` as `os = "windows"` and normalized into the
824/// reserved `os:windows` tag on load, so it rides the existing tag vector
825/// through every path that already carries a [`WorkerConfig`].
826///
827/// Declaring it makes the worker **exclusive**: selection admits it only for
828/// commands that require that OS, so a Windows box cannot quietly absorb a
829/// `cargo check` dispatched from Linux and return wrong-platform artifacts.
830/// Workers without the tag — every historical worker — keep taking anything.
831#[must_use]
832pub fn declared_os(tags: &[String]) -> Option<String> {
833    // `find` over the mapped values, not `find_map` over the tags: a malformed
834    // `os:` entry must not mask a valid declaration later in the list, because
835    // failing open here is precisely the "Windows worker silently takes Linux
836    // jobs" outcome the gate exists to prevent.
837    tags.iter()
838        .filter_map(|tag| tag.strip_prefix(OS_TAG_PREFIX))
839        .map(|os| os.trim().to_ascii_lowercase())
840        .find(|os| !os.is_empty())
841}
842
843/// Build the reserved tag for a declared OS.
844#[must_use]
845pub fn os_tag(os: &str) -> String {
846    format!("{OS_TAG_PREFIX}{}", os.trim().to_ascii_lowercase())
847}
848
849impl Default for WorkerConfig {
850    fn default() -> Self {
851        Self {
852            id: WorkerId::new("default-worker"),
853            host: "localhost".to_string(),
854            user: "user".to_string(),
855            identity_file: "~/.ssh/id_rsa".to_string(),
856            total_slots: 4,
857            priority: default_priority(),
858            tags: Vec::new(),
859            tools: Vec::new(),
860        }
861    }
862}
863
864/// Runtime capabilities detected on a worker.
865///
866/// These are probed during health checks and cached for routing decisions.
867/// Commands requiring specific runtimes (e.g., `bun test`) can be routed
868/// only to workers with the corresponding capability.
869#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)]
870pub struct WorkerCapabilities {
871    /// Rust compiler version (from `rustc --version`).
872    #[serde(default, skip_serializing_if = "Option::is_none")]
873    pub rustc_version: Option<String>,
874    /// Installed rustup toolchains, including their host triple when rustup
875    /// reports one (for example `nightly-2026-07-05-x86_64-unknown-linux-gnu`).
876    #[serde(default, skip_serializing_if = "Vec::is_empty")]
877    pub rustup_toolchains: Vec<String>,
878    /// Installed rustup components as `<toolchain>:<component>` facts. Component
879    /// names are normalized by removing only the probed host-triple suffix, so
880    /// `clippy-preview` can never satisfy a `clippy` requirement.
881    #[serde(default, skip_serializing_if = "Vec::is_empty")]
882    pub rustup_components: Vec<String>,
883    /// Non-fatal diagnostics gathered while probing (for example a tool binary
884    /// that had to be resolved from a well-known fallback location because the
885    /// caller's PATH was minimal). Empty in the common case; clients may
886    /// surface these so "no facts" is distinguishable from "facts show
887    /// absence" (bd-deft5).
888    #[serde(default, skip_serializing_if = "Vec::is_empty")]
889    pub probe_warnings: Vec<String>,
890    /// Bun runtime version (from `bun --version`).
891    #[serde(default, skip_serializing_if = "Option::is_none")]
892    pub bun_version: Option<String>,
893    /// Node.js version (from `node --version`).
894    #[serde(default, skip_serializing_if = "Option::is_none")]
895    pub node_version: Option<String>,
896    /// npm version (from `npm --version`).
897    #[serde(default, skip_serializing_if = "Option::is_none")]
898    pub npm_version: Option<String>,
899    /// Nix version (from `nix --version`). Presence implies a usable nix binary
900    /// with an accompanying `/nix/store`, gating `nix build`/`nix develop -c`
901    /// routing to this worker.
902    #[serde(default, skip_serializing_if = "Option::is_none")]
903    pub nix_version: Option<String>,
904    /// Go toolchain version (from `go version`). Presence gates `go build`/
905    /// `go test`/`go vet` routing to this worker.
906    #[serde(default, skip_serializing_if = "Option::is_none")]
907    pub go_version: Option<String>,
908    /// Zig version (from `zig version`). `cargo zigbuild` shells out to `zig`
909    /// as the linker/C toolchain, so the bare `zig` binary is necessary but not
910    /// sufficient — see [`WorkerCapabilities::has_zig`].
911    #[serde(default, skip_serializing_if = "Option::is_none")]
912    pub zig_version: Option<String>,
913    /// cargo-zigbuild version (from `cargo-zigbuild --version`). Probed as the
914    /// hyphenated binary rather than `cargo zigbuild --version`, which the
915    /// subcommand's own parser rejects.
916    #[serde(default, skip_serializing_if = "Option::is_none")]
917    pub cargo_zigbuild_version: Option<String>,
918    /// x86-64 microarchitecture level (1..=4) parsed from `/proc/cpuinfo`
919    /// flags on the worker (bd-6qchz / bd-68hon item 4). `Some(2)` marks a
920    /// pre-AVX2 box (e.g. ovh-b's Ivy Bridge) whose build scripts SIGILL on
921    /// x86-64-v3 codegen; selection deprioritizes such workers proactively
922    /// while the reactive SIGILL quarantine remains the primary mechanism.
923    /// `None` on non-x86 workers or when the probe cannot read cpuinfo.
924    #[serde(default, skip_serializing_if = "Option::is_none")]
925    pub cpu_microarch_level: Option<u8>,
926    /// Operator-declared named tools whose probe exited ZERO on this worker.
927    ///
928    /// The list is evidence, not configuration: a name appears here only
929    /// because the worker ran the declared argv and it succeeded. Selection
930    /// gates `--require-tool` on this, which is the difference between a
931    /// verified capability and a tag somebody wrote by hand.
932    #[serde(default, skip_serializing_if = "Vec::is_empty")]
933    pub tools_present: Vec<String>,
934    /// Operator-declared named tools whose probe RAN and did not exit zero.
935    ///
936    /// Kept separate from "never declared" so an operator can tell a worker
937    /// that is missing a tool from a fleet that never probed for it — the two
938    /// have different fixes.
939    #[serde(default, skip_serializing_if = "Vec::is_empty")]
940    pub tools_absent: Vec<String>,
941
942    // Health metrics (bd-3eaa)
943    /// Number of CPU cores on the worker.
944    #[serde(default, skip_serializing_if = "Option::is_none")]
945    pub num_cpus: Option<u32>,
946    /// 1-minute load average.
947    #[serde(default, skip_serializing_if = "Option::is_none")]
948    pub load_avg_1: Option<f64>,
949    /// 5-minute load average.
950    #[serde(default, skip_serializing_if = "Option::is_none")]
951    pub load_avg_5: Option<f64>,
952    /// 15-minute load average.
953    #[serde(default, skip_serializing_if = "Option::is_none")]
954    pub load_avg_15: Option<f64>,
955    /// Free space in GB on the tightest mount a build touches: whichever of
956    /// the projects root, its alias and `/tmp` has the lowest free ratio.
957    /// Drives pressure classification (a full `/tmp` breaks scp/mktemp).
958    #[serde(default, skip_serializing_if = "Option::is_none")]
959    pub disk_free_gb: Option<f64>,
960    /// Total size in GB of the mount reported in `disk_free_gb`.
961    #[serde(default, skip_serializing_if = "Option::is_none")]
962    pub disk_total_gb: Option<f64>,
963    /// Free space in GB on the filesystem that holds build trees (the fuller
964    /// of the projects root and its alias; `/tmp` excluded). Sizes capacity:
965    /// slot derating, disk headroom and the free-space floor (GH #78).
966    /// `None` from an `rch-wkr` that predates the field.
967    #[serde(default, skip_serializing_if = "Option::is_none")]
968    pub build_disk_free_gb: Option<f64>,
969    /// Total size in GB of the mount reported in `build_disk_free_gb`.
970    #[serde(default, skip_serializing_if = "Option::is_none")]
971    pub build_disk_total_gb: Option<f64>,
972    /// Canonical path-topology preflight status (`/data/projects` + `/dp` alias).
973    ///
974    /// `Some(false)` indicates a hard preflight failure that should exclude this
975    /// worker from remote scheduling until a subsequent successful revalidation.
976    #[serde(default, skip_serializing_if = "Option::is_none")]
977    pub projects_root_ok: Option<bool>,
978    /// Machine-readable failure reason for path-topology preflight.
979    #[serde(default, skip_serializing_if = "Option::is_none")]
980    pub projects_root_issue: Option<String>,
981    /// Unix timestamp (ms) when topology preflight last executed.
982    #[serde(default, skip_serializing_if = "Option::is_none")]
983    pub projects_root_checked_at_unix_ms: Option<i64>,
984}
985
986impl WorkerCapabilities {
987    /// Create new empty capabilities.
988    pub fn new() -> Self {
989        Self::default()
990    }
991
992    /// Create mock capabilities with Rust installed.
993    ///
994    /// Used by mock transport to simulate a worker with Rust toolchain.
995    pub fn mock_with_rust() -> Self {
996        Self {
997            rustc_version: Some("1.85.0-nightly (mock)".to_string()),
998            projects_root_ok: Some(true),
999            ..Self::default()
1000        }
1001    }
1002
1003    /// Whether this is a known pre-x86-64-v3 (no AVX2) x86 worker — the class
1004    /// whose build-script/proc-macro binaries SIGILL on v3 codegen (bd-68hon).
1005    /// `None` (unknown/non-x86) is NOT pre-v3: only affirmative evidence
1006    /// deprioritizes a worker.
1007    pub fn is_pre_v3_x86(&self) -> bool {
1008        matches!(self.cpu_microarch_level, Some(1 | 2))
1009    }
1010
1011    /// Check if this worker has Bun installed.
1012    pub fn has_bun(&self) -> bool {
1013        self.bun_version.is_some()
1014    }
1015
1016    /// Check if this worker has Node.js installed.
1017    pub fn has_node(&self) -> bool {
1018        self.node_version.is_some()
1019    }
1020
1021    /// Check if this worker has Rust installed.
1022    pub fn has_rust(&self) -> bool {
1023        self.rustc_version.is_some()
1024    }
1025
1026    /// Check whether `component` is installed for `toolchain`.
1027    ///
1028    /// Rustup reports pinned toolchains with a host-triple suffix. The request
1029    /// carries the project pin without that suffix, so the owner half uses the
1030    /// same exact-or-`-`-boundary prefix rule as toolchain admission. The
1031    /// component half is exact.
1032    pub fn has_rustup_component(&self, toolchain: &str, component: &str) -> bool {
1033        let owner_prefix = format!("{toolchain}-");
1034        self.rustup_components.iter().any(|fact| {
1035            fact.split_once(':').is_some_and(|(owner, name)| {
1036                name == component && (owner == toolchain || owner.starts_with(&owner_prefix))
1037            })
1038        })
1039    }
1040
1041    /// Check if this worker has Nix installed (nix binary + `/nix/store`).
1042    pub fn has_nix(&self) -> bool {
1043        self.nix_version.is_some()
1044    }
1045
1046    /// Check if this worker has the Go toolchain installed.
1047    pub fn has_go(&self) -> bool {
1048        self.go_version.is_some()
1049    }
1050
1051    /// Check if this worker can run `cargo zigbuild` cross-compiles.
1052    ///
1053    /// Requires BOTH halves of the toolchain: the `cargo-zigbuild` cargo
1054    /// subcommand and the `zig` binary it drives as the linker. Either one
1055    /// alone fails at build time with an error the remote-result heuristics do
1056    /// not recognize as infrastructure (`error: no such command: 'zigbuild'` is
1057    /// not a rustup message), so the exit code would be surfaced verbatim
1058    /// instead of falling open to local. Gating both here keeps that failure
1059    /// out of the user's build.
1060    pub fn has_zig(&self) -> bool {
1061        self.zig_version.is_some() && self.cargo_zigbuild_version.is_some()
1062    }
1063
1064    /// Calculate load per core (1-minute load average / num_cpus).
1065    /// Returns None if metrics are unavailable.
1066    pub fn load_per_core(&self) -> Option<f64> {
1067        match (self.load_avg_1, self.num_cpus) {
1068            (Some(load), Some(cpus)) if cpus > 0 => Some(load / cpus as f64),
1069            _ => None,
1070        }
1071    }
1072
1073    /// Check if worker has high load (exceeds threshold).
1074    /// Returns None if metrics unavailable (fail-open).
1075    pub fn is_high_load(&self, max_load_per_core: f64) -> Option<bool> {
1076        self.load_per_core().map(|lpc| lpc > max_load_per_core)
1077    }
1078
1079    /// `(free_gb, total_gb)` of the filesystem that holds build trees.
1080    ///
1081    /// Capacity questions (how many builds fit, is the free-space floor met)
1082    /// are about that filesystem, not about a small tmpfs `/tmp` that happens
1083    /// to be the tightest mount. Falls back to the tightest-mount sample when
1084    /// the worker does not report a build-disk sample (older `rch-wkr`), which
1085    /// is the conservative pre-GH-#78 behaviour.
1086    pub fn build_disk_gb(&self) -> (Option<f64>, Option<f64>) {
1087        if self.build_disk_free_gb.is_some() {
1088            (self.build_disk_free_gb, self.build_disk_total_gb)
1089        } else {
1090            (self.disk_free_gb, self.disk_total_gb)
1091        }
1092    }
1093
1094    /// Check if the build disk is below the free-space threshold.
1095    /// Returns None if metrics unavailable (fail-open).
1096    pub fn is_low_disk(&self, min_free_gb: f64) -> Option<bool> {
1097        self.build_disk_gb().0.map(|free| free < min_free_gb)
1098    }
1099
1100    /// Check if canonical path-topology preflight is healthy.
1101    ///
1102    /// Returns `None` when probe data is unavailable (fail-open behavior).
1103    pub fn is_topology_healthy(&self) -> Option<bool> {
1104        self.projects_root_ok
1105    }
1106}
1107
1108/// Classify an x86-64 microarchitecture level (1..=4) from a `/proc/cpuinfo`
1109/// `flags` line (bd-6qchz).
1110///
1111/// Levels follow the psABI x86-64-v2/v3/v4 feature baselines, restricted to
1112/// flags as they appear in Linux cpuinfo. A CPU missing any flag of a level is
1113/// classified at the level below — e.g. ovh-b's Ivy Bridge E3-1245 V2 has AVX
1114/// but no `avx2`/`bmi2`/`fma`, so it classifies as 2 and its v3-codegen build
1115/// scripts SIGILL (the bd-68hon incident class).
1116#[must_use]
1117pub fn x86_64_microarch_level_from_flags(flags: &str) -> u8 {
1118    let set: std::collections::HashSet<&str> = flags.split_whitespace().collect();
1119    let has_all = |required: &[&str]| required.iter().all(|flag| set.contains(flag));
1120    // x86-64-v2: CMPXCHG16B, LAHF/SAHF, POPCNT, SSE3/SSSE3/SSE4.x.
1121    const V2: &[&str] = &["cx16", "lahf_lm", "popcnt", "sse4_1", "sse4_2", "ssse3"];
1122    // x86-64-v3: AVX/AVX2, BMI1/2, F16C, FMA, MOVBE, XSAVE (LZCNT is `abm`).
1123    const V3: &[&str] = &[
1124        "abm", "avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "movbe", "xsave",
1125    ];
1126    // x86-64-v4: the AVX-512 F/BW/CD/DQ/VL baseline.
1127    const V4: &[&str] = &["avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl"];
1128    if !has_all(V2) {
1129        return 1;
1130    }
1131    if !has_all(V3) {
1132        return 2;
1133    }
1134    if !has_all(V4) {
1135        return 3;
1136    }
1137    4
1138}
1139
1140/// Path topology configuration.
1141///
1142/// Overrides the default canonical and alias project roots used by
1143/// [`crate::path_topology::PathTopologyPolicy`].  This is useful on platforms
1144/// such as macOS where SIP prevents using `/data/projects` and the user must
1145/// choose a different root (e.g. `~/Projects`).
1146///
1147/// ```toml
1148/// [path_topology]
1149/// canonical_root = "/Users/me/Projects"
1150/// alias_root = "/Users/me/p"
1151/// ```
1152#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1153pub struct PathTopologyConfig {
1154    /// Canonical project root directory.
1155    /// Defaults to `/data/projects` when absent.
1156    #[serde(default)]
1157    pub canonical_root: Option<String>,
1158
1159    /// Alias (symlink) root that points at the canonical root.
1160    /// Defaults to `/dp` when absent.
1161    #[serde(default)]
1162    pub alias_root: Option<String>,
1163}
1164
1165impl PathTopologyConfig {
1166    /// Build a [`crate::path_topology::PathTopologyPolicy`] from this config,
1167    /// falling back to the compiled-in defaults for any field left unset.
1168    ///
1169    /// Empty strings are treated as unset (fall back to defaults).
1170    /// Tilde prefixes (`~/...`) are expanded to the user's home directory.
1171    pub fn to_policy(&self) -> crate::path_topology::PathTopologyPolicy {
1172        let canonical = self
1173            .canonical_root
1174            .as_deref()
1175            .filter(|s| !s.is_empty())
1176            .map(|s| shellexpand::tilde(s).into_owned())
1177            .unwrap_or_else(|| crate::path_topology::DEFAULT_CANONICAL_PROJECT_ROOT.to_string());
1178        let alias = self
1179            .alias_root
1180            .as_deref()
1181            .filter(|s| !s.is_empty())
1182            .map(|s| shellexpand::tilde(s).into_owned())
1183            .unwrap_or_else(|| crate::path_topology::DEFAULT_ALIAS_PROJECT_ROOT.to_string());
1184        crate::path_topology::PathTopologyPolicy::new(
1185            PathBuf::from(canonical),
1186            PathBuf::from(alias),
1187        )
1188    }
1189}
1190
1191/// RCH configuration.
1192#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1193pub struct RchConfig {
1194    #[serde(default)]
1195    pub general: GeneralConfig,
1196    #[serde(default)]
1197    pub compilation: CompilationConfig,
1198    #[serde(default)]
1199    pub transfer: TransferConfig,
1200    #[serde(default)]
1201    pub environment: EnvironmentConfig,
1202    #[serde(default)]
1203    pub circuit: CircuitBreakerConfig,
1204    #[serde(default)]
1205    pub output: OutputConfig,
1206    #[serde(default)]
1207    pub self_healing: SelfHealingConfig,
1208    #[serde(default)]
1209    pub self_test: SelfTestConfig,
1210    /// Worker selection algorithm configuration.
1211    #[serde(default)]
1212    pub selection: SelectionConfig,
1213    /// Execution allowlist configuration (bd-785w).
1214    #[serde(default)]
1215    pub execution: ExecutionConfig,
1216    /// Worker health alerting configuration (daemon).
1217    #[serde(default)]
1218    pub alerts: AlertsConfig,
1219    /// Fleet operation configuration (bd-rs7w.2).
1220    #[serde(default)]
1221    pub fleet: FleetConfig,
1222    /// Path topology overrides for project root directories.
1223    #[serde(default)]
1224    pub path_topology: PathTopologyConfig,
1225    /// Doctor / reliability subsystem configuration (verdict webhooks).
1226    #[serde(default)]
1227    pub doctor: DoctorConfig,
1228    /// Session-history remediation knobs: the central schema and default policy
1229    /// for temporary bypass, auto-rejoin, reconciliation, proof, incident
1230    /// ledger, build-root, pooled targets, telemetry freshness, log retention,
1231    /// disk pressure, and smoke defaults (bd-...remediation-ocv9i.17.1).
1232    #[serde(default)]
1233    pub remediation: crate::remediation_config::RemediationConfig,
1234    /// Layer 0 configuration pack knobs (bd-bqu38 / M-1 §149). Defaults OFF;
1235    /// injected as CARGO_PROFILE_* env vars into remote cargo builds.
1236    #[serde(default)]
1237    pub layer0: Layer0Config,
1238    /// Daemon status API over TCP for other machines on the tailnet
1239    /// (`[api]`, bd-2f5ms). Off unless `bind` is set.
1240    #[serde(default)]
1241    pub api: ApiConfig,
1242    /// Where the fleet dashboard lives (`[dashboard]`), for `rch web`.
1243    #[serde(default)]
1244    pub dashboard: DashboardConfig,
1245    /// Job-mode defaults (`[jobs]`), for `rch exec --job`.
1246    #[serde(default)]
1247    pub jobs: JobsConfig,
1248}
1249
1250/// Job-mode configuration (`[jobs]`).
1251#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1252pub struct JobsConfig {
1253    /// Named tools every job from this project requires a worker to have
1254    /// VERIFIED, as though each were passed with `--require-tool`.
1255    ///
1256    /// This is the project saying "my jobs cannot run on a worker without
1257    /// these", so it applies on top of any `--require-tool` given on the
1258    /// command line rather than being replaced by it. It affects job mode
1259    /// ONLY: ordinary offloaded compilation keeps its existing admission
1260    /// behavior, because a project-wide requirement that silently narrowed
1261    /// every build's worker pool would be a surprising way to lose the fleet.
1262    #[serde(default, skip_serializing_if = "Vec::is_empty")]
1263    pub required_tools: Vec<String>,
1264}
1265
1266/// Daemon status API over TCP (`[api]`).
1267///
1268/// rchd's rich state (`/status`, `/workers/capabilities`) lives on a `0600`
1269/// Unix socket, which only the local user can read. This section opens the
1270/// same JSON on a TCP listener so agents and the fleet dashboard on OTHER
1271/// machines can ask a dispatcher "why are you local-only right now?" without
1272/// ssh. It is meant for a tailnet: the bind address must be loopback or a
1273/// Tailscale address (100.64.0.0/10, fd7a:115c:a1e0::/48) unless
1274/// `allow_any_addr` is set, and every status route requires the bearer token
1275/// unless `token` is unset AND `no_token` is explicitly true.
1276///
1277/// ```toml
1278/// [api]
1279/// bind = "tailscale"       # or "tailscale:9101", "100.68.51.94:9101", "127.0.0.1:9101"
1280/// token = "…"              # or token_file = "~/.config/rch/api.token"
1281/// ```
1282#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1283#[serde(default)]
1284pub struct ApiConfig {
1285    /// Where to listen. `"tailscale"` resolves this machine's Tailscale IPv4
1286    /// at startup (default port 9101); `"tailscale:PORT"`, `"IP:PORT"` or
1287    /// `"[v6]:PORT"` are taken literally. Unset (empty) = API off.
1288    pub bind: String,
1289    /// Bearer token required on `/status` and `/workers/capabilities`.
1290    pub token: Option<String>,
1291    /// File holding the bearer token (trimmed). Takes precedence over `token`.
1292    pub token_file: Option<String>,
1293    /// Serve the status routes with NO token. Only honoured when no token is
1294    /// configured; must be set explicitly so an empty token is never an
1295    /// accidental open door.
1296    pub no_token: bool,
1297    /// Permit a bind address outside loopback and the Tailscale ranges.
1298    pub allow_any_addr: bool,
1299}
1300
1301/// Where the fleet dashboard is served (`[dashboard]`).
1302#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1303#[serde(default)]
1304pub struct DashboardConfig {
1305    /// URL `rch web` opens, e.g. `https://rch-fleet.vercel.app`.
1306    pub url: Option<String>,
1307}
1308
1309/// Doctor reliability subsystem configuration.
1310///
1311/// Currently carries the verdict-transition webhook surface. Kept as its own
1312/// section (`[doctor]`) so future doctor knobs land here rather than bloating
1313/// the top-level config.
1314#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1315pub struct DoctorConfig {
1316    /// Verdict-transition webhook destinations.
1317    #[serde(default)]
1318    pub webhooks: DoctorWebhooksConfig,
1319}
1320
1321/// Webhook notification surface for reliability verdict transitions.
1322///
1323/// Fires when `rch doctor --reliability --watch` observes a verdict change
1324/// (`Healthy → Degraded`, `Degraded → Failing`, etc.). Each endpoint selects
1325/// which transitions it cares about and how the payload is formatted.
1326#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1327pub struct DoctorWebhooksConfig {
1328    /// Named webhook destinations. Empty = no webhook dispatch.
1329    #[serde(default)]
1330    pub endpoints: Vec<DoctorWebhookEndpoint>,
1331    /// Maximum number of in-flight webhook deliveries. Bounds memory and
1332    /// outbound connection pressure: when the queue is full, a transition's
1333    /// delivery is dropped (and journaled) rather than growing unboundedly.
1334    /// Per-process, shared across all endpoints.
1335    #[serde(default = "default_webhook_max_inflight")]
1336    pub max_inflight: usize,
1337}
1338
1339// Manual `Default` so the in-code default matches the serde default
1340// (`#[derive(Default)]` would give `max_inflight = 0`, diverging from a
1341// from-TOML load that fills in 16).
1342impl Default for DoctorWebhooksConfig {
1343    fn default() -> Self {
1344        Self {
1345            endpoints: Vec::new(),
1346            max_inflight: default_webhook_max_inflight(),
1347        }
1348    }
1349}
1350
1351/// A single webhook destination for verdict transitions.
1352///
1353/// SECRETS ARE NEVER STORED HERE. Routing keys, bearer tokens, and signing
1354/// secrets are referenced by the *name* of an environment variable
1355/// (`*_env` fields) read at dispatch time. There is deliberately no
1356/// `secret` / `routing_key` / `bearer_token` / `password` field on this
1357/// struct (enforced by `test_no_inline_secret_fields`).
1358#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1359pub struct DoctorWebhookEndpoint {
1360    /// Operator-facing name (appears in logs and the audit trail).
1361    pub name: String,
1362    /// Destination URL.
1363    pub url: String,
1364    /// Payload format.
1365    #[serde(default)]
1366    pub format: DoctorWebhookFormat,
1367    /// Transition predicates that arm this endpoint. Empty = never fire.
1368    /// Supported: `healthy_to_degraded`, `degraded_to_failing`,
1369    /// `any_to_failing`, `any_to_degraded`, `failing_to_healthy`,
1370    /// `degraded_to_healthy`, `any_to_healthy`, `any_to_any`.
1371    #[serde(default)]
1372    pub on_transitions: Vec<String>,
1373    /// Maximum retry attempts after the first try fails.
1374    #[serde(default = "default_webhook_retry_max")]
1375    pub retry_max: u32,
1376    /// Base backoff between retries (milliseconds); doubles each attempt.
1377    #[serde(default = "default_webhook_retry_backoff_ms")]
1378    pub retry_backoff_ms: u64,
1379    /// Per-attempt request timeout (milliseconds).
1380    #[serde(default = "default_webhook_timeout_ms")]
1381    pub timeout_ms: u64,
1382    /// Per-endpoint kill switch.
1383    #[serde(default = "default_true")]
1384    pub enabled: bool,
1385    /// Name of the env var holding the PagerDuty Events API routing key.
1386    /// Required for `format = "pagerduty"`. Read at runtime, never stored.
1387    #[serde(default)]
1388    pub routing_key_env: Option<String>,
1389    /// Name of the env var holding a bearer token sent as
1390    /// `Authorization: Bearer <token>`. Read at runtime, never stored.
1391    #[serde(default)]
1392    pub bearer_token_env: Option<String>,
1393    /// Name of the env var holding an HMAC-SHA256 signing secret. When set,
1394    /// the body is signed and the signature sent as `X-RCH-Signature:
1395    /// sha256=<hex>`. Read at runtime, never stored.
1396    #[serde(default)]
1397    pub signing_secret_env: Option<String>,
1398}
1399
1400/// Built-in webhook payload formats.
1401#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
1402#[serde(rename_all = "snake_case")]
1403pub enum DoctorWebhookFormat {
1404    /// Opinionated RCH JSON schema (default).
1405    #[default]
1406    GenericJson,
1407    /// Slack Block Kit payload.
1408    Slack,
1409    /// PagerDuty Events API v2 payload.
1410    Pagerduty,
1411}
1412
1413fn default_webhook_max_inflight() -> usize {
1414    16
1415}
1416
1417fn default_webhook_retry_max() -> u32 {
1418    3
1419}
1420
1421fn default_webhook_retry_backoff_ms() -> u64 {
1422    200
1423}
1424
1425fn default_webhook_timeout_ms() -> u64 {
1426    5000
1427}
1428
1429/// Worker health alerting configuration.
1430#[derive(Debug, Clone, Serialize, Deserialize)]
1431pub struct AlertsConfig {
1432    /// Enable or disable alert generation in the daemon.
1433    #[serde(default = "default_true")]
1434    pub enabled: bool,
1435    /// Suppress duplicate alerts for this many seconds.
1436    #[serde(default = "default_alert_suppress_duplicates_secs")]
1437    pub suppress_duplicates_secs: u64,
1438    /// How long to keep a cleared alert visible (with state
1439    /// `cleared_pending_clean`) before removing it entirely. Prevents
1440    /// transient, self-healing circuit-breaker warnings from lingering on
1441    /// `rch status` long after the fleet returned to healthy (bd-3ogaz).
1442    #[serde(default = "default_alert_cleared_retention_secs")]
1443    pub cleared_retention_secs: u64,
1444    /// Webhook configuration for external notifications.
1445    #[serde(default)]
1446    pub webhook: Option<WebhookConfig>,
1447}
1448
1449/// Webhook configuration for alert dispatch.
1450#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1451pub struct WebhookConfig {
1452    /// Webhook endpoint URL (e.g., Slack/Discord webhook URL).
1453    pub url: Option<String>,
1454    /// Secret for HMAC-SHA256 signing (optional).
1455    #[serde(default)]
1456    pub secret: Option<String>,
1457    /// Timeout in seconds for webhook requests.
1458    #[serde(default = "default_webhook_timeout_secs")]
1459    pub timeout_secs: u64,
1460    /// Number of retries on failure.
1461    #[serde(default = "default_webhook_retry_count")]
1462    pub retry_count: u32,
1463    /// Events to send (empty = all events).
1464    /// Supported: worker_offline, worker_degraded, circuit_open, all_workers_offline
1465    #[serde(default)]
1466    pub events: Vec<String>,
1467}
1468
1469fn default_webhook_timeout_secs() -> u64 {
1470    5
1471}
1472
1473fn default_webhook_retry_count() -> u32 {
1474    3
1475}
1476
1477fn default_alert_suppress_duplicates_secs() -> u64 {
1478    300
1479}
1480
1481fn default_alert_cleared_retention_secs() -> u64 {
1482    300
1483}
1484
1485impl Default for AlertsConfig {
1486    fn default() -> Self {
1487        Self {
1488            enabled: true,
1489            suppress_duplicates_secs: default_alert_suppress_duplicates_secs(),
1490            cleared_retention_secs: default_alert_cleared_retention_secs(),
1491            webhook: None,
1492        }
1493    }
1494}
1495
1496/// Fleet operation configuration.
1497///
1498/// Controls timeouts, thresholds, and behavior for fleet management commands
1499/// like `rch fleet preflight`, `rch fleet status`, and `rch fleet deploy`.
1500#[derive(Debug, Clone, Serialize, Deserialize)]
1501pub struct FleetConfig {
1502    /// SSH connection timeout in seconds.
1503    #[serde(default = "default_fleet_ssh_connect_timeout_secs")]
1504    pub ssh_connect_timeout_secs: u64,
1505    /// SSH command execution timeout in seconds.
1506    #[serde(default = "default_fleet_ssh_command_timeout_secs")]
1507    pub ssh_command_timeout_secs: u64,
1508    /// Minimum required disk space in MB for worker health.
1509    #[serde(default = "default_fleet_min_disk_space_mb")]
1510    pub min_disk_space_mb: u64,
1511    /// Maximum acceptable load average for worker health.
1512    #[serde(default = "default_fleet_max_load_average")]
1513    pub max_load_average: f64,
1514    /// Maximum number of concurrent SSH connections.
1515    #[serde(default = "default_fleet_max_concurrent_workers")]
1516    pub max_concurrent_workers: usize,
1517    /// Number of retry attempts for transient failures.
1518    #[serde(default = "default_fleet_retry_count")]
1519    pub retry_count: u32,
1520    /// Delay between retries in milliseconds.
1521    #[serde(default = "default_fleet_retry_delay_ms")]
1522    pub retry_delay_ms: u64,
1523}
1524
1525fn default_fleet_ssh_connect_timeout_secs() -> u64 {
1526    10
1527}
1528
1529fn default_fleet_ssh_command_timeout_secs() -> u64 {
1530    30
1531}
1532
1533fn default_fleet_min_disk_space_mb() -> u64 {
1534    500
1535}
1536
1537fn default_fleet_max_load_average() -> f64 {
1538    10.0
1539}
1540
1541fn default_fleet_max_concurrent_workers() -> usize {
1542    10
1543}
1544
1545fn default_fleet_retry_count() -> u32 {
1546    2
1547}
1548
1549fn default_fleet_retry_delay_ms() -> u64 {
1550    1000
1551}
1552
1553impl Default for FleetConfig {
1554    fn default() -> Self {
1555        Self {
1556            ssh_connect_timeout_secs: default_fleet_ssh_connect_timeout_secs(),
1557            ssh_command_timeout_secs: default_fleet_ssh_command_timeout_secs(),
1558            min_disk_space_mb: default_fleet_min_disk_space_mb(),
1559            max_load_average: default_fleet_max_load_average(),
1560            max_concurrent_workers: default_fleet_max_concurrent_workers(),
1561            retry_count: default_fleet_retry_count(),
1562            retry_delay_ms: default_fleet_retry_delay_ms(),
1563        }
1564    }
1565}
1566
1567impl FleetConfig {
1568    /// Get SSH connect timeout as Duration.
1569    pub fn ssh_connect_timeout(&self) -> std::time::Duration {
1570        std::time::Duration::from_secs(self.ssh_connect_timeout_secs)
1571    }
1572
1573    /// Get SSH command timeout as Duration.
1574    pub fn ssh_command_timeout(&self) -> std::time::Duration {
1575        std::time::Duration::from_secs(self.ssh_command_timeout_secs)
1576    }
1577
1578    /// Get retry delay as Duration.
1579    pub fn retry_delay(&self) -> std::time::Duration {
1580        std::time::Duration::from_millis(self.retry_delay_ms)
1581    }
1582}
1583
1584/// Box-level role (bd-wywsj): what this MACHINE is for, so "never
1585/// build locally" is configuration rather than a per-call env var.
1586#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
1587#[serde(rename_all = "lowercase")]
1588pub enum BoxRole {
1589    /// Orchestrator box: offloadable builds default FAIL-CLOSED +
1590    /// queue (RCH_REQUIRE_REMOTE semantics) — a saturated fleet waits
1591    /// for a worker instead of repopulating local rustc. Explicit
1592    /// per-call env still overrides.
1593    Dispatcher,
1594    /// The box IS the compute: fail-open/local; `rch shim install`
1595    /// refuses here.
1596    Worker,
1597    /// Today's behavior (fully backward compatible).
1598    #[default]
1599    Hybrid,
1600}
1601
1602impl BoxRole {
1603    /// Stable lowercase token.
1604    #[must_use]
1605    pub fn as_str(self) -> &'static str {
1606        match self {
1607            BoxRole::Dispatcher => "dispatcher",
1608            BoxRole::Worker => "worker",
1609            BoxRole::Hybrid => "hybrid",
1610        }
1611    }
1612}
1613
1614#[derive(Debug, Clone, Serialize, Deserialize)]
1615pub struct GeneralConfig {
1616    /// Whether RCH is enabled.
1617    #[serde(default = "default_true")]
1618    pub enabled: bool,
1619    /// Box-level role (bd-wywsj): dispatcher boxes default fail-closed
1620    /// + queue for offloadable builds; worker boxes are the compute.
1621    #[serde(default)]
1622    pub role: BoxRole,
1623    /// Force local execution for this project (never offload), even if the command is classified.
1624    ///
1625    /// Intended for sensitive projects or cases where local state must be authoritative.
1626    #[serde(default)]
1627    pub force_local: bool,
1628    /// Force remote execution for this project (always attempt offload) when safe.
1629    ///
1630    /// This bypasses heuristic gating (e.g. confidence threshold overrides) but still respects
1631    /// structural safety checks and NEVER_INTERCEPT patterns.
1632    #[serde(default)]
1633    pub force_remote: bool,
1634    /// Log level (trace, debug, info, warn, error).
1635    #[serde(default = "default_log_level")]
1636    pub log_level: String,
1637    /// Path to Unix socket for daemon communication.
1638    #[serde(default = "default_socket_path")]
1639    pub socket_path: String,
1640}
1641
1642/// Environment variable passthrough configuration.
1643#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1644pub struct EnvironmentConfig {
1645    /// Allowlist of environment variables to forward to remote workers.
1646    #[serde(default)]
1647    pub allowlist: Vec<String>,
1648}
1649
1650impl Default for GeneralConfig {
1651    fn default() -> Self {
1652        Self {
1653            enabled: true,
1654            role: BoxRole::default(),
1655            force_local: false,
1656            force_remote: false,
1657            log_level: "info".to_string(),
1658            socket_path: default_socket_path(),
1659        }
1660    }
1661}
1662
1663/// Visibility level for hook output.
1664#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
1665#[serde(rename_all = "lowercase")]
1666pub enum OutputVisibility {
1667    #[default]
1668    None,
1669    Summary,
1670    Verbose,
1671}
1672
1673impl std::fmt::Display for OutputVisibility {
1674    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1675        let value = match self {
1676            OutputVisibility::None => "none",
1677            OutputVisibility::Summary => "summary",
1678            OutputVisibility::Verbose => "verbose",
1679        };
1680        write!(f, "{}", value)
1681    }
1682}
1683
1684impl std::str::FromStr for OutputVisibility {
1685    type Err = ();
1686
1687    fn from_str(s: &str) -> Result<Self, Self::Err> {
1688        match s.trim().to_lowercase().as_str() {
1689            "none" | "silent" | "quiet" => Ok(Self::None),
1690            "summary" | "short" => Ok(Self::Summary),
1691            "verbose" | "debug" => Ok(Self::Verbose),
1692            _ => Err(()),
1693        }
1694    }
1695}
1696
1697/// Color mode for remote command output.
1698///
1699/// Controls whether ANSI color codes are preserved when streaming
1700/// output from remote compilation commands.
1701#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1702#[serde(rename_all = "lowercase")]
1703pub enum ColorMode {
1704    /// Force color output regardless of terminal detection.
1705    /// Sets CARGO_TERM_COLOR=always and similar environment variables.
1706    #[default]
1707    Always,
1708    /// Let the remote command detect terminal capabilities (may lose colors).
1709    Auto,
1710    /// Disable color output entirely.
1711    Never,
1712}
1713
1714impl std::fmt::Display for ColorMode {
1715    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1716        let value = match self {
1717            ColorMode::Always => "always",
1718            ColorMode::Auto => "auto",
1719            ColorMode::Never => "never",
1720        };
1721        write!(f, "{}", value)
1722    }
1723}
1724
1725impl std::str::FromStr for ColorMode {
1726    type Err = ();
1727
1728    fn from_str(s: &str) -> Result<Self, Self::Err> {
1729        match s.trim().to_lowercase().as_str() {
1730            "always" | "force" | "yes" | "true" => Ok(Self::Always),
1731            "auto" | "detect" => Ok(Self::Auto),
1732            "never" | "none" | "no" | "false" => Ok(Self::Never),
1733            _ => Err(()),
1734        }
1735    }
1736}
1737
1738fn default_color_mode() -> ColorMode {
1739    ColorMode::default()
1740}
1741
1742/// Hook output configuration.
1743#[derive(Debug, Clone, Serialize, Deserialize)]
1744pub struct OutputConfig {
1745    /// Output visibility level for hook messages.
1746    #[serde(default = "default_output_visibility")]
1747    pub visibility: OutputVisibility,
1748    /// Whether the first-run success message has been shown.
1749    #[serde(default)]
1750    pub first_run_complete: bool,
1751    /// Color mode for remote command output.
1752    /// Controls whether ANSI color codes are preserved in remote output.
1753    #[serde(default = "default_color_mode")]
1754    pub color_mode: ColorMode,
1755}
1756
1757impl Default for OutputConfig {
1758    fn default() -> Self {
1759        Self {
1760            visibility: OutputVisibility::None,
1761            first_run_complete: false,
1762            color_mode: ColorMode::default(),
1763        }
1764    }
1765}
1766
1767// ============================================================================
1768// Self-Healing Configuration
1769// ============================================================================
1770
1771fn default_autostart_cooldown_secs() -> u64 {
1772    30
1773}
1774
1775fn default_autostart_timeout_secs() -> u64 {
1776    3
1777}
1778
1779/// Log level for self-healing diagnostic output.
1780///
1781/// Used to filter `tracing` events emitted by the self-healing subsystem
1782/// (hook auto-start, daemon hook-install). Read from
1783/// `RCH_SELF_HEALING_LOG_LEVEL=debug|info|warn|error` (case-insensitive).
1784#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
1785#[serde(rename_all = "lowercase")]
1786pub enum SelfHealingLogLevel {
1787    Debug,
1788    #[default]
1789    Info,
1790    Warn,
1791    Error,
1792}
1793
1794impl SelfHealingLogLevel {
1795    /// Parse a human-friendly level name. Returns `None` for unknown input.
1796    pub fn from_env_str(s: &str) -> Option<Self> {
1797        match s.trim().to_ascii_lowercase().as_str() {
1798            "debug" => Some(Self::Debug),
1799            "info" => Some(Self::Info),
1800            "warn" | "warning" => Some(Self::Warn),
1801            "error" => Some(Self::Error),
1802            _ => None,
1803        }
1804    }
1805}
1806
1807/// Self-healing behaviors for the hook and daemon.
1808#[derive(Debug, Clone, Serialize, Deserialize)]
1809pub struct SelfHealingConfig {
1810    /// Whether the hook should attempt to auto-start the daemon.
1811    #[serde(default = "default_true")]
1812    pub hook_starts_daemon: bool,
1813    /// Whether the daemon should auto-install Claude Code hooks on startup.
1814    #[serde(default = "default_true")]
1815    pub daemon_installs_hooks: bool,
1816    /// Cooldown between auto-start attempts in seconds.
1817    #[serde(default = "default_autostart_cooldown_secs")]
1818    pub auto_start_cooldown_secs: u64,
1819    /// Time to wait for the daemon socket after spawning in seconds.
1820    /// Also aliased as `daemon_start_timeout` for backwards compatibility.
1821    #[serde(
1822        default = "default_autostart_timeout_secs",
1823        alias = "daemon_start_timeout"
1824    )]
1825    pub auto_start_timeout_secs: u64,
1826    /// Log level for self-healing diagnostic events. Default: info.
1827    /// Override via `RCH_SELF_HEALING_LOG_LEVEL=debug|info|warn|error`.
1828    #[serde(default)]
1829    pub self_healing_log_level: SelfHealingLogLevel,
1830}
1831
1832impl Default for SelfHealingConfig {
1833    fn default() -> Self {
1834        Self {
1835            hook_starts_daemon: default_true(),
1836            daemon_installs_hooks: default_true(),
1837            auto_start_cooldown_secs: default_autostart_cooldown_secs(),
1838            auto_start_timeout_secs: default_autostart_timeout_secs(),
1839            self_healing_log_level: SelfHealingLogLevel::default(),
1840        }
1841    }
1842}
1843
1844impl SelfHealingConfig {
1845    /// Apply environment variable overrides to this config.
1846    ///
1847    /// Supported environment variables:
1848    /// - `RCH_NO_SELF_HEALING=1` - Master switch to disable all self-healing
1849    /// - `RCH_HOOK_STARTS_DAEMON=0|1` - Control hook auto-starting daemon
1850    /// - `RCH_DAEMON_INSTALLS_HOOKS=0|1` - Control daemon auto-installing hooks
1851    /// - `RCH_AUTO_START_TIMEOUT_SECS=<seconds>` - Max wait for daemon start
1852    /// - `RCH_AUTO_START_COOLDOWN_SECS=<seconds>` - Min time between auto-starts
1853    pub fn with_env_overrides(mut self) -> Self {
1854        // Master disable switch
1855        if let Ok(val) = std::env::var("RCH_NO_SELF_HEALING")
1856            && (val == "1" || val.eq_ignore_ascii_case("true"))
1857        {
1858            self.hook_starts_daemon = false;
1859            self.daemon_installs_hooks = false;
1860            return self;
1861        }
1862
1863        // Individual toggles
1864        if let Ok(val) = std::env::var("RCH_HOOK_STARTS_DAEMON") {
1865            self.hook_starts_daemon = val != "0" && !val.eq_ignore_ascii_case("false");
1866        }
1867        if let Ok(val) = std::env::var("RCH_DAEMON_INSTALLS_HOOKS") {
1868            self.daemon_installs_hooks = val != "0" && !val.eq_ignore_ascii_case("false");
1869        }
1870
1871        // Numeric settings
1872        if let Ok(val) = std::env::var("RCH_AUTO_START_TIMEOUT_SECS")
1873            && let Ok(secs) = val.parse()
1874        {
1875            self.auto_start_timeout_secs = secs;
1876        }
1877        if let Ok(val) = std::env::var("RCH_AUTO_START_COOLDOWN_SECS")
1878            && let Ok(secs) = val.parse()
1879        {
1880            self.auto_start_cooldown_secs = secs;
1881        }
1882        // br-4zf3p: log-level override.
1883        if let Ok(val) = std::env::var("RCH_SELF_HEALING_LOG_LEVEL") {
1884            if let Some(level) = SelfHealingLogLevel::from_env_str(&val) {
1885                self.self_healing_log_level = level;
1886            } else {
1887                tracing::warn!(
1888                    target: "rch::self_healing",
1889                    invalid = %val,
1890                    "RCH_SELF_HEALING_LOG_LEVEL has invalid value; using default=info"
1891                );
1892            }
1893        }
1894
1895        self
1896    }
1897}
1898
1899// ============================================================================
1900// Self-Test Configuration
1901// ============================================================================
1902
1903/// Action to take when a scheduled self-test fails.
1904#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1905#[serde(rename_all = "snake_case")]
1906pub enum SelfTestFailureAction {
1907    /// Only emit an alert/log entry.
1908    #[default]
1909    Alert,
1910    /// Disable the failing worker automatically.
1911    DisableWorker,
1912    /// Alert and disable the worker.
1913    AlertAndDisable,
1914}
1915
1916/// Which workers to include in self-test runs.
1917#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1918#[serde(untagged)]
1919pub enum SelfTestWorkers {
1920    /// Special value "all" (or a single worker id string).
1921    All(String),
1922    /// Explicit worker list.
1923    List(Vec<WorkerId>),
1924}
1925
1926impl Default for SelfTestWorkers {
1927    fn default() -> Self {
1928        Self::All("all".to_string())
1929    }
1930}
1931
1932impl SelfTestWorkers {
1933    /// Resolve to an explicit list if configured, or None to indicate "all".
1934    pub fn resolve(&self) -> Option<Vec<WorkerId>> {
1935        match self {
1936            SelfTestWorkers::All(value) => {
1937                if value.eq_ignore_ascii_case("all") {
1938                    None
1939                } else {
1940                    Some(vec![WorkerId::new(value.clone())])
1941                }
1942            }
1943            SelfTestWorkers::List(list) => Some(list.clone()),
1944        }
1945    }
1946}
1947
1948fn default_self_test_retry_count() -> u32 {
1949    3
1950}
1951
1952fn default_self_test_retry_delay() -> String {
1953    "5m".to_string()
1954}
1955
1956fn default_self_test_enabled() -> bool {
1957    false
1958}
1959
1960/// Self-test scheduling and behavior configuration.
1961#[derive(Debug, Clone, Serialize, Deserialize)]
1962pub struct SelfTestConfig {
1963    /// Enable scheduled self-tests.
1964    #[serde(default = "default_self_test_enabled")]
1965    pub enabled: bool,
1966    /// Cron schedule (e.g., "0 3 * * *").
1967    #[serde(default)]
1968    pub schedule: Option<String>,
1969    /// Interval duration (e.g., "24h").
1970    #[serde(default)]
1971    pub interval: Option<String>,
1972    /// Which workers to test (default: "all").
1973    #[serde(default)]
1974    pub workers: SelfTestWorkers,
1975    /// Action on failure.
1976    #[serde(default)]
1977    pub on_failure: SelfTestFailureAction,
1978    /// Retry count for failed tests.
1979    #[serde(default = "default_self_test_retry_count")]
1980    pub retry_count: u32,
1981    /// Delay between retries.
1982    #[serde(default = "default_self_test_retry_delay")]
1983    pub retry_delay: String,
1984}
1985
1986impl Default for SelfTestConfig {
1987    fn default() -> Self {
1988        Self {
1989            enabled: default_self_test_enabled(),
1990            schedule: None,
1991            interval: None,
1992            workers: SelfTestWorkers::default(),
1993            on_failure: SelfTestFailureAction::default(),
1994            retry_count: default_self_test_retry_count(),
1995            retry_delay: default_self_test_retry_delay(),
1996        }
1997    }
1998}
1999
2000/// Circuit breaker configuration.
2001#[derive(Debug, Clone, Serialize, Deserialize)]
2002pub struct CircuitBreakerConfig {
2003    /// Consecutive failures to open circuit.
2004    #[serde(default = "default_circuit_failure_threshold")]
2005    pub failure_threshold: u32,
2006    /// Consecutive successes to close circuit.
2007    #[serde(default = "default_circuit_success_threshold")]
2008    pub success_threshold: u32,
2009    /// Error rate threshold (0.0-1.0) to open circuit.
2010    #[serde(default = "default_circuit_error_rate_threshold")]
2011    pub error_rate_threshold: f64,
2012    /// Rolling window size in seconds.
2013    #[serde(default = "default_circuit_window_secs")]
2014    pub window_secs: u64,
2015    /// Cooldown duration before half-open (seconds).
2016    #[serde(default = "default_circuit_open_cooldown_secs")]
2017    pub open_cooldown_secs: u64,
2018    /// Max concurrent probes in half-open.
2019    #[serde(default = "default_circuit_half_open_max_probes")]
2020    pub half_open_max_probes: u32,
2021    /// Consecutive *command-class* failures to open the circuit.
2022    ///
2023    /// Command-class evidence is an operation of the same shape as dispatched
2024    /// work — a remote binary that must run to completion (the telemetry poll,
2025    /// or a health probe whose session connected but whose command never
2026    /// finished). It is tracked separately from the plain liveness probe
2027    /// because a host under fork exhaustion or swap thrash keeps answering
2028    /// `echo` over a warm connection while every real command times out;
2029    /// interleaved liveness successes would otherwise keep resetting
2030    /// `failure_threshold` and the breaker would never trip.
2031    #[serde(default = "default_circuit_command_failure_threshold")]
2032    pub command_failure_threshold: u32,
2033    /// Upper bound (seconds) for the open cooldown after repeated reopens.
2034    ///
2035    /// Each Open -> HalfOpen -> Open cycle without an intervening Closed
2036    /// doubles the cooldown (`open_cooldown_secs`, x2, x4, ...) up to this
2037    /// cap, so a flapping worker converges to Open instead of being
2038    /// re-admitted every cooldown. Set equal to `open_cooldown_secs` to
2039    /// disable the backoff.
2040    #[serde(default = "default_circuit_open_cooldown_max_secs")]
2041    pub open_cooldown_max_secs: u64,
2042}
2043
2044impl Default for CircuitBreakerConfig {
2045    fn default() -> Self {
2046        Self {
2047            failure_threshold: default_circuit_failure_threshold(),
2048            success_threshold: default_circuit_success_threshold(),
2049            error_rate_threshold: default_circuit_error_rate_threshold(),
2050            window_secs: default_circuit_window_secs(),
2051            open_cooldown_secs: default_circuit_open_cooldown_secs(),
2052            half_open_max_probes: default_circuit_half_open_max_probes(),
2053            command_failure_threshold: default_circuit_command_failure_threshold(),
2054            open_cooldown_max_secs: default_circuit_open_cooldown_max_secs(),
2055        }
2056    }
2057}
2058
2059impl CircuitBreakerConfig {
2060    /// Effective cooldown (ms) for the given reopen streak, with exponential
2061    /// backoff capped at `open_cooldown_max_secs`. `reopen_streak` counts
2062    /// Open transitions since the circuit last closed (1 = first open).
2063    pub fn cooldown_ms_for_streak(&self, reopen_streak: u32) -> u64 {
2064        let base_ms = self.open_cooldown_secs.saturating_mul(1000);
2065        let cap_ms = self
2066            .open_cooldown_max_secs
2067            .saturating_mul(1000)
2068            .max(base_ms);
2069        // Exponent bounded so the shift cannot overflow; the cap clamps anyway.
2070        let exponent = reopen_streak.saturating_sub(1).min(16);
2071        base_ms.saturating_mul(1u64 << exponent).min(cap_ms)
2072    }
2073}
2074
2075/// Circuit breaker statistics for tracking state transitions.
2076///
2077/// Maintains a rolling window of successes and failures to determine
2078/// when the circuit should open, close, or enter half-open state.
2079#[derive(Debug, Clone, Default)]
2080pub struct CircuitStats {
2081    /// Current circuit state.
2082    state: CircuitState,
2083    /// Consecutive failure count (reset on success).
2084    consecutive_failures: u32,
2085    /// Consecutive success count in half-open state (reset on failure).
2086    consecutive_successes: u32,
2087    /// Total successes in the current rolling window.
2088    window_successes: u32,
2089    /// Total failures in the current rolling window.
2090    window_failures: u32,
2091    /// Timestamp when circuit entered Open state (epoch millis).
2092    opened_at: Option<u64>,
2093    /// Timestamp of last state change (epoch millis).
2094    last_state_change: u64,
2095    /// Current active probes in half-open state.
2096    active_probes: u32,
2097    /// Consecutive command-class failures (telemetry poll exhaustion, probe
2098    /// sessions that connected but never completed). Reset only by a
2099    /// command-class success or a circuit close — NOT by a liveness success,
2100    /// which is the whole point (see `CircuitBreakerConfig::command_failure_threshold`).
2101    consecutive_command_failures: u32,
2102    /// Number of Open transitions since the circuit last closed. Drives the
2103    /// cooldown backoff; 0 while Closed-since-recovery.
2104    reopen_streak: u32,
2105    /// Recent health check results (true=success, false=failure).
2106    /// Used for history visualization in status display.
2107    recent_results: Vec<bool>,
2108}
2109
2110/// Maximum number of recent results to keep for history visualization.
2111const CIRCUIT_HISTORY_SIZE: usize = 10;
2112
2113impl CircuitStats {
2114    /// Create new circuit stats in the closed state.
2115    pub fn new() -> Self {
2116        Self {
2117            state: CircuitState::Closed,
2118            consecutive_failures: 0,
2119            consecutive_successes: 0,
2120            window_successes: 0,
2121            window_failures: 0,
2122            opened_at: None,
2123            last_state_change: Self::now_millis(),
2124            active_probes: 0,
2125            consecutive_command_failures: 0,
2126            reopen_streak: 0,
2127            recent_results: Vec::with_capacity(CIRCUIT_HISTORY_SIZE),
2128        }
2129    }
2130
2131    /// Consecutive command-class failures observed since the last
2132    /// command-class success (or circuit close).
2133    pub fn consecutive_command_failures(&self) -> u32 {
2134        self.consecutive_command_failures
2135    }
2136
2137    /// Open transitions since the circuit last closed (0 when it has not
2138    /// opened since recovery). Exposed for diagnostics.
2139    pub fn reopen_streak(&self) -> u32 {
2140        self.reopen_streak
2141    }
2142
2143    /// Cooldown currently in force (ms), including reopen backoff.
2144    fn effective_cooldown_ms(&self, config: &CircuitBreakerConfig) -> u64 {
2145        config.cooldown_ms_for_streak(self.reopen_streak.max(1))
2146    }
2147
2148    /// Get the current circuit state.
2149    pub fn state(&self) -> CircuitState {
2150        self.state
2151    }
2152
2153    /// Get the timestamp when the circuit opened (if open).
2154    pub fn opened_at(&self) -> Option<u64> {
2155        self.opened_at
2156    }
2157
2158    /// Get the timestamp of the last state change.
2159    pub fn last_state_change(&self) -> u64 {
2160        self.last_state_change
2161    }
2162
2163    /// Get the consecutive failure count.
2164    pub fn consecutive_failures(&self) -> u32 {
2165        self.consecutive_failures
2166    }
2167
2168    /// Get the error rate in the current window (0.0-1.0).
2169    ///
2170    /// Computed over the bounded recent-results window (the last
2171    /// `CIRCUIT_HISTORY_SIZE` health checks) rather than the unbounded lifetime
2172    /// `window_successes`/`window_failures` counters. A long-lived daemon polling
2173    /// distant workers accumulates occasional transient failures over days; with a
2174    /// lifetime ratio a currently-healthy worker could stay gated (circuit kept
2175    /// open, depressed health score) until the daemon was restarted. A recency
2176    /// window lets the rate self-heal as fresh successes age out old failures.
2177    pub fn error_rate(&self) -> f64 {
2178        if self.recent_results.is_empty() {
2179            return 0.0;
2180        }
2181        let failures = self.recent_results.iter().filter(|&&ok| !ok).count();
2182        failures as f64 / self.recent_results.len() as f64
2183    }
2184
2185    /// Record a successful operation.
2186    ///
2187    /// In closed state: resets consecutive failures, increments window successes.
2188    /// In half-open state: increments consecutive successes.
2189    pub fn record_success(&mut self) {
2190        self.window_successes += 1;
2191        self.consecutive_failures = 0;
2192        self.push_result(true);
2193
2194        if self.state == CircuitState::HalfOpen {
2195            self.consecutive_successes += 1;
2196            if self.active_probes > 0 {
2197                self.active_probes -= 1;
2198            }
2199        }
2200    }
2201
2202    /// Record a failed operation.
2203    ///
2204    /// Increments consecutive failures and window failures.
2205    /// In half-open state, resets consecutive successes.
2206    pub fn record_failure(&mut self) {
2207        self.window_failures += 1;
2208        self.consecutive_failures += 1;
2209        self.push_result(false);
2210
2211        if self.state == CircuitState::HalfOpen {
2212            self.consecutive_successes = 0;
2213            if self.active_probes > 0 {
2214                self.active_probes -= 1;
2215            }
2216        }
2217    }
2218
2219    /// Check if the circuit should open based on config thresholds.
2220    ///
2221    /// Returns true if:
2222    /// - Consecutive failures exceed threshold, OR
2223    /// - Error rate exceeds threshold (with minimum sample size)
2224    pub fn should_open(&self, config: &CircuitBreakerConfig) -> bool {
2225        if self.state != CircuitState::Closed {
2226            return false;
2227        }
2228
2229        // Open if consecutive failures exceed threshold
2230        if self.consecutive_failures >= config.failure_threshold {
2231            return true;
2232        }
2233
2234        // Open if error rate exceeds threshold (with a minimum sample size).
2235        // Counts samples in the bounded recent-results window so the gate
2236        // reflects recent behavior, matching error_rate() above.
2237        if self.recent_results.len() >= 5 && self.error_rate() >= config.error_rate_threshold {
2238            return true;
2239        }
2240
2241        false
2242    }
2243
2244    /// Check if the circuit should transition to half-open.
2245    ///
2246    /// Returns true if the circuit is open and the cooldown period has elapsed.
2247    pub fn should_half_open(&self, config: &CircuitBreakerConfig) -> bool {
2248        if self.state != CircuitState::Open {
2249            return false;
2250        }
2251
2252        let now = Self::now_millis();
2253        let cooldown_ms = self.effective_cooldown_ms(config);
2254
2255        if let Some(opened_at) = self.opened_at {
2256            now.saturating_sub(opened_at) >= cooldown_ms
2257        } else {
2258            false
2259        }
2260    }
2261
2262    /// Check if the circuit should close.
2263    ///
2264    /// Returns true if in half-open state, consecutive successes meet or
2265    /// exceed the success threshold, AND no command-class failure is
2266    /// outstanding. A worker whose last real command timed out is not
2267    /// recovered no matter how many liveness probes it answers; it stays
2268    /// HalfOpen (degraded, probe-limited) until a command-class success
2269    /// clears the evidence.
2270    pub fn should_close(&self, config: &CircuitBreakerConfig) -> bool {
2271        if self.state != CircuitState::HalfOpen {
2272            return false;
2273        }
2274
2275        self.consecutive_successes >= config.success_threshold
2276            && self.consecutive_command_failures == 0
2277    }
2278
2279    /// Check if a probe request can be made in half-open state.
2280    ///
2281    /// Returns true if active probes are below the maximum allowed.
2282    pub fn can_probe(&self, config: &CircuitBreakerConfig) -> bool {
2283        if self.state != CircuitState::HalfOpen {
2284            return false;
2285        }
2286
2287        self.active_probes < config.half_open_max_probes
2288    }
2289
2290    /// Start a probe request in half-open state.
2291    ///
2292    /// Returns true if the probe was started, false if already at max probes.
2293    pub fn start_probe(&mut self, config: &CircuitBreakerConfig) -> bool {
2294        if !self.can_probe(config) {
2295            return false;
2296        }
2297        self.active_probes += 1;
2298        true
2299    }
2300
2301    /// Transition the circuit to open state.
2302    pub fn open(&mut self) {
2303        if self.state != CircuitState::Open {
2304            self.state = CircuitState::Open;
2305            self.opened_at = Some(Self::now_millis());
2306            self.last_state_change = Self::now_millis();
2307            self.consecutive_successes = 0;
2308            self.active_probes = 0;
2309            // Every open without an intervening close lengthens the next
2310            // cooldown (see `CircuitBreakerConfig::cooldown_ms_for_streak`).
2311            self.reopen_streak = self.reopen_streak.saturating_add(1);
2312        }
2313    }
2314
2315    /// Transition the circuit to half-open state.
2316    pub fn half_open(&mut self) {
2317        if self.state != CircuitState::HalfOpen {
2318            self.state = CircuitState::HalfOpen;
2319            self.last_state_change = Self::now_millis();
2320            self.consecutive_successes = 0;
2321            self.active_probes = 0;
2322        }
2323    }
2324
2325    /// Transition the circuit to closed state.
2326    pub fn close(&mut self) {
2327        if self.state != CircuitState::Closed {
2328            self.state = CircuitState::Closed;
2329            self.last_state_change = Self::now_millis();
2330            self.opened_at = None;
2331            self.consecutive_failures = 0;
2332            self.consecutive_successes = 0;
2333            self.active_probes = 0;
2334            self.consecutive_command_failures = 0;
2335            self.reopen_streak = 0;
2336            // Reset the window on close, INCLUDING the recent-results history
2337            // that error_rate() now reads. Otherwise a freshly-recovered circuit
2338            // would still compute a high error rate from the pre-failure samples
2339            // and immediately re-open (flapping). Matches the original intent of
2340            // resetting the error-rate inputs when the circuit recovers.
2341            self.window_successes = 0;
2342            self.window_failures = 0;
2343            self.recent_results.clear();
2344        }
2345    }
2346
2347    /// Reset the rolling window counters.
2348    ///
2349    /// Called periodically to ensure the window reflects recent activity.
2350    /// Also clears the recent-results history that `error_rate()` is computed
2351    /// from, so a reset truly zeroes the observed error rate.
2352    pub fn reset_window(&mut self) {
2353        self.window_successes = 0;
2354        self.window_failures = 0;
2355        self.recent_results.clear();
2356    }
2357
2358    /// Get recent health check results for history visualization.
2359    ///
2360    /// Returns a slice of recent results (true=success, false=failure),
2361    /// with the most recent result at the end.
2362    pub fn recent_results(&self) -> &[bool] {
2363        &self.recent_results
2364    }
2365
2366    /// Calculate seconds remaining until circuit auto-transitions to half-open.
2367    ///
2368    /// Returns None if circuit is not open or cooldown has already elapsed.
2369    pub fn recovery_remaining_secs(&self, config: &CircuitBreakerConfig) -> Option<u64> {
2370        if self.state != CircuitState::Open {
2371            return None;
2372        }
2373
2374        let now = Self::now_millis();
2375        let cooldown_ms = self.effective_cooldown_ms(config);
2376
2377        if let Some(opened_at) = self.opened_at {
2378            let elapsed_ms = now.saturating_sub(opened_at);
2379            if elapsed_ms < cooldown_ms {
2380                return Some((cooldown_ms - elapsed_ms) / 1000);
2381            }
2382        }
2383        None
2384    }
2385
2386    /// Push a result to the history, maintaining the maximum size.
2387    fn push_result(&mut self, success: bool) {
2388        if self.recent_results.len() >= CIRCUIT_HISTORY_SIZE {
2389            self.recent_results.remove(0);
2390        }
2391        self.recent_results.push(success);
2392    }
2393
2394    fn now_millis() -> u64 {
2395        std::time::SystemTime::now()
2396            .duration_since(std::time::UNIX_EPOCH)
2397            .map(|d| d.as_millis() as u64)
2398            .unwrap_or(0)
2399    }
2400
2401    /// Drive the circuit through a single health-check outcome and return the
2402    /// resulting [`CircuitState`].
2403    ///
2404    /// This is the single, self-contained transition engine shared by every
2405    /// circuit consumer (the daemon health monitor via
2406    /// `WorkerState::record_health_check`, and the diagnostic `WorkerHealth`
2407    /// panel via `WorkerHealth::observe`). Keeping the transition logic in ONE
2408    /// place is what makes `WorkerState.circuit` (which selection reads) and the
2409    /// diagnostic `WorkerHealth.circuit` agree.
2410    ///
2411    /// Behaviour:
2412    /// - On a **healthy** outcome: if the circuit is Open and its cooldown has
2413    ///   elapsed it first transitions Open → HalfOpen *before* crediting the
2414    ///   success, so recovery needs exactly `success_threshold` clean probes
2415    ///   (crediting a success against an Open circuit would be lost). Then the
2416    ///   success is recorded and the circuit closes once the threshold is met.
2417    /// - On an **unhealthy** outcome: the failure is recorded, the circuit opens
2418    ///   if the closed-state open condition is met, and a HalfOpen probe failure
2419    ///   reopens the circuit — UNLESS the failure was `transient` (a retryable
2420    ///   transport blip that survived the in-check retries). A transient
2421    ///   half-open failure neither reopens nor resets the cooldown, so a single
2422    ///   network hiccup during recovery cannot bounce a worker back to Open.
2423    /// - After either branch, an Open circuit past its cooldown is nudged to
2424    ///   HalfOpen so probing resumes on the next poll even for a still-failing
2425    ///   worker.
2426    pub fn apply_health_outcome(
2427        &mut self,
2428        healthy: bool,
2429        transient: bool,
2430        config: &CircuitBreakerConfig,
2431    ) -> CircuitState {
2432        if healthy {
2433            // Transition to half-open BEFORE crediting the success so the probe
2434            // counts toward recovery. record_success() only increments the
2435            // consecutive-success counter while in HalfOpen.
2436            if self.state == CircuitState::Open && self.should_half_open(config) {
2437                self.half_open();
2438            }
2439            self.record_success();
2440            if self.should_close(config) {
2441                self.close();
2442            }
2443        } else {
2444            let was_half_open = self.state == CircuitState::HalfOpen;
2445            self.record_failure();
2446            if self.should_open(config) {
2447                self.open();
2448            } else if was_half_open && !transient {
2449                // A real (non-transient) probe failure in half-open reopens the
2450                // circuit and restarts the cooldown. A transient failure is
2451                // ignored so a network blip can't undo an in-progress recovery.
2452                self.open();
2453            }
2454        }
2455
2456        // Even for a still-failing worker, resume probing once the cooldown
2457        // elapses so a recovered worker is retried promptly.
2458        if self.state == CircuitState::Open && self.should_half_open(config) {
2459            self.half_open();
2460        }
2461
2462        self.state
2463    }
2464
2465    /// Drive the circuit through a single *command-class* outcome and return
2466    /// the resulting [`CircuitState`].
2467    ///
2468    /// Command-class evidence comes from an operation of the same shape as
2469    /// dispatched work: a remote binary that must run to completion (the
2470    /// telemetry poll, or a health probe whose SSH session connected but whose
2471    /// command never finished). It is the signal the plain liveness probe
2472    /// cannot provide — a host under fork exhaustion or swap thrash keeps
2473    /// answering `echo` over a warm connection while every real command hangs,
2474    /// and on that evidence alone the breaker would happily re-admit it.
2475    ///
2476    /// Behaviour:
2477    /// - **Success**: clears the outstanding command-failure count and then
2478    ///   counts as an ordinary healthy outcome (may advance Open -> HalfOpen ->
2479    ///   Closed exactly like a liveness success).
2480    /// - **Failure**: increments the command-failure count and records a plain
2481    ///   failure. The circuit opens when either the ordinary open condition or
2482    ///   `command_failure_threshold` is met. In HalfOpen a command failure
2483    ///   ALWAYS reopens — there is no transient exemption, because "connected
2484    ///   but the command did not complete" is precisely the condition the
2485    ///   half-open trial exists to detect. While any command failure is
2486    ///   outstanding, [`Self::should_close`] refuses to close on liveness
2487    ///   successes alone.
2488    pub fn apply_command_outcome(
2489        &mut self,
2490        success: bool,
2491        config: &CircuitBreakerConfig,
2492    ) -> CircuitState {
2493        if success {
2494            self.consecutive_command_failures = 0;
2495            return self.apply_health_outcome(true, false, config);
2496        }
2497
2498        self.consecutive_command_failures = self.consecutive_command_failures.saturating_add(1);
2499        let was_half_open = self.state == CircuitState::HalfOpen;
2500        self.record_failure();
2501        let threshold = config.command_failure_threshold.max(1);
2502        if was_half_open
2503            || self.should_open(config)
2504            || (self.state == CircuitState::Closed
2505                && self.consecutive_command_failures >= threshold)
2506        {
2507            self.open();
2508        }
2509
2510        if self.state == CircuitState::Open && self.should_half_open(config) {
2511            self.half_open();
2512        }
2513
2514        self.state
2515    }
2516}
2517
2518#[derive(Debug, Clone, Serialize, Deserialize)]
2519pub struct CompilationConfig {
2520    /// Minimum confidence score to intercept (0.0-1.0).
2521    #[serde(default = "default_confidence")]
2522    pub confidence_threshold: f64,
2523    /// Run locally when this build's measured local median is below this (ms).
2524    /// Consulted only by the speed gate, i.e. when `local_slots > 0`.
2525    #[serde(default = "default_min_local_time")]
2526    pub min_local_time_ms: u64,
2527    /// Minimum measured speedup ratio (local_time / remote_time) to offload.
2528    /// If the measured speedup < threshold, run locally. Default: 1.2 (20% faster).
2529    /// Consulted only by the speed gate, i.e. when `local_slots > 0`.
2530    #[serde(default = "default_speedup_threshold")]
2531    pub remote_speedup_threshold: f64,
2532    /// Speed gate (bd-2spi4): how many offloadable builds may run concurrently
2533    /// on THIS machine because its own measured build history says remote is
2534    /// not worth it. Zero (default) disables the gate, so every offloadable
2535    /// build is offloaded as before. Set it only on a dispatcher that is faster
2536    /// than its workers; local runs also need free memory (see the hook).
2537    #[serde(default)]
2538    pub local_slots: u32,
2539    /// Default slot estimate for build commands (cargo build, gcc, etc.).
2540    #[serde(default = "default_build_slots")]
2541    pub build_slots: u32,
2542    /// Default slot estimate for test commands (cargo test, nextest).
2543    /// Tests typically use more parallelism than builds.
2544    #[serde(default = "default_test_slots")]
2545    pub test_slots: u32,
2546    /// Default slot estimate for check/lint commands (cargo check, clippy).
2547    /// These are typically faster and use fewer resources.
2548    #[serde(default = "default_check_slots")]
2549    pub check_slots: u32,
2550    /// Per-build additional free-space budget on the worker's reported build
2551    /// filesystem, in GiB. Include expected output growth and a safety margin.
2552    /// Zero (default) disables this explicit budget. Positive budgets require
2553    /// a fresh disk observation and are held in durable daemon ownership until
2554    /// completion. They are admission accounting, not filesystem quotas.
2555    #[serde(default)]
2556    pub disk_headroom_gib: u32,
2557    /// Timeout in seconds for build commands.
2558    /// Build commands (cargo build, gcc, etc.) typically complete faster than tests.
2559    #[serde(default = "default_build_timeout")]
2560    pub build_timeout_sec: u64,
2561    /// Timeout in seconds for test commands.
2562    /// Test commands often need longer timeouts due to test suite execution time.
2563    #[serde(default = "default_test_timeout")]
2564    pub test_timeout_sec: u64,
2565    /// Timeout in seconds for bun commands (bun test, bun typecheck).
2566    /// Bun has known issues where its internal --timeout flag doesn't work for
2567    /// CPU-bound hangs, so we use a shorter external timeout by default.
2568    #[serde(default = "default_bun_timeout")]
2569    pub bun_timeout_sec: u64,
2570    /// Whether to wrap remote commands with external timeout protection.
2571    /// This prevents runaway/stuck processes from consuming worker slots indefinitely.
2572    /// Default: true. Set to false to disable timeout wrapping entirely.
2573    #[serde(default = "default_external_timeout_enabled")]
2574    pub external_timeout_enabled: bool,
2575    /// Whether a remote build FAILURE may fall back to LOCAL execution on the
2576    /// orchestrator once every eligible remote worker has been exhausted.
2577    ///
2578    /// On a remote failure the hook first retries the build on a *different,
2579    /// higher-capacity* worker (see `run_exec`'s worker-fallback loop). Local
2580    /// execution runs the heavy compile on the orchestrator that coordinates the
2581    /// whole fleet, so it is a genuine last resort and can flood that box. This
2582    /// flag gates it. Default `true` preserves the historical fail-open behavior
2583    /// (local only when no worker can serve); set to `false` on an orchestrator
2584    /// to refuse local fallback and fail the build closed instead. `force_remote`
2585    /// / proof mode / `RCH_REQUIRE_REMOTE` still override this by refusing local
2586    /// unconditionally.
2587    #[serde(default = "default_allow_local_fallback")]
2588    pub allow_local_fallback: bool,
2589    /// Per-job rustc parallelism cap applied on the WORKER (issue #49).
2590    ///
2591    /// `total_slots` bounds concurrent *jobs* per worker, but cargo defaults
2592    /// `-j` to the worker's `nproc`, so one offloaded `cargo test --workspace`
2593    /// forks `nproc` rustc regardless of slots and can swap-thrash a
2594    /// RAM-light box. This policy injects `CARGO_BUILD_JOBS` into the remote
2595    /// build environment; see [`RemoteBuildJobs`] for the modes. It never
2596    /// overrides a `CARGO_BUILD_JOBS` the worker session already carries, an
2597    /// inline `CARGO_BUILD_JOBS=N` or `-j N` in the command, or a project
2598    /// `.cargo/config.toml` `[build] jobs` — explicit intent always wins.
2599    #[serde(default)]
2600    pub remote_build_jobs: RemoteBuildJobs,
2601}
2602
2603/// How rch derives the `CARGO_BUILD_JOBS` it injects into a remote build.
2604///
2605/// - `auto` (default): computed on the worker from its live `nproc` and
2606///   total RAM as `clamp(ram_gib / 8, 2, min(nproc, 8))` — budgeting roughly
2607///   8 GiB of RAM per concurrent rustc leaves headroom for two to three
2608///   concurrent jobs at a realistic per-rustc peak of ~3 GiB.
2609/// - `off`: inject nothing (historical behavior; cargo uses `nproc`).
2610/// - `<n>`: inject a fixed job count fleet-wide.
2611///
2612/// In every mode an explicitly set `CARGO_BUILD_JOBS` on the worker (for
2613/// example from `/etc/environment`) or in the command is left untouched.
2614#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
2615pub enum RemoteBuildJobs {
2616    /// Derive from the worker's cores and RAM at execution time.
2617    #[default]
2618    Auto,
2619    /// Do not inject `CARGO_BUILD_JOBS`.
2620    Off,
2621    /// Inject a fixed job count (must be >= 1).
2622    Fixed(u32),
2623}
2624
2625impl RemoteBuildJobs {
2626    /// Parse the config/CLI spelling: `auto`, `off`, or a positive integer.
2627    pub fn parse(value: &str) -> Result<Self, String> {
2628        let trimmed = value.trim();
2629        match trimmed.to_ascii_lowercase().as_str() {
2630            "auto" => Ok(Self::Auto),
2631            "off" | "none" | "disabled" => Ok(Self::Off),
2632            _ => match trimmed.parse::<u32>() {
2633                Ok(0) => Err(
2634                    "remote_build_jobs must be >= 1 (cargo rejects `-j 0`); use `off` to disable"
2635                        .to_string(),
2636                ),
2637                Ok(n) => Ok(Self::Fixed(n)),
2638                Err(_) => Err(format!(
2639                    "'{trimmed}' is not a valid remote_build_jobs value (expected `auto`, `off`, or a positive integer)"
2640                )),
2641            },
2642        }
2643    }
2644}
2645
2646impl std::fmt::Display for RemoteBuildJobs {
2647    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2648        match self {
2649            Self::Auto => f.write_str("auto"),
2650            Self::Off => f.write_str("off"),
2651            Self::Fixed(n) => write!(f, "{n}"),
2652        }
2653    }
2654}
2655
2656impl std::str::FromStr for RemoteBuildJobs {
2657    type Err = String;
2658
2659    fn from_str(value: &str) -> Result<Self, Self::Err> {
2660        Self::parse(value)
2661    }
2662}
2663
2664impl Serialize for RemoteBuildJobs {
2665    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
2666        match self {
2667            Self::Auto => serializer.serialize_str("auto"),
2668            Self::Off => serializer.serialize_str("off"),
2669            Self::Fixed(n) => serializer.serialize_u32(*n),
2670        }
2671    }
2672}
2673
2674impl<'de> Deserialize<'de> for RemoteBuildJobs {
2675    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
2676        #[derive(Deserialize)]
2677        #[serde(untagged)]
2678        enum Raw {
2679            Count(u64),
2680            Spelling(String),
2681        }
2682        match Raw::deserialize(deserializer)? {
2683            Raw::Count(n) => {
2684                let n = u32::try_from(n).map_err(|_| {
2685                    serde::de::Error::custom(format!("remote_build_jobs {n} is out of range"))
2686                })?;
2687                Self::parse(&n.to_string()).map_err(serde::de::Error::custom)
2688            }
2689            Raw::Spelling(s) => Self::parse(&s).map_err(serde::de::Error::custom),
2690        }
2691    }
2692}
2693
2694impl Default for CompilationConfig {
2695    fn default() -> Self {
2696        Self {
2697            confidence_threshold: 0.85,
2698            min_local_time_ms: 2000,
2699            remote_speedup_threshold: default_speedup_threshold(),
2700            local_slots: 0,
2701            build_slots: default_build_slots(),
2702            test_slots: default_test_slots(),
2703            check_slots: default_check_slots(),
2704            disk_headroom_gib: 0,
2705            build_timeout_sec: default_build_timeout(),
2706            test_timeout_sec: default_test_timeout(),
2707            bun_timeout_sec: default_bun_timeout(),
2708            external_timeout_enabled: default_external_timeout_enabled(),
2709            allow_local_fallback: default_allow_local_fallback(),
2710            remote_build_jobs: RemoteBuildJobs::default(),
2711        }
2712    }
2713}
2714
2715fn default_build_slots() -> u32 {
2716    4
2717}
2718
2719fn default_test_slots() -> u32 {
2720    8
2721}
2722
2723fn default_check_slots() -> u32 {
2724    2
2725}
2726
2727/// Default build timeout: 5 minutes (300 seconds).
2728/// Most builds complete well within this time.
2729fn default_build_timeout() -> u64 {
2730    300
2731}
2732
2733/// Default test timeout: 30 minutes (1800 seconds).
2734/// Test suites often take significantly longer than builds.
2735fn default_test_timeout() -> u64 {
2736    1800
2737}
2738
2739/// Default bun timeout: 10 minutes (600 seconds).
2740/// Bun's internal --timeout flag doesn't work for CPU-bound hangs,
2741/// so we use a shorter external timeout to prevent zombies.
2742fn default_bun_timeout() -> u64 {
2743    600
2744}
2745
2746/// Default: external timeout protection is enabled.
2747fn default_external_timeout_enabled() -> bool {
2748    true
2749}
2750
2751/// Default: permit last-resort local fallback after remote workers are
2752/// exhausted. Set `compilation.allow_local_fallback = false` to fail closed and
2753/// keep heavy compiles off the orchestrator entirely.
2754fn default_allow_local_fallback() -> bool {
2755    true
2756}
2757
2758impl CompilationConfig {
2759    /// Returns the appropriate external timeout for the given compilation kind.
2760    ///
2761    /// - Bun commands get the shorter bun_timeout (to protect against known hang issues)
2762    /// - Test commands get the longer test_timeout
2763    /// - All other commands (builds, checks, clippy) get the build_timeout
2764    pub fn timeout_for_kind(&self, kind: Option<CompilationKind>) -> std::time::Duration {
2765        let secs = match kind {
2766            // Bun has known issues where internal timeout doesn't work for CPU hangs
2767            Some(CompilationKind::BunTest) | Some(CompilationKind::BunTypecheck) => {
2768                self.bun_timeout_sec
2769            }
2770            // Other test commands use the standard test timeout
2771            Some(CompilationKind::CargoTest) | Some(CompilationKind::CargoNextest) => {
2772                self.test_timeout_sec
2773            }
2774            // Nix derivation builds can take a long time (fetch + compile large
2775            // closures); give them the generous test-timeout bucket rather than
2776            // the short build timeout so legitimate builds are not killed early.
2777            Some(CompilationKind::NixBuild) => self.test_timeout_sec,
2778            // Go test suites are long-running like cargo test; give them the
2779            // test bucket rather than the short build timeout.
2780            Some(CompilationKind::GoTest) => self.test_timeout_sec,
2781            // Builds, checks, clippy, and unknown use build timeout
2782            _ => self.build_timeout_sec,
2783        };
2784        std::time::Duration::from_secs(secs)
2785    }
2786
2787    /// Returns whether external timeout wrapping is enabled.
2788    pub fn external_timeout_enabled(&self) -> bool {
2789        self.external_timeout_enabled
2790    }
2791}
2792
2793/// Version of the Layer 0 configuration pack schema (bd-bqu38 / M-1 §149).
2794///
2795/// Bump when a knob is added, removed, or its meaning changes; consumers
2796/// record the version alongside emitted baselines so regressions are
2797/// attributable to a specific pack revision.
2798pub const LAYER0_PACK_VERSION: &str = "1";
2799
2800/// Layer 0 configuration pack knobs (bd-bqu38).
2801///
2802/// Every knob defaults to OFF and toggles independently — the kill/rollback
2803/// contract from the master plan (§149: removed from defaults if it regresses
2804/// representative p95, output equivalence, debugger behavior, or
2805/// compatibility). Knobs are injected into remote cargo builds as
2806/// `CARGO_PROFILE_*` environment variables, so enabling one never mutates any
2807/// user-owned `.cargo/config.toml`.
2808#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
2809#[serde(default)]
2810pub struct Layer0Config {
2811    /// Master switch: nothing is injected unless this is on.
2812    pub enabled: bool,
2813    /// `profile.release.lto = "thin"` via `CARGO_PROFILE_RELEASE_LTO=thin`.
2814    pub release_lto_thin: bool,
2815    /// `profile.release.codegen-units = 1` via
2816    /// `CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1`.
2817    pub release_codegen_units_1: bool,
2818}
2819
2820impl Layer0Config {
2821    /// Environment pairs to inject into a remote cargo build.
2822    ///
2823    /// Empty unless [`Self::enabled`] — the master switch gates every knob so
2824    /// an operator can disable the whole pack with one key. Order is stable
2825    /// (declaration order) for deterministic emission and test assertions.
2826    #[must_use]
2827    pub fn active_env(&self) -> Vec<(&'static str, &'static str)> {
2828        if !self.enabled {
2829            return Vec::new();
2830        }
2831        let mut env = Vec::new();
2832        if self.release_lto_thin {
2833            env.push(("CARGO_PROFILE_RELEASE_LTO", "thin"));
2834        }
2835        if self.release_codegen_units_1 {
2836            env.push(("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"));
2837        }
2838        env
2839    }
2840}
2841
2842#[derive(Debug, Clone, Serialize, Deserialize)]
2843pub struct TransferConfig {
2844    /// Durable controller retrieval staging base. Unset uses the job-lease
2845    /// directory's retrieval subtree. Must be absolute; TMPDIR is not used.
2846    #[serde(default, skip_serializing_if = "Option::is_none")]
2847    pub retrieval_staging_dir: Option<PathBuf>,
2848    /// Free bytes to retain on the staging filesystem in addition to selected
2849    /// uncompressed output bytes. The preflight never deletes retained data.
2850    #[serde(default)]
2851    pub retrieval_min_free_bytes: u64,
2852    /// zstd compression level (1-19).
2853    #[serde(default = "default_compression")]
2854    pub compression_level: u32,
2855    /// Patterns to exclude from transfer.
2856    #[serde(default = "default_excludes")]
2857    pub exclude_patterns: Vec<String>,
2858    /// SSH keepalive interval in seconds (`ssh -o ServerAliveInterval=<N>`).
2859    ///
2860    /// When unset, build transfers use 15 seconds. Zero defers to OpenSSH configuration.
2861    #[serde(default, skip_serializing_if = "Option::is_none")]
2862    pub ssh_server_alive_interval_secs: Option<u64>,
2863    /// SSH ControlPersist idle timeout in seconds (`ssh -o ControlPersist=<N>s`).
2864    ///
2865    /// - Unset preserves OpenSSH defaults (ControlPersist enabled by default via mux).
2866    /// - `0` disables persistence (`ControlPersist=no`).
2867    #[serde(default, skip_serializing_if = "Option::is_none")]
2868    pub ssh_control_persist_secs: Option<u64>,
2869    /// Staging base on the worker for the transfer paths that do NOT mirror
2870    /// the client's project layout: `--clean-overlay` roots, Windows workers,
2871    /// and `rch cache warm`. Must be an absolute path; defaults to
2872    /// [`default_remote_base`] (`/data/tmp/rch`).
2873    ///
2874    /// An ordinary `rch exec` does not use this: it mirrors the project under
2875    /// the worker's canonical root (`[path_topology] canonical_root`), and the
2876    /// pooled Cargo target store rides along inside that mirror. To move
2877    /// pooled stores onto another filesystem, set
2878    /// `[remediation.pooled_target] store_base` (issue #64).
2879    #[serde(default = "default_remote_base")]
2880    pub remote_base: String,
2881    /// Per-attempt timeout for source synchronization, in milliseconds.
2882    ///
2883    /// This is deliberately separate from remote compilation and artifact-return
2884    /// timeouts. When unset, RCH derives a conservative timeout from the source
2885    /// payload size (30 seconds plus one second per MiB, capped at one hour).
2886    #[serde(default, skip_serializing_if = "Option::is_none")]
2887    pub sync_timeout_ms: Option<u64>,
2888    /// Silence-based stall timeout for source synchronization, in seconds
2889    /// (issue #59).
2890    ///
2891    /// Aborts a source sync when rsync emits NO output at all — no progress
2892    /// refresh, no stats, no itemized line — for this long. This is deliberately
2893    /// far tighter than the wall-clock `sync_timeout_ms` cap (which scales with
2894    /// payload size up to an hour): a large first-time sync keeps refreshing its
2895    /// progress line and is never killed, while a dead channel or wedged rsync
2896    /// is detected within this window instead of holding the reservation until
2897    /// the wall-clock cap. On a stall the hook releases the worker and re-enters
2898    /// selection excluding it. `0` disables silence detection.
2899    #[serde(default = "default_source_sync_silence_timeout_secs")]
2900    pub source_sync_silence_timeout_secs: u64,
2901    /// Retry policy for transient network errors during transfer.
2902    #[serde(default)]
2903    pub retry: RetryConfig,
2904    /// Verify artifact integrity using blake3 hashes after transfer (bd-377q).
2905    ///
2906    /// When enabled, computes blake3 hashes of key artifacts (binaries, test outputs)
2907    /// on the worker after build, then verifies them locally after download.
2908    /// Disabled by default to avoid extra overhead.
2909    #[serde(default)]
2910    pub verify_artifacts: bool,
2911    /// Maximum file size (bytes) for artifact verification (bd-377q).
2912    ///
2913    /// Files larger than this limit are skipped during verification to avoid
2914    /// excessive I/O. Defaults to 100MB.
2915    #[serde(default = "default_verify_max_size")]
2916    pub verify_max_size_bytes: u64,
2917
2918    // =========================================================================
2919    // Transfer Optimization (bd-3hho)
2920    // =========================================================================
2921    /// Maximum transfer size in MB before skipping remote execution.
2922    ///
2923    /// When set, runs `rsync --dry-run --stats` to estimate transfer size.
2924    /// If the estimated size exceeds this threshold, remote offload is skipped
2925    /// and the command runs locally. Set to `None` (default) to disable.
2926    #[serde(default, skip_serializing_if = "Option::is_none")]
2927    pub max_transfer_mb: Option<u64>,
2928
2929    /// Maximum estimated transfer time in milliseconds before skipping remote.
2930    ///
2931    /// Uses `estimated_bandwidth_bps` (or measured link speed) to calculate
2932    /// expected transfer time. If it exceeds this threshold, remote offload
2933    /// is skipped. Set to `None` (default) to disable.
2934    #[serde(default, skip_serializing_if = "Option::is_none")]
2935    pub max_transfer_time_ms: Option<u64>,
2936
2937    /// Bandwidth limit for rsync in KB/s.
2938    ///
2939    /// Passed as `rsync --bwlimit=<N>` to limit transfer speed and prevent
2940    /// network saturation. Set to `None` or `0` (default) for unlimited.
2941    #[serde(default, skip_serializing_if = "Option::is_none")]
2942    pub bwlimit_kbps: Option<u64>,
2943
2944    /// Explicit rsync binary for every local transfer (issue #66).
2945    ///
2946    /// Unset (the default) lets rch pick: the PATH `rsync` when it is a
2947    /// modern (3.1+) rsync, otherwise a modern rsync from the well-known
2948    /// Homebrew/MacPorts locations, otherwise the PATH binary driven with the
2949    /// openrsync/2.6.9-compatible argv. Set this to force one binary
2950    /// (`~` is expanded; a bare name is looked up on PATH). The
2951    /// `RCH_RSYNC_BIN` environment variable overrides this setting.
2952    #[serde(default, skip_serializing_if = "Option::is_none")]
2953    pub rsync_bin: Option<String>,
2954
2955    /// Estimated link bandwidth in bytes per second.
2956    ///
2957    /// Used for transfer time estimation when `max_transfer_time_ms` is set.
2958    /// If not provided, defaults to 10 MB/s (10485760 bytes/sec).
2959    #[serde(default, skip_serializing_if = "Option::is_none")]
2960    pub estimated_bandwidth_bps: Option<u64>,
2961
2962    // =========================================================================
2963    // Adaptive Compression (bd-243w)
2964    // =========================================================================
2965    /// Enable adaptive zstd compression based on transfer size (bd-243w).
2966    ///
2967    /// When enabled, automatically selects compression level based on estimated
2968    /// payload size:
2969    /// - < 10MB: level 1 (fast, minimal compression)
2970    /// - 10-200MB: level 3 (balanced, default)
2971    /// - > 200MB: level 7 (slower, better compression)
2972    ///
2973    /// When disabled (default), uses the fixed `compression_level` setting.
2974    #[serde(default)]
2975    pub adaptive_compression: bool,
2976
2977    /// Minimum compression level for adaptive mode (bd-243w).
2978    ///
2979    /// Even in adaptive mode, compression level won't go below this.
2980    /// Defaults to 1.
2981    #[serde(default = "default_min_compression")]
2982    pub min_compression_level: u32,
2983
2984    /// Maximum compression level for adaptive mode (bd-243w).
2985    ///
2986    /// Even in adaptive mode, compression level won't exceed this.
2987    /// Defaults to 9 (avoids CPU-intensive levels 10+).
2988    #[serde(default = "default_max_compression")]
2989    pub max_compression_level: u32,
2990}
2991
2992impl Default for TransferConfig {
2993    fn default() -> Self {
2994        Self {
2995            retrieval_staging_dir: None,
2996            retrieval_min_free_bytes: 0,
2997            compression_level: 3,
2998            exclude_patterns: default_excludes(),
2999            ssh_server_alive_interval_secs: None,
3000            ssh_control_persist_secs: None,
3001            remote_base: default_remote_base(),
3002            sync_timeout_ms: None,
3003            source_sync_silence_timeout_secs: default_source_sync_silence_timeout_secs(),
3004            retry: RetryConfig::default(),
3005            verify_artifacts: false,
3006            verify_max_size_bytes: default_verify_max_size(),
3007            // Transfer optimization (bd-3hho)
3008            max_transfer_mb: None,
3009            max_transfer_time_ms: None,
3010            bwlimit_kbps: None,
3011            rsync_bin: None,
3012            estimated_bandwidth_bps: None,
3013            // Adaptive compression (bd-243w)
3014            adaptive_compression: false,
3015            min_compression_level: default_min_compression(),
3016            max_compression_level: default_max_compression(),
3017        }
3018    }
3019}
3020
3021impl TransferConfig {
3022    /// Smallest accepted explicit source-sync timeout (one second).
3023    pub const MIN_SYNC_TIMEOUT_MS: u64 = 1_000;
3024    /// Largest accepted explicit or derived source-sync timeout (one hour).
3025    pub const MAX_SYNC_TIMEOUT_MS: u64 = 3_600_000;
3026    /// Fixed floor used by the payload-aware default policy.
3027    pub const DEFAULT_SYNC_TIMEOUT_FLOOR_MS: u64 = 30_000;
3028    const DEFAULT_SYNC_BYTES_PER_SECOND: u64 = 1024 * 1024;
3029
3030    /// Whether an explicit source-sync timeout is within the supported range.
3031    #[must_use]
3032    pub fn valid_sync_timeout_ms(value: u64) -> bool {
3033        (Self::MIN_SYNC_TIMEOUT_MS..=Self::MAX_SYNC_TIMEOUT_MS).contains(&value)
3034    }
3035
3036    /// Resolve the per-attempt source-sync timeout for an exact payload size.
3037    ///
3038    /// Explicit configuration wins. Otherwise the default grants 30 seconds of
3039    /// setup time plus one second per MiB at a conservative 1 MiB/s floor. This
3040    /// prevents a cold, large pinned tree from inheriting the old fixed 30-second
3041    /// ceiling while still bounding every individual attempt.
3042    #[must_use]
3043    pub fn sync_timeout_for_payload(&self, payload_bytes: u64) -> std::time::Duration {
3044        let timeout_ms = self
3045            .sync_timeout_ms
3046            .filter(|value| Self::valid_sync_timeout_ms(*value))
3047            .unwrap_or_else(|| {
3048                let transfer_seconds = payload_bytes
3049                    .saturating_add(Self::DEFAULT_SYNC_BYTES_PER_SECOND - 1)
3050                    / Self::DEFAULT_SYNC_BYTES_PER_SECOND;
3051                Self::DEFAULT_SYNC_TIMEOUT_FLOOR_MS
3052                    .saturating_add(transfer_seconds.saturating_mul(1_000))
3053                    .min(Self::MAX_SYNC_TIMEOUT_MS)
3054            });
3055        std::time::Duration::from_millis(timeout_ms)
3056    }
3057
3058    /// Select compression level based on estimated transfer size (bd-243w).
3059    ///
3060    /// When adaptive compression is enabled, selects an appropriate level
3061    /// based on payload size:
3062    /// - < 10MB: level 1 (fast, minimal compression)
3063    /// - 10-200MB: level 3 (balanced)
3064    /// - > 200MB: level 7 (better compression)
3065    ///
3066    /// The result is clamped between `min_compression_level` and `max_compression_level`.
3067    ///
3068    /// When adaptive compression is disabled, returns the fixed `compression_level`.
3069    pub fn select_compression_level(&self, estimated_bytes: Option<u64>) -> u32 {
3070        if !self.adaptive_compression {
3071            return self.compression_level;
3072        }
3073
3074        let Some(bytes) = estimated_bytes else {
3075            // No estimate available, use default
3076            return self.compression_level;
3077        };
3078
3079        // Size thresholds
3080        const SMALL_THRESHOLD: u64 = 10_000_000; // 10 MB (decimal)
3081        const LARGE_THRESHOLD: u64 = 200_000_000; // 200 MB (decimal)
3082
3083        let level = if bytes < SMALL_THRESHOLD {
3084            1 // Fast compression for small payloads
3085        } else if bytes < LARGE_THRESHOLD {
3086            3 // Balanced for medium payloads
3087        } else {
3088            7 // Better compression for large payloads
3089        };
3090
3091        // Clamp to configured bounds
3092        level.clamp(self.min_compression_level, self.max_compression_level)
3093    }
3094}
3095
3096// =============================================================================
3097// Execution Configuration (bd-785w)
3098// =============================================================================
3099
3100/// Default allowlist of commands that can be executed remotely.
3101/// Aligned with the classifier's supported CompilationKind values.
3102/// Command bases of every compilation kind rch knows how to offload.
3103///
3104/// This is the built-in `execution.allowlist`. It is also what a persisted,
3105/// explicit allowlist is compared against: a config file written before a tool
3106/// family was added (Go, Nix, TypeScript, ...) silently keeps that family local,
3107/// and the only symptom is a `[RCH] local (... not in execution.allowlist)` line
3108/// (issue #51). `ExecutionConfig::missing_builtin_bases` surfaces the gap.
3109#[rustfmt::skip]
3110pub const BUILTIN_EXECUTION_BASES: &[&str] = &[
3111    // Rust
3112    "cargo",
3113    "rustc",
3114    // cargo-nextest
3115    "nextest",
3116    // C/C++
3117    "gcc",
3118    "g++",
3119    "clang",
3120    "clang++",
3121    "cc",
3122    "c++",
3123    // Build systems
3124    "make",
3125    "cmake",
3126    "ninja",
3127    "meson",
3128    // Bun
3129    "bun",
3130    // Nix (covers both `nix build` and the legacy `nix-build`)
3131    "nix",
3132    // Go
3133    "go",
3134    // TypeScript (covers both `tsc` and `npx tsc`; command_base is "tsc")
3135    "tsc",
3136];
3137
3138fn default_execution_allowlist() -> Vec<String> {
3139    BUILTIN_EXECUTION_BASES
3140        .iter()
3141        .map(|base| (*base).to_string())
3142        .collect()
3143}
3144
3145/// Execution allowlist configuration for remote builds (bd-785w).
3146///
3147/// Controls which command base names are permitted for remote execution.
3148/// Commands not in the allowlist will fail-open to local execution.
3149#[derive(Debug, Clone, Serialize, Deserialize)]
3150pub struct ExecutionConfig {
3151    /// Allowlist of command base names permitted for remote execution.
3152    ///
3153    /// Examples: "cargo", "rustc", "gcc", "bun"
3154    ///
3155    /// If a classified compilation command's base name is not in this list,
3156    /// RCH will fail-open to local execution with a clear reason.
3157    ///
3158    /// An empty allowlist disables all remote execution (local only).
3159    #[serde(default = "default_execution_allowlist")]
3160    pub allowlist: Vec<String>,
3161}
3162
3163impl Default for ExecutionConfig {
3164    fn default() -> Self {
3165        Self {
3166            allowlist: default_execution_allowlist(),
3167        }
3168    }
3169}
3170
3171impl ExecutionConfig {
3172    /// Check if a command base name is allowed for remote execution.
3173    pub fn is_allowed(&self, command_base: &str) -> bool {
3174        self.allowlist
3175            .iter()
3176            .any(|allowed| allowed.eq_ignore_ascii_case(command_base))
3177    }
3178
3179    /// Built-in compilation tool bases that this allowlist does NOT admit.
3180    ///
3181    /// Empty for the default allowlist. Non-empty means some command family rch
3182    /// classifies as compilation will always fall open to local execution on
3183    /// this box — usually because the config file predates that family.
3184    pub fn missing_builtin_bases(&self) -> Vec<&'static str> {
3185        BUILTIN_EXECUTION_BASES
3186            .iter()
3187            .copied()
3188            .filter(|base| !self.is_allowed(base))
3189            .collect()
3190    }
3191}
3192
3193// =============================================================================
3194// Retry Configuration (bd-x1ek)
3195// =============================================================================
3196
3197fn default_retry_max_attempts() -> u32 {
3198    3
3199}
3200
3201fn default_retry_base_delay_ms() -> u64 {
3202    100
3203}
3204
3205fn default_retry_max_delay_ms() -> u64 {
3206    5000
3207}
3208
3209fn default_retry_jitter_factor() -> f64 {
3210    0.1
3211}
3212
3213fn default_retry_total_timeout_ms() -> u64 {
3214    30000
3215}
3216
3217/// Configuration for retry behavior on transient errors.
3218///
3219/// Used by rsync and SSH operations to recover from transient network failures
3220/// without blocking too long or retrying non-recoverable errors.
3221#[derive(Debug, Clone, Serialize, Deserialize)]
3222pub struct RetryConfig {
3223    /// Maximum number of retry attempts (1 = no retries, just initial attempt).
3224    #[serde(default = "default_retry_max_attempts")]
3225    pub max_attempts: u32,
3226
3227    /// Initial delay between retries in milliseconds.
3228    #[serde(default = "default_retry_base_delay_ms")]
3229    pub base_delay_ms: u64,
3230
3231    /// Maximum delay between retries in milliseconds (caps exponential backoff).
3232    #[serde(default = "default_retry_max_delay_ms")]
3233    pub max_delay_ms: u64,
3234
3235    /// Jitter factor (0.0 - 1.0) to randomize delay.
3236    /// A value of 0.1 means ±10% randomization.
3237    #[serde(default = "default_retry_jitter_factor")]
3238    pub jitter_factor: f64,
3239
3240    /// Total timeout for all retry attempts in milliseconds.
3241    /// After this, retries stop even if max_attempts not reached.
3242    #[serde(default = "default_retry_total_timeout_ms")]
3243    pub total_timeout_ms: u64,
3244}
3245
3246impl Default for RetryConfig {
3247    fn default() -> Self {
3248        Self {
3249            max_attempts: default_retry_max_attempts(),
3250            base_delay_ms: default_retry_base_delay_ms(),
3251            max_delay_ms: default_retry_max_delay_ms(),
3252            jitter_factor: default_retry_jitter_factor(),
3253            total_timeout_ms: default_retry_total_timeout_ms(),
3254        }
3255    }
3256}
3257
3258impl RetryConfig {
3259    /// Create a config that disables retries (single attempt only).
3260    pub fn no_retry() -> Self {
3261        Self {
3262            max_attempts: 1,
3263            ..Default::default()
3264        }
3265    }
3266
3267    /// Calculate delay for a given attempt number (0-indexed).
3268    ///
3269    /// Uses exponential backoff with optional jitter.
3270    pub fn delay_for_attempt(&self, attempt: u32) -> std::time::Duration {
3271        if attempt == 0 {
3272            return std::time::Duration::ZERO;
3273        }
3274
3275        // Exponential backoff: base * 2^attempt
3276        let base = self.base_delay_ms as f64;
3277        let delay = base * (2.0_f64.powi(attempt as i32 - 1));
3278        let capped = delay.min(self.max_delay_ms as f64);
3279
3280        // Apply jitter: delay ± (delay * jitter_factor)
3281        let jittered = if self.jitter_factor > 0.0 {
3282            let jitter_range = capped * self.jitter_factor;
3283            let jitter = (rand::rng().random::<f64>() * 2.0 - 1.0) * jitter_range;
3284            (capped + jitter).max(0.0)
3285        } else {
3286            capped
3287        };
3288
3289        std::time::Duration::from_millis(jittered as u64)
3290    }
3291
3292    /// Check if we should retry given elapsed time since start.
3293    pub fn should_retry(&self, attempt: u32, elapsed: std::time::Duration) -> bool {
3294        if attempt >= self.max_attempts {
3295            return false;
3296        }
3297        elapsed.as_millis() < self.total_timeout_ms as u128
3298    }
3299}
3300
3301/// Default source-sync silence timeout (issue #59): two minutes without a
3302/// single byte of rsync output means a dead channel or wedged rsync, not a
3303/// large transfer (progress2 refreshes its line continuously while bytes
3304/// flow, and each refresh counts as forward progress).
3305fn default_source_sync_silence_timeout_secs() -> u64 {
3306    120
3307}
3308
3309/// Default remote base path for project transfers.
3310///
3311/// Rooted under `/data/tmp` (the fleet-standard managed ephemeral-build zone)
3312/// rather than `/tmp`: on the Linux workers `/tmp` is a RAM-backed tmpfs, so
3313/// dispatched build artifacts placed there consume RAM and evade the disk-GC
3314/// (sbh / datatmp-janitor) that watches `/data/tmp`. Keeping the remote base in
3315/// the managed zone lets those reapers reclaim orphaned build trees. Must agree
3316/// with the rchd cache-cleanup and reclaim `remote_base` defaults.
3317pub fn default_remote_base() -> String {
3318    "/data/tmp/rch".to_string()
3319}
3320
3321/// Validate and normalize a remote base path.
3322/// Returns an error if the path is not absolute or contains traversal sequences.
3323pub fn validate_remote_base(path: &str) -> Result<String, String> {
3324    // Expand tilde for user convenience
3325    let expanded = shellexpand::tilde(path).into_owned();
3326
3327    // Check for absolute path
3328    if !expanded.starts_with('/') {
3329        return Err(format!(
3330            "remote_base must be an absolute path, got: {}",
3331            path
3332        ));
3333    }
3334
3335    // Check for path traversal attempts
3336    if expanded.contains("..") {
3337        return Err(format!(
3338            "remote_base must not contain path traversal (..): {}",
3339            path
3340        ));
3341    }
3342
3343    // Normalize: remove trailing slashes
3344    let normalized = expanded.trim_end_matches('/').to_string();
3345
3346    // Safety checks
3347    if normalized.is_empty() || normalized == "/" {
3348        return Err("remote_base cannot be the root directory (safety restriction)".to_string());
3349    }
3350
3351    // Enforce at least 2 levels deep (e.g. /tmp/rch, /home/user/rch)
3352    // /tmp -> depth 1 (unsafe for recursive delete)
3353    // /tmp/rch -> depth 2 (safe)
3354    let components: Vec<&str> = normalized.split('/').filter(|c| !c.is_empty()).collect();
3355    if components.len() < 2 {
3356        return Err(format!(
3357            "remote_base must be at least 2 levels deep (e.g. /tmp/rch), got: {}",
3358            normalized
3359        ));
3360    }
3361
3362    Ok(normalized)
3363}
3364
3365fn default_circuit_failure_threshold() -> u32 {
3366    3
3367}
3368
3369fn default_circuit_success_threshold() -> u32 {
3370    2
3371}
3372
3373fn default_circuit_error_rate_threshold() -> f64 {
3374    0.5
3375}
3376
3377fn default_circuit_window_secs() -> u64 {
3378    60
3379}
3380
3381fn default_circuit_open_cooldown_secs() -> u64 {
3382    30
3383}
3384
3385fn default_circuit_half_open_max_probes() -> u32 {
3386    1
3387}
3388
3389fn default_circuit_command_failure_threshold() -> u32 {
3390    2
3391}
3392
3393fn default_circuit_open_cooldown_max_secs() -> u64 {
3394    // 30s base doubled four times: 30, 60, 120, 240, 480.
3395    480
3396}
3397
3398fn default_true() -> bool {
3399    true
3400}
3401
3402fn default_log_level() -> String {
3403    "info".to_string()
3404}
3405
3406pub fn default_socket_path() -> String {
3407    // Prefer XDG_RUNTIME_DIR if available (per-user runtime directory).
3408    if let Ok(runtime_dir) = std::env::var("XDG_RUNTIME_DIR")
3409        && !runtime_dir.trim().is_empty()
3410    {
3411        let path = PathBuf::from(runtime_dir).join("rch.sock");
3412        return path.to_string_lossy().to_string();
3413    }
3414
3415    // Fallback to ~/.cache/rch/rch.sock (persistent across reboots).
3416    if let Some(cache_dir) = dirs::cache_dir() {
3417        let rch_cache = cache_dir.join("rch");
3418        let _ = std::fs::create_dir_all(&rch_cache);
3419        return rch_cache.join("rch.sock").to_string_lossy().to_string();
3420    }
3421
3422    // Last resort: /tmp/rch.sock
3423    "/tmp/rch.sock".to_string()
3424}
3425
3426#[allow(dead_code)]
3427fn default_output_visibility() -> OutputVisibility {
3428    OutputVisibility::None
3429}
3430
3431fn default_confidence() -> f64 {
3432    0.85
3433}
3434
3435fn default_min_local_time() -> u64 {
3436    2000
3437}
3438
3439/// Default speedup threshold: 1.2 (require 20% speedup to offload).
3440/// This is conservative to avoid offloading builds that would be only
3441/// marginally faster remotely when accounting for transfer overhead.
3442fn default_speedup_threshold() -> f64 {
3443    1.2
3444}
3445
3446fn default_compression() -> u32 {
3447    3
3448}
3449
3450/// Default maximum file size for artifact verification (100MB).
3451fn default_verify_max_size() -> u64 {
3452    100 * 1024 * 1024 // 100MB
3453}
3454
3455/// Default minimum compression level for adaptive mode.
3456fn default_min_compression() -> u32 {
3457    1
3458}
3459
3460/// Default maximum compression level for adaptive mode.
3461fn default_max_compression() -> u32 {
3462    9
3463}
3464
3465fn default_excludes() -> Vec<String> {
3466    vec![
3467        // Rust build artifacts
3468        "target/".to_string(),
3469        "*.rlib".to_string(),
3470        "*.rmeta".to_string(),
3471        // Git metadata is large, often permission-sensitive on workers, and
3472        // a partial `.git/` tree is not reliable for build scripts anyway.
3473        ".git/".to_string(),
3474        // Node.js / Bun dependencies (massive, reinstalled on worker)
3475        "node_modules/".to_string(),
3476        // Bun cache and runtime files
3477        ".bun/".to_string(),
3478        // npm / pnpm caches
3479        ".npm/".to_string(),
3480        ".pnpm-store/".to_string(),
3481        // Common build output directories
3482        "dist/".to_string(),
3483        "build/".to_string(),
3484        // Framework-specific caches
3485        ".next/".to_string(),
3486        ".nuxt/".to_string(),
3487        ".turbo/".to_string(),
3488        ".parcel-cache/".to_string(),
3489        // Beads runtime state (local issue DB + lock/temp files); not needed remotely.
3490        ".beads/".to_string(),
3491        // Local diagnostic runs may contain host-specific symlinks and are not build inputs.
3492        ".doctor/".to_string(),
3493        // Coverage reports (generated during tests)
3494        "coverage/".to_string(),
3495        ".nyc_output/".to_string(),
3496        // Credentials and secrets (must never be transferred to workers)
3497        ".cargo/credentials".to_string(),
3498        ".cargo/credentials.toml".to_string(),
3499        ".env".to_string(),
3500        ".env.*".to_string(),
3501        "*.pem".to_string(),
3502        "*.key".to_string(),
3503        "credentials.json".to_string(),
3504        "secrets.json".to_string(),
3505        "secrets.yaml".to_string(),
3506        "secrets.yml".to_string(),
3507        "secrets.toml".to_string(),
3508        "secrets.env".to_string(),
3509        "secrets.txt".to_string(),
3510        "secrets.conf".to_string(),
3511        "secrets.cfg".to_string(),
3512        "secrets.properties".to_string(),
3513        "secrets.xml".to_string(),
3514        // The classes above were already covered. These are the ones
3515        // RABS's E027 source-capture seed set denies
3516        // (rabs-sandbox/src/source_capture.rs) that this list did not,
3517        // so the two halves of the product now agree about what a
3518        // secret is. Two components disagreeing on that is the kind of
3519        // gap nobody notices until one of them is the path that ran.
3520        //
3521        // rsync patterns without a `/` match a basename at any depth,
3522        // so these cover nested copies too — a key checked into a
3523        // fixtures directory is excluded exactly like one at the root.
3524        //
3525        // SSH private keys. The public halves (.pub) are deliberately
3526        // not excluded: they are not secret and some builds read them.
3527        "id_rsa".to_string(),
3528        "id_ed25519".to_string(),
3529        "id_ecdsa".to_string(),
3530        "id_dsa".to_string(),
3531        // Key/certificate container formats (*.pem and *.key are above).
3532        "*.p12".to_string(),
3533        "*.pfx".to_string(),
3534        "*.keystore".to_string(),
3535        "*.jks".to_string(),
3536        // Machine credentials and cloud/container config that carries
3537        // long-lived tokens.
3538        ".netrc".to_string(),
3539        "_netrc".to_string(),
3540        ".aws/credentials".to_string(),
3541        ".docker/config.json".to_string(),
3542        // Package-registry auth tokens. These sit in project roots far
3543        // more often than the key formats above and are the likeliest
3544        // real leak in a normal repo.
3545        ".npmrc".to_string(),
3546        ".pypirc".to_string(),
3547        // GnuPG private keyring.
3548        "secring.*".to_string(),
3549    ]
3550}
3551
3552// ============================================================================
3553// Build History Types
3554// ============================================================================
3555
3556/// Location where a build was executed.
3557#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
3558#[serde(rename_all = "snake_case")]
3559pub enum BuildLocation {
3560    /// Build executed locally.
3561    #[default]
3562    Local,
3563    /// Build executed on a remote worker.
3564    Remote,
3565}
3566
3567/// Worker health snapshot captured when a cancellation completes.
3568#[derive(Debug, Clone, Serialize, Deserialize)]
3569pub struct BuildCancellationWorkerHealth {
3570    /// Worker status at cancellation completion.
3571    pub status: String,
3572    /// Worker speed score at cancellation completion.
3573    pub speed_score: f64,
3574    /// Slots currently used on the worker.
3575    pub used_slots: u32,
3576    /// Slots currently available on the worker.
3577    pub available_slots: u32,
3578    /// Normalized pressure state.
3579    pub pressure_state: String,
3580    /// Stable pressure reason code.
3581    pub pressure_reason_code: String,
3582}
3583
3584/// Structured cancellation metadata attached to cancelled build records.
3585#[derive(Debug, Clone, Serialize, Deserialize)]
3586pub struct BuildCancellationMetadata {
3587    /// Stable cancellation operation identifier.
3588    pub operation_id: String,
3589    /// Cancellation origin (for example `user`, `timeout`).
3590    pub origin: String,
3591    /// Stable reason code for cancellation.
3592    pub reason_code: String,
3593    /// Ordered cancellation state path.
3594    #[serde(default)]
3595    pub decision_path: Vec<String>,
3596    /// Highest escalation stage reached (`term`, `remote_kill`, `sigkill`).
3597    pub escalation_stage: String,
3598    /// Number of escalation transitions taken.
3599    pub escalation_count: u32,
3600    /// Whether remote kill was attempted.
3601    pub remote_kill_attempted: bool,
3602    /// Whether cleanup completed successfully.
3603    pub cleanup_ok: bool,
3604    /// Whether active build history was successfully claimed/cancelled.
3605    pub history_cancelled: bool,
3606    /// Final cancellation state.
3607    pub final_state: String,
3608    /// Worker health snapshot at completion.
3609    #[serde(default, skip_serializing_if = "Option::is_none")]
3610    pub worker_health: Option<BuildCancellationWorkerHealth>,
3611}
3612
3613/// Record of a completed build.
3614#[derive(Debug, Clone, Serialize, Deserialize)]
3615pub struct BuildRecord {
3616    /// Unique build identifier.
3617    pub id: u64,
3618    /// When the build started (ISO 8601 timestamp).
3619    pub started_at: String,
3620    /// When the build completed (ISO 8601 timestamp).
3621    pub completed_at: String,
3622    /// Project identifier (usually directory name or hash).
3623    pub project_id: String,
3624    /// Worker that executed the build (None if local).
3625    pub worker_id: Option<String>,
3626    /// Full command executed.
3627    pub command: String,
3628    /// Exit code (0 = success).
3629    pub exit_code: i32,
3630    /// Duration in milliseconds.
3631    pub duration_ms: u64,
3632    /// Build location (local or remote).
3633    pub location: BuildLocation,
3634    /// Bytes transferred (if remote).
3635    pub bytes_transferred: Option<u64>,
3636    /// Optional timing breakdown for the pipeline.
3637    #[serde(default)]
3638    pub timing: Option<CommandTimingBreakdown>,
3639    /// Structured cancellation metadata for cancelled builds.
3640    #[serde(default, skip_serializing_if = "Option::is_none")]
3641    pub cancellation: Option<BuildCancellationMetadata>,
3642}
3643
3644/// Input payload for recording a completed build.
3645#[derive(Debug, Clone, Serialize, Deserialize)]
3646pub struct BuildRecordInput {
3647    /// When the build started (ISO 8601 timestamp).
3648    pub started_at: String,
3649    /// When the build completed (ISO 8601 timestamp).
3650    pub completed_at: String,
3651    /// Project identifier (usually directory name or hash).
3652    pub project_id: String,
3653    /// Worker that executed the build (None if local).
3654    pub worker_id: Option<String>,
3655    /// Full command executed.
3656    pub command: String,
3657    /// Exit code (0 = success).
3658    pub exit_code: i32,
3659    /// Duration in milliseconds.
3660    pub duration_ms: u64,
3661    /// Build location (local or remote).
3662    pub location: BuildLocation,
3663    /// Bytes transferred (if remote).
3664    pub bytes_transferred: Option<u64>,
3665    /// Optional timing breakdown for the pipeline.
3666    #[serde(default)]
3667    pub timing: Option<CommandTimingBreakdown>,
3668    /// Structured cancellation metadata for cancelled builds.
3669    #[serde(default, skip_serializing_if = "Option::is_none")]
3670    pub cancellation: Option<BuildCancellationMetadata>,
3671}
3672
3673impl BuildRecordInput {
3674    /// Convert input payload into a build record with a supplied ID.
3675    pub fn into_record(self, id: u64) -> BuildRecord {
3676        BuildRecord {
3677            id,
3678            started_at: self.started_at,
3679            completed_at: self.completed_at,
3680            project_id: self.project_id,
3681            worker_id: self.worker_id,
3682            command: self.command,
3683            exit_code: self.exit_code,
3684            duration_ms: self.duration_ms,
3685            location: self.location,
3686            bytes_transferred: self.bytes_transferred,
3687            timing: self.timing,
3688            cancellation: self.cancellation,
3689        }
3690    }
3691}
3692
3693/// Aggregate terminal-command statistics for build history.
3694///
3695/// These counts do not infer whether compilation completed before a failing
3696/// test or interrupted command. The denominator includes every stored record.
3697#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3698pub struct BuildStats {
3699    /// Total number of builds in history.
3700    pub total_builds: usize,
3701    /// Number of successfully completed commands (exit_code == 0).
3702    pub success_count: usize,
3703    /// Number of unsuccessful commands, including interruption (exit_code != 0).
3704    pub failure_count: usize,
3705    /// Number of remote builds.
3706    pub remote_count: usize,
3707    /// Number of local builds.
3708    pub local_count: usize,
3709    /// Average build duration in milliseconds.
3710    pub avg_duration_ms: u64,
3711}
3712
3713/// Saved time statistics from remote builds.
3714///
3715/// Tracks estimated time savings from offloading builds to remote workers.
3716/// Estimates are only produced from an observed local baseline; without one the
3717/// stats carry measured remote durations with `estimate_basis = "none"` and
3718/// zero estimated savings rather than a fabricated speedup factor.
3719#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3720pub struct SavedTimeStats {
3721    /// Total time spent on remote builds (milliseconds).
3722    pub total_remote_duration_ms: u64,
3723    /// Estimated total time if those builds ran locally (milliseconds).
3724    /// Zero when no local baseline exists.
3725    pub estimated_local_duration_ms: u64,
3726    /// Time saved by offloading (milliseconds).
3727    /// Computed as max(0, estimated_local - remote); zero without a baseline.
3728    pub time_saved_ms: u64,
3729    /// Number of remote builds included in calculation.
3730    pub builds_counted: usize,
3731    /// Average speedup factor (local_estimate / remote_duration).
3732    /// A value of 2.0 means remote builds are ~2x faster than local.
3733    /// Zero when the estimate basis is "none".
3734    pub avg_speedup: f64,
3735    /// Total time saved today (milliseconds).
3736    pub today_saved_ms: u64,
3737    /// Total time saved this week (milliseconds).
3738    pub week_saved_ms: u64,
3739    /// How the local-time estimate was derived: "observed_local_mean" when
3740    /// successful local builds back the baseline, "none" otherwise.
3741    #[serde(default)]
3742    pub estimate_basis: String,
3743    /// Number of successful local builds backing the baseline (0 for "none").
3744    #[serde(default)]
3745    pub local_baseline_builds: usize,
3746}
3747
3748// ============================================================================
3749// Compilation Timing and Metrics
3750// ============================================================================
3751
3752use std::collections::VecDeque;
3753use std::time::{Duration, Instant};
3754
3755/// Breakdown of timing for each compilation phase.
3756#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3757pub struct CompilationTimingBreakdown {
3758    /// Time to sync source files to worker.
3759    #[serde(with = "duration_millis")]
3760    pub rsync_up: Duration,
3761    /// Time for cargo build on worker.
3762    #[serde(with = "duration_millis")]
3763    pub remote_build: Duration,
3764    /// Time to sync artifacts back.
3765    #[serde(with = "duration_millis")]
3766    pub rsync_down: Duration,
3767    /// Total end-to-end latency.
3768    #[serde(with = "duration_millis")]
3769    pub total: Duration,
3770}
3771
3772/// Timing breakdown for a single command pipeline.
3773///
3774/// Uses optional durations so missing phases serialize as `null`.
3775#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3776pub struct CommandTimingBreakdown {
3777    /// Time spent classifying the command.
3778    #[serde(with = "option_duration_millis")]
3779    pub classify: Option<Duration>,
3780    /// Time spent selecting a worker.
3781    #[serde(with = "option_duration_millis")]
3782    pub select: Option<Duration>,
3783    /// Time spent syncing source files to the worker.
3784    #[serde(with = "option_duration_millis")]
3785    pub sync_up: Option<Duration>,
3786    /// Time spent executing the remote command.
3787    #[serde(with = "option_duration_millis")]
3788    pub exec: Option<Duration>,
3789    /// Time spent syncing artifacts back.
3790    #[serde(with = "option_duration_millis")]
3791    pub sync_down: Option<Duration>,
3792    /// Time spent on cleanup (slot release, bookkeeping).
3793    #[serde(with = "option_duration_millis")]
3794    pub cleanup: Option<Duration>,
3795    /// Total end-to-end time for the pipeline.
3796    #[serde(with = "option_duration_millis")]
3797    pub total: Option<Duration>,
3798}
3799
3800mod duration_millis {
3801    use serde::{Deserialize, Deserializer, Serialize, Serializer};
3802    use std::time::Duration;
3803
3804    pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
3805    where
3806        S: Serializer,
3807    {
3808        duration.as_millis().serialize(serializer)
3809    }
3810
3811    pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
3812    where
3813        D: Deserializer<'de>,
3814    {
3815        let millis = u64::deserialize(deserializer)?;
3816        Ok(Duration::from_millis(millis))
3817    }
3818}
3819
3820/// Comprehensive metrics for a single compilation.
3821#[derive(Debug, Clone, Serialize, Deserialize)]
3822pub struct CompilationMetrics {
3823    /// Project identifier (usually directory name or hash).
3824    pub project_id: String,
3825    /// Worker that performed the compilation.
3826    pub worker_id: String,
3827    /// When the compilation occurred.
3828    pub timestamp: chrono::DateTime<chrono::Utc>,
3829    /// Timing breakdown for each phase.
3830    pub timing: CompilationTimingBreakdown,
3831    /// Local build time for comparison (if available).
3832    #[serde(with = "option_duration_millis")]
3833    pub local_build_time: Option<Duration>,
3834    /// Speedup ratio (local_time / remote_time).
3835    pub speedup: Option<f64>,
3836    /// Number of files synced to worker.
3837    pub files_synced: u64,
3838    /// Total bytes transferred.
3839    pub bytes_transferred: u64,
3840    /// Exit code from compilation.
3841    pub exit_code: i32,
3842    /// Whether compilation succeeded.
3843    pub success: bool,
3844}
3845
3846mod option_duration_millis {
3847    use serde::{Deserialize, Deserializer, Serialize, Serializer};
3848    use std::time::Duration;
3849
3850    pub fn serialize<S>(duration: &Option<Duration>, serializer: S) -> Result<S::Ok, S::Error>
3851    where
3852        S: Serializer,
3853    {
3854        match duration {
3855            Some(d) => Some(d.as_millis() as u64).serialize(serializer),
3856            None => Option::<u64>::None.serialize(serializer),
3857        }
3858    }
3859
3860    pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Duration>, D::Error>
3861    where
3862        D: Deserializer<'de>,
3863    {
3864        let opt: Option<u64> = Option::deserialize(deserializer)?;
3865        Ok(opt.map(Duration::from_millis))
3866    }
3867}
3868
3869impl CompilationMetrics {
3870    /// Calculate speedup based on local vs remote build time.
3871    pub fn calculate_speedup(&mut self) {
3872        if let Some(local) = self.local_build_time
3873            && self.timing.total.as_millis() > 0
3874        {
3875            self.speedup = Some(local.as_secs_f64() / self.timing.total.as_secs_f64());
3876        }
3877    }
3878
3879    /// Check if remote compilation was beneficial (faster than local).
3880    pub fn is_beneficial(&self) -> bool {
3881        self.speedup.map(|s| s > 1.0).unwrap_or(false)
3882    }
3883}
3884
3885impl Default for CompilationMetrics {
3886    fn default() -> Self {
3887        Self {
3888            project_id: String::new(),
3889            worker_id: String::new(),
3890            timestamp: chrono::Utc::now(),
3891            timing: CompilationTimingBreakdown::default(),
3892            local_build_time: None,
3893            speedup: None,
3894            files_synced: 0,
3895            bytes_transferred: 0,
3896            exit_code: 0,
3897            success: true,
3898        }
3899    }
3900}
3901
3902/// Timer for tracking compilation phases.
3903///
3904/// Use this to measure the duration of each phase of remote compilation.
3905///
3906/// # Example
3907/// ```ignore
3908/// let mut timer = CompilationTimer::new("my-project", "worker-1");
3909/// // ... rsync source files ...
3910/// timer.end_rsync_up();
3911/// // ... run cargo build ...
3912/// timer.end_remote_build();
3913/// // ... rsync artifacts back ...
3914/// timer.end_rsync_down();
3915/// let metrics = timer.finish(0, 100, 1_000_000);
3916/// ```
3917#[derive(Debug)]
3918pub struct CompilationTimer {
3919    project_id: String,
3920    worker_id: String,
3921    start: Instant,
3922    phase_start: Instant,
3923    rsync_up: Option<Duration>,
3924    remote_build: Option<Duration>,
3925    rsync_down: Option<Duration>,
3926}
3927
3928impl CompilationTimer {
3929    /// Create a new timer for a compilation.
3930    pub fn new(project_id: &str, worker_id: &str) -> Self {
3931        let now = Instant::now();
3932        Self {
3933            project_id: project_id.to_string(),
3934            worker_id: worker_id.to_string(),
3935            start: now,
3936            phase_start: now,
3937            rsync_up: None,
3938            remote_build: None,
3939            rsync_down: None,
3940        }
3941    }
3942
3943    /// Mark the end of the rsync upload phase.
3944    pub fn end_rsync_up(&mut self) {
3945        self.rsync_up = Some(self.phase_start.elapsed());
3946        self.phase_start = Instant::now();
3947        tracing::info!(
3948            rsync_up_ms = %self.rsync_up.unwrap().as_millis(),
3949            "TIMING: rsync_up completed"
3950        );
3951    }
3952
3953    /// Mark the end of the remote build phase.
3954    pub fn end_remote_build(&mut self) {
3955        self.remote_build = Some(self.phase_start.elapsed());
3956        self.phase_start = Instant::now();
3957        tracing::info!(
3958            remote_build_ms = %self.remote_build.unwrap().as_millis(),
3959            "TIMING: remote_build completed"
3960        );
3961    }
3962
3963    /// Mark the end of the rsync download phase.
3964    pub fn end_rsync_down(&mut self) {
3965        self.rsync_down = Some(self.phase_start.elapsed());
3966        tracing::info!(
3967            rsync_down_ms = %self.rsync_down.unwrap().as_millis(),
3968            "TIMING: rsync_down completed"
3969        );
3970    }
3971
3972    /// Finish timing and produce metrics.
3973    pub fn finish(self, exit_code: i32, files: u64, bytes: u64) -> CompilationMetrics {
3974        let total = self.start.elapsed();
3975        tracing::info!(
3976            total_ms = %total.as_millis(),
3977            exit_code = %exit_code,
3978            "TIMING: compilation completed"
3979        );
3980
3981        CompilationMetrics {
3982            project_id: self.project_id,
3983            worker_id: self.worker_id,
3984            timestamp: chrono::Utc::now(),
3985            timing: CompilationTimingBreakdown {
3986                rsync_up: self.rsync_up.unwrap_or_default(),
3987                remote_build: self.remote_build.unwrap_or_default(),
3988                rsync_down: self.rsync_down.unwrap_or_default(),
3989                total,
3990            },
3991            local_build_time: None,
3992            speedup: None,
3993            files_synced: files,
3994            bytes_transferred: bytes,
3995            exit_code,
3996            success: exit_code == 0,
3997        }
3998    }
3999}
4000
4001/// Aggregator for compilation metrics with statistics.
4002#[derive(Debug)]
4003pub struct MetricsAggregator {
4004    history: VecDeque<CompilationMetrics>,
4005    max_history: usize,
4006}
4007
4008impl MetricsAggregator {
4009    /// Create a new aggregator with a maximum history size.
4010    pub fn new(max_history: usize) -> Self {
4011        Self {
4012            history: VecDeque::with_capacity(max_history),
4013            max_history,
4014        }
4015    }
4016
4017    /// Record a new compilation's metrics.
4018    pub fn record(&mut self, metrics: CompilationMetrics) {
4019        if self.history.len() >= self.max_history {
4020            self.history.pop_front();
4021        }
4022        self.history.push_back(metrics);
4023    }
4024
4025    /// Get the average speedup across all recorded compilations.
4026    pub fn average_speedup(&self) -> Option<f64> {
4027        let speedups: Vec<f64> = self.history.iter().filter_map(|m| m.speedup).collect();
4028
4029        if speedups.is_empty() {
4030            None
4031        } else {
4032            Some(speedups.iter().sum::<f64>() / speedups.len() as f64)
4033        }
4034    }
4035
4036    /// Get the p50 (median) total compilation time.
4037    pub fn p50_total_time(&self) -> Option<Duration> {
4038        self.percentile_total_time(0.50)
4039    }
4040
4041    /// Get the p95 total compilation time.
4042    pub fn p95_total_time(&self) -> Option<Duration> {
4043        self.percentile_total_time(0.95)
4044    }
4045
4046    /// Get the p99 total compilation time.
4047    pub fn p99_total_time(&self) -> Option<Duration> {
4048        self.percentile_total_time(0.99)
4049    }
4050
4051    /// Get a percentile of total compilation times.
4052    ///
4053    /// Uses the "inclusive" percentile method (matching numpy.percentile, Excel PERCENTILE.INC):
4054    /// index = (n - 1) * percentile, where n is the number of samples.
4055    fn percentile_total_time(&self, percentile: f64) -> Option<Duration> {
4056        let mut times: Vec<_> = self.history.iter().map(|m| m.timing.total).collect();
4057
4058        if times.is_empty() {
4059            return None;
4060        }
4061
4062        times.sort();
4063        // Use inclusive percentile: index = (n - 1) * p
4064        // For p50 with 100 items: (99 * 0.50) = 49.5 -> 49 (true median position)
4065        // For p99 with 100 items: (99 * 0.99) = 98.01 -> 98
4066        let idx = (((times.len() - 1) as f64 * percentile).round() as usize).min(times.len() - 1);
4067        Some(times[idx])
4068    }
4069
4070    /// Get the success rate as a percentage.
4071    pub fn success_rate(&self) -> f64 {
4072        if self.history.is_empty() {
4073            return 100.0;
4074        }
4075        let successes = self.history.iter().filter(|m| m.success).count();
4076        (successes as f64 / self.history.len() as f64) * 100.0
4077    }
4078
4079    /// Get the number of recorded compilations.
4080    pub fn count(&self) -> usize {
4081        self.history.len()
4082    }
4083
4084    /// Get all recorded metrics.
4085    pub fn metrics(&self) -> &VecDeque<CompilationMetrics> {
4086        &self.history
4087    }
4088
4089    /// Clear all recorded metrics.
4090    pub fn clear(&mut self) {
4091        self.history.clear();
4092    }
4093}
4094
4095impl Default for MetricsAggregator {
4096    fn default() -> Self {
4097        Self::new(1000)
4098    }
4099}
4100
4101#[cfg(test)]
4102mod tests {
4103    use super::*;
4104    use crate::test_guard;
4105
4106    #[test]
4107    fn test_declared_os_reads_reserved_tag() {
4108        assert_eq!(
4109            declared_os(&["rust".to_string(), "os:windows".to_string()]),
4110            Some("windows".to_string())
4111        );
4112        assert_eq!(
4113            declared_os(&["os:WINDOWS".to_string()]),
4114            Some("windows".to_string())
4115        );
4116    }
4117
4118    #[test]
4119    fn test_declared_os_absent_for_ordinary_tags() {
4120        // Every historical worker: no reserved tag, so no exclusivity and the
4121        // worker stays a candidate for any job.
4122        assert_eq!(declared_os(&[]), None);
4123        assert_eq!(
4124            declared_os(&["rust".to_string(), "bun".to_string(), "go".to_string()]),
4125            None
4126        );
4127        // A malformed declaration must not silently fence the worker off.
4128        assert_eq!(declared_os(&["os:".to_string()]), None);
4129        assert_eq!(declared_os(&["os:  ".to_string()]), None);
4130    }
4131
4132    #[test]
4133    fn test_os_tag_roundtrips_through_declared_os() {
4134        for os in ["windows", "Darwin", " linux "] {
4135            let tag = os_tag(os);
4136            assert_eq!(
4137                declared_os(std::slice::from_ref(&tag)).as_deref(),
4138                Some(os.trim().to_ascii_lowercase().as_str())
4139            );
4140        }
4141    }
4142
4143    /// Verdict-webhook config must NEVER carry an inline secret. Operators
4144    /// reference secrets by env-var name (`*_env`); the resolved value is
4145    /// read at dispatch time and never persisted to the cache or config TOML.
4146    /// This guards against a future careless `pub secret: String` addition.
4147    #[test]
4148    fn test_no_inline_secret_fields() {
4149        let endpoint = DoctorWebhookEndpoint {
4150            name: "ops".to_string(),
4151            url: "https://example.com/hook".to_string(),
4152            format: DoctorWebhookFormat::Pagerduty,
4153            on_transitions: vec!["any_to_failing".to_string()],
4154            retry_max: 3,
4155            retry_backoff_ms: 200,
4156            timeout_ms: 5000,
4157            enabled: true,
4158            routing_key_env: Some("RCH_PAGERDUTY_ROUTING_KEY".to_string()),
4159            bearer_token_env: Some("RCH_WEBHOOK_BEARER".to_string()),
4160            signing_secret_env: Some("RCH_WEBHOOK_HMAC".to_string()),
4161        };
4162        // Serialize to a JSON object and inspect the *keys*. The forbidden
4163        // field names must be entirely absent; only `*_env` references survive.
4164        let value = serde_json::to_value(&endpoint).unwrap();
4165        let obj = value.as_object().expect("endpoint serializes to an object");
4166        for forbidden in ["secret", "routing_key", "bearer_token", "password", "token"] {
4167            assert!(
4168                !obj.contains_key(forbidden),
4169                "endpoint config must not expose an inline `{forbidden}` field"
4170            );
4171        }
4172        assert!(obj.contains_key("routing_key_env"));
4173        assert!(obj.contains_key("bearer_token_env"));
4174        assert!(obj.contains_key("signing_secret_env"));
4175    }
4176
4177    #[test]
4178    fn test_webhook_format_toml_spelling() {
4179        // The config TOML uses snake_case spellings. TOML documents are tables
4180        // at the root, so a bare enum cannot be (de)serialized standalone —
4181        // verify the wire spelling through a `format = "..."` table field, the
4182        // shape it actually appears in within the config.
4183        #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)]
4184        struct FormatWrap {
4185            format: DoctorWebhookFormat,
4186        }
4187        let rendered = toml::to_string(&FormatWrap {
4188            format: DoctorWebhookFormat::GenericJson,
4189        })
4190        .unwrap();
4191        assert!(
4192            rendered.contains("format = \"generic_json\""),
4193            "unexpected TOML: {rendered}"
4194        );
4195        let slack: FormatWrap = toml::from_str("format = \"slack\"").unwrap();
4196        assert_eq!(slack.format, DoctorWebhookFormat::Slack);
4197        let pd: FormatWrap = toml::from_str("format = \"pagerduty\"").unwrap();
4198        assert_eq!(pd.format, DoctorWebhookFormat::Pagerduty);
4199    }
4200
4201    #[test]
4202    fn test_doctor_webhooks_defaults() {
4203        let cfg = DoctorWebhooksConfig::default();
4204        assert!(cfg.endpoints.is_empty());
4205        assert_eq!(cfg.max_inflight, 16);
4206        // RchConfig wires the section in with serde(default), so an empty
4207        // config still yields an empty (never-firing) webhook surface.
4208        let rch: RchConfig = toml::from_str("").unwrap();
4209        assert!(rch.doctor.webhooks.endpoints.is_empty());
4210    }
4211
4212    #[test]
4213    fn test_doctor_webhook_endpoint_parses_from_toml() {
4214        let toml_src = r#"
4215[doctor.webhooks]
4216max_inflight = 8
4217
4218[[doctor.webhooks.endpoints]]
4219name = "ops-slack"
4220url = "https://hooks.slack.com/services/T0/B0/x"
4221format = "slack"
4222on_transitions = ["healthy_to_degraded", "any_to_failing"]
4223retry_max = 2
4224"#;
4225        let cfg: RchConfig = toml::from_str(toml_src).unwrap();
4226        assert_eq!(cfg.doctor.webhooks.max_inflight, 8);
4227        assert_eq!(cfg.doctor.webhooks.endpoints.len(), 1);
4228        let ep = &cfg.doctor.webhooks.endpoints[0];
4229        assert_eq!(ep.name, "ops-slack");
4230        assert_eq!(ep.format, DoctorWebhookFormat::Slack);
4231        assert_eq!(ep.on_transitions.len(), 2);
4232        assert_eq!(ep.retry_max, 2);
4233        // Defaults filled in for omitted fields.
4234        assert_eq!(ep.retry_backoff_ms, 200);
4235        assert_eq!(ep.timeout_ms, 5000);
4236        assert!(ep.enabled);
4237    }
4238
4239    fn assert_golden_json(
4240        actual: serde_json::Value,
4241        expected: &serde_json::Value,
4242        fixture_path: &str,
4243    ) {
4244        if &actual == expected {
4245            return;
4246        }
4247
4248        let actual_json = serde_json::to_string_pretty(&actual).expect("actual JSON renders");
4249        let expected_json = serde_json::to_string_pretty(expected).expect("expected JSON renders");
4250        assert_eq!(
4251            &actual,
4252            expected,
4253            "golden mismatch in {fixture_path}\n\
4254             expected_blake3={}\n\
4255             actual_blake3={}\n\
4256             bless path: review the semantic diff, update the fixture expected_* field, \
4257             and record both hashes in the Beads closeout.\n\
4258             expected:\n{expected_json}\n\
4259             actual:\n{actual_json}",
4260            blake3::hash(expected_json.as_bytes()),
4261            blake3::hash(actual_json.as_bytes())
4262        );
4263    }
4264
4265    #[test]
4266    fn test_circuit_state_default() {
4267        let _guard = test_guard!();
4268        assert_eq!(CircuitState::default(), CircuitState::Closed);
4269    }
4270
4271    #[test]
4272    fn test_circuit_config_defaults() {
4273        let _guard = test_guard!();
4274        let config = CircuitBreakerConfig::default();
4275        assert_eq!(config.failure_threshold, 3);
4276        assert_eq!(config.success_threshold, 2);
4277        assert_eq!(config.error_rate_threshold, 0.5);
4278        assert_eq!(config.window_secs, 60);
4279        assert_eq!(config.open_cooldown_secs, 30);
4280        assert_eq!(config.half_open_max_probes, 1);
4281    }
4282
4283    #[test]
4284    fn test_rch_config_has_circuit_defaults() {
4285        let _guard = test_guard!();
4286        let config = RchConfig::default();
4287        assert_eq!(config.circuit.failure_threshold, 3);
4288    }
4289
4290    #[test]
4291    fn test_circuit_config_serde_roundtrip() {
4292        let _guard = test_guard!();
4293        let config = CircuitBreakerConfig {
4294            failure_threshold: 5,
4295            success_threshold: 3,
4296            error_rate_threshold: 0.75,
4297            window_secs: 120,
4298            open_cooldown_secs: 45,
4299            half_open_max_probes: 2,
4300            command_failure_threshold: 4,
4301            open_cooldown_max_secs: 900,
4302        };
4303        let json = serde_json::to_string(&config).unwrap();
4304        let parsed: CircuitBreakerConfig = serde_json::from_str(&json).unwrap();
4305        assert_eq!(parsed.command_failure_threshold, 4);
4306        assert_eq!(parsed.open_cooldown_max_secs, 900);
4307        assert_eq!(parsed.failure_threshold, 5);
4308        assert_eq!(parsed.success_threshold, 3);
4309        assert_eq!(parsed.error_rate_threshold, 0.75);
4310        assert_eq!(parsed.window_secs, 120);
4311        assert_eq!(parsed.open_cooldown_secs, 45);
4312        assert_eq!(parsed.half_open_max_probes, 2);
4313    }
4314
4315    // ========================================================================
4316    // WorkerCapabilities Tests
4317    // ========================================================================
4318
4319    #[test]
4320    fn test_worker_capabilities_default() {
4321        let _guard = test_guard!();
4322        let caps = WorkerCapabilities::default();
4323        assert!(caps.rustc_version.is_none());
4324        assert!(caps.rustup_toolchains.is_empty());
4325        assert!(caps.rustup_components.is_empty());
4326        assert!(caps.bun_version.is_none());
4327        assert!(caps.node_version.is_none());
4328        assert!(caps.npm_version.is_none());
4329        assert!(caps.projects_root_ok.is_none());
4330        assert!(caps.projects_root_issue.is_none());
4331        assert!(caps.projects_root_checked_at_unix_ms.is_none());
4332    }
4333
4334    #[test]
4335    fn test_worker_capabilities_new() {
4336        let _guard = test_guard!();
4337        let caps = WorkerCapabilities::new();
4338        assert!(caps.rustc_version.is_none());
4339        assert!(!caps.has_rust());
4340        assert!(!caps.has_bun());
4341        assert!(!caps.has_node());
4342        assert!(caps.is_topology_healthy().is_none());
4343    }
4344
4345    #[test]
4346    fn test_worker_capabilities_has_rust() {
4347        let _guard = test_guard!();
4348        let mut caps = WorkerCapabilities::new();
4349        assert!(!caps.has_rust());
4350
4351        caps.rustc_version = Some("rustc 1.76.0 (07dca489a 2024-02-04)".to_string());
4352        assert!(caps.has_rust());
4353    }
4354
4355    #[test]
4356    fn test_worker_capabilities_components_are_toolchain_scoped_and_exact() {
4357        let _guard = test_guard!();
4358        let caps = WorkerCapabilities {
4359            rustup_components: vec![
4360                "stable-x86_64-unknown-linux-gnu:clippy".to_string(),
4361                "nightly-2026-07-05-x86_64-unknown-linux-gnu:rustfmt".to_string(),
4362                "beta-x86_64-unknown-linux-gnu:clippy-preview".to_string(),
4363            ],
4364            ..Default::default()
4365        };
4366
4367        assert!(caps.has_rustup_component("stable", "clippy"));
4368        assert!(caps.has_rustup_component("nightly-2026-07-05", "rustfmt"));
4369        assert!(!caps.has_rustup_component("nightly-2026-07-05", "clippy"));
4370        assert!(!caps.has_rustup_component("beta", "clippy"));
4371        assert!(!caps.has_rustup_component("nightly-2026-07-0", "rustfmt"));
4372    }
4373
4374    #[test]
4375    fn test_worker_capabilities_has_bun() {
4376        let _guard = test_guard!();
4377        let mut caps = WorkerCapabilities::new();
4378        assert!(!caps.has_bun());
4379
4380        caps.bun_version = Some("1.0.25".to_string());
4381        assert!(caps.has_bun());
4382    }
4383
4384    #[test]
4385    fn test_worker_capabilities_has_nix() {
4386        let _guard = test_guard!();
4387        let mut caps = WorkerCapabilities::new();
4388        assert!(!caps.has_nix());
4389
4390        caps.nix_version = Some("nix (Nix) 2.24.9".to_string());
4391        assert!(caps.has_nix());
4392    }
4393
4394    #[test]
4395    fn test_worker_capabilities_has_zig_requires_both_halves() {
4396        let _guard = test_guard!();
4397        let mut caps = WorkerCapabilities::new();
4398        assert!(!caps.has_zig());
4399
4400        // `zig` alone cannot run `cargo zigbuild` — cargo would report
4401        // `no such command: 'zigbuild'`.
4402        caps.zig_version = Some("0.14.1".to_string());
4403        assert!(!caps.has_zig());
4404
4405        // cargo-zigbuild alone cannot link — it shells out to `zig cc`.
4406        caps.zig_version = None;
4407        caps.cargo_zigbuild_version = Some("cargo-zigbuild 0.23.0".to_string());
4408        assert!(!caps.has_zig());
4409
4410        caps.zig_version = Some("0.14.1".to_string());
4411        assert!(caps.has_zig());
4412    }
4413
4414    #[test]
4415    fn test_nix_build_uses_generous_timeout() {
4416        let _guard = test_guard!();
4417        let cfg = CompilationConfig::default();
4418        // Nix builds get the long test-timeout bucket, not the short build timeout.
4419        assert_eq!(
4420            cfg.timeout_for_kind(Some(CompilationKind::NixBuild)),
4421            std::time::Duration::from_secs(cfg.test_timeout_sec)
4422        );
4423    }
4424
4425    #[test]
4426    fn test_job_uses_build_timeout_bucket() {
4427        let _guard = test_guard!();
4428        // Jobs (explicit `rch exec --job`) reuse the standard build-timeout
4429        // bucket, so `RCH_BUILD_TIMEOUT_SEC` governs them without new config.
4430        let cfg = CompilationConfig::default();
4431        assert_eq!(
4432            cfg.timeout_for_kind(Some(CompilationKind::Job)),
4433            std::time::Duration::from_secs(cfg.build_timeout_sec)
4434        );
4435    }
4436
4437    #[test]
4438    fn test_layer0_disabled_by_default_and_env_gated() {
4439        let _guard = test_guard!();
4440        let cfg = Layer0Config::default();
4441        assert!(!cfg.enabled);
4442        // Master switch off ⇒ nothing injects even with knobs on.
4443        let knobs_on = Layer0Config {
4444            enabled: false,
4445            release_lto_thin: true,
4446            release_codegen_units_1: true,
4447        };
4448        assert!(cfg.active_env().is_empty());
4449        assert!(knobs_on.active_env().is_empty());
4450    }
4451
4452    #[test]
4453    fn test_layer0_knobs_emit_stable_env_pairs() {
4454        let _guard = test_guard!();
4455        let lto_only = Layer0Config {
4456            enabled: true,
4457            release_lto_thin: true,
4458            release_codegen_units_1: false,
4459        };
4460        assert_eq!(
4461            lto_only.active_env(),
4462            vec![("CARGO_PROFILE_RELEASE_LTO", "thin")]
4463        );
4464        let both = Layer0Config {
4465            enabled: true,
4466            release_lto_thin: true,
4467            release_codegen_units_1: true,
4468        };
4469        assert_eq!(
4470            both.active_env(),
4471            vec![
4472                ("CARGO_PROFILE_RELEASE_LTO", "thin"),
4473                ("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"),
4474            ],
4475            "emission order must stay stable for deterministic baselines"
4476        );
4477    }
4478
4479    #[test]
4480    fn test_layer0_pack_version_is_recorded() {
4481        let _guard = test_guard!();
4482        // Baselines record the pack version; it must exist and be non-empty.
4483        assert!(!LAYER0_PACK_VERSION.is_empty());
4484    }
4485    #[test]
4486    fn test_default_allowlist_includes_nix() {
4487        let _guard = test_guard!();
4488        let cfg = ExecutionConfig::default();
4489        assert!(cfg.is_allowed("nix"));
4490    }
4491
4492    #[test]
4493    fn test_worker_capabilities_has_node() {
4494        let _guard = test_guard!();
4495        let mut caps = WorkerCapabilities::new();
4496        assert!(!caps.has_node());
4497
4498        caps.node_version = Some("v20.11.0".to_string());
4499        assert!(caps.has_node());
4500    }
4501
4502    #[test]
4503    fn test_worker_capabilities_multiple_runtimes() {
4504        let _guard = test_guard!();
4505        let caps = WorkerCapabilities {
4506            rustc_version: Some("rustc 1.76.0".to_string()),
4507            bun_version: Some("1.0.25".to_string()),
4508            node_version: Some("v20.11.0".to_string()),
4509            npm_version: Some("10.2.4".to_string()),
4510            ..Default::default()
4511        };
4512
4513        assert!(caps.has_rust());
4514        assert!(caps.has_bun());
4515        assert!(caps.has_node());
4516    }
4517
4518    #[test]
4519    fn test_worker_capabilities_serialization_empty() {
4520        let _guard = test_guard!();
4521        let caps = WorkerCapabilities::new();
4522        let json = serde_json::to_string(&caps).unwrap();
4523        // Empty capabilities should serialize to {}
4524        assert_eq!(json, "{}");
4525
4526        // And deserialize back correctly
4527        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4528        assert!(!parsed.has_rust());
4529        assert!(!parsed.has_bun());
4530        assert!(!parsed.has_node());
4531    }
4532
4533    #[test]
4534    fn test_worker_capabilities_serialization_with_versions() {
4535        let _guard = test_guard!();
4536        let caps = WorkerCapabilities {
4537            rustc_version: Some("rustc 1.76.0-nightly".to_string()),
4538            bun_version: Some("1.0.25".to_string()),
4539            node_version: None,
4540            npm_version: None,
4541            ..Default::default()
4542        };
4543
4544        let json = serde_json::to_string(&caps).unwrap();
4545        assert!(json.contains("rustc_version"));
4546        assert!(json.contains("rustc 1.76.0-nightly"));
4547        assert!(json.contains("bun_version"));
4548        assert!(json.contains("1.0.25"));
4549        // None fields should be skipped
4550        assert!(!json.contains("node_version"));
4551        assert!(!json.contains("npm_version"));
4552
4553        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4554        assert!(parsed.has_rust());
4555        assert!(parsed.has_bun());
4556        assert!(!parsed.has_node());
4557    }
4558
4559    #[test]
4560    fn test_worker_capabilities_deserialization_partial() {
4561        let _guard = test_guard!();
4562        // Deserialize JSON with only some fields
4563        let json = r#"{"bun_version": "1.0.0"}"#;
4564        let caps: WorkerCapabilities = serde_json::from_str(json).unwrap();
4565
4566        assert!(!caps.has_rust());
4567        assert!(caps.has_bun());
4568        assert!(!caps.has_node());
4569        assert_eq!(caps.bun_version, Some("1.0.0".to_string()));
4570    }
4571
4572    #[test]
4573    fn test_worker_capabilities_clone() {
4574        let _guard = test_guard!();
4575        let caps = WorkerCapabilities {
4576            rustc_version: Some("1.76.0".to_string()),
4577            bun_version: None,
4578            node_version: Some("v20".to_string()),
4579            npm_version: None,
4580            projects_root_ok: Some(false),
4581            projects_root_issue: Some("alias_missing".to_string()),
4582            projects_root_checked_at_unix_ms: Some(123),
4583            ..Default::default()
4584        };
4585
4586        let cloned = caps.clone();
4587        assert_eq!(cloned.rustc_version, caps.rustc_version);
4588        assert_eq!(cloned.bun_version, caps.bun_version);
4589        assert_eq!(cloned.node_version, caps.node_version);
4590        assert_eq!(cloned.npm_version, caps.npm_version);
4591        assert_eq!(cloned.projects_root_ok, caps.projects_root_ok);
4592        assert_eq!(cloned.projects_root_issue, caps.projects_root_issue);
4593        assert_eq!(
4594            cloned.projects_root_checked_at_unix_ms,
4595            caps.projects_root_checked_at_unix_ms
4596        );
4597    }
4598
4599    #[test]
4600    fn test_worker_capabilities_topology_health_helper() {
4601        let _guard = test_guard!();
4602        let mut caps = WorkerCapabilities::new();
4603        assert_eq!(caps.is_topology_healthy(), None);
4604
4605        caps.projects_root_ok = Some(true);
4606        assert_eq!(caps.is_topology_healthy(), Some(true));
4607
4608        caps.projects_root_ok = Some(false);
4609        assert_eq!(caps.is_topology_healthy(), Some(false));
4610    }
4611
4612    #[test]
4613    fn test_selection_reason_serialization() {
4614        let _guard = test_guard!();
4615        // Test that all variants serialize to snake_case
4616        assert_eq!(
4617            serde_json::to_string(&SelectionReason::Success).unwrap(),
4618            "\"success\""
4619        );
4620        assert_eq!(
4621            serde_json::to_string(&SelectionReason::NoWorkersConfigured).unwrap(),
4622            "\"no_workers_configured\""
4623        );
4624        assert_eq!(
4625            serde_json::to_string(&SelectionReason::AllWorkersUnreachable).unwrap(),
4626            "\"all_workers_unreachable\""
4627        );
4628        assert_eq!(
4629            serde_json::to_string(&SelectionReason::AllCircuitsOpen).unwrap(),
4630            "\"all_circuits_open\""
4631        );
4632        assert_eq!(
4633            serde_json::to_string(&SelectionReason::AllWorkersBusy).unwrap(),
4634            "\"all_workers_busy\""
4635        );
4636        assert_eq!(
4637            serde_json::to_string(&SelectionReason::NoWorkersPassedHealth).unwrap(),
4638            "\"no_workers_passed_health\""
4639        );
4640        assert_eq!(
4641            serde_json::to_string(&SelectionReason::AllWorkersFailedPreflight).unwrap(),
4642            "\"all_workers_failed_preflight\""
4643        );
4644        assert_eq!(
4645            serde_json::to_string(&SelectionReason::AllWorkersFailedConvergence).unwrap(),
4646            "\"all_workers_failed_convergence\""
4647        );
4648        assert_eq!(
4649            serde_json::to_string(&SelectionReason::NoAdmissibleWorkers(
4650                "critical_pressure=1,insufficient_slots=1".to_string()
4651            ))
4652            .unwrap(),
4653            "{\"no_admissible_workers\":\"critical_pressure=1,insufficient_slots=1\"}"
4654        );
4655        assert_eq!(
4656            serde_json::to_string(&SelectionReason::NoMatchingWorkers).unwrap(),
4657            "\"no_matching_workers\""
4658        );
4659    }
4660
4661    #[test]
4662    fn test_selection_reason_with_error() {
4663        let _guard = test_guard!();
4664        let reason = SelectionReason::SelectionError("test error".to_string());
4665        let json = serde_json::to_string(&reason).unwrap();
4666        assert!(json.contains("selection_error"));
4667        assert!(json.contains("test error"));
4668    }
4669
4670    #[test]
4671    fn test_selection_reason_deserialization() {
4672        let _guard = test_guard!();
4673        assert_eq!(
4674            serde_json::from_str::<SelectionReason>("\"success\"").unwrap(),
4675            SelectionReason::Success
4676        );
4677        assert_eq!(
4678            serde_json::from_str::<SelectionReason>("\"all_workers_busy\"").unwrap(),
4679            SelectionReason::AllWorkersBusy
4680        );
4681        assert_eq!(
4682            serde_json::from_str::<SelectionReason>("\"no_workers_passed_health\"").unwrap(),
4683            SelectionReason::NoWorkersPassedHealth
4684        );
4685        assert_eq!(
4686            serde_json::from_str::<SelectionReason>("\"all_workers_failed_preflight\"").unwrap(),
4687            SelectionReason::AllWorkersFailedPreflight
4688        );
4689        assert_eq!(
4690            serde_json::from_str::<SelectionReason>("\"all_workers_failed_convergence\"").unwrap(),
4691            SelectionReason::AllWorkersFailedConvergence
4692        );
4693        assert_eq!(
4694            serde_json::from_str::<SelectionReason>(
4695                "{\"no_admissible_workers\":\"critical_pressure=1\"}"
4696            )
4697            .unwrap(),
4698            SelectionReason::NoAdmissibleWorkers("critical_pressure=1".to_string())
4699        );
4700    }
4701
4702    #[test]
4703    fn test_selection_reason_display() {
4704        let _guard = test_guard!();
4705        assert_eq!(
4706            SelectionReason::Success.to_string(),
4707            "worker assigned successfully"
4708        );
4709        assert_eq!(
4710            SelectionReason::NoWorkersConfigured.to_string(),
4711            "no workers configured"
4712        );
4713        assert_eq!(
4714            SelectionReason::AllWorkersUnreachable.to_string(),
4715            "all workers unreachable"
4716        );
4717        assert_eq!(
4718            SelectionReason::AllWorkersBusy.to_string(),
4719            "all workers at capacity"
4720        );
4721        assert_eq!(
4722            SelectionReason::NoWorkersPassedHealth.to_string(),
4723            "no workers passed health thresholds"
4724        );
4725        assert_eq!(
4726            SelectionReason::AllWorkersFailedPreflight.to_string(),
4727            "all workers failed preflight checks"
4728        );
4729        assert_eq!(
4730            SelectionReason::AllWorkersFailedConvergence.to_string(),
4731            "all workers failed repo convergence checks"
4732        );
4733        assert_eq!(
4734            SelectionReason::NoAdmissibleWorkers(
4735                "critical_pressure=1,insufficient_slots=1".to_string()
4736            )
4737            .to_string(),
4738            "no admissible workers: critical_pressure=1,insufficient_slots=1"
4739        );
4740        assert_eq!(
4741            SelectionReason::SelectionError("oops".to_string()).to_string(),
4742            "selection error: oops"
4743        );
4744    }
4745
4746    #[test]
4747    fn test_selection_response_with_worker() {
4748        let _guard = test_guard!();
4749        let response = SelectionResponse {
4750            worker: Some(SelectedWorker {
4751                id: WorkerId::new("test"),
4752                host: "localhost".to_string(),
4753                user: "user".to_string(),
4754                identity_file: "~/.ssh/id_rsa".to_string(),
4755                slots_available: 8,
4756                speed_score: 75.0,
4757                declared_os: None,
4758            }),
4759            reason: SelectionReason::Success,
4760            build_id: None,
4761            diagnostics: None,
4762        };
4763
4764        let json = serde_json::to_string(&response).unwrap();
4765        assert!(json.contains("\"reason\":\"success\""));
4766        assert!(json.contains("\"id\":\"test\""));
4767    }
4768
4769    #[test]
4770    fn test_selection_response_without_worker() {
4771        let _guard = test_guard!();
4772        let response = SelectionResponse {
4773            worker: None,
4774            reason: SelectionReason::AllWorkersBusy,
4775            build_id: None,
4776            diagnostics: None,
4777        };
4778
4779        let json = serde_json::to_string(&response).unwrap();
4780        assert!(json.contains("\"worker\":null"));
4781        assert!(json.contains("\"reason\":\"all_workers_busy\""));
4782    }
4783
4784    #[test]
4785    fn golden_selection_response_with_no_workers_passed_health_diagnostics() {
4786        let _guard = test_guard!();
4787        const FIXTURE_PATH: &str =
4788            "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json";
4789        let fixture: serde_json::Value = serde_json::from_str(include_str!(
4790            "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json"
4791        ))
4792        .expect("selection diagnostics golden fixture parses");
4793        assert_eq!(
4794            fixture["schema_version"],
4795            "rch.golden.selection_diagnostics.v1"
4796        );
4797
4798        let response = SelectionResponse {
4799            worker: None,
4800            reason: SelectionReason::NoWorkersPassedHealth,
4801            build_id: None,
4802            diagnostics: Some(SelectionDiagnostics {
4803                required_runtime: RequiredRuntime::Rust,
4804                estimated_cores: 2,
4805                min_success_rate: 0.8,
4806                fallback_min_success_rate: 0.5,
4807                active_project_exclusion_count: 0,
4808                workers: vec![WorkerSelectionDiagnostic {
4809                    worker_id: WorkerId::new("fixture-worker"),
4810                    status: "healthy".to_string(),
4811                    circuit_state: "closed".to_string(),
4812                    pressure_state: "healthy".to_string(),
4813                    pressure_reason_code: "pressure.ok".to_string(),
4814                    success_rate: Some(0.2),
4815                    min_success_rate: 0.8,
4816                    fallback_min_success_rate: 0.5,
4817                    required_runtime: RequiredRuntime::Rust,
4818                    runtime_available: true,
4819                    available_slots: 8,
4820                    total_slots: 8,
4821                    estimated_cores: 2,
4822                    active_project_excluded: false,
4823                    final_decision: WorkerSelectionDiagnosticDecision::Deny,
4824                    final_reason: "success_rate 0.20 < fallback 0.50".to_string(),
4825                    reason_codes: vec![
4826                        "health.below_min_success_rate".to_string(),
4827                        "health.below_fallback_min_success_rate".to_string(),
4828                    ],
4829                }],
4830            }),
4831        };
4832
4833        assert_golden_json(
4834            serde_json::to_value(&response).expect("response serializes"),
4835            &fixture["expected_response"],
4836            FIXTURE_PATH,
4837        );
4838    }
4839
4840    #[test]
4841    fn test_selection_response_roundtrip() {
4842        let _guard = test_guard!();
4843        let original = SelectionResponse {
4844            worker: Some(SelectedWorker {
4845                id: WorkerId::new("worker-1"),
4846                host: "192.168.1.100".to_string(),
4847                user: "ubuntu".to_string(),
4848                identity_file: "/path/to/key".to_string(),
4849                slots_available: 16,
4850                speed_score: 90.5,
4851                declared_os: None,
4852            }),
4853            reason: SelectionReason::Success,
4854            build_id: None,
4855            diagnostics: None,
4856        };
4857
4858        let json = serde_json::to_string(&original).unwrap();
4859        let parsed: SelectionResponse = serde_json::from_str(&json).unwrap();
4860
4861        assert!(parsed.worker.is_some());
4862        let worker = parsed.worker.unwrap();
4863        assert_eq!(worker.id.as_str(), "worker-1");
4864        assert_eq!(worker.host, "192.168.1.100");
4865        assert_eq!(worker.slots_available, 16);
4866        assert_eq!(parsed.reason, SelectionReason::Success);
4867    }
4868
4869    #[test]
4870    fn test_build_heartbeat_phase_roundtrip() {
4871        let _guard = test_guard!();
4872        let json = serde_json::to_string(&BuildHeartbeatPhase::SyncUp).unwrap();
4873        assert_eq!(json, "\"sync_up\"");
4874        let parsed: BuildHeartbeatPhase = serde_json::from_str(&json).unwrap();
4875        assert_eq!(parsed, BuildHeartbeatPhase::SyncUp);
4876    }
4877
4878    #[test]
4879    fn test_build_heartbeat_request_roundtrip() {
4880        let _guard = test_guard!();
4881        let request = BuildHeartbeatRequest {
4882            build_id: 42,
4883            worker_id: WorkerId::new("worker-a"),
4884            hook_pid: Some(12345),
4885            local_wrapper_id: Some("wrapper-abc123".to_string()),
4886            remote_pgid_file: Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string()),
4887            phase: BuildHeartbeatPhase::Execute,
4888            detail: Some("Compiling crates".to_string()),
4889            progress_counter: Some(7),
4890            progress_percent: Some(42.5),
4891        };
4892
4893        let json = serde_json::to_string(&request).unwrap();
4894        let parsed: BuildHeartbeatRequest = serde_json::from_str(&json).unwrap();
4895        assert_eq!(parsed.build_id, 42);
4896        assert_eq!(parsed.worker_id.as_str(), "worker-a");
4897        assert_eq!(
4898            parsed.remote_pgid_file,
4899            Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string())
4900        );
4901        assert_eq!(parsed.phase, BuildHeartbeatPhase::Execute);
4902        assert_eq!(parsed.progress_counter, Some(7));
4903        assert_eq!(parsed.progress_percent, Some(42.5));
4904    }
4905
4906    // CircuitStats tests
4907
4908    #[test]
4909    fn test_circuit_stats_new() {
4910        let _guard = test_guard!();
4911        let stats = CircuitStats::new();
4912        assert_eq!(stats.state(), CircuitState::Closed);
4913        assert_eq!(stats.consecutive_failures(), 0);
4914        assert_eq!(stats.error_rate(), 0.0);
4915        assert!(stats.opened_at().is_none());
4916    }
4917
4918    #[test]
4919    fn test_circuit_stats_record_success() {
4920        let _guard = test_guard!();
4921        let mut stats = CircuitStats::new();
4922        stats.record_success();
4923        stats.record_success();
4924        assert_eq!(stats.error_rate(), 0.0);
4925        assert_eq!(stats.consecutive_failures(), 0);
4926    }
4927
4928    #[test]
4929    fn test_circuit_stats_record_failure() {
4930        let _guard = test_guard!();
4931        let mut stats = CircuitStats::new();
4932        stats.record_failure();
4933        stats.record_failure();
4934        assert_eq!(stats.consecutive_failures(), 2);
4935        assert_eq!(stats.error_rate(), 1.0); // 2 failures, 0 successes
4936    }
4937
4938    #[test]
4939    fn test_circuit_stats_error_rate() {
4940        let _guard = test_guard!();
4941        let mut stats = CircuitStats::new();
4942        stats.record_success();
4943        stats.record_success();
4944        stats.record_failure();
4945        // 1 failure / 3 total = 0.333...
4946        assert!((stats.error_rate() - 0.333).abs() < 0.01);
4947    }
4948
4949    #[test]
4950    fn test_circuit_stats_should_open_consecutive_failures() {
4951        let _guard = test_guard!();
4952        let mut stats = CircuitStats::new();
4953        let config = CircuitBreakerConfig::default(); // threshold = 3
4954
4955        stats.record_failure();
4956        assert!(!stats.should_open(&config));
4957
4958        stats.record_failure();
4959        assert!(!stats.should_open(&config));
4960
4961        stats.record_failure();
4962        assert!(stats.should_open(&config)); // 3 consecutive failures
4963    }
4964
4965    #[test]
4966    fn test_circuit_stats_should_open_error_rate() {
4967        let _guard = test_guard!();
4968        let mut stats = CircuitStats::new();
4969        let config = CircuitBreakerConfig {
4970            error_rate_threshold: 0.5,
4971            ..Default::default()
4972        };
4973
4974        // Need at least 5 samples
4975        stats.record_success();
4976        stats.record_success();
4977        stats.record_failure();
4978        stats.record_failure();
4979        assert!(!stats.should_open(&config)); // Only 4 samples
4980
4981        stats.record_failure(); // 5 samples: 2 success, 3 failures = 60% error rate
4982        assert!(stats.should_open(&config));
4983    }
4984
4985    #[test]
4986    fn test_circuit_close_clears_recent_results_no_flap() {
4987        let _guard = test_guard!();
4988        // Regression: error_rate() is computed over recent_results, so close()
4989        // must clear that history or a recovered circuit immediately re-opens
4990        // on stale pre-failure samples (flapping).
4991        let mut stats = CircuitStats::new();
4992        let config = CircuitBreakerConfig {
4993            error_rate_threshold: 0.5,
4994            ..Default::default()
4995        };
4996        for _ in 0..6 {
4997            stats.record_failure();
4998        }
4999        assert!(stats.should_open(&config));
5000        stats.open();
5001        // Recover via half-open probes, then close.
5002        stats.half_open();
5003        stats.record_success();
5004        stats.record_success();
5005        stats.close();
5006        // Error rate must be reset and the circuit must NOT immediately re-open.
5007        assert_eq!(stats.error_rate(), 0.0);
5008        assert!(!stats.should_open(&config));
5009    }
5010
5011    #[test]
5012    fn test_apply_health_outcome_opens_on_failure_threshold() {
5013        let _guard = test_guard!();
5014        // failure_threshold=3: three unhealthy outcomes open the circuit.
5015        let config = CircuitBreakerConfig {
5016            failure_threshold: 3,
5017            open_cooldown_secs: 300,
5018            ..Default::default()
5019        };
5020        let mut stats = CircuitStats::new();
5021        assert_eq!(
5022            stats.apply_health_outcome(false, false, &config),
5023            CircuitState::Closed
5024        );
5025        assert_eq!(
5026            stats.apply_health_outcome(false, false, &config),
5027            CircuitState::Closed
5028        );
5029        assert_eq!(
5030            stats.apply_health_outcome(false, false, &config),
5031            CircuitState::Open
5032        );
5033    }
5034
5035    #[test]
5036    fn test_apply_health_outcome_half_open_two_probe_recovery() {
5037        let _guard = test_guard!();
5038        // success_threshold=2: recovery from Open needs exactly two clean probes.
5039        // open_cooldown_secs=0 makes should_half_open() true immediately, so the
5040        // engine's trailing nudge leaves an opened circuit already in HalfOpen —
5041        // exactly the state a real daemon would be in on the next poll once the
5042        // cooldown had elapsed. From HalfOpen the engine credits successes.
5043        let config = CircuitBreakerConfig {
5044            failure_threshold: 3,
5045            success_threshold: 2,
5046            open_cooldown_secs: 0,
5047            ..Default::default()
5048        };
5049        let mut stats = CircuitStats::new();
5050        for _ in 0..3 {
5051            stats.apply_health_outcome(false, false, &config);
5052        }
5053        // With a 0s cooldown the engine nudges Open -> HalfOpen immediately.
5054        assert_eq!(stats.state(), CircuitState::HalfOpen);
5055
5056        // First healthy probe in half-open: one success credited, not yet closed.
5057        let after_first = stats.apply_health_outcome(true, false, &config);
5058        assert_eq!(after_first, CircuitState::HalfOpen);
5059
5060        // Second healthy probe: threshold met -> Closed.
5061        let after_second = stats.apply_health_outcome(true, false, &config);
5062        assert_eq!(after_second, CircuitState::Closed);
5063    }
5064
5065    #[test]
5066    fn test_apply_health_outcome_transient_half_open_does_not_reopen() {
5067        let _guard = test_guard!();
5068        // A transient failure while half-open must NOT reset the recovery streak
5069        // — a network blip can't undo an in-progress recovery.
5070        let config = CircuitBreakerConfig {
5071            failure_threshold: 3,
5072            success_threshold: 2,
5073            open_cooldown_secs: 0,
5074            ..Default::default()
5075        };
5076        let mut stats = CircuitStats::new();
5077        for _ in 0..3 {
5078            stats.apply_health_outcome(false, false, &config);
5079        }
5080        // 0s cooldown -> already nudged to HalfOpen.
5081        assert_eq!(stats.state(), CircuitState::HalfOpen);
5082
5083        // One clean probe (1 of 2 needed).
5084        assert_eq!(
5085            stats.apply_health_outcome(true, false, &config),
5086            CircuitState::HalfOpen
5087        );
5088
5089        // Transient failure in half-open: stays HalfOpen (does NOT reopen the
5090        // circuit or restart the cooldown). The key contrast with a real failure
5091        // is that the circuit does not drop back to Open — probing continues in
5092        // place, so recovery resumes immediately on the next clean probes rather
5093        // than waiting out a fresh cooldown.
5094        let after_transient = stats.apply_health_outcome(false, true, &config);
5095        assert_eq!(after_transient, CircuitState::HalfOpen);
5096
5097        // Recovery still completes from half-open with the required clean probes.
5098        assert_eq!(
5099            stats.apply_health_outcome(true, false, &config),
5100            CircuitState::HalfOpen
5101        );
5102        assert_eq!(
5103            stats.apply_health_outcome(true, false, &config),
5104            CircuitState::Closed
5105        );
5106    }
5107
5108    #[test]
5109    fn test_apply_health_outcome_real_vs_transient_half_open_failure() {
5110        let _guard = test_guard!();
5111        // With a positive cooldown the difference is directly observable: a REAL
5112        // half-open failure reopens the circuit (state -> Open, must wait out the
5113        // cooldown again), whereas a TRANSIENT half-open failure keeps it in
5114        // HalfOpen so recovery continues without a fresh cooldown.
5115        //
5116        // Manually drive to HalfOpen (open() + half_open()) so the positive
5117        // cooldown doesn't force a real wallclock wait to reach half-open.
5118        let config = CircuitBreakerConfig {
5119            failure_threshold: 3,
5120            success_threshold: 2,
5121            open_cooldown_secs: 300,
5122            ..Default::default()
5123        };
5124
5125        // Real failure path.
5126        let mut real = CircuitStats::new();
5127        real.open();
5128        real.half_open();
5129        assert_eq!(real.state(), CircuitState::HalfOpen);
5130        let after_real = real.apply_health_outcome(false, false, &config);
5131        assert_eq!(
5132            after_real,
5133            CircuitState::Open,
5134            "a real half-open failure must reopen the circuit"
5135        );
5136
5137        // Transient failure path (same starting state).
5138        let mut transient = CircuitStats::new();
5139        transient.open();
5140        transient.half_open();
5141        assert_eq!(transient.state(), CircuitState::HalfOpen);
5142        let after_transient = transient.apply_health_outcome(false, true, &config);
5143        assert_eq!(
5144            after_transient,
5145            CircuitState::HalfOpen,
5146            "a transient half-open failure must NOT reopen the circuit"
5147        );
5148    }
5149
5150    /// Issue #48 regression: a worker whose liveness probe (`echo` over a warm
5151    /// session) keeps succeeding while every real command times out must NOT
5152    /// be re-admitted. Liveness successes alone cannot close a half-open
5153    /// circuit while command-class evidence is outstanding.
5154    #[test]
5155    fn test_command_failure_blocks_half_open_close_on_liveness_successes() {
5156        let _guard = test_guard!();
5157        let config = CircuitBreakerConfig {
5158            failure_threshold: 3,
5159            success_threshold: 2,
5160            open_cooldown_secs: 0,
5161            command_failure_threshold: 2,
5162            ..Default::default()
5163        };
5164        let mut stats = CircuitStats::new();
5165        stats.open();
5166        stats.half_open();
5167
5168        // The half-open trial: a real command hangs on the worker.
5169        assert_eq!(
5170            stats.apply_command_outcome(false, &config),
5171            CircuitState::HalfOpen,
5172            "cooldown 0: reopen then immediately half-open again"
5173        );
5174        assert_eq!(stats.consecutive_command_failures(), 1);
5175
5176        // Liveness keeps answering — many times over.
5177        for _ in 0..10 {
5178            assert_eq!(
5179                stats.apply_health_outcome(true, false, &config),
5180                CircuitState::HalfOpen,
5181                "liveness successes must not close the circuit while a command failure is outstanding"
5182            );
5183        }
5184
5185        // Only a command-class success clears the evidence and recovers.
5186        assert_eq!(
5187            stats.apply_command_outcome(true, &config),
5188            CircuitState::Closed
5189        );
5190        assert_eq!(stats.consecutive_command_failures(), 0);
5191        assert_eq!(stats.reopen_streak(), 0);
5192    }
5193
5194    /// Issue #48: interleaved liveness successes reset `failure_threshold`, so
5195    /// command failures need their own threshold to open a Closed circuit.
5196    #[test]
5197    fn test_command_failures_open_closed_circuit_despite_liveness_successes() {
5198        let _guard = test_guard!();
5199        let config = CircuitBreakerConfig {
5200            failure_threshold: 3,
5201            command_failure_threshold: 2,
5202            open_cooldown_secs: 300,
5203            ..Default::default()
5204        };
5205        let mut stats = CircuitStats::new();
5206
5207        assert_eq!(
5208            stats.apply_command_outcome(false, &config),
5209            CircuitState::Closed
5210        );
5211        // Two liveness successes in between: consecutive_failures is reset...
5212        stats.apply_health_outcome(true, false, &config);
5213        stats.apply_health_outcome(true, false, &config);
5214        assert_eq!(stats.consecutive_failures(), 0);
5215        // ...but the command-failure count is not, and the second one opens.
5216        assert_eq!(
5217            stats.apply_command_outcome(false, &config),
5218            CircuitState::Open
5219        );
5220        assert_eq!(stats.reopen_streak(), 1);
5221    }
5222
5223    /// A command failure in HalfOpen always reopens: "connected but the
5224    /// command never completed" is exactly what the trial exists to detect,
5225    /// so there is no transient exemption on this channel.
5226    #[test]
5227    fn test_command_failure_in_half_open_always_reopens() {
5228        let _guard = test_guard!();
5229        let config = CircuitBreakerConfig {
5230            open_cooldown_secs: 300,
5231            ..Default::default()
5232        };
5233        let mut stats = CircuitStats::new();
5234        stats.open();
5235        stats.half_open();
5236        assert_eq!(
5237            stats.apply_command_outcome(false, &config),
5238            CircuitState::Open
5239        );
5240        assert_eq!(stats.reopen_streak(), 2);
5241    }
5242
5243    /// Repeated Open -> HalfOpen -> Open cycles lengthen the cooldown
5244    /// (exponential, capped) so a flapping worker converges to Open instead of
5245    /// being re-admitted on every base cooldown.
5246    #[test]
5247    fn test_reopen_streak_backs_off_cooldown_until_cap() {
5248        let _guard = test_guard!();
5249        let config = CircuitBreakerConfig {
5250            open_cooldown_secs: 30,
5251            open_cooldown_max_secs: 100,
5252            ..Default::default()
5253        };
5254        assert_eq!(config.cooldown_ms_for_streak(0), 30_000);
5255        assert_eq!(config.cooldown_ms_for_streak(1), 30_000);
5256        assert_eq!(config.cooldown_ms_for_streak(2), 60_000);
5257        assert_eq!(config.cooldown_ms_for_streak(3), 100_000, "capped");
5258        assert_eq!(config.cooldown_ms_for_streak(40), 100_000, "no overflow");
5259
5260        // A cap below the base never shortens the base cooldown.
5261        let inverted = CircuitBreakerConfig {
5262            open_cooldown_secs: 30,
5263            open_cooldown_max_secs: 5,
5264            ..Default::default()
5265        };
5266        assert_eq!(inverted.cooldown_ms_for_streak(3), 30_000);
5267
5268        // The streak is observable through recovery_remaining_secs: after two
5269        // opens the remaining cooldown exceeds the base.
5270        let mut stats = CircuitStats::new();
5271        stats.open();
5272        stats.half_open();
5273        stats.open();
5274        assert_eq!(stats.reopen_streak(), 2);
5275        let remaining = stats.recovery_remaining_secs(&config).unwrap();
5276        assert!(
5277            remaining > 30 && remaining <= 60,
5278            "second open should wait ~60s, got {remaining}"
5279        );
5280
5281        // Closing resets the streak, so the next open starts from the base.
5282        stats.close();
5283        assert_eq!(stats.reopen_streak(), 0);
5284        stats.open();
5285        assert!(stats.recovery_remaining_secs(&config).unwrap() <= 30);
5286    }
5287
5288    #[test]
5289    fn test_circuit_stats_success_resets_consecutive_failures() {
5290        let _guard = test_guard!();
5291        let mut stats = CircuitStats::new();
5292        stats.record_failure();
5293        stats.record_failure();
5294        assert_eq!(stats.consecutive_failures(), 2);
5295
5296        stats.record_success();
5297        assert_eq!(stats.consecutive_failures(), 0);
5298    }
5299
5300    #[test]
5301    fn test_circuit_stats_open_transition() {
5302        let _guard = test_guard!();
5303        let mut stats = CircuitStats::new();
5304        stats.open();
5305
5306        assert_eq!(stats.state(), CircuitState::Open);
5307        assert!(stats.opened_at().is_some());
5308    }
5309
5310    #[test]
5311    fn test_circuit_stats_half_open_transition() {
5312        let _guard = test_guard!();
5313        let mut stats = CircuitStats::new();
5314        stats.open();
5315        stats.half_open();
5316
5317        assert_eq!(stats.state(), CircuitState::HalfOpen);
5318    }
5319
5320    #[test]
5321    fn test_circuit_stats_close_transition() {
5322        let _guard = test_guard!();
5323        let mut stats = CircuitStats::new();
5324        stats.open();
5325        stats.half_open();
5326        stats.close();
5327
5328        assert_eq!(stats.state(), CircuitState::Closed);
5329        assert!(stats.opened_at().is_none());
5330        assert_eq!(stats.consecutive_failures(), 0);
5331    }
5332
5333    #[test]
5334    fn test_circuit_stats_should_close() {
5335        let _guard = test_guard!();
5336        let mut stats = CircuitStats::new();
5337        let config = CircuitBreakerConfig {
5338            success_threshold: 2,
5339            ..Default::default()
5340        };
5341
5342        stats.open();
5343        stats.half_open();
5344
5345        assert!(!stats.should_close(&config)); // 0 successes
5346
5347        stats.record_success();
5348        assert!(!stats.should_close(&config)); // 1 success
5349
5350        stats.record_success();
5351        assert!(stats.should_close(&config)); // 2 successes
5352    }
5353
5354    #[test]
5355    fn test_circuit_stats_can_probe() {
5356        let _guard = test_guard!();
5357        let mut stats = CircuitStats::new();
5358        let config = CircuitBreakerConfig {
5359            half_open_max_probes: 1,
5360            ..Default::default()
5361        };
5362
5363        // Can't probe when closed
5364        assert!(!stats.can_probe(&config));
5365
5366        stats.open();
5367        // Can't probe when open
5368        assert!(!stats.can_probe(&config));
5369
5370        stats.half_open();
5371        // Can probe when half-open
5372        assert!(stats.can_probe(&config));
5373
5374        // Start a probe
5375        assert!(stats.start_probe(&config));
5376        // Can't start another probe
5377        assert!(!stats.can_probe(&config));
5378        assert!(!stats.start_probe(&config));
5379    }
5380
5381    #[test]
5382    fn test_circuit_stats_probe_completion() {
5383        let _guard = test_guard!();
5384        let mut stats = CircuitStats::new();
5385        let config = CircuitBreakerConfig {
5386            half_open_max_probes: 1,
5387            ..Default::default()
5388        };
5389
5390        stats.open();
5391        stats.half_open();
5392        stats.start_probe(&config);
5393
5394        // Probe completes with success
5395        stats.record_success();
5396
5397        // Can start another probe
5398        assert!(stats.can_probe(&config));
5399    }
5400
5401    #[test]
5402    fn test_circuit_stats_failure_in_half_open() {
5403        let _guard = test_guard!();
5404        let mut stats = CircuitStats::new();
5405        let config = CircuitBreakerConfig {
5406            success_threshold: 2,
5407            ..Default::default()
5408        };
5409
5410        stats.open();
5411        stats.half_open();
5412
5413        stats.record_success();
5414        assert_eq!(stats.consecutive_failures(), 0);
5415
5416        // Failure resets consecutive successes
5417        stats.record_failure();
5418        assert!(!stats.should_close(&config));
5419
5420        // Need 2 more successes
5421        stats.record_success();
5422        stats.record_success();
5423        assert!(stats.should_close(&config));
5424    }
5425
5426    #[test]
5427    fn test_circuit_stats_reset_window() {
5428        let _guard = test_guard!();
5429        let mut stats = CircuitStats::new();
5430        stats.record_success();
5431        stats.record_failure();
5432
5433        assert!(stats.error_rate() > 0.0);
5434
5435        stats.reset_window();
5436        assert_eq!(stats.error_rate(), 0.0);
5437    }
5438
5439    #[test]
5440    fn test_circuit_state_transitions_are_deterministic() {
5441        let _guard = test_guard!();
5442        let config = CircuitBreakerConfig::default();
5443        let mut stats = CircuitStats::new();
5444
5445        // Start closed
5446        assert_eq!(stats.state(), CircuitState::Closed);
5447
5448        // Cause failures to open
5449        for _ in 0..3 {
5450            stats.record_failure();
5451        }
5452        assert!(stats.should_open(&config));
5453        stats.open();
5454        assert_eq!(stats.state(), CircuitState::Open);
5455
5456        // Transition to half-open after cooldown would be checked
5457        stats.half_open();
5458        assert_eq!(stats.state(), CircuitState::HalfOpen);
5459
5460        // Successes to close
5461        for _ in 0..2 {
5462            stats.record_success();
5463        }
5464        assert!(stats.should_close(&config));
5465        stats.close();
5466        assert_eq!(stats.state(), CircuitState::Closed);
5467    }
5468
5469    // ========================================================================
5470    // Compilation Timing Tests
5471    // ========================================================================
5472
5473    fn make_test_metrics(
5474        speedup: Option<f64>,
5475        total_secs: u64,
5476        success: bool,
5477    ) -> CompilationMetrics {
5478        CompilationMetrics {
5479            project_id: "test-project".to_string(),
5480            worker_id: "worker-1".to_string(),
5481            timestamp: chrono::Utc::now(),
5482            timing: CompilationTimingBreakdown {
5483                rsync_up: Duration::from_millis(100),
5484                remote_build: Duration::from_millis(800),
5485                rsync_down: Duration::from_millis(100),
5486                total: Duration::from_secs(total_secs),
5487            },
5488            local_build_time: speedup.map(|s| Duration::from_secs_f64(total_secs as f64 * s)),
5489            speedup,
5490            files_synced: 100,
5491            bytes_transferred: 1_000_000,
5492            exit_code: if success { 0 } else { 1 },
5493            success,
5494        }
5495    }
5496
5497    #[test]
5498    fn test_compilation_timing_breakdown_default() {
5499        let _guard = test_guard!();
5500        let timing = CompilationTimingBreakdown::default();
5501        assert_eq!(timing.rsync_up, Duration::ZERO);
5502        assert_eq!(timing.remote_build, Duration::ZERO);
5503        assert_eq!(timing.rsync_down, Duration::ZERO);
5504        assert_eq!(timing.total, Duration::ZERO);
5505    }
5506
5507    #[test]
5508    fn test_compilation_timing_breakdown_serialization() {
5509        let _guard = test_guard!();
5510        let timing = CompilationTimingBreakdown {
5511            rsync_up: Duration::from_millis(100),
5512            remote_build: Duration::from_millis(2000),
5513            rsync_down: Duration::from_millis(50),
5514            total: Duration::from_millis(2150),
5515        };
5516
5517        let json = serde_json::to_string(&timing).unwrap();
5518        assert!(json.contains("100")); // rsync_up
5519        assert!(json.contains("2000")); // remote_build
5520        assert!(json.contains("50")); // rsync_down
5521        assert!(json.contains("2150")); // total
5522
5523        let parsed: CompilationTimingBreakdown = serde_json::from_str(&json).unwrap();
5524        assert_eq!(parsed.rsync_up, timing.rsync_up);
5525        assert_eq!(parsed.remote_build, timing.remote_build);
5526        assert_eq!(parsed.rsync_down, timing.rsync_down);
5527        assert_eq!(parsed.total, timing.total);
5528    }
5529
5530    #[test]
5531    fn test_command_timing_breakdown_serialization() {
5532        let _guard = test_guard!();
5533        let timing = CommandTimingBreakdown {
5534            classify: Some(Duration::from_millis(2)),
5535            select: None,
5536            sync_up: Some(Duration::from_millis(120)),
5537            exec: Some(Duration::from_millis(900)),
5538            sync_down: None,
5539            cleanup: Some(Duration::from_millis(5)),
5540            total: Some(Duration::from_millis(1027)),
5541        };
5542
5543        let json = serde_json::to_string(&timing).unwrap();
5544        assert!(json.contains("\"classify\":2"));
5545        assert!(json.contains("\"select\":null"));
5546        assert!(json.contains("\"sync_up\":120"));
5547        assert!(json.contains("\"exec\":900"));
5548        assert!(json.contains("\"sync_down\":null"));
5549        assert!(json.contains("\"cleanup\":5"));
5550        assert!(json.contains("\"total\":1027"));
5551
5552        let parsed: CommandTimingBreakdown = serde_json::from_str(&json).unwrap();
5553        assert_eq!(parsed.classify, timing.classify);
5554        assert_eq!(parsed.select, timing.select);
5555        assert_eq!(parsed.sync_up, timing.sync_up);
5556        assert_eq!(parsed.exec, timing.exec);
5557        assert_eq!(parsed.sync_down, timing.sync_down);
5558        assert_eq!(parsed.cleanup, timing.cleanup);
5559        assert_eq!(parsed.total, timing.total);
5560    }
5561
5562    #[test]
5563    fn test_compilation_metrics_calculate_speedup() {
5564        let _guard = test_guard!();
5565        let mut metrics = CompilationMetrics {
5566            timing: CompilationTimingBreakdown {
5567                total: Duration::from_secs(10),
5568                ..Default::default()
5569            },
5570            local_build_time: Some(Duration::from_secs(30)),
5571            ..Default::default()
5572        };
5573
5574        metrics.calculate_speedup();
5575
5576        assert!(metrics.speedup.is_some());
5577        let speedup = metrics.speedup.unwrap();
5578        assert!(
5579            (speedup - 3.0).abs() < 0.01,
5580            "Expected 3.0x speedup, got {}",
5581            speedup
5582        );
5583    }
5584
5585    #[test]
5586    fn test_compilation_metrics_is_beneficial() {
5587        let _guard = test_guard!();
5588        let mut metrics = CompilationMetrics::default();
5589
5590        // No speedup calculated
5591        assert!(!metrics.is_beneficial());
5592
5593        // Speedup > 1 is beneficial
5594        metrics.speedup = Some(1.5);
5595        assert!(metrics.is_beneficial());
5596
5597        // Speedup < 1 is not beneficial
5598        metrics.speedup = Some(0.8);
5599        assert!(!metrics.is_beneficial());
5600
5601        // Speedup = 1 is not beneficial (exactly same speed)
5602        metrics.speedup = Some(1.0);
5603        assert!(!metrics.is_beneficial());
5604    }
5605
5606    #[test]
5607    fn test_compilation_metrics_serialization() {
5608        let _guard = test_guard!();
5609        let metrics = make_test_metrics(Some(2.5), 10, true);
5610        let json = serde_json::to_string(&metrics).unwrap();
5611
5612        assert!(json.contains("test-project"));
5613        assert!(json.contains("worker-1"));
5614        assert!(json.contains("2.5")); // speedup
5615
5616        let parsed: CompilationMetrics = serde_json::from_str(&json).unwrap();
5617        assert_eq!(parsed.project_id, "test-project");
5618        assert_eq!(parsed.worker_id, "worker-1");
5619        assert!(parsed.success);
5620    }
5621
5622    #[test]
5623    fn test_compilation_timer_new() {
5624        let _guard = test_guard!();
5625        let timer = CompilationTimer::new("my-project", "my-worker");
5626        assert_eq!(timer.project_id, "my-project");
5627        assert_eq!(timer.worker_id, "my-worker");
5628        assert!(timer.rsync_up.is_none());
5629        assert!(timer.remote_build.is_none());
5630        assert!(timer.rsync_down.is_none());
5631    }
5632
5633    #[test]
5634    fn test_compilation_timer_phases() {
5635        let _guard = test_guard!();
5636        let mut timer = CompilationTimer::new("test", "worker");
5637
5638        // Simulate rsync up
5639        std::thread::sleep(Duration::from_millis(5));
5640        timer.end_rsync_up();
5641        assert!(timer.rsync_up.is_some());
5642        assert!(timer.rsync_up.unwrap() >= Duration::from_millis(5));
5643
5644        // Simulate remote build
5645        std::thread::sleep(Duration::from_millis(10));
5646        timer.end_remote_build();
5647        assert!(timer.remote_build.is_some());
5648        assert!(timer.remote_build.unwrap() >= Duration::from_millis(10));
5649
5650        // Simulate rsync down
5651        std::thread::sleep(Duration::from_millis(5));
5652        timer.end_rsync_down();
5653        assert!(timer.rsync_down.is_some());
5654        assert!(timer.rsync_down.unwrap() >= Duration::from_millis(5));
5655
5656        // Finish
5657        let metrics = timer.finish(0, 50, 500_000);
5658        assert_eq!(metrics.project_id, "test");
5659        assert_eq!(metrics.worker_id, "worker");
5660        assert!(metrics.success);
5661        assert!(metrics.timing.total >= Duration::from_millis(20));
5662        assert_eq!(metrics.files_synced, 50);
5663        assert_eq!(metrics.bytes_transferred, 500_000);
5664    }
5665
5666    #[test]
5667    fn test_metrics_aggregator_new() {
5668        let _guard = test_guard!();
5669        let agg = MetricsAggregator::new(100);
5670        assert_eq!(agg.count(), 0);
5671        assert!(agg.average_speedup().is_none());
5672    }
5673
5674    #[test]
5675    fn test_metrics_aggregator_record() {
5676        let _guard = test_guard!();
5677        let mut agg = MetricsAggregator::new(3);
5678
5679        // Record first metric
5680        agg.record(make_test_metrics(Some(2.0), 10, true));
5681        assert_eq!(agg.count(), 1);
5682
5683        // Record up to max
5684        agg.record(make_test_metrics(Some(3.0), 20, true));
5685        agg.record(make_test_metrics(Some(4.0), 30, true));
5686        assert_eq!(agg.count(), 3);
5687
5688        // Exceed max should drop oldest
5689        agg.record(make_test_metrics(Some(5.0), 40, true));
5690        assert_eq!(agg.count(), 3);
5691    }
5692
5693    #[test]
5694    fn test_metrics_aggregator_average_speedup() {
5695        let _guard = test_guard!();
5696        let mut agg = MetricsAggregator::new(100);
5697
5698        // Empty case
5699        assert!(agg.average_speedup().is_none());
5700
5701        // Add metrics with speedups 1, 2, 3, 4, 5 -> average = 3
5702        for i in 1..=5 {
5703            agg.record(make_test_metrics(Some(i as f64), 10, true));
5704        }
5705
5706        let avg = agg.average_speedup().unwrap();
5707        assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5708    }
5709
5710    #[test]
5711    fn test_metrics_aggregator_average_speedup_with_none() {
5712        let _guard = test_guard!();
5713        let mut agg = MetricsAggregator::new(100);
5714
5715        // Mix of Some and None speedups
5716        agg.record(make_test_metrics(Some(2.0), 10, true));
5717        agg.record(make_test_metrics(None, 10, true)); // No speedup
5718        agg.record(make_test_metrics(Some(4.0), 10, true));
5719
5720        // Average should only consider metrics with speedup
5721        let avg = agg.average_speedup().unwrap();
5722        assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5723    }
5724
5725    #[test]
5726    fn test_metrics_aggregator_percentiles() {
5727        let _guard = test_guard!();
5728        let mut agg = MetricsAggregator::new(100);
5729
5730        // Add metrics with total times 1s, 2s, 3s, ..., 10s
5731        for i in 1..=10 {
5732            agg.record(make_test_metrics(Some(1.0), i, true));
5733        }
5734
5735        // p50 (median) should be around 5s
5736        let p50 = agg.p50_total_time().unwrap();
5737        assert!(p50 >= Duration::from_secs(5) && p50 <= Duration::from_secs(6));
5738
5739        // p95 should be around 9s or 10s
5740        let p95 = agg.p95_total_time().unwrap();
5741        assert!(p95 >= Duration::from_secs(9) && p95 <= Duration::from_secs(10));
5742
5743        // p99 should be 10s
5744        let p99 = agg.p99_total_time().unwrap();
5745        assert!(p99 >= Duration::from_secs(9));
5746    }
5747
5748    #[test]
5749    fn test_metrics_aggregator_success_rate() {
5750        let _guard = test_guard!();
5751        let mut agg = MetricsAggregator::new(100);
5752
5753        // Empty case
5754        assert_eq!(agg.success_rate(), 100.0);
5755
5756        // All successful
5757        agg.record(make_test_metrics(Some(1.0), 10, true));
5758        agg.record(make_test_metrics(Some(1.0), 10, true));
5759        assert_eq!(agg.success_rate(), 100.0);
5760
5761        // Add a failure (2 success, 1 failure = 66.67%)
5762        agg.record(make_test_metrics(Some(1.0), 10, false));
5763        let rate = agg.success_rate();
5764        assert!((rate - 66.67).abs() < 1.0, "Expected ~66.67%, got {}", rate);
5765    }
5766
5767    #[test]
5768    fn test_metrics_aggregator_clear() {
5769        let _guard = test_guard!();
5770        let mut agg = MetricsAggregator::new(100);
5771        agg.record(make_test_metrics(Some(1.0), 10, true));
5772        agg.record(make_test_metrics(Some(2.0), 20, true));
5773
5774        assert_eq!(agg.count(), 2);
5775
5776        agg.clear();
5777        assert_eq!(agg.count(), 0);
5778        assert!(agg.average_speedup().is_none());
5779    }
5780
5781    // CompilationConfig timeout tests
5782
5783    #[test]
5784    fn test_remote_build_jobs_parse_display_and_serde() {
5785        let _guard = test_guard!();
5786        assert_eq!(RemoteBuildJobs::default(), RemoteBuildJobs::Auto);
5787        for (spelling, expected) in [
5788            ("auto", RemoteBuildJobs::Auto),
5789            (" AUTO ", RemoteBuildJobs::Auto),
5790            ("off", RemoteBuildJobs::Off),
5791            ("none", RemoteBuildJobs::Off),
5792            ("4", RemoteBuildJobs::Fixed(4)),
5793        ] {
5794            assert_eq!(
5795                RemoteBuildJobs::parse(spelling).unwrap(),
5796                expected,
5797                "{spelling}"
5798            );
5799        }
5800        assert!(RemoteBuildJobs::parse("0").is_err(), "cargo rejects -j 0");
5801        assert!(RemoteBuildJobs::parse("many").is_err());
5802        assert_eq!(RemoteBuildJobs::Fixed(4).to_string(), "4");
5803        assert_eq!(RemoteBuildJobs::Off.to_string(), "off");
5804
5805        // TOML accepts both the string spellings and a bare integer, and an
5806        // absent key keeps the default.
5807        #[derive(Deserialize)]
5808        struct Wrap {
5809            compilation: CompilationConfig,
5810        }
5811        let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = \"off\"\n").unwrap();
5812        assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Off);
5813        let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = 6\n").unwrap();
5814        assert_eq!(
5815            parsed.compilation.remote_build_jobs,
5816            RemoteBuildJobs::Fixed(6)
5817        );
5818        let parsed: Wrap = toml::from_str("[compilation]\nbuild_slots = 4\n").unwrap();
5819        assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Auto);
5820        assert!(toml::from_str::<Wrap>("[compilation]\nremote_build_jobs = 0\n").is_err());
5821
5822        // Serialization round-trips through the same spellings.
5823        let json = serde_json::to_string(&RemoteBuildJobs::Fixed(3)).unwrap();
5824        assert_eq!(json, "3");
5825        assert_eq!(
5826            serde_json::to_string(&RemoteBuildJobs::Auto).unwrap(),
5827            "\"auto\""
5828        );
5829        let back: RemoteBuildJobs = serde_json::from_str("\"auto\"").unwrap();
5830        assert_eq!(back, RemoteBuildJobs::Auto);
5831    }
5832
5833    #[test]
5834    fn test_compilation_config_default_timeouts() {
5835        let _guard = test_guard!();
5836        let config = CompilationConfig::default();
5837        // Default build timeout: 5 minutes
5838        assert_eq!(config.build_timeout_sec, 300);
5839        // Default test timeout: 30 minutes
5840        assert_eq!(config.test_timeout_sec, 1800);
5841    }
5842
5843    #[test]
5844    fn test_compilation_config_timeout_for_test_kinds() {
5845        let _guard = test_guard!();
5846        let config = CompilationConfig::default();
5847
5848        // Test commands should get test_timeout_sec
5849        assert_eq!(
5850            config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5851            std::time::Duration::from_secs(1800)
5852        );
5853        assert_eq!(
5854            config.timeout_for_kind(Some(crate::CompilationKind::CargoNextest)),
5855            std::time::Duration::from_secs(1800)
5856        );
5857        assert_eq!(
5858            config.timeout_for_kind(Some(crate::CompilationKind::BunTest)),
5859            std::time::Duration::from_secs(config.bun_timeout_sec)
5860        );
5861    }
5862
5863    #[test]
5864    fn test_compilation_config_timeout_for_build_kinds() {
5865        let _guard = test_guard!();
5866        let config = CompilationConfig::default();
5867
5868        // Build/check commands should get build_timeout_sec
5869        assert_eq!(
5870            config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5871            std::time::Duration::from_secs(300)
5872        );
5873        assert_eq!(
5874            config.timeout_for_kind(Some(crate::CompilationKind::CargoCheck)),
5875            std::time::Duration::from_secs(300)
5876        );
5877        assert_eq!(
5878            config.timeout_for_kind(Some(crate::CompilationKind::CargoClippy)),
5879            std::time::Duration::from_secs(300)
5880        );
5881        assert_eq!(
5882            config.timeout_for_kind(None),
5883            std::time::Duration::from_secs(300)
5884        );
5885    }
5886
5887    #[test]
5888    fn test_compilation_config_custom_timeouts() {
5889        let _guard = test_guard!();
5890        let config = CompilationConfig {
5891            build_timeout_sec: 600, // 10 minutes
5892            test_timeout_sec: 3600, // 1 hour
5893            ..Default::default()
5894        };
5895
5896        assert_eq!(
5897            config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5898            std::time::Duration::from_secs(600)
5899        );
5900        assert_eq!(
5901            config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5902            std::time::Duration::from_secs(3600)
5903        );
5904    }
5905
5906    #[test]
5907    fn test_compilation_config_speedup_threshold_default() {
5908        let _guard = test_guard!();
5909        let config = CompilationConfig::default();
5910        // Default speedup threshold: 1.2 (20% faster required)
5911        assert!((config.remote_speedup_threshold - 1.2).abs() < 0.001);
5912    }
5913
5914    #[test]
5915    fn test_compilation_config_speedup_threshold_custom() {
5916        let _guard = test_guard!();
5917        let config = CompilationConfig {
5918            remote_speedup_threshold: 1.5, // Require 50% faster
5919            ..Default::default()
5920        };
5921        assert!((config.remote_speedup_threshold - 1.5).abs() < 0.001);
5922    }
5923
5924    #[test]
5925    fn test_compilation_config_speedup_threshold_no_minimum() {
5926        let _guard = test_guard!();
5927        // Setting to 1.0 means "always offload when other criteria met"
5928        let config = CompilationConfig {
5929            remote_speedup_threshold: 1.0,
5930            ..Default::default()
5931        };
5932        assert!((config.remote_speedup_threshold - 1.0).abs() < 0.001);
5933    }
5934
5935    #[test]
5936    fn test_compilation_config_min_local_time_ms_default() {
5937        let _guard = test_guard!();
5938        let config = CompilationConfig::default();
5939        // Default min_local_time_ms: 2000ms
5940        assert_eq!(config.min_local_time_ms, 2000);
5941    }
5942
5943    #[test]
5944    fn test_compilation_config_serde_roundtrip_speedup_threshold() {
5945        let _guard = test_guard!();
5946        let config = CompilationConfig {
5947            remote_speedup_threshold: 2.5,
5948            min_local_time_ms: 5000,
5949            ..Default::default()
5950        };
5951        let json = serde_json::to_string(&config).unwrap();
5952        let parsed: CompilationConfig = serde_json::from_str(&json).unwrap();
5953        assert!((parsed.remote_speedup_threshold - 2.5).abs() < 0.001);
5954        assert_eq!(parsed.min_local_time_ms, 5000);
5955    }
5956
5957    // ========================================================================
5958    // validate_remote_base Tests
5959    // ========================================================================
5960
5961    #[test]
5962    fn test_validate_remote_base_absolute_path() {
5963        let _guard = test_guard!();
5964        assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5965        assert_eq!(
5966            validate_remote_base("/var/rch-builds").unwrap(),
5967            "/var/rch-builds"
5968        );
5969        assert_eq!(
5970            validate_remote_base("/home/builder/.rch").unwrap(),
5971            "/home/builder/.rch"
5972        );
5973    }
5974
5975    #[test]
5976    fn test_validate_remote_base_tilde_expansion() {
5977        let _guard = test_guard!();
5978        // Tilde expansion should work
5979        let result = validate_remote_base("~/rch");
5980        assert!(result.is_ok());
5981        let path = result.unwrap();
5982        assert!(
5983            path.starts_with('/'),
5984            "Path should be absolute after expansion: {}",
5985            path
5986        );
5987        assert!(!path.contains('~'), "Tilde should be expanded: {}", path);
5988    }
5989
5990    #[test]
5991    fn test_validate_remote_base_rejects_relative_path() {
5992        let _guard = test_guard!();
5993        let result = validate_remote_base("tmp/rch");
5994        assert!(result.is_err());
5995        assert!(result.unwrap_err().contains("absolute path"));
5996    }
5997
5998    #[test]
5999    fn test_validate_remote_base_rejects_path_traversal() {
6000        let _guard = test_guard!();
6001        let result = validate_remote_base("/tmp/../etc/rch");
6002        assert!(result.is_err());
6003        assert!(result.unwrap_err().contains("path traversal"));
6004
6005        let result = validate_remote_base("/tmp/rch/../other");
6006        assert!(result.is_err());
6007        assert!(result.unwrap_err().contains("path traversal"));
6008    }
6009
6010    #[test]
6011    fn test_validate_remote_base_normalizes_trailing_slash() {
6012        let _guard = test_guard!();
6013        assert_eq!(validate_remote_base("/tmp/rch/").unwrap(), "/tmp/rch");
6014        assert_eq!(validate_remote_base("/tmp/rch///").unwrap(), "/tmp/rch");
6015    }
6016
6017    #[test]
6018    fn test_validate_remote_base_root_path() {
6019        let _guard = test_guard!();
6020        // Root path should be rejected
6021        let result = validate_remote_base("/");
6022        assert!(result.is_err());
6023        assert!(result.unwrap_err().contains("root directory"));
6024    }
6025
6026    #[test]
6027    fn test_validate_remote_base_rejects_top_level() {
6028        let _guard = test_guard!();
6029        // Top-level directories should be rejected (depth 1)
6030        let result = validate_remote_base("/tmp");
6031        assert!(result.is_err());
6032        assert!(result.unwrap_err().contains("at least 2 levels deep"));
6033
6034        let result = validate_remote_base("/home");
6035        assert!(result.is_err());
6036
6037        // Depth 2 should be allowed
6038        assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
6039    }
6040
6041    #[test]
6042    fn test_default_remote_base() {
6043        let _guard = test_guard!();
6044        assert_eq!(default_remote_base(), "/data/tmp/rch");
6045    }
6046
6047    #[test]
6048    fn test_transfer_config_default_has_remote_base() {
6049        let _guard = test_guard!();
6050        let config = TransferConfig::default();
6051        assert_eq!(config.remote_base, "/data/tmp/rch");
6052    }
6053
6054    #[test]
6055    fn test_source_sync_timeout_validation_boundaries() {
6056        let _guard = test_guard!();
6057        assert!(!TransferConfig::valid_sync_timeout_ms(0));
6058        assert!(!TransferConfig::valid_sync_timeout_ms(999));
6059        assert!(TransferConfig::valid_sync_timeout_ms(1_000));
6060        assert!(TransferConfig::valid_sync_timeout_ms(3_600_000));
6061        assert!(!TransferConfig::valid_sync_timeout_ms(3_600_001));
6062    }
6063
6064    #[test]
6065    fn test_source_sync_timeout_is_explicit_or_payload_aware() {
6066        let _guard = test_guard!();
6067        let default = TransferConfig::default();
6068        assert_eq!(
6069            default.sync_timeout_for_payload(0),
6070            std::time::Duration::from_millis(30_000)
6071        );
6072        assert_eq!(
6073            default.sync_timeout_for_payload(10 * 1024 * 1024),
6074            std::time::Duration::from_millis(40_000)
6075        );
6076        assert_eq!(
6077            default.sync_timeout_for_payload(u64::MAX),
6078            std::time::Duration::from_millis(3_600_000)
6079        );
6080
6081        let explicit = TransferConfig {
6082            sync_timeout_ms: Some(120_000),
6083            ..TransferConfig::default()
6084        };
6085        assert_eq!(
6086            explicit.sync_timeout_for_payload(u64::MAX),
6087            std::time::Duration::from_millis(120_000)
6088        );
6089    }
6090
6091    // ========================================================================
6092    // RetryConfig Tests (bd-x1ek)
6093    // ========================================================================
6094
6095    #[test]
6096    fn test_retry_config_default() {
6097        let _guard = test_guard!();
6098        let config = RetryConfig::default();
6099        assert_eq!(config.max_attempts, 3);
6100        assert_eq!(config.base_delay_ms, 100);
6101        assert_eq!(config.max_delay_ms, 5000);
6102        assert_eq!(config.jitter_factor, 0.1);
6103        assert_eq!(config.total_timeout_ms, 30000);
6104    }
6105
6106    #[test]
6107    fn test_retry_config_no_retry() {
6108        let _guard = test_guard!();
6109        let config = RetryConfig::no_retry();
6110        assert_eq!(config.max_attempts, 1);
6111    }
6112
6113    #[test]
6114    fn test_retry_config_delay_for_attempt_zero() {
6115        let _guard = test_guard!();
6116        let config = RetryConfig::default();
6117        assert_eq!(config.delay_for_attempt(0), std::time::Duration::ZERO);
6118    }
6119
6120    #[test]
6121    fn test_retry_config_delay_for_attempt_exponential() {
6122        let _guard = test_guard!();
6123        let config = RetryConfig {
6124            base_delay_ms: 100,
6125            max_delay_ms: 10000,
6126            jitter_factor: 0.0, // No jitter for deterministic test
6127            ..Default::default()
6128        };
6129
6130        // attempt 1: 100 * 2^0 = 100ms
6131        assert_eq!(
6132            config.delay_for_attempt(1),
6133            std::time::Duration::from_millis(100)
6134        );
6135        // attempt 2: 100 * 2^1 = 200ms
6136        assert_eq!(
6137            config.delay_for_attempt(2),
6138            std::time::Duration::from_millis(200)
6139        );
6140        // attempt 3: 100 * 2^2 = 400ms
6141        assert_eq!(
6142            config.delay_for_attempt(3),
6143            std::time::Duration::from_millis(400)
6144        );
6145    }
6146
6147    #[test]
6148    fn test_retry_config_delay_capped_at_max() {
6149        let _guard = test_guard!();
6150        let config = RetryConfig {
6151            base_delay_ms: 1000,
6152            max_delay_ms: 2000,
6153            jitter_factor: 0.0,
6154            ..Default::default()
6155        };
6156
6157        // attempt 2: 1000 * 2^1 = 2000ms (at cap)
6158        assert_eq!(
6159            config.delay_for_attempt(2),
6160            std::time::Duration::from_millis(2000)
6161        );
6162        // attempt 3: 1000 * 2^2 = 4000ms, capped to 2000ms
6163        assert_eq!(
6164            config.delay_for_attempt(3),
6165            std::time::Duration::from_millis(2000)
6166        );
6167    }
6168
6169    #[test]
6170    fn test_retry_config_random_jitter_stays_within_bounds() {
6171        let _guard = test_guard!();
6172        let config = RetryConfig {
6173            base_delay_ms: 100,
6174            max_delay_ms: 200,
6175            jitter_factor: 0.25,
6176            ..Default::default()
6177        };
6178
6179        // Jitter applies after the exponential delay is capped. Exercise the
6180        // real RNG without requiring any particular random value or sequence.
6181        for (attempt, minimum_ms, maximum_ms) in [(1, 75, 125), (3, 150, 250)] {
6182            for _ in 0..128 {
6183                let delay = config.delay_for_attempt(attempt);
6184                assert!(delay >= std::time::Duration::from_millis(minimum_ms));
6185                assert!(delay <= std::time::Duration::from_millis(maximum_ms));
6186            }
6187        }
6188    }
6189
6190    #[test]
6191    fn test_retry_config_should_retry() {
6192        let _guard = test_guard!();
6193        let config = RetryConfig {
6194            max_attempts: 3,
6195            total_timeout_ms: 1000,
6196            ..Default::default()
6197        };
6198
6199        // Before max attempts and within timeout
6200        assert!(config.should_retry(0, std::time::Duration::from_millis(0)));
6201        assert!(config.should_retry(1, std::time::Duration::from_millis(500)));
6202        assert!(config.should_retry(2, std::time::Duration::from_millis(900)));
6203
6204        // At max attempts
6205        assert!(!config.should_retry(3, std::time::Duration::from_millis(0)));
6206
6207        // Past timeout
6208        assert!(!config.should_retry(1, std::time::Duration::from_millis(1001)));
6209    }
6210
6211    #[test]
6212    fn test_retry_config_serde_roundtrip() {
6213        let _guard = test_guard!();
6214        let config = RetryConfig {
6215            max_attempts: 5,
6216            base_delay_ms: 200,
6217            max_delay_ms: 8000,
6218            jitter_factor: 0.2,
6219            total_timeout_ms: 60000,
6220        };
6221        let json = serde_json::to_string(&config).unwrap();
6222        let parsed: RetryConfig = serde_json::from_str(&json).unwrap();
6223        assert_eq!(parsed.max_attempts, 5);
6224        assert_eq!(parsed.base_delay_ms, 200);
6225        assert_eq!(parsed.max_delay_ms, 8000);
6226        assert_eq!(parsed.jitter_factor, 0.2);
6227        assert_eq!(parsed.total_timeout_ms, 60000);
6228    }
6229
6230    #[test]
6231    fn test_transfer_config_includes_retry() {
6232        let _guard = test_guard!();
6233        let config = TransferConfig::default();
6234        assert_eq!(config.retry.max_attempts, 3);
6235    }
6236
6237    // =========================================================================
6238    // ExecutionConfig Tests (bd-785w)
6239    // =========================================================================
6240
6241    #[test]
6242    fn test_execution_config_default_allowlist() {
6243        let _guard = test_guard!();
6244        let config = ExecutionConfig::default();
6245        // All supported compilers should be in the default allowlist
6246        assert!(config.is_allowed("cargo"));
6247        assert!(config.is_allowed("rustc"));
6248        assert!(config.is_allowed("gcc"));
6249        assert!(config.is_allowed("g++"));
6250        assert!(config.is_allowed("clang"));
6251        assert!(config.is_allowed("clang++"));
6252        assert!(config.is_allowed("make"));
6253        assert!(config.is_allowed("cmake"));
6254        assert!(config.is_allowed("ninja"));
6255        assert!(config.is_allowed("meson"));
6256        assert!(config.is_allowed("bun"));
6257        assert!(config.is_allowed("nextest"));
6258        assert!(config.is_allowed("cc"));
6259        assert!(config.is_allowed("c++"));
6260    }
6261
6262    #[test]
6263    fn test_execution_config_is_allowed_case_insensitive() {
6264        let _guard = test_guard!();
6265        let config = ExecutionConfig::default();
6266        // Should be case-insensitive
6267        assert!(config.is_allowed("CARGO"));
6268        assert!(config.is_allowed("Cargo"));
6269        assert!(config.is_allowed("GCC"));
6270        assert!(config.is_allowed("Gcc"));
6271    }
6272
6273    #[test]
6274    fn test_execution_config_is_not_allowed() {
6275        let _guard = test_guard!();
6276        let config = ExecutionConfig::default();
6277        // Unknown commands should not be allowed
6278        assert!(!config.is_allowed("python"));
6279        assert!(!config.is_allowed("npm"));
6280        assert!(!config.is_allowed(""));
6281        // `go` and `tsc` ARE allowed now that Go/TypeScript are offload targets.
6282        // If they were missing from the allowlist the hook would fail open to
6283        // LOCAL execution — classification would look correct while every build
6284        // silently ran on the orchestrator.
6285        assert!(config.is_allowed("go"));
6286        assert!(config.is_allowed("tsc"));
6287    }
6288
6289    #[test]
6290    fn test_execution_config_empty_allowlist() {
6291        let _guard = test_guard!();
6292        let config = ExecutionConfig { allowlist: vec![] };
6293        // Empty allowlist should block everything
6294        assert!(!config.is_allowed("cargo"));
6295        assert!(!config.is_allowed("gcc"));
6296    }
6297
6298    #[test]
6299    fn test_execution_config_custom_allowlist() {
6300        let _guard = test_guard!();
6301        let config = ExecutionConfig {
6302            allowlist: vec!["cargo".to_string(), "custom_tool".to_string()],
6303        };
6304        assert!(config.is_allowed("cargo"));
6305        assert!(config.is_allowed("custom_tool"));
6306        assert!(!config.is_allowed("gcc"));
6307    }
6308
6309    #[test]
6310    fn test_execution_config_serde() {
6311        let _guard = test_guard!();
6312        let config = ExecutionConfig {
6313            allowlist: vec!["cargo".to_string(), "rustc".to_string()],
6314        };
6315        let json = serde_json::to_string(&config).unwrap();
6316        let parsed: ExecutionConfig = serde_json::from_str(&json).unwrap();
6317        assert_eq!(parsed.allowlist, config.allowlist);
6318    }
6319
6320    #[test]
6321    fn test_rch_config_includes_execution() {
6322        let _guard = test_guard!();
6323        let config = RchConfig::default();
6324        // RchConfig should include ExecutionConfig with default allowlist
6325        assert!(config.execution.is_allowed("cargo"));
6326    }
6327
6328    // ========================================================================
6329    // Adaptive Compression Tests (bd-243w)
6330    // ========================================================================
6331
6332    #[test]
6333    fn test_adaptive_compression_disabled_by_default() {
6334        let _guard = test_guard!();
6335        let config = TransferConfig::default();
6336        assert!(!config.adaptive_compression);
6337        // Should use fixed level when disabled
6338        assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6339        assert_eq!(config.select_compression_level(Some(500_000_000)), 3);
6340    }
6341
6342    #[test]
6343    fn test_adaptive_compression_small_payload() {
6344        let _guard = test_guard!();
6345        let config = TransferConfig {
6346            adaptive_compression: true,
6347            ..Default::default()
6348        };
6349        // < 10MB should use level 1
6350        assert_eq!(config.select_compression_level(Some(0)), 1);
6351        assert_eq!(config.select_compression_level(Some(1_000_000)), 1); // 1MB
6352        assert_eq!(config.select_compression_level(Some(9_999_999)), 1); // ~10MB
6353    }
6354
6355    #[test]
6356    fn test_adaptive_compression_medium_payload() {
6357        let _guard = test_guard!();
6358        let config = TransferConfig {
6359            adaptive_compression: true,
6360            ..Default::default()
6361        };
6362        // 10-200MB should use level 3
6363        assert_eq!(config.select_compression_level(Some(10_000_000)), 3); // 10MB
6364        assert_eq!(config.select_compression_level(Some(100_000_000)), 3); // 100MB
6365        assert_eq!(config.select_compression_level(Some(199_999_999)), 3); // ~200MB
6366    }
6367
6368    #[test]
6369    fn test_adaptive_compression_large_payload() {
6370        let _guard = test_guard!();
6371        let config = TransferConfig {
6372            adaptive_compression: true,
6373            ..Default::default()
6374        };
6375        // > 200MB should use level 7
6376        assert_eq!(config.select_compression_level(Some(200_000_000)), 7); // 200MB
6377        assert_eq!(config.select_compression_level(Some(500_000_000)), 7); // 500MB
6378        assert_eq!(config.select_compression_level(Some(1_000_000_000)), 7); // 1GB
6379    }
6380
6381    #[test]
6382    fn test_adaptive_compression_no_estimate() {
6383        let _guard = test_guard!();
6384        let config = TransferConfig {
6385            adaptive_compression: true,
6386            compression_level: 5,
6387            ..Default::default()
6388        };
6389        // No estimate should fallback to default level
6390        assert_eq!(config.select_compression_level(None), 5);
6391    }
6392
6393    #[test]
6394    fn test_adaptive_compression_respects_min_level() {
6395        let _guard = test_guard!();
6396        let config = TransferConfig {
6397            adaptive_compression: true,
6398            min_compression_level: 3,
6399            ..Default::default()
6400        };
6401        // Small payload would want level 1, but min is 3
6402        assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6403    }
6404
6405    #[test]
6406    fn test_adaptive_compression_respects_max_level() {
6407        let _guard = test_guard!();
6408        let config = TransferConfig {
6409            adaptive_compression: true,
6410            max_compression_level: 5,
6411            ..Default::default()
6412        };
6413        // Large payload would want level 7, but max is 5
6414        assert_eq!(config.select_compression_level(Some(500_000_000)), 5);
6415    }
6416
6417    #[test]
6418    fn test_adaptive_compression_serde_roundtrip() {
6419        let _guard = test_guard!();
6420        let config = TransferConfig {
6421            adaptive_compression: true,
6422            min_compression_level: 2,
6423            max_compression_level: 8,
6424            ..Default::default()
6425        };
6426        let json = serde_json::to_string(&config).unwrap();
6427        let parsed: TransferConfig = serde_json::from_str(&json).unwrap();
6428        assert!(parsed.adaptive_compression);
6429        assert_eq!(parsed.min_compression_level, 2);
6430        assert_eq!(parsed.max_compression_level, 8);
6431    }
6432
6433    // -------------------------------------------------------------------------
6434    // Self-Healing Config Tests (bd-59kg)
6435    // -------------------------------------------------------------------------
6436
6437    #[test]
6438    fn test_self_healing_config_defaults() {
6439        let _guard = test_guard!();
6440        // TEST START: SelfHealingConfig::default() returns expected values
6441        let config = SelfHealingConfig::default();
6442
6443        // Both self-healing behaviors enabled by default
6444        assert!(
6445            config.hook_starts_daemon,
6446            "hook_starts_daemon should be true by default"
6447        );
6448        assert!(
6449            config.daemon_installs_hooks,
6450            "daemon_installs_hooks should be true by default"
6451        );
6452
6453        // Timing defaults
6454        assert_eq!(
6455            config.auto_start_cooldown_secs, 30,
6456            "auto_start_cooldown_secs should default to 30"
6457        );
6458        assert_eq!(
6459            config.auto_start_timeout_secs, 3,
6460            "auto_start_timeout_secs should default to 3"
6461        );
6462        // TEST PASS: SelfHealingConfig defaults
6463    }
6464
6465    #[test]
6466    fn test_self_healing_config_serde_full() {
6467        let _guard = test_guard!();
6468        // TEST START: Full SelfHealingConfig serialization/deserialization
6469        let config = SelfHealingConfig {
6470            hook_starts_daemon: false,
6471            daemon_installs_hooks: false,
6472            auto_start_cooldown_secs: 60,
6473            auto_start_timeout_secs: 10,
6474            self_healing_log_level: SelfHealingLogLevel::Debug,
6475        };
6476
6477        let json = serde_json::to_string(&config).unwrap();
6478        assert!(json.contains("\"hook_starts_daemon\":false"));
6479        assert!(json.contains("\"daemon_installs_hooks\":false"));
6480        assert!(json.contains("\"auto_start_cooldown_secs\":60"));
6481        assert!(json.contains("\"auto_start_timeout_secs\":10"));
6482        assert!(json.contains("\"self_healing_log_level\":\"debug\""));
6483
6484        let parsed: SelfHealingConfig = serde_json::from_str(&json).unwrap();
6485        assert!(!parsed.hook_starts_daemon);
6486        assert!(!parsed.daemon_installs_hooks);
6487        assert_eq!(parsed.auto_start_cooldown_secs, 60);
6488        assert_eq!(parsed.auto_start_timeout_secs, 10);
6489        assert_eq!(parsed.self_healing_log_level, SelfHealingLogLevel::Debug);
6490        // TEST PASS: Full SelfHealingConfig serde
6491    }
6492
6493    // br-4zf3p: Tests for the self_healing_log_level field.
6494
6495    #[test]
6496    fn test_self_healing_log_level_default_is_info() {
6497        // TEST START: SelfHealingLogLevel::default() == Info
6498        let level = SelfHealingLogLevel::default();
6499        assert_eq!(level, SelfHealingLogLevel::Info);
6500        // Also verify the SelfHealingConfig default surfaces it.
6501        let config = SelfHealingConfig::default();
6502        assert_eq!(
6503            config.self_healing_log_level,
6504            SelfHealingLogLevel::Info,
6505            "config default should be Info"
6506        );
6507        // TEST PASS
6508    }
6509
6510    #[test]
6511    fn test_self_healing_log_level_serializes_lowercase() {
6512        // TEST START: serde uses lowercase variants
6513        let json = serde_json::to_string(&SelfHealingLogLevel::Warn).unwrap();
6514        assert_eq!(json, "\"warn\"");
6515        let parsed: SelfHealingLogLevel = serde_json::from_str("\"error\"").unwrap();
6516        assert_eq!(parsed, SelfHealingLogLevel::Error);
6517        // TEST PASS
6518    }
6519
6520    #[test]
6521    fn test_self_healing_log_level_from_env_str_accepts_known() {
6522        // TEST START: from_env_str parses each level (case-insensitive) and aliases
6523        assert_eq!(
6524            SelfHealingLogLevel::from_env_str("debug"),
6525            Some(SelfHealingLogLevel::Debug)
6526        );
6527        assert_eq!(
6528            SelfHealingLogLevel::from_env_str("DEBUG"),
6529            Some(SelfHealingLogLevel::Debug)
6530        );
6531        assert_eq!(
6532            SelfHealingLogLevel::from_env_str("info"),
6533            Some(SelfHealingLogLevel::Info)
6534        );
6535        assert_eq!(
6536            SelfHealingLogLevel::from_env_str("warn"),
6537            Some(SelfHealingLogLevel::Warn)
6538        );
6539        assert_eq!(
6540            SelfHealingLogLevel::from_env_str("warning"),
6541            Some(SelfHealingLogLevel::Warn),
6542            "warning should map to Warn"
6543        );
6544        assert_eq!(
6545            SelfHealingLogLevel::from_env_str("error"),
6546            Some(SelfHealingLogLevel::Error)
6547        );
6548        // Whitespace tolerance
6549        assert_eq!(
6550            SelfHealingLogLevel::from_env_str("  debug  "),
6551            Some(SelfHealingLogLevel::Debug)
6552        );
6553        // TEST PASS
6554    }
6555
6556    #[test]
6557    fn test_self_healing_log_level_from_env_str_rejects_unknown() {
6558        // TEST START: unknown values return None (caller falls back to default)
6559        assert!(SelfHealingLogLevel::from_env_str("banana").is_none());
6560        assert!(SelfHealingLogLevel::from_env_str("").is_none());
6561        assert!(SelfHealingLogLevel::from_env_str("trace").is_none());
6562        // TEST PASS
6563    }
6564
6565    #[test]
6566    fn test_self_healing_config_serde_partial_uses_defaults() {
6567        let _guard = test_guard!();
6568        // TEST START: Partial TOML/JSON uses defaults for missing fields
6569        let json = r#"{"hook_starts_daemon": false}"#;
6570        let config: SelfHealingConfig = serde_json::from_str(json).unwrap();
6571
6572        assert!(
6573            !config.hook_starts_daemon,
6574            "Explicit false should be parsed"
6575        );
6576        assert!(
6577            config.daemon_installs_hooks,
6578            "Missing field should use default (true)"
6579        );
6580        assert_eq!(
6581            config.auto_start_cooldown_secs, 30,
6582            "Missing field should use default (30)"
6583        );
6584        assert_eq!(
6585            config.auto_start_timeout_secs, 3,
6586            "Missing field should use default (3)"
6587        );
6588        // TEST PASS: Partial SelfHealingConfig uses defaults
6589    }
6590
6591    #[test]
6592    fn test_self_healing_config_toml_with_alias() {
6593        let _guard = test_guard!();
6594        // TEST START: daemon_start_timeout alias works for auto_start_timeout_secs
6595        let toml_str = r#"
6596            hook_starts_daemon = true
6597            daemon_installs_hooks = false
6598            auto_start_cooldown_secs = 45
6599            daemon_start_timeout = 7
6600        "#;
6601
6602        let config: SelfHealingConfig = toml::from_str(toml_str).unwrap();
6603        assert!(config.hook_starts_daemon);
6604        assert!(!config.daemon_installs_hooks);
6605        assert_eq!(config.auto_start_cooldown_secs, 45);
6606        assert_eq!(
6607            config.auto_start_timeout_secs, 7,
6608            "daemon_start_timeout alias should set auto_start_timeout_secs"
6609        );
6610        // TEST PASS: TOML alias works
6611    }
6612
6613    #[allow(unsafe_code)]
6614    mod self_healing_env_override_tests {
6615        use super::*;
6616        use crate::config::env_test_lock;
6617
6618        fn env_guard() -> std::sync::MutexGuard<'static, ()> {
6619            env_test_lock()
6620        }
6621
6622        fn set_env(key: &str, value: &str) {
6623            // SAFETY: Tests are serialized with env_guard().
6624            unsafe { std::env::set_var(key, value) };
6625        }
6626
6627        fn remove_env(key: &str) {
6628            // SAFETY: Tests are serialized with env_guard().
6629            unsafe { std::env::remove_var(key) };
6630        }
6631
6632        struct EnvVarGuard {
6633            key: &'static str,
6634            old: Option<String>,
6635        }
6636
6637        impl EnvVarGuard {
6638            fn set(key: &'static str, value: &str) -> Self {
6639                let old = std::env::var(key).ok();
6640                set_env(key, value);
6641                Self { key, old }
6642            }
6643        }
6644
6645        impl Drop for EnvVarGuard {
6646            fn drop(&mut self) {
6647                if let Some(old) = &self.old {
6648                    set_env(self.key, old);
6649                } else {
6650                    remove_env(self.key);
6651                }
6652            }
6653        }
6654
6655        #[test]
6656        fn test_self_healing_config_with_env_overrides_master_disable() {
6657            let _guard = test_guard!();
6658            let _guard = env_guard();
6659
6660            let _no_self_healing = EnvVarGuard::set("RCH_NO_SELF_HEALING", "1");
6661            let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "1");
6662            let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "1");
6663            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "99");
6664            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "99");
6665
6666            let config = SelfHealingConfig::default().with_env_overrides();
6667            assert!(
6668                !config.hook_starts_daemon,
6669                "RCH_NO_SELF_HEALING should disable hook auto-start"
6670            );
6671            assert!(
6672                !config.daemon_installs_hooks,
6673                "RCH_NO_SELF_HEALING should disable daemon hook installation"
6674            );
6675
6676            // Master disable returns early; numeric overrides should not apply.
6677            assert_eq!(config.auto_start_cooldown_secs, 30);
6678            assert_eq!(config.auto_start_timeout_secs, 3);
6679        }
6680
6681        #[test]
6682        fn test_self_healing_config_with_env_overrides_toggles_and_numbers() {
6683            let _guard = test_guard!();
6684            let _guard = env_guard();
6685
6686            let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "0");
6687            let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "false");
6688            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "45");
6689            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "7");
6690
6691            let config = SelfHealingConfig::default().with_env_overrides();
6692            assert!(!config.hook_starts_daemon);
6693            assert!(!config.daemon_installs_hooks);
6694            assert_eq!(config.auto_start_cooldown_secs, 45);
6695            assert_eq!(config.auto_start_timeout_secs, 7);
6696        }
6697
6698        #[test]
6699        fn test_self_healing_config_with_env_overrides_invalid_numbers_ignored() {
6700            let _guard = test_guard!();
6701            let _guard = env_guard();
6702
6703            let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "not-a-number");
6704            let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "nope");
6705
6706            let config = SelfHealingConfig::default().with_env_overrides();
6707            assert_eq!(config.auto_start_cooldown_secs, 30);
6708            assert_eq!(config.auto_start_timeout_secs, 3);
6709        }
6710    }
6711
6712    // -------------------------------------------------------------------------
6713    // Preflight Health Guard Tests (bd-3eaa)
6714    // -------------------------------------------------------------------------
6715
6716    #[test]
6717    fn test_load_per_core_calculation() {
6718        let _guard = test_guard!();
6719        let mut caps = WorkerCapabilities::new();
6720        // No metrics -> None
6721        assert!(caps.load_per_core().is_none());
6722
6723        // Only load, no cpus -> None
6724        caps.load_avg_1 = Some(4.0);
6725        assert!(caps.load_per_core().is_none());
6726
6727        // Both available
6728        caps.num_cpus = Some(4);
6729        assert_eq!(caps.load_per_core(), Some(1.0)); // 4.0 / 4 = 1.0
6730
6731        // High load
6732        caps.load_avg_1 = Some(16.0);
6733        assert_eq!(caps.load_per_core(), Some(4.0)); // 16.0 / 4 = 4.0
6734    }
6735
6736    #[test]
6737    fn test_is_high_load() {
6738        let _guard = test_guard!();
6739        let mut caps = WorkerCapabilities::new();
6740        caps.load_avg_1 = Some(8.0);
6741        caps.num_cpus = Some(4);
6742        // load_per_core = 2.0
6743
6744        // Below threshold
6745        assert_eq!(caps.is_high_load(3.0), Some(false));
6746        // At threshold
6747        assert_eq!(caps.is_high_load(2.0), Some(false)); // equal is OK
6748        // Above threshold
6749        assert_eq!(caps.is_high_load(1.5), Some(true));
6750
6751        // No metrics -> None (fail-open)
6752        caps.load_avg_1 = None;
6753        assert!(caps.is_high_load(1.0).is_none());
6754    }
6755
6756    #[test]
6757    fn test_is_low_disk() {
6758        let _guard = test_guard!();
6759        let mut caps = WorkerCapabilities::new();
6760        caps.disk_free_gb = Some(15.0);
6761
6762        // Above threshold
6763        assert_eq!(caps.is_low_disk(10.0), Some(false));
6764        // At threshold
6765        assert_eq!(caps.is_low_disk(15.0), Some(false)); // equal is OK
6766        // Below threshold
6767        assert_eq!(caps.is_low_disk(20.0), Some(true));
6768
6769        // No metrics -> None (fail-open)
6770        caps.disk_free_gb = None;
6771        assert!(caps.is_low_disk(10.0).is_none());
6772    }
6773
6774    #[test]
6775    fn low_disk_judges_the_build_disk_not_a_small_tmpfs() {
6776        let _guard = test_guard!();
6777        // GH #78: a 16 GB tmpfs /tmp is the tightest mount by ratio, but the
6778        // build trees live on a roomy data disk.
6779        let mut caps = WorkerCapabilities::new();
6780        caps.disk_free_gb = Some(8.0);
6781        caps.disk_total_gb = Some(15.5);
6782        caps.build_disk_free_gb = Some(1700.0);
6783        caps.build_disk_total_gb = Some(1900.0);
6784        assert_eq!(caps.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6785        assert_eq!(caps.is_low_disk(10.0), Some(false));
6786
6787        // A full build disk is low even when /tmp is roomy.
6788        caps.build_disk_free_gb = Some(4.0);
6789        assert_eq!(caps.is_low_disk(10.0), Some(true));
6790
6791        // An older rch-wkr without the build sample keeps the old behaviour.
6792        caps.build_disk_free_gb = None;
6793        caps.build_disk_total_gb = None;
6794        assert_eq!(caps.build_disk_gb(), (Some(8.0), Some(15.5)));
6795        assert_eq!(caps.is_low_disk(10.0), Some(true));
6796    }
6797
6798    #[test]
6799    fn build_disk_sample_is_optional_on_the_wire() {
6800        let _guard = test_guard!();
6801        let old: WorkerCapabilities =
6802            serde_json::from_str(r#"{"disk_free_gb":13.9,"disk_total_gb":15.5}"#).unwrap();
6803        assert_eq!(old.build_disk_free_gb, None);
6804        assert_eq!(old.build_disk_gb(), (Some(13.9), Some(15.5)));
6805
6806        let new: WorkerCapabilities = serde_json::from_str(
6807            r#"{"disk_free_gb":13.9,"disk_total_gb":15.5,"build_disk_free_gb":1700.0,"build_disk_total_gb":1900.0}"#,
6808        )
6809        .unwrap();
6810        assert_eq!(new.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6811        let json = serde_json::to_string(&new).unwrap();
6812        assert!(json.contains("\"build_disk_free_gb\":1700.0"), "{json}");
6813    }
6814
6815    #[test]
6816    fn test_selection_config_preflight_defaults() {
6817        let _guard = test_guard!();
6818        let config = SelectionConfig::default();
6819        // Default thresholds
6820        assert_eq!(config.max_load_per_core, Some(2.0));
6821        assert_eq!(config.min_free_gb, Some(10.0));
6822        assert_eq!(config.disk_gb_per_slot, 10.0);
6823    }
6824
6825    #[test]
6826    fn test_selection_config_preflight_serde() {
6827        let _guard = test_guard!();
6828        let config = SelectionConfig {
6829            max_load_per_core: Some(3.5),
6830            min_free_gb: Some(25.0),
6831            disk_gb_per_slot: 0.83,
6832            ..Default::default()
6833        };
6834        let json = serde_json::to_string(&config).unwrap();
6835        let parsed: SelectionConfig = serde_json::from_str(&json).unwrap();
6836        assert_eq!(parsed.max_load_per_core, Some(3.5));
6837        assert_eq!(parsed.min_free_gb, Some(25.0));
6838        assert_eq!(parsed.disk_gb_per_slot, 0.83);
6839    }
6840
6841    #[test]
6842    fn test_selection_disk_slot_budget_validation() {
6843        let _guard = test_guard!();
6844        for value in ["0.0", "-1.0", "nan", "inf", "-inf"] {
6845            let result = toml::from_str::<SelectionConfig>(&format!("disk_gb_per_slot = {value}"));
6846            assert!(result.is_err(), "invalid disk slot budget {value}");
6847        }
6848        let default: SelectionConfig = toml::from_str("").unwrap();
6849        assert_eq!(default.disk_gb_per_slot, 10.0);
6850    }
6851
6852    #[test]
6853    fn test_selection_disk_weight_validation_and_roundtrip() {
6854        for value in ["-0.1", "1.1", "nan", "inf", "-inf"] {
6855            assert!(toml::from_str::<SelectionWeightConfig>(&format!("disk = {value}")).is_err());
6856        }
6857        for value in [0.0, 0.2, 1.0] {
6858            let weights: SelectionWeightConfig =
6859                toml::from_str(&format!("disk = {value}")).unwrap();
6860            assert_eq!(weights.disk, value);
6861            let roundtrip: SelectionWeightConfig =
6862                serde_json::from_str(&serde_json::to_string(&weights).unwrap()).unwrap();
6863            assert_eq!(roundtrip.disk, value);
6864        }
6865        let defaults: SelectionWeightConfig = toml::from_str("").unwrap();
6866        assert_eq!(defaults.disk, 0.2);
6867    }
6868
6869    #[test]
6870    fn test_worker_capabilities_health_metrics_serde() {
6871        let _guard = test_guard!();
6872        let caps = WorkerCapabilities {
6873            num_cpus: Some(8),
6874            load_avg_1: Some(1.5),
6875            load_avg_5: Some(2.0),
6876            load_avg_15: Some(1.8),
6877            disk_free_gb: Some(50.5),
6878            disk_total_gb: Some(100.0),
6879            ..Default::default()
6880        };
6881        let json = serde_json::to_string(&caps).unwrap();
6882        let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
6883        assert_eq!(parsed.num_cpus, Some(8));
6884        assert_eq!(parsed.load_avg_1, Some(1.5));
6885        assert_eq!(parsed.disk_free_gb, Some(50.5));
6886    }
6887
6888    #[test]
6889    fn microarch_level_classifies_ivy_bridge_as_v2() {
6890        // Abbreviated real flag set from ovh-b (Xeon E3-1245 V2, Ivy Bridge):
6891        // AVX present, but no avx2/bmi/fma — the bd-68hon SIGILL class.
6892        let ivy = "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov \
6893                   pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx rdtscp lm \
6894                   constant_tsc pni pclmulqdq ssse3 cx16 sse4_1 sse4_2 x2apic \
6895                   popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm";
6896        assert_eq!(x86_64_microarch_level_from_flags(ivy), 2);
6897    }
6898
6899    #[test]
6900    fn microarch_level_classifies_v3_and_v4() {
6901        let v3 = "cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 abm avx avx2 bmi1 bmi2 \
6902                  f16c fma movbe xsave";
6903        assert_eq!(x86_64_microarch_level_from_flags(v3), 3);
6904
6905        let v4 = format!("{v3} avx512f avx512bw avx512cd avx512dq avx512vl");
6906        assert_eq!(x86_64_microarch_level_from_flags(&v4), 4);
6907    }
6908
6909    #[test]
6910    fn microarch_level_v1_for_ancient_or_empty_flags() {
6911        assert_eq!(x86_64_microarch_level_from_flags(""), 1);
6912        assert_eq!(x86_64_microarch_level_from_flags("fpu mmx sse sse2"), 1);
6913    }
6914
6915    #[test]
6916    fn pre_v3_predicate_only_fires_on_affirmative_evidence() {
6917        let mut caps = WorkerCapabilities::new();
6918        assert!(!caps.is_pre_v3_x86(), "unknown must not deprioritize");
6919        caps.cpu_microarch_level = Some(2);
6920        assert!(caps.is_pre_v3_x86());
6921        caps.cpu_microarch_level = Some(3);
6922        assert!(!caps.is_pre_v3_x86());
6923    }
6924}