1use crate::{CompilationKind, toolchain::ToolchainInfo};
4use rand::RngExt;
5use schemars::JsonSchema;
6use serde::{Deserialize, Serialize};
7use std::path::PathBuf;
8
9#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
11pub struct WorkerId(pub String);
12
13impl WorkerId {
14 pub fn new(id: impl Into<String>) -> Self {
15 Self(id.into())
16 }
17
18 pub fn as_str(&self) -> &str {
19 &self.0
20 }
21}
22
23impl std::fmt::Display for WorkerId {
24 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
25 write!(f, "{}", self.0)
26 }
27}
28
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
46#[serde(rename_all = "snake_case")]
47pub enum WorkerStatus {
48 #[default]
50 Healthy,
51 Degraded,
53 Unreachable,
55 Draining,
57 Drained,
59 Disabled,
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
65#[serde(rename_all = "snake_case")]
66pub enum CircuitState {
67 #[default]
69 Closed,
70 Open,
72 HalfOpen,
74}
75
76#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
78#[serde(rename_all = "lowercase")]
79pub enum RequiredRuntime {
80 #[default]
82 None,
83 Rust,
85 Bun,
87 Node,
89 Nix,
91 Go,
93 Zig,
95}
96
97#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
102#[serde(rename_all = "lowercase")]
103pub enum CommandPriority {
104 Low,
105 #[default]
106 Normal,
107 High,
108}
109
110impl std::fmt::Display for CommandPriority {
111 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112 let value = match self {
113 Self::Low => "low",
114 Self::Normal => "normal",
115 Self::High => "high",
116 };
117 write!(f, "{}", value)
118 }
119}
120
121impl std::str::FromStr for CommandPriority {
122 type Err = ();
123
124 fn from_str(s: &str) -> Result<Self, Self::Err> {
125 match s.trim().to_lowercase().as_str() {
126 "low" => Ok(Self::Low),
127 "normal" => Ok(Self::Normal),
128 "high" => Ok(Self::High),
129 _ => Err(()),
130 }
131 }
132}
133
134#[derive(Debug, Clone, Serialize, Deserialize)]
136pub struct SelectionRequest {
137 pub project: String,
139 #[serde(default, skip_serializing_if = "Option::is_none")]
141 pub command: Option<String>,
142 #[serde(default)]
144 pub command_priority: CommandPriority,
145 pub estimated_cores: u32,
147 #[serde(default)]
149 pub preferred_workers: Vec<WorkerId>,
150 #[serde(default)]
152 pub toolchain: Option<ToolchainInfo>,
153 #[serde(default)]
155 pub required_runtime: RequiredRuntime,
156 #[serde(default, skip_serializing_if = "Option::is_none")]
159 pub classification_duration_us: Option<u64>,
160 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
166 pub job_mode: bool,
167 #[serde(default, skip_serializing_if = "Vec::is_empty")]
177 pub required_tools: Vec<String>,
178 #[serde(default, skip_serializing_if = "Option::is_none")]
180 pub hook_pid: Option<u32>,
181}
182
183#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
188#[serde(rename_all = "snake_case")]
189pub enum SelectionReason {
190 Success,
192 NoWorkersConfigured,
194 AllWorkersUnreachable,
196 AllCircuitsOpen,
198 AllWorkersBusy,
200 NoWorkersPassedHealth,
202 AllWorkersFailedPreflight,
204 AllWorkersFailedConvergence,
206 NoAdmissibleWorkers(String),
208 NoMatchingWorkers,
210 NoWorkersWithRuntime(String),
212 SelectionError(String),
214 AffinityPinned,
216 AffinityFallback,
218}
219
220pub const SELECTION_RESPONSE_PROTOCOL_VERSION: u64 = 1;
226
227impl std::fmt::Display for SelectionReason {
228 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
229 match self {
230 Self::Success => write!(f, "worker assigned successfully"),
231 Self::NoWorkersConfigured => write!(f, "no workers configured"),
232 Self::AllWorkersUnreachable => write!(f, "all workers unreachable"),
233 Self::AllCircuitsOpen => write!(f, "all worker circuits open"),
234 Self::AllWorkersBusy => write!(f, "all workers at capacity"),
235 Self::NoWorkersPassedHealth => write!(f, "no workers passed health thresholds"),
236 Self::AllWorkersFailedPreflight => write!(f, "all workers failed preflight checks"),
237 Self::AllWorkersFailedConvergence => {
238 write!(f, "all workers failed repo convergence checks")
239 }
240 Self::NoAdmissibleWorkers(summary) => {
241 write!(f, "no admissible workers: {}", summary)
242 }
243 Self::NoMatchingWorkers => write!(f, "no matching workers found"),
244 Self::NoWorkersWithRuntime(rt) => write!(f, "no workers with {} installed", rt),
245 Self::SelectionError(e) => write!(f, "selection error: {}", e),
246 Self::AffinityPinned => write!(f, "worker assigned via affinity pinning"),
247 Self::AffinityFallback => write!(f, "worker assigned via last-success fallback"),
248 }
249 }
250}
251
252#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
258#[serde(rename_all = "snake_case")]
259pub enum SelectionStrategy {
260 Priority,
263 Fastest,
266 #[default]
269 Balanced,
270 CacheAffinity,
273 FairFastest,
276}
277
278impl std::fmt::Display for SelectionStrategy {
279 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
280 let name = match self {
281 Self::Priority => "priority",
282 Self::Fastest => "fastest",
283 Self::Balanced => "balanced",
284 Self::CacheAffinity => "cache_affinity",
285 Self::FairFastest => "fair_fastest",
286 };
287 write!(f, "{}", name)
288 }
289}
290
291impl std::str::FromStr for SelectionStrategy {
292 type Err = String;
293
294 fn from_str(s: &str) -> Result<Self, Self::Err> {
295 match s.trim().to_lowercase().as_str() {
296 "priority" => Ok(Self::Priority),
297 "fastest" => Ok(Self::Fastest),
298 "balanced" => Ok(Self::Balanced),
299 "cache_affinity" | "cache-affinity" | "cacheaffinity" => Ok(Self::CacheAffinity),
300 "fair_fastest" | "fair-fastest" | "fairfastest" => Ok(Self::FairFastest),
301 _ => Err(format!(
302 "unknown selection strategy '{}', expected one of: priority, fastest, balanced, cache_affinity, fair_fastest",
303 s
304 )),
305 }
306 }
307}
308
309#[derive(Debug, Clone, Serialize, Deserialize)]
311pub struct SelectionConfig {
312 #[serde(default)]
314 pub strategy: SelectionStrategy,
315 #[serde(default = "default_min_success_rate")]
317 pub min_success_rate: f64,
318 #[serde(default)]
320 pub weights: SelectionWeightConfig,
321 #[serde(default)]
323 pub fairness: FairnessConfig,
324 #[serde(default)]
326 pub affinity: AffinityConfig,
327
328 #[serde(default = "default_max_load_per_core")]
332 pub max_load_per_core: Option<f64>,
333 #[serde(default = "default_min_free_gb")]
336 pub min_free_gb: Option<f64>,
337 #[serde(
340 default = "default_disk_gb_per_slot",
341 deserialize_with = "deserialize_disk_gb_per_slot"
342 )]
343 pub disk_gb_per_slot: f64,
344}
345
346impl Default for SelectionConfig {
347 fn default() -> Self {
348 Self {
349 strategy: SelectionStrategy::default(),
350 min_success_rate: default_min_success_rate(),
351 weights: SelectionWeightConfig::default(),
352 fairness: FairnessConfig::default(),
353 affinity: AffinityConfig::default(),
354 max_load_per_core: default_max_load_per_core(),
355 min_free_gb: default_min_free_gb(),
356 disk_gb_per_slot: default_disk_gb_per_slot(),
357 }
358 }
359}
360
361fn default_min_success_rate() -> f64 {
362 0.8
363}
364
365fn default_max_load_per_core() -> Option<f64> {
366 Some(2.0) }
368
369fn default_min_free_gb() -> Option<f64> {
370 Some(10.0) }
372
373fn default_disk_gb_per_slot() -> f64 {
374 10.0
375}
376
377fn deserialize_disk_gb_per_slot<'de, D>(deserializer: D) -> Result<f64, D::Error>
378where
379 D: serde::Deserializer<'de>,
380{
381 let budget = f64::deserialize(deserializer)?;
382 if budget.is_finite() && budget > 0.0 {
383 Ok(budget)
384 } else {
385 Err(serde::de::Error::custom(
386 "disk_gb_per_slot must be finite and greater than zero",
387 ))
388 }
389}
390
391#[derive(Debug, Clone, Serialize, Deserialize)]
393pub struct SelectionWeightConfig {
394 #[serde(default = "default_weight_speedscore")]
396 pub speedscore: f64,
397 #[serde(default = "default_weight_slots")]
399 pub slots: f64,
400 #[serde(default = "default_weight_health")]
402 pub health: f64,
403 #[serde(default = "default_weight_cache")]
405 pub cache: f64,
406 #[serde(default = "default_weight_network")]
408 pub network: f64,
409 #[serde(default = "default_weight_priority")]
411 pub priority: f64,
412 #[serde(
414 default = "default_weight_disk",
415 deserialize_with = "deserialize_weight_disk"
416 )]
417 pub disk: f64,
418 #[serde(default = "default_half_open_penalty")]
420 pub half_open_penalty: f64,
421}
422
423impl Default for SelectionWeightConfig {
424 fn default() -> Self {
425 Self {
426 speedscore: default_weight_speedscore(),
427 slots: default_weight_slots(),
428 health: default_weight_health(),
429 cache: default_weight_cache(),
430 network: default_weight_network(),
431 priority: default_weight_priority(),
432 disk: default_weight_disk(),
433 half_open_penalty: default_half_open_penalty(),
434 }
435 }
436}
437
438fn default_weight_speedscore() -> f64 {
439 0.5
440}
441fn default_weight_slots() -> f64 {
442 0.4
443}
444fn default_weight_health() -> f64 {
445 0.3
446}
447fn default_weight_cache() -> f64 {
448 0.2
449}
450fn default_weight_network() -> f64 {
451 0.1
452}
453fn default_weight_priority() -> f64 {
454 0.5
461}
462fn default_half_open_penalty() -> f64 {
463 0.5
464}
465
466fn default_weight_disk() -> f64 {
467 0.2
468}
469
470fn deserialize_weight_disk<'de, D>(deserializer: D) -> Result<f64, D::Error>
471where
472 D: serde::Deserializer<'de>,
473{
474 let weight = f64::deserialize(deserializer)?;
475 if weight.is_finite() && (0.0..=1.0).contains(&weight) {
476 Ok(weight)
477 } else {
478 Err(serde::de::Error::custom(
479 "disk weight must be finite and between zero and one",
480 ))
481 }
482}
483
484#[derive(Debug, Clone, Serialize, Deserialize)]
486pub struct FairnessConfig {
487 #[serde(default = "default_fairness_lookback_secs")]
489 pub lookback_secs: u64,
490 #[serde(default = "default_max_consecutive_selections")]
492 pub max_consecutive_selections: u32,
493}
494
495impl Default for FairnessConfig {
496 fn default() -> Self {
497 Self {
498 lookback_secs: default_fairness_lookback_secs(),
499 max_consecutive_selections: default_max_consecutive_selections(),
500 }
501 }
502}
503
504fn default_fairness_lookback_secs() -> u64 {
505 300 }
507
508fn default_max_consecutive_selections() -> u32 {
509 3
510}
511
512#[derive(Debug, Clone, Serialize, Deserialize)]
514pub struct AffinityConfig {
515 #[serde(default = "default_affinity_enabled")]
517 pub enabled: bool,
518 #[serde(default = "default_affinity_pin_minutes")]
521 pub pin_minutes: u64,
522 #[serde(default = "default_last_success_fallback")]
526 pub enable_last_success_fallback: bool,
527 #[serde(default = "default_fallback_min_success_rate")]
530 pub fallback_min_success_rate: f64,
531}
532
533impl Default for AffinityConfig {
534 fn default() -> Self {
535 Self {
536 enabled: default_affinity_enabled(),
537 pin_minutes: default_affinity_pin_minutes(),
538 enable_last_success_fallback: default_last_success_fallback(),
539 fallback_min_success_rate: default_fallback_min_success_rate(),
540 }
541 }
542}
543
544fn default_affinity_enabled() -> bool {
545 true
546}
547
548fn default_affinity_pin_minutes() -> u64 {
549 60 }
551
552fn default_last_success_fallback() -> bool {
553 true
554}
555
556fn default_fallback_min_success_rate() -> f64 {
557 0.5 }
559
560#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
562pub struct SelectedWorker {
563 pub id: WorkerId,
565 pub host: String,
567 pub user: String,
569 pub identity_file: String,
571 pub slots_available: u32,
573 pub speed_score: f64,
575 #[serde(default)]
581 pub declared_os: Option<String>,
582}
583
584#[derive(Debug, Clone, Serialize, Deserialize)]
586pub struct SelectionResponse {
587 pub worker: Option<SelectedWorker>,
589 pub reason: SelectionReason,
591 #[serde(default, skip_serializing_if = "Option::is_none")]
593 pub build_id: Option<u64>,
594 #[serde(default, skip_serializing_if = "Option::is_none")]
596 pub diagnostics: Option<SelectionDiagnostics>,
597}
598
599#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
601#[serde(rename_all = "snake_case")]
602pub enum WorkerSelectionDiagnosticDecision {
603 Allow,
605 FallbackCandidate,
607 Deny,
609}
610
611#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
613pub struct WorkerSelectionDiagnostic {
614 pub worker_id: WorkerId,
616 pub status: String,
618 pub circuit_state: String,
620 pub pressure_state: String,
622 pub pressure_reason_code: String,
624 #[serde(skip_serializing_if = "Option::is_none")]
626 pub success_rate: Option<f64>,
627 pub min_success_rate: f64,
629 pub fallback_min_success_rate: f64,
631 pub required_runtime: RequiredRuntime,
633 pub runtime_available: bool,
635 pub available_slots: u32,
637 pub total_slots: u32,
639 pub estimated_cores: u32,
641 pub active_project_excluded: bool,
643 pub final_decision: WorkerSelectionDiagnosticDecision,
645 pub final_reason: String,
647 pub reason_codes: Vec<String>,
649}
650
651#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
653pub struct SelectionDiagnostics {
654 pub required_runtime: RequiredRuntime,
656 pub estimated_cores: u32,
658 pub min_success_rate: f64,
660 pub fallback_min_success_rate: f64,
662 pub active_project_exclusion_count: usize,
664 pub workers: Vec<WorkerSelectionDiagnostic>,
666}
667
668#[derive(Debug, Clone, Serialize, Deserialize)]
670pub struct ReleaseRequest {
671 pub worker_id: WorkerId,
673 pub slots: u32,
675 #[serde(default, skip_serializing_if = "Option::is_none")]
677 pub build_id: Option<u64>,
678 #[serde(default, skip_serializing_if = "Option::is_none")]
680 pub local_wrapper_id: Option<String>,
681 #[serde(default, skip_serializing_if = "Option::is_none")]
683 pub exit_code: Option<i32>,
684 #[serde(default, skip_serializing_if = "Option::is_none")]
688 pub duration_ms: Option<u64>,
689 #[serde(default, skip_serializing_if = "Option::is_none")]
691 pub bytes_transferred: Option<u64>,
692 #[serde(default, skip_serializing_if = "Option::is_none")]
694 pub timing: Option<CommandTimingBreakdown>,
695}
696
697#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
699#[serde(rename_all = "snake_case")]
700pub enum BuildHeartbeatPhase {
701 SyncUp,
703 Execute,
705 SyncDown,
707 Finalize,
709}
710
711#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
713pub struct BuildHeartbeatRequest {
714 pub build_id: u64,
716 pub worker_id: WorkerId,
718 #[serde(default, skip_serializing_if = "Option::is_none")]
720 pub hook_pid: Option<u32>,
721 #[serde(default, skip_serializing_if = "Option::is_none")]
723 pub local_wrapper_id: Option<String>,
724 #[serde(default, skip_serializing_if = "Option::is_none")]
726 pub remote_pgid_file: Option<String>,
727 pub phase: BuildHeartbeatPhase,
729 #[serde(default, skip_serializing_if = "Option::is_none")]
731 pub detail: Option<String>,
732 #[serde(default, skip_serializing_if = "Option::is_none")]
737 pub progress_counter: Option<u64>,
738 #[serde(default, skip_serializing_if = "Option::is_none")]
740 pub progress_percent: Option<f64>,
741}
742
743#[derive(Debug, Clone, Serialize, Deserialize)]
745pub struct WorkerConfig {
746 pub id: WorkerId,
748 pub host: String,
750 pub user: String,
752 pub identity_file: String,
754 pub total_slots: u32,
756 #[serde(default = "default_priority")]
758 pub priority: u32,
759 #[serde(default)]
765 pub tags: Vec<String>,
766 #[serde(default, skip_serializing_if = "Vec::is_empty")]
775 pub tools: Vec<WorkerToolProbe>,
776}
777
778#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
784pub struct WorkerToolProbe {
785 pub name: String,
787 pub command: Vec<String>,
789}
790
791fn default_priority() -> u32 {
792 100
793}
794
795pub const OS_TAG_PREFIX: &str = "os:";
797
798#[must_use]
809pub fn declared_os(tags: &[String]) -> Option<String> {
810 tags.iter()
815 .filter_map(|tag| tag.strip_prefix(OS_TAG_PREFIX))
816 .map(|os| os.trim().to_ascii_lowercase())
817 .find(|os| !os.is_empty())
818}
819
820#[must_use]
822pub fn os_tag(os: &str) -> String {
823 format!("{OS_TAG_PREFIX}{}", os.trim().to_ascii_lowercase())
824}
825
826impl Default for WorkerConfig {
827 fn default() -> Self {
828 Self {
829 id: WorkerId::new("default-worker"),
830 host: "localhost".to_string(),
831 user: "user".to_string(),
832 identity_file: "~/.ssh/id_rsa".to_string(),
833 total_slots: 4,
834 priority: default_priority(),
835 tags: Vec::new(),
836 tools: Vec::new(),
837 }
838 }
839}
840
841#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)]
847pub struct WorkerCapabilities {
848 #[serde(default, skip_serializing_if = "Option::is_none")]
850 pub rustc_version: Option<String>,
851 #[serde(default, skip_serializing_if = "Vec::is_empty")]
854 pub rustup_toolchains: Vec<String>,
855 #[serde(default, skip_serializing_if = "Vec::is_empty")]
859 pub rustup_components: Vec<String>,
860 #[serde(default, skip_serializing_if = "Vec::is_empty")]
866 pub probe_warnings: Vec<String>,
867 #[serde(default, skip_serializing_if = "Option::is_none")]
869 pub bun_version: Option<String>,
870 #[serde(default, skip_serializing_if = "Option::is_none")]
872 pub node_version: Option<String>,
873 #[serde(default, skip_serializing_if = "Option::is_none")]
875 pub npm_version: Option<String>,
876 #[serde(default, skip_serializing_if = "Option::is_none")]
880 pub nix_version: Option<String>,
881 #[serde(default, skip_serializing_if = "Option::is_none")]
884 pub go_version: Option<String>,
885 #[serde(default, skip_serializing_if = "Option::is_none")]
889 pub zig_version: Option<String>,
890 #[serde(default, skip_serializing_if = "Option::is_none")]
894 pub cargo_zigbuild_version: Option<String>,
895 #[serde(default, skip_serializing_if = "Option::is_none")]
902 pub cpu_microarch_level: Option<u8>,
903 #[serde(default, skip_serializing_if = "Vec::is_empty")]
910 pub tools_present: Vec<String>,
911 #[serde(default, skip_serializing_if = "Vec::is_empty")]
917 pub tools_absent: Vec<String>,
918
919 #[serde(default, skip_serializing_if = "Option::is_none")]
922 pub num_cpus: Option<u32>,
923 #[serde(default, skip_serializing_if = "Option::is_none")]
925 pub load_avg_1: Option<f64>,
926 #[serde(default, skip_serializing_if = "Option::is_none")]
928 pub load_avg_5: Option<f64>,
929 #[serde(default, skip_serializing_if = "Option::is_none")]
931 pub load_avg_15: Option<f64>,
932 #[serde(default, skip_serializing_if = "Option::is_none")]
936 pub disk_free_gb: Option<f64>,
937 #[serde(default, skip_serializing_if = "Option::is_none")]
939 pub disk_total_gb: Option<f64>,
940 #[serde(default, skip_serializing_if = "Option::is_none")]
945 pub build_disk_free_gb: Option<f64>,
946 #[serde(default, skip_serializing_if = "Option::is_none")]
948 pub build_disk_total_gb: Option<f64>,
949 #[serde(default, skip_serializing_if = "Option::is_none")]
954 pub projects_root_ok: Option<bool>,
955 #[serde(default, skip_serializing_if = "Option::is_none")]
957 pub projects_root_issue: Option<String>,
958 #[serde(default, skip_serializing_if = "Option::is_none")]
960 pub projects_root_checked_at_unix_ms: Option<i64>,
961}
962
963impl WorkerCapabilities {
964 pub fn new() -> Self {
966 Self::default()
967 }
968
969 pub fn mock_with_rust() -> Self {
973 Self {
974 rustc_version: Some("1.85.0-nightly (mock)".to_string()),
975 projects_root_ok: Some(true),
976 ..Self::default()
977 }
978 }
979
980 pub fn is_pre_v3_x86(&self) -> bool {
985 matches!(self.cpu_microarch_level, Some(1 | 2))
986 }
987
988 pub fn has_bun(&self) -> bool {
990 self.bun_version.is_some()
991 }
992
993 pub fn has_node(&self) -> bool {
995 self.node_version.is_some()
996 }
997
998 pub fn has_rust(&self) -> bool {
1000 self.rustc_version.is_some()
1001 }
1002
1003 pub fn has_rustup_component(&self, toolchain: &str, component: &str) -> bool {
1010 let owner_prefix = format!("{toolchain}-");
1011 self.rustup_components.iter().any(|fact| {
1012 fact.split_once(':').is_some_and(|(owner, name)| {
1013 name == component && (owner == toolchain || owner.starts_with(&owner_prefix))
1014 })
1015 })
1016 }
1017
1018 pub fn has_nix(&self) -> bool {
1020 self.nix_version.is_some()
1021 }
1022
1023 pub fn has_go(&self) -> bool {
1025 self.go_version.is_some()
1026 }
1027
1028 pub fn has_zig(&self) -> bool {
1038 self.zig_version.is_some() && self.cargo_zigbuild_version.is_some()
1039 }
1040
1041 pub fn load_per_core(&self) -> Option<f64> {
1044 match (self.load_avg_1, self.num_cpus) {
1045 (Some(load), Some(cpus)) if cpus > 0 => Some(load / cpus as f64),
1046 _ => None,
1047 }
1048 }
1049
1050 pub fn is_high_load(&self, max_load_per_core: f64) -> Option<bool> {
1053 self.load_per_core().map(|lpc| lpc > max_load_per_core)
1054 }
1055
1056 pub fn build_disk_gb(&self) -> (Option<f64>, Option<f64>) {
1064 if self.build_disk_free_gb.is_some() {
1065 (self.build_disk_free_gb, self.build_disk_total_gb)
1066 } else {
1067 (self.disk_free_gb, self.disk_total_gb)
1068 }
1069 }
1070
1071 pub fn is_low_disk(&self, min_free_gb: f64) -> Option<bool> {
1074 self.build_disk_gb().0.map(|free| free < min_free_gb)
1075 }
1076
1077 pub fn is_topology_healthy(&self) -> Option<bool> {
1081 self.projects_root_ok
1082 }
1083}
1084
1085#[must_use]
1094pub fn x86_64_microarch_level_from_flags(flags: &str) -> u8 {
1095 let set: std::collections::HashSet<&str> = flags.split_whitespace().collect();
1096 let has_all = |required: &[&str]| required.iter().all(|flag| set.contains(flag));
1097 const V2: &[&str] = &["cx16", "lahf_lm", "popcnt", "sse4_1", "sse4_2", "ssse3"];
1099 const V3: &[&str] = &[
1101 "abm", "avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "movbe", "xsave",
1102 ];
1103 const V4: &[&str] = &["avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl"];
1105 if !has_all(V2) {
1106 return 1;
1107 }
1108 if !has_all(V3) {
1109 return 2;
1110 }
1111 if !has_all(V4) {
1112 return 3;
1113 }
1114 4
1115}
1116
1117#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1130pub struct PathTopologyConfig {
1131 #[serde(default)]
1134 pub canonical_root: Option<String>,
1135
1136 #[serde(default)]
1139 pub alias_root: Option<String>,
1140}
1141
1142impl PathTopologyConfig {
1143 pub fn to_policy(&self) -> crate::path_topology::PathTopologyPolicy {
1149 let canonical = self
1150 .canonical_root
1151 .as_deref()
1152 .filter(|s| !s.is_empty())
1153 .map(|s| shellexpand::tilde(s).into_owned())
1154 .unwrap_or_else(|| crate::path_topology::DEFAULT_CANONICAL_PROJECT_ROOT.to_string());
1155 let alias = self
1156 .alias_root
1157 .as_deref()
1158 .filter(|s| !s.is_empty())
1159 .map(|s| shellexpand::tilde(s).into_owned())
1160 .unwrap_or_else(|| crate::path_topology::DEFAULT_ALIAS_PROJECT_ROOT.to_string());
1161 crate::path_topology::PathTopologyPolicy::new(
1162 PathBuf::from(canonical),
1163 PathBuf::from(alias),
1164 )
1165 }
1166}
1167
1168#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1170pub struct RchConfig {
1171 #[serde(default)]
1172 pub general: GeneralConfig,
1173 #[serde(default)]
1174 pub compilation: CompilationConfig,
1175 #[serde(default)]
1176 pub transfer: TransferConfig,
1177 #[serde(default)]
1178 pub environment: EnvironmentConfig,
1179 #[serde(default)]
1180 pub circuit: CircuitBreakerConfig,
1181 #[serde(default)]
1182 pub output: OutputConfig,
1183 #[serde(default)]
1184 pub self_healing: SelfHealingConfig,
1185 #[serde(default)]
1186 pub self_test: SelfTestConfig,
1187 #[serde(default)]
1189 pub selection: SelectionConfig,
1190 #[serde(default)]
1192 pub execution: ExecutionConfig,
1193 #[serde(default)]
1195 pub alerts: AlertsConfig,
1196 #[serde(default)]
1198 pub fleet: FleetConfig,
1199 #[serde(default)]
1201 pub path_topology: PathTopologyConfig,
1202 #[serde(default)]
1204 pub doctor: DoctorConfig,
1205 #[serde(default)]
1210 pub remediation: crate::remediation_config::RemediationConfig,
1211 #[serde(default)]
1214 pub layer0: Layer0Config,
1215 #[serde(default)]
1218 pub api: ApiConfig,
1219 #[serde(default)]
1221 pub dashboard: DashboardConfig,
1222 #[serde(default)]
1224 pub jobs: JobsConfig,
1225}
1226
1227#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1229pub struct JobsConfig {
1230 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1240 pub required_tools: Vec<String>,
1241}
1242
1243#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1260#[serde(default)]
1261pub struct ApiConfig {
1262 pub bind: String,
1266 pub token: Option<String>,
1268 pub token_file: Option<String>,
1270 pub no_token: bool,
1274 pub allow_any_addr: bool,
1276}
1277
1278#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1280#[serde(default)]
1281pub struct DashboardConfig {
1282 pub url: Option<String>,
1284}
1285
1286#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1292pub struct DoctorConfig {
1293 #[serde(default)]
1295 pub webhooks: DoctorWebhooksConfig,
1296}
1297
1298#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1304pub struct DoctorWebhooksConfig {
1305 #[serde(default)]
1307 pub endpoints: Vec<DoctorWebhookEndpoint>,
1308 #[serde(default = "default_webhook_max_inflight")]
1313 pub max_inflight: usize,
1314}
1315
1316impl Default for DoctorWebhooksConfig {
1320 fn default() -> Self {
1321 Self {
1322 endpoints: Vec::new(),
1323 max_inflight: default_webhook_max_inflight(),
1324 }
1325 }
1326}
1327
1328#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1336pub struct DoctorWebhookEndpoint {
1337 pub name: String,
1339 pub url: String,
1341 #[serde(default)]
1343 pub format: DoctorWebhookFormat,
1344 #[serde(default)]
1349 pub on_transitions: Vec<String>,
1350 #[serde(default = "default_webhook_retry_max")]
1352 pub retry_max: u32,
1353 #[serde(default = "default_webhook_retry_backoff_ms")]
1355 pub retry_backoff_ms: u64,
1356 #[serde(default = "default_webhook_timeout_ms")]
1358 pub timeout_ms: u64,
1359 #[serde(default = "default_true")]
1361 pub enabled: bool,
1362 #[serde(default)]
1365 pub routing_key_env: Option<String>,
1366 #[serde(default)]
1369 pub bearer_token_env: Option<String>,
1370 #[serde(default)]
1374 pub signing_secret_env: Option<String>,
1375}
1376
1377#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
1379#[serde(rename_all = "snake_case")]
1380pub enum DoctorWebhookFormat {
1381 #[default]
1383 GenericJson,
1384 Slack,
1386 Pagerduty,
1388}
1389
1390fn default_webhook_max_inflight() -> usize {
1391 16
1392}
1393
1394fn default_webhook_retry_max() -> u32 {
1395 3
1396}
1397
1398fn default_webhook_retry_backoff_ms() -> u64 {
1399 200
1400}
1401
1402fn default_webhook_timeout_ms() -> u64 {
1403 5000
1404}
1405
1406#[derive(Debug, Clone, Serialize, Deserialize)]
1408pub struct AlertsConfig {
1409 #[serde(default = "default_true")]
1411 pub enabled: bool,
1412 #[serde(default = "default_alert_suppress_duplicates_secs")]
1414 pub suppress_duplicates_secs: u64,
1415 #[serde(default = "default_alert_cleared_retention_secs")]
1420 pub cleared_retention_secs: u64,
1421 #[serde(default)]
1423 pub webhook: Option<WebhookConfig>,
1424}
1425
1426#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1428pub struct WebhookConfig {
1429 pub url: Option<String>,
1431 #[serde(default)]
1433 pub secret: Option<String>,
1434 #[serde(default = "default_webhook_timeout_secs")]
1436 pub timeout_secs: u64,
1437 #[serde(default = "default_webhook_retry_count")]
1439 pub retry_count: u32,
1440 #[serde(default)]
1443 pub events: Vec<String>,
1444}
1445
1446fn default_webhook_timeout_secs() -> u64 {
1447 5
1448}
1449
1450fn default_webhook_retry_count() -> u32 {
1451 3
1452}
1453
1454fn default_alert_suppress_duplicates_secs() -> u64 {
1455 300
1456}
1457
1458fn default_alert_cleared_retention_secs() -> u64 {
1459 300
1460}
1461
1462impl Default for AlertsConfig {
1463 fn default() -> Self {
1464 Self {
1465 enabled: true,
1466 suppress_duplicates_secs: default_alert_suppress_duplicates_secs(),
1467 cleared_retention_secs: default_alert_cleared_retention_secs(),
1468 webhook: None,
1469 }
1470 }
1471}
1472
1473#[derive(Debug, Clone, Serialize, Deserialize)]
1478pub struct FleetConfig {
1479 #[serde(default = "default_fleet_ssh_connect_timeout_secs")]
1481 pub ssh_connect_timeout_secs: u64,
1482 #[serde(default = "default_fleet_ssh_command_timeout_secs")]
1484 pub ssh_command_timeout_secs: u64,
1485 #[serde(default = "default_fleet_min_disk_space_mb")]
1487 pub min_disk_space_mb: u64,
1488 #[serde(default = "default_fleet_max_load_average")]
1490 pub max_load_average: f64,
1491 #[serde(default = "default_fleet_max_concurrent_workers")]
1493 pub max_concurrent_workers: usize,
1494 #[serde(default = "default_fleet_retry_count")]
1496 pub retry_count: u32,
1497 #[serde(default = "default_fleet_retry_delay_ms")]
1499 pub retry_delay_ms: u64,
1500}
1501
1502fn default_fleet_ssh_connect_timeout_secs() -> u64 {
1503 10
1504}
1505
1506fn default_fleet_ssh_command_timeout_secs() -> u64 {
1507 30
1508}
1509
1510fn default_fleet_min_disk_space_mb() -> u64 {
1511 500
1512}
1513
1514fn default_fleet_max_load_average() -> f64 {
1515 10.0
1516}
1517
1518fn default_fleet_max_concurrent_workers() -> usize {
1519 10
1520}
1521
1522fn default_fleet_retry_count() -> u32 {
1523 2
1524}
1525
1526fn default_fleet_retry_delay_ms() -> u64 {
1527 1000
1528}
1529
1530impl Default for FleetConfig {
1531 fn default() -> Self {
1532 Self {
1533 ssh_connect_timeout_secs: default_fleet_ssh_connect_timeout_secs(),
1534 ssh_command_timeout_secs: default_fleet_ssh_command_timeout_secs(),
1535 min_disk_space_mb: default_fleet_min_disk_space_mb(),
1536 max_load_average: default_fleet_max_load_average(),
1537 max_concurrent_workers: default_fleet_max_concurrent_workers(),
1538 retry_count: default_fleet_retry_count(),
1539 retry_delay_ms: default_fleet_retry_delay_ms(),
1540 }
1541 }
1542}
1543
1544impl FleetConfig {
1545 pub fn ssh_connect_timeout(&self) -> std::time::Duration {
1547 std::time::Duration::from_secs(self.ssh_connect_timeout_secs)
1548 }
1549
1550 pub fn ssh_command_timeout(&self) -> std::time::Duration {
1552 std::time::Duration::from_secs(self.ssh_command_timeout_secs)
1553 }
1554
1555 pub fn retry_delay(&self) -> std::time::Duration {
1557 std::time::Duration::from_millis(self.retry_delay_ms)
1558 }
1559}
1560
1561#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
1564#[serde(rename_all = "lowercase")]
1565pub enum BoxRole {
1566 Dispatcher,
1571 Worker,
1574 #[default]
1576 Hybrid,
1577}
1578
1579impl BoxRole {
1580 #[must_use]
1582 pub fn as_str(self) -> &'static str {
1583 match self {
1584 BoxRole::Dispatcher => "dispatcher",
1585 BoxRole::Worker => "worker",
1586 BoxRole::Hybrid => "hybrid",
1587 }
1588 }
1589}
1590
1591#[derive(Debug, Clone, Serialize, Deserialize)]
1592pub struct GeneralConfig {
1593 #[serde(default = "default_true")]
1595 pub enabled: bool,
1596 #[serde(default)]
1599 pub role: BoxRole,
1600 #[serde(default)]
1604 pub force_local: bool,
1605 #[serde(default)]
1610 pub force_remote: bool,
1611 #[serde(default = "default_log_level")]
1613 pub log_level: String,
1614 #[serde(default = "default_socket_path")]
1616 pub socket_path: String,
1617}
1618
1619#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1621pub struct EnvironmentConfig {
1622 #[serde(default)]
1624 pub allowlist: Vec<String>,
1625}
1626
1627impl Default for GeneralConfig {
1628 fn default() -> Self {
1629 Self {
1630 enabled: true,
1631 role: BoxRole::default(),
1632 force_local: false,
1633 force_remote: false,
1634 log_level: "info".to_string(),
1635 socket_path: default_socket_path(),
1636 }
1637 }
1638}
1639
1640#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
1642#[serde(rename_all = "lowercase")]
1643pub enum OutputVisibility {
1644 #[default]
1645 None,
1646 Summary,
1647 Verbose,
1648}
1649
1650impl std::fmt::Display for OutputVisibility {
1651 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1652 let value = match self {
1653 OutputVisibility::None => "none",
1654 OutputVisibility::Summary => "summary",
1655 OutputVisibility::Verbose => "verbose",
1656 };
1657 write!(f, "{}", value)
1658 }
1659}
1660
1661impl std::str::FromStr for OutputVisibility {
1662 type Err = ();
1663
1664 fn from_str(s: &str) -> Result<Self, Self::Err> {
1665 match s.trim().to_lowercase().as_str() {
1666 "none" | "silent" | "quiet" => Ok(Self::None),
1667 "summary" | "short" => Ok(Self::Summary),
1668 "verbose" | "debug" => Ok(Self::Verbose),
1669 _ => Err(()),
1670 }
1671 }
1672}
1673
1674#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1679#[serde(rename_all = "lowercase")]
1680pub enum ColorMode {
1681 #[default]
1684 Always,
1685 Auto,
1687 Never,
1689}
1690
1691impl std::fmt::Display for ColorMode {
1692 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1693 let value = match self {
1694 ColorMode::Always => "always",
1695 ColorMode::Auto => "auto",
1696 ColorMode::Never => "never",
1697 };
1698 write!(f, "{}", value)
1699 }
1700}
1701
1702impl std::str::FromStr for ColorMode {
1703 type Err = ();
1704
1705 fn from_str(s: &str) -> Result<Self, Self::Err> {
1706 match s.trim().to_lowercase().as_str() {
1707 "always" | "force" | "yes" | "true" => Ok(Self::Always),
1708 "auto" | "detect" => Ok(Self::Auto),
1709 "never" | "none" | "no" | "false" => Ok(Self::Never),
1710 _ => Err(()),
1711 }
1712 }
1713}
1714
1715fn default_color_mode() -> ColorMode {
1716 ColorMode::default()
1717}
1718
1719#[derive(Debug, Clone, Serialize, Deserialize)]
1721pub struct OutputConfig {
1722 #[serde(default = "default_output_visibility")]
1724 pub visibility: OutputVisibility,
1725 #[serde(default)]
1727 pub first_run_complete: bool,
1728 #[serde(default = "default_color_mode")]
1731 pub color_mode: ColorMode,
1732}
1733
1734impl Default for OutputConfig {
1735 fn default() -> Self {
1736 Self {
1737 visibility: OutputVisibility::None,
1738 first_run_complete: false,
1739 color_mode: ColorMode::default(),
1740 }
1741 }
1742}
1743
1744fn default_autostart_cooldown_secs() -> u64 {
1749 30
1750}
1751
1752fn default_autostart_timeout_secs() -> u64 {
1753 3
1754}
1755
1756#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
1762#[serde(rename_all = "lowercase")]
1763pub enum SelfHealingLogLevel {
1764 Debug,
1765 #[default]
1766 Info,
1767 Warn,
1768 Error,
1769}
1770
1771impl SelfHealingLogLevel {
1772 pub fn from_env_str(s: &str) -> Option<Self> {
1774 match s.trim().to_ascii_lowercase().as_str() {
1775 "debug" => Some(Self::Debug),
1776 "info" => Some(Self::Info),
1777 "warn" | "warning" => Some(Self::Warn),
1778 "error" => Some(Self::Error),
1779 _ => None,
1780 }
1781 }
1782}
1783
1784#[derive(Debug, Clone, Serialize, Deserialize)]
1786pub struct SelfHealingConfig {
1787 #[serde(default = "default_true")]
1789 pub hook_starts_daemon: bool,
1790 #[serde(default = "default_true")]
1792 pub daemon_installs_hooks: bool,
1793 #[serde(default = "default_autostart_cooldown_secs")]
1795 pub auto_start_cooldown_secs: u64,
1796 #[serde(
1799 default = "default_autostart_timeout_secs",
1800 alias = "daemon_start_timeout"
1801 )]
1802 pub auto_start_timeout_secs: u64,
1803 #[serde(default)]
1806 pub self_healing_log_level: SelfHealingLogLevel,
1807}
1808
1809impl Default for SelfHealingConfig {
1810 fn default() -> Self {
1811 Self {
1812 hook_starts_daemon: default_true(),
1813 daemon_installs_hooks: default_true(),
1814 auto_start_cooldown_secs: default_autostart_cooldown_secs(),
1815 auto_start_timeout_secs: default_autostart_timeout_secs(),
1816 self_healing_log_level: SelfHealingLogLevel::default(),
1817 }
1818 }
1819}
1820
1821impl SelfHealingConfig {
1822 pub fn with_env_overrides(mut self) -> Self {
1831 if let Ok(val) = std::env::var("RCH_NO_SELF_HEALING")
1833 && (val == "1" || val.eq_ignore_ascii_case("true"))
1834 {
1835 self.hook_starts_daemon = false;
1836 self.daemon_installs_hooks = false;
1837 return self;
1838 }
1839
1840 if let Ok(val) = std::env::var("RCH_HOOK_STARTS_DAEMON") {
1842 self.hook_starts_daemon = val != "0" && !val.eq_ignore_ascii_case("false");
1843 }
1844 if let Ok(val) = std::env::var("RCH_DAEMON_INSTALLS_HOOKS") {
1845 self.daemon_installs_hooks = val != "0" && !val.eq_ignore_ascii_case("false");
1846 }
1847
1848 if let Ok(val) = std::env::var("RCH_AUTO_START_TIMEOUT_SECS")
1850 && let Ok(secs) = val.parse()
1851 {
1852 self.auto_start_timeout_secs = secs;
1853 }
1854 if let Ok(val) = std::env::var("RCH_AUTO_START_COOLDOWN_SECS")
1855 && let Ok(secs) = val.parse()
1856 {
1857 self.auto_start_cooldown_secs = secs;
1858 }
1859 if let Ok(val) = std::env::var("RCH_SELF_HEALING_LOG_LEVEL") {
1861 if let Some(level) = SelfHealingLogLevel::from_env_str(&val) {
1862 self.self_healing_log_level = level;
1863 } else {
1864 tracing::warn!(
1865 target: "rch::self_healing",
1866 invalid = %val,
1867 "RCH_SELF_HEALING_LOG_LEVEL has invalid value; using default=info"
1868 );
1869 }
1870 }
1871
1872 self
1873 }
1874}
1875
1876#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1882#[serde(rename_all = "snake_case")]
1883pub enum SelfTestFailureAction {
1884 #[default]
1886 Alert,
1887 DisableWorker,
1889 AlertAndDisable,
1891}
1892
1893#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1895#[serde(untagged)]
1896pub enum SelfTestWorkers {
1897 All(String),
1899 List(Vec<WorkerId>),
1901}
1902
1903impl Default for SelfTestWorkers {
1904 fn default() -> Self {
1905 Self::All("all".to_string())
1906 }
1907}
1908
1909impl SelfTestWorkers {
1910 pub fn resolve(&self) -> Option<Vec<WorkerId>> {
1912 match self {
1913 SelfTestWorkers::All(value) => {
1914 if value.eq_ignore_ascii_case("all") {
1915 None
1916 } else {
1917 Some(vec![WorkerId::new(value.clone())])
1918 }
1919 }
1920 SelfTestWorkers::List(list) => Some(list.clone()),
1921 }
1922 }
1923}
1924
1925fn default_self_test_retry_count() -> u32 {
1926 3
1927}
1928
1929fn default_self_test_retry_delay() -> String {
1930 "5m".to_string()
1931}
1932
1933fn default_self_test_enabled() -> bool {
1934 false
1935}
1936
1937#[derive(Debug, Clone, Serialize, Deserialize)]
1939pub struct SelfTestConfig {
1940 #[serde(default = "default_self_test_enabled")]
1942 pub enabled: bool,
1943 #[serde(default)]
1945 pub schedule: Option<String>,
1946 #[serde(default)]
1948 pub interval: Option<String>,
1949 #[serde(default)]
1951 pub workers: SelfTestWorkers,
1952 #[serde(default)]
1954 pub on_failure: SelfTestFailureAction,
1955 #[serde(default = "default_self_test_retry_count")]
1957 pub retry_count: u32,
1958 #[serde(default = "default_self_test_retry_delay")]
1960 pub retry_delay: String,
1961}
1962
1963impl Default for SelfTestConfig {
1964 fn default() -> Self {
1965 Self {
1966 enabled: default_self_test_enabled(),
1967 schedule: None,
1968 interval: None,
1969 workers: SelfTestWorkers::default(),
1970 on_failure: SelfTestFailureAction::default(),
1971 retry_count: default_self_test_retry_count(),
1972 retry_delay: default_self_test_retry_delay(),
1973 }
1974 }
1975}
1976
1977#[derive(Debug, Clone, Serialize, Deserialize)]
1979pub struct CircuitBreakerConfig {
1980 #[serde(default = "default_circuit_failure_threshold")]
1982 pub failure_threshold: u32,
1983 #[serde(default = "default_circuit_success_threshold")]
1985 pub success_threshold: u32,
1986 #[serde(default = "default_circuit_error_rate_threshold")]
1988 pub error_rate_threshold: f64,
1989 #[serde(default = "default_circuit_window_secs")]
1991 pub window_secs: u64,
1992 #[serde(default = "default_circuit_open_cooldown_secs")]
1994 pub open_cooldown_secs: u64,
1995 #[serde(default = "default_circuit_half_open_max_probes")]
1997 pub half_open_max_probes: u32,
1998 #[serde(default = "default_circuit_command_failure_threshold")]
2009 pub command_failure_threshold: u32,
2010 #[serde(default = "default_circuit_open_cooldown_max_secs")]
2018 pub open_cooldown_max_secs: u64,
2019}
2020
2021impl Default for CircuitBreakerConfig {
2022 fn default() -> Self {
2023 Self {
2024 failure_threshold: default_circuit_failure_threshold(),
2025 success_threshold: default_circuit_success_threshold(),
2026 error_rate_threshold: default_circuit_error_rate_threshold(),
2027 window_secs: default_circuit_window_secs(),
2028 open_cooldown_secs: default_circuit_open_cooldown_secs(),
2029 half_open_max_probes: default_circuit_half_open_max_probes(),
2030 command_failure_threshold: default_circuit_command_failure_threshold(),
2031 open_cooldown_max_secs: default_circuit_open_cooldown_max_secs(),
2032 }
2033 }
2034}
2035
2036impl CircuitBreakerConfig {
2037 pub fn cooldown_ms_for_streak(&self, reopen_streak: u32) -> u64 {
2041 let base_ms = self.open_cooldown_secs.saturating_mul(1000);
2042 let cap_ms = self
2043 .open_cooldown_max_secs
2044 .saturating_mul(1000)
2045 .max(base_ms);
2046 let exponent = reopen_streak.saturating_sub(1).min(16);
2048 base_ms.saturating_mul(1u64 << exponent).min(cap_ms)
2049 }
2050}
2051
2052#[derive(Debug, Clone, Default)]
2057pub struct CircuitStats {
2058 state: CircuitState,
2060 consecutive_failures: u32,
2062 consecutive_successes: u32,
2064 window_successes: u32,
2066 window_failures: u32,
2068 opened_at: Option<u64>,
2070 last_state_change: u64,
2072 active_probes: u32,
2074 consecutive_command_failures: u32,
2079 reopen_streak: u32,
2082 recent_results: Vec<bool>,
2085}
2086
2087const CIRCUIT_HISTORY_SIZE: usize = 10;
2089
2090impl CircuitStats {
2091 pub fn new() -> Self {
2093 Self {
2094 state: CircuitState::Closed,
2095 consecutive_failures: 0,
2096 consecutive_successes: 0,
2097 window_successes: 0,
2098 window_failures: 0,
2099 opened_at: None,
2100 last_state_change: Self::now_millis(),
2101 active_probes: 0,
2102 consecutive_command_failures: 0,
2103 reopen_streak: 0,
2104 recent_results: Vec::with_capacity(CIRCUIT_HISTORY_SIZE),
2105 }
2106 }
2107
2108 pub fn consecutive_command_failures(&self) -> u32 {
2111 self.consecutive_command_failures
2112 }
2113
2114 pub fn reopen_streak(&self) -> u32 {
2117 self.reopen_streak
2118 }
2119
2120 fn effective_cooldown_ms(&self, config: &CircuitBreakerConfig) -> u64 {
2122 config.cooldown_ms_for_streak(self.reopen_streak.max(1))
2123 }
2124
2125 pub fn state(&self) -> CircuitState {
2127 self.state
2128 }
2129
2130 pub fn opened_at(&self) -> Option<u64> {
2132 self.opened_at
2133 }
2134
2135 pub fn last_state_change(&self) -> u64 {
2137 self.last_state_change
2138 }
2139
2140 pub fn consecutive_failures(&self) -> u32 {
2142 self.consecutive_failures
2143 }
2144
2145 pub fn error_rate(&self) -> f64 {
2155 if self.recent_results.is_empty() {
2156 return 0.0;
2157 }
2158 let failures = self.recent_results.iter().filter(|&&ok| !ok).count();
2159 failures as f64 / self.recent_results.len() as f64
2160 }
2161
2162 pub fn record_success(&mut self) {
2167 self.window_successes += 1;
2168 self.consecutive_failures = 0;
2169 self.push_result(true);
2170
2171 if self.state == CircuitState::HalfOpen {
2172 self.consecutive_successes += 1;
2173 if self.active_probes > 0 {
2174 self.active_probes -= 1;
2175 }
2176 }
2177 }
2178
2179 pub fn record_failure(&mut self) {
2184 self.window_failures += 1;
2185 self.consecutive_failures += 1;
2186 self.push_result(false);
2187
2188 if self.state == CircuitState::HalfOpen {
2189 self.consecutive_successes = 0;
2190 if self.active_probes > 0 {
2191 self.active_probes -= 1;
2192 }
2193 }
2194 }
2195
2196 pub fn should_open(&self, config: &CircuitBreakerConfig) -> bool {
2202 if self.state != CircuitState::Closed {
2203 return false;
2204 }
2205
2206 if self.consecutive_failures >= config.failure_threshold {
2208 return true;
2209 }
2210
2211 if self.recent_results.len() >= 5 && self.error_rate() >= config.error_rate_threshold {
2215 return true;
2216 }
2217
2218 false
2219 }
2220
2221 pub fn should_half_open(&self, config: &CircuitBreakerConfig) -> bool {
2225 if self.state != CircuitState::Open {
2226 return false;
2227 }
2228
2229 let now = Self::now_millis();
2230 let cooldown_ms = self.effective_cooldown_ms(config);
2231
2232 if let Some(opened_at) = self.opened_at {
2233 now.saturating_sub(opened_at) >= cooldown_ms
2234 } else {
2235 false
2236 }
2237 }
2238
2239 pub fn should_close(&self, config: &CircuitBreakerConfig) -> bool {
2248 if self.state != CircuitState::HalfOpen {
2249 return false;
2250 }
2251
2252 self.consecutive_successes >= config.success_threshold
2253 && self.consecutive_command_failures == 0
2254 }
2255
2256 pub fn can_probe(&self, config: &CircuitBreakerConfig) -> bool {
2260 if self.state != CircuitState::HalfOpen {
2261 return false;
2262 }
2263
2264 self.active_probes < config.half_open_max_probes
2265 }
2266
2267 pub fn start_probe(&mut self, config: &CircuitBreakerConfig) -> bool {
2271 if !self.can_probe(config) {
2272 return false;
2273 }
2274 self.active_probes += 1;
2275 true
2276 }
2277
2278 pub fn open(&mut self) {
2280 if self.state != CircuitState::Open {
2281 self.state = CircuitState::Open;
2282 self.opened_at = Some(Self::now_millis());
2283 self.last_state_change = Self::now_millis();
2284 self.consecutive_successes = 0;
2285 self.active_probes = 0;
2286 self.reopen_streak = self.reopen_streak.saturating_add(1);
2289 }
2290 }
2291
2292 pub fn half_open(&mut self) {
2294 if self.state != CircuitState::HalfOpen {
2295 self.state = CircuitState::HalfOpen;
2296 self.last_state_change = Self::now_millis();
2297 self.consecutive_successes = 0;
2298 self.active_probes = 0;
2299 }
2300 }
2301
2302 pub fn close(&mut self) {
2304 if self.state != CircuitState::Closed {
2305 self.state = CircuitState::Closed;
2306 self.last_state_change = Self::now_millis();
2307 self.opened_at = None;
2308 self.consecutive_failures = 0;
2309 self.consecutive_successes = 0;
2310 self.active_probes = 0;
2311 self.consecutive_command_failures = 0;
2312 self.reopen_streak = 0;
2313 self.window_successes = 0;
2319 self.window_failures = 0;
2320 self.recent_results.clear();
2321 }
2322 }
2323
2324 pub fn reset_window(&mut self) {
2330 self.window_successes = 0;
2331 self.window_failures = 0;
2332 self.recent_results.clear();
2333 }
2334
2335 pub fn recent_results(&self) -> &[bool] {
2340 &self.recent_results
2341 }
2342
2343 pub fn recovery_remaining_secs(&self, config: &CircuitBreakerConfig) -> Option<u64> {
2347 if self.state != CircuitState::Open {
2348 return None;
2349 }
2350
2351 let now = Self::now_millis();
2352 let cooldown_ms = self.effective_cooldown_ms(config);
2353
2354 if let Some(opened_at) = self.opened_at {
2355 let elapsed_ms = now.saturating_sub(opened_at);
2356 if elapsed_ms < cooldown_ms {
2357 return Some((cooldown_ms - elapsed_ms) / 1000);
2358 }
2359 }
2360 None
2361 }
2362
2363 fn push_result(&mut self, success: bool) {
2365 if self.recent_results.len() >= CIRCUIT_HISTORY_SIZE {
2366 self.recent_results.remove(0);
2367 }
2368 self.recent_results.push(success);
2369 }
2370
2371 fn now_millis() -> u64 {
2372 std::time::SystemTime::now()
2373 .duration_since(std::time::UNIX_EPOCH)
2374 .map(|d| d.as_millis() as u64)
2375 .unwrap_or(0)
2376 }
2377
2378 pub fn apply_health_outcome(
2404 &mut self,
2405 healthy: bool,
2406 transient: bool,
2407 config: &CircuitBreakerConfig,
2408 ) -> CircuitState {
2409 if healthy {
2410 if self.state == CircuitState::Open && self.should_half_open(config) {
2414 self.half_open();
2415 }
2416 self.record_success();
2417 if self.should_close(config) {
2418 self.close();
2419 }
2420 } else {
2421 let was_half_open = self.state == CircuitState::HalfOpen;
2422 self.record_failure();
2423 if self.should_open(config) {
2424 self.open();
2425 } else if was_half_open && !transient {
2426 self.open();
2430 }
2431 }
2432
2433 if self.state == CircuitState::Open && self.should_half_open(config) {
2436 self.half_open();
2437 }
2438
2439 self.state
2440 }
2441
2442 pub fn apply_command_outcome(
2466 &mut self,
2467 success: bool,
2468 config: &CircuitBreakerConfig,
2469 ) -> CircuitState {
2470 if success {
2471 self.consecutive_command_failures = 0;
2472 return self.apply_health_outcome(true, false, config);
2473 }
2474
2475 self.consecutive_command_failures = self.consecutive_command_failures.saturating_add(1);
2476 let was_half_open = self.state == CircuitState::HalfOpen;
2477 self.record_failure();
2478 let threshold = config.command_failure_threshold.max(1);
2479 if was_half_open
2480 || self.should_open(config)
2481 || (self.state == CircuitState::Closed
2482 && self.consecutive_command_failures >= threshold)
2483 {
2484 self.open();
2485 }
2486
2487 if self.state == CircuitState::Open && self.should_half_open(config) {
2488 self.half_open();
2489 }
2490
2491 self.state
2492 }
2493}
2494
2495#[derive(Debug, Clone, Serialize, Deserialize)]
2496pub struct CompilationConfig {
2497 #[serde(default = "default_confidence")]
2499 pub confidence_threshold: f64,
2500 #[serde(default = "default_min_local_time")]
2502 pub min_local_time_ms: u64,
2503 #[serde(default = "default_speedup_threshold")]
2507 pub remote_speedup_threshold: f64,
2508 #[serde(default = "default_build_slots")]
2510 pub build_slots: u32,
2511 #[serde(default = "default_test_slots")]
2514 pub test_slots: u32,
2515 #[serde(default = "default_check_slots")]
2518 pub check_slots: u32,
2519 #[serde(default = "default_build_timeout")]
2522 pub build_timeout_sec: u64,
2523 #[serde(default = "default_test_timeout")]
2526 pub test_timeout_sec: u64,
2527 #[serde(default = "default_bun_timeout")]
2531 pub bun_timeout_sec: u64,
2532 #[serde(default = "default_external_timeout_enabled")]
2536 pub external_timeout_enabled: bool,
2537 #[serde(default = "default_allow_local_fallback")]
2550 pub allow_local_fallback: bool,
2551 #[serde(default)]
2562 pub remote_build_jobs: RemoteBuildJobs,
2563}
2564
2565#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
2577pub enum RemoteBuildJobs {
2578 #[default]
2580 Auto,
2581 Off,
2583 Fixed(u32),
2585}
2586
2587impl RemoteBuildJobs {
2588 pub fn parse(value: &str) -> Result<Self, String> {
2590 let trimmed = value.trim();
2591 match trimmed.to_ascii_lowercase().as_str() {
2592 "auto" => Ok(Self::Auto),
2593 "off" | "none" | "disabled" => Ok(Self::Off),
2594 _ => match trimmed.parse::<u32>() {
2595 Ok(0) => Err(
2596 "remote_build_jobs must be >= 1 (cargo rejects `-j 0`); use `off` to disable"
2597 .to_string(),
2598 ),
2599 Ok(n) => Ok(Self::Fixed(n)),
2600 Err(_) => Err(format!(
2601 "'{trimmed}' is not a valid remote_build_jobs value (expected `auto`, `off`, or a positive integer)"
2602 )),
2603 },
2604 }
2605 }
2606}
2607
2608impl std::fmt::Display for RemoteBuildJobs {
2609 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2610 match self {
2611 Self::Auto => f.write_str("auto"),
2612 Self::Off => f.write_str("off"),
2613 Self::Fixed(n) => write!(f, "{n}"),
2614 }
2615 }
2616}
2617
2618impl std::str::FromStr for RemoteBuildJobs {
2619 type Err = String;
2620
2621 fn from_str(value: &str) -> Result<Self, Self::Err> {
2622 Self::parse(value)
2623 }
2624}
2625
2626impl Serialize for RemoteBuildJobs {
2627 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
2628 match self {
2629 Self::Auto => serializer.serialize_str("auto"),
2630 Self::Off => serializer.serialize_str("off"),
2631 Self::Fixed(n) => serializer.serialize_u32(*n),
2632 }
2633 }
2634}
2635
2636impl<'de> Deserialize<'de> for RemoteBuildJobs {
2637 fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
2638 #[derive(Deserialize)]
2639 #[serde(untagged)]
2640 enum Raw {
2641 Count(u64),
2642 Spelling(String),
2643 }
2644 match Raw::deserialize(deserializer)? {
2645 Raw::Count(n) => {
2646 let n = u32::try_from(n).map_err(|_| {
2647 serde::de::Error::custom(format!("remote_build_jobs {n} is out of range"))
2648 })?;
2649 Self::parse(&n.to_string()).map_err(serde::de::Error::custom)
2650 }
2651 Raw::Spelling(s) => Self::parse(&s).map_err(serde::de::Error::custom),
2652 }
2653 }
2654}
2655
2656impl Default for CompilationConfig {
2657 fn default() -> Self {
2658 Self {
2659 confidence_threshold: 0.85,
2660 min_local_time_ms: 2000,
2661 remote_speedup_threshold: default_speedup_threshold(),
2662 build_slots: default_build_slots(),
2663 test_slots: default_test_slots(),
2664 check_slots: default_check_slots(),
2665 build_timeout_sec: default_build_timeout(),
2666 test_timeout_sec: default_test_timeout(),
2667 bun_timeout_sec: default_bun_timeout(),
2668 external_timeout_enabled: default_external_timeout_enabled(),
2669 allow_local_fallback: default_allow_local_fallback(),
2670 remote_build_jobs: RemoteBuildJobs::default(),
2671 }
2672 }
2673}
2674
2675fn default_build_slots() -> u32 {
2676 4
2677}
2678
2679fn default_test_slots() -> u32 {
2680 8
2681}
2682
2683fn default_check_slots() -> u32 {
2684 2
2685}
2686
2687fn default_build_timeout() -> u64 {
2690 300
2691}
2692
2693fn default_test_timeout() -> u64 {
2696 1800
2697}
2698
2699fn default_bun_timeout() -> u64 {
2703 600
2704}
2705
2706fn default_external_timeout_enabled() -> bool {
2708 true
2709}
2710
2711fn default_allow_local_fallback() -> bool {
2715 true
2716}
2717
2718impl CompilationConfig {
2719 pub fn timeout_for_kind(&self, kind: Option<CompilationKind>) -> std::time::Duration {
2725 let secs = match kind {
2726 Some(CompilationKind::BunTest) | Some(CompilationKind::BunTypecheck) => {
2728 self.bun_timeout_sec
2729 }
2730 Some(CompilationKind::CargoTest) | Some(CompilationKind::CargoNextest) => {
2732 self.test_timeout_sec
2733 }
2734 Some(CompilationKind::NixBuild) => self.test_timeout_sec,
2738 Some(CompilationKind::GoTest) => self.test_timeout_sec,
2741 _ => self.build_timeout_sec,
2743 };
2744 std::time::Duration::from_secs(secs)
2745 }
2746
2747 pub fn external_timeout_enabled(&self) -> bool {
2749 self.external_timeout_enabled
2750 }
2751}
2752
2753pub const LAYER0_PACK_VERSION: &str = "1";
2759
2760#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
2769#[serde(default)]
2770pub struct Layer0Config {
2771 pub enabled: bool,
2773 pub release_lto_thin: bool,
2775 pub release_codegen_units_1: bool,
2778}
2779
2780impl Layer0Config {
2781 #[must_use]
2787 pub fn active_env(&self) -> Vec<(&'static str, &'static str)> {
2788 if !self.enabled {
2789 return Vec::new();
2790 }
2791 let mut env = Vec::new();
2792 if self.release_lto_thin {
2793 env.push(("CARGO_PROFILE_RELEASE_LTO", "thin"));
2794 }
2795 if self.release_codegen_units_1 {
2796 env.push(("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"));
2797 }
2798 env
2799 }
2800}
2801
2802#[derive(Debug, Clone, Serialize, Deserialize)]
2803pub struct TransferConfig {
2804 #[serde(default = "default_compression")]
2806 pub compression_level: u32,
2807 #[serde(default = "default_excludes")]
2809 pub exclude_patterns: Vec<String>,
2810 #[serde(default, skip_serializing_if = "Option::is_none")]
2814 pub ssh_server_alive_interval_secs: Option<u64>,
2815 #[serde(default, skip_serializing_if = "Option::is_none")]
2820 pub ssh_control_persist_secs: Option<u64>,
2821 #[serde(default = "default_remote_base")]
2832 pub remote_base: String,
2833 #[serde(default, skip_serializing_if = "Option::is_none")]
2839 pub sync_timeout_ms: Option<u64>,
2840 #[serde(default = "default_source_sync_silence_timeout_secs")]
2852 pub source_sync_silence_timeout_secs: u64,
2853 #[serde(default)]
2855 pub retry: RetryConfig,
2856 #[serde(default)]
2862 pub verify_artifacts: bool,
2863 #[serde(default = "default_verify_max_size")]
2868 pub verify_max_size_bytes: u64,
2869
2870 #[serde(default, skip_serializing_if = "Option::is_none")]
2879 pub max_transfer_mb: Option<u64>,
2880
2881 #[serde(default, skip_serializing_if = "Option::is_none")]
2887 pub max_transfer_time_ms: Option<u64>,
2888
2889 #[serde(default, skip_serializing_if = "Option::is_none")]
2894 pub bwlimit_kbps: Option<u64>,
2895
2896 #[serde(default, skip_serializing_if = "Option::is_none")]
2905 pub rsync_bin: Option<String>,
2906
2907 #[serde(default, skip_serializing_if = "Option::is_none")]
2912 pub estimated_bandwidth_bps: Option<u64>,
2913
2914 #[serde(default)]
2927 pub adaptive_compression: bool,
2928
2929 #[serde(default = "default_min_compression")]
2934 pub min_compression_level: u32,
2935
2936 #[serde(default = "default_max_compression")]
2941 pub max_compression_level: u32,
2942}
2943
2944impl Default for TransferConfig {
2945 fn default() -> Self {
2946 Self {
2947 compression_level: 3,
2948 exclude_patterns: default_excludes(),
2949 ssh_server_alive_interval_secs: None,
2950 ssh_control_persist_secs: None,
2951 remote_base: default_remote_base(),
2952 sync_timeout_ms: None,
2953 source_sync_silence_timeout_secs: default_source_sync_silence_timeout_secs(),
2954 retry: RetryConfig::default(),
2955 verify_artifacts: false,
2956 verify_max_size_bytes: default_verify_max_size(),
2957 max_transfer_mb: None,
2959 max_transfer_time_ms: None,
2960 bwlimit_kbps: None,
2961 rsync_bin: None,
2962 estimated_bandwidth_bps: None,
2963 adaptive_compression: false,
2965 min_compression_level: default_min_compression(),
2966 max_compression_level: default_max_compression(),
2967 }
2968 }
2969}
2970
2971impl TransferConfig {
2972 pub const MIN_SYNC_TIMEOUT_MS: u64 = 1_000;
2974 pub const MAX_SYNC_TIMEOUT_MS: u64 = 3_600_000;
2976 pub const DEFAULT_SYNC_TIMEOUT_FLOOR_MS: u64 = 30_000;
2978 const DEFAULT_SYNC_BYTES_PER_SECOND: u64 = 1024 * 1024;
2979
2980 #[must_use]
2982 pub fn valid_sync_timeout_ms(value: u64) -> bool {
2983 (Self::MIN_SYNC_TIMEOUT_MS..=Self::MAX_SYNC_TIMEOUT_MS).contains(&value)
2984 }
2985
2986 #[must_use]
2993 pub fn sync_timeout_for_payload(&self, payload_bytes: u64) -> std::time::Duration {
2994 let timeout_ms = self
2995 .sync_timeout_ms
2996 .filter(|value| Self::valid_sync_timeout_ms(*value))
2997 .unwrap_or_else(|| {
2998 let transfer_seconds = payload_bytes
2999 .saturating_add(Self::DEFAULT_SYNC_BYTES_PER_SECOND - 1)
3000 / Self::DEFAULT_SYNC_BYTES_PER_SECOND;
3001 Self::DEFAULT_SYNC_TIMEOUT_FLOOR_MS
3002 .saturating_add(transfer_seconds.saturating_mul(1_000))
3003 .min(Self::MAX_SYNC_TIMEOUT_MS)
3004 });
3005 std::time::Duration::from_millis(timeout_ms)
3006 }
3007
3008 pub fn select_compression_level(&self, estimated_bytes: Option<u64>) -> u32 {
3020 if !self.adaptive_compression {
3021 return self.compression_level;
3022 }
3023
3024 let Some(bytes) = estimated_bytes else {
3025 return self.compression_level;
3027 };
3028
3029 const SMALL_THRESHOLD: u64 = 10_000_000; const LARGE_THRESHOLD: u64 = 200_000_000; let level = if bytes < SMALL_THRESHOLD {
3034 1 } else if bytes < LARGE_THRESHOLD {
3036 3 } else {
3038 7 };
3040
3041 level.clamp(self.min_compression_level, self.max_compression_level)
3043 }
3044}
3045
3046#[rustfmt::skip]
3060pub const BUILTIN_EXECUTION_BASES: &[&str] = &[
3061 "cargo",
3063 "rustc",
3064 "nextest",
3066 "gcc",
3068 "g++",
3069 "clang",
3070 "clang++",
3071 "cc",
3072 "c++",
3073 "make",
3075 "cmake",
3076 "ninja",
3077 "meson",
3078 "bun",
3080 "nix",
3082 "go",
3084 "tsc",
3086];
3087
3088fn default_execution_allowlist() -> Vec<String> {
3089 BUILTIN_EXECUTION_BASES
3090 .iter()
3091 .map(|base| (*base).to_string())
3092 .collect()
3093}
3094
3095#[derive(Debug, Clone, Serialize, Deserialize)]
3100pub struct ExecutionConfig {
3101 #[serde(default = "default_execution_allowlist")]
3110 pub allowlist: Vec<String>,
3111}
3112
3113impl Default for ExecutionConfig {
3114 fn default() -> Self {
3115 Self {
3116 allowlist: default_execution_allowlist(),
3117 }
3118 }
3119}
3120
3121impl ExecutionConfig {
3122 pub fn is_allowed(&self, command_base: &str) -> bool {
3124 self.allowlist
3125 .iter()
3126 .any(|allowed| allowed.eq_ignore_ascii_case(command_base))
3127 }
3128
3129 pub fn missing_builtin_bases(&self) -> Vec<&'static str> {
3135 BUILTIN_EXECUTION_BASES
3136 .iter()
3137 .copied()
3138 .filter(|base| !self.is_allowed(base))
3139 .collect()
3140 }
3141}
3142
3143fn default_retry_max_attempts() -> u32 {
3148 3
3149}
3150
3151fn default_retry_base_delay_ms() -> u64 {
3152 100
3153}
3154
3155fn default_retry_max_delay_ms() -> u64 {
3156 5000
3157}
3158
3159fn default_retry_jitter_factor() -> f64 {
3160 0.1
3161}
3162
3163fn default_retry_total_timeout_ms() -> u64 {
3164 30000
3165}
3166
3167#[derive(Debug, Clone, Serialize, Deserialize)]
3172pub struct RetryConfig {
3173 #[serde(default = "default_retry_max_attempts")]
3175 pub max_attempts: u32,
3176
3177 #[serde(default = "default_retry_base_delay_ms")]
3179 pub base_delay_ms: u64,
3180
3181 #[serde(default = "default_retry_max_delay_ms")]
3183 pub max_delay_ms: u64,
3184
3185 #[serde(default = "default_retry_jitter_factor")]
3188 pub jitter_factor: f64,
3189
3190 #[serde(default = "default_retry_total_timeout_ms")]
3193 pub total_timeout_ms: u64,
3194}
3195
3196impl Default for RetryConfig {
3197 fn default() -> Self {
3198 Self {
3199 max_attempts: default_retry_max_attempts(),
3200 base_delay_ms: default_retry_base_delay_ms(),
3201 max_delay_ms: default_retry_max_delay_ms(),
3202 jitter_factor: default_retry_jitter_factor(),
3203 total_timeout_ms: default_retry_total_timeout_ms(),
3204 }
3205 }
3206}
3207
3208impl RetryConfig {
3209 pub fn no_retry() -> Self {
3211 Self {
3212 max_attempts: 1,
3213 ..Default::default()
3214 }
3215 }
3216
3217 pub fn delay_for_attempt(&self, attempt: u32) -> std::time::Duration {
3221 if attempt == 0 {
3222 return std::time::Duration::ZERO;
3223 }
3224
3225 let base = self.base_delay_ms as f64;
3227 let delay = base * (2.0_f64.powi(attempt as i32 - 1));
3228 let capped = delay.min(self.max_delay_ms as f64);
3229
3230 let jittered = if self.jitter_factor > 0.0 {
3232 let jitter_range = capped * self.jitter_factor;
3233 let jitter = (rand::rng().random::<f64>() * 2.0 - 1.0) * jitter_range;
3234 (capped + jitter).max(0.0)
3235 } else {
3236 capped
3237 };
3238
3239 std::time::Duration::from_millis(jittered as u64)
3240 }
3241
3242 pub fn should_retry(&self, attempt: u32, elapsed: std::time::Duration) -> bool {
3244 if attempt >= self.max_attempts {
3245 return false;
3246 }
3247 elapsed.as_millis() < self.total_timeout_ms as u128
3248 }
3249}
3250
3251fn default_source_sync_silence_timeout_secs() -> u64 {
3256 120
3257}
3258
3259pub fn default_remote_base() -> String {
3268 "/data/tmp/rch".to_string()
3269}
3270
3271pub fn validate_remote_base(path: &str) -> Result<String, String> {
3274 let expanded = shellexpand::tilde(path).into_owned();
3276
3277 if !expanded.starts_with('/') {
3279 return Err(format!(
3280 "remote_base must be an absolute path, got: {}",
3281 path
3282 ));
3283 }
3284
3285 if expanded.contains("..") {
3287 return Err(format!(
3288 "remote_base must not contain path traversal (..): {}",
3289 path
3290 ));
3291 }
3292
3293 let normalized = expanded.trim_end_matches('/').to_string();
3295
3296 if normalized.is_empty() || normalized == "/" {
3298 return Err("remote_base cannot be the root directory (safety restriction)".to_string());
3299 }
3300
3301 let components: Vec<&str> = normalized.split('/').filter(|c| !c.is_empty()).collect();
3305 if components.len() < 2 {
3306 return Err(format!(
3307 "remote_base must be at least 2 levels deep (e.g. /tmp/rch), got: {}",
3308 normalized
3309 ));
3310 }
3311
3312 Ok(normalized)
3313}
3314
3315fn default_circuit_failure_threshold() -> u32 {
3316 3
3317}
3318
3319fn default_circuit_success_threshold() -> u32 {
3320 2
3321}
3322
3323fn default_circuit_error_rate_threshold() -> f64 {
3324 0.5
3325}
3326
3327fn default_circuit_window_secs() -> u64 {
3328 60
3329}
3330
3331fn default_circuit_open_cooldown_secs() -> u64 {
3332 30
3333}
3334
3335fn default_circuit_half_open_max_probes() -> u32 {
3336 1
3337}
3338
3339fn default_circuit_command_failure_threshold() -> u32 {
3340 2
3341}
3342
3343fn default_circuit_open_cooldown_max_secs() -> u64 {
3344 480
3346}
3347
3348fn default_true() -> bool {
3349 true
3350}
3351
3352fn default_log_level() -> String {
3353 "info".to_string()
3354}
3355
3356pub fn default_socket_path() -> String {
3357 if let Ok(runtime_dir) = std::env::var("XDG_RUNTIME_DIR")
3359 && !runtime_dir.trim().is_empty()
3360 {
3361 let path = PathBuf::from(runtime_dir).join("rch.sock");
3362 return path.to_string_lossy().to_string();
3363 }
3364
3365 if let Some(cache_dir) = dirs::cache_dir() {
3367 let rch_cache = cache_dir.join("rch");
3368 let _ = std::fs::create_dir_all(&rch_cache);
3369 return rch_cache.join("rch.sock").to_string_lossy().to_string();
3370 }
3371
3372 "/tmp/rch.sock".to_string()
3374}
3375
3376#[allow(dead_code)]
3377fn default_output_visibility() -> OutputVisibility {
3378 OutputVisibility::None
3379}
3380
3381fn default_confidence() -> f64 {
3382 0.85
3383}
3384
3385fn default_min_local_time() -> u64 {
3386 2000
3387}
3388
3389fn default_speedup_threshold() -> f64 {
3393 1.2
3394}
3395
3396fn default_compression() -> u32 {
3397 3
3398}
3399
3400fn default_verify_max_size() -> u64 {
3402 100 * 1024 * 1024 }
3404
3405fn default_min_compression() -> u32 {
3407 1
3408}
3409
3410fn default_max_compression() -> u32 {
3412 9
3413}
3414
3415fn default_excludes() -> Vec<String> {
3416 vec![
3417 "target/".to_string(),
3419 "*.rlib".to_string(),
3420 "*.rmeta".to_string(),
3421 ".git/".to_string(),
3424 "node_modules/".to_string(),
3426 ".bun/".to_string(),
3428 ".npm/".to_string(),
3430 ".pnpm-store/".to_string(),
3431 "dist/".to_string(),
3433 "build/".to_string(),
3434 ".next/".to_string(),
3436 ".nuxt/".to_string(),
3437 ".turbo/".to_string(),
3438 ".parcel-cache/".to_string(),
3439 ".beads/".to_string(),
3441 ".doctor/".to_string(),
3443 "coverage/".to_string(),
3445 ".nyc_output/".to_string(),
3446 ".cargo/credentials".to_string(),
3448 ".cargo/credentials.toml".to_string(),
3449 ".env".to_string(),
3450 ".env.*".to_string(),
3451 "*.pem".to_string(),
3452 "*.key".to_string(),
3453 "credentials.json".to_string(),
3454 "secrets.json".to_string(),
3455 "secrets.yaml".to_string(),
3456 "secrets.yml".to_string(),
3457 "secrets.toml".to_string(),
3458 "secrets.env".to_string(),
3459 "secrets.txt".to_string(),
3460 "secrets.conf".to_string(),
3461 "secrets.cfg".to_string(),
3462 "secrets.properties".to_string(),
3463 "secrets.xml".to_string(),
3464 "id_rsa".to_string(),
3478 "id_ed25519".to_string(),
3479 "id_ecdsa".to_string(),
3480 "id_dsa".to_string(),
3481 "*.p12".to_string(),
3483 "*.pfx".to_string(),
3484 "*.keystore".to_string(),
3485 "*.jks".to_string(),
3486 ".netrc".to_string(),
3489 "_netrc".to_string(),
3490 ".aws/credentials".to_string(),
3491 ".docker/config.json".to_string(),
3492 ".npmrc".to_string(),
3496 ".pypirc".to_string(),
3497 "secring.*".to_string(),
3499 ]
3500}
3501
3502#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
3508#[serde(rename_all = "snake_case")]
3509pub enum BuildLocation {
3510 #[default]
3512 Local,
3513 Remote,
3515}
3516
3517#[derive(Debug, Clone, Serialize, Deserialize)]
3519pub struct BuildCancellationWorkerHealth {
3520 pub status: String,
3522 pub speed_score: f64,
3524 pub used_slots: u32,
3526 pub available_slots: u32,
3528 pub pressure_state: String,
3530 pub pressure_reason_code: String,
3532}
3533
3534#[derive(Debug, Clone, Serialize, Deserialize)]
3536pub struct BuildCancellationMetadata {
3537 pub operation_id: String,
3539 pub origin: String,
3541 pub reason_code: String,
3543 #[serde(default)]
3545 pub decision_path: Vec<String>,
3546 pub escalation_stage: String,
3548 pub escalation_count: u32,
3550 pub remote_kill_attempted: bool,
3552 pub cleanup_ok: bool,
3554 pub history_cancelled: bool,
3556 pub final_state: String,
3558 #[serde(default, skip_serializing_if = "Option::is_none")]
3560 pub worker_health: Option<BuildCancellationWorkerHealth>,
3561}
3562
3563#[derive(Debug, Clone, Serialize, Deserialize)]
3565pub struct BuildRecord {
3566 pub id: u64,
3568 pub started_at: String,
3570 pub completed_at: String,
3572 pub project_id: String,
3574 pub worker_id: Option<String>,
3576 pub command: String,
3578 pub exit_code: i32,
3580 pub duration_ms: u64,
3582 pub location: BuildLocation,
3584 pub bytes_transferred: Option<u64>,
3586 #[serde(default)]
3588 pub timing: Option<CommandTimingBreakdown>,
3589 #[serde(default, skip_serializing_if = "Option::is_none")]
3591 pub cancellation: Option<BuildCancellationMetadata>,
3592}
3593
3594#[derive(Debug, Clone, Serialize, Deserialize)]
3596pub struct BuildRecordInput {
3597 pub started_at: String,
3599 pub completed_at: String,
3601 pub project_id: String,
3603 pub worker_id: Option<String>,
3605 pub command: String,
3607 pub exit_code: i32,
3609 pub duration_ms: u64,
3611 pub location: BuildLocation,
3613 pub bytes_transferred: Option<u64>,
3615 #[serde(default)]
3617 pub timing: Option<CommandTimingBreakdown>,
3618 #[serde(default, skip_serializing_if = "Option::is_none")]
3620 pub cancellation: Option<BuildCancellationMetadata>,
3621}
3622
3623impl BuildRecordInput {
3624 pub fn into_record(self, id: u64) -> BuildRecord {
3626 BuildRecord {
3627 id,
3628 started_at: self.started_at,
3629 completed_at: self.completed_at,
3630 project_id: self.project_id,
3631 worker_id: self.worker_id,
3632 command: self.command,
3633 exit_code: self.exit_code,
3634 duration_ms: self.duration_ms,
3635 location: self.location,
3636 bytes_transferred: self.bytes_transferred,
3637 timing: self.timing,
3638 cancellation: self.cancellation,
3639 }
3640 }
3641}
3642
3643#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3648pub struct BuildStats {
3649 pub total_builds: usize,
3651 pub success_count: usize,
3653 pub failure_count: usize,
3655 pub remote_count: usize,
3657 pub local_count: usize,
3659 pub avg_duration_ms: u64,
3661}
3662
3663#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3670pub struct SavedTimeStats {
3671 pub total_remote_duration_ms: u64,
3673 pub estimated_local_duration_ms: u64,
3676 pub time_saved_ms: u64,
3679 pub builds_counted: usize,
3681 pub avg_speedup: f64,
3685 pub today_saved_ms: u64,
3687 pub week_saved_ms: u64,
3689 #[serde(default)]
3692 pub estimate_basis: String,
3693 #[serde(default)]
3695 pub local_baseline_builds: usize,
3696}
3697
3698use std::collections::VecDeque;
3703use std::time::{Duration, Instant};
3704
3705#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3707pub struct CompilationTimingBreakdown {
3708 #[serde(with = "duration_millis")]
3710 pub rsync_up: Duration,
3711 #[serde(with = "duration_millis")]
3713 pub remote_build: Duration,
3714 #[serde(with = "duration_millis")]
3716 pub rsync_down: Duration,
3717 #[serde(with = "duration_millis")]
3719 pub total: Duration,
3720}
3721
3722#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3726pub struct CommandTimingBreakdown {
3727 #[serde(with = "option_duration_millis")]
3729 pub classify: Option<Duration>,
3730 #[serde(with = "option_duration_millis")]
3732 pub select: Option<Duration>,
3733 #[serde(with = "option_duration_millis")]
3735 pub sync_up: Option<Duration>,
3736 #[serde(with = "option_duration_millis")]
3738 pub exec: Option<Duration>,
3739 #[serde(with = "option_duration_millis")]
3741 pub sync_down: Option<Duration>,
3742 #[serde(with = "option_duration_millis")]
3744 pub cleanup: Option<Duration>,
3745 #[serde(with = "option_duration_millis")]
3747 pub total: Option<Duration>,
3748}
3749
3750mod duration_millis {
3751 use serde::{Deserialize, Deserializer, Serialize, Serializer};
3752 use std::time::Duration;
3753
3754 pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
3755 where
3756 S: Serializer,
3757 {
3758 duration.as_millis().serialize(serializer)
3759 }
3760
3761 pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
3762 where
3763 D: Deserializer<'de>,
3764 {
3765 let millis = u64::deserialize(deserializer)?;
3766 Ok(Duration::from_millis(millis))
3767 }
3768}
3769
3770#[derive(Debug, Clone, Serialize, Deserialize)]
3772pub struct CompilationMetrics {
3773 pub project_id: String,
3775 pub worker_id: String,
3777 pub timestamp: chrono::DateTime<chrono::Utc>,
3779 pub timing: CompilationTimingBreakdown,
3781 #[serde(with = "option_duration_millis")]
3783 pub local_build_time: Option<Duration>,
3784 pub speedup: Option<f64>,
3786 pub files_synced: u64,
3788 pub bytes_transferred: u64,
3790 pub exit_code: i32,
3792 pub success: bool,
3794}
3795
3796mod option_duration_millis {
3797 use serde::{Deserialize, Deserializer, Serialize, Serializer};
3798 use std::time::Duration;
3799
3800 pub fn serialize<S>(duration: &Option<Duration>, serializer: S) -> Result<S::Ok, S::Error>
3801 where
3802 S: Serializer,
3803 {
3804 match duration {
3805 Some(d) => Some(d.as_millis() as u64).serialize(serializer),
3806 None => Option::<u64>::None.serialize(serializer),
3807 }
3808 }
3809
3810 pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Duration>, D::Error>
3811 where
3812 D: Deserializer<'de>,
3813 {
3814 let opt: Option<u64> = Option::deserialize(deserializer)?;
3815 Ok(opt.map(Duration::from_millis))
3816 }
3817}
3818
3819impl CompilationMetrics {
3820 pub fn calculate_speedup(&mut self) {
3822 if let Some(local) = self.local_build_time
3823 && self.timing.total.as_millis() > 0
3824 {
3825 self.speedup = Some(local.as_secs_f64() / self.timing.total.as_secs_f64());
3826 }
3827 }
3828
3829 pub fn is_beneficial(&self) -> bool {
3831 self.speedup.map(|s| s > 1.0).unwrap_or(false)
3832 }
3833}
3834
3835impl Default for CompilationMetrics {
3836 fn default() -> Self {
3837 Self {
3838 project_id: String::new(),
3839 worker_id: String::new(),
3840 timestamp: chrono::Utc::now(),
3841 timing: CompilationTimingBreakdown::default(),
3842 local_build_time: None,
3843 speedup: None,
3844 files_synced: 0,
3845 bytes_transferred: 0,
3846 exit_code: 0,
3847 success: true,
3848 }
3849 }
3850}
3851
3852#[derive(Debug)]
3868pub struct CompilationTimer {
3869 project_id: String,
3870 worker_id: String,
3871 start: Instant,
3872 phase_start: Instant,
3873 rsync_up: Option<Duration>,
3874 remote_build: Option<Duration>,
3875 rsync_down: Option<Duration>,
3876}
3877
3878impl CompilationTimer {
3879 pub fn new(project_id: &str, worker_id: &str) -> Self {
3881 let now = Instant::now();
3882 Self {
3883 project_id: project_id.to_string(),
3884 worker_id: worker_id.to_string(),
3885 start: now,
3886 phase_start: now,
3887 rsync_up: None,
3888 remote_build: None,
3889 rsync_down: None,
3890 }
3891 }
3892
3893 pub fn end_rsync_up(&mut self) {
3895 self.rsync_up = Some(self.phase_start.elapsed());
3896 self.phase_start = Instant::now();
3897 tracing::info!(
3898 rsync_up_ms = %self.rsync_up.unwrap().as_millis(),
3899 "TIMING: rsync_up completed"
3900 );
3901 }
3902
3903 pub fn end_remote_build(&mut self) {
3905 self.remote_build = Some(self.phase_start.elapsed());
3906 self.phase_start = Instant::now();
3907 tracing::info!(
3908 remote_build_ms = %self.remote_build.unwrap().as_millis(),
3909 "TIMING: remote_build completed"
3910 );
3911 }
3912
3913 pub fn end_rsync_down(&mut self) {
3915 self.rsync_down = Some(self.phase_start.elapsed());
3916 tracing::info!(
3917 rsync_down_ms = %self.rsync_down.unwrap().as_millis(),
3918 "TIMING: rsync_down completed"
3919 );
3920 }
3921
3922 pub fn finish(self, exit_code: i32, files: u64, bytes: u64) -> CompilationMetrics {
3924 let total = self.start.elapsed();
3925 tracing::info!(
3926 total_ms = %total.as_millis(),
3927 exit_code = %exit_code,
3928 "TIMING: compilation completed"
3929 );
3930
3931 CompilationMetrics {
3932 project_id: self.project_id,
3933 worker_id: self.worker_id,
3934 timestamp: chrono::Utc::now(),
3935 timing: CompilationTimingBreakdown {
3936 rsync_up: self.rsync_up.unwrap_or_default(),
3937 remote_build: self.remote_build.unwrap_or_default(),
3938 rsync_down: self.rsync_down.unwrap_or_default(),
3939 total,
3940 },
3941 local_build_time: None,
3942 speedup: None,
3943 files_synced: files,
3944 bytes_transferred: bytes,
3945 exit_code,
3946 success: exit_code == 0,
3947 }
3948 }
3949}
3950
3951#[derive(Debug)]
3953pub struct MetricsAggregator {
3954 history: VecDeque<CompilationMetrics>,
3955 max_history: usize,
3956}
3957
3958impl MetricsAggregator {
3959 pub fn new(max_history: usize) -> Self {
3961 Self {
3962 history: VecDeque::with_capacity(max_history),
3963 max_history,
3964 }
3965 }
3966
3967 pub fn record(&mut self, metrics: CompilationMetrics) {
3969 if self.history.len() >= self.max_history {
3970 self.history.pop_front();
3971 }
3972 self.history.push_back(metrics);
3973 }
3974
3975 pub fn average_speedup(&self) -> Option<f64> {
3977 let speedups: Vec<f64> = self.history.iter().filter_map(|m| m.speedup).collect();
3978
3979 if speedups.is_empty() {
3980 None
3981 } else {
3982 Some(speedups.iter().sum::<f64>() / speedups.len() as f64)
3983 }
3984 }
3985
3986 pub fn p50_total_time(&self) -> Option<Duration> {
3988 self.percentile_total_time(0.50)
3989 }
3990
3991 pub fn p95_total_time(&self) -> Option<Duration> {
3993 self.percentile_total_time(0.95)
3994 }
3995
3996 pub fn p99_total_time(&self) -> Option<Duration> {
3998 self.percentile_total_time(0.99)
3999 }
4000
4001 fn percentile_total_time(&self, percentile: f64) -> Option<Duration> {
4006 let mut times: Vec<_> = self.history.iter().map(|m| m.timing.total).collect();
4007
4008 if times.is_empty() {
4009 return None;
4010 }
4011
4012 times.sort();
4013 let idx = (((times.len() - 1) as f64 * percentile).round() as usize).min(times.len() - 1);
4017 Some(times[idx])
4018 }
4019
4020 pub fn success_rate(&self) -> f64 {
4022 if self.history.is_empty() {
4023 return 100.0;
4024 }
4025 let successes = self.history.iter().filter(|m| m.success).count();
4026 (successes as f64 / self.history.len() as f64) * 100.0
4027 }
4028
4029 pub fn count(&self) -> usize {
4031 self.history.len()
4032 }
4033
4034 pub fn metrics(&self) -> &VecDeque<CompilationMetrics> {
4036 &self.history
4037 }
4038
4039 pub fn clear(&mut self) {
4041 self.history.clear();
4042 }
4043}
4044
4045impl Default for MetricsAggregator {
4046 fn default() -> Self {
4047 Self::new(1000)
4048 }
4049}
4050
4051#[cfg(test)]
4052mod tests {
4053 use super::*;
4054 use crate::test_guard;
4055
4056 #[test]
4057 fn test_declared_os_reads_reserved_tag() {
4058 assert_eq!(
4059 declared_os(&["rust".to_string(), "os:windows".to_string()]),
4060 Some("windows".to_string())
4061 );
4062 assert_eq!(
4063 declared_os(&["os:WINDOWS".to_string()]),
4064 Some("windows".to_string())
4065 );
4066 }
4067
4068 #[test]
4069 fn test_declared_os_absent_for_ordinary_tags() {
4070 assert_eq!(declared_os(&[]), None);
4073 assert_eq!(
4074 declared_os(&["rust".to_string(), "bun".to_string(), "go".to_string()]),
4075 None
4076 );
4077 assert_eq!(declared_os(&["os:".to_string()]), None);
4079 assert_eq!(declared_os(&["os: ".to_string()]), None);
4080 }
4081
4082 #[test]
4083 fn test_os_tag_roundtrips_through_declared_os() {
4084 for os in ["windows", "Darwin", " linux "] {
4085 let tag = os_tag(os);
4086 assert_eq!(
4087 declared_os(std::slice::from_ref(&tag)).as_deref(),
4088 Some(os.trim().to_ascii_lowercase().as_str())
4089 );
4090 }
4091 }
4092
4093 #[test]
4098 fn test_no_inline_secret_fields() {
4099 let endpoint = DoctorWebhookEndpoint {
4100 name: "ops".to_string(),
4101 url: "https://example.com/hook".to_string(),
4102 format: DoctorWebhookFormat::Pagerduty,
4103 on_transitions: vec!["any_to_failing".to_string()],
4104 retry_max: 3,
4105 retry_backoff_ms: 200,
4106 timeout_ms: 5000,
4107 enabled: true,
4108 routing_key_env: Some("RCH_PAGERDUTY_ROUTING_KEY".to_string()),
4109 bearer_token_env: Some("RCH_WEBHOOK_BEARER".to_string()),
4110 signing_secret_env: Some("RCH_WEBHOOK_HMAC".to_string()),
4111 };
4112 let value = serde_json::to_value(&endpoint).unwrap();
4115 let obj = value.as_object().expect("endpoint serializes to an object");
4116 for forbidden in ["secret", "routing_key", "bearer_token", "password", "token"] {
4117 assert!(
4118 !obj.contains_key(forbidden),
4119 "endpoint config must not expose an inline `{forbidden}` field"
4120 );
4121 }
4122 assert!(obj.contains_key("routing_key_env"));
4123 assert!(obj.contains_key("bearer_token_env"));
4124 assert!(obj.contains_key("signing_secret_env"));
4125 }
4126
4127 #[test]
4128 fn test_webhook_format_toml_spelling() {
4129 #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)]
4134 struct FormatWrap {
4135 format: DoctorWebhookFormat,
4136 }
4137 let rendered = toml::to_string(&FormatWrap {
4138 format: DoctorWebhookFormat::GenericJson,
4139 })
4140 .unwrap();
4141 assert!(
4142 rendered.contains("format = \"generic_json\""),
4143 "unexpected TOML: {rendered}"
4144 );
4145 let slack: FormatWrap = toml::from_str("format = \"slack\"").unwrap();
4146 assert_eq!(slack.format, DoctorWebhookFormat::Slack);
4147 let pd: FormatWrap = toml::from_str("format = \"pagerduty\"").unwrap();
4148 assert_eq!(pd.format, DoctorWebhookFormat::Pagerduty);
4149 }
4150
4151 #[test]
4152 fn test_doctor_webhooks_defaults() {
4153 let cfg = DoctorWebhooksConfig::default();
4154 assert!(cfg.endpoints.is_empty());
4155 assert_eq!(cfg.max_inflight, 16);
4156 let rch: RchConfig = toml::from_str("").unwrap();
4159 assert!(rch.doctor.webhooks.endpoints.is_empty());
4160 }
4161
4162 #[test]
4163 fn test_doctor_webhook_endpoint_parses_from_toml() {
4164 let toml_src = r#"
4165[doctor.webhooks]
4166max_inflight = 8
4167
4168[[doctor.webhooks.endpoints]]
4169name = "ops-slack"
4170url = "https://hooks.slack.com/services/T0/B0/x"
4171format = "slack"
4172on_transitions = ["healthy_to_degraded", "any_to_failing"]
4173retry_max = 2
4174"#;
4175 let cfg: RchConfig = toml::from_str(toml_src).unwrap();
4176 assert_eq!(cfg.doctor.webhooks.max_inflight, 8);
4177 assert_eq!(cfg.doctor.webhooks.endpoints.len(), 1);
4178 let ep = &cfg.doctor.webhooks.endpoints[0];
4179 assert_eq!(ep.name, "ops-slack");
4180 assert_eq!(ep.format, DoctorWebhookFormat::Slack);
4181 assert_eq!(ep.on_transitions.len(), 2);
4182 assert_eq!(ep.retry_max, 2);
4183 assert_eq!(ep.retry_backoff_ms, 200);
4185 assert_eq!(ep.timeout_ms, 5000);
4186 assert!(ep.enabled);
4187 }
4188
4189 fn assert_golden_json(
4190 actual: serde_json::Value,
4191 expected: &serde_json::Value,
4192 fixture_path: &str,
4193 ) {
4194 if &actual == expected {
4195 return;
4196 }
4197
4198 let actual_json = serde_json::to_string_pretty(&actual).expect("actual JSON renders");
4199 let expected_json = serde_json::to_string_pretty(expected).expect("expected JSON renders");
4200 assert_eq!(
4201 &actual,
4202 expected,
4203 "golden mismatch in {fixture_path}\n\
4204 expected_blake3={}\n\
4205 actual_blake3={}\n\
4206 bless path: review the semantic diff, update the fixture expected_* field, \
4207 and record both hashes in the Beads closeout.\n\
4208 expected:\n{expected_json}\n\
4209 actual:\n{actual_json}",
4210 blake3::hash(expected_json.as_bytes()),
4211 blake3::hash(actual_json.as_bytes())
4212 );
4213 }
4214
4215 #[test]
4216 fn test_circuit_state_default() {
4217 let _guard = test_guard!();
4218 assert_eq!(CircuitState::default(), CircuitState::Closed);
4219 }
4220
4221 #[test]
4222 fn test_circuit_config_defaults() {
4223 let _guard = test_guard!();
4224 let config = CircuitBreakerConfig::default();
4225 assert_eq!(config.failure_threshold, 3);
4226 assert_eq!(config.success_threshold, 2);
4227 assert_eq!(config.error_rate_threshold, 0.5);
4228 assert_eq!(config.window_secs, 60);
4229 assert_eq!(config.open_cooldown_secs, 30);
4230 assert_eq!(config.half_open_max_probes, 1);
4231 }
4232
4233 #[test]
4234 fn test_rch_config_has_circuit_defaults() {
4235 let _guard = test_guard!();
4236 let config = RchConfig::default();
4237 assert_eq!(config.circuit.failure_threshold, 3);
4238 }
4239
4240 #[test]
4241 fn test_circuit_config_serde_roundtrip() {
4242 let _guard = test_guard!();
4243 let config = CircuitBreakerConfig {
4244 failure_threshold: 5,
4245 success_threshold: 3,
4246 error_rate_threshold: 0.75,
4247 window_secs: 120,
4248 open_cooldown_secs: 45,
4249 half_open_max_probes: 2,
4250 command_failure_threshold: 4,
4251 open_cooldown_max_secs: 900,
4252 };
4253 let json = serde_json::to_string(&config).unwrap();
4254 let parsed: CircuitBreakerConfig = serde_json::from_str(&json).unwrap();
4255 assert_eq!(parsed.command_failure_threshold, 4);
4256 assert_eq!(parsed.open_cooldown_max_secs, 900);
4257 assert_eq!(parsed.failure_threshold, 5);
4258 assert_eq!(parsed.success_threshold, 3);
4259 assert_eq!(parsed.error_rate_threshold, 0.75);
4260 assert_eq!(parsed.window_secs, 120);
4261 assert_eq!(parsed.open_cooldown_secs, 45);
4262 assert_eq!(parsed.half_open_max_probes, 2);
4263 }
4264
4265 #[test]
4270 fn test_worker_capabilities_default() {
4271 let _guard = test_guard!();
4272 let caps = WorkerCapabilities::default();
4273 assert!(caps.rustc_version.is_none());
4274 assert!(caps.rustup_toolchains.is_empty());
4275 assert!(caps.rustup_components.is_empty());
4276 assert!(caps.bun_version.is_none());
4277 assert!(caps.node_version.is_none());
4278 assert!(caps.npm_version.is_none());
4279 assert!(caps.projects_root_ok.is_none());
4280 assert!(caps.projects_root_issue.is_none());
4281 assert!(caps.projects_root_checked_at_unix_ms.is_none());
4282 }
4283
4284 #[test]
4285 fn test_worker_capabilities_new() {
4286 let _guard = test_guard!();
4287 let caps = WorkerCapabilities::new();
4288 assert!(caps.rustc_version.is_none());
4289 assert!(!caps.has_rust());
4290 assert!(!caps.has_bun());
4291 assert!(!caps.has_node());
4292 assert!(caps.is_topology_healthy().is_none());
4293 }
4294
4295 #[test]
4296 fn test_worker_capabilities_has_rust() {
4297 let _guard = test_guard!();
4298 let mut caps = WorkerCapabilities::new();
4299 assert!(!caps.has_rust());
4300
4301 caps.rustc_version = Some("rustc 1.76.0 (07dca489a 2024-02-04)".to_string());
4302 assert!(caps.has_rust());
4303 }
4304
4305 #[test]
4306 fn test_worker_capabilities_components_are_toolchain_scoped_and_exact() {
4307 let _guard = test_guard!();
4308 let caps = WorkerCapabilities {
4309 rustup_components: vec![
4310 "stable-x86_64-unknown-linux-gnu:clippy".to_string(),
4311 "nightly-2026-07-05-x86_64-unknown-linux-gnu:rustfmt".to_string(),
4312 "beta-x86_64-unknown-linux-gnu:clippy-preview".to_string(),
4313 ],
4314 ..Default::default()
4315 };
4316
4317 assert!(caps.has_rustup_component("stable", "clippy"));
4318 assert!(caps.has_rustup_component("nightly-2026-07-05", "rustfmt"));
4319 assert!(!caps.has_rustup_component("nightly-2026-07-05", "clippy"));
4320 assert!(!caps.has_rustup_component("beta", "clippy"));
4321 assert!(!caps.has_rustup_component("nightly-2026-07-0", "rustfmt"));
4322 }
4323
4324 #[test]
4325 fn test_worker_capabilities_has_bun() {
4326 let _guard = test_guard!();
4327 let mut caps = WorkerCapabilities::new();
4328 assert!(!caps.has_bun());
4329
4330 caps.bun_version = Some("1.0.25".to_string());
4331 assert!(caps.has_bun());
4332 }
4333
4334 #[test]
4335 fn test_worker_capabilities_has_nix() {
4336 let _guard = test_guard!();
4337 let mut caps = WorkerCapabilities::new();
4338 assert!(!caps.has_nix());
4339
4340 caps.nix_version = Some("nix (Nix) 2.24.9".to_string());
4341 assert!(caps.has_nix());
4342 }
4343
4344 #[test]
4345 fn test_worker_capabilities_has_zig_requires_both_halves() {
4346 let _guard = test_guard!();
4347 let mut caps = WorkerCapabilities::new();
4348 assert!(!caps.has_zig());
4349
4350 caps.zig_version = Some("0.14.1".to_string());
4353 assert!(!caps.has_zig());
4354
4355 caps.zig_version = None;
4357 caps.cargo_zigbuild_version = Some("cargo-zigbuild 0.23.0".to_string());
4358 assert!(!caps.has_zig());
4359
4360 caps.zig_version = Some("0.14.1".to_string());
4361 assert!(caps.has_zig());
4362 }
4363
4364 #[test]
4365 fn test_nix_build_uses_generous_timeout() {
4366 let _guard = test_guard!();
4367 let cfg = CompilationConfig::default();
4368 assert_eq!(
4370 cfg.timeout_for_kind(Some(CompilationKind::NixBuild)),
4371 std::time::Duration::from_secs(cfg.test_timeout_sec)
4372 );
4373 }
4374
4375 #[test]
4376 fn test_job_uses_build_timeout_bucket() {
4377 let _guard = test_guard!();
4378 let cfg = CompilationConfig::default();
4381 assert_eq!(
4382 cfg.timeout_for_kind(Some(CompilationKind::Job)),
4383 std::time::Duration::from_secs(cfg.build_timeout_sec)
4384 );
4385 }
4386
4387 #[test]
4388 fn test_layer0_disabled_by_default_and_env_gated() {
4389 let _guard = test_guard!();
4390 let cfg = Layer0Config::default();
4391 assert!(!cfg.enabled);
4392 let knobs_on = Layer0Config {
4394 enabled: false,
4395 release_lto_thin: true,
4396 release_codegen_units_1: true,
4397 };
4398 assert!(cfg.active_env().is_empty());
4399 assert!(knobs_on.active_env().is_empty());
4400 }
4401
4402 #[test]
4403 fn test_layer0_knobs_emit_stable_env_pairs() {
4404 let _guard = test_guard!();
4405 let lto_only = Layer0Config {
4406 enabled: true,
4407 release_lto_thin: true,
4408 release_codegen_units_1: false,
4409 };
4410 assert_eq!(
4411 lto_only.active_env(),
4412 vec![("CARGO_PROFILE_RELEASE_LTO", "thin")]
4413 );
4414 let both = Layer0Config {
4415 enabled: true,
4416 release_lto_thin: true,
4417 release_codegen_units_1: true,
4418 };
4419 assert_eq!(
4420 both.active_env(),
4421 vec![
4422 ("CARGO_PROFILE_RELEASE_LTO", "thin"),
4423 ("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"),
4424 ],
4425 "emission order must stay stable for deterministic baselines"
4426 );
4427 }
4428
4429 #[test]
4430 fn test_layer0_pack_version_is_recorded() {
4431 let _guard = test_guard!();
4432 assert!(!LAYER0_PACK_VERSION.is_empty());
4434 }
4435 #[test]
4436 fn test_default_allowlist_includes_nix() {
4437 let _guard = test_guard!();
4438 let cfg = ExecutionConfig::default();
4439 assert!(cfg.is_allowed("nix"));
4440 }
4441
4442 #[test]
4443 fn test_worker_capabilities_has_node() {
4444 let _guard = test_guard!();
4445 let mut caps = WorkerCapabilities::new();
4446 assert!(!caps.has_node());
4447
4448 caps.node_version = Some("v20.11.0".to_string());
4449 assert!(caps.has_node());
4450 }
4451
4452 #[test]
4453 fn test_worker_capabilities_multiple_runtimes() {
4454 let _guard = test_guard!();
4455 let caps = WorkerCapabilities {
4456 rustc_version: Some("rustc 1.76.0".to_string()),
4457 bun_version: Some("1.0.25".to_string()),
4458 node_version: Some("v20.11.0".to_string()),
4459 npm_version: Some("10.2.4".to_string()),
4460 ..Default::default()
4461 };
4462
4463 assert!(caps.has_rust());
4464 assert!(caps.has_bun());
4465 assert!(caps.has_node());
4466 }
4467
4468 #[test]
4469 fn test_worker_capabilities_serialization_empty() {
4470 let _guard = test_guard!();
4471 let caps = WorkerCapabilities::new();
4472 let json = serde_json::to_string(&caps).unwrap();
4473 assert_eq!(json, "{}");
4475
4476 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4478 assert!(!parsed.has_rust());
4479 assert!(!parsed.has_bun());
4480 assert!(!parsed.has_node());
4481 }
4482
4483 #[test]
4484 fn test_worker_capabilities_serialization_with_versions() {
4485 let _guard = test_guard!();
4486 let caps = WorkerCapabilities {
4487 rustc_version: Some("rustc 1.76.0-nightly".to_string()),
4488 bun_version: Some("1.0.25".to_string()),
4489 node_version: None,
4490 npm_version: None,
4491 ..Default::default()
4492 };
4493
4494 let json = serde_json::to_string(&caps).unwrap();
4495 assert!(json.contains("rustc_version"));
4496 assert!(json.contains("rustc 1.76.0-nightly"));
4497 assert!(json.contains("bun_version"));
4498 assert!(json.contains("1.0.25"));
4499 assert!(!json.contains("node_version"));
4501 assert!(!json.contains("npm_version"));
4502
4503 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4504 assert!(parsed.has_rust());
4505 assert!(parsed.has_bun());
4506 assert!(!parsed.has_node());
4507 }
4508
4509 #[test]
4510 fn test_worker_capabilities_deserialization_partial() {
4511 let _guard = test_guard!();
4512 let json = r#"{"bun_version": "1.0.0"}"#;
4514 let caps: WorkerCapabilities = serde_json::from_str(json).unwrap();
4515
4516 assert!(!caps.has_rust());
4517 assert!(caps.has_bun());
4518 assert!(!caps.has_node());
4519 assert_eq!(caps.bun_version, Some("1.0.0".to_string()));
4520 }
4521
4522 #[test]
4523 fn test_worker_capabilities_clone() {
4524 let _guard = test_guard!();
4525 let caps = WorkerCapabilities {
4526 rustc_version: Some("1.76.0".to_string()),
4527 bun_version: None,
4528 node_version: Some("v20".to_string()),
4529 npm_version: None,
4530 projects_root_ok: Some(false),
4531 projects_root_issue: Some("alias_missing".to_string()),
4532 projects_root_checked_at_unix_ms: Some(123),
4533 ..Default::default()
4534 };
4535
4536 let cloned = caps.clone();
4537 assert_eq!(cloned.rustc_version, caps.rustc_version);
4538 assert_eq!(cloned.bun_version, caps.bun_version);
4539 assert_eq!(cloned.node_version, caps.node_version);
4540 assert_eq!(cloned.npm_version, caps.npm_version);
4541 assert_eq!(cloned.projects_root_ok, caps.projects_root_ok);
4542 assert_eq!(cloned.projects_root_issue, caps.projects_root_issue);
4543 assert_eq!(
4544 cloned.projects_root_checked_at_unix_ms,
4545 caps.projects_root_checked_at_unix_ms
4546 );
4547 }
4548
4549 #[test]
4550 fn test_worker_capabilities_topology_health_helper() {
4551 let _guard = test_guard!();
4552 let mut caps = WorkerCapabilities::new();
4553 assert_eq!(caps.is_topology_healthy(), None);
4554
4555 caps.projects_root_ok = Some(true);
4556 assert_eq!(caps.is_topology_healthy(), Some(true));
4557
4558 caps.projects_root_ok = Some(false);
4559 assert_eq!(caps.is_topology_healthy(), Some(false));
4560 }
4561
4562 #[test]
4563 fn test_selection_reason_serialization() {
4564 let _guard = test_guard!();
4565 assert_eq!(
4567 serde_json::to_string(&SelectionReason::Success).unwrap(),
4568 "\"success\""
4569 );
4570 assert_eq!(
4571 serde_json::to_string(&SelectionReason::NoWorkersConfigured).unwrap(),
4572 "\"no_workers_configured\""
4573 );
4574 assert_eq!(
4575 serde_json::to_string(&SelectionReason::AllWorkersUnreachable).unwrap(),
4576 "\"all_workers_unreachable\""
4577 );
4578 assert_eq!(
4579 serde_json::to_string(&SelectionReason::AllCircuitsOpen).unwrap(),
4580 "\"all_circuits_open\""
4581 );
4582 assert_eq!(
4583 serde_json::to_string(&SelectionReason::AllWorkersBusy).unwrap(),
4584 "\"all_workers_busy\""
4585 );
4586 assert_eq!(
4587 serde_json::to_string(&SelectionReason::NoWorkersPassedHealth).unwrap(),
4588 "\"no_workers_passed_health\""
4589 );
4590 assert_eq!(
4591 serde_json::to_string(&SelectionReason::AllWorkersFailedPreflight).unwrap(),
4592 "\"all_workers_failed_preflight\""
4593 );
4594 assert_eq!(
4595 serde_json::to_string(&SelectionReason::AllWorkersFailedConvergence).unwrap(),
4596 "\"all_workers_failed_convergence\""
4597 );
4598 assert_eq!(
4599 serde_json::to_string(&SelectionReason::NoAdmissibleWorkers(
4600 "critical_pressure=1,insufficient_slots=1".to_string()
4601 ))
4602 .unwrap(),
4603 "{\"no_admissible_workers\":\"critical_pressure=1,insufficient_slots=1\"}"
4604 );
4605 assert_eq!(
4606 serde_json::to_string(&SelectionReason::NoMatchingWorkers).unwrap(),
4607 "\"no_matching_workers\""
4608 );
4609 }
4610
4611 #[test]
4612 fn test_selection_reason_with_error() {
4613 let _guard = test_guard!();
4614 let reason = SelectionReason::SelectionError("test error".to_string());
4615 let json = serde_json::to_string(&reason).unwrap();
4616 assert!(json.contains("selection_error"));
4617 assert!(json.contains("test error"));
4618 }
4619
4620 #[test]
4621 fn test_selection_reason_deserialization() {
4622 let _guard = test_guard!();
4623 assert_eq!(
4624 serde_json::from_str::<SelectionReason>("\"success\"").unwrap(),
4625 SelectionReason::Success
4626 );
4627 assert_eq!(
4628 serde_json::from_str::<SelectionReason>("\"all_workers_busy\"").unwrap(),
4629 SelectionReason::AllWorkersBusy
4630 );
4631 assert_eq!(
4632 serde_json::from_str::<SelectionReason>("\"no_workers_passed_health\"").unwrap(),
4633 SelectionReason::NoWorkersPassedHealth
4634 );
4635 assert_eq!(
4636 serde_json::from_str::<SelectionReason>("\"all_workers_failed_preflight\"").unwrap(),
4637 SelectionReason::AllWorkersFailedPreflight
4638 );
4639 assert_eq!(
4640 serde_json::from_str::<SelectionReason>("\"all_workers_failed_convergence\"").unwrap(),
4641 SelectionReason::AllWorkersFailedConvergence
4642 );
4643 assert_eq!(
4644 serde_json::from_str::<SelectionReason>(
4645 "{\"no_admissible_workers\":\"critical_pressure=1\"}"
4646 )
4647 .unwrap(),
4648 SelectionReason::NoAdmissibleWorkers("critical_pressure=1".to_string())
4649 );
4650 }
4651
4652 #[test]
4653 fn test_selection_reason_display() {
4654 let _guard = test_guard!();
4655 assert_eq!(
4656 SelectionReason::Success.to_string(),
4657 "worker assigned successfully"
4658 );
4659 assert_eq!(
4660 SelectionReason::NoWorkersConfigured.to_string(),
4661 "no workers configured"
4662 );
4663 assert_eq!(
4664 SelectionReason::AllWorkersUnreachable.to_string(),
4665 "all workers unreachable"
4666 );
4667 assert_eq!(
4668 SelectionReason::AllWorkersBusy.to_string(),
4669 "all workers at capacity"
4670 );
4671 assert_eq!(
4672 SelectionReason::NoWorkersPassedHealth.to_string(),
4673 "no workers passed health thresholds"
4674 );
4675 assert_eq!(
4676 SelectionReason::AllWorkersFailedPreflight.to_string(),
4677 "all workers failed preflight checks"
4678 );
4679 assert_eq!(
4680 SelectionReason::AllWorkersFailedConvergence.to_string(),
4681 "all workers failed repo convergence checks"
4682 );
4683 assert_eq!(
4684 SelectionReason::NoAdmissibleWorkers(
4685 "critical_pressure=1,insufficient_slots=1".to_string()
4686 )
4687 .to_string(),
4688 "no admissible workers: critical_pressure=1,insufficient_slots=1"
4689 );
4690 assert_eq!(
4691 SelectionReason::SelectionError("oops".to_string()).to_string(),
4692 "selection error: oops"
4693 );
4694 }
4695
4696 #[test]
4697 fn test_selection_response_with_worker() {
4698 let _guard = test_guard!();
4699 let response = SelectionResponse {
4700 worker: Some(SelectedWorker {
4701 id: WorkerId::new("test"),
4702 host: "localhost".to_string(),
4703 user: "user".to_string(),
4704 identity_file: "~/.ssh/id_rsa".to_string(),
4705 slots_available: 8,
4706 speed_score: 75.0,
4707 declared_os: None,
4708 }),
4709 reason: SelectionReason::Success,
4710 build_id: None,
4711 diagnostics: None,
4712 };
4713
4714 let json = serde_json::to_string(&response).unwrap();
4715 assert!(json.contains("\"reason\":\"success\""));
4716 assert!(json.contains("\"id\":\"test\""));
4717 }
4718
4719 #[test]
4720 fn test_selection_response_without_worker() {
4721 let _guard = test_guard!();
4722 let response = SelectionResponse {
4723 worker: None,
4724 reason: SelectionReason::AllWorkersBusy,
4725 build_id: None,
4726 diagnostics: None,
4727 };
4728
4729 let json = serde_json::to_string(&response).unwrap();
4730 assert!(json.contains("\"worker\":null"));
4731 assert!(json.contains("\"reason\":\"all_workers_busy\""));
4732 }
4733
4734 #[test]
4735 fn golden_selection_response_with_no_workers_passed_health_diagnostics() {
4736 let _guard = test_guard!();
4737 const FIXTURE_PATH: &str =
4738 "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json";
4739 let fixture: serde_json::Value = serde_json::from_str(include_str!(
4740 "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json"
4741 ))
4742 .expect("selection diagnostics golden fixture parses");
4743 assert_eq!(
4744 fixture["schema_version"],
4745 "rch.golden.selection_diagnostics.v1"
4746 );
4747
4748 let response = SelectionResponse {
4749 worker: None,
4750 reason: SelectionReason::NoWorkersPassedHealth,
4751 build_id: None,
4752 diagnostics: Some(SelectionDiagnostics {
4753 required_runtime: RequiredRuntime::Rust,
4754 estimated_cores: 2,
4755 min_success_rate: 0.8,
4756 fallback_min_success_rate: 0.5,
4757 active_project_exclusion_count: 0,
4758 workers: vec![WorkerSelectionDiagnostic {
4759 worker_id: WorkerId::new("fixture-worker"),
4760 status: "healthy".to_string(),
4761 circuit_state: "closed".to_string(),
4762 pressure_state: "healthy".to_string(),
4763 pressure_reason_code: "pressure.ok".to_string(),
4764 success_rate: Some(0.2),
4765 min_success_rate: 0.8,
4766 fallback_min_success_rate: 0.5,
4767 required_runtime: RequiredRuntime::Rust,
4768 runtime_available: true,
4769 available_slots: 8,
4770 total_slots: 8,
4771 estimated_cores: 2,
4772 active_project_excluded: false,
4773 final_decision: WorkerSelectionDiagnosticDecision::Deny,
4774 final_reason: "success_rate 0.20 < fallback 0.50".to_string(),
4775 reason_codes: vec![
4776 "health.below_min_success_rate".to_string(),
4777 "health.below_fallback_min_success_rate".to_string(),
4778 ],
4779 }],
4780 }),
4781 };
4782
4783 assert_golden_json(
4784 serde_json::to_value(&response).expect("response serializes"),
4785 &fixture["expected_response"],
4786 FIXTURE_PATH,
4787 );
4788 }
4789
4790 #[test]
4791 fn test_selection_response_roundtrip() {
4792 let _guard = test_guard!();
4793 let original = SelectionResponse {
4794 worker: Some(SelectedWorker {
4795 id: WorkerId::new("worker-1"),
4796 host: "192.168.1.100".to_string(),
4797 user: "ubuntu".to_string(),
4798 identity_file: "/path/to/key".to_string(),
4799 slots_available: 16,
4800 speed_score: 90.5,
4801 declared_os: None,
4802 }),
4803 reason: SelectionReason::Success,
4804 build_id: None,
4805 diagnostics: None,
4806 };
4807
4808 let json = serde_json::to_string(&original).unwrap();
4809 let parsed: SelectionResponse = serde_json::from_str(&json).unwrap();
4810
4811 assert!(parsed.worker.is_some());
4812 let worker = parsed.worker.unwrap();
4813 assert_eq!(worker.id.as_str(), "worker-1");
4814 assert_eq!(worker.host, "192.168.1.100");
4815 assert_eq!(worker.slots_available, 16);
4816 assert_eq!(parsed.reason, SelectionReason::Success);
4817 }
4818
4819 #[test]
4820 fn test_build_heartbeat_phase_roundtrip() {
4821 let _guard = test_guard!();
4822 let json = serde_json::to_string(&BuildHeartbeatPhase::SyncUp).unwrap();
4823 assert_eq!(json, "\"sync_up\"");
4824 let parsed: BuildHeartbeatPhase = serde_json::from_str(&json).unwrap();
4825 assert_eq!(parsed, BuildHeartbeatPhase::SyncUp);
4826 }
4827
4828 #[test]
4829 fn test_build_heartbeat_request_roundtrip() {
4830 let _guard = test_guard!();
4831 let request = BuildHeartbeatRequest {
4832 build_id: 42,
4833 worker_id: WorkerId::new("worker-a"),
4834 hook_pid: Some(12345),
4835 local_wrapper_id: Some("wrapper-abc123".to_string()),
4836 remote_pgid_file: Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string()),
4837 phase: BuildHeartbeatPhase::Execute,
4838 detail: Some("Compiling crates".to_string()),
4839 progress_counter: Some(7),
4840 progress_percent: Some(42.5),
4841 };
4842
4843 let json = serde_json::to_string(&request).unwrap();
4844 let parsed: BuildHeartbeatRequest = serde_json::from_str(&json).unwrap();
4845 assert_eq!(parsed.build_id, 42);
4846 assert_eq!(parsed.worker_id.as_str(), "worker-a");
4847 assert_eq!(
4848 parsed.remote_pgid_file,
4849 Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string())
4850 );
4851 assert_eq!(parsed.phase, BuildHeartbeatPhase::Execute);
4852 assert_eq!(parsed.progress_counter, Some(7));
4853 assert_eq!(parsed.progress_percent, Some(42.5));
4854 }
4855
4856 #[test]
4859 fn test_circuit_stats_new() {
4860 let _guard = test_guard!();
4861 let stats = CircuitStats::new();
4862 assert_eq!(stats.state(), CircuitState::Closed);
4863 assert_eq!(stats.consecutive_failures(), 0);
4864 assert_eq!(stats.error_rate(), 0.0);
4865 assert!(stats.opened_at().is_none());
4866 }
4867
4868 #[test]
4869 fn test_circuit_stats_record_success() {
4870 let _guard = test_guard!();
4871 let mut stats = CircuitStats::new();
4872 stats.record_success();
4873 stats.record_success();
4874 assert_eq!(stats.error_rate(), 0.0);
4875 assert_eq!(stats.consecutive_failures(), 0);
4876 }
4877
4878 #[test]
4879 fn test_circuit_stats_record_failure() {
4880 let _guard = test_guard!();
4881 let mut stats = CircuitStats::new();
4882 stats.record_failure();
4883 stats.record_failure();
4884 assert_eq!(stats.consecutive_failures(), 2);
4885 assert_eq!(stats.error_rate(), 1.0); }
4887
4888 #[test]
4889 fn test_circuit_stats_error_rate() {
4890 let _guard = test_guard!();
4891 let mut stats = CircuitStats::new();
4892 stats.record_success();
4893 stats.record_success();
4894 stats.record_failure();
4895 assert!((stats.error_rate() - 0.333).abs() < 0.01);
4897 }
4898
4899 #[test]
4900 fn test_circuit_stats_should_open_consecutive_failures() {
4901 let _guard = test_guard!();
4902 let mut stats = CircuitStats::new();
4903 let config = CircuitBreakerConfig::default(); stats.record_failure();
4906 assert!(!stats.should_open(&config));
4907
4908 stats.record_failure();
4909 assert!(!stats.should_open(&config));
4910
4911 stats.record_failure();
4912 assert!(stats.should_open(&config)); }
4914
4915 #[test]
4916 fn test_circuit_stats_should_open_error_rate() {
4917 let _guard = test_guard!();
4918 let mut stats = CircuitStats::new();
4919 let config = CircuitBreakerConfig {
4920 error_rate_threshold: 0.5,
4921 ..Default::default()
4922 };
4923
4924 stats.record_success();
4926 stats.record_success();
4927 stats.record_failure();
4928 stats.record_failure();
4929 assert!(!stats.should_open(&config)); stats.record_failure(); assert!(stats.should_open(&config));
4933 }
4934
4935 #[test]
4936 fn test_circuit_close_clears_recent_results_no_flap() {
4937 let _guard = test_guard!();
4938 let mut stats = CircuitStats::new();
4942 let config = CircuitBreakerConfig {
4943 error_rate_threshold: 0.5,
4944 ..Default::default()
4945 };
4946 for _ in 0..6 {
4947 stats.record_failure();
4948 }
4949 assert!(stats.should_open(&config));
4950 stats.open();
4951 stats.half_open();
4953 stats.record_success();
4954 stats.record_success();
4955 stats.close();
4956 assert_eq!(stats.error_rate(), 0.0);
4958 assert!(!stats.should_open(&config));
4959 }
4960
4961 #[test]
4962 fn test_apply_health_outcome_opens_on_failure_threshold() {
4963 let _guard = test_guard!();
4964 let config = CircuitBreakerConfig {
4966 failure_threshold: 3,
4967 open_cooldown_secs: 300,
4968 ..Default::default()
4969 };
4970 let mut stats = CircuitStats::new();
4971 assert_eq!(
4972 stats.apply_health_outcome(false, false, &config),
4973 CircuitState::Closed
4974 );
4975 assert_eq!(
4976 stats.apply_health_outcome(false, false, &config),
4977 CircuitState::Closed
4978 );
4979 assert_eq!(
4980 stats.apply_health_outcome(false, false, &config),
4981 CircuitState::Open
4982 );
4983 }
4984
4985 #[test]
4986 fn test_apply_health_outcome_half_open_two_probe_recovery() {
4987 let _guard = test_guard!();
4988 let config = CircuitBreakerConfig {
4994 failure_threshold: 3,
4995 success_threshold: 2,
4996 open_cooldown_secs: 0,
4997 ..Default::default()
4998 };
4999 let mut stats = CircuitStats::new();
5000 for _ in 0..3 {
5001 stats.apply_health_outcome(false, false, &config);
5002 }
5003 assert_eq!(stats.state(), CircuitState::HalfOpen);
5005
5006 let after_first = stats.apply_health_outcome(true, false, &config);
5008 assert_eq!(after_first, CircuitState::HalfOpen);
5009
5010 let after_second = stats.apply_health_outcome(true, false, &config);
5012 assert_eq!(after_second, CircuitState::Closed);
5013 }
5014
5015 #[test]
5016 fn test_apply_health_outcome_transient_half_open_does_not_reopen() {
5017 let _guard = test_guard!();
5018 let config = CircuitBreakerConfig {
5021 failure_threshold: 3,
5022 success_threshold: 2,
5023 open_cooldown_secs: 0,
5024 ..Default::default()
5025 };
5026 let mut stats = CircuitStats::new();
5027 for _ in 0..3 {
5028 stats.apply_health_outcome(false, false, &config);
5029 }
5030 assert_eq!(stats.state(), CircuitState::HalfOpen);
5032
5033 assert_eq!(
5035 stats.apply_health_outcome(true, false, &config),
5036 CircuitState::HalfOpen
5037 );
5038
5039 let after_transient = stats.apply_health_outcome(false, true, &config);
5045 assert_eq!(after_transient, CircuitState::HalfOpen);
5046
5047 assert_eq!(
5049 stats.apply_health_outcome(true, false, &config),
5050 CircuitState::HalfOpen
5051 );
5052 assert_eq!(
5053 stats.apply_health_outcome(true, false, &config),
5054 CircuitState::Closed
5055 );
5056 }
5057
5058 #[test]
5059 fn test_apply_health_outcome_real_vs_transient_half_open_failure() {
5060 let _guard = test_guard!();
5061 let config = CircuitBreakerConfig {
5069 failure_threshold: 3,
5070 success_threshold: 2,
5071 open_cooldown_secs: 300,
5072 ..Default::default()
5073 };
5074
5075 let mut real = CircuitStats::new();
5077 real.open();
5078 real.half_open();
5079 assert_eq!(real.state(), CircuitState::HalfOpen);
5080 let after_real = real.apply_health_outcome(false, false, &config);
5081 assert_eq!(
5082 after_real,
5083 CircuitState::Open,
5084 "a real half-open failure must reopen the circuit"
5085 );
5086
5087 let mut transient = CircuitStats::new();
5089 transient.open();
5090 transient.half_open();
5091 assert_eq!(transient.state(), CircuitState::HalfOpen);
5092 let after_transient = transient.apply_health_outcome(false, true, &config);
5093 assert_eq!(
5094 after_transient,
5095 CircuitState::HalfOpen,
5096 "a transient half-open failure must NOT reopen the circuit"
5097 );
5098 }
5099
5100 #[test]
5105 fn test_command_failure_blocks_half_open_close_on_liveness_successes() {
5106 let _guard = test_guard!();
5107 let config = CircuitBreakerConfig {
5108 failure_threshold: 3,
5109 success_threshold: 2,
5110 open_cooldown_secs: 0,
5111 command_failure_threshold: 2,
5112 ..Default::default()
5113 };
5114 let mut stats = CircuitStats::new();
5115 stats.open();
5116 stats.half_open();
5117
5118 assert_eq!(
5120 stats.apply_command_outcome(false, &config),
5121 CircuitState::HalfOpen,
5122 "cooldown 0: reopen then immediately half-open again"
5123 );
5124 assert_eq!(stats.consecutive_command_failures(), 1);
5125
5126 for _ in 0..10 {
5128 assert_eq!(
5129 stats.apply_health_outcome(true, false, &config),
5130 CircuitState::HalfOpen,
5131 "liveness successes must not close the circuit while a command failure is outstanding"
5132 );
5133 }
5134
5135 assert_eq!(
5137 stats.apply_command_outcome(true, &config),
5138 CircuitState::Closed
5139 );
5140 assert_eq!(stats.consecutive_command_failures(), 0);
5141 assert_eq!(stats.reopen_streak(), 0);
5142 }
5143
5144 #[test]
5147 fn test_command_failures_open_closed_circuit_despite_liveness_successes() {
5148 let _guard = test_guard!();
5149 let config = CircuitBreakerConfig {
5150 failure_threshold: 3,
5151 command_failure_threshold: 2,
5152 open_cooldown_secs: 300,
5153 ..Default::default()
5154 };
5155 let mut stats = CircuitStats::new();
5156
5157 assert_eq!(
5158 stats.apply_command_outcome(false, &config),
5159 CircuitState::Closed
5160 );
5161 stats.apply_health_outcome(true, false, &config);
5163 stats.apply_health_outcome(true, false, &config);
5164 assert_eq!(stats.consecutive_failures(), 0);
5165 assert_eq!(
5167 stats.apply_command_outcome(false, &config),
5168 CircuitState::Open
5169 );
5170 assert_eq!(stats.reopen_streak(), 1);
5171 }
5172
5173 #[test]
5177 fn test_command_failure_in_half_open_always_reopens() {
5178 let _guard = test_guard!();
5179 let config = CircuitBreakerConfig {
5180 open_cooldown_secs: 300,
5181 ..Default::default()
5182 };
5183 let mut stats = CircuitStats::new();
5184 stats.open();
5185 stats.half_open();
5186 assert_eq!(
5187 stats.apply_command_outcome(false, &config),
5188 CircuitState::Open
5189 );
5190 assert_eq!(stats.reopen_streak(), 2);
5191 }
5192
5193 #[test]
5197 fn test_reopen_streak_backs_off_cooldown_until_cap() {
5198 let _guard = test_guard!();
5199 let config = CircuitBreakerConfig {
5200 open_cooldown_secs: 30,
5201 open_cooldown_max_secs: 100,
5202 ..Default::default()
5203 };
5204 assert_eq!(config.cooldown_ms_for_streak(0), 30_000);
5205 assert_eq!(config.cooldown_ms_for_streak(1), 30_000);
5206 assert_eq!(config.cooldown_ms_for_streak(2), 60_000);
5207 assert_eq!(config.cooldown_ms_for_streak(3), 100_000, "capped");
5208 assert_eq!(config.cooldown_ms_for_streak(40), 100_000, "no overflow");
5209
5210 let inverted = CircuitBreakerConfig {
5212 open_cooldown_secs: 30,
5213 open_cooldown_max_secs: 5,
5214 ..Default::default()
5215 };
5216 assert_eq!(inverted.cooldown_ms_for_streak(3), 30_000);
5217
5218 let mut stats = CircuitStats::new();
5221 stats.open();
5222 stats.half_open();
5223 stats.open();
5224 assert_eq!(stats.reopen_streak(), 2);
5225 let remaining = stats.recovery_remaining_secs(&config).unwrap();
5226 assert!(
5227 remaining > 30 && remaining <= 60,
5228 "second open should wait ~60s, got {remaining}"
5229 );
5230
5231 stats.close();
5233 assert_eq!(stats.reopen_streak(), 0);
5234 stats.open();
5235 assert!(stats.recovery_remaining_secs(&config).unwrap() <= 30);
5236 }
5237
5238 #[test]
5239 fn test_circuit_stats_success_resets_consecutive_failures() {
5240 let _guard = test_guard!();
5241 let mut stats = CircuitStats::new();
5242 stats.record_failure();
5243 stats.record_failure();
5244 assert_eq!(stats.consecutive_failures(), 2);
5245
5246 stats.record_success();
5247 assert_eq!(stats.consecutive_failures(), 0);
5248 }
5249
5250 #[test]
5251 fn test_circuit_stats_open_transition() {
5252 let _guard = test_guard!();
5253 let mut stats = CircuitStats::new();
5254 stats.open();
5255
5256 assert_eq!(stats.state(), CircuitState::Open);
5257 assert!(stats.opened_at().is_some());
5258 }
5259
5260 #[test]
5261 fn test_circuit_stats_half_open_transition() {
5262 let _guard = test_guard!();
5263 let mut stats = CircuitStats::new();
5264 stats.open();
5265 stats.half_open();
5266
5267 assert_eq!(stats.state(), CircuitState::HalfOpen);
5268 }
5269
5270 #[test]
5271 fn test_circuit_stats_close_transition() {
5272 let _guard = test_guard!();
5273 let mut stats = CircuitStats::new();
5274 stats.open();
5275 stats.half_open();
5276 stats.close();
5277
5278 assert_eq!(stats.state(), CircuitState::Closed);
5279 assert!(stats.opened_at().is_none());
5280 assert_eq!(stats.consecutive_failures(), 0);
5281 }
5282
5283 #[test]
5284 fn test_circuit_stats_should_close() {
5285 let _guard = test_guard!();
5286 let mut stats = CircuitStats::new();
5287 let config = CircuitBreakerConfig {
5288 success_threshold: 2,
5289 ..Default::default()
5290 };
5291
5292 stats.open();
5293 stats.half_open();
5294
5295 assert!(!stats.should_close(&config)); stats.record_success();
5298 assert!(!stats.should_close(&config)); stats.record_success();
5301 assert!(stats.should_close(&config)); }
5303
5304 #[test]
5305 fn test_circuit_stats_can_probe() {
5306 let _guard = test_guard!();
5307 let mut stats = CircuitStats::new();
5308 let config = CircuitBreakerConfig {
5309 half_open_max_probes: 1,
5310 ..Default::default()
5311 };
5312
5313 assert!(!stats.can_probe(&config));
5315
5316 stats.open();
5317 assert!(!stats.can_probe(&config));
5319
5320 stats.half_open();
5321 assert!(stats.can_probe(&config));
5323
5324 assert!(stats.start_probe(&config));
5326 assert!(!stats.can_probe(&config));
5328 assert!(!stats.start_probe(&config));
5329 }
5330
5331 #[test]
5332 fn test_circuit_stats_probe_completion() {
5333 let _guard = test_guard!();
5334 let mut stats = CircuitStats::new();
5335 let config = CircuitBreakerConfig {
5336 half_open_max_probes: 1,
5337 ..Default::default()
5338 };
5339
5340 stats.open();
5341 stats.half_open();
5342 stats.start_probe(&config);
5343
5344 stats.record_success();
5346
5347 assert!(stats.can_probe(&config));
5349 }
5350
5351 #[test]
5352 fn test_circuit_stats_failure_in_half_open() {
5353 let _guard = test_guard!();
5354 let mut stats = CircuitStats::new();
5355 let config = CircuitBreakerConfig {
5356 success_threshold: 2,
5357 ..Default::default()
5358 };
5359
5360 stats.open();
5361 stats.half_open();
5362
5363 stats.record_success();
5364 assert_eq!(stats.consecutive_failures(), 0);
5365
5366 stats.record_failure();
5368 assert!(!stats.should_close(&config));
5369
5370 stats.record_success();
5372 stats.record_success();
5373 assert!(stats.should_close(&config));
5374 }
5375
5376 #[test]
5377 fn test_circuit_stats_reset_window() {
5378 let _guard = test_guard!();
5379 let mut stats = CircuitStats::new();
5380 stats.record_success();
5381 stats.record_failure();
5382
5383 assert!(stats.error_rate() > 0.0);
5384
5385 stats.reset_window();
5386 assert_eq!(stats.error_rate(), 0.0);
5387 }
5388
5389 #[test]
5390 fn test_circuit_state_transitions_are_deterministic() {
5391 let _guard = test_guard!();
5392 let config = CircuitBreakerConfig::default();
5393 let mut stats = CircuitStats::new();
5394
5395 assert_eq!(stats.state(), CircuitState::Closed);
5397
5398 for _ in 0..3 {
5400 stats.record_failure();
5401 }
5402 assert!(stats.should_open(&config));
5403 stats.open();
5404 assert_eq!(stats.state(), CircuitState::Open);
5405
5406 stats.half_open();
5408 assert_eq!(stats.state(), CircuitState::HalfOpen);
5409
5410 for _ in 0..2 {
5412 stats.record_success();
5413 }
5414 assert!(stats.should_close(&config));
5415 stats.close();
5416 assert_eq!(stats.state(), CircuitState::Closed);
5417 }
5418
5419 fn make_test_metrics(
5424 speedup: Option<f64>,
5425 total_secs: u64,
5426 success: bool,
5427 ) -> CompilationMetrics {
5428 CompilationMetrics {
5429 project_id: "test-project".to_string(),
5430 worker_id: "worker-1".to_string(),
5431 timestamp: chrono::Utc::now(),
5432 timing: CompilationTimingBreakdown {
5433 rsync_up: Duration::from_millis(100),
5434 remote_build: Duration::from_millis(800),
5435 rsync_down: Duration::from_millis(100),
5436 total: Duration::from_secs(total_secs),
5437 },
5438 local_build_time: speedup.map(|s| Duration::from_secs_f64(total_secs as f64 * s)),
5439 speedup,
5440 files_synced: 100,
5441 bytes_transferred: 1_000_000,
5442 exit_code: if success { 0 } else { 1 },
5443 success,
5444 }
5445 }
5446
5447 #[test]
5448 fn test_compilation_timing_breakdown_default() {
5449 let _guard = test_guard!();
5450 let timing = CompilationTimingBreakdown::default();
5451 assert_eq!(timing.rsync_up, Duration::ZERO);
5452 assert_eq!(timing.remote_build, Duration::ZERO);
5453 assert_eq!(timing.rsync_down, Duration::ZERO);
5454 assert_eq!(timing.total, Duration::ZERO);
5455 }
5456
5457 #[test]
5458 fn test_compilation_timing_breakdown_serialization() {
5459 let _guard = test_guard!();
5460 let timing = CompilationTimingBreakdown {
5461 rsync_up: Duration::from_millis(100),
5462 remote_build: Duration::from_millis(2000),
5463 rsync_down: Duration::from_millis(50),
5464 total: Duration::from_millis(2150),
5465 };
5466
5467 let json = serde_json::to_string(&timing).unwrap();
5468 assert!(json.contains("100")); assert!(json.contains("2000")); assert!(json.contains("50")); assert!(json.contains("2150")); let parsed: CompilationTimingBreakdown = serde_json::from_str(&json).unwrap();
5474 assert_eq!(parsed.rsync_up, timing.rsync_up);
5475 assert_eq!(parsed.remote_build, timing.remote_build);
5476 assert_eq!(parsed.rsync_down, timing.rsync_down);
5477 assert_eq!(parsed.total, timing.total);
5478 }
5479
5480 #[test]
5481 fn test_command_timing_breakdown_serialization() {
5482 let _guard = test_guard!();
5483 let timing = CommandTimingBreakdown {
5484 classify: Some(Duration::from_millis(2)),
5485 select: None,
5486 sync_up: Some(Duration::from_millis(120)),
5487 exec: Some(Duration::from_millis(900)),
5488 sync_down: None,
5489 cleanup: Some(Duration::from_millis(5)),
5490 total: Some(Duration::from_millis(1027)),
5491 };
5492
5493 let json = serde_json::to_string(&timing).unwrap();
5494 assert!(json.contains("\"classify\":2"));
5495 assert!(json.contains("\"select\":null"));
5496 assert!(json.contains("\"sync_up\":120"));
5497 assert!(json.contains("\"exec\":900"));
5498 assert!(json.contains("\"sync_down\":null"));
5499 assert!(json.contains("\"cleanup\":5"));
5500 assert!(json.contains("\"total\":1027"));
5501
5502 let parsed: CommandTimingBreakdown = serde_json::from_str(&json).unwrap();
5503 assert_eq!(parsed.classify, timing.classify);
5504 assert_eq!(parsed.select, timing.select);
5505 assert_eq!(parsed.sync_up, timing.sync_up);
5506 assert_eq!(parsed.exec, timing.exec);
5507 assert_eq!(parsed.sync_down, timing.sync_down);
5508 assert_eq!(parsed.cleanup, timing.cleanup);
5509 assert_eq!(parsed.total, timing.total);
5510 }
5511
5512 #[test]
5513 fn test_compilation_metrics_calculate_speedup() {
5514 let _guard = test_guard!();
5515 let mut metrics = CompilationMetrics {
5516 timing: CompilationTimingBreakdown {
5517 total: Duration::from_secs(10),
5518 ..Default::default()
5519 },
5520 local_build_time: Some(Duration::from_secs(30)),
5521 ..Default::default()
5522 };
5523
5524 metrics.calculate_speedup();
5525
5526 assert!(metrics.speedup.is_some());
5527 let speedup = metrics.speedup.unwrap();
5528 assert!(
5529 (speedup - 3.0).abs() < 0.01,
5530 "Expected 3.0x speedup, got {}",
5531 speedup
5532 );
5533 }
5534
5535 #[test]
5536 fn test_compilation_metrics_is_beneficial() {
5537 let _guard = test_guard!();
5538 let mut metrics = CompilationMetrics::default();
5539
5540 assert!(!metrics.is_beneficial());
5542
5543 metrics.speedup = Some(1.5);
5545 assert!(metrics.is_beneficial());
5546
5547 metrics.speedup = Some(0.8);
5549 assert!(!metrics.is_beneficial());
5550
5551 metrics.speedup = Some(1.0);
5553 assert!(!metrics.is_beneficial());
5554 }
5555
5556 #[test]
5557 fn test_compilation_metrics_serialization() {
5558 let _guard = test_guard!();
5559 let metrics = make_test_metrics(Some(2.5), 10, true);
5560 let json = serde_json::to_string(&metrics).unwrap();
5561
5562 assert!(json.contains("test-project"));
5563 assert!(json.contains("worker-1"));
5564 assert!(json.contains("2.5")); let parsed: CompilationMetrics = serde_json::from_str(&json).unwrap();
5567 assert_eq!(parsed.project_id, "test-project");
5568 assert_eq!(parsed.worker_id, "worker-1");
5569 assert!(parsed.success);
5570 }
5571
5572 #[test]
5573 fn test_compilation_timer_new() {
5574 let _guard = test_guard!();
5575 let timer = CompilationTimer::new("my-project", "my-worker");
5576 assert_eq!(timer.project_id, "my-project");
5577 assert_eq!(timer.worker_id, "my-worker");
5578 assert!(timer.rsync_up.is_none());
5579 assert!(timer.remote_build.is_none());
5580 assert!(timer.rsync_down.is_none());
5581 }
5582
5583 #[test]
5584 fn test_compilation_timer_phases() {
5585 let _guard = test_guard!();
5586 let mut timer = CompilationTimer::new("test", "worker");
5587
5588 std::thread::sleep(Duration::from_millis(5));
5590 timer.end_rsync_up();
5591 assert!(timer.rsync_up.is_some());
5592 assert!(timer.rsync_up.unwrap() >= Duration::from_millis(5));
5593
5594 std::thread::sleep(Duration::from_millis(10));
5596 timer.end_remote_build();
5597 assert!(timer.remote_build.is_some());
5598 assert!(timer.remote_build.unwrap() >= Duration::from_millis(10));
5599
5600 std::thread::sleep(Duration::from_millis(5));
5602 timer.end_rsync_down();
5603 assert!(timer.rsync_down.is_some());
5604 assert!(timer.rsync_down.unwrap() >= Duration::from_millis(5));
5605
5606 let metrics = timer.finish(0, 50, 500_000);
5608 assert_eq!(metrics.project_id, "test");
5609 assert_eq!(metrics.worker_id, "worker");
5610 assert!(metrics.success);
5611 assert!(metrics.timing.total >= Duration::from_millis(20));
5612 assert_eq!(metrics.files_synced, 50);
5613 assert_eq!(metrics.bytes_transferred, 500_000);
5614 }
5615
5616 #[test]
5617 fn test_metrics_aggregator_new() {
5618 let _guard = test_guard!();
5619 let agg = MetricsAggregator::new(100);
5620 assert_eq!(agg.count(), 0);
5621 assert!(agg.average_speedup().is_none());
5622 }
5623
5624 #[test]
5625 fn test_metrics_aggregator_record() {
5626 let _guard = test_guard!();
5627 let mut agg = MetricsAggregator::new(3);
5628
5629 agg.record(make_test_metrics(Some(2.0), 10, true));
5631 assert_eq!(agg.count(), 1);
5632
5633 agg.record(make_test_metrics(Some(3.0), 20, true));
5635 agg.record(make_test_metrics(Some(4.0), 30, true));
5636 assert_eq!(agg.count(), 3);
5637
5638 agg.record(make_test_metrics(Some(5.0), 40, true));
5640 assert_eq!(agg.count(), 3);
5641 }
5642
5643 #[test]
5644 fn test_metrics_aggregator_average_speedup() {
5645 let _guard = test_guard!();
5646 let mut agg = MetricsAggregator::new(100);
5647
5648 assert!(agg.average_speedup().is_none());
5650
5651 for i in 1..=5 {
5653 agg.record(make_test_metrics(Some(i as f64), 10, true));
5654 }
5655
5656 let avg = agg.average_speedup().unwrap();
5657 assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5658 }
5659
5660 #[test]
5661 fn test_metrics_aggregator_average_speedup_with_none() {
5662 let _guard = test_guard!();
5663 let mut agg = MetricsAggregator::new(100);
5664
5665 agg.record(make_test_metrics(Some(2.0), 10, true));
5667 agg.record(make_test_metrics(None, 10, true)); agg.record(make_test_metrics(Some(4.0), 10, true));
5669
5670 let avg = agg.average_speedup().unwrap();
5672 assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5673 }
5674
5675 #[test]
5676 fn test_metrics_aggregator_percentiles() {
5677 let _guard = test_guard!();
5678 let mut agg = MetricsAggregator::new(100);
5679
5680 for i in 1..=10 {
5682 agg.record(make_test_metrics(Some(1.0), i, true));
5683 }
5684
5685 let p50 = agg.p50_total_time().unwrap();
5687 assert!(p50 >= Duration::from_secs(5) && p50 <= Duration::from_secs(6));
5688
5689 let p95 = agg.p95_total_time().unwrap();
5691 assert!(p95 >= Duration::from_secs(9) && p95 <= Duration::from_secs(10));
5692
5693 let p99 = agg.p99_total_time().unwrap();
5695 assert!(p99 >= Duration::from_secs(9));
5696 }
5697
5698 #[test]
5699 fn test_metrics_aggregator_success_rate() {
5700 let _guard = test_guard!();
5701 let mut agg = MetricsAggregator::new(100);
5702
5703 assert_eq!(agg.success_rate(), 100.0);
5705
5706 agg.record(make_test_metrics(Some(1.0), 10, true));
5708 agg.record(make_test_metrics(Some(1.0), 10, true));
5709 assert_eq!(agg.success_rate(), 100.0);
5710
5711 agg.record(make_test_metrics(Some(1.0), 10, false));
5713 let rate = agg.success_rate();
5714 assert!((rate - 66.67).abs() < 1.0, "Expected ~66.67%, got {}", rate);
5715 }
5716
5717 #[test]
5718 fn test_metrics_aggregator_clear() {
5719 let _guard = test_guard!();
5720 let mut agg = MetricsAggregator::new(100);
5721 agg.record(make_test_metrics(Some(1.0), 10, true));
5722 agg.record(make_test_metrics(Some(2.0), 20, true));
5723
5724 assert_eq!(agg.count(), 2);
5725
5726 agg.clear();
5727 assert_eq!(agg.count(), 0);
5728 assert!(agg.average_speedup().is_none());
5729 }
5730
5731 #[test]
5734 fn test_remote_build_jobs_parse_display_and_serde() {
5735 let _guard = test_guard!();
5736 assert_eq!(RemoteBuildJobs::default(), RemoteBuildJobs::Auto);
5737 for (spelling, expected) in [
5738 ("auto", RemoteBuildJobs::Auto),
5739 (" AUTO ", RemoteBuildJobs::Auto),
5740 ("off", RemoteBuildJobs::Off),
5741 ("none", RemoteBuildJobs::Off),
5742 ("4", RemoteBuildJobs::Fixed(4)),
5743 ] {
5744 assert_eq!(
5745 RemoteBuildJobs::parse(spelling).unwrap(),
5746 expected,
5747 "{spelling}"
5748 );
5749 }
5750 assert!(RemoteBuildJobs::parse("0").is_err(), "cargo rejects -j 0");
5751 assert!(RemoteBuildJobs::parse("many").is_err());
5752 assert_eq!(RemoteBuildJobs::Fixed(4).to_string(), "4");
5753 assert_eq!(RemoteBuildJobs::Off.to_string(), "off");
5754
5755 #[derive(Deserialize)]
5758 struct Wrap {
5759 compilation: CompilationConfig,
5760 }
5761 let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = \"off\"\n").unwrap();
5762 assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Off);
5763 let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = 6\n").unwrap();
5764 assert_eq!(
5765 parsed.compilation.remote_build_jobs,
5766 RemoteBuildJobs::Fixed(6)
5767 );
5768 let parsed: Wrap = toml::from_str("[compilation]\nbuild_slots = 4\n").unwrap();
5769 assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Auto);
5770 assert!(toml::from_str::<Wrap>("[compilation]\nremote_build_jobs = 0\n").is_err());
5771
5772 let json = serde_json::to_string(&RemoteBuildJobs::Fixed(3)).unwrap();
5774 assert_eq!(json, "3");
5775 assert_eq!(
5776 serde_json::to_string(&RemoteBuildJobs::Auto).unwrap(),
5777 "\"auto\""
5778 );
5779 let back: RemoteBuildJobs = serde_json::from_str("\"auto\"").unwrap();
5780 assert_eq!(back, RemoteBuildJobs::Auto);
5781 }
5782
5783 #[test]
5784 fn test_compilation_config_default_timeouts() {
5785 let _guard = test_guard!();
5786 let config = CompilationConfig::default();
5787 assert_eq!(config.build_timeout_sec, 300);
5789 assert_eq!(config.test_timeout_sec, 1800);
5791 }
5792
5793 #[test]
5794 fn test_compilation_config_timeout_for_test_kinds() {
5795 let _guard = test_guard!();
5796 let config = CompilationConfig::default();
5797
5798 assert_eq!(
5800 config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5801 std::time::Duration::from_secs(1800)
5802 );
5803 assert_eq!(
5804 config.timeout_for_kind(Some(crate::CompilationKind::CargoNextest)),
5805 std::time::Duration::from_secs(1800)
5806 );
5807 assert_eq!(
5808 config.timeout_for_kind(Some(crate::CompilationKind::BunTest)),
5809 std::time::Duration::from_secs(config.bun_timeout_sec)
5810 );
5811 }
5812
5813 #[test]
5814 fn test_compilation_config_timeout_for_build_kinds() {
5815 let _guard = test_guard!();
5816 let config = CompilationConfig::default();
5817
5818 assert_eq!(
5820 config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5821 std::time::Duration::from_secs(300)
5822 );
5823 assert_eq!(
5824 config.timeout_for_kind(Some(crate::CompilationKind::CargoCheck)),
5825 std::time::Duration::from_secs(300)
5826 );
5827 assert_eq!(
5828 config.timeout_for_kind(Some(crate::CompilationKind::CargoClippy)),
5829 std::time::Duration::from_secs(300)
5830 );
5831 assert_eq!(
5832 config.timeout_for_kind(None),
5833 std::time::Duration::from_secs(300)
5834 );
5835 }
5836
5837 #[test]
5838 fn test_compilation_config_custom_timeouts() {
5839 let _guard = test_guard!();
5840 let config = CompilationConfig {
5841 build_timeout_sec: 600, test_timeout_sec: 3600, ..Default::default()
5844 };
5845
5846 assert_eq!(
5847 config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5848 std::time::Duration::from_secs(600)
5849 );
5850 assert_eq!(
5851 config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5852 std::time::Duration::from_secs(3600)
5853 );
5854 }
5855
5856 #[test]
5857 fn test_compilation_config_speedup_threshold_default() {
5858 let _guard = test_guard!();
5859 let config = CompilationConfig::default();
5860 assert!((config.remote_speedup_threshold - 1.2).abs() < 0.001);
5862 }
5863
5864 #[test]
5865 fn test_compilation_config_speedup_threshold_custom() {
5866 let _guard = test_guard!();
5867 let config = CompilationConfig {
5868 remote_speedup_threshold: 1.5, ..Default::default()
5870 };
5871 assert!((config.remote_speedup_threshold - 1.5).abs() < 0.001);
5872 }
5873
5874 #[test]
5875 fn test_compilation_config_speedup_threshold_no_minimum() {
5876 let _guard = test_guard!();
5877 let config = CompilationConfig {
5879 remote_speedup_threshold: 1.0,
5880 ..Default::default()
5881 };
5882 assert!((config.remote_speedup_threshold - 1.0).abs() < 0.001);
5883 }
5884
5885 #[test]
5886 fn test_compilation_config_min_local_time_ms_default() {
5887 let _guard = test_guard!();
5888 let config = CompilationConfig::default();
5889 assert_eq!(config.min_local_time_ms, 2000);
5891 }
5892
5893 #[test]
5894 fn test_compilation_config_serde_roundtrip_speedup_threshold() {
5895 let _guard = test_guard!();
5896 let config = CompilationConfig {
5897 remote_speedup_threshold: 2.5,
5898 min_local_time_ms: 5000,
5899 ..Default::default()
5900 };
5901 let json = serde_json::to_string(&config).unwrap();
5902 let parsed: CompilationConfig = serde_json::from_str(&json).unwrap();
5903 assert!((parsed.remote_speedup_threshold - 2.5).abs() < 0.001);
5904 assert_eq!(parsed.min_local_time_ms, 5000);
5905 }
5906
5907 #[test]
5912 fn test_validate_remote_base_absolute_path() {
5913 let _guard = test_guard!();
5914 assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5915 assert_eq!(
5916 validate_remote_base("/var/rch-builds").unwrap(),
5917 "/var/rch-builds"
5918 );
5919 assert_eq!(
5920 validate_remote_base("/home/builder/.rch").unwrap(),
5921 "/home/builder/.rch"
5922 );
5923 }
5924
5925 #[test]
5926 fn test_validate_remote_base_tilde_expansion() {
5927 let _guard = test_guard!();
5928 let result = validate_remote_base("~/rch");
5930 assert!(result.is_ok());
5931 let path = result.unwrap();
5932 assert!(
5933 path.starts_with('/'),
5934 "Path should be absolute after expansion: {}",
5935 path
5936 );
5937 assert!(!path.contains('~'), "Tilde should be expanded: {}", path);
5938 }
5939
5940 #[test]
5941 fn test_validate_remote_base_rejects_relative_path() {
5942 let _guard = test_guard!();
5943 let result = validate_remote_base("tmp/rch");
5944 assert!(result.is_err());
5945 assert!(result.unwrap_err().contains("absolute path"));
5946 }
5947
5948 #[test]
5949 fn test_validate_remote_base_rejects_path_traversal() {
5950 let _guard = test_guard!();
5951 let result = validate_remote_base("/tmp/../etc/rch");
5952 assert!(result.is_err());
5953 assert!(result.unwrap_err().contains("path traversal"));
5954
5955 let result = validate_remote_base("/tmp/rch/../other");
5956 assert!(result.is_err());
5957 assert!(result.unwrap_err().contains("path traversal"));
5958 }
5959
5960 #[test]
5961 fn test_validate_remote_base_normalizes_trailing_slash() {
5962 let _guard = test_guard!();
5963 assert_eq!(validate_remote_base("/tmp/rch/").unwrap(), "/tmp/rch");
5964 assert_eq!(validate_remote_base("/tmp/rch///").unwrap(), "/tmp/rch");
5965 }
5966
5967 #[test]
5968 fn test_validate_remote_base_root_path() {
5969 let _guard = test_guard!();
5970 let result = validate_remote_base("/");
5972 assert!(result.is_err());
5973 assert!(result.unwrap_err().contains("root directory"));
5974 }
5975
5976 #[test]
5977 fn test_validate_remote_base_rejects_top_level() {
5978 let _guard = test_guard!();
5979 let result = validate_remote_base("/tmp");
5981 assert!(result.is_err());
5982 assert!(result.unwrap_err().contains("at least 2 levels deep"));
5983
5984 let result = validate_remote_base("/home");
5985 assert!(result.is_err());
5986
5987 assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5989 }
5990
5991 #[test]
5992 fn test_default_remote_base() {
5993 let _guard = test_guard!();
5994 assert_eq!(default_remote_base(), "/data/tmp/rch");
5995 }
5996
5997 #[test]
5998 fn test_transfer_config_default_has_remote_base() {
5999 let _guard = test_guard!();
6000 let config = TransferConfig::default();
6001 assert_eq!(config.remote_base, "/data/tmp/rch");
6002 }
6003
6004 #[test]
6005 fn test_source_sync_timeout_validation_boundaries() {
6006 let _guard = test_guard!();
6007 assert!(!TransferConfig::valid_sync_timeout_ms(0));
6008 assert!(!TransferConfig::valid_sync_timeout_ms(999));
6009 assert!(TransferConfig::valid_sync_timeout_ms(1_000));
6010 assert!(TransferConfig::valid_sync_timeout_ms(3_600_000));
6011 assert!(!TransferConfig::valid_sync_timeout_ms(3_600_001));
6012 }
6013
6014 #[test]
6015 fn test_source_sync_timeout_is_explicit_or_payload_aware() {
6016 let _guard = test_guard!();
6017 let default = TransferConfig::default();
6018 assert_eq!(
6019 default.sync_timeout_for_payload(0),
6020 std::time::Duration::from_millis(30_000)
6021 );
6022 assert_eq!(
6023 default.sync_timeout_for_payload(10 * 1024 * 1024),
6024 std::time::Duration::from_millis(40_000)
6025 );
6026 assert_eq!(
6027 default.sync_timeout_for_payload(u64::MAX),
6028 std::time::Duration::from_millis(3_600_000)
6029 );
6030
6031 let explicit = TransferConfig {
6032 sync_timeout_ms: Some(120_000),
6033 ..TransferConfig::default()
6034 };
6035 assert_eq!(
6036 explicit.sync_timeout_for_payload(u64::MAX),
6037 std::time::Duration::from_millis(120_000)
6038 );
6039 }
6040
6041 #[test]
6046 fn test_retry_config_default() {
6047 let _guard = test_guard!();
6048 let config = RetryConfig::default();
6049 assert_eq!(config.max_attempts, 3);
6050 assert_eq!(config.base_delay_ms, 100);
6051 assert_eq!(config.max_delay_ms, 5000);
6052 assert_eq!(config.jitter_factor, 0.1);
6053 assert_eq!(config.total_timeout_ms, 30000);
6054 }
6055
6056 #[test]
6057 fn test_retry_config_no_retry() {
6058 let _guard = test_guard!();
6059 let config = RetryConfig::no_retry();
6060 assert_eq!(config.max_attempts, 1);
6061 }
6062
6063 #[test]
6064 fn test_retry_config_delay_for_attempt_zero() {
6065 let _guard = test_guard!();
6066 let config = RetryConfig::default();
6067 assert_eq!(config.delay_for_attempt(0), std::time::Duration::ZERO);
6068 }
6069
6070 #[test]
6071 fn test_retry_config_delay_for_attempt_exponential() {
6072 let _guard = test_guard!();
6073 let config = RetryConfig {
6074 base_delay_ms: 100,
6075 max_delay_ms: 10000,
6076 jitter_factor: 0.0, ..Default::default()
6078 };
6079
6080 assert_eq!(
6082 config.delay_for_attempt(1),
6083 std::time::Duration::from_millis(100)
6084 );
6085 assert_eq!(
6087 config.delay_for_attempt(2),
6088 std::time::Duration::from_millis(200)
6089 );
6090 assert_eq!(
6092 config.delay_for_attempt(3),
6093 std::time::Duration::from_millis(400)
6094 );
6095 }
6096
6097 #[test]
6098 fn test_retry_config_delay_capped_at_max() {
6099 let _guard = test_guard!();
6100 let config = RetryConfig {
6101 base_delay_ms: 1000,
6102 max_delay_ms: 2000,
6103 jitter_factor: 0.0,
6104 ..Default::default()
6105 };
6106
6107 assert_eq!(
6109 config.delay_for_attempt(2),
6110 std::time::Duration::from_millis(2000)
6111 );
6112 assert_eq!(
6114 config.delay_for_attempt(3),
6115 std::time::Duration::from_millis(2000)
6116 );
6117 }
6118
6119 #[test]
6120 fn test_retry_config_random_jitter_stays_within_bounds() {
6121 let _guard = test_guard!();
6122 let config = RetryConfig {
6123 base_delay_ms: 100,
6124 max_delay_ms: 200,
6125 jitter_factor: 0.25,
6126 ..Default::default()
6127 };
6128
6129 for (attempt, minimum_ms, maximum_ms) in [(1, 75, 125), (3, 150, 250)] {
6132 for _ in 0..128 {
6133 let delay = config.delay_for_attempt(attempt);
6134 assert!(delay >= std::time::Duration::from_millis(minimum_ms));
6135 assert!(delay <= std::time::Duration::from_millis(maximum_ms));
6136 }
6137 }
6138 }
6139
6140 #[test]
6141 fn test_retry_config_should_retry() {
6142 let _guard = test_guard!();
6143 let config = RetryConfig {
6144 max_attempts: 3,
6145 total_timeout_ms: 1000,
6146 ..Default::default()
6147 };
6148
6149 assert!(config.should_retry(0, std::time::Duration::from_millis(0)));
6151 assert!(config.should_retry(1, std::time::Duration::from_millis(500)));
6152 assert!(config.should_retry(2, std::time::Duration::from_millis(900)));
6153
6154 assert!(!config.should_retry(3, std::time::Duration::from_millis(0)));
6156
6157 assert!(!config.should_retry(1, std::time::Duration::from_millis(1001)));
6159 }
6160
6161 #[test]
6162 fn test_retry_config_serde_roundtrip() {
6163 let _guard = test_guard!();
6164 let config = RetryConfig {
6165 max_attempts: 5,
6166 base_delay_ms: 200,
6167 max_delay_ms: 8000,
6168 jitter_factor: 0.2,
6169 total_timeout_ms: 60000,
6170 };
6171 let json = serde_json::to_string(&config).unwrap();
6172 let parsed: RetryConfig = serde_json::from_str(&json).unwrap();
6173 assert_eq!(parsed.max_attempts, 5);
6174 assert_eq!(parsed.base_delay_ms, 200);
6175 assert_eq!(parsed.max_delay_ms, 8000);
6176 assert_eq!(parsed.jitter_factor, 0.2);
6177 assert_eq!(parsed.total_timeout_ms, 60000);
6178 }
6179
6180 #[test]
6181 fn test_transfer_config_includes_retry() {
6182 let _guard = test_guard!();
6183 let config = TransferConfig::default();
6184 assert_eq!(config.retry.max_attempts, 3);
6185 }
6186
6187 #[test]
6192 fn test_execution_config_default_allowlist() {
6193 let _guard = test_guard!();
6194 let config = ExecutionConfig::default();
6195 assert!(config.is_allowed("cargo"));
6197 assert!(config.is_allowed("rustc"));
6198 assert!(config.is_allowed("gcc"));
6199 assert!(config.is_allowed("g++"));
6200 assert!(config.is_allowed("clang"));
6201 assert!(config.is_allowed("clang++"));
6202 assert!(config.is_allowed("make"));
6203 assert!(config.is_allowed("cmake"));
6204 assert!(config.is_allowed("ninja"));
6205 assert!(config.is_allowed("meson"));
6206 assert!(config.is_allowed("bun"));
6207 assert!(config.is_allowed("nextest"));
6208 assert!(config.is_allowed("cc"));
6209 assert!(config.is_allowed("c++"));
6210 }
6211
6212 #[test]
6213 fn test_execution_config_is_allowed_case_insensitive() {
6214 let _guard = test_guard!();
6215 let config = ExecutionConfig::default();
6216 assert!(config.is_allowed("CARGO"));
6218 assert!(config.is_allowed("Cargo"));
6219 assert!(config.is_allowed("GCC"));
6220 assert!(config.is_allowed("Gcc"));
6221 }
6222
6223 #[test]
6224 fn test_execution_config_is_not_allowed() {
6225 let _guard = test_guard!();
6226 let config = ExecutionConfig::default();
6227 assert!(!config.is_allowed("python"));
6229 assert!(!config.is_allowed("npm"));
6230 assert!(!config.is_allowed(""));
6231 assert!(config.is_allowed("go"));
6236 assert!(config.is_allowed("tsc"));
6237 }
6238
6239 #[test]
6240 fn test_execution_config_empty_allowlist() {
6241 let _guard = test_guard!();
6242 let config = ExecutionConfig { allowlist: vec![] };
6243 assert!(!config.is_allowed("cargo"));
6245 assert!(!config.is_allowed("gcc"));
6246 }
6247
6248 #[test]
6249 fn test_execution_config_custom_allowlist() {
6250 let _guard = test_guard!();
6251 let config = ExecutionConfig {
6252 allowlist: vec!["cargo".to_string(), "custom_tool".to_string()],
6253 };
6254 assert!(config.is_allowed("cargo"));
6255 assert!(config.is_allowed("custom_tool"));
6256 assert!(!config.is_allowed("gcc"));
6257 }
6258
6259 #[test]
6260 fn test_execution_config_serde() {
6261 let _guard = test_guard!();
6262 let config = ExecutionConfig {
6263 allowlist: vec!["cargo".to_string(), "rustc".to_string()],
6264 };
6265 let json = serde_json::to_string(&config).unwrap();
6266 let parsed: ExecutionConfig = serde_json::from_str(&json).unwrap();
6267 assert_eq!(parsed.allowlist, config.allowlist);
6268 }
6269
6270 #[test]
6271 fn test_rch_config_includes_execution() {
6272 let _guard = test_guard!();
6273 let config = RchConfig::default();
6274 assert!(config.execution.is_allowed("cargo"));
6276 }
6277
6278 #[test]
6283 fn test_adaptive_compression_disabled_by_default() {
6284 let _guard = test_guard!();
6285 let config = TransferConfig::default();
6286 assert!(!config.adaptive_compression);
6287 assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6289 assert_eq!(config.select_compression_level(Some(500_000_000)), 3);
6290 }
6291
6292 #[test]
6293 fn test_adaptive_compression_small_payload() {
6294 let _guard = test_guard!();
6295 let config = TransferConfig {
6296 adaptive_compression: true,
6297 ..Default::default()
6298 };
6299 assert_eq!(config.select_compression_level(Some(0)), 1);
6301 assert_eq!(config.select_compression_level(Some(1_000_000)), 1); assert_eq!(config.select_compression_level(Some(9_999_999)), 1); }
6304
6305 #[test]
6306 fn test_adaptive_compression_medium_payload() {
6307 let _guard = test_guard!();
6308 let config = TransferConfig {
6309 adaptive_compression: true,
6310 ..Default::default()
6311 };
6312 assert_eq!(config.select_compression_level(Some(10_000_000)), 3); assert_eq!(config.select_compression_level(Some(100_000_000)), 3); assert_eq!(config.select_compression_level(Some(199_999_999)), 3); }
6317
6318 #[test]
6319 fn test_adaptive_compression_large_payload() {
6320 let _guard = test_guard!();
6321 let config = TransferConfig {
6322 adaptive_compression: true,
6323 ..Default::default()
6324 };
6325 assert_eq!(config.select_compression_level(Some(200_000_000)), 7); assert_eq!(config.select_compression_level(Some(500_000_000)), 7); assert_eq!(config.select_compression_level(Some(1_000_000_000)), 7); }
6330
6331 #[test]
6332 fn test_adaptive_compression_no_estimate() {
6333 let _guard = test_guard!();
6334 let config = TransferConfig {
6335 adaptive_compression: true,
6336 compression_level: 5,
6337 ..Default::default()
6338 };
6339 assert_eq!(config.select_compression_level(None), 5);
6341 }
6342
6343 #[test]
6344 fn test_adaptive_compression_respects_min_level() {
6345 let _guard = test_guard!();
6346 let config = TransferConfig {
6347 adaptive_compression: true,
6348 min_compression_level: 3,
6349 ..Default::default()
6350 };
6351 assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6353 }
6354
6355 #[test]
6356 fn test_adaptive_compression_respects_max_level() {
6357 let _guard = test_guard!();
6358 let config = TransferConfig {
6359 adaptive_compression: true,
6360 max_compression_level: 5,
6361 ..Default::default()
6362 };
6363 assert_eq!(config.select_compression_level(Some(500_000_000)), 5);
6365 }
6366
6367 #[test]
6368 fn test_adaptive_compression_serde_roundtrip() {
6369 let _guard = test_guard!();
6370 let config = TransferConfig {
6371 adaptive_compression: true,
6372 min_compression_level: 2,
6373 max_compression_level: 8,
6374 ..Default::default()
6375 };
6376 let json = serde_json::to_string(&config).unwrap();
6377 let parsed: TransferConfig = serde_json::from_str(&json).unwrap();
6378 assert!(parsed.adaptive_compression);
6379 assert_eq!(parsed.min_compression_level, 2);
6380 assert_eq!(parsed.max_compression_level, 8);
6381 }
6382
6383 #[test]
6388 fn test_self_healing_config_defaults() {
6389 let _guard = test_guard!();
6390 let config = SelfHealingConfig::default();
6392
6393 assert!(
6395 config.hook_starts_daemon,
6396 "hook_starts_daemon should be true by default"
6397 );
6398 assert!(
6399 config.daemon_installs_hooks,
6400 "daemon_installs_hooks should be true by default"
6401 );
6402
6403 assert_eq!(
6405 config.auto_start_cooldown_secs, 30,
6406 "auto_start_cooldown_secs should default to 30"
6407 );
6408 assert_eq!(
6409 config.auto_start_timeout_secs, 3,
6410 "auto_start_timeout_secs should default to 3"
6411 );
6412 }
6414
6415 #[test]
6416 fn test_self_healing_config_serde_full() {
6417 let _guard = test_guard!();
6418 let config = SelfHealingConfig {
6420 hook_starts_daemon: false,
6421 daemon_installs_hooks: false,
6422 auto_start_cooldown_secs: 60,
6423 auto_start_timeout_secs: 10,
6424 self_healing_log_level: SelfHealingLogLevel::Debug,
6425 };
6426
6427 let json = serde_json::to_string(&config).unwrap();
6428 assert!(json.contains("\"hook_starts_daemon\":false"));
6429 assert!(json.contains("\"daemon_installs_hooks\":false"));
6430 assert!(json.contains("\"auto_start_cooldown_secs\":60"));
6431 assert!(json.contains("\"auto_start_timeout_secs\":10"));
6432 assert!(json.contains("\"self_healing_log_level\":\"debug\""));
6433
6434 let parsed: SelfHealingConfig = serde_json::from_str(&json).unwrap();
6435 assert!(!parsed.hook_starts_daemon);
6436 assert!(!parsed.daemon_installs_hooks);
6437 assert_eq!(parsed.auto_start_cooldown_secs, 60);
6438 assert_eq!(parsed.auto_start_timeout_secs, 10);
6439 assert_eq!(parsed.self_healing_log_level, SelfHealingLogLevel::Debug);
6440 }
6442
6443 #[test]
6446 fn test_self_healing_log_level_default_is_info() {
6447 let level = SelfHealingLogLevel::default();
6449 assert_eq!(level, SelfHealingLogLevel::Info);
6450 let config = SelfHealingConfig::default();
6452 assert_eq!(
6453 config.self_healing_log_level,
6454 SelfHealingLogLevel::Info,
6455 "config default should be Info"
6456 );
6457 }
6459
6460 #[test]
6461 fn test_self_healing_log_level_serializes_lowercase() {
6462 let json = serde_json::to_string(&SelfHealingLogLevel::Warn).unwrap();
6464 assert_eq!(json, "\"warn\"");
6465 let parsed: SelfHealingLogLevel = serde_json::from_str("\"error\"").unwrap();
6466 assert_eq!(parsed, SelfHealingLogLevel::Error);
6467 }
6469
6470 #[test]
6471 fn test_self_healing_log_level_from_env_str_accepts_known() {
6472 assert_eq!(
6474 SelfHealingLogLevel::from_env_str("debug"),
6475 Some(SelfHealingLogLevel::Debug)
6476 );
6477 assert_eq!(
6478 SelfHealingLogLevel::from_env_str("DEBUG"),
6479 Some(SelfHealingLogLevel::Debug)
6480 );
6481 assert_eq!(
6482 SelfHealingLogLevel::from_env_str("info"),
6483 Some(SelfHealingLogLevel::Info)
6484 );
6485 assert_eq!(
6486 SelfHealingLogLevel::from_env_str("warn"),
6487 Some(SelfHealingLogLevel::Warn)
6488 );
6489 assert_eq!(
6490 SelfHealingLogLevel::from_env_str("warning"),
6491 Some(SelfHealingLogLevel::Warn),
6492 "warning should map to Warn"
6493 );
6494 assert_eq!(
6495 SelfHealingLogLevel::from_env_str("error"),
6496 Some(SelfHealingLogLevel::Error)
6497 );
6498 assert_eq!(
6500 SelfHealingLogLevel::from_env_str(" debug "),
6501 Some(SelfHealingLogLevel::Debug)
6502 );
6503 }
6505
6506 #[test]
6507 fn test_self_healing_log_level_from_env_str_rejects_unknown() {
6508 assert!(SelfHealingLogLevel::from_env_str("banana").is_none());
6510 assert!(SelfHealingLogLevel::from_env_str("").is_none());
6511 assert!(SelfHealingLogLevel::from_env_str("trace").is_none());
6512 }
6514
6515 #[test]
6516 fn test_self_healing_config_serde_partial_uses_defaults() {
6517 let _guard = test_guard!();
6518 let json = r#"{"hook_starts_daemon": false}"#;
6520 let config: SelfHealingConfig = serde_json::from_str(json).unwrap();
6521
6522 assert!(
6523 !config.hook_starts_daemon,
6524 "Explicit false should be parsed"
6525 );
6526 assert!(
6527 config.daemon_installs_hooks,
6528 "Missing field should use default (true)"
6529 );
6530 assert_eq!(
6531 config.auto_start_cooldown_secs, 30,
6532 "Missing field should use default (30)"
6533 );
6534 assert_eq!(
6535 config.auto_start_timeout_secs, 3,
6536 "Missing field should use default (3)"
6537 );
6538 }
6540
6541 #[test]
6542 fn test_self_healing_config_toml_with_alias() {
6543 let _guard = test_guard!();
6544 let toml_str = r#"
6546 hook_starts_daemon = true
6547 daemon_installs_hooks = false
6548 auto_start_cooldown_secs = 45
6549 daemon_start_timeout = 7
6550 "#;
6551
6552 let config: SelfHealingConfig = toml::from_str(toml_str).unwrap();
6553 assert!(config.hook_starts_daemon);
6554 assert!(!config.daemon_installs_hooks);
6555 assert_eq!(config.auto_start_cooldown_secs, 45);
6556 assert_eq!(
6557 config.auto_start_timeout_secs, 7,
6558 "daemon_start_timeout alias should set auto_start_timeout_secs"
6559 );
6560 }
6562
6563 #[allow(unsafe_code)]
6564 mod self_healing_env_override_tests {
6565 use super::*;
6566 use crate::config::env_test_lock;
6567
6568 fn env_guard() -> std::sync::MutexGuard<'static, ()> {
6569 env_test_lock()
6570 }
6571
6572 fn set_env(key: &str, value: &str) {
6573 unsafe { std::env::set_var(key, value) };
6575 }
6576
6577 fn remove_env(key: &str) {
6578 unsafe { std::env::remove_var(key) };
6580 }
6581
6582 struct EnvVarGuard {
6583 key: &'static str,
6584 old: Option<String>,
6585 }
6586
6587 impl EnvVarGuard {
6588 fn set(key: &'static str, value: &str) -> Self {
6589 let old = std::env::var(key).ok();
6590 set_env(key, value);
6591 Self { key, old }
6592 }
6593 }
6594
6595 impl Drop for EnvVarGuard {
6596 fn drop(&mut self) {
6597 if let Some(old) = &self.old {
6598 set_env(self.key, old);
6599 } else {
6600 remove_env(self.key);
6601 }
6602 }
6603 }
6604
6605 #[test]
6606 fn test_self_healing_config_with_env_overrides_master_disable() {
6607 let _guard = test_guard!();
6608 let _guard = env_guard();
6609
6610 let _no_self_healing = EnvVarGuard::set("RCH_NO_SELF_HEALING", "1");
6611 let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "1");
6612 let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "1");
6613 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "99");
6614 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "99");
6615
6616 let config = SelfHealingConfig::default().with_env_overrides();
6617 assert!(
6618 !config.hook_starts_daemon,
6619 "RCH_NO_SELF_HEALING should disable hook auto-start"
6620 );
6621 assert!(
6622 !config.daemon_installs_hooks,
6623 "RCH_NO_SELF_HEALING should disable daemon hook installation"
6624 );
6625
6626 assert_eq!(config.auto_start_cooldown_secs, 30);
6628 assert_eq!(config.auto_start_timeout_secs, 3);
6629 }
6630
6631 #[test]
6632 fn test_self_healing_config_with_env_overrides_toggles_and_numbers() {
6633 let _guard = test_guard!();
6634 let _guard = env_guard();
6635
6636 let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "0");
6637 let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "false");
6638 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "45");
6639 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "7");
6640
6641 let config = SelfHealingConfig::default().with_env_overrides();
6642 assert!(!config.hook_starts_daemon);
6643 assert!(!config.daemon_installs_hooks);
6644 assert_eq!(config.auto_start_cooldown_secs, 45);
6645 assert_eq!(config.auto_start_timeout_secs, 7);
6646 }
6647
6648 #[test]
6649 fn test_self_healing_config_with_env_overrides_invalid_numbers_ignored() {
6650 let _guard = test_guard!();
6651 let _guard = env_guard();
6652
6653 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "not-a-number");
6654 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "nope");
6655
6656 let config = SelfHealingConfig::default().with_env_overrides();
6657 assert_eq!(config.auto_start_cooldown_secs, 30);
6658 assert_eq!(config.auto_start_timeout_secs, 3);
6659 }
6660 }
6661
6662 #[test]
6667 fn test_load_per_core_calculation() {
6668 let _guard = test_guard!();
6669 let mut caps = WorkerCapabilities::new();
6670 assert!(caps.load_per_core().is_none());
6672
6673 caps.load_avg_1 = Some(4.0);
6675 assert!(caps.load_per_core().is_none());
6676
6677 caps.num_cpus = Some(4);
6679 assert_eq!(caps.load_per_core(), Some(1.0)); caps.load_avg_1 = Some(16.0);
6683 assert_eq!(caps.load_per_core(), Some(4.0)); }
6685
6686 #[test]
6687 fn test_is_high_load() {
6688 let _guard = test_guard!();
6689 let mut caps = WorkerCapabilities::new();
6690 caps.load_avg_1 = Some(8.0);
6691 caps.num_cpus = Some(4);
6692 assert_eq!(caps.is_high_load(3.0), Some(false));
6696 assert_eq!(caps.is_high_load(2.0), Some(false)); assert_eq!(caps.is_high_load(1.5), Some(true));
6700
6701 caps.load_avg_1 = None;
6703 assert!(caps.is_high_load(1.0).is_none());
6704 }
6705
6706 #[test]
6707 fn test_is_low_disk() {
6708 let _guard = test_guard!();
6709 let mut caps = WorkerCapabilities::new();
6710 caps.disk_free_gb = Some(15.0);
6711
6712 assert_eq!(caps.is_low_disk(10.0), Some(false));
6714 assert_eq!(caps.is_low_disk(15.0), Some(false)); assert_eq!(caps.is_low_disk(20.0), Some(true));
6718
6719 caps.disk_free_gb = None;
6721 assert!(caps.is_low_disk(10.0).is_none());
6722 }
6723
6724 #[test]
6725 fn low_disk_judges_the_build_disk_not_a_small_tmpfs() {
6726 let _guard = test_guard!();
6727 let mut caps = WorkerCapabilities::new();
6730 caps.disk_free_gb = Some(8.0);
6731 caps.disk_total_gb = Some(15.5);
6732 caps.build_disk_free_gb = Some(1700.0);
6733 caps.build_disk_total_gb = Some(1900.0);
6734 assert_eq!(caps.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6735 assert_eq!(caps.is_low_disk(10.0), Some(false));
6736
6737 caps.build_disk_free_gb = Some(4.0);
6739 assert_eq!(caps.is_low_disk(10.0), Some(true));
6740
6741 caps.build_disk_free_gb = None;
6743 caps.build_disk_total_gb = None;
6744 assert_eq!(caps.build_disk_gb(), (Some(8.0), Some(15.5)));
6745 assert_eq!(caps.is_low_disk(10.0), Some(true));
6746 }
6747
6748 #[test]
6749 fn build_disk_sample_is_optional_on_the_wire() {
6750 let _guard = test_guard!();
6751 let old: WorkerCapabilities =
6752 serde_json::from_str(r#"{"disk_free_gb":13.9,"disk_total_gb":15.5}"#).unwrap();
6753 assert_eq!(old.build_disk_free_gb, None);
6754 assert_eq!(old.build_disk_gb(), (Some(13.9), Some(15.5)));
6755
6756 let new: WorkerCapabilities = serde_json::from_str(
6757 r#"{"disk_free_gb":13.9,"disk_total_gb":15.5,"build_disk_free_gb":1700.0,"build_disk_total_gb":1900.0}"#,
6758 )
6759 .unwrap();
6760 assert_eq!(new.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6761 let json = serde_json::to_string(&new).unwrap();
6762 assert!(json.contains("\"build_disk_free_gb\":1700.0"), "{json}");
6763 }
6764
6765 #[test]
6766 fn test_selection_config_preflight_defaults() {
6767 let _guard = test_guard!();
6768 let config = SelectionConfig::default();
6769 assert_eq!(config.max_load_per_core, Some(2.0));
6771 assert_eq!(config.min_free_gb, Some(10.0));
6772 assert_eq!(config.disk_gb_per_slot, 10.0);
6773 }
6774
6775 #[test]
6776 fn test_selection_config_preflight_serde() {
6777 let _guard = test_guard!();
6778 let config = SelectionConfig {
6779 max_load_per_core: Some(3.5),
6780 min_free_gb: Some(25.0),
6781 disk_gb_per_slot: 0.83,
6782 ..Default::default()
6783 };
6784 let json = serde_json::to_string(&config).unwrap();
6785 let parsed: SelectionConfig = serde_json::from_str(&json).unwrap();
6786 assert_eq!(parsed.max_load_per_core, Some(3.5));
6787 assert_eq!(parsed.min_free_gb, Some(25.0));
6788 assert_eq!(parsed.disk_gb_per_slot, 0.83);
6789 }
6790
6791 #[test]
6792 fn test_selection_disk_slot_budget_validation() {
6793 let _guard = test_guard!();
6794 for value in ["0.0", "-1.0", "nan", "inf", "-inf"] {
6795 let result = toml::from_str::<SelectionConfig>(&format!("disk_gb_per_slot = {value}"));
6796 assert!(result.is_err(), "invalid disk slot budget {value}");
6797 }
6798 let default: SelectionConfig = toml::from_str("").unwrap();
6799 assert_eq!(default.disk_gb_per_slot, 10.0);
6800 }
6801
6802 #[test]
6803 fn test_selection_disk_weight_validation_and_roundtrip() {
6804 for value in ["-0.1", "1.1", "nan", "inf", "-inf"] {
6805 assert!(toml::from_str::<SelectionWeightConfig>(&format!("disk = {value}")).is_err());
6806 }
6807 for value in [0.0, 0.2, 1.0] {
6808 let weights: SelectionWeightConfig =
6809 toml::from_str(&format!("disk = {value}")).unwrap();
6810 assert_eq!(weights.disk, value);
6811 let roundtrip: SelectionWeightConfig =
6812 serde_json::from_str(&serde_json::to_string(&weights).unwrap()).unwrap();
6813 assert_eq!(roundtrip.disk, value);
6814 }
6815 let defaults: SelectionWeightConfig = toml::from_str("").unwrap();
6816 assert_eq!(defaults.disk, 0.2);
6817 }
6818
6819 #[test]
6820 fn test_worker_capabilities_health_metrics_serde() {
6821 let _guard = test_guard!();
6822 let caps = WorkerCapabilities {
6823 num_cpus: Some(8),
6824 load_avg_1: Some(1.5),
6825 load_avg_5: Some(2.0),
6826 load_avg_15: Some(1.8),
6827 disk_free_gb: Some(50.5),
6828 disk_total_gb: Some(100.0),
6829 ..Default::default()
6830 };
6831 let json = serde_json::to_string(&caps).unwrap();
6832 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
6833 assert_eq!(parsed.num_cpus, Some(8));
6834 assert_eq!(parsed.load_avg_1, Some(1.5));
6835 assert_eq!(parsed.disk_free_gb, Some(50.5));
6836 }
6837
6838 #[test]
6839 fn microarch_level_classifies_ivy_bridge_as_v2() {
6840 let ivy = "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov \
6843 pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx rdtscp lm \
6844 constant_tsc pni pclmulqdq ssse3 cx16 sse4_1 sse4_2 x2apic \
6845 popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm";
6846 assert_eq!(x86_64_microarch_level_from_flags(ivy), 2);
6847 }
6848
6849 #[test]
6850 fn microarch_level_classifies_v3_and_v4() {
6851 let v3 = "cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 abm avx avx2 bmi1 bmi2 \
6852 f16c fma movbe xsave";
6853 assert_eq!(x86_64_microarch_level_from_flags(v3), 3);
6854
6855 let v4 = format!("{v3} avx512f avx512bw avx512cd avx512dq avx512vl");
6856 assert_eq!(x86_64_microarch_level_from_flags(&v4), 4);
6857 }
6858
6859 #[test]
6860 fn microarch_level_v1_for_ancient_or_empty_flags() {
6861 assert_eq!(x86_64_microarch_level_from_flags(""), 1);
6862 assert_eq!(x86_64_microarch_level_from_flags("fpu mmx sse sse2"), 1);
6863 }
6864
6865 #[test]
6866 fn pre_v3_predicate_only_fires_on_affirmative_evidence() {
6867 let mut caps = WorkerCapabilities::new();
6868 assert!(!caps.is_pre_v3_x86(), "unknown must not deprioritize");
6869 caps.cpu_microarch_level = Some(2);
6870 assert!(caps.is_pre_v3_x86());
6871 caps.cpu_microarch_level = Some(3);
6872 assert!(!caps.is_pre_v3_x86());
6873 }
6874}