1use crate::{CompilationKind, toolchain::ToolchainInfo};
4use rand::RngExt;
5use schemars::JsonSchema;
6use serde::{Deserialize, Serialize};
7use std::path::PathBuf;
8
9#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
11pub struct WorkerId(pub String);
12
13impl WorkerId {
14 pub fn new(id: impl Into<String>) -> Self {
15 Self(id.into())
16 }
17
18 pub fn as_str(&self) -> &str {
19 &self.0
20 }
21}
22
23impl std::fmt::Display for WorkerId {
24 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
25 write!(f, "{}", self.0)
26 }
27}
28
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
46#[serde(rename_all = "snake_case")]
47pub enum WorkerStatus {
48 #[default]
50 Healthy,
51 Degraded,
53 Unreachable,
55 Draining,
57 Drained,
59 Disabled,
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
65#[serde(rename_all = "snake_case")]
66pub enum CircuitState {
67 #[default]
69 Closed,
70 Open,
72 HalfOpen,
74}
75
76#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
78#[serde(rename_all = "lowercase")]
79pub enum RequiredRuntime {
80 #[default]
82 None,
83 Rust,
85 Bun,
87 Node,
89 Nix,
91 Go,
93 Zig,
95}
96
97#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
102#[serde(rename_all = "lowercase")]
103pub enum CommandPriority {
104 Low,
105 #[default]
106 Normal,
107 High,
108}
109
110impl std::fmt::Display for CommandPriority {
111 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112 let value = match self {
113 Self::Low => "low",
114 Self::Normal => "normal",
115 Self::High => "high",
116 };
117 write!(f, "{}", value)
118 }
119}
120
121impl std::str::FromStr for CommandPriority {
122 type Err = ();
123
124 fn from_str(s: &str) -> Result<Self, Self::Err> {
125 match s.trim().to_lowercase().as_str() {
126 "low" => Ok(Self::Low),
127 "normal" => Ok(Self::Normal),
128 "high" => Ok(Self::High),
129 _ => Err(()),
130 }
131 }
132}
133
134#[derive(Debug, Clone, Serialize, Deserialize)]
136pub struct SelectionRequest {
137 pub project: String,
139 #[serde(default, skip_serializing_if = "Option::is_none")]
141 pub command: Option<String>,
142 #[serde(default)]
144 pub command_priority: CommandPriority,
145 pub estimated_cores: u32,
147 #[serde(default, skip_serializing_if = "disk_headroom_disabled")]
151 pub disk_headroom_gib: u32,
152 #[serde(default)]
154 pub preferred_workers: Vec<WorkerId>,
155 #[serde(default)]
157 pub toolchain: Option<ToolchainInfo>,
158 #[serde(default)]
160 pub required_runtime: RequiredRuntime,
161 #[serde(default, skip_serializing_if = "Option::is_none")]
164 pub classification_duration_us: Option<u64>,
165 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
171 pub job_mode: bool,
172 #[serde(default, skip_serializing_if = "Vec::is_empty")]
182 pub required_tools: Vec<String>,
183 #[serde(default, skip_serializing_if = "Option::is_none")]
185 pub hook_pid: Option<u32>,
186}
187
188fn disk_headroom_disabled(gib: &u32) -> bool {
189 *gib == 0
190}
191
192#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
197#[serde(rename_all = "snake_case")]
198pub enum SelectionReason {
199 Success,
201 NoWorkersConfigured,
203 AllWorkersUnreachable,
205 AllCircuitsOpen,
207 AllWorkersBusy,
209 NoWorkersPassedHealth,
211 AllWorkersFailedPreflight,
213 AllWorkersFailedConvergence,
215 NoAdmissibleWorkers(String),
217 NoMatchingWorkers,
219 NoWorkersWithRuntime(String),
221 SelectionError(String),
223 AffinityPinned,
225 AffinityFallback,
227}
228
229pub const SELECTION_RESPONSE_PROTOCOL_VERSION: u64 = 1;
235
236impl std::fmt::Display for SelectionReason {
237 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
238 match self {
239 Self::Success => write!(f, "worker assigned successfully"),
240 Self::NoWorkersConfigured => write!(f, "no workers configured"),
241 Self::AllWorkersUnreachable => write!(f, "all workers unreachable"),
242 Self::AllCircuitsOpen => write!(f, "all worker circuits open"),
243 Self::AllWorkersBusy => write!(f, "all workers at capacity"),
244 Self::NoWorkersPassedHealth => write!(f, "no workers passed health thresholds"),
245 Self::AllWorkersFailedPreflight => write!(f, "all workers failed preflight checks"),
246 Self::AllWorkersFailedConvergence => {
247 write!(f, "all workers failed repo convergence checks")
248 }
249 Self::NoAdmissibleWorkers(summary) => {
250 write!(f, "no admissible workers: {}", summary)
251 }
252 Self::NoMatchingWorkers => write!(f, "no matching workers found"),
253 Self::NoWorkersWithRuntime(rt) => write!(f, "no workers with {} installed", rt),
254 Self::SelectionError(e) => write!(f, "selection error: {}", e),
255 Self::AffinityPinned => write!(f, "worker assigned via affinity pinning"),
256 Self::AffinityFallback => write!(f, "worker assigned via last-success fallback"),
257 }
258 }
259}
260
261#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
267#[serde(rename_all = "snake_case")]
268pub enum SelectionStrategy {
269 Priority,
272 Fastest,
275 #[default]
278 Balanced,
279 CacheAffinity,
282 FairFastest,
285}
286
287impl std::fmt::Display for SelectionStrategy {
288 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
289 let name = match self {
290 Self::Priority => "priority",
291 Self::Fastest => "fastest",
292 Self::Balanced => "balanced",
293 Self::CacheAffinity => "cache_affinity",
294 Self::FairFastest => "fair_fastest",
295 };
296 write!(f, "{}", name)
297 }
298}
299
300impl std::str::FromStr for SelectionStrategy {
301 type Err = String;
302
303 fn from_str(s: &str) -> Result<Self, Self::Err> {
304 match s.trim().to_lowercase().as_str() {
305 "priority" => Ok(Self::Priority),
306 "fastest" => Ok(Self::Fastest),
307 "balanced" => Ok(Self::Balanced),
308 "cache_affinity" | "cache-affinity" | "cacheaffinity" => Ok(Self::CacheAffinity),
309 "fair_fastest" | "fair-fastest" | "fairfastest" => Ok(Self::FairFastest),
310 _ => Err(format!(
311 "unknown selection strategy '{}', expected one of: priority, fastest, balanced, cache_affinity, fair_fastest",
312 s
313 )),
314 }
315 }
316}
317
318#[derive(Debug, Clone, Serialize, Deserialize)]
320pub struct SelectionConfig {
321 #[serde(default)]
323 pub strategy: SelectionStrategy,
324 #[serde(default = "default_min_success_rate")]
326 pub min_success_rate: f64,
327 #[serde(default)]
329 pub weights: SelectionWeightConfig,
330 #[serde(default)]
332 pub fairness: FairnessConfig,
333 #[serde(default)]
335 pub affinity: AffinityConfig,
336
337 #[serde(default = "default_max_load_per_core")]
341 pub max_load_per_core: Option<f64>,
342 #[serde(default = "default_min_free_gb")]
345 pub min_free_gb: Option<f64>,
346 #[serde(
349 default = "default_disk_gb_per_slot",
350 deserialize_with = "deserialize_disk_gb_per_slot"
351 )]
352 pub disk_gb_per_slot: f64,
353}
354
355impl Default for SelectionConfig {
356 fn default() -> Self {
357 Self {
358 strategy: SelectionStrategy::default(),
359 min_success_rate: default_min_success_rate(),
360 weights: SelectionWeightConfig::default(),
361 fairness: FairnessConfig::default(),
362 affinity: AffinityConfig::default(),
363 max_load_per_core: default_max_load_per_core(),
364 min_free_gb: default_min_free_gb(),
365 disk_gb_per_slot: default_disk_gb_per_slot(),
366 }
367 }
368}
369
370fn default_min_success_rate() -> f64 {
371 0.8
372}
373
374fn default_max_load_per_core() -> Option<f64> {
375 Some(2.0) }
377
378fn default_min_free_gb() -> Option<f64> {
379 Some(10.0) }
381
382fn default_disk_gb_per_slot() -> f64 {
383 10.0
384}
385
386fn deserialize_disk_gb_per_slot<'de, D>(deserializer: D) -> Result<f64, D::Error>
387where
388 D: serde::Deserializer<'de>,
389{
390 let budget = f64::deserialize(deserializer)?;
391 if budget.is_finite() && budget > 0.0 {
392 Ok(budget)
393 } else {
394 Err(serde::de::Error::custom(
395 "disk_gb_per_slot must be finite and greater than zero",
396 ))
397 }
398}
399
400#[derive(Debug, Clone, Serialize, Deserialize)]
402pub struct SelectionWeightConfig {
403 #[serde(default = "default_weight_speedscore")]
405 pub speedscore: f64,
406 #[serde(default = "default_weight_slots")]
408 pub slots: f64,
409 #[serde(default = "default_weight_health")]
411 pub health: f64,
412 #[serde(default = "default_weight_cache")]
414 pub cache: f64,
415 #[serde(default = "default_weight_network")]
417 pub network: f64,
418 #[serde(default = "default_weight_priority")]
420 pub priority: f64,
421 #[serde(
423 default = "default_weight_disk",
424 deserialize_with = "deserialize_weight_disk"
425 )]
426 pub disk: f64,
427 #[serde(default = "default_half_open_penalty")]
429 pub half_open_penalty: f64,
430}
431
432impl Default for SelectionWeightConfig {
433 fn default() -> Self {
434 Self {
435 speedscore: default_weight_speedscore(),
436 slots: default_weight_slots(),
437 health: default_weight_health(),
438 cache: default_weight_cache(),
439 network: default_weight_network(),
440 priority: default_weight_priority(),
441 disk: default_weight_disk(),
442 half_open_penalty: default_half_open_penalty(),
443 }
444 }
445}
446
447fn default_weight_speedscore() -> f64 {
448 0.5
449}
450fn default_weight_slots() -> f64 {
451 0.4
452}
453fn default_weight_health() -> f64 {
454 0.3
455}
456fn default_weight_cache() -> f64 {
457 0.2
458}
459fn default_weight_network() -> f64 {
460 0.1
461}
462fn default_weight_priority() -> f64 {
463 0.5
470}
471fn default_half_open_penalty() -> f64 {
472 0.5
473}
474
475fn default_weight_disk() -> f64 {
476 0.2
477}
478
479fn deserialize_weight_disk<'de, D>(deserializer: D) -> Result<f64, D::Error>
480where
481 D: serde::Deserializer<'de>,
482{
483 let weight = f64::deserialize(deserializer)?;
484 if weight.is_finite() && (0.0..=1.0).contains(&weight) {
485 Ok(weight)
486 } else {
487 Err(serde::de::Error::custom(
488 "disk weight must be finite and between zero and one",
489 ))
490 }
491}
492
493#[derive(Debug, Clone, Serialize, Deserialize)]
495pub struct FairnessConfig {
496 #[serde(default = "default_fairness_lookback_secs")]
498 pub lookback_secs: u64,
499 #[serde(default = "default_max_consecutive_selections")]
501 pub max_consecutive_selections: u32,
502}
503
504impl Default for FairnessConfig {
505 fn default() -> Self {
506 Self {
507 lookback_secs: default_fairness_lookback_secs(),
508 max_consecutive_selections: default_max_consecutive_selections(),
509 }
510 }
511}
512
513fn default_fairness_lookback_secs() -> u64 {
514 300 }
516
517fn default_max_consecutive_selections() -> u32 {
518 3
519}
520
521#[derive(Debug, Clone, Serialize, Deserialize)]
523pub struct AffinityConfig {
524 #[serde(default = "default_affinity_enabled")]
526 pub enabled: bool,
527 #[serde(default = "default_affinity_pin_minutes")]
530 pub pin_minutes: u64,
531 #[serde(default = "default_last_success_fallback")]
535 pub enable_last_success_fallback: bool,
536 #[serde(default = "default_fallback_min_success_rate")]
539 pub fallback_min_success_rate: f64,
540}
541
542impl Default for AffinityConfig {
543 fn default() -> Self {
544 Self {
545 enabled: default_affinity_enabled(),
546 pin_minutes: default_affinity_pin_minutes(),
547 enable_last_success_fallback: default_last_success_fallback(),
548 fallback_min_success_rate: default_fallback_min_success_rate(),
549 }
550 }
551}
552
553fn default_affinity_enabled() -> bool {
554 true
555}
556
557fn default_affinity_pin_minutes() -> u64 {
558 60 }
560
561fn default_last_success_fallback() -> bool {
562 true
563}
564
565fn default_fallback_min_success_rate() -> f64 {
566 0.5 }
568
569#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
571pub struct SelectedWorker {
572 pub id: WorkerId,
574 pub host: String,
576 pub user: String,
578 pub identity_file: String,
580 pub slots_available: u32,
582 pub speed_score: f64,
584 #[serde(default)]
590 pub declared_os: Option<String>,
591}
592
593#[derive(Debug, Clone, Serialize, Deserialize)]
595pub struct SelectionResponse {
596 pub worker: Option<SelectedWorker>,
598 pub reason: SelectionReason,
600 #[serde(default, skip_serializing_if = "Option::is_none")]
602 pub build_id: Option<u64>,
603 #[serde(default, skip_serializing_if = "Option::is_none")]
605 pub diagnostics: Option<SelectionDiagnostics>,
606}
607
608#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
610#[serde(rename_all = "snake_case")]
611pub enum WorkerSelectionDiagnosticDecision {
612 Allow,
614 FallbackCandidate,
616 Deny,
618}
619
620#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
622pub struct WorkerSelectionDiagnostic {
623 pub worker_id: WorkerId,
625 pub status: String,
627 pub circuit_state: String,
629 pub pressure_state: String,
631 pub pressure_reason_code: String,
633 #[serde(skip_serializing_if = "Option::is_none")]
635 pub success_rate: Option<f64>,
636 pub min_success_rate: f64,
638 pub fallback_min_success_rate: f64,
640 pub required_runtime: RequiredRuntime,
642 pub runtime_available: bool,
644 pub available_slots: u32,
646 pub total_slots: u32,
648 pub estimated_cores: u32,
650 pub active_project_excluded: bool,
652 pub final_decision: WorkerSelectionDiagnosticDecision,
654 pub final_reason: String,
656 pub reason_codes: Vec<String>,
658}
659
660#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
662pub struct SelectionDiagnostics {
663 pub required_runtime: RequiredRuntime,
665 pub estimated_cores: u32,
667 pub min_success_rate: f64,
669 pub fallback_min_success_rate: f64,
671 pub active_project_exclusion_count: usize,
673 pub workers: Vec<WorkerSelectionDiagnostic>,
675}
676
677#[derive(Debug, Clone, Serialize, Deserialize)]
679pub struct ReleaseRequest {
680 pub worker_id: WorkerId,
682 pub slots: u32,
684 #[serde(default, skip_serializing_if = "Option::is_none")]
686 pub build_id: Option<u64>,
687 #[serde(default, skip_serializing_if = "Option::is_none")]
689 pub local_wrapper_id: Option<String>,
690 #[serde(default, skip_serializing_if = "Option::is_none")]
692 pub exit_code: Option<i32>,
693 #[serde(default, skip_serializing_if = "Option::is_none")]
697 pub duration_ms: Option<u64>,
698 #[serde(default, skip_serializing_if = "Option::is_none")]
700 pub bytes_transferred: Option<u64>,
701 #[serde(default, skip_serializing_if = "Option::is_none")]
703 pub timing: Option<CommandTimingBreakdown>,
704 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
708 pub worker_fault: bool,
709 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
713 pub worker_disk_full: bool,
714 #[serde(default, skip_serializing_if = "Vec::is_empty")]
717 pub worker_disk_roots: Vec<String>,
718}
719
720#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
722#[serde(rename_all = "snake_case")]
723pub enum BuildHeartbeatPhase {
724 SyncUp,
726 Execute,
728 SyncDown,
730 Finalize,
732}
733
734#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
736pub struct BuildHeartbeatRequest {
737 pub build_id: u64,
739 pub worker_id: WorkerId,
741 #[serde(default, skip_serializing_if = "Option::is_none")]
743 pub hook_pid: Option<u32>,
744 #[serde(default, skip_serializing_if = "Option::is_none")]
746 pub local_wrapper_id: Option<String>,
747 #[serde(default, skip_serializing_if = "Option::is_none")]
749 pub remote_pgid_file: Option<String>,
750 pub phase: BuildHeartbeatPhase,
752 #[serde(default, skip_serializing_if = "Option::is_none")]
754 pub detail: Option<String>,
755 #[serde(default, skip_serializing_if = "Option::is_none")]
760 pub progress_counter: Option<u64>,
761 #[serde(default, skip_serializing_if = "Option::is_none")]
763 pub progress_percent: Option<f64>,
764}
765
766#[derive(Debug, Clone, Serialize, Deserialize)]
768pub struct WorkerConfig {
769 pub id: WorkerId,
771 pub host: String,
773 pub user: String,
775 pub identity_file: String,
777 pub total_slots: u32,
779 #[serde(default = "default_priority")]
781 pub priority: u32,
782 #[serde(default)]
788 pub tags: Vec<String>,
789 #[serde(default, skip_serializing_if = "Vec::is_empty")]
798 pub tools: Vec<WorkerToolProbe>,
799}
800
801#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
807pub struct WorkerToolProbe {
808 pub name: String,
810 pub command: Vec<String>,
812}
813
814fn default_priority() -> u32 {
815 100
816}
817
818pub const OS_TAG_PREFIX: &str = "os:";
820
821#[must_use]
832pub fn declared_os(tags: &[String]) -> Option<String> {
833 tags.iter()
838 .filter_map(|tag| tag.strip_prefix(OS_TAG_PREFIX))
839 .map(|os| os.trim().to_ascii_lowercase())
840 .find(|os| !os.is_empty())
841}
842
843#[must_use]
845pub fn os_tag(os: &str) -> String {
846 format!("{OS_TAG_PREFIX}{}", os.trim().to_ascii_lowercase())
847}
848
849impl Default for WorkerConfig {
850 fn default() -> Self {
851 Self {
852 id: WorkerId::new("default-worker"),
853 host: "localhost".to_string(),
854 user: "user".to_string(),
855 identity_file: "~/.ssh/id_rsa".to_string(),
856 total_slots: 4,
857 priority: default_priority(),
858 tags: Vec::new(),
859 tools: Vec::new(),
860 }
861 }
862}
863
864#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)]
870pub struct WorkerCapabilities {
871 #[serde(default, skip_serializing_if = "Option::is_none")]
873 pub rustc_version: Option<String>,
874 #[serde(default, skip_serializing_if = "Vec::is_empty")]
877 pub rustup_toolchains: Vec<String>,
878 #[serde(default, skip_serializing_if = "Vec::is_empty")]
882 pub rustup_components: Vec<String>,
883 #[serde(default, skip_serializing_if = "Vec::is_empty")]
889 pub probe_warnings: Vec<String>,
890 #[serde(default, skip_serializing_if = "Option::is_none")]
892 pub bun_version: Option<String>,
893 #[serde(default, skip_serializing_if = "Option::is_none")]
895 pub node_version: Option<String>,
896 #[serde(default, skip_serializing_if = "Option::is_none")]
898 pub npm_version: Option<String>,
899 #[serde(default, skip_serializing_if = "Option::is_none")]
903 pub nix_version: Option<String>,
904 #[serde(default, skip_serializing_if = "Option::is_none")]
907 pub go_version: Option<String>,
908 #[serde(default, skip_serializing_if = "Option::is_none")]
912 pub zig_version: Option<String>,
913 #[serde(default, skip_serializing_if = "Option::is_none")]
917 pub cargo_zigbuild_version: Option<String>,
918 #[serde(default, skip_serializing_if = "Option::is_none")]
925 pub cpu_microarch_level: Option<u8>,
926 #[serde(default, skip_serializing_if = "Vec::is_empty")]
933 pub tools_present: Vec<String>,
934 #[serde(default, skip_serializing_if = "Vec::is_empty")]
940 pub tools_absent: Vec<String>,
941
942 #[serde(default, skip_serializing_if = "Option::is_none")]
945 pub num_cpus: Option<u32>,
946 #[serde(default, skip_serializing_if = "Option::is_none")]
948 pub load_avg_1: Option<f64>,
949 #[serde(default, skip_serializing_if = "Option::is_none")]
951 pub load_avg_5: Option<f64>,
952 #[serde(default, skip_serializing_if = "Option::is_none")]
954 pub load_avg_15: Option<f64>,
955 #[serde(default, skip_serializing_if = "Option::is_none")]
959 pub disk_free_gb: Option<f64>,
960 #[serde(default, skip_serializing_if = "Option::is_none")]
962 pub disk_total_gb: Option<f64>,
963 #[serde(default, skip_serializing_if = "Option::is_none")]
968 pub build_disk_free_gb: Option<f64>,
969 #[serde(default, skip_serializing_if = "Option::is_none")]
971 pub build_disk_total_gb: Option<f64>,
972 #[serde(default, skip_serializing_if = "Option::is_none")]
977 pub projects_root_ok: Option<bool>,
978 #[serde(default, skip_serializing_if = "Option::is_none")]
980 pub projects_root_issue: Option<String>,
981 #[serde(default, skip_serializing_if = "Option::is_none")]
983 pub projects_root_checked_at_unix_ms: Option<i64>,
984}
985
986impl WorkerCapabilities {
987 pub fn new() -> Self {
989 Self::default()
990 }
991
992 pub fn mock_with_rust() -> Self {
996 Self {
997 rustc_version: Some("1.85.0-nightly (mock)".to_string()),
998 projects_root_ok: Some(true),
999 ..Self::default()
1000 }
1001 }
1002
1003 pub fn is_pre_v3_x86(&self) -> bool {
1008 matches!(self.cpu_microarch_level, Some(1 | 2))
1009 }
1010
1011 pub fn has_bun(&self) -> bool {
1013 self.bun_version.is_some()
1014 }
1015
1016 pub fn has_node(&self) -> bool {
1018 self.node_version.is_some()
1019 }
1020
1021 pub fn has_rust(&self) -> bool {
1023 self.rustc_version.is_some()
1024 }
1025
1026 pub fn has_rustup_component(&self, toolchain: &str, component: &str) -> bool {
1033 let owner_prefix = format!("{toolchain}-");
1034 self.rustup_components.iter().any(|fact| {
1035 fact.split_once(':').is_some_and(|(owner, name)| {
1036 name == component && (owner == toolchain || owner.starts_with(&owner_prefix))
1037 })
1038 })
1039 }
1040
1041 pub fn has_nix(&self) -> bool {
1043 self.nix_version.is_some()
1044 }
1045
1046 pub fn has_go(&self) -> bool {
1048 self.go_version.is_some()
1049 }
1050
1051 pub fn has_zig(&self) -> bool {
1061 self.zig_version.is_some() && self.cargo_zigbuild_version.is_some()
1062 }
1063
1064 pub fn load_per_core(&self) -> Option<f64> {
1067 match (self.load_avg_1, self.num_cpus) {
1068 (Some(load), Some(cpus)) if cpus > 0 => Some(load / cpus as f64),
1069 _ => None,
1070 }
1071 }
1072
1073 pub fn is_high_load(&self, max_load_per_core: f64) -> Option<bool> {
1076 self.load_per_core().map(|lpc| lpc > max_load_per_core)
1077 }
1078
1079 pub fn build_disk_gb(&self) -> (Option<f64>, Option<f64>) {
1087 if self.build_disk_free_gb.is_some() {
1088 (self.build_disk_free_gb, self.build_disk_total_gb)
1089 } else {
1090 (self.disk_free_gb, self.disk_total_gb)
1091 }
1092 }
1093
1094 pub fn is_low_disk(&self, min_free_gb: f64) -> Option<bool> {
1097 self.build_disk_gb().0.map(|free| free < min_free_gb)
1098 }
1099
1100 pub fn is_topology_healthy(&self) -> Option<bool> {
1104 self.projects_root_ok
1105 }
1106}
1107
1108#[must_use]
1117pub fn x86_64_microarch_level_from_flags(flags: &str) -> u8 {
1118 let set: std::collections::HashSet<&str> = flags.split_whitespace().collect();
1119 let has_all = |required: &[&str]| required.iter().all(|flag| set.contains(flag));
1120 const V2: &[&str] = &["cx16", "lahf_lm", "popcnt", "sse4_1", "sse4_2", "ssse3"];
1122 const V3: &[&str] = &[
1124 "abm", "avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "movbe", "xsave",
1125 ];
1126 const V4: &[&str] = &["avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl"];
1128 if !has_all(V2) {
1129 return 1;
1130 }
1131 if !has_all(V3) {
1132 return 2;
1133 }
1134 if !has_all(V4) {
1135 return 3;
1136 }
1137 4
1138}
1139
1140#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1153pub struct PathTopologyConfig {
1154 #[serde(default)]
1157 pub canonical_root: Option<String>,
1158
1159 #[serde(default)]
1162 pub alias_root: Option<String>,
1163}
1164
1165impl PathTopologyConfig {
1166 pub fn to_policy(&self) -> crate::path_topology::PathTopologyPolicy {
1172 let canonical = self
1173 .canonical_root
1174 .as_deref()
1175 .filter(|s| !s.is_empty())
1176 .map(|s| shellexpand::tilde(s).into_owned())
1177 .unwrap_or_else(|| crate::path_topology::DEFAULT_CANONICAL_PROJECT_ROOT.to_string());
1178 let alias = self
1179 .alias_root
1180 .as_deref()
1181 .filter(|s| !s.is_empty())
1182 .map(|s| shellexpand::tilde(s).into_owned())
1183 .unwrap_or_else(|| crate::path_topology::DEFAULT_ALIAS_PROJECT_ROOT.to_string());
1184 crate::path_topology::PathTopologyPolicy::new(
1185 PathBuf::from(canonical),
1186 PathBuf::from(alias),
1187 )
1188 }
1189}
1190
1191#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1193pub struct RchConfig {
1194 #[serde(default)]
1195 pub general: GeneralConfig,
1196 #[serde(default)]
1197 pub compilation: CompilationConfig,
1198 #[serde(default)]
1199 pub transfer: TransferConfig,
1200 #[serde(default)]
1201 pub environment: EnvironmentConfig,
1202 #[serde(default)]
1203 pub circuit: CircuitBreakerConfig,
1204 #[serde(default)]
1205 pub output: OutputConfig,
1206 #[serde(default)]
1207 pub self_healing: SelfHealingConfig,
1208 #[serde(default)]
1209 pub self_test: SelfTestConfig,
1210 #[serde(default)]
1212 pub selection: SelectionConfig,
1213 #[serde(default)]
1215 pub execution: ExecutionConfig,
1216 #[serde(default)]
1218 pub alerts: AlertsConfig,
1219 #[serde(default)]
1221 pub fleet: FleetConfig,
1222 #[serde(default)]
1224 pub path_topology: PathTopologyConfig,
1225 #[serde(default)]
1227 pub doctor: DoctorConfig,
1228 #[serde(default)]
1233 pub remediation: crate::remediation_config::RemediationConfig,
1234 #[serde(default)]
1237 pub layer0: Layer0Config,
1238 #[serde(default)]
1241 pub api: ApiConfig,
1242 #[serde(default)]
1244 pub dashboard: DashboardConfig,
1245 #[serde(default)]
1247 pub jobs: JobsConfig,
1248}
1249
1250#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1252pub struct JobsConfig {
1253 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1263 pub required_tools: Vec<String>,
1264}
1265
1266#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1283#[serde(default)]
1284pub struct ApiConfig {
1285 pub bind: String,
1289 pub token: Option<String>,
1291 pub token_file: Option<String>,
1293 pub no_token: bool,
1297 pub allow_any_addr: bool,
1299}
1300
1301#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1303#[serde(default)]
1304pub struct DashboardConfig {
1305 pub url: Option<String>,
1307}
1308
1309#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1315pub struct DoctorConfig {
1316 #[serde(default)]
1318 pub webhooks: DoctorWebhooksConfig,
1319}
1320
1321#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1327pub struct DoctorWebhooksConfig {
1328 #[serde(default)]
1330 pub endpoints: Vec<DoctorWebhookEndpoint>,
1331 #[serde(default = "default_webhook_max_inflight")]
1336 pub max_inflight: usize,
1337}
1338
1339impl Default for DoctorWebhooksConfig {
1343 fn default() -> Self {
1344 Self {
1345 endpoints: Vec::new(),
1346 max_inflight: default_webhook_max_inflight(),
1347 }
1348 }
1349}
1350
1351#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1359pub struct DoctorWebhookEndpoint {
1360 pub name: String,
1362 pub url: String,
1364 #[serde(default)]
1366 pub format: DoctorWebhookFormat,
1367 #[serde(default)]
1372 pub on_transitions: Vec<String>,
1373 #[serde(default = "default_webhook_retry_max")]
1375 pub retry_max: u32,
1376 #[serde(default = "default_webhook_retry_backoff_ms")]
1378 pub retry_backoff_ms: u64,
1379 #[serde(default = "default_webhook_timeout_ms")]
1381 pub timeout_ms: u64,
1382 #[serde(default = "default_true")]
1384 pub enabled: bool,
1385 #[serde(default)]
1388 pub routing_key_env: Option<String>,
1389 #[serde(default)]
1392 pub bearer_token_env: Option<String>,
1393 #[serde(default)]
1397 pub signing_secret_env: Option<String>,
1398}
1399
1400#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
1402#[serde(rename_all = "snake_case")]
1403pub enum DoctorWebhookFormat {
1404 #[default]
1406 GenericJson,
1407 Slack,
1409 Pagerduty,
1411}
1412
1413fn default_webhook_max_inflight() -> usize {
1414 16
1415}
1416
1417fn default_webhook_retry_max() -> u32 {
1418 3
1419}
1420
1421fn default_webhook_retry_backoff_ms() -> u64 {
1422 200
1423}
1424
1425fn default_webhook_timeout_ms() -> u64 {
1426 5000
1427}
1428
1429#[derive(Debug, Clone, Serialize, Deserialize)]
1431pub struct AlertsConfig {
1432 #[serde(default = "default_true")]
1434 pub enabled: bool,
1435 #[serde(default = "default_alert_suppress_duplicates_secs")]
1437 pub suppress_duplicates_secs: u64,
1438 #[serde(default = "default_alert_cleared_retention_secs")]
1443 pub cleared_retention_secs: u64,
1444 #[serde(default)]
1446 pub webhook: Option<WebhookConfig>,
1447}
1448
1449#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1451pub struct WebhookConfig {
1452 pub url: Option<String>,
1454 #[serde(default)]
1456 pub secret: Option<String>,
1457 #[serde(default = "default_webhook_timeout_secs")]
1459 pub timeout_secs: u64,
1460 #[serde(default = "default_webhook_retry_count")]
1462 pub retry_count: u32,
1463 #[serde(default)]
1466 pub events: Vec<String>,
1467}
1468
1469fn default_webhook_timeout_secs() -> u64 {
1470 5
1471}
1472
1473fn default_webhook_retry_count() -> u32 {
1474 3
1475}
1476
1477fn default_alert_suppress_duplicates_secs() -> u64 {
1478 300
1479}
1480
1481fn default_alert_cleared_retention_secs() -> u64 {
1482 300
1483}
1484
1485impl Default for AlertsConfig {
1486 fn default() -> Self {
1487 Self {
1488 enabled: true,
1489 suppress_duplicates_secs: default_alert_suppress_duplicates_secs(),
1490 cleared_retention_secs: default_alert_cleared_retention_secs(),
1491 webhook: None,
1492 }
1493 }
1494}
1495
1496#[derive(Debug, Clone, Serialize, Deserialize)]
1501pub struct FleetConfig {
1502 #[serde(default = "default_fleet_ssh_connect_timeout_secs")]
1504 pub ssh_connect_timeout_secs: u64,
1505 #[serde(default = "default_fleet_ssh_command_timeout_secs")]
1507 pub ssh_command_timeout_secs: u64,
1508 #[serde(default = "default_fleet_min_disk_space_mb")]
1510 pub min_disk_space_mb: u64,
1511 #[serde(default = "default_fleet_max_load_average")]
1513 pub max_load_average: f64,
1514 #[serde(default = "default_fleet_max_concurrent_workers")]
1516 pub max_concurrent_workers: usize,
1517 #[serde(default = "default_fleet_retry_count")]
1519 pub retry_count: u32,
1520 #[serde(default = "default_fleet_retry_delay_ms")]
1522 pub retry_delay_ms: u64,
1523}
1524
1525fn default_fleet_ssh_connect_timeout_secs() -> u64 {
1526 10
1527}
1528
1529fn default_fleet_ssh_command_timeout_secs() -> u64 {
1530 30
1531}
1532
1533fn default_fleet_min_disk_space_mb() -> u64 {
1534 500
1535}
1536
1537fn default_fleet_max_load_average() -> f64 {
1538 10.0
1539}
1540
1541fn default_fleet_max_concurrent_workers() -> usize {
1542 10
1543}
1544
1545fn default_fleet_retry_count() -> u32 {
1546 2
1547}
1548
1549fn default_fleet_retry_delay_ms() -> u64 {
1550 1000
1551}
1552
1553impl Default for FleetConfig {
1554 fn default() -> Self {
1555 Self {
1556 ssh_connect_timeout_secs: default_fleet_ssh_connect_timeout_secs(),
1557 ssh_command_timeout_secs: default_fleet_ssh_command_timeout_secs(),
1558 min_disk_space_mb: default_fleet_min_disk_space_mb(),
1559 max_load_average: default_fleet_max_load_average(),
1560 max_concurrent_workers: default_fleet_max_concurrent_workers(),
1561 retry_count: default_fleet_retry_count(),
1562 retry_delay_ms: default_fleet_retry_delay_ms(),
1563 }
1564 }
1565}
1566
1567impl FleetConfig {
1568 pub fn ssh_connect_timeout(&self) -> std::time::Duration {
1570 std::time::Duration::from_secs(self.ssh_connect_timeout_secs)
1571 }
1572
1573 pub fn ssh_command_timeout(&self) -> std::time::Duration {
1575 std::time::Duration::from_secs(self.ssh_command_timeout_secs)
1576 }
1577
1578 pub fn retry_delay(&self) -> std::time::Duration {
1580 std::time::Duration::from_millis(self.retry_delay_ms)
1581 }
1582}
1583
1584#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
1587#[serde(rename_all = "lowercase")]
1588pub enum BoxRole {
1589 Dispatcher,
1594 Worker,
1597 #[default]
1599 Hybrid,
1600}
1601
1602impl BoxRole {
1603 #[must_use]
1605 pub fn as_str(self) -> &'static str {
1606 match self {
1607 BoxRole::Dispatcher => "dispatcher",
1608 BoxRole::Worker => "worker",
1609 BoxRole::Hybrid => "hybrid",
1610 }
1611 }
1612}
1613
1614#[derive(Debug, Clone, Serialize, Deserialize)]
1615pub struct GeneralConfig {
1616 #[serde(default = "default_true")]
1618 pub enabled: bool,
1619 #[serde(default)]
1622 pub role: BoxRole,
1623 #[serde(default)]
1627 pub force_local: bool,
1628 #[serde(default)]
1633 pub force_remote: bool,
1634 #[serde(default = "default_log_level")]
1636 pub log_level: String,
1637 #[serde(default = "default_socket_path")]
1639 pub socket_path: String,
1640}
1641
1642#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1644pub struct EnvironmentConfig {
1645 #[serde(default)]
1647 pub allowlist: Vec<String>,
1648}
1649
1650impl Default for GeneralConfig {
1651 fn default() -> Self {
1652 Self {
1653 enabled: true,
1654 role: BoxRole::default(),
1655 force_local: false,
1656 force_remote: false,
1657 log_level: "info".to_string(),
1658 socket_path: default_socket_path(),
1659 }
1660 }
1661}
1662
1663#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
1665#[serde(rename_all = "lowercase")]
1666pub enum OutputVisibility {
1667 #[default]
1668 None,
1669 Summary,
1670 Verbose,
1671}
1672
1673impl std::fmt::Display for OutputVisibility {
1674 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1675 let value = match self {
1676 OutputVisibility::None => "none",
1677 OutputVisibility::Summary => "summary",
1678 OutputVisibility::Verbose => "verbose",
1679 };
1680 write!(f, "{}", value)
1681 }
1682}
1683
1684impl std::str::FromStr for OutputVisibility {
1685 type Err = ();
1686
1687 fn from_str(s: &str) -> Result<Self, Self::Err> {
1688 match s.trim().to_lowercase().as_str() {
1689 "none" | "silent" | "quiet" => Ok(Self::None),
1690 "summary" | "short" => Ok(Self::Summary),
1691 "verbose" | "debug" => Ok(Self::Verbose),
1692 _ => Err(()),
1693 }
1694 }
1695}
1696
1697#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1702#[serde(rename_all = "lowercase")]
1703pub enum ColorMode {
1704 #[default]
1707 Always,
1708 Auto,
1710 Never,
1712}
1713
1714impl std::fmt::Display for ColorMode {
1715 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1716 let value = match self {
1717 ColorMode::Always => "always",
1718 ColorMode::Auto => "auto",
1719 ColorMode::Never => "never",
1720 };
1721 write!(f, "{}", value)
1722 }
1723}
1724
1725impl std::str::FromStr for ColorMode {
1726 type Err = ();
1727
1728 fn from_str(s: &str) -> Result<Self, Self::Err> {
1729 match s.trim().to_lowercase().as_str() {
1730 "always" | "force" | "yes" | "true" => Ok(Self::Always),
1731 "auto" | "detect" => Ok(Self::Auto),
1732 "never" | "none" | "no" | "false" => Ok(Self::Never),
1733 _ => Err(()),
1734 }
1735 }
1736}
1737
1738fn default_color_mode() -> ColorMode {
1739 ColorMode::default()
1740}
1741
1742#[derive(Debug, Clone, Serialize, Deserialize)]
1744pub struct OutputConfig {
1745 #[serde(default = "default_output_visibility")]
1747 pub visibility: OutputVisibility,
1748 #[serde(default)]
1750 pub first_run_complete: bool,
1751 #[serde(default = "default_color_mode")]
1754 pub color_mode: ColorMode,
1755}
1756
1757impl Default for OutputConfig {
1758 fn default() -> Self {
1759 Self {
1760 visibility: OutputVisibility::None,
1761 first_run_complete: false,
1762 color_mode: ColorMode::default(),
1763 }
1764 }
1765}
1766
1767fn default_autostart_cooldown_secs() -> u64 {
1772 30
1773}
1774
1775fn default_autostart_timeout_secs() -> u64 {
1776 3
1777}
1778
1779#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
1785#[serde(rename_all = "lowercase")]
1786pub enum SelfHealingLogLevel {
1787 Debug,
1788 #[default]
1789 Info,
1790 Warn,
1791 Error,
1792}
1793
1794impl SelfHealingLogLevel {
1795 pub fn from_env_str(s: &str) -> Option<Self> {
1797 match s.trim().to_ascii_lowercase().as_str() {
1798 "debug" => Some(Self::Debug),
1799 "info" => Some(Self::Info),
1800 "warn" | "warning" => Some(Self::Warn),
1801 "error" => Some(Self::Error),
1802 _ => None,
1803 }
1804 }
1805}
1806
1807#[derive(Debug, Clone, Serialize, Deserialize)]
1809pub struct SelfHealingConfig {
1810 #[serde(default = "default_true")]
1812 pub hook_starts_daemon: bool,
1813 #[serde(default = "default_true")]
1815 pub daemon_installs_hooks: bool,
1816 #[serde(default = "default_autostart_cooldown_secs")]
1818 pub auto_start_cooldown_secs: u64,
1819 #[serde(
1822 default = "default_autostart_timeout_secs",
1823 alias = "daemon_start_timeout"
1824 )]
1825 pub auto_start_timeout_secs: u64,
1826 #[serde(default)]
1829 pub self_healing_log_level: SelfHealingLogLevel,
1830}
1831
1832impl Default for SelfHealingConfig {
1833 fn default() -> Self {
1834 Self {
1835 hook_starts_daemon: default_true(),
1836 daemon_installs_hooks: default_true(),
1837 auto_start_cooldown_secs: default_autostart_cooldown_secs(),
1838 auto_start_timeout_secs: default_autostart_timeout_secs(),
1839 self_healing_log_level: SelfHealingLogLevel::default(),
1840 }
1841 }
1842}
1843
1844impl SelfHealingConfig {
1845 pub fn with_env_overrides(mut self) -> Self {
1854 if let Ok(val) = std::env::var("RCH_NO_SELF_HEALING")
1856 && (val == "1" || val.eq_ignore_ascii_case("true"))
1857 {
1858 self.hook_starts_daemon = false;
1859 self.daemon_installs_hooks = false;
1860 return self;
1861 }
1862
1863 if let Ok(val) = std::env::var("RCH_HOOK_STARTS_DAEMON") {
1865 self.hook_starts_daemon = val != "0" && !val.eq_ignore_ascii_case("false");
1866 }
1867 if let Ok(val) = std::env::var("RCH_DAEMON_INSTALLS_HOOKS") {
1868 self.daemon_installs_hooks = val != "0" && !val.eq_ignore_ascii_case("false");
1869 }
1870
1871 if let Ok(val) = std::env::var("RCH_AUTO_START_TIMEOUT_SECS")
1873 && let Ok(secs) = val.parse()
1874 {
1875 self.auto_start_timeout_secs = secs;
1876 }
1877 if let Ok(val) = std::env::var("RCH_AUTO_START_COOLDOWN_SECS")
1878 && let Ok(secs) = val.parse()
1879 {
1880 self.auto_start_cooldown_secs = secs;
1881 }
1882 if let Ok(val) = std::env::var("RCH_SELF_HEALING_LOG_LEVEL") {
1884 if let Some(level) = SelfHealingLogLevel::from_env_str(&val) {
1885 self.self_healing_log_level = level;
1886 } else {
1887 tracing::warn!(
1888 target: "rch::self_healing",
1889 invalid = %val,
1890 "RCH_SELF_HEALING_LOG_LEVEL has invalid value; using default=info"
1891 );
1892 }
1893 }
1894
1895 self
1896 }
1897}
1898
1899#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1905#[serde(rename_all = "snake_case")]
1906pub enum SelfTestFailureAction {
1907 #[default]
1909 Alert,
1910 DisableWorker,
1912 AlertAndDisable,
1914}
1915
1916#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1918#[serde(untagged)]
1919pub enum SelfTestWorkers {
1920 All(String),
1922 List(Vec<WorkerId>),
1924}
1925
1926impl Default for SelfTestWorkers {
1927 fn default() -> Self {
1928 Self::All("all".to_string())
1929 }
1930}
1931
1932impl SelfTestWorkers {
1933 pub fn resolve(&self) -> Option<Vec<WorkerId>> {
1935 match self {
1936 SelfTestWorkers::All(value) => {
1937 if value.eq_ignore_ascii_case("all") {
1938 None
1939 } else {
1940 Some(vec![WorkerId::new(value.clone())])
1941 }
1942 }
1943 SelfTestWorkers::List(list) => Some(list.clone()),
1944 }
1945 }
1946}
1947
1948fn default_self_test_retry_count() -> u32 {
1949 3
1950}
1951
1952fn default_self_test_retry_delay() -> String {
1953 "5m".to_string()
1954}
1955
1956fn default_self_test_enabled() -> bool {
1957 false
1958}
1959
1960#[derive(Debug, Clone, Serialize, Deserialize)]
1962pub struct SelfTestConfig {
1963 #[serde(default = "default_self_test_enabled")]
1965 pub enabled: bool,
1966 #[serde(default)]
1968 pub schedule: Option<String>,
1969 #[serde(default)]
1971 pub interval: Option<String>,
1972 #[serde(default)]
1974 pub workers: SelfTestWorkers,
1975 #[serde(default)]
1977 pub on_failure: SelfTestFailureAction,
1978 #[serde(default = "default_self_test_retry_count")]
1980 pub retry_count: u32,
1981 #[serde(default = "default_self_test_retry_delay")]
1983 pub retry_delay: String,
1984}
1985
1986impl Default for SelfTestConfig {
1987 fn default() -> Self {
1988 Self {
1989 enabled: default_self_test_enabled(),
1990 schedule: None,
1991 interval: None,
1992 workers: SelfTestWorkers::default(),
1993 on_failure: SelfTestFailureAction::default(),
1994 retry_count: default_self_test_retry_count(),
1995 retry_delay: default_self_test_retry_delay(),
1996 }
1997 }
1998}
1999
2000#[derive(Debug, Clone, Serialize, Deserialize)]
2002pub struct CircuitBreakerConfig {
2003 #[serde(default = "default_circuit_failure_threshold")]
2005 pub failure_threshold: u32,
2006 #[serde(default = "default_circuit_success_threshold")]
2008 pub success_threshold: u32,
2009 #[serde(default = "default_circuit_error_rate_threshold")]
2011 pub error_rate_threshold: f64,
2012 #[serde(default = "default_circuit_window_secs")]
2014 pub window_secs: u64,
2015 #[serde(default = "default_circuit_open_cooldown_secs")]
2017 pub open_cooldown_secs: u64,
2018 #[serde(default = "default_circuit_half_open_max_probes")]
2020 pub half_open_max_probes: u32,
2021 #[serde(default = "default_circuit_command_failure_threshold")]
2032 pub command_failure_threshold: u32,
2033 #[serde(default = "default_circuit_open_cooldown_max_secs")]
2041 pub open_cooldown_max_secs: u64,
2042}
2043
2044impl Default for CircuitBreakerConfig {
2045 fn default() -> Self {
2046 Self {
2047 failure_threshold: default_circuit_failure_threshold(),
2048 success_threshold: default_circuit_success_threshold(),
2049 error_rate_threshold: default_circuit_error_rate_threshold(),
2050 window_secs: default_circuit_window_secs(),
2051 open_cooldown_secs: default_circuit_open_cooldown_secs(),
2052 half_open_max_probes: default_circuit_half_open_max_probes(),
2053 command_failure_threshold: default_circuit_command_failure_threshold(),
2054 open_cooldown_max_secs: default_circuit_open_cooldown_max_secs(),
2055 }
2056 }
2057}
2058
2059impl CircuitBreakerConfig {
2060 pub fn cooldown_ms_for_streak(&self, reopen_streak: u32) -> u64 {
2064 let base_ms = self.open_cooldown_secs.saturating_mul(1000);
2065 let cap_ms = self
2066 .open_cooldown_max_secs
2067 .saturating_mul(1000)
2068 .max(base_ms);
2069 let exponent = reopen_streak.saturating_sub(1).min(16);
2071 base_ms.saturating_mul(1u64 << exponent).min(cap_ms)
2072 }
2073}
2074
2075#[derive(Debug, Clone, Default)]
2080pub struct CircuitStats {
2081 state: CircuitState,
2083 consecutive_failures: u32,
2085 consecutive_successes: u32,
2087 window_successes: u32,
2089 window_failures: u32,
2091 opened_at: Option<u64>,
2093 last_state_change: u64,
2095 active_probes: u32,
2097 consecutive_command_failures: u32,
2102 reopen_streak: u32,
2105 recent_results: Vec<bool>,
2108}
2109
2110const CIRCUIT_HISTORY_SIZE: usize = 10;
2112
2113impl CircuitStats {
2114 pub fn new() -> Self {
2116 Self {
2117 state: CircuitState::Closed,
2118 consecutive_failures: 0,
2119 consecutive_successes: 0,
2120 window_successes: 0,
2121 window_failures: 0,
2122 opened_at: None,
2123 last_state_change: Self::now_millis(),
2124 active_probes: 0,
2125 consecutive_command_failures: 0,
2126 reopen_streak: 0,
2127 recent_results: Vec::with_capacity(CIRCUIT_HISTORY_SIZE),
2128 }
2129 }
2130
2131 pub fn consecutive_command_failures(&self) -> u32 {
2134 self.consecutive_command_failures
2135 }
2136
2137 pub fn reopen_streak(&self) -> u32 {
2140 self.reopen_streak
2141 }
2142
2143 fn effective_cooldown_ms(&self, config: &CircuitBreakerConfig) -> u64 {
2145 config.cooldown_ms_for_streak(self.reopen_streak.max(1))
2146 }
2147
2148 pub fn state(&self) -> CircuitState {
2150 self.state
2151 }
2152
2153 pub fn opened_at(&self) -> Option<u64> {
2155 self.opened_at
2156 }
2157
2158 pub fn last_state_change(&self) -> u64 {
2160 self.last_state_change
2161 }
2162
2163 pub fn consecutive_failures(&self) -> u32 {
2165 self.consecutive_failures
2166 }
2167
2168 pub fn error_rate(&self) -> f64 {
2178 if self.recent_results.is_empty() {
2179 return 0.0;
2180 }
2181 let failures = self.recent_results.iter().filter(|&&ok| !ok).count();
2182 failures as f64 / self.recent_results.len() as f64
2183 }
2184
2185 pub fn record_success(&mut self) {
2190 self.window_successes += 1;
2191 self.consecutive_failures = 0;
2192 self.push_result(true);
2193
2194 if self.state == CircuitState::HalfOpen {
2195 self.consecutive_successes += 1;
2196 if self.active_probes > 0 {
2197 self.active_probes -= 1;
2198 }
2199 }
2200 }
2201
2202 pub fn record_failure(&mut self) {
2207 self.window_failures += 1;
2208 self.consecutive_failures += 1;
2209 self.push_result(false);
2210
2211 if self.state == CircuitState::HalfOpen {
2212 self.consecutive_successes = 0;
2213 if self.active_probes > 0 {
2214 self.active_probes -= 1;
2215 }
2216 }
2217 }
2218
2219 pub fn should_open(&self, config: &CircuitBreakerConfig) -> bool {
2225 if self.state != CircuitState::Closed {
2226 return false;
2227 }
2228
2229 if self.consecutive_failures >= config.failure_threshold {
2231 return true;
2232 }
2233
2234 if self.recent_results.len() >= 5 && self.error_rate() >= config.error_rate_threshold {
2238 return true;
2239 }
2240
2241 false
2242 }
2243
2244 pub fn should_half_open(&self, config: &CircuitBreakerConfig) -> bool {
2248 if self.state != CircuitState::Open {
2249 return false;
2250 }
2251
2252 let now = Self::now_millis();
2253 let cooldown_ms = self.effective_cooldown_ms(config);
2254
2255 if let Some(opened_at) = self.opened_at {
2256 now.saturating_sub(opened_at) >= cooldown_ms
2257 } else {
2258 false
2259 }
2260 }
2261
2262 pub fn should_close(&self, config: &CircuitBreakerConfig) -> bool {
2271 if self.state != CircuitState::HalfOpen {
2272 return false;
2273 }
2274
2275 self.consecutive_successes >= config.success_threshold
2276 && self.consecutive_command_failures == 0
2277 }
2278
2279 pub fn can_probe(&self, config: &CircuitBreakerConfig) -> bool {
2283 if self.state != CircuitState::HalfOpen {
2284 return false;
2285 }
2286
2287 self.active_probes < config.half_open_max_probes
2288 }
2289
2290 pub fn start_probe(&mut self, config: &CircuitBreakerConfig) -> bool {
2294 if !self.can_probe(config) {
2295 return false;
2296 }
2297 self.active_probes += 1;
2298 true
2299 }
2300
2301 pub fn open(&mut self) {
2303 if self.state != CircuitState::Open {
2304 self.state = CircuitState::Open;
2305 self.opened_at = Some(Self::now_millis());
2306 self.last_state_change = Self::now_millis();
2307 self.consecutive_successes = 0;
2308 self.active_probes = 0;
2309 self.reopen_streak = self.reopen_streak.saturating_add(1);
2312 }
2313 }
2314
2315 pub fn half_open(&mut self) {
2317 if self.state != CircuitState::HalfOpen {
2318 self.state = CircuitState::HalfOpen;
2319 self.last_state_change = Self::now_millis();
2320 self.consecutive_successes = 0;
2321 self.active_probes = 0;
2322 }
2323 }
2324
2325 pub fn close(&mut self) {
2327 if self.state != CircuitState::Closed {
2328 self.state = CircuitState::Closed;
2329 self.last_state_change = Self::now_millis();
2330 self.opened_at = None;
2331 self.consecutive_failures = 0;
2332 self.consecutive_successes = 0;
2333 self.active_probes = 0;
2334 self.consecutive_command_failures = 0;
2335 self.reopen_streak = 0;
2336 self.window_successes = 0;
2342 self.window_failures = 0;
2343 self.recent_results.clear();
2344 }
2345 }
2346
2347 pub fn reset_window(&mut self) {
2353 self.window_successes = 0;
2354 self.window_failures = 0;
2355 self.recent_results.clear();
2356 }
2357
2358 pub fn recent_results(&self) -> &[bool] {
2363 &self.recent_results
2364 }
2365
2366 pub fn recovery_remaining_secs(&self, config: &CircuitBreakerConfig) -> Option<u64> {
2370 if self.state != CircuitState::Open {
2371 return None;
2372 }
2373
2374 let now = Self::now_millis();
2375 let cooldown_ms = self.effective_cooldown_ms(config);
2376
2377 if let Some(opened_at) = self.opened_at {
2378 let elapsed_ms = now.saturating_sub(opened_at);
2379 if elapsed_ms < cooldown_ms {
2380 return Some((cooldown_ms - elapsed_ms) / 1000);
2381 }
2382 }
2383 None
2384 }
2385
2386 fn push_result(&mut self, success: bool) {
2388 if self.recent_results.len() >= CIRCUIT_HISTORY_SIZE {
2389 self.recent_results.remove(0);
2390 }
2391 self.recent_results.push(success);
2392 }
2393
2394 fn now_millis() -> u64 {
2395 std::time::SystemTime::now()
2396 .duration_since(std::time::UNIX_EPOCH)
2397 .map(|d| d.as_millis() as u64)
2398 .unwrap_or(0)
2399 }
2400
2401 pub fn apply_health_outcome(
2427 &mut self,
2428 healthy: bool,
2429 transient: bool,
2430 config: &CircuitBreakerConfig,
2431 ) -> CircuitState {
2432 if healthy {
2433 if self.state == CircuitState::Open && self.should_half_open(config) {
2437 self.half_open();
2438 }
2439 self.record_success();
2440 if self.should_close(config) {
2441 self.close();
2442 }
2443 } else {
2444 let was_half_open = self.state == CircuitState::HalfOpen;
2445 self.record_failure();
2446 if self.should_open(config) {
2447 self.open();
2448 } else if was_half_open && !transient {
2449 self.open();
2453 }
2454 }
2455
2456 if self.state == CircuitState::Open && self.should_half_open(config) {
2459 self.half_open();
2460 }
2461
2462 self.state
2463 }
2464
2465 pub fn apply_command_outcome(
2489 &mut self,
2490 success: bool,
2491 config: &CircuitBreakerConfig,
2492 ) -> CircuitState {
2493 if success {
2494 self.consecutive_command_failures = 0;
2495 return self.apply_health_outcome(true, false, config);
2496 }
2497
2498 self.consecutive_command_failures = self.consecutive_command_failures.saturating_add(1);
2499 let was_half_open = self.state == CircuitState::HalfOpen;
2500 self.record_failure();
2501 let threshold = config.command_failure_threshold.max(1);
2502 if was_half_open
2503 || self.should_open(config)
2504 || (self.state == CircuitState::Closed
2505 && self.consecutive_command_failures >= threshold)
2506 {
2507 self.open();
2508 }
2509
2510 if self.state == CircuitState::Open && self.should_half_open(config) {
2511 self.half_open();
2512 }
2513
2514 self.state
2515 }
2516}
2517
2518#[derive(Debug, Clone, Serialize, Deserialize)]
2519pub struct CompilationConfig {
2520 #[serde(default = "default_confidence")]
2522 pub confidence_threshold: f64,
2523 #[serde(default = "default_min_local_time")]
2526 pub min_local_time_ms: u64,
2527 #[serde(default = "default_speedup_threshold")]
2531 pub remote_speedup_threshold: f64,
2532 #[serde(default)]
2538 pub local_slots: u32,
2539 #[serde(default = "default_build_slots")]
2541 pub build_slots: u32,
2542 #[serde(default = "default_test_slots")]
2545 pub test_slots: u32,
2546 #[serde(default = "default_check_slots")]
2549 pub check_slots: u32,
2550 #[serde(default)]
2556 pub disk_headroom_gib: u32,
2557 #[serde(default = "default_build_timeout")]
2560 pub build_timeout_sec: u64,
2561 #[serde(default = "default_test_timeout")]
2564 pub test_timeout_sec: u64,
2565 #[serde(default = "default_bun_timeout")]
2569 pub bun_timeout_sec: u64,
2570 #[serde(default = "default_external_timeout_enabled")]
2574 pub external_timeout_enabled: bool,
2575 #[serde(default = "default_allow_local_fallback")]
2588 pub allow_local_fallback: bool,
2589 #[serde(default)]
2600 pub remote_build_jobs: RemoteBuildJobs,
2601}
2602
2603#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
2615pub enum RemoteBuildJobs {
2616 #[default]
2618 Auto,
2619 Off,
2621 Fixed(u32),
2623}
2624
2625impl RemoteBuildJobs {
2626 pub fn parse(value: &str) -> Result<Self, String> {
2628 let trimmed = value.trim();
2629 match trimmed.to_ascii_lowercase().as_str() {
2630 "auto" => Ok(Self::Auto),
2631 "off" | "none" | "disabled" => Ok(Self::Off),
2632 _ => match trimmed.parse::<u32>() {
2633 Ok(0) => Err(
2634 "remote_build_jobs must be >= 1 (cargo rejects `-j 0`); use `off` to disable"
2635 .to_string(),
2636 ),
2637 Ok(n) => Ok(Self::Fixed(n)),
2638 Err(_) => Err(format!(
2639 "'{trimmed}' is not a valid remote_build_jobs value (expected `auto`, `off`, or a positive integer)"
2640 )),
2641 },
2642 }
2643 }
2644}
2645
2646impl std::fmt::Display for RemoteBuildJobs {
2647 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2648 match self {
2649 Self::Auto => f.write_str("auto"),
2650 Self::Off => f.write_str("off"),
2651 Self::Fixed(n) => write!(f, "{n}"),
2652 }
2653 }
2654}
2655
2656impl std::str::FromStr for RemoteBuildJobs {
2657 type Err = String;
2658
2659 fn from_str(value: &str) -> Result<Self, Self::Err> {
2660 Self::parse(value)
2661 }
2662}
2663
2664impl Serialize for RemoteBuildJobs {
2665 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
2666 match self {
2667 Self::Auto => serializer.serialize_str("auto"),
2668 Self::Off => serializer.serialize_str("off"),
2669 Self::Fixed(n) => serializer.serialize_u32(*n),
2670 }
2671 }
2672}
2673
2674impl<'de> Deserialize<'de> for RemoteBuildJobs {
2675 fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
2676 #[derive(Deserialize)]
2677 #[serde(untagged)]
2678 enum Raw {
2679 Count(u64),
2680 Spelling(String),
2681 }
2682 match Raw::deserialize(deserializer)? {
2683 Raw::Count(n) => {
2684 let n = u32::try_from(n).map_err(|_| {
2685 serde::de::Error::custom(format!("remote_build_jobs {n} is out of range"))
2686 })?;
2687 Self::parse(&n.to_string()).map_err(serde::de::Error::custom)
2688 }
2689 Raw::Spelling(s) => Self::parse(&s).map_err(serde::de::Error::custom),
2690 }
2691 }
2692}
2693
2694impl Default for CompilationConfig {
2695 fn default() -> Self {
2696 Self {
2697 confidence_threshold: 0.85,
2698 min_local_time_ms: 2000,
2699 remote_speedup_threshold: default_speedup_threshold(),
2700 local_slots: 0,
2701 build_slots: default_build_slots(),
2702 test_slots: default_test_slots(),
2703 check_slots: default_check_slots(),
2704 disk_headroom_gib: 0,
2705 build_timeout_sec: default_build_timeout(),
2706 test_timeout_sec: default_test_timeout(),
2707 bun_timeout_sec: default_bun_timeout(),
2708 external_timeout_enabled: default_external_timeout_enabled(),
2709 allow_local_fallback: default_allow_local_fallback(),
2710 remote_build_jobs: RemoteBuildJobs::default(),
2711 }
2712 }
2713}
2714
2715fn default_build_slots() -> u32 {
2716 4
2717}
2718
2719fn default_test_slots() -> u32 {
2720 8
2721}
2722
2723fn default_check_slots() -> u32 {
2724 2
2725}
2726
2727fn default_build_timeout() -> u64 {
2730 300
2731}
2732
2733fn default_test_timeout() -> u64 {
2736 1800
2737}
2738
2739fn default_bun_timeout() -> u64 {
2743 600
2744}
2745
2746fn default_external_timeout_enabled() -> bool {
2748 true
2749}
2750
2751fn default_allow_local_fallback() -> bool {
2755 true
2756}
2757
2758impl CompilationConfig {
2759 pub fn timeout_for_kind(&self, kind: Option<CompilationKind>) -> std::time::Duration {
2765 let secs = match kind {
2766 Some(CompilationKind::BunTest) | Some(CompilationKind::BunTypecheck) => {
2768 self.bun_timeout_sec
2769 }
2770 Some(CompilationKind::CargoTest) | Some(CompilationKind::CargoNextest) => {
2772 self.test_timeout_sec
2773 }
2774 Some(CompilationKind::NixBuild) => self.test_timeout_sec,
2778 Some(CompilationKind::GoTest) => self.test_timeout_sec,
2781 _ => self.build_timeout_sec,
2783 };
2784 std::time::Duration::from_secs(secs)
2785 }
2786
2787 pub fn external_timeout_enabled(&self) -> bool {
2789 self.external_timeout_enabled
2790 }
2791}
2792
2793pub const LAYER0_PACK_VERSION: &str = "1";
2799
2800#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
2809#[serde(default)]
2810pub struct Layer0Config {
2811 pub enabled: bool,
2813 pub release_lto_thin: bool,
2815 pub release_codegen_units_1: bool,
2818}
2819
2820impl Layer0Config {
2821 #[must_use]
2827 pub fn active_env(&self) -> Vec<(&'static str, &'static str)> {
2828 if !self.enabled {
2829 return Vec::new();
2830 }
2831 let mut env = Vec::new();
2832 if self.release_lto_thin {
2833 env.push(("CARGO_PROFILE_RELEASE_LTO", "thin"));
2834 }
2835 if self.release_codegen_units_1 {
2836 env.push(("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"));
2837 }
2838 env
2839 }
2840}
2841
2842#[derive(Debug, Clone, Serialize, Deserialize)]
2843pub struct TransferConfig {
2844 #[serde(default = "default_compression")]
2846 pub compression_level: u32,
2847 #[serde(default = "default_excludes")]
2849 pub exclude_patterns: Vec<String>,
2850 #[serde(default, skip_serializing_if = "Option::is_none")]
2854 pub ssh_server_alive_interval_secs: Option<u64>,
2855 #[serde(default, skip_serializing_if = "Option::is_none")]
2860 pub ssh_control_persist_secs: Option<u64>,
2861 #[serde(default = "default_remote_base")]
2872 pub remote_base: String,
2873 #[serde(default, skip_serializing_if = "Option::is_none")]
2879 pub sync_timeout_ms: Option<u64>,
2880 #[serde(default = "default_source_sync_silence_timeout_secs")]
2892 pub source_sync_silence_timeout_secs: u64,
2893 #[serde(default)]
2895 pub retry: RetryConfig,
2896 #[serde(default)]
2902 pub verify_artifacts: bool,
2903 #[serde(default = "default_verify_max_size")]
2908 pub verify_max_size_bytes: u64,
2909
2910 #[serde(default, skip_serializing_if = "Option::is_none")]
2919 pub max_transfer_mb: Option<u64>,
2920
2921 #[serde(default, skip_serializing_if = "Option::is_none")]
2927 pub max_transfer_time_ms: Option<u64>,
2928
2929 #[serde(default, skip_serializing_if = "Option::is_none")]
2934 pub bwlimit_kbps: Option<u64>,
2935
2936 #[serde(default, skip_serializing_if = "Option::is_none")]
2945 pub rsync_bin: Option<String>,
2946
2947 #[serde(default, skip_serializing_if = "Option::is_none")]
2952 pub estimated_bandwidth_bps: Option<u64>,
2953
2954 #[serde(default)]
2967 pub adaptive_compression: bool,
2968
2969 #[serde(default = "default_min_compression")]
2974 pub min_compression_level: u32,
2975
2976 #[serde(default = "default_max_compression")]
2981 pub max_compression_level: u32,
2982}
2983
2984impl Default for TransferConfig {
2985 fn default() -> Self {
2986 Self {
2987 compression_level: 3,
2988 exclude_patterns: default_excludes(),
2989 ssh_server_alive_interval_secs: None,
2990 ssh_control_persist_secs: None,
2991 remote_base: default_remote_base(),
2992 sync_timeout_ms: None,
2993 source_sync_silence_timeout_secs: default_source_sync_silence_timeout_secs(),
2994 retry: RetryConfig::default(),
2995 verify_artifacts: false,
2996 verify_max_size_bytes: default_verify_max_size(),
2997 max_transfer_mb: None,
2999 max_transfer_time_ms: None,
3000 bwlimit_kbps: None,
3001 rsync_bin: None,
3002 estimated_bandwidth_bps: None,
3003 adaptive_compression: false,
3005 min_compression_level: default_min_compression(),
3006 max_compression_level: default_max_compression(),
3007 }
3008 }
3009}
3010
3011impl TransferConfig {
3012 pub const MIN_SYNC_TIMEOUT_MS: u64 = 1_000;
3014 pub const MAX_SYNC_TIMEOUT_MS: u64 = 3_600_000;
3016 pub const DEFAULT_SYNC_TIMEOUT_FLOOR_MS: u64 = 30_000;
3018 const DEFAULT_SYNC_BYTES_PER_SECOND: u64 = 1024 * 1024;
3019
3020 #[must_use]
3022 pub fn valid_sync_timeout_ms(value: u64) -> bool {
3023 (Self::MIN_SYNC_TIMEOUT_MS..=Self::MAX_SYNC_TIMEOUT_MS).contains(&value)
3024 }
3025
3026 #[must_use]
3033 pub fn sync_timeout_for_payload(&self, payload_bytes: u64) -> std::time::Duration {
3034 let timeout_ms = self
3035 .sync_timeout_ms
3036 .filter(|value| Self::valid_sync_timeout_ms(*value))
3037 .unwrap_or_else(|| {
3038 let transfer_seconds = payload_bytes
3039 .saturating_add(Self::DEFAULT_SYNC_BYTES_PER_SECOND - 1)
3040 / Self::DEFAULT_SYNC_BYTES_PER_SECOND;
3041 Self::DEFAULT_SYNC_TIMEOUT_FLOOR_MS
3042 .saturating_add(transfer_seconds.saturating_mul(1_000))
3043 .min(Self::MAX_SYNC_TIMEOUT_MS)
3044 });
3045 std::time::Duration::from_millis(timeout_ms)
3046 }
3047
3048 pub fn select_compression_level(&self, estimated_bytes: Option<u64>) -> u32 {
3060 if !self.adaptive_compression {
3061 return self.compression_level;
3062 }
3063
3064 let Some(bytes) = estimated_bytes else {
3065 return self.compression_level;
3067 };
3068
3069 const SMALL_THRESHOLD: u64 = 10_000_000; const LARGE_THRESHOLD: u64 = 200_000_000; let level = if bytes < SMALL_THRESHOLD {
3074 1 } else if bytes < LARGE_THRESHOLD {
3076 3 } else {
3078 7 };
3080
3081 level.clamp(self.min_compression_level, self.max_compression_level)
3083 }
3084}
3085
3086#[rustfmt::skip]
3100pub const BUILTIN_EXECUTION_BASES: &[&str] = &[
3101 "cargo",
3103 "rustc",
3104 "nextest",
3106 "gcc",
3108 "g++",
3109 "clang",
3110 "clang++",
3111 "cc",
3112 "c++",
3113 "make",
3115 "cmake",
3116 "ninja",
3117 "meson",
3118 "bun",
3120 "nix",
3122 "go",
3124 "tsc",
3126];
3127
3128fn default_execution_allowlist() -> Vec<String> {
3129 BUILTIN_EXECUTION_BASES
3130 .iter()
3131 .map(|base| (*base).to_string())
3132 .collect()
3133}
3134
3135#[derive(Debug, Clone, Serialize, Deserialize)]
3140pub struct ExecutionConfig {
3141 #[serde(default = "default_execution_allowlist")]
3150 pub allowlist: Vec<String>,
3151}
3152
3153impl Default for ExecutionConfig {
3154 fn default() -> Self {
3155 Self {
3156 allowlist: default_execution_allowlist(),
3157 }
3158 }
3159}
3160
3161impl ExecutionConfig {
3162 pub fn is_allowed(&self, command_base: &str) -> bool {
3164 self.allowlist
3165 .iter()
3166 .any(|allowed| allowed.eq_ignore_ascii_case(command_base))
3167 }
3168
3169 pub fn missing_builtin_bases(&self) -> Vec<&'static str> {
3175 BUILTIN_EXECUTION_BASES
3176 .iter()
3177 .copied()
3178 .filter(|base| !self.is_allowed(base))
3179 .collect()
3180 }
3181}
3182
3183fn default_retry_max_attempts() -> u32 {
3188 3
3189}
3190
3191fn default_retry_base_delay_ms() -> u64 {
3192 100
3193}
3194
3195fn default_retry_max_delay_ms() -> u64 {
3196 5000
3197}
3198
3199fn default_retry_jitter_factor() -> f64 {
3200 0.1
3201}
3202
3203fn default_retry_total_timeout_ms() -> u64 {
3204 30000
3205}
3206
3207#[derive(Debug, Clone, Serialize, Deserialize)]
3212pub struct RetryConfig {
3213 #[serde(default = "default_retry_max_attempts")]
3215 pub max_attempts: u32,
3216
3217 #[serde(default = "default_retry_base_delay_ms")]
3219 pub base_delay_ms: u64,
3220
3221 #[serde(default = "default_retry_max_delay_ms")]
3223 pub max_delay_ms: u64,
3224
3225 #[serde(default = "default_retry_jitter_factor")]
3228 pub jitter_factor: f64,
3229
3230 #[serde(default = "default_retry_total_timeout_ms")]
3233 pub total_timeout_ms: u64,
3234}
3235
3236impl Default for RetryConfig {
3237 fn default() -> Self {
3238 Self {
3239 max_attempts: default_retry_max_attempts(),
3240 base_delay_ms: default_retry_base_delay_ms(),
3241 max_delay_ms: default_retry_max_delay_ms(),
3242 jitter_factor: default_retry_jitter_factor(),
3243 total_timeout_ms: default_retry_total_timeout_ms(),
3244 }
3245 }
3246}
3247
3248impl RetryConfig {
3249 pub fn no_retry() -> Self {
3251 Self {
3252 max_attempts: 1,
3253 ..Default::default()
3254 }
3255 }
3256
3257 pub fn delay_for_attempt(&self, attempt: u32) -> std::time::Duration {
3261 if attempt == 0 {
3262 return std::time::Duration::ZERO;
3263 }
3264
3265 let base = self.base_delay_ms as f64;
3267 let delay = base * (2.0_f64.powi(attempt as i32 - 1));
3268 let capped = delay.min(self.max_delay_ms as f64);
3269
3270 let jittered = if self.jitter_factor > 0.0 {
3272 let jitter_range = capped * self.jitter_factor;
3273 let jitter = (rand::rng().random::<f64>() * 2.0 - 1.0) * jitter_range;
3274 (capped + jitter).max(0.0)
3275 } else {
3276 capped
3277 };
3278
3279 std::time::Duration::from_millis(jittered as u64)
3280 }
3281
3282 pub fn should_retry(&self, attempt: u32, elapsed: std::time::Duration) -> bool {
3284 if attempt >= self.max_attempts {
3285 return false;
3286 }
3287 elapsed.as_millis() < self.total_timeout_ms as u128
3288 }
3289}
3290
3291fn default_source_sync_silence_timeout_secs() -> u64 {
3296 120
3297}
3298
3299pub fn default_remote_base() -> String {
3308 "/data/tmp/rch".to_string()
3309}
3310
3311pub fn validate_remote_base(path: &str) -> Result<String, String> {
3314 let expanded = shellexpand::tilde(path).into_owned();
3316
3317 if !expanded.starts_with('/') {
3319 return Err(format!(
3320 "remote_base must be an absolute path, got: {}",
3321 path
3322 ));
3323 }
3324
3325 if expanded.contains("..") {
3327 return Err(format!(
3328 "remote_base must not contain path traversal (..): {}",
3329 path
3330 ));
3331 }
3332
3333 let normalized = expanded.trim_end_matches('/').to_string();
3335
3336 if normalized.is_empty() || normalized == "/" {
3338 return Err("remote_base cannot be the root directory (safety restriction)".to_string());
3339 }
3340
3341 let components: Vec<&str> = normalized.split('/').filter(|c| !c.is_empty()).collect();
3345 if components.len() < 2 {
3346 return Err(format!(
3347 "remote_base must be at least 2 levels deep (e.g. /tmp/rch), got: {}",
3348 normalized
3349 ));
3350 }
3351
3352 Ok(normalized)
3353}
3354
3355fn default_circuit_failure_threshold() -> u32 {
3356 3
3357}
3358
3359fn default_circuit_success_threshold() -> u32 {
3360 2
3361}
3362
3363fn default_circuit_error_rate_threshold() -> f64 {
3364 0.5
3365}
3366
3367fn default_circuit_window_secs() -> u64 {
3368 60
3369}
3370
3371fn default_circuit_open_cooldown_secs() -> u64 {
3372 30
3373}
3374
3375fn default_circuit_half_open_max_probes() -> u32 {
3376 1
3377}
3378
3379fn default_circuit_command_failure_threshold() -> u32 {
3380 2
3381}
3382
3383fn default_circuit_open_cooldown_max_secs() -> u64 {
3384 480
3386}
3387
3388fn default_true() -> bool {
3389 true
3390}
3391
3392fn default_log_level() -> String {
3393 "info".to_string()
3394}
3395
3396pub fn default_socket_path() -> String {
3397 if let Ok(runtime_dir) = std::env::var("XDG_RUNTIME_DIR")
3399 && !runtime_dir.trim().is_empty()
3400 {
3401 let path = PathBuf::from(runtime_dir).join("rch.sock");
3402 return path.to_string_lossy().to_string();
3403 }
3404
3405 if let Some(cache_dir) = dirs::cache_dir() {
3407 let rch_cache = cache_dir.join("rch");
3408 let _ = std::fs::create_dir_all(&rch_cache);
3409 return rch_cache.join("rch.sock").to_string_lossy().to_string();
3410 }
3411
3412 "/tmp/rch.sock".to_string()
3414}
3415
3416#[allow(dead_code)]
3417fn default_output_visibility() -> OutputVisibility {
3418 OutputVisibility::None
3419}
3420
3421fn default_confidence() -> f64 {
3422 0.85
3423}
3424
3425fn default_min_local_time() -> u64 {
3426 2000
3427}
3428
3429fn default_speedup_threshold() -> f64 {
3433 1.2
3434}
3435
3436fn default_compression() -> u32 {
3437 3
3438}
3439
3440fn default_verify_max_size() -> u64 {
3442 100 * 1024 * 1024 }
3444
3445fn default_min_compression() -> u32 {
3447 1
3448}
3449
3450fn default_max_compression() -> u32 {
3452 9
3453}
3454
3455fn default_excludes() -> Vec<String> {
3456 vec![
3457 "target/".to_string(),
3459 "*.rlib".to_string(),
3460 "*.rmeta".to_string(),
3461 ".git/".to_string(),
3464 "node_modules/".to_string(),
3466 ".bun/".to_string(),
3468 ".npm/".to_string(),
3470 ".pnpm-store/".to_string(),
3471 "dist/".to_string(),
3473 "build/".to_string(),
3474 ".next/".to_string(),
3476 ".nuxt/".to_string(),
3477 ".turbo/".to_string(),
3478 ".parcel-cache/".to_string(),
3479 ".beads/".to_string(),
3481 ".doctor/".to_string(),
3483 "coverage/".to_string(),
3485 ".nyc_output/".to_string(),
3486 ".cargo/credentials".to_string(),
3488 ".cargo/credentials.toml".to_string(),
3489 ".env".to_string(),
3490 ".env.*".to_string(),
3491 "*.pem".to_string(),
3492 "*.key".to_string(),
3493 "credentials.json".to_string(),
3494 "secrets.json".to_string(),
3495 "secrets.yaml".to_string(),
3496 "secrets.yml".to_string(),
3497 "secrets.toml".to_string(),
3498 "secrets.env".to_string(),
3499 "secrets.txt".to_string(),
3500 "secrets.conf".to_string(),
3501 "secrets.cfg".to_string(),
3502 "secrets.properties".to_string(),
3503 "secrets.xml".to_string(),
3504 "id_rsa".to_string(),
3518 "id_ed25519".to_string(),
3519 "id_ecdsa".to_string(),
3520 "id_dsa".to_string(),
3521 "*.p12".to_string(),
3523 "*.pfx".to_string(),
3524 "*.keystore".to_string(),
3525 "*.jks".to_string(),
3526 ".netrc".to_string(),
3529 "_netrc".to_string(),
3530 ".aws/credentials".to_string(),
3531 ".docker/config.json".to_string(),
3532 ".npmrc".to_string(),
3536 ".pypirc".to_string(),
3537 "secring.*".to_string(),
3539 ]
3540}
3541
3542#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
3548#[serde(rename_all = "snake_case")]
3549pub enum BuildLocation {
3550 #[default]
3552 Local,
3553 Remote,
3555}
3556
3557#[derive(Debug, Clone, Serialize, Deserialize)]
3559pub struct BuildCancellationWorkerHealth {
3560 pub status: String,
3562 pub speed_score: f64,
3564 pub used_slots: u32,
3566 pub available_slots: u32,
3568 pub pressure_state: String,
3570 pub pressure_reason_code: String,
3572}
3573
3574#[derive(Debug, Clone, Serialize, Deserialize)]
3576pub struct BuildCancellationMetadata {
3577 pub operation_id: String,
3579 pub origin: String,
3581 pub reason_code: String,
3583 #[serde(default)]
3585 pub decision_path: Vec<String>,
3586 pub escalation_stage: String,
3588 pub escalation_count: u32,
3590 pub remote_kill_attempted: bool,
3592 pub cleanup_ok: bool,
3594 pub history_cancelled: bool,
3596 pub final_state: String,
3598 #[serde(default, skip_serializing_if = "Option::is_none")]
3600 pub worker_health: Option<BuildCancellationWorkerHealth>,
3601}
3602
3603#[derive(Debug, Clone, Serialize, Deserialize)]
3605pub struct BuildRecord {
3606 pub id: u64,
3608 pub started_at: String,
3610 pub completed_at: String,
3612 pub project_id: String,
3614 pub worker_id: Option<String>,
3616 pub command: String,
3618 pub exit_code: i32,
3620 pub duration_ms: u64,
3622 pub location: BuildLocation,
3624 pub bytes_transferred: Option<u64>,
3626 #[serde(default)]
3628 pub timing: Option<CommandTimingBreakdown>,
3629 #[serde(default, skip_serializing_if = "Option::is_none")]
3631 pub cancellation: Option<BuildCancellationMetadata>,
3632}
3633
3634#[derive(Debug, Clone, Serialize, Deserialize)]
3636pub struct BuildRecordInput {
3637 pub started_at: String,
3639 pub completed_at: String,
3641 pub project_id: String,
3643 pub worker_id: Option<String>,
3645 pub command: String,
3647 pub exit_code: i32,
3649 pub duration_ms: u64,
3651 pub location: BuildLocation,
3653 pub bytes_transferred: Option<u64>,
3655 #[serde(default)]
3657 pub timing: Option<CommandTimingBreakdown>,
3658 #[serde(default, skip_serializing_if = "Option::is_none")]
3660 pub cancellation: Option<BuildCancellationMetadata>,
3661}
3662
3663impl BuildRecordInput {
3664 pub fn into_record(self, id: u64) -> BuildRecord {
3666 BuildRecord {
3667 id,
3668 started_at: self.started_at,
3669 completed_at: self.completed_at,
3670 project_id: self.project_id,
3671 worker_id: self.worker_id,
3672 command: self.command,
3673 exit_code: self.exit_code,
3674 duration_ms: self.duration_ms,
3675 location: self.location,
3676 bytes_transferred: self.bytes_transferred,
3677 timing: self.timing,
3678 cancellation: self.cancellation,
3679 }
3680 }
3681}
3682
3683#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3688pub struct BuildStats {
3689 pub total_builds: usize,
3691 pub success_count: usize,
3693 pub failure_count: usize,
3695 pub remote_count: usize,
3697 pub local_count: usize,
3699 pub avg_duration_ms: u64,
3701}
3702
3703#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3710pub struct SavedTimeStats {
3711 pub total_remote_duration_ms: u64,
3713 pub estimated_local_duration_ms: u64,
3716 pub time_saved_ms: u64,
3719 pub builds_counted: usize,
3721 pub avg_speedup: f64,
3725 pub today_saved_ms: u64,
3727 pub week_saved_ms: u64,
3729 #[serde(default)]
3732 pub estimate_basis: String,
3733 #[serde(default)]
3735 pub local_baseline_builds: usize,
3736}
3737
3738use std::collections::VecDeque;
3743use std::time::{Duration, Instant};
3744
3745#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3747pub struct CompilationTimingBreakdown {
3748 #[serde(with = "duration_millis")]
3750 pub rsync_up: Duration,
3751 #[serde(with = "duration_millis")]
3753 pub remote_build: Duration,
3754 #[serde(with = "duration_millis")]
3756 pub rsync_down: Duration,
3757 #[serde(with = "duration_millis")]
3759 pub total: Duration,
3760}
3761
3762#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3766pub struct CommandTimingBreakdown {
3767 #[serde(with = "option_duration_millis")]
3769 pub classify: Option<Duration>,
3770 #[serde(with = "option_duration_millis")]
3772 pub select: Option<Duration>,
3773 #[serde(with = "option_duration_millis")]
3775 pub sync_up: Option<Duration>,
3776 #[serde(with = "option_duration_millis")]
3778 pub exec: Option<Duration>,
3779 #[serde(with = "option_duration_millis")]
3781 pub sync_down: Option<Duration>,
3782 #[serde(with = "option_duration_millis")]
3784 pub cleanup: Option<Duration>,
3785 #[serde(with = "option_duration_millis")]
3787 pub total: Option<Duration>,
3788}
3789
3790mod duration_millis {
3791 use serde::{Deserialize, Deserializer, Serialize, Serializer};
3792 use std::time::Duration;
3793
3794 pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
3795 where
3796 S: Serializer,
3797 {
3798 duration.as_millis().serialize(serializer)
3799 }
3800
3801 pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
3802 where
3803 D: Deserializer<'de>,
3804 {
3805 let millis = u64::deserialize(deserializer)?;
3806 Ok(Duration::from_millis(millis))
3807 }
3808}
3809
3810#[derive(Debug, Clone, Serialize, Deserialize)]
3812pub struct CompilationMetrics {
3813 pub project_id: String,
3815 pub worker_id: String,
3817 pub timestamp: chrono::DateTime<chrono::Utc>,
3819 pub timing: CompilationTimingBreakdown,
3821 #[serde(with = "option_duration_millis")]
3823 pub local_build_time: Option<Duration>,
3824 pub speedup: Option<f64>,
3826 pub files_synced: u64,
3828 pub bytes_transferred: u64,
3830 pub exit_code: i32,
3832 pub success: bool,
3834}
3835
3836mod option_duration_millis {
3837 use serde::{Deserialize, Deserializer, Serialize, Serializer};
3838 use std::time::Duration;
3839
3840 pub fn serialize<S>(duration: &Option<Duration>, serializer: S) -> Result<S::Ok, S::Error>
3841 where
3842 S: Serializer,
3843 {
3844 match duration {
3845 Some(d) => Some(d.as_millis() as u64).serialize(serializer),
3846 None => Option::<u64>::None.serialize(serializer),
3847 }
3848 }
3849
3850 pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Duration>, D::Error>
3851 where
3852 D: Deserializer<'de>,
3853 {
3854 let opt: Option<u64> = Option::deserialize(deserializer)?;
3855 Ok(opt.map(Duration::from_millis))
3856 }
3857}
3858
3859impl CompilationMetrics {
3860 pub fn calculate_speedup(&mut self) {
3862 if let Some(local) = self.local_build_time
3863 && self.timing.total.as_millis() > 0
3864 {
3865 self.speedup = Some(local.as_secs_f64() / self.timing.total.as_secs_f64());
3866 }
3867 }
3868
3869 pub fn is_beneficial(&self) -> bool {
3871 self.speedup.map(|s| s > 1.0).unwrap_or(false)
3872 }
3873}
3874
3875impl Default for CompilationMetrics {
3876 fn default() -> Self {
3877 Self {
3878 project_id: String::new(),
3879 worker_id: String::new(),
3880 timestamp: chrono::Utc::now(),
3881 timing: CompilationTimingBreakdown::default(),
3882 local_build_time: None,
3883 speedup: None,
3884 files_synced: 0,
3885 bytes_transferred: 0,
3886 exit_code: 0,
3887 success: true,
3888 }
3889 }
3890}
3891
3892#[derive(Debug)]
3908pub struct CompilationTimer {
3909 project_id: String,
3910 worker_id: String,
3911 start: Instant,
3912 phase_start: Instant,
3913 rsync_up: Option<Duration>,
3914 remote_build: Option<Duration>,
3915 rsync_down: Option<Duration>,
3916}
3917
3918impl CompilationTimer {
3919 pub fn new(project_id: &str, worker_id: &str) -> Self {
3921 let now = Instant::now();
3922 Self {
3923 project_id: project_id.to_string(),
3924 worker_id: worker_id.to_string(),
3925 start: now,
3926 phase_start: now,
3927 rsync_up: None,
3928 remote_build: None,
3929 rsync_down: None,
3930 }
3931 }
3932
3933 pub fn end_rsync_up(&mut self) {
3935 self.rsync_up = Some(self.phase_start.elapsed());
3936 self.phase_start = Instant::now();
3937 tracing::info!(
3938 rsync_up_ms = %self.rsync_up.unwrap().as_millis(),
3939 "TIMING: rsync_up completed"
3940 );
3941 }
3942
3943 pub fn end_remote_build(&mut self) {
3945 self.remote_build = Some(self.phase_start.elapsed());
3946 self.phase_start = Instant::now();
3947 tracing::info!(
3948 remote_build_ms = %self.remote_build.unwrap().as_millis(),
3949 "TIMING: remote_build completed"
3950 );
3951 }
3952
3953 pub fn end_rsync_down(&mut self) {
3955 self.rsync_down = Some(self.phase_start.elapsed());
3956 tracing::info!(
3957 rsync_down_ms = %self.rsync_down.unwrap().as_millis(),
3958 "TIMING: rsync_down completed"
3959 );
3960 }
3961
3962 pub fn finish(self, exit_code: i32, files: u64, bytes: u64) -> CompilationMetrics {
3964 let total = self.start.elapsed();
3965 tracing::info!(
3966 total_ms = %total.as_millis(),
3967 exit_code = %exit_code,
3968 "TIMING: compilation completed"
3969 );
3970
3971 CompilationMetrics {
3972 project_id: self.project_id,
3973 worker_id: self.worker_id,
3974 timestamp: chrono::Utc::now(),
3975 timing: CompilationTimingBreakdown {
3976 rsync_up: self.rsync_up.unwrap_or_default(),
3977 remote_build: self.remote_build.unwrap_or_default(),
3978 rsync_down: self.rsync_down.unwrap_or_default(),
3979 total,
3980 },
3981 local_build_time: None,
3982 speedup: None,
3983 files_synced: files,
3984 bytes_transferred: bytes,
3985 exit_code,
3986 success: exit_code == 0,
3987 }
3988 }
3989}
3990
3991#[derive(Debug)]
3993pub struct MetricsAggregator {
3994 history: VecDeque<CompilationMetrics>,
3995 max_history: usize,
3996}
3997
3998impl MetricsAggregator {
3999 pub fn new(max_history: usize) -> Self {
4001 Self {
4002 history: VecDeque::with_capacity(max_history),
4003 max_history,
4004 }
4005 }
4006
4007 pub fn record(&mut self, metrics: CompilationMetrics) {
4009 if self.history.len() >= self.max_history {
4010 self.history.pop_front();
4011 }
4012 self.history.push_back(metrics);
4013 }
4014
4015 pub fn average_speedup(&self) -> Option<f64> {
4017 let speedups: Vec<f64> = self.history.iter().filter_map(|m| m.speedup).collect();
4018
4019 if speedups.is_empty() {
4020 None
4021 } else {
4022 Some(speedups.iter().sum::<f64>() / speedups.len() as f64)
4023 }
4024 }
4025
4026 pub fn p50_total_time(&self) -> Option<Duration> {
4028 self.percentile_total_time(0.50)
4029 }
4030
4031 pub fn p95_total_time(&self) -> Option<Duration> {
4033 self.percentile_total_time(0.95)
4034 }
4035
4036 pub fn p99_total_time(&self) -> Option<Duration> {
4038 self.percentile_total_time(0.99)
4039 }
4040
4041 fn percentile_total_time(&self, percentile: f64) -> Option<Duration> {
4046 let mut times: Vec<_> = self.history.iter().map(|m| m.timing.total).collect();
4047
4048 if times.is_empty() {
4049 return None;
4050 }
4051
4052 times.sort();
4053 let idx = (((times.len() - 1) as f64 * percentile).round() as usize).min(times.len() - 1);
4057 Some(times[idx])
4058 }
4059
4060 pub fn success_rate(&self) -> f64 {
4062 if self.history.is_empty() {
4063 return 100.0;
4064 }
4065 let successes = self.history.iter().filter(|m| m.success).count();
4066 (successes as f64 / self.history.len() as f64) * 100.0
4067 }
4068
4069 pub fn count(&self) -> usize {
4071 self.history.len()
4072 }
4073
4074 pub fn metrics(&self) -> &VecDeque<CompilationMetrics> {
4076 &self.history
4077 }
4078
4079 pub fn clear(&mut self) {
4081 self.history.clear();
4082 }
4083}
4084
4085impl Default for MetricsAggregator {
4086 fn default() -> Self {
4087 Self::new(1000)
4088 }
4089}
4090
4091#[cfg(test)]
4092mod tests {
4093 use super::*;
4094 use crate::test_guard;
4095
4096 #[test]
4097 fn test_declared_os_reads_reserved_tag() {
4098 assert_eq!(
4099 declared_os(&["rust".to_string(), "os:windows".to_string()]),
4100 Some("windows".to_string())
4101 );
4102 assert_eq!(
4103 declared_os(&["os:WINDOWS".to_string()]),
4104 Some("windows".to_string())
4105 );
4106 }
4107
4108 #[test]
4109 fn test_declared_os_absent_for_ordinary_tags() {
4110 assert_eq!(declared_os(&[]), None);
4113 assert_eq!(
4114 declared_os(&["rust".to_string(), "bun".to_string(), "go".to_string()]),
4115 None
4116 );
4117 assert_eq!(declared_os(&["os:".to_string()]), None);
4119 assert_eq!(declared_os(&["os: ".to_string()]), None);
4120 }
4121
4122 #[test]
4123 fn test_os_tag_roundtrips_through_declared_os() {
4124 for os in ["windows", "Darwin", " linux "] {
4125 let tag = os_tag(os);
4126 assert_eq!(
4127 declared_os(std::slice::from_ref(&tag)).as_deref(),
4128 Some(os.trim().to_ascii_lowercase().as_str())
4129 );
4130 }
4131 }
4132
4133 #[test]
4138 fn test_no_inline_secret_fields() {
4139 let endpoint = DoctorWebhookEndpoint {
4140 name: "ops".to_string(),
4141 url: "https://example.com/hook".to_string(),
4142 format: DoctorWebhookFormat::Pagerduty,
4143 on_transitions: vec!["any_to_failing".to_string()],
4144 retry_max: 3,
4145 retry_backoff_ms: 200,
4146 timeout_ms: 5000,
4147 enabled: true,
4148 routing_key_env: Some("RCH_PAGERDUTY_ROUTING_KEY".to_string()),
4149 bearer_token_env: Some("RCH_WEBHOOK_BEARER".to_string()),
4150 signing_secret_env: Some("RCH_WEBHOOK_HMAC".to_string()),
4151 };
4152 let value = serde_json::to_value(&endpoint).unwrap();
4155 let obj = value.as_object().expect("endpoint serializes to an object");
4156 for forbidden in ["secret", "routing_key", "bearer_token", "password", "token"] {
4157 assert!(
4158 !obj.contains_key(forbidden),
4159 "endpoint config must not expose an inline `{forbidden}` field"
4160 );
4161 }
4162 assert!(obj.contains_key("routing_key_env"));
4163 assert!(obj.contains_key("bearer_token_env"));
4164 assert!(obj.contains_key("signing_secret_env"));
4165 }
4166
4167 #[test]
4168 fn test_webhook_format_toml_spelling() {
4169 #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)]
4174 struct FormatWrap {
4175 format: DoctorWebhookFormat,
4176 }
4177 let rendered = toml::to_string(&FormatWrap {
4178 format: DoctorWebhookFormat::GenericJson,
4179 })
4180 .unwrap();
4181 assert!(
4182 rendered.contains("format = \"generic_json\""),
4183 "unexpected TOML: {rendered}"
4184 );
4185 let slack: FormatWrap = toml::from_str("format = \"slack\"").unwrap();
4186 assert_eq!(slack.format, DoctorWebhookFormat::Slack);
4187 let pd: FormatWrap = toml::from_str("format = \"pagerduty\"").unwrap();
4188 assert_eq!(pd.format, DoctorWebhookFormat::Pagerduty);
4189 }
4190
4191 #[test]
4192 fn test_doctor_webhooks_defaults() {
4193 let cfg = DoctorWebhooksConfig::default();
4194 assert!(cfg.endpoints.is_empty());
4195 assert_eq!(cfg.max_inflight, 16);
4196 let rch: RchConfig = toml::from_str("").unwrap();
4199 assert!(rch.doctor.webhooks.endpoints.is_empty());
4200 }
4201
4202 #[test]
4203 fn test_doctor_webhook_endpoint_parses_from_toml() {
4204 let toml_src = r#"
4205[doctor.webhooks]
4206max_inflight = 8
4207
4208[[doctor.webhooks.endpoints]]
4209name = "ops-slack"
4210url = "https://hooks.slack.com/services/T0/B0/x"
4211format = "slack"
4212on_transitions = ["healthy_to_degraded", "any_to_failing"]
4213retry_max = 2
4214"#;
4215 let cfg: RchConfig = toml::from_str(toml_src).unwrap();
4216 assert_eq!(cfg.doctor.webhooks.max_inflight, 8);
4217 assert_eq!(cfg.doctor.webhooks.endpoints.len(), 1);
4218 let ep = &cfg.doctor.webhooks.endpoints[0];
4219 assert_eq!(ep.name, "ops-slack");
4220 assert_eq!(ep.format, DoctorWebhookFormat::Slack);
4221 assert_eq!(ep.on_transitions.len(), 2);
4222 assert_eq!(ep.retry_max, 2);
4223 assert_eq!(ep.retry_backoff_ms, 200);
4225 assert_eq!(ep.timeout_ms, 5000);
4226 assert!(ep.enabled);
4227 }
4228
4229 fn assert_golden_json(
4230 actual: serde_json::Value,
4231 expected: &serde_json::Value,
4232 fixture_path: &str,
4233 ) {
4234 if &actual == expected {
4235 return;
4236 }
4237
4238 let actual_json = serde_json::to_string_pretty(&actual).expect("actual JSON renders");
4239 let expected_json = serde_json::to_string_pretty(expected).expect("expected JSON renders");
4240 assert_eq!(
4241 &actual,
4242 expected,
4243 "golden mismatch in {fixture_path}\n\
4244 expected_blake3={}\n\
4245 actual_blake3={}\n\
4246 bless path: review the semantic diff, update the fixture expected_* field, \
4247 and record both hashes in the Beads closeout.\n\
4248 expected:\n{expected_json}\n\
4249 actual:\n{actual_json}",
4250 blake3::hash(expected_json.as_bytes()),
4251 blake3::hash(actual_json.as_bytes())
4252 );
4253 }
4254
4255 #[test]
4256 fn test_circuit_state_default() {
4257 let _guard = test_guard!();
4258 assert_eq!(CircuitState::default(), CircuitState::Closed);
4259 }
4260
4261 #[test]
4262 fn test_circuit_config_defaults() {
4263 let _guard = test_guard!();
4264 let config = CircuitBreakerConfig::default();
4265 assert_eq!(config.failure_threshold, 3);
4266 assert_eq!(config.success_threshold, 2);
4267 assert_eq!(config.error_rate_threshold, 0.5);
4268 assert_eq!(config.window_secs, 60);
4269 assert_eq!(config.open_cooldown_secs, 30);
4270 assert_eq!(config.half_open_max_probes, 1);
4271 }
4272
4273 #[test]
4274 fn test_rch_config_has_circuit_defaults() {
4275 let _guard = test_guard!();
4276 let config = RchConfig::default();
4277 assert_eq!(config.circuit.failure_threshold, 3);
4278 }
4279
4280 #[test]
4281 fn test_circuit_config_serde_roundtrip() {
4282 let _guard = test_guard!();
4283 let config = CircuitBreakerConfig {
4284 failure_threshold: 5,
4285 success_threshold: 3,
4286 error_rate_threshold: 0.75,
4287 window_secs: 120,
4288 open_cooldown_secs: 45,
4289 half_open_max_probes: 2,
4290 command_failure_threshold: 4,
4291 open_cooldown_max_secs: 900,
4292 };
4293 let json = serde_json::to_string(&config).unwrap();
4294 let parsed: CircuitBreakerConfig = serde_json::from_str(&json).unwrap();
4295 assert_eq!(parsed.command_failure_threshold, 4);
4296 assert_eq!(parsed.open_cooldown_max_secs, 900);
4297 assert_eq!(parsed.failure_threshold, 5);
4298 assert_eq!(parsed.success_threshold, 3);
4299 assert_eq!(parsed.error_rate_threshold, 0.75);
4300 assert_eq!(parsed.window_secs, 120);
4301 assert_eq!(parsed.open_cooldown_secs, 45);
4302 assert_eq!(parsed.half_open_max_probes, 2);
4303 }
4304
4305 #[test]
4310 fn test_worker_capabilities_default() {
4311 let _guard = test_guard!();
4312 let caps = WorkerCapabilities::default();
4313 assert!(caps.rustc_version.is_none());
4314 assert!(caps.rustup_toolchains.is_empty());
4315 assert!(caps.rustup_components.is_empty());
4316 assert!(caps.bun_version.is_none());
4317 assert!(caps.node_version.is_none());
4318 assert!(caps.npm_version.is_none());
4319 assert!(caps.projects_root_ok.is_none());
4320 assert!(caps.projects_root_issue.is_none());
4321 assert!(caps.projects_root_checked_at_unix_ms.is_none());
4322 }
4323
4324 #[test]
4325 fn test_worker_capabilities_new() {
4326 let _guard = test_guard!();
4327 let caps = WorkerCapabilities::new();
4328 assert!(caps.rustc_version.is_none());
4329 assert!(!caps.has_rust());
4330 assert!(!caps.has_bun());
4331 assert!(!caps.has_node());
4332 assert!(caps.is_topology_healthy().is_none());
4333 }
4334
4335 #[test]
4336 fn test_worker_capabilities_has_rust() {
4337 let _guard = test_guard!();
4338 let mut caps = WorkerCapabilities::new();
4339 assert!(!caps.has_rust());
4340
4341 caps.rustc_version = Some("rustc 1.76.0 (07dca489a 2024-02-04)".to_string());
4342 assert!(caps.has_rust());
4343 }
4344
4345 #[test]
4346 fn test_worker_capabilities_components_are_toolchain_scoped_and_exact() {
4347 let _guard = test_guard!();
4348 let caps = WorkerCapabilities {
4349 rustup_components: vec![
4350 "stable-x86_64-unknown-linux-gnu:clippy".to_string(),
4351 "nightly-2026-07-05-x86_64-unknown-linux-gnu:rustfmt".to_string(),
4352 "beta-x86_64-unknown-linux-gnu:clippy-preview".to_string(),
4353 ],
4354 ..Default::default()
4355 };
4356
4357 assert!(caps.has_rustup_component("stable", "clippy"));
4358 assert!(caps.has_rustup_component("nightly-2026-07-05", "rustfmt"));
4359 assert!(!caps.has_rustup_component("nightly-2026-07-05", "clippy"));
4360 assert!(!caps.has_rustup_component("beta", "clippy"));
4361 assert!(!caps.has_rustup_component("nightly-2026-07-0", "rustfmt"));
4362 }
4363
4364 #[test]
4365 fn test_worker_capabilities_has_bun() {
4366 let _guard = test_guard!();
4367 let mut caps = WorkerCapabilities::new();
4368 assert!(!caps.has_bun());
4369
4370 caps.bun_version = Some("1.0.25".to_string());
4371 assert!(caps.has_bun());
4372 }
4373
4374 #[test]
4375 fn test_worker_capabilities_has_nix() {
4376 let _guard = test_guard!();
4377 let mut caps = WorkerCapabilities::new();
4378 assert!(!caps.has_nix());
4379
4380 caps.nix_version = Some("nix (Nix) 2.24.9".to_string());
4381 assert!(caps.has_nix());
4382 }
4383
4384 #[test]
4385 fn test_worker_capabilities_has_zig_requires_both_halves() {
4386 let _guard = test_guard!();
4387 let mut caps = WorkerCapabilities::new();
4388 assert!(!caps.has_zig());
4389
4390 caps.zig_version = Some("0.14.1".to_string());
4393 assert!(!caps.has_zig());
4394
4395 caps.zig_version = None;
4397 caps.cargo_zigbuild_version = Some("cargo-zigbuild 0.23.0".to_string());
4398 assert!(!caps.has_zig());
4399
4400 caps.zig_version = Some("0.14.1".to_string());
4401 assert!(caps.has_zig());
4402 }
4403
4404 #[test]
4405 fn test_nix_build_uses_generous_timeout() {
4406 let _guard = test_guard!();
4407 let cfg = CompilationConfig::default();
4408 assert_eq!(
4410 cfg.timeout_for_kind(Some(CompilationKind::NixBuild)),
4411 std::time::Duration::from_secs(cfg.test_timeout_sec)
4412 );
4413 }
4414
4415 #[test]
4416 fn test_job_uses_build_timeout_bucket() {
4417 let _guard = test_guard!();
4418 let cfg = CompilationConfig::default();
4421 assert_eq!(
4422 cfg.timeout_for_kind(Some(CompilationKind::Job)),
4423 std::time::Duration::from_secs(cfg.build_timeout_sec)
4424 );
4425 }
4426
4427 #[test]
4428 fn test_layer0_disabled_by_default_and_env_gated() {
4429 let _guard = test_guard!();
4430 let cfg = Layer0Config::default();
4431 assert!(!cfg.enabled);
4432 let knobs_on = Layer0Config {
4434 enabled: false,
4435 release_lto_thin: true,
4436 release_codegen_units_1: true,
4437 };
4438 assert!(cfg.active_env().is_empty());
4439 assert!(knobs_on.active_env().is_empty());
4440 }
4441
4442 #[test]
4443 fn test_layer0_knobs_emit_stable_env_pairs() {
4444 let _guard = test_guard!();
4445 let lto_only = Layer0Config {
4446 enabled: true,
4447 release_lto_thin: true,
4448 release_codegen_units_1: false,
4449 };
4450 assert_eq!(
4451 lto_only.active_env(),
4452 vec![("CARGO_PROFILE_RELEASE_LTO", "thin")]
4453 );
4454 let both = Layer0Config {
4455 enabled: true,
4456 release_lto_thin: true,
4457 release_codegen_units_1: true,
4458 };
4459 assert_eq!(
4460 both.active_env(),
4461 vec![
4462 ("CARGO_PROFILE_RELEASE_LTO", "thin"),
4463 ("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"),
4464 ],
4465 "emission order must stay stable for deterministic baselines"
4466 );
4467 }
4468
4469 #[test]
4470 fn test_layer0_pack_version_is_recorded() {
4471 let _guard = test_guard!();
4472 assert!(!LAYER0_PACK_VERSION.is_empty());
4474 }
4475 #[test]
4476 fn test_default_allowlist_includes_nix() {
4477 let _guard = test_guard!();
4478 let cfg = ExecutionConfig::default();
4479 assert!(cfg.is_allowed("nix"));
4480 }
4481
4482 #[test]
4483 fn test_worker_capabilities_has_node() {
4484 let _guard = test_guard!();
4485 let mut caps = WorkerCapabilities::new();
4486 assert!(!caps.has_node());
4487
4488 caps.node_version = Some("v20.11.0".to_string());
4489 assert!(caps.has_node());
4490 }
4491
4492 #[test]
4493 fn test_worker_capabilities_multiple_runtimes() {
4494 let _guard = test_guard!();
4495 let caps = WorkerCapabilities {
4496 rustc_version: Some("rustc 1.76.0".to_string()),
4497 bun_version: Some("1.0.25".to_string()),
4498 node_version: Some("v20.11.0".to_string()),
4499 npm_version: Some("10.2.4".to_string()),
4500 ..Default::default()
4501 };
4502
4503 assert!(caps.has_rust());
4504 assert!(caps.has_bun());
4505 assert!(caps.has_node());
4506 }
4507
4508 #[test]
4509 fn test_worker_capabilities_serialization_empty() {
4510 let _guard = test_guard!();
4511 let caps = WorkerCapabilities::new();
4512 let json = serde_json::to_string(&caps).unwrap();
4513 assert_eq!(json, "{}");
4515
4516 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4518 assert!(!parsed.has_rust());
4519 assert!(!parsed.has_bun());
4520 assert!(!parsed.has_node());
4521 }
4522
4523 #[test]
4524 fn test_worker_capabilities_serialization_with_versions() {
4525 let _guard = test_guard!();
4526 let caps = WorkerCapabilities {
4527 rustc_version: Some("rustc 1.76.0-nightly".to_string()),
4528 bun_version: Some("1.0.25".to_string()),
4529 node_version: None,
4530 npm_version: None,
4531 ..Default::default()
4532 };
4533
4534 let json = serde_json::to_string(&caps).unwrap();
4535 assert!(json.contains("rustc_version"));
4536 assert!(json.contains("rustc 1.76.0-nightly"));
4537 assert!(json.contains("bun_version"));
4538 assert!(json.contains("1.0.25"));
4539 assert!(!json.contains("node_version"));
4541 assert!(!json.contains("npm_version"));
4542
4543 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4544 assert!(parsed.has_rust());
4545 assert!(parsed.has_bun());
4546 assert!(!parsed.has_node());
4547 }
4548
4549 #[test]
4550 fn test_worker_capabilities_deserialization_partial() {
4551 let _guard = test_guard!();
4552 let json = r#"{"bun_version": "1.0.0"}"#;
4554 let caps: WorkerCapabilities = serde_json::from_str(json).unwrap();
4555
4556 assert!(!caps.has_rust());
4557 assert!(caps.has_bun());
4558 assert!(!caps.has_node());
4559 assert_eq!(caps.bun_version, Some("1.0.0".to_string()));
4560 }
4561
4562 #[test]
4563 fn test_worker_capabilities_clone() {
4564 let _guard = test_guard!();
4565 let caps = WorkerCapabilities {
4566 rustc_version: Some("1.76.0".to_string()),
4567 bun_version: None,
4568 node_version: Some("v20".to_string()),
4569 npm_version: None,
4570 projects_root_ok: Some(false),
4571 projects_root_issue: Some("alias_missing".to_string()),
4572 projects_root_checked_at_unix_ms: Some(123),
4573 ..Default::default()
4574 };
4575
4576 let cloned = caps.clone();
4577 assert_eq!(cloned.rustc_version, caps.rustc_version);
4578 assert_eq!(cloned.bun_version, caps.bun_version);
4579 assert_eq!(cloned.node_version, caps.node_version);
4580 assert_eq!(cloned.npm_version, caps.npm_version);
4581 assert_eq!(cloned.projects_root_ok, caps.projects_root_ok);
4582 assert_eq!(cloned.projects_root_issue, caps.projects_root_issue);
4583 assert_eq!(
4584 cloned.projects_root_checked_at_unix_ms,
4585 caps.projects_root_checked_at_unix_ms
4586 );
4587 }
4588
4589 #[test]
4590 fn test_worker_capabilities_topology_health_helper() {
4591 let _guard = test_guard!();
4592 let mut caps = WorkerCapabilities::new();
4593 assert_eq!(caps.is_topology_healthy(), None);
4594
4595 caps.projects_root_ok = Some(true);
4596 assert_eq!(caps.is_topology_healthy(), Some(true));
4597
4598 caps.projects_root_ok = Some(false);
4599 assert_eq!(caps.is_topology_healthy(), Some(false));
4600 }
4601
4602 #[test]
4603 fn test_selection_reason_serialization() {
4604 let _guard = test_guard!();
4605 assert_eq!(
4607 serde_json::to_string(&SelectionReason::Success).unwrap(),
4608 "\"success\""
4609 );
4610 assert_eq!(
4611 serde_json::to_string(&SelectionReason::NoWorkersConfigured).unwrap(),
4612 "\"no_workers_configured\""
4613 );
4614 assert_eq!(
4615 serde_json::to_string(&SelectionReason::AllWorkersUnreachable).unwrap(),
4616 "\"all_workers_unreachable\""
4617 );
4618 assert_eq!(
4619 serde_json::to_string(&SelectionReason::AllCircuitsOpen).unwrap(),
4620 "\"all_circuits_open\""
4621 );
4622 assert_eq!(
4623 serde_json::to_string(&SelectionReason::AllWorkersBusy).unwrap(),
4624 "\"all_workers_busy\""
4625 );
4626 assert_eq!(
4627 serde_json::to_string(&SelectionReason::NoWorkersPassedHealth).unwrap(),
4628 "\"no_workers_passed_health\""
4629 );
4630 assert_eq!(
4631 serde_json::to_string(&SelectionReason::AllWorkersFailedPreflight).unwrap(),
4632 "\"all_workers_failed_preflight\""
4633 );
4634 assert_eq!(
4635 serde_json::to_string(&SelectionReason::AllWorkersFailedConvergence).unwrap(),
4636 "\"all_workers_failed_convergence\""
4637 );
4638 assert_eq!(
4639 serde_json::to_string(&SelectionReason::NoAdmissibleWorkers(
4640 "critical_pressure=1,insufficient_slots=1".to_string()
4641 ))
4642 .unwrap(),
4643 "{\"no_admissible_workers\":\"critical_pressure=1,insufficient_slots=1\"}"
4644 );
4645 assert_eq!(
4646 serde_json::to_string(&SelectionReason::NoMatchingWorkers).unwrap(),
4647 "\"no_matching_workers\""
4648 );
4649 }
4650
4651 #[test]
4652 fn test_selection_reason_with_error() {
4653 let _guard = test_guard!();
4654 let reason = SelectionReason::SelectionError("test error".to_string());
4655 let json = serde_json::to_string(&reason).unwrap();
4656 assert!(json.contains("selection_error"));
4657 assert!(json.contains("test error"));
4658 }
4659
4660 #[test]
4661 fn test_selection_reason_deserialization() {
4662 let _guard = test_guard!();
4663 assert_eq!(
4664 serde_json::from_str::<SelectionReason>("\"success\"").unwrap(),
4665 SelectionReason::Success
4666 );
4667 assert_eq!(
4668 serde_json::from_str::<SelectionReason>("\"all_workers_busy\"").unwrap(),
4669 SelectionReason::AllWorkersBusy
4670 );
4671 assert_eq!(
4672 serde_json::from_str::<SelectionReason>("\"no_workers_passed_health\"").unwrap(),
4673 SelectionReason::NoWorkersPassedHealth
4674 );
4675 assert_eq!(
4676 serde_json::from_str::<SelectionReason>("\"all_workers_failed_preflight\"").unwrap(),
4677 SelectionReason::AllWorkersFailedPreflight
4678 );
4679 assert_eq!(
4680 serde_json::from_str::<SelectionReason>("\"all_workers_failed_convergence\"").unwrap(),
4681 SelectionReason::AllWorkersFailedConvergence
4682 );
4683 assert_eq!(
4684 serde_json::from_str::<SelectionReason>(
4685 "{\"no_admissible_workers\":\"critical_pressure=1\"}"
4686 )
4687 .unwrap(),
4688 SelectionReason::NoAdmissibleWorkers("critical_pressure=1".to_string())
4689 );
4690 }
4691
4692 #[test]
4693 fn test_selection_reason_display() {
4694 let _guard = test_guard!();
4695 assert_eq!(
4696 SelectionReason::Success.to_string(),
4697 "worker assigned successfully"
4698 );
4699 assert_eq!(
4700 SelectionReason::NoWorkersConfigured.to_string(),
4701 "no workers configured"
4702 );
4703 assert_eq!(
4704 SelectionReason::AllWorkersUnreachable.to_string(),
4705 "all workers unreachable"
4706 );
4707 assert_eq!(
4708 SelectionReason::AllWorkersBusy.to_string(),
4709 "all workers at capacity"
4710 );
4711 assert_eq!(
4712 SelectionReason::NoWorkersPassedHealth.to_string(),
4713 "no workers passed health thresholds"
4714 );
4715 assert_eq!(
4716 SelectionReason::AllWorkersFailedPreflight.to_string(),
4717 "all workers failed preflight checks"
4718 );
4719 assert_eq!(
4720 SelectionReason::AllWorkersFailedConvergence.to_string(),
4721 "all workers failed repo convergence checks"
4722 );
4723 assert_eq!(
4724 SelectionReason::NoAdmissibleWorkers(
4725 "critical_pressure=1,insufficient_slots=1".to_string()
4726 )
4727 .to_string(),
4728 "no admissible workers: critical_pressure=1,insufficient_slots=1"
4729 );
4730 assert_eq!(
4731 SelectionReason::SelectionError("oops".to_string()).to_string(),
4732 "selection error: oops"
4733 );
4734 }
4735
4736 #[test]
4737 fn test_selection_response_with_worker() {
4738 let _guard = test_guard!();
4739 let response = SelectionResponse {
4740 worker: Some(SelectedWorker {
4741 id: WorkerId::new("test"),
4742 host: "localhost".to_string(),
4743 user: "user".to_string(),
4744 identity_file: "~/.ssh/id_rsa".to_string(),
4745 slots_available: 8,
4746 speed_score: 75.0,
4747 declared_os: None,
4748 }),
4749 reason: SelectionReason::Success,
4750 build_id: None,
4751 diagnostics: None,
4752 };
4753
4754 let json = serde_json::to_string(&response).unwrap();
4755 assert!(json.contains("\"reason\":\"success\""));
4756 assert!(json.contains("\"id\":\"test\""));
4757 }
4758
4759 #[test]
4760 fn test_selection_response_without_worker() {
4761 let _guard = test_guard!();
4762 let response = SelectionResponse {
4763 worker: None,
4764 reason: SelectionReason::AllWorkersBusy,
4765 build_id: None,
4766 diagnostics: None,
4767 };
4768
4769 let json = serde_json::to_string(&response).unwrap();
4770 assert!(json.contains("\"worker\":null"));
4771 assert!(json.contains("\"reason\":\"all_workers_busy\""));
4772 }
4773
4774 #[test]
4775 fn golden_selection_response_with_no_workers_passed_health_diagnostics() {
4776 let _guard = test_guard!();
4777 const FIXTURE_PATH: &str =
4778 "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json";
4779 let fixture: serde_json::Value = serde_json::from_str(include_str!(
4780 "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json"
4781 ))
4782 .expect("selection diagnostics golden fixture parses");
4783 assert_eq!(
4784 fixture["schema_version"],
4785 "rch.golden.selection_diagnostics.v1"
4786 );
4787
4788 let response = SelectionResponse {
4789 worker: None,
4790 reason: SelectionReason::NoWorkersPassedHealth,
4791 build_id: None,
4792 diagnostics: Some(SelectionDiagnostics {
4793 required_runtime: RequiredRuntime::Rust,
4794 estimated_cores: 2,
4795 min_success_rate: 0.8,
4796 fallback_min_success_rate: 0.5,
4797 active_project_exclusion_count: 0,
4798 workers: vec![WorkerSelectionDiagnostic {
4799 worker_id: WorkerId::new("fixture-worker"),
4800 status: "healthy".to_string(),
4801 circuit_state: "closed".to_string(),
4802 pressure_state: "healthy".to_string(),
4803 pressure_reason_code: "pressure.ok".to_string(),
4804 success_rate: Some(0.2),
4805 min_success_rate: 0.8,
4806 fallback_min_success_rate: 0.5,
4807 required_runtime: RequiredRuntime::Rust,
4808 runtime_available: true,
4809 available_slots: 8,
4810 total_slots: 8,
4811 estimated_cores: 2,
4812 active_project_excluded: false,
4813 final_decision: WorkerSelectionDiagnosticDecision::Deny,
4814 final_reason: "success_rate 0.20 < fallback 0.50".to_string(),
4815 reason_codes: vec![
4816 "health.below_min_success_rate".to_string(),
4817 "health.below_fallback_min_success_rate".to_string(),
4818 ],
4819 }],
4820 }),
4821 };
4822
4823 assert_golden_json(
4824 serde_json::to_value(&response).expect("response serializes"),
4825 &fixture["expected_response"],
4826 FIXTURE_PATH,
4827 );
4828 }
4829
4830 #[test]
4831 fn test_selection_response_roundtrip() {
4832 let _guard = test_guard!();
4833 let original = SelectionResponse {
4834 worker: Some(SelectedWorker {
4835 id: WorkerId::new("worker-1"),
4836 host: "192.168.1.100".to_string(),
4837 user: "ubuntu".to_string(),
4838 identity_file: "/path/to/key".to_string(),
4839 slots_available: 16,
4840 speed_score: 90.5,
4841 declared_os: None,
4842 }),
4843 reason: SelectionReason::Success,
4844 build_id: None,
4845 diagnostics: None,
4846 };
4847
4848 let json = serde_json::to_string(&original).unwrap();
4849 let parsed: SelectionResponse = serde_json::from_str(&json).unwrap();
4850
4851 assert!(parsed.worker.is_some());
4852 let worker = parsed.worker.unwrap();
4853 assert_eq!(worker.id.as_str(), "worker-1");
4854 assert_eq!(worker.host, "192.168.1.100");
4855 assert_eq!(worker.slots_available, 16);
4856 assert_eq!(parsed.reason, SelectionReason::Success);
4857 }
4858
4859 #[test]
4860 fn test_build_heartbeat_phase_roundtrip() {
4861 let _guard = test_guard!();
4862 let json = serde_json::to_string(&BuildHeartbeatPhase::SyncUp).unwrap();
4863 assert_eq!(json, "\"sync_up\"");
4864 let parsed: BuildHeartbeatPhase = serde_json::from_str(&json).unwrap();
4865 assert_eq!(parsed, BuildHeartbeatPhase::SyncUp);
4866 }
4867
4868 #[test]
4869 fn test_build_heartbeat_request_roundtrip() {
4870 let _guard = test_guard!();
4871 let request = BuildHeartbeatRequest {
4872 build_id: 42,
4873 worker_id: WorkerId::new("worker-a"),
4874 hook_pid: Some(12345),
4875 local_wrapper_id: Some("wrapper-abc123".to_string()),
4876 remote_pgid_file: Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string()),
4877 phase: BuildHeartbeatPhase::Execute,
4878 detail: Some("Compiling crates".to_string()),
4879 progress_counter: Some(7),
4880 progress_percent: Some(42.5),
4881 };
4882
4883 let json = serde_json::to_string(&request).unwrap();
4884 let parsed: BuildHeartbeatRequest = serde_json::from_str(&json).unwrap();
4885 assert_eq!(parsed.build_id, 42);
4886 assert_eq!(parsed.worker_id.as_str(), "worker-a");
4887 assert_eq!(
4888 parsed.remote_pgid_file,
4889 Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string())
4890 );
4891 assert_eq!(parsed.phase, BuildHeartbeatPhase::Execute);
4892 assert_eq!(parsed.progress_counter, Some(7));
4893 assert_eq!(parsed.progress_percent, Some(42.5));
4894 }
4895
4896 #[test]
4899 fn test_circuit_stats_new() {
4900 let _guard = test_guard!();
4901 let stats = CircuitStats::new();
4902 assert_eq!(stats.state(), CircuitState::Closed);
4903 assert_eq!(stats.consecutive_failures(), 0);
4904 assert_eq!(stats.error_rate(), 0.0);
4905 assert!(stats.opened_at().is_none());
4906 }
4907
4908 #[test]
4909 fn test_circuit_stats_record_success() {
4910 let _guard = test_guard!();
4911 let mut stats = CircuitStats::new();
4912 stats.record_success();
4913 stats.record_success();
4914 assert_eq!(stats.error_rate(), 0.0);
4915 assert_eq!(stats.consecutive_failures(), 0);
4916 }
4917
4918 #[test]
4919 fn test_circuit_stats_record_failure() {
4920 let _guard = test_guard!();
4921 let mut stats = CircuitStats::new();
4922 stats.record_failure();
4923 stats.record_failure();
4924 assert_eq!(stats.consecutive_failures(), 2);
4925 assert_eq!(stats.error_rate(), 1.0); }
4927
4928 #[test]
4929 fn test_circuit_stats_error_rate() {
4930 let _guard = test_guard!();
4931 let mut stats = CircuitStats::new();
4932 stats.record_success();
4933 stats.record_success();
4934 stats.record_failure();
4935 assert!((stats.error_rate() - 0.333).abs() < 0.01);
4937 }
4938
4939 #[test]
4940 fn test_circuit_stats_should_open_consecutive_failures() {
4941 let _guard = test_guard!();
4942 let mut stats = CircuitStats::new();
4943 let config = CircuitBreakerConfig::default(); stats.record_failure();
4946 assert!(!stats.should_open(&config));
4947
4948 stats.record_failure();
4949 assert!(!stats.should_open(&config));
4950
4951 stats.record_failure();
4952 assert!(stats.should_open(&config)); }
4954
4955 #[test]
4956 fn test_circuit_stats_should_open_error_rate() {
4957 let _guard = test_guard!();
4958 let mut stats = CircuitStats::new();
4959 let config = CircuitBreakerConfig {
4960 error_rate_threshold: 0.5,
4961 ..Default::default()
4962 };
4963
4964 stats.record_success();
4966 stats.record_success();
4967 stats.record_failure();
4968 stats.record_failure();
4969 assert!(!stats.should_open(&config)); stats.record_failure(); assert!(stats.should_open(&config));
4973 }
4974
4975 #[test]
4976 fn test_circuit_close_clears_recent_results_no_flap() {
4977 let _guard = test_guard!();
4978 let mut stats = CircuitStats::new();
4982 let config = CircuitBreakerConfig {
4983 error_rate_threshold: 0.5,
4984 ..Default::default()
4985 };
4986 for _ in 0..6 {
4987 stats.record_failure();
4988 }
4989 assert!(stats.should_open(&config));
4990 stats.open();
4991 stats.half_open();
4993 stats.record_success();
4994 stats.record_success();
4995 stats.close();
4996 assert_eq!(stats.error_rate(), 0.0);
4998 assert!(!stats.should_open(&config));
4999 }
5000
5001 #[test]
5002 fn test_apply_health_outcome_opens_on_failure_threshold() {
5003 let _guard = test_guard!();
5004 let config = CircuitBreakerConfig {
5006 failure_threshold: 3,
5007 open_cooldown_secs: 300,
5008 ..Default::default()
5009 };
5010 let mut stats = CircuitStats::new();
5011 assert_eq!(
5012 stats.apply_health_outcome(false, false, &config),
5013 CircuitState::Closed
5014 );
5015 assert_eq!(
5016 stats.apply_health_outcome(false, false, &config),
5017 CircuitState::Closed
5018 );
5019 assert_eq!(
5020 stats.apply_health_outcome(false, false, &config),
5021 CircuitState::Open
5022 );
5023 }
5024
5025 #[test]
5026 fn test_apply_health_outcome_half_open_two_probe_recovery() {
5027 let _guard = test_guard!();
5028 let config = CircuitBreakerConfig {
5034 failure_threshold: 3,
5035 success_threshold: 2,
5036 open_cooldown_secs: 0,
5037 ..Default::default()
5038 };
5039 let mut stats = CircuitStats::new();
5040 for _ in 0..3 {
5041 stats.apply_health_outcome(false, false, &config);
5042 }
5043 assert_eq!(stats.state(), CircuitState::HalfOpen);
5045
5046 let after_first = stats.apply_health_outcome(true, false, &config);
5048 assert_eq!(after_first, CircuitState::HalfOpen);
5049
5050 let after_second = stats.apply_health_outcome(true, false, &config);
5052 assert_eq!(after_second, CircuitState::Closed);
5053 }
5054
5055 #[test]
5056 fn test_apply_health_outcome_transient_half_open_does_not_reopen() {
5057 let _guard = test_guard!();
5058 let config = CircuitBreakerConfig {
5061 failure_threshold: 3,
5062 success_threshold: 2,
5063 open_cooldown_secs: 0,
5064 ..Default::default()
5065 };
5066 let mut stats = CircuitStats::new();
5067 for _ in 0..3 {
5068 stats.apply_health_outcome(false, false, &config);
5069 }
5070 assert_eq!(stats.state(), CircuitState::HalfOpen);
5072
5073 assert_eq!(
5075 stats.apply_health_outcome(true, false, &config),
5076 CircuitState::HalfOpen
5077 );
5078
5079 let after_transient = stats.apply_health_outcome(false, true, &config);
5085 assert_eq!(after_transient, CircuitState::HalfOpen);
5086
5087 assert_eq!(
5089 stats.apply_health_outcome(true, false, &config),
5090 CircuitState::HalfOpen
5091 );
5092 assert_eq!(
5093 stats.apply_health_outcome(true, false, &config),
5094 CircuitState::Closed
5095 );
5096 }
5097
5098 #[test]
5099 fn test_apply_health_outcome_real_vs_transient_half_open_failure() {
5100 let _guard = test_guard!();
5101 let config = CircuitBreakerConfig {
5109 failure_threshold: 3,
5110 success_threshold: 2,
5111 open_cooldown_secs: 300,
5112 ..Default::default()
5113 };
5114
5115 let mut real = CircuitStats::new();
5117 real.open();
5118 real.half_open();
5119 assert_eq!(real.state(), CircuitState::HalfOpen);
5120 let after_real = real.apply_health_outcome(false, false, &config);
5121 assert_eq!(
5122 after_real,
5123 CircuitState::Open,
5124 "a real half-open failure must reopen the circuit"
5125 );
5126
5127 let mut transient = CircuitStats::new();
5129 transient.open();
5130 transient.half_open();
5131 assert_eq!(transient.state(), CircuitState::HalfOpen);
5132 let after_transient = transient.apply_health_outcome(false, true, &config);
5133 assert_eq!(
5134 after_transient,
5135 CircuitState::HalfOpen,
5136 "a transient half-open failure must NOT reopen the circuit"
5137 );
5138 }
5139
5140 #[test]
5145 fn test_command_failure_blocks_half_open_close_on_liveness_successes() {
5146 let _guard = test_guard!();
5147 let config = CircuitBreakerConfig {
5148 failure_threshold: 3,
5149 success_threshold: 2,
5150 open_cooldown_secs: 0,
5151 command_failure_threshold: 2,
5152 ..Default::default()
5153 };
5154 let mut stats = CircuitStats::new();
5155 stats.open();
5156 stats.half_open();
5157
5158 assert_eq!(
5160 stats.apply_command_outcome(false, &config),
5161 CircuitState::HalfOpen,
5162 "cooldown 0: reopen then immediately half-open again"
5163 );
5164 assert_eq!(stats.consecutive_command_failures(), 1);
5165
5166 for _ in 0..10 {
5168 assert_eq!(
5169 stats.apply_health_outcome(true, false, &config),
5170 CircuitState::HalfOpen,
5171 "liveness successes must not close the circuit while a command failure is outstanding"
5172 );
5173 }
5174
5175 assert_eq!(
5177 stats.apply_command_outcome(true, &config),
5178 CircuitState::Closed
5179 );
5180 assert_eq!(stats.consecutive_command_failures(), 0);
5181 assert_eq!(stats.reopen_streak(), 0);
5182 }
5183
5184 #[test]
5187 fn test_command_failures_open_closed_circuit_despite_liveness_successes() {
5188 let _guard = test_guard!();
5189 let config = CircuitBreakerConfig {
5190 failure_threshold: 3,
5191 command_failure_threshold: 2,
5192 open_cooldown_secs: 300,
5193 ..Default::default()
5194 };
5195 let mut stats = CircuitStats::new();
5196
5197 assert_eq!(
5198 stats.apply_command_outcome(false, &config),
5199 CircuitState::Closed
5200 );
5201 stats.apply_health_outcome(true, false, &config);
5203 stats.apply_health_outcome(true, false, &config);
5204 assert_eq!(stats.consecutive_failures(), 0);
5205 assert_eq!(
5207 stats.apply_command_outcome(false, &config),
5208 CircuitState::Open
5209 );
5210 assert_eq!(stats.reopen_streak(), 1);
5211 }
5212
5213 #[test]
5217 fn test_command_failure_in_half_open_always_reopens() {
5218 let _guard = test_guard!();
5219 let config = CircuitBreakerConfig {
5220 open_cooldown_secs: 300,
5221 ..Default::default()
5222 };
5223 let mut stats = CircuitStats::new();
5224 stats.open();
5225 stats.half_open();
5226 assert_eq!(
5227 stats.apply_command_outcome(false, &config),
5228 CircuitState::Open
5229 );
5230 assert_eq!(stats.reopen_streak(), 2);
5231 }
5232
5233 #[test]
5237 fn test_reopen_streak_backs_off_cooldown_until_cap() {
5238 let _guard = test_guard!();
5239 let config = CircuitBreakerConfig {
5240 open_cooldown_secs: 30,
5241 open_cooldown_max_secs: 100,
5242 ..Default::default()
5243 };
5244 assert_eq!(config.cooldown_ms_for_streak(0), 30_000);
5245 assert_eq!(config.cooldown_ms_for_streak(1), 30_000);
5246 assert_eq!(config.cooldown_ms_for_streak(2), 60_000);
5247 assert_eq!(config.cooldown_ms_for_streak(3), 100_000, "capped");
5248 assert_eq!(config.cooldown_ms_for_streak(40), 100_000, "no overflow");
5249
5250 let inverted = CircuitBreakerConfig {
5252 open_cooldown_secs: 30,
5253 open_cooldown_max_secs: 5,
5254 ..Default::default()
5255 };
5256 assert_eq!(inverted.cooldown_ms_for_streak(3), 30_000);
5257
5258 let mut stats = CircuitStats::new();
5261 stats.open();
5262 stats.half_open();
5263 stats.open();
5264 assert_eq!(stats.reopen_streak(), 2);
5265 let remaining = stats.recovery_remaining_secs(&config).unwrap();
5266 assert!(
5267 remaining > 30 && remaining <= 60,
5268 "second open should wait ~60s, got {remaining}"
5269 );
5270
5271 stats.close();
5273 assert_eq!(stats.reopen_streak(), 0);
5274 stats.open();
5275 assert!(stats.recovery_remaining_secs(&config).unwrap() <= 30);
5276 }
5277
5278 #[test]
5279 fn test_circuit_stats_success_resets_consecutive_failures() {
5280 let _guard = test_guard!();
5281 let mut stats = CircuitStats::new();
5282 stats.record_failure();
5283 stats.record_failure();
5284 assert_eq!(stats.consecutive_failures(), 2);
5285
5286 stats.record_success();
5287 assert_eq!(stats.consecutive_failures(), 0);
5288 }
5289
5290 #[test]
5291 fn test_circuit_stats_open_transition() {
5292 let _guard = test_guard!();
5293 let mut stats = CircuitStats::new();
5294 stats.open();
5295
5296 assert_eq!(stats.state(), CircuitState::Open);
5297 assert!(stats.opened_at().is_some());
5298 }
5299
5300 #[test]
5301 fn test_circuit_stats_half_open_transition() {
5302 let _guard = test_guard!();
5303 let mut stats = CircuitStats::new();
5304 stats.open();
5305 stats.half_open();
5306
5307 assert_eq!(stats.state(), CircuitState::HalfOpen);
5308 }
5309
5310 #[test]
5311 fn test_circuit_stats_close_transition() {
5312 let _guard = test_guard!();
5313 let mut stats = CircuitStats::new();
5314 stats.open();
5315 stats.half_open();
5316 stats.close();
5317
5318 assert_eq!(stats.state(), CircuitState::Closed);
5319 assert!(stats.opened_at().is_none());
5320 assert_eq!(stats.consecutive_failures(), 0);
5321 }
5322
5323 #[test]
5324 fn test_circuit_stats_should_close() {
5325 let _guard = test_guard!();
5326 let mut stats = CircuitStats::new();
5327 let config = CircuitBreakerConfig {
5328 success_threshold: 2,
5329 ..Default::default()
5330 };
5331
5332 stats.open();
5333 stats.half_open();
5334
5335 assert!(!stats.should_close(&config)); stats.record_success();
5338 assert!(!stats.should_close(&config)); stats.record_success();
5341 assert!(stats.should_close(&config)); }
5343
5344 #[test]
5345 fn test_circuit_stats_can_probe() {
5346 let _guard = test_guard!();
5347 let mut stats = CircuitStats::new();
5348 let config = CircuitBreakerConfig {
5349 half_open_max_probes: 1,
5350 ..Default::default()
5351 };
5352
5353 assert!(!stats.can_probe(&config));
5355
5356 stats.open();
5357 assert!(!stats.can_probe(&config));
5359
5360 stats.half_open();
5361 assert!(stats.can_probe(&config));
5363
5364 assert!(stats.start_probe(&config));
5366 assert!(!stats.can_probe(&config));
5368 assert!(!stats.start_probe(&config));
5369 }
5370
5371 #[test]
5372 fn test_circuit_stats_probe_completion() {
5373 let _guard = test_guard!();
5374 let mut stats = CircuitStats::new();
5375 let config = CircuitBreakerConfig {
5376 half_open_max_probes: 1,
5377 ..Default::default()
5378 };
5379
5380 stats.open();
5381 stats.half_open();
5382 stats.start_probe(&config);
5383
5384 stats.record_success();
5386
5387 assert!(stats.can_probe(&config));
5389 }
5390
5391 #[test]
5392 fn test_circuit_stats_failure_in_half_open() {
5393 let _guard = test_guard!();
5394 let mut stats = CircuitStats::new();
5395 let config = CircuitBreakerConfig {
5396 success_threshold: 2,
5397 ..Default::default()
5398 };
5399
5400 stats.open();
5401 stats.half_open();
5402
5403 stats.record_success();
5404 assert_eq!(stats.consecutive_failures(), 0);
5405
5406 stats.record_failure();
5408 assert!(!stats.should_close(&config));
5409
5410 stats.record_success();
5412 stats.record_success();
5413 assert!(stats.should_close(&config));
5414 }
5415
5416 #[test]
5417 fn test_circuit_stats_reset_window() {
5418 let _guard = test_guard!();
5419 let mut stats = CircuitStats::new();
5420 stats.record_success();
5421 stats.record_failure();
5422
5423 assert!(stats.error_rate() > 0.0);
5424
5425 stats.reset_window();
5426 assert_eq!(stats.error_rate(), 0.0);
5427 }
5428
5429 #[test]
5430 fn test_circuit_state_transitions_are_deterministic() {
5431 let _guard = test_guard!();
5432 let config = CircuitBreakerConfig::default();
5433 let mut stats = CircuitStats::new();
5434
5435 assert_eq!(stats.state(), CircuitState::Closed);
5437
5438 for _ in 0..3 {
5440 stats.record_failure();
5441 }
5442 assert!(stats.should_open(&config));
5443 stats.open();
5444 assert_eq!(stats.state(), CircuitState::Open);
5445
5446 stats.half_open();
5448 assert_eq!(stats.state(), CircuitState::HalfOpen);
5449
5450 for _ in 0..2 {
5452 stats.record_success();
5453 }
5454 assert!(stats.should_close(&config));
5455 stats.close();
5456 assert_eq!(stats.state(), CircuitState::Closed);
5457 }
5458
5459 fn make_test_metrics(
5464 speedup: Option<f64>,
5465 total_secs: u64,
5466 success: bool,
5467 ) -> CompilationMetrics {
5468 CompilationMetrics {
5469 project_id: "test-project".to_string(),
5470 worker_id: "worker-1".to_string(),
5471 timestamp: chrono::Utc::now(),
5472 timing: CompilationTimingBreakdown {
5473 rsync_up: Duration::from_millis(100),
5474 remote_build: Duration::from_millis(800),
5475 rsync_down: Duration::from_millis(100),
5476 total: Duration::from_secs(total_secs),
5477 },
5478 local_build_time: speedup.map(|s| Duration::from_secs_f64(total_secs as f64 * s)),
5479 speedup,
5480 files_synced: 100,
5481 bytes_transferred: 1_000_000,
5482 exit_code: if success { 0 } else { 1 },
5483 success,
5484 }
5485 }
5486
5487 #[test]
5488 fn test_compilation_timing_breakdown_default() {
5489 let _guard = test_guard!();
5490 let timing = CompilationTimingBreakdown::default();
5491 assert_eq!(timing.rsync_up, Duration::ZERO);
5492 assert_eq!(timing.remote_build, Duration::ZERO);
5493 assert_eq!(timing.rsync_down, Duration::ZERO);
5494 assert_eq!(timing.total, Duration::ZERO);
5495 }
5496
5497 #[test]
5498 fn test_compilation_timing_breakdown_serialization() {
5499 let _guard = test_guard!();
5500 let timing = CompilationTimingBreakdown {
5501 rsync_up: Duration::from_millis(100),
5502 remote_build: Duration::from_millis(2000),
5503 rsync_down: Duration::from_millis(50),
5504 total: Duration::from_millis(2150),
5505 };
5506
5507 let json = serde_json::to_string(&timing).unwrap();
5508 assert!(json.contains("100")); assert!(json.contains("2000")); assert!(json.contains("50")); assert!(json.contains("2150")); let parsed: CompilationTimingBreakdown = serde_json::from_str(&json).unwrap();
5514 assert_eq!(parsed.rsync_up, timing.rsync_up);
5515 assert_eq!(parsed.remote_build, timing.remote_build);
5516 assert_eq!(parsed.rsync_down, timing.rsync_down);
5517 assert_eq!(parsed.total, timing.total);
5518 }
5519
5520 #[test]
5521 fn test_command_timing_breakdown_serialization() {
5522 let _guard = test_guard!();
5523 let timing = CommandTimingBreakdown {
5524 classify: Some(Duration::from_millis(2)),
5525 select: None,
5526 sync_up: Some(Duration::from_millis(120)),
5527 exec: Some(Duration::from_millis(900)),
5528 sync_down: None,
5529 cleanup: Some(Duration::from_millis(5)),
5530 total: Some(Duration::from_millis(1027)),
5531 };
5532
5533 let json = serde_json::to_string(&timing).unwrap();
5534 assert!(json.contains("\"classify\":2"));
5535 assert!(json.contains("\"select\":null"));
5536 assert!(json.contains("\"sync_up\":120"));
5537 assert!(json.contains("\"exec\":900"));
5538 assert!(json.contains("\"sync_down\":null"));
5539 assert!(json.contains("\"cleanup\":5"));
5540 assert!(json.contains("\"total\":1027"));
5541
5542 let parsed: CommandTimingBreakdown = serde_json::from_str(&json).unwrap();
5543 assert_eq!(parsed.classify, timing.classify);
5544 assert_eq!(parsed.select, timing.select);
5545 assert_eq!(parsed.sync_up, timing.sync_up);
5546 assert_eq!(parsed.exec, timing.exec);
5547 assert_eq!(parsed.sync_down, timing.sync_down);
5548 assert_eq!(parsed.cleanup, timing.cleanup);
5549 assert_eq!(parsed.total, timing.total);
5550 }
5551
5552 #[test]
5553 fn test_compilation_metrics_calculate_speedup() {
5554 let _guard = test_guard!();
5555 let mut metrics = CompilationMetrics {
5556 timing: CompilationTimingBreakdown {
5557 total: Duration::from_secs(10),
5558 ..Default::default()
5559 },
5560 local_build_time: Some(Duration::from_secs(30)),
5561 ..Default::default()
5562 };
5563
5564 metrics.calculate_speedup();
5565
5566 assert!(metrics.speedup.is_some());
5567 let speedup = metrics.speedup.unwrap();
5568 assert!(
5569 (speedup - 3.0).abs() < 0.01,
5570 "Expected 3.0x speedup, got {}",
5571 speedup
5572 );
5573 }
5574
5575 #[test]
5576 fn test_compilation_metrics_is_beneficial() {
5577 let _guard = test_guard!();
5578 let mut metrics = CompilationMetrics::default();
5579
5580 assert!(!metrics.is_beneficial());
5582
5583 metrics.speedup = Some(1.5);
5585 assert!(metrics.is_beneficial());
5586
5587 metrics.speedup = Some(0.8);
5589 assert!(!metrics.is_beneficial());
5590
5591 metrics.speedup = Some(1.0);
5593 assert!(!metrics.is_beneficial());
5594 }
5595
5596 #[test]
5597 fn test_compilation_metrics_serialization() {
5598 let _guard = test_guard!();
5599 let metrics = make_test_metrics(Some(2.5), 10, true);
5600 let json = serde_json::to_string(&metrics).unwrap();
5601
5602 assert!(json.contains("test-project"));
5603 assert!(json.contains("worker-1"));
5604 assert!(json.contains("2.5")); let parsed: CompilationMetrics = serde_json::from_str(&json).unwrap();
5607 assert_eq!(parsed.project_id, "test-project");
5608 assert_eq!(parsed.worker_id, "worker-1");
5609 assert!(parsed.success);
5610 }
5611
5612 #[test]
5613 fn test_compilation_timer_new() {
5614 let _guard = test_guard!();
5615 let timer = CompilationTimer::new("my-project", "my-worker");
5616 assert_eq!(timer.project_id, "my-project");
5617 assert_eq!(timer.worker_id, "my-worker");
5618 assert!(timer.rsync_up.is_none());
5619 assert!(timer.remote_build.is_none());
5620 assert!(timer.rsync_down.is_none());
5621 }
5622
5623 #[test]
5624 fn test_compilation_timer_phases() {
5625 let _guard = test_guard!();
5626 let mut timer = CompilationTimer::new("test", "worker");
5627
5628 std::thread::sleep(Duration::from_millis(5));
5630 timer.end_rsync_up();
5631 assert!(timer.rsync_up.is_some());
5632 assert!(timer.rsync_up.unwrap() >= Duration::from_millis(5));
5633
5634 std::thread::sleep(Duration::from_millis(10));
5636 timer.end_remote_build();
5637 assert!(timer.remote_build.is_some());
5638 assert!(timer.remote_build.unwrap() >= Duration::from_millis(10));
5639
5640 std::thread::sleep(Duration::from_millis(5));
5642 timer.end_rsync_down();
5643 assert!(timer.rsync_down.is_some());
5644 assert!(timer.rsync_down.unwrap() >= Duration::from_millis(5));
5645
5646 let metrics = timer.finish(0, 50, 500_000);
5648 assert_eq!(metrics.project_id, "test");
5649 assert_eq!(metrics.worker_id, "worker");
5650 assert!(metrics.success);
5651 assert!(metrics.timing.total >= Duration::from_millis(20));
5652 assert_eq!(metrics.files_synced, 50);
5653 assert_eq!(metrics.bytes_transferred, 500_000);
5654 }
5655
5656 #[test]
5657 fn test_metrics_aggregator_new() {
5658 let _guard = test_guard!();
5659 let agg = MetricsAggregator::new(100);
5660 assert_eq!(agg.count(), 0);
5661 assert!(agg.average_speedup().is_none());
5662 }
5663
5664 #[test]
5665 fn test_metrics_aggregator_record() {
5666 let _guard = test_guard!();
5667 let mut agg = MetricsAggregator::new(3);
5668
5669 agg.record(make_test_metrics(Some(2.0), 10, true));
5671 assert_eq!(agg.count(), 1);
5672
5673 agg.record(make_test_metrics(Some(3.0), 20, true));
5675 agg.record(make_test_metrics(Some(4.0), 30, true));
5676 assert_eq!(agg.count(), 3);
5677
5678 agg.record(make_test_metrics(Some(5.0), 40, true));
5680 assert_eq!(agg.count(), 3);
5681 }
5682
5683 #[test]
5684 fn test_metrics_aggregator_average_speedup() {
5685 let _guard = test_guard!();
5686 let mut agg = MetricsAggregator::new(100);
5687
5688 assert!(agg.average_speedup().is_none());
5690
5691 for i in 1..=5 {
5693 agg.record(make_test_metrics(Some(i as f64), 10, true));
5694 }
5695
5696 let avg = agg.average_speedup().unwrap();
5697 assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5698 }
5699
5700 #[test]
5701 fn test_metrics_aggregator_average_speedup_with_none() {
5702 let _guard = test_guard!();
5703 let mut agg = MetricsAggregator::new(100);
5704
5705 agg.record(make_test_metrics(Some(2.0), 10, true));
5707 agg.record(make_test_metrics(None, 10, true)); agg.record(make_test_metrics(Some(4.0), 10, true));
5709
5710 let avg = agg.average_speedup().unwrap();
5712 assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5713 }
5714
5715 #[test]
5716 fn test_metrics_aggregator_percentiles() {
5717 let _guard = test_guard!();
5718 let mut agg = MetricsAggregator::new(100);
5719
5720 for i in 1..=10 {
5722 agg.record(make_test_metrics(Some(1.0), i, true));
5723 }
5724
5725 let p50 = agg.p50_total_time().unwrap();
5727 assert!(p50 >= Duration::from_secs(5) && p50 <= Duration::from_secs(6));
5728
5729 let p95 = agg.p95_total_time().unwrap();
5731 assert!(p95 >= Duration::from_secs(9) && p95 <= Duration::from_secs(10));
5732
5733 let p99 = agg.p99_total_time().unwrap();
5735 assert!(p99 >= Duration::from_secs(9));
5736 }
5737
5738 #[test]
5739 fn test_metrics_aggregator_success_rate() {
5740 let _guard = test_guard!();
5741 let mut agg = MetricsAggregator::new(100);
5742
5743 assert_eq!(agg.success_rate(), 100.0);
5745
5746 agg.record(make_test_metrics(Some(1.0), 10, true));
5748 agg.record(make_test_metrics(Some(1.0), 10, true));
5749 assert_eq!(agg.success_rate(), 100.0);
5750
5751 agg.record(make_test_metrics(Some(1.0), 10, false));
5753 let rate = agg.success_rate();
5754 assert!((rate - 66.67).abs() < 1.0, "Expected ~66.67%, got {}", rate);
5755 }
5756
5757 #[test]
5758 fn test_metrics_aggregator_clear() {
5759 let _guard = test_guard!();
5760 let mut agg = MetricsAggregator::new(100);
5761 agg.record(make_test_metrics(Some(1.0), 10, true));
5762 agg.record(make_test_metrics(Some(2.0), 20, true));
5763
5764 assert_eq!(agg.count(), 2);
5765
5766 agg.clear();
5767 assert_eq!(agg.count(), 0);
5768 assert!(agg.average_speedup().is_none());
5769 }
5770
5771 #[test]
5774 fn test_remote_build_jobs_parse_display_and_serde() {
5775 let _guard = test_guard!();
5776 assert_eq!(RemoteBuildJobs::default(), RemoteBuildJobs::Auto);
5777 for (spelling, expected) in [
5778 ("auto", RemoteBuildJobs::Auto),
5779 (" AUTO ", RemoteBuildJobs::Auto),
5780 ("off", RemoteBuildJobs::Off),
5781 ("none", RemoteBuildJobs::Off),
5782 ("4", RemoteBuildJobs::Fixed(4)),
5783 ] {
5784 assert_eq!(
5785 RemoteBuildJobs::parse(spelling).unwrap(),
5786 expected,
5787 "{spelling}"
5788 );
5789 }
5790 assert!(RemoteBuildJobs::parse("0").is_err(), "cargo rejects -j 0");
5791 assert!(RemoteBuildJobs::parse("many").is_err());
5792 assert_eq!(RemoteBuildJobs::Fixed(4).to_string(), "4");
5793 assert_eq!(RemoteBuildJobs::Off.to_string(), "off");
5794
5795 #[derive(Deserialize)]
5798 struct Wrap {
5799 compilation: CompilationConfig,
5800 }
5801 let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = \"off\"\n").unwrap();
5802 assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Off);
5803 let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = 6\n").unwrap();
5804 assert_eq!(
5805 parsed.compilation.remote_build_jobs,
5806 RemoteBuildJobs::Fixed(6)
5807 );
5808 let parsed: Wrap = toml::from_str("[compilation]\nbuild_slots = 4\n").unwrap();
5809 assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Auto);
5810 assert!(toml::from_str::<Wrap>("[compilation]\nremote_build_jobs = 0\n").is_err());
5811
5812 let json = serde_json::to_string(&RemoteBuildJobs::Fixed(3)).unwrap();
5814 assert_eq!(json, "3");
5815 assert_eq!(
5816 serde_json::to_string(&RemoteBuildJobs::Auto).unwrap(),
5817 "\"auto\""
5818 );
5819 let back: RemoteBuildJobs = serde_json::from_str("\"auto\"").unwrap();
5820 assert_eq!(back, RemoteBuildJobs::Auto);
5821 }
5822
5823 #[test]
5824 fn test_compilation_config_default_timeouts() {
5825 let _guard = test_guard!();
5826 let config = CompilationConfig::default();
5827 assert_eq!(config.build_timeout_sec, 300);
5829 assert_eq!(config.test_timeout_sec, 1800);
5831 }
5832
5833 #[test]
5834 fn test_compilation_config_timeout_for_test_kinds() {
5835 let _guard = test_guard!();
5836 let config = CompilationConfig::default();
5837
5838 assert_eq!(
5840 config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5841 std::time::Duration::from_secs(1800)
5842 );
5843 assert_eq!(
5844 config.timeout_for_kind(Some(crate::CompilationKind::CargoNextest)),
5845 std::time::Duration::from_secs(1800)
5846 );
5847 assert_eq!(
5848 config.timeout_for_kind(Some(crate::CompilationKind::BunTest)),
5849 std::time::Duration::from_secs(config.bun_timeout_sec)
5850 );
5851 }
5852
5853 #[test]
5854 fn test_compilation_config_timeout_for_build_kinds() {
5855 let _guard = test_guard!();
5856 let config = CompilationConfig::default();
5857
5858 assert_eq!(
5860 config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5861 std::time::Duration::from_secs(300)
5862 );
5863 assert_eq!(
5864 config.timeout_for_kind(Some(crate::CompilationKind::CargoCheck)),
5865 std::time::Duration::from_secs(300)
5866 );
5867 assert_eq!(
5868 config.timeout_for_kind(Some(crate::CompilationKind::CargoClippy)),
5869 std::time::Duration::from_secs(300)
5870 );
5871 assert_eq!(
5872 config.timeout_for_kind(None),
5873 std::time::Duration::from_secs(300)
5874 );
5875 }
5876
5877 #[test]
5878 fn test_compilation_config_custom_timeouts() {
5879 let _guard = test_guard!();
5880 let config = CompilationConfig {
5881 build_timeout_sec: 600, test_timeout_sec: 3600, ..Default::default()
5884 };
5885
5886 assert_eq!(
5887 config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5888 std::time::Duration::from_secs(600)
5889 );
5890 assert_eq!(
5891 config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5892 std::time::Duration::from_secs(3600)
5893 );
5894 }
5895
5896 #[test]
5897 fn test_compilation_config_speedup_threshold_default() {
5898 let _guard = test_guard!();
5899 let config = CompilationConfig::default();
5900 assert!((config.remote_speedup_threshold - 1.2).abs() < 0.001);
5902 }
5903
5904 #[test]
5905 fn test_compilation_config_speedup_threshold_custom() {
5906 let _guard = test_guard!();
5907 let config = CompilationConfig {
5908 remote_speedup_threshold: 1.5, ..Default::default()
5910 };
5911 assert!((config.remote_speedup_threshold - 1.5).abs() < 0.001);
5912 }
5913
5914 #[test]
5915 fn test_compilation_config_speedup_threshold_no_minimum() {
5916 let _guard = test_guard!();
5917 let config = CompilationConfig {
5919 remote_speedup_threshold: 1.0,
5920 ..Default::default()
5921 };
5922 assert!((config.remote_speedup_threshold - 1.0).abs() < 0.001);
5923 }
5924
5925 #[test]
5926 fn test_compilation_config_min_local_time_ms_default() {
5927 let _guard = test_guard!();
5928 let config = CompilationConfig::default();
5929 assert_eq!(config.min_local_time_ms, 2000);
5931 }
5932
5933 #[test]
5934 fn test_compilation_config_serde_roundtrip_speedup_threshold() {
5935 let _guard = test_guard!();
5936 let config = CompilationConfig {
5937 remote_speedup_threshold: 2.5,
5938 min_local_time_ms: 5000,
5939 ..Default::default()
5940 };
5941 let json = serde_json::to_string(&config).unwrap();
5942 let parsed: CompilationConfig = serde_json::from_str(&json).unwrap();
5943 assert!((parsed.remote_speedup_threshold - 2.5).abs() < 0.001);
5944 assert_eq!(parsed.min_local_time_ms, 5000);
5945 }
5946
5947 #[test]
5952 fn test_validate_remote_base_absolute_path() {
5953 let _guard = test_guard!();
5954 assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5955 assert_eq!(
5956 validate_remote_base("/var/rch-builds").unwrap(),
5957 "/var/rch-builds"
5958 );
5959 assert_eq!(
5960 validate_remote_base("/home/builder/.rch").unwrap(),
5961 "/home/builder/.rch"
5962 );
5963 }
5964
5965 #[test]
5966 fn test_validate_remote_base_tilde_expansion() {
5967 let _guard = test_guard!();
5968 let result = validate_remote_base("~/rch");
5970 assert!(result.is_ok());
5971 let path = result.unwrap();
5972 assert!(
5973 path.starts_with('/'),
5974 "Path should be absolute after expansion: {}",
5975 path
5976 );
5977 assert!(!path.contains('~'), "Tilde should be expanded: {}", path);
5978 }
5979
5980 #[test]
5981 fn test_validate_remote_base_rejects_relative_path() {
5982 let _guard = test_guard!();
5983 let result = validate_remote_base("tmp/rch");
5984 assert!(result.is_err());
5985 assert!(result.unwrap_err().contains("absolute path"));
5986 }
5987
5988 #[test]
5989 fn test_validate_remote_base_rejects_path_traversal() {
5990 let _guard = test_guard!();
5991 let result = validate_remote_base("/tmp/../etc/rch");
5992 assert!(result.is_err());
5993 assert!(result.unwrap_err().contains("path traversal"));
5994
5995 let result = validate_remote_base("/tmp/rch/../other");
5996 assert!(result.is_err());
5997 assert!(result.unwrap_err().contains("path traversal"));
5998 }
5999
6000 #[test]
6001 fn test_validate_remote_base_normalizes_trailing_slash() {
6002 let _guard = test_guard!();
6003 assert_eq!(validate_remote_base("/tmp/rch/").unwrap(), "/tmp/rch");
6004 assert_eq!(validate_remote_base("/tmp/rch///").unwrap(), "/tmp/rch");
6005 }
6006
6007 #[test]
6008 fn test_validate_remote_base_root_path() {
6009 let _guard = test_guard!();
6010 let result = validate_remote_base("/");
6012 assert!(result.is_err());
6013 assert!(result.unwrap_err().contains("root directory"));
6014 }
6015
6016 #[test]
6017 fn test_validate_remote_base_rejects_top_level() {
6018 let _guard = test_guard!();
6019 let result = validate_remote_base("/tmp");
6021 assert!(result.is_err());
6022 assert!(result.unwrap_err().contains("at least 2 levels deep"));
6023
6024 let result = validate_remote_base("/home");
6025 assert!(result.is_err());
6026
6027 assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
6029 }
6030
6031 #[test]
6032 fn test_default_remote_base() {
6033 let _guard = test_guard!();
6034 assert_eq!(default_remote_base(), "/data/tmp/rch");
6035 }
6036
6037 #[test]
6038 fn test_transfer_config_default_has_remote_base() {
6039 let _guard = test_guard!();
6040 let config = TransferConfig::default();
6041 assert_eq!(config.remote_base, "/data/tmp/rch");
6042 }
6043
6044 #[test]
6045 fn test_source_sync_timeout_validation_boundaries() {
6046 let _guard = test_guard!();
6047 assert!(!TransferConfig::valid_sync_timeout_ms(0));
6048 assert!(!TransferConfig::valid_sync_timeout_ms(999));
6049 assert!(TransferConfig::valid_sync_timeout_ms(1_000));
6050 assert!(TransferConfig::valid_sync_timeout_ms(3_600_000));
6051 assert!(!TransferConfig::valid_sync_timeout_ms(3_600_001));
6052 }
6053
6054 #[test]
6055 fn test_source_sync_timeout_is_explicit_or_payload_aware() {
6056 let _guard = test_guard!();
6057 let default = TransferConfig::default();
6058 assert_eq!(
6059 default.sync_timeout_for_payload(0),
6060 std::time::Duration::from_millis(30_000)
6061 );
6062 assert_eq!(
6063 default.sync_timeout_for_payload(10 * 1024 * 1024),
6064 std::time::Duration::from_millis(40_000)
6065 );
6066 assert_eq!(
6067 default.sync_timeout_for_payload(u64::MAX),
6068 std::time::Duration::from_millis(3_600_000)
6069 );
6070
6071 let explicit = TransferConfig {
6072 sync_timeout_ms: Some(120_000),
6073 ..TransferConfig::default()
6074 };
6075 assert_eq!(
6076 explicit.sync_timeout_for_payload(u64::MAX),
6077 std::time::Duration::from_millis(120_000)
6078 );
6079 }
6080
6081 #[test]
6086 fn test_retry_config_default() {
6087 let _guard = test_guard!();
6088 let config = RetryConfig::default();
6089 assert_eq!(config.max_attempts, 3);
6090 assert_eq!(config.base_delay_ms, 100);
6091 assert_eq!(config.max_delay_ms, 5000);
6092 assert_eq!(config.jitter_factor, 0.1);
6093 assert_eq!(config.total_timeout_ms, 30000);
6094 }
6095
6096 #[test]
6097 fn test_retry_config_no_retry() {
6098 let _guard = test_guard!();
6099 let config = RetryConfig::no_retry();
6100 assert_eq!(config.max_attempts, 1);
6101 }
6102
6103 #[test]
6104 fn test_retry_config_delay_for_attempt_zero() {
6105 let _guard = test_guard!();
6106 let config = RetryConfig::default();
6107 assert_eq!(config.delay_for_attempt(0), std::time::Duration::ZERO);
6108 }
6109
6110 #[test]
6111 fn test_retry_config_delay_for_attempt_exponential() {
6112 let _guard = test_guard!();
6113 let config = RetryConfig {
6114 base_delay_ms: 100,
6115 max_delay_ms: 10000,
6116 jitter_factor: 0.0, ..Default::default()
6118 };
6119
6120 assert_eq!(
6122 config.delay_for_attempt(1),
6123 std::time::Duration::from_millis(100)
6124 );
6125 assert_eq!(
6127 config.delay_for_attempt(2),
6128 std::time::Duration::from_millis(200)
6129 );
6130 assert_eq!(
6132 config.delay_for_attempt(3),
6133 std::time::Duration::from_millis(400)
6134 );
6135 }
6136
6137 #[test]
6138 fn test_retry_config_delay_capped_at_max() {
6139 let _guard = test_guard!();
6140 let config = RetryConfig {
6141 base_delay_ms: 1000,
6142 max_delay_ms: 2000,
6143 jitter_factor: 0.0,
6144 ..Default::default()
6145 };
6146
6147 assert_eq!(
6149 config.delay_for_attempt(2),
6150 std::time::Duration::from_millis(2000)
6151 );
6152 assert_eq!(
6154 config.delay_for_attempt(3),
6155 std::time::Duration::from_millis(2000)
6156 );
6157 }
6158
6159 #[test]
6160 fn test_retry_config_random_jitter_stays_within_bounds() {
6161 let _guard = test_guard!();
6162 let config = RetryConfig {
6163 base_delay_ms: 100,
6164 max_delay_ms: 200,
6165 jitter_factor: 0.25,
6166 ..Default::default()
6167 };
6168
6169 for (attempt, minimum_ms, maximum_ms) in [(1, 75, 125), (3, 150, 250)] {
6172 for _ in 0..128 {
6173 let delay = config.delay_for_attempt(attempt);
6174 assert!(delay >= std::time::Duration::from_millis(minimum_ms));
6175 assert!(delay <= std::time::Duration::from_millis(maximum_ms));
6176 }
6177 }
6178 }
6179
6180 #[test]
6181 fn test_retry_config_should_retry() {
6182 let _guard = test_guard!();
6183 let config = RetryConfig {
6184 max_attempts: 3,
6185 total_timeout_ms: 1000,
6186 ..Default::default()
6187 };
6188
6189 assert!(config.should_retry(0, std::time::Duration::from_millis(0)));
6191 assert!(config.should_retry(1, std::time::Duration::from_millis(500)));
6192 assert!(config.should_retry(2, std::time::Duration::from_millis(900)));
6193
6194 assert!(!config.should_retry(3, std::time::Duration::from_millis(0)));
6196
6197 assert!(!config.should_retry(1, std::time::Duration::from_millis(1001)));
6199 }
6200
6201 #[test]
6202 fn test_retry_config_serde_roundtrip() {
6203 let _guard = test_guard!();
6204 let config = RetryConfig {
6205 max_attempts: 5,
6206 base_delay_ms: 200,
6207 max_delay_ms: 8000,
6208 jitter_factor: 0.2,
6209 total_timeout_ms: 60000,
6210 };
6211 let json = serde_json::to_string(&config).unwrap();
6212 let parsed: RetryConfig = serde_json::from_str(&json).unwrap();
6213 assert_eq!(parsed.max_attempts, 5);
6214 assert_eq!(parsed.base_delay_ms, 200);
6215 assert_eq!(parsed.max_delay_ms, 8000);
6216 assert_eq!(parsed.jitter_factor, 0.2);
6217 assert_eq!(parsed.total_timeout_ms, 60000);
6218 }
6219
6220 #[test]
6221 fn test_transfer_config_includes_retry() {
6222 let _guard = test_guard!();
6223 let config = TransferConfig::default();
6224 assert_eq!(config.retry.max_attempts, 3);
6225 }
6226
6227 #[test]
6232 fn test_execution_config_default_allowlist() {
6233 let _guard = test_guard!();
6234 let config = ExecutionConfig::default();
6235 assert!(config.is_allowed("cargo"));
6237 assert!(config.is_allowed("rustc"));
6238 assert!(config.is_allowed("gcc"));
6239 assert!(config.is_allowed("g++"));
6240 assert!(config.is_allowed("clang"));
6241 assert!(config.is_allowed("clang++"));
6242 assert!(config.is_allowed("make"));
6243 assert!(config.is_allowed("cmake"));
6244 assert!(config.is_allowed("ninja"));
6245 assert!(config.is_allowed("meson"));
6246 assert!(config.is_allowed("bun"));
6247 assert!(config.is_allowed("nextest"));
6248 assert!(config.is_allowed("cc"));
6249 assert!(config.is_allowed("c++"));
6250 }
6251
6252 #[test]
6253 fn test_execution_config_is_allowed_case_insensitive() {
6254 let _guard = test_guard!();
6255 let config = ExecutionConfig::default();
6256 assert!(config.is_allowed("CARGO"));
6258 assert!(config.is_allowed("Cargo"));
6259 assert!(config.is_allowed("GCC"));
6260 assert!(config.is_allowed("Gcc"));
6261 }
6262
6263 #[test]
6264 fn test_execution_config_is_not_allowed() {
6265 let _guard = test_guard!();
6266 let config = ExecutionConfig::default();
6267 assert!(!config.is_allowed("python"));
6269 assert!(!config.is_allowed("npm"));
6270 assert!(!config.is_allowed(""));
6271 assert!(config.is_allowed("go"));
6276 assert!(config.is_allowed("tsc"));
6277 }
6278
6279 #[test]
6280 fn test_execution_config_empty_allowlist() {
6281 let _guard = test_guard!();
6282 let config = ExecutionConfig { allowlist: vec![] };
6283 assert!(!config.is_allowed("cargo"));
6285 assert!(!config.is_allowed("gcc"));
6286 }
6287
6288 #[test]
6289 fn test_execution_config_custom_allowlist() {
6290 let _guard = test_guard!();
6291 let config = ExecutionConfig {
6292 allowlist: vec!["cargo".to_string(), "custom_tool".to_string()],
6293 };
6294 assert!(config.is_allowed("cargo"));
6295 assert!(config.is_allowed("custom_tool"));
6296 assert!(!config.is_allowed("gcc"));
6297 }
6298
6299 #[test]
6300 fn test_execution_config_serde() {
6301 let _guard = test_guard!();
6302 let config = ExecutionConfig {
6303 allowlist: vec!["cargo".to_string(), "rustc".to_string()],
6304 };
6305 let json = serde_json::to_string(&config).unwrap();
6306 let parsed: ExecutionConfig = serde_json::from_str(&json).unwrap();
6307 assert_eq!(parsed.allowlist, config.allowlist);
6308 }
6309
6310 #[test]
6311 fn test_rch_config_includes_execution() {
6312 let _guard = test_guard!();
6313 let config = RchConfig::default();
6314 assert!(config.execution.is_allowed("cargo"));
6316 }
6317
6318 #[test]
6323 fn test_adaptive_compression_disabled_by_default() {
6324 let _guard = test_guard!();
6325 let config = TransferConfig::default();
6326 assert!(!config.adaptive_compression);
6327 assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6329 assert_eq!(config.select_compression_level(Some(500_000_000)), 3);
6330 }
6331
6332 #[test]
6333 fn test_adaptive_compression_small_payload() {
6334 let _guard = test_guard!();
6335 let config = TransferConfig {
6336 adaptive_compression: true,
6337 ..Default::default()
6338 };
6339 assert_eq!(config.select_compression_level(Some(0)), 1);
6341 assert_eq!(config.select_compression_level(Some(1_000_000)), 1); assert_eq!(config.select_compression_level(Some(9_999_999)), 1); }
6344
6345 #[test]
6346 fn test_adaptive_compression_medium_payload() {
6347 let _guard = test_guard!();
6348 let config = TransferConfig {
6349 adaptive_compression: true,
6350 ..Default::default()
6351 };
6352 assert_eq!(config.select_compression_level(Some(10_000_000)), 3); assert_eq!(config.select_compression_level(Some(100_000_000)), 3); assert_eq!(config.select_compression_level(Some(199_999_999)), 3); }
6357
6358 #[test]
6359 fn test_adaptive_compression_large_payload() {
6360 let _guard = test_guard!();
6361 let config = TransferConfig {
6362 adaptive_compression: true,
6363 ..Default::default()
6364 };
6365 assert_eq!(config.select_compression_level(Some(200_000_000)), 7); assert_eq!(config.select_compression_level(Some(500_000_000)), 7); assert_eq!(config.select_compression_level(Some(1_000_000_000)), 7); }
6370
6371 #[test]
6372 fn test_adaptive_compression_no_estimate() {
6373 let _guard = test_guard!();
6374 let config = TransferConfig {
6375 adaptive_compression: true,
6376 compression_level: 5,
6377 ..Default::default()
6378 };
6379 assert_eq!(config.select_compression_level(None), 5);
6381 }
6382
6383 #[test]
6384 fn test_adaptive_compression_respects_min_level() {
6385 let _guard = test_guard!();
6386 let config = TransferConfig {
6387 adaptive_compression: true,
6388 min_compression_level: 3,
6389 ..Default::default()
6390 };
6391 assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6393 }
6394
6395 #[test]
6396 fn test_adaptive_compression_respects_max_level() {
6397 let _guard = test_guard!();
6398 let config = TransferConfig {
6399 adaptive_compression: true,
6400 max_compression_level: 5,
6401 ..Default::default()
6402 };
6403 assert_eq!(config.select_compression_level(Some(500_000_000)), 5);
6405 }
6406
6407 #[test]
6408 fn test_adaptive_compression_serde_roundtrip() {
6409 let _guard = test_guard!();
6410 let config = TransferConfig {
6411 adaptive_compression: true,
6412 min_compression_level: 2,
6413 max_compression_level: 8,
6414 ..Default::default()
6415 };
6416 let json = serde_json::to_string(&config).unwrap();
6417 let parsed: TransferConfig = serde_json::from_str(&json).unwrap();
6418 assert!(parsed.adaptive_compression);
6419 assert_eq!(parsed.min_compression_level, 2);
6420 assert_eq!(parsed.max_compression_level, 8);
6421 }
6422
6423 #[test]
6428 fn test_self_healing_config_defaults() {
6429 let _guard = test_guard!();
6430 let config = SelfHealingConfig::default();
6432
6433 assert!(
6435 config.hook_starts_daemon,
6436 "hook_starts_daemon should be true by default"
6437 );
6438 assert!(
6439 config.daemon_installs_hooks,
6440 "daemon_installs_hooks should be true by default"
6441 );
6442
6443 assert_eq!(
6445 config.auto_start_cooldown_secs, 30,
6446 "auto_start_cooldown_secs should default to 30"
6447 );
6448 assert_eq!(
6449 config.auto_start_timeout_secs, 3,
6450 "auto_start_timeout_secs should default to 3"
6451 );
6452 }
6454
6455 #[test]
6456 fn test_self_healing_config_serde_full() {
6457 let _guard = test_guard!();
6458 let config = SelfHealingConfig {
6460 hook_starts_daemon: false,
6461 daemon_installs_hooks: false,
6462 auto_start_cooldown_secs: 60,
6463 auto_start_timeout_secs: 10,
6464 self_healing_log_level: SelfHealingLogLevel::Debug,
6465 };
6466
6467 let json = serde_json::to_string(&config).unwrap();
6468 assert!(json.contains("\"hook_starts_daemon\":false"));
6469 assert!(json.contains("\"daemon_installs_hooks\":false"));
6470 assert!(json.contains("\"auto_start_cooldown_secs\":60"));
6471 assert!(json.contains("\"auto_start_timeout_secs\":10"));
6472 assert!(json.contains("\"self_healing_log_level\":\"debug\""));
6473
6474 let parsed: SelfHealingConfig = serde_json::from_str(&json).unwrap();
6475 assert!(!parsed.hook_starts_daemon);
6476 assert!(!parsed.daemon_installs_hooks);
6477 assert_eq!(parsed.auto_start_cooldown_secs, 60);
6478 assert_eq!(parsed.auto_start_timeout_secs, 10);
6479 assert_eq!(parsed.self_healing_log_level, SelfHealingLogLevel::Debug);
6480 }
6482
6483 #[test]
6486 fn test_self_healing_log_level_default_is_info() {
6487 let level = SelfHealingLogLevel::default();
6489 assert_eq!(level, SelfHealingLogLevel::Info);
6490 let config = SelfHealingConfig::default();
6492 assert_eq!(
6493 config.self_healing_log_level,
6494 SelfHealingLogLevel::Info,
6495 "config default should be Info"
6496 );
6497 }
6499
6500 #[test]
6501 fn test_self_healing_log_level_serializes_lowercase() {
6502 let json = serde_json::to_string(&SelfHealingLogLevel::Warn).unwrap();
6504 assert_eq!(json, "\"warn\"");
6505 let parsed: SelfHealingLogLevel = serde_json::from_str("\"error\"").unwrap();
6506 assert_eq!(parsed, SelfHealingLogLevel::Error);
6507 }
6509
6510 #[test]
6511 fn test_self_healing_log_level_from_env_str_accepts_known() {
6512 assert_eq!(
6514 SelfHealingLogLevel::from_env_str("debug"),
6515 Some(SelfHealingLogLevel::Debug)
6516 );
6517 assert_eq!(
6518 SelfHealingLogLevel::from_env_str("DEBUG"),
6519 Some(SelfHealingLogLevel::Debug)
6520 );
6521 assert_eq!(
6522 SelfHealingLogLevel::from_env_str("info"),
6523 Some(SelfHealingLogLevel::Info)
6524 );
6525 assert_eq!(
6526 SelfHealingLogLevel::from_env_str("warn"),
6527 Some(SelfHealingLogLevel::Warn)
6528 );
6529 assert_eq!(
6530 SelfHealingLogLevel::from_env_str("warning"),
6531 Some(SelfHealingLogLevel::Warn),
6532 "warning should map to Warn"
6533 );
6534 assert_eq!(
6535 SelfHealingLogLevel::from_env_str("error"),
6536 Some(SelfHealingLogLevel::Error)
6537 );
6538 assert_eq!(
6540 SelfHealingLogLevel::from_env_str(" debug "),
6541 Some(SelfHealingLogLevel::Debug)
6542 );
6543 }
6545
6546 #[test]
6547 fn test_self_healing_log_level_from_env_str_rejects_unknown() {
6548 assert!(SelfHealingLogLevel::from_env_str("banana").is_none());
6550 assert!(SelfHealingLogLevel::from_env_str("").is_none());
6551 assert!(SelfHealingLogLevel::from_env_str("trace").is_none());
6552 }
6554
6555 #[test]
6556 fn test_self_healing_config_serde_partial_uses_defaults() {
6557 let _guard = test_guard!();
6558 let json = r#"{"hook_starts_daemon": false}"#;
6560 let config: SelfHealingConfig = serde_json::from_str(json).unwrap();
6561
6562 assert!(
6563 !config.hook_starts_daemon,
6564 "Explicit false should be parsed"
6565 );
6566 assert!(
6567 config.daemon_installs_hooks,
6568 "Missing field should use default (true)"
6569 );
6570 assert_eq!(
6571 config.auto_start_cooldown_secs, 30,
6572 "Missing field should use default (30)"
6573 );
6574 assert_eq!(
6575 config.auto_start_timeout_secs, 3,
6576 "Missing field should use default (3)"
6577 );
6578 }
6580
6581 #[test]
6582 fn test_self_healing_config_toml_with_alias() {
6583 let _guard = test_guard!();
6584 let toml_str = r#"
6586 hook_starts_daemon = true
6587 daemon_installs_hooks = false
6588 auto_start_cooldown_secs = 45
6589 daemon_start_timeout = 7
6590 "#;
6591
6592 let config: SelfHealingConfig = toml::from_str(toml_str).unwrap();
6593 assert!(config.hook_starts_daemon);
6594 assert!(!config.daemon_installs_hooks);
6595 assert_eq!(config.auto_start_cooldown_secs, 45);
6596 assert_eq!(
6597 config.auto_start_timeout_secs, 7,
6598 "daemon_start_timeout alias should set auto_start_timeout_secs"
6599 );
6600 }
6602
6603 #[allow(unsafe_code)]
6604 mod self_healing_env_override_tests {
6605 use super::*;
6606 use crate::config::env_test_lock;
6607
6608 fn env_guard() -> std::sync::MutexGuard<'static, ()> {
6609 env_test_lock()
6610 }
6611
6612 fn set_env(key: &str, value: &str) {
6613 unsafe { std::env::set_var(key, value) };
6615 }
6616
6617 fn remove_env(key: &str) {
6618 unsafe { std::env::remove_var(key) };
6620 }
6621
6622 struct EnvVarGuard {
6623 key: &'static str,
6624 old: Option<String>,
6625 }
6626
6627 impl EnvVarGuard {
6628 fn set(key: &'static str, value: &str) -> Self {
6629 let old = std::env::var(key).ok();
6630 set_env(key, value);
6631 Self { key, old }
6632 }
6633 }
6634
6635 impl Drop for EnvVarGuard {
6636 fn drop(&mut self) {
6637 if let Some(old) = &self.old {
6638 set_env(self.key, old);
6639 } else {
6640 remove_env(self.key);
6641 }
6642 }
6643 }
6644
6645 #[test]
6646 fn test_self_healing_config_with_env_overrides_master_disable() {
6647 let _guard = test_guard!();
6648 let _guard = env_guard();
6649
6650 let _no_self_healing = EnvVarGuard::set("RCH_NO_SELF_HEALING", "1");
6651 let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "1");
6652 let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "1");
6653 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "99");
6654 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "99");
6655
6656 let config = SelfHealingConfig::default().with_env_overrides();
6657 assert!(
6658 !config.hook_starts_daemon,
6659 "RCH_NO_SELF_HEALING should disable hook auto-start"
6660 );
6661 assert!(
6662 !config.daemon_installs_hooks,
6663 "RCH_NO_SELF_HEALING should disable daemon hook installation"
6664 );
6665
6666 assert_eq!(config.auto_start_cooldown_secs, 30);
6668 assert_eq!(config.auto_start_timeout_secs, 3);
6669 }
6670
6671 #[test]
6672 fn test_self_healing_config_with_env_overrides_toggles_and_numbers() {
6673 let _guard = test_guard!();
6674 let _guard = env_guard();
6675
6676 let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "0");
6677 let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "false");
6678 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "45");
6679 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "7");
6680
6681 let config = SelfHealingConfig::default().with_env_overrides();
6682 assert!(!config.hook_starts_daemon);
6683 assert!(!config.daemon_installs_hooks);
6684 assert_eq!(config.auto_start_cooldown_secs, 45);
6685 assert_eq!(config.auto_start_timeout_secs, 7);
6686 }
6687
6688 #[test]
6689 fn test_self_healing_config_with_env_overrides_invalid_numbers_ignored() {
6690 let _guard = test_guard!();
6691 let _guard = env_guard();
6692
6693 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "not-a-number");
6694 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "nope");
6695
6696 let config = SelfHealingConfig::default().with_env_overrides();
6697 assert_eq!(config.auto_start_cooldown_secs, 30);
6698 assert_eq!(config.auto_start_timeout_secs, 3);
6699 }
6700 }
6701
6702 #[test]
6707 fn test_load_per_core_calculation() {
6708 let _guard = test_guard!();
6709 let mut caps = WorkerCapabilities::new();
6710 assert!(caps.load_per_core().is_none());
6712
6713 caps.load_avg_1 = Some(4.0);
6715 assert!(caps.load_per_core().is_none());
6716
6717 caps.num_cpus = Some(4);
6719 assert_eq!(caps.load_per_core(), Some(1.0)); caps.load_avg_1 = Some(16.0);
6723 assert_eq!(caps.load_per_core(), Some(4.0)); }
6725
6726 #[test]
6727 fn test_is_high_load() {
6728 let _guard = test_guard!();
6729 let mut caps = WorkerCapabilities::new();
6730 caps.load_avg_1 = Some(8.0);
6731 caps.num_cpus = Some(4);
6732 assert_eq!(caps.is_high_load(3.0), Some(false));
6736 assert_eq!(caps.is_high_load(2.0), Some(false)); assert_eq!(caps.is_high_load(1.5), Some(true));
6740
6741 caps.load_avg_1 = None;
6743 assert!(caps.is_high_load(1.0).is_none());
6744 }
6745
6746 #[test]
6747 fn test_is_low_disk() {
6748 let _guard = test_guard!();
6749 let mut caps = WorkerCapabilities::new();
6750 caps.disk_free_gb = Some(15.0);
6751
6752 assert_eq!(caps.is_low_disk(10.0), Some(false));
6754 assert_eq!(caps.is_low_disk(15.0), Some(false)); assert_eq!(caps.is_low_disk(20.0), Some(true));
6758
6759 caps.disk_free_gb = None;
6761 assert!(caps.is_low_disk(10.0).is_none());
6762 }
6763
6764 #[test]
6765 fn low_disk_judges_the_build_disk_not_a_small_tmpfs() {
6766 let _guard = test_guard!();
6767 let mut caps = WorkerCapabilities::new();
6770 caps.disk_free_gb = Some(8.0);
6771 caps.disk_total_gb = Some(15.5);
6772 caps.build_disk_free_gb = Some(1700.0);
6773 caps.build_disk_total_gb = Some(1900.0);
6774 assert_eq!(caps.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6775 assert_eq!(caps.is_low_disk(10.0), Some(false));
6776
6777 caps.build_disk_free_gb = Some(4.0);
6779 assert_eq!(caps.is_low_disk(10.0), Some(true));
6780
6781 caps.build_disk_free_gb = None;
6783 caps.build_disk_total_gb = None;
6784 assert_eq!(caps.build_disk_gb(), (Some(8.0), Some(15.5)));
6785 assert_eq!(caps.is_low_disk(10.0), Some(true));
6786 }
6787
6788 #[test]
6789 fn build_disk_sample_is_optional_on_the_wire() {
6790 let _guard = test_guard!();
6791 let old: WorkerCapabilities =
6792 serde_json::from_str(r#"{"disk_free_gb":13.9,"disk_total_gb":15.5}"#).unwrap();
6793 assert_eq!(old.build_disk_free_gb, None);
6794 assert_eq!(old.build_disk_gb(), (Some(13.9), Some(15.5)));
6795
6796 let new: WorkerCapabilities = serde_json::from_str(
6797 r#"{"disk_free_gb":13.9,"disk_total_gb":15.5,"build_disk_free_gb":1700.0,"build_disk_total_gb":1900.0}"#,
6798 )
6799 .unwrap();
6800 assert_eq!(new.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6801 let json = serde_json::to_string(&new).unwrap();
6802 assert!(json.contains("\"build_disk_free_gb\":1700.0"), "{json}");
6803 }
6804
6805 #[test]
6806 fn test_selection_config_preflight_defaults() {
6807 let _guard = test_guard!();
6808 let config = SelectionConfig::default();
6809 assert_eq!(config.max_load_per_core, Some(2.0));
6811 assert_eq!(config.min_free_gb, Some(10.0));
6812 assert_eq!(config.disk_gb_per_slot, 10.0);
6813 }
6814
6815 #[test]
6816 fn test_selection_config_preflight_serde() {
6817 let _guard = test_guard!();
6818 let config = SelectionConfig {
6819 max_load_per_core: Some(3.5),
6820 min_free_gb: Some(25.0),
6821 disk_gb_per_slot: 0.83,
6822 ..Default::default()
6823 };
6824 let json = serde_json::to_string(&config).unwrap();
6825 let parsed: SelectionConfig = serde_json::from_str(&json).unwrap();
6826 assert_eq!(parsed.max_load_per_core, Some(3.5));
6827 assert_eq!(parsed.min_free_gb, Some(25.0));
6828 assert_eq!(parsed.disk_gb_per_slot, 0.83);
6829 }
6830
6831 #[test]
6832 fn test_selection_disk_slot_budget_validation() {
6833 let _guard = test_guard!();
6834 for value in ["0.0", "-1.0", "nan", "inf", "-inf"] {
6835 let result = toml::from_str::<SelectionConfig>(&format!("disk_gb_per_slot = {value}"));
6836 assert!(result.is_err(), "invalid disk slot budget {value}");
6837 }
6838 let default: SelectionConfig = toml::from_str("").unwrap();
6839 assert_eq!(default.disk_gb_per_slot, 10.0);
6840 }
6841
6842 #[test]
6843 fn test_selection_disk_weight_validation_and_roundtrip() {
6844 for value in ["-0.1", "1.1", "nan", "inf", "-inf"] {
6845 assert!(toml::from_str::<SelectionWeightConfig>(&format!("disk = {value}")).is_err());
6846 }
6847 for value in [0.0, 0.2, 1.0] {
6848 let weights: SelectionWeightConfig =
6849 toml::from_str(&format!("disk = {value}")).unwrap();
6850 assert_eq!(weights.disk, value);
6851 let roundtrip: SelectionWeightConfig =
6852 serde_json::from_str(&serde_json::to_string(&weights).unwrap()).unwrap();
6853 assert_eq!(roundtrip.disk, value);
6854 }
6855 let defaults: SelectionWeightConfig = toml::from_str("").unwrap();
6856 assert_eq!(defaults.disk, 0.2);
6857 }
6858
6859 #[test]
6860 fn test_worker_capabilities_health_metrics_serde() {
6861 let _guard = test_guard!();
6862 let caps = WorkerCapabilities {
6863 num_cpus: Some(8),
6864 load_avg_1: Some(1.5),
6865 load_avg_5: Some(2.0),
6866 load_avg_15: Some(1.8),
6867 disk_free_gb: Some(50.5),
6868 disk_total_gb: Some(100.0),
6869 ..Default::default()
6870 };
6871 let json = serde_json::to_string(&caps).unwrap();
6872 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
6873 assert_eq!(parsed.num_cpus, Some(8));
6874 assert_eq!(parsed.load_avg_1, Some(1.5));
6875 assert_eq!(parsed.disk_free_gb, Some(50.5));
6876 }
6877
6878 #[test]
6879 fn microarch_level_classifies_ivy_bridge_as_v2() {
6880 let ivy = "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov \
6883 pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx rdtscp lm \
6884 constant_tsc pni pclmulqdq ssse3 cx16 sse4_1 sse4_2 x2apic \
6885 popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm";
6886 assert_eq!(x86_64_microarch_level_from_flags(ivy), 2);
6887 }
6888
6889 #[test]
6890 fn microarch_level_classifies_v3_and_v4() {
6891 let v3 = "cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 abm avx avx2 bmi1 bmi2 \
6892 f16c fma movbe xsave";
6893 assert_eq!(x86_64_microarch_level_from_flags(v3), 3);
6894
6895 let v4 = format!("{v3} avx512f avx512bw avx512cd avx512dq avx512vl");
6896 assert_eq!(x86_64_microarch_level_from_flags(&v4), 4);
6897 }
6898
6899 #[test]
6900 fn microarch_level_v1_for_ancient_or_empty_flags() {
6901 assert_eq!(x86_64_microarch_level_from_flags(""), 1);
6902 assert_eq!(x86_64_microarch_level_from_flags("fpu mmx sse sse2"), 1);
6903 }
6904
6905 #[test]
6906 fn pre_v3_predicate_only_fires_on_affirmative_evidence() {
6907 let mut caps = WorkerCapabilities::new();
6908 assert!(!caps.is_pre_v3_x86(), "unknown must not deprioritize");
6909 caps.cpu_microarch_level = Some(2);
6910 assert!(caps.is_pre_v3_x86());
6911 caps.cpu_microarch_level = Some(3);
6912 assert!(!caps.is_pre_v3_x86());
6913 }
6914}