1use crate::{CompilationKind, toolchain::ToolchainInfo};
4use rand::RngExt;
5use schemars::JsonSchema;
6use serde::{Deserialize, Serialize};
7use std::path::PathBuf;
8
9#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
11pub struct WorkerId(pub String);
12
13impl WorkerId {
14 pub fn new(id: impl Into<String>) -> Self {
15 Self(id.into())
16 }
17
18 pub fn as_str(&self) -> &str {
19 &self.0
20 }
21}
22
23impl std::fmt::Display for WorkerId {
24 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
25 write!(f, "{}", self.0)
26 }
27}
28
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
46#[serde(rename_all = "snake_case")]
47pub enum WorkerStatus {
48 #[default]
50 Healthy,
51 Degraded,
53 Unreachable,
55 Draining,
57 Drained,
59 Disabled,
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
65#[serde(rename_all = "snake_case")]
66pub enum CircuitState {
67 #[default]
69 Closed,
70 Open,
72 HalfOpen,
74}
75
76#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
78#[serde(rename_all = "lowercase")]
79pub enum RequiredRuntime {
80 #[default]
82 None,
83 Rust,
85 Bun,
87 Node,
89 Nix,
91 Go,
93 Zig,
95}
96
97#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
102#[serde(rename_all = "lowercase")]
103pub enum CommandPriority {
104 Low,
105 #[default]
106 Normal,
107 High,
108}
109
110impl std::fmt::Display for CommandPriority {
111 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112 let value = match self {
113 Self::Low => "low",
114 Self::Normal => "normal",
115 Self::High => "high",
116 };
117 write!(f, "{}", value)
118 }
119}
120
121impl std::str::FromStr for CommandPriority {
122 type Err = ();
123
124 fn from_str(s: &str) -> Result<Self, Self::Err> {
125 match s.trim().to_lowercase().as_str() {
126 "low" => Ok(Self::Low),
127 "normal" => Ok(Self::Normal),
128 "high" => Ok(Self::High),
129 _ => Err(()),
130 }
131 }
132}
133
134#[derive(Debug, Clone, Serialize, Deserialize)]
136pub struct SelectionRequest {
137 pub project: String,
139 #[serde(default, skip_serializing_if = "Option::is_none")]
141 pub command: Option<String>,
142 #[serde(default)]
144 pub command_priority: CommandPriority,
145 pub estimated_cores: u32,
147 #[serde(default, skip_serializing_if = "disk_headroom_disabled")]
151 pub disk_headroom_gib: u32,
152 #[serde(default)]
154 pub preferred_workers: Vec<WorkerId>,
155 #[serde(default)]
157 pub toolchain: Option<ToolchainInfo>,
158 #[serde(default)]
160 pub required_runtime: RequiredRuntime,
161 #[serde(default, skip_serializing_if = "Option::is_none")]
164 pub classification_duration_us: Option<u64>,
165 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
171 pub job_mode: bool,
172 #[serde(default, skip_serializing_if = "Vec::is_empty")]
182 pub required_tools: Vec<String>,
183 #[serde(default, skip_serializing_if = "Option::is_none")]
185 pub hook_pid: Option<u32>,
186}
187
188fn disk_headroom_disabled(gib: &u32) -> bool {
189 *gib == 0
190}
191
192#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
197#[serde(rename_all = "snake_case")]
198pub enum SelectionReason {
199 Success,
201 NoWorkersConfigured,
203 AllWorkersUnreachable,
205 AllCircuitsOpen,
207 AllWorkersBusy,
209 NoWorkersPassedHealth,
211 AllWorkersFailedPreflight,
213 AllWorkersFailedConvergence,
215 NoAdmissibleWorkers(String),
217 NoMatchingWorkers,
219 NoWorkersWithRuntime(String),
221 SelectionError(String),
223 AffinityPinned,
225 AffinityFallback,
227}
228
229pub const SELECTION_RESPONSE_PROTOCOL_VERSION: u64 = 1;
235
236impl std::fmt::Display for SelectionReason {
237 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
238 match self {
239 Self::Success => write!(f, "worker assigned successfully"),
240 Self::NoWorkersConfigured => write!(f, "no workers configured"),
241 Self::AllWorkersUnreachable => write!(f, "all workers unreachable"),
242 Self::AllCircuitsOpen => write!(f, "all worker circuits open"),
243 Self::AllWorkersBusy => write!(f, "all workers at capacity"),
244 Self::NoWorkersPassedHealth => write!(f, "no workers passed health thresholds"),
245 Self::AllWorkersFailedPreflight => write!(f, "all workers failed preflight checks"),
246 Self::AllWorkersFailedConvergence => {
247 write!(f, "all workers failed repo convergence checks")
248 }
249 Self::NoAdmissibleWorkers(summary) => {
250 write!(f, "no admissible workers: {}", summary)
251 }
252 Self::NoMatchingWorkers => write!(f, "no matching workers found"),
253 Self::NoWorkersWithRuntime(rt) => write!(f, "no workers with {} installed", rt),
254 Self::SelectionError(e) => write!(f, "selection error: {}", e),
255 Self::AffinityPinned => write!(f, "worker assigned via affinity pinning"),
256 Self::AffinityFallback => write!(f, "worker assigned via last-success fallback"),
257 }
258 }
259}
260
261#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
267#[serde(rename_all = "snake_case")]
268pub enum SelectionStrategy {
269 Priority,
272 Fastest,
275 #[default]
278 Balanced,
279 CacheAffinity,
282 FairFastest,
285}
286
287impl std::fmt::Display for SelectionStrategy {
288 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
289 let name = match self {
290 Self::Priority => "priority",
291 Self::Fastest => "fastest",
292 Self::Balanced => "balanced",
293 Self::CacheAffinity => "cache_affinity",
294 Self::FairFastest => "fair_fastest",
295 };
296 write!(f, "{}", name)
297 }
298}
299
300impl std::str::FromStr for SelectionStrategy {
301 type Err = String;
302
303 fn from_str(s: &str) -> Result<Self, Self::Err> {
304 match s.trim().to_lowercase().as_str() {
305 "priority" => Ok(Self::Priority),
306 "fastest" => Ok(Self::Fastest),
307 "balanced" => Ok(Self::Balanced),
308 "cache_affinity" | "cache-affinity" | "cacheaffinity" => Ok(Self::CacheAffinity),
309 "fair_fastest" | "fair-fastest" | "fairfastest" => Ok(Self::FairFastest),
310 _ => Err(format!(
311 "unknown selection strategy '{}', expected one of: priority, fastest, balanced, cache_affinity, fair_fastest",
312 s
313 )),
314 }
315 }
316}
317
318#[derive(Debug, Clone, Serialize, Deserialize)]
320pub struct SelectionConfig {
321 #[serde(default)]
323 pub strategy: SelectionStrategy,
324 #[serde(default = "default_min_success_rate")]
326 pub min_success_rate: f64,
327 #[serde(default)]
329 pub weights: SelectionWeightConfig,
330 #[serde(default)]
332 pub fairness: FairnessConfig,
333 #[serde(default)]
335 pub affinity: AffinityConfig,
336
337 #[serde(default = "default_max_load_per_core")]
341 pub max_load_per_core: Option<f64>,
342 #[serde(default = "default_min_free_gb")]
345 pub min_free_gb: Option<f64>,
346 #[serde(
349 default = "default_disk_gb_per_slot",
350 deserialize_with = "deserialize_disk_gb_per_slot"
351 )]
352 pub disk_gb_per_slot: f64,
353}
354
355impl Default for SelectionConfig {
356 fn default() -> Self {
357 Self {
358 strategy: SelectionStrategy::default(),
359 min_success_rate: default_min_success_rate(),
360 weights: SelectionWeightConfig::default(),
361 fairness: FairnessConfig::default(),
362 affinity: AffinityConfig::default(),
363 max_load_per_core: default_max_load_per_core(),
364 min_free_gb: default_min_free_gb(),
365 disk_gb_per_slot: default_disk_gb_per_slot(),
366 }
367 }
368}
369
370fn default_min_success_rate() -> f64 {
371 0.8
372}
373
374fn default_max_load_per_core() -> Option<f64> {
375 Some(2.0) }
377
378fn default_min_free_gb() -> Option<f64> {
379 Some(10.0) }
381
382fn default_disk_gb_per_slot() -> f64 {
383 10.0
384}
385
386fn deserialize_disk_gb_per_slot<'de, D>(deserializer: D) -> Result<f64, D::Error>
387where
388 D: serde::Deserializer<'de>,
389{
390 let budget = f64::deserialize(deserializer)?;
391 if budget.is_finite() && budget > 0.0 {
392 Ok(budget)
393 } else {
394 Err(serde::de::Error::custom(
395 "disk_gb_per_slot must be finite and greater than zero",
396 ))
397 }
398}
399
400#[derive(Debug, Clone, Serialize, Deserialize)]
402pub struct SelectionWeightConfig {
403 #[serde(default = "default_weight_speedscore")]
405 pub speedscore: f64,
406 #[serde(default = "default_weight_slots")]
408 pub slots: f64,
409 #[serde(default = "default_weight_health")]
411 pub health: f64,
412 #[serde(default = "default_weight_cache")]
414 pub cache: f64,
415 #[serde(default = "default_weight_network")]
417 pub network: f64,
418 #[serde(default = "default_weight_priority")]
420 pub priority: f64,
421 #[serde(
423 default = "default_weight_disk",
424 deserialize_with = "deserialize_weight_disk"
425 )]
426 pub disk: f64,
427 #[serde(default = "default_half_open_penalty")]
429 pub half_open_penalty: f64,
430}
431
432impl Default for SelectionWeightConfig {
433 fn default() -> Self {
434 Self {
435 speedscore: default_weight_speedscore(),
436 slots: default_weight_slots(),
437 health: default_weight_health(),
438 cache: default_weight_cache(),
439 network: default_weight_network(),
440 priority: default_weight_priority(),
441 disk: default_weight_disk(),
442 half_open_penalty: default_half_open_penalty(),
443 }
444 }
445}
446
447fn default_weight_speedscore() -> f64 {
448 0.5
449}
450fn default_weight_slots() -> f64 {
451 0.4
452}
453fn default_weight_health() -> f64 {
454 0.3
455}
456fn default_weight_cache() -> f64 {
457 0.2
458}
459fn default_weight_network() -> f64 {
460 0.1
461}
462fn default_weight_priority() -> f64 {
463 0.5
470}
471fn default_half_open_penalty() -> f64 {
472 0.5
473}
474
475fn default_weight_disk() -> f64 {
476 0.2
477}
478
479fn deserialize_weight_disk<'de, D>(deserializer: D) -> Result<f64, D::Error>
480where
481 D: serde::Deserializer<'de>,
482{
483 let weight = f64::deserialize(deserializer)?;
484 if weight.is_finite() && (0.0..=1.0).contains(&weight) {
485 Ok(weight)
486 } else {
487 Err(serde::de::Error::custom(
488 "disk weight must be finite and between zero and one",
489 ))
490 }
491}
492
493#[derive(Debug, Clone, Serialize, Deserialize)]
495pub struct FairnessConfig {
496 #[serde(default = "default_fairness_lookback_secs")]
498 pub lookback_secs: u64,
499 #[serde(default = "default_max_consecutive_selections")]
501 pub max_consecutive_selections: u32,
502}
503
504impl Default for FairnessConfig {
505 fn default() -> Self {
506 Self {
507 lookback_secs: default_fairness_lookback_secs(),
508 max_consecutive_selections: default_max_consecutive_selections(),
509 }
510 }
511}
512
513fn default_fairness_lookback_secs() -> u64 {
514 300 }
516
517fn default_max_consecutive_selections() -> u32 {
518 3
519}
520
521#[derive(Debug, Clone, Serialize, Deserialize)]
523pub struct AffinityConfig {
524 #[serde(default = "default_affinity_enabled")]
526 pub enabled: bool,
527 #[serde(default = "default_affinity_pin_minutes")]
530 pub pin_minutes: u64,
531 #[serde(default = "default_last_success_fallback")]
535 pub enable_last_success_fallback: bool,
536 #[serde(default = "default_fallback_min_success_rate")]
539 pub fallback_min_success_rate: f64,
540}
541
542impl Default for AffinityConfig {
543 fn default() -> Self {
544 Self {
545 enabled: default_affinity_enabled(),
546 pin_minutes: default_affinity_pin_minutes(),
547 enable_last_success_fallback: default_last_success_fallback(),
548 fallback_min_success_rate: default_fallback_min_success_rate(),
549 }
550 }
551}
552
553fn default_affinity_enabled() -> bool {
554 true
555}
556
557fn default_affinity_pin_minutes() -> u64 {
558 60 }
560
561fn default_last_success_fallback() -> bool {
562 true
563}
564
565fn default_fallback_min_success_rate() -> f64 {
566 0.5 }
568
569#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
571pub struct SelectedWorker {
572 pub id: WorkerId,
574 pub host: String,
576 pub user: String,
578 pub identity_file: String,
580 pub slots_available: u32,
582 pub speed_score: f64,
584 #[serde(default)]
590 pub declared_os: Option<String>,
591}
592
593#[derive(Debug, Clone, Serialize, Deserialize)]
595pub struct SelectionResponse {
596 pub worker: Option<SelectedWorker>,
598 pub reason: SelectionReason,
600 #[serde(default, skip_serializing_if = "Option::is_none")]
602 pub build_id: Option<u64>,
603 #[serde(default, skip_serializing_if = "Option::is_none")]
605 pub diagnostics: Option<SelectionDiagnostics>,
606}
607
608#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
610#[serde(rename_all = "snake_case")]
611pub enum WorkerSelectionDiagnosticDecision {
612 Allow,
614 FallbackCandidate,
616 Deny,
618}
619
620#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
622pub struct WorkerSelectionDiagnostic {
623 pub worker_id: WorkerId,
625 pub status: String,
627 pub circuit_state: String,
629 pub pressure_state: String,
631 pub pressure_reason_code: String,
633 #[serde(skip_serializing_if = "Option::is_none")]
635 pub success_rate: Option<f64>,
636 pub min_success_rate: f64,
638 pub fallback_min_success_rate: f64,
640 pub required_runtime: RequiredRuntime,
642 pub runtime_available: bool,
644 pub available_slots: u32,
646 pub total_slots: u32,
648 pub estimated_cores: u32,
650 pub active_project_excluded: bool,
652 pub final_decision: WorkerSelectionDiagnosticDecision,
654 pub final_reason: String,
656 pub reason_codes: Vec<String>,
658}
659
660#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
662pub struct SelectionDiagnostics {
663 pub required_runtime: RequiredRuntime,
665 pub estimated_cores: u32,
667 pub min_success_rate: f64,
669 pub fallback_min_success_rate: f64,
671 pub active_project_exclusion_count: usize,
673 pub workers: Vec<WorkerSelectionDiagnostic>,
675}
676
677#[derive(Debug, Clone, Serialize, Deserialize)]
679pub struct ReleaseRequest {
680 pub worker_id: WorkerId,
682 pub slots: u32,
684 #[serde(default, skip_serializing_if = "Option::is_none")]
686 pub build_id: Option<u64>,
687 #[serde(default, skip_serializing_if = "Option::is_none")]
689 pub local_wrapper_id: Option<String>,
690 #[serde(default, skip_serializing_if = "Option::is_none")]
692 pub exit_code: Option<i32>,
693 #[serde(default, skip_serializing_if = "Option::is_none")]
697 pub duration_ms: Option<u64>,
698 #[serde(default, skip_serializing_if = "Option::is_none")]
700 pub bytes_transferred: Option<u64>,
701 #[serde(default, skip_serializing_if = "Option::is_none")]
703 pub timing: Option<CommandTimingBreakdown>,
704 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
708 pub worker_fault: bool,
709 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
713 pub worker_disk_full: bool,
714 #[serde(default, skip_serializing_if = "Vec::is_empty")]
717 pub worker_disk_roots: Vec<String>,
718}
719
720#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
722#[serde(rename_all = "snake_case")]
723pub enum BuildHeartbeatPhase {
724 SyncUp,
726 Execute,
728 SyncDown,
730 Finalize,
732}
733
734#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
736pub struct BuildHeartbeatRequest {
737 pub build_id: u64,
739 pub worker_id: WorkerId,
741 #[serde(default, skip_serializing_if = "Option::is_none")]
743 pub hook_pid: Option<u32>,
744 #[serde(default, skip_serializing_if = "Option::is_none")]
746 pub local_wrapper_id: Option<String>,
747 #[serde(default, skip_serializing_if = "Option::is_none")]
749 pub remote_pgid_file: Option<String>,
750 pub phase: BuildHeartbeatPhase,
752 #[serde(default, skip_serializing_if = "Option::is_none")]
754 pub detail: Option<String>,
755 #[serde(default, skip_serializing_if = "Option::is_none")]
760 pub progress_counter: Option<u64>,
761 #[serde(default, skip_serializing_if = "Option::is_none")]
763 pub progress_percent: Option<f64>,
764}
765
766#[derive(Debug, Clone, Serialize, Deserialize)]
768pub struct WorkerConfig {
769 pub id: WorkerId,
771 pub host: String,
773 pub user: String,
775 pub identity_file: String,
777 pub total_slots: u32,
779 #[serde(default = "default_priority")]
781 pub priority: u32,
782 #[serde(default)]
788 pub tags: Vec<String>,
789 #[serde(default, skip_serializing_if = "Vec::is_empty")]
798 pub tools: Vec<WorkerToolProbe>,
799}
800
801#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
807pub struct WorkerToolProbe {
808 pub name: String,
810 pub command: Vec<String>,
812}
813
814fn default_priority() -> u32 {
815 100
816}
817
818pub const OS_TAG_PREFIX: &str = "os:";
820
821#[must_use]
832pub fn declared_os(tags: &[String]) -> Option<String> {
833 tags.iter()
838 .filter_map(|tag| tag.strip_prefix(OS_TAG_PREFIX))
839 .map(|os| os.trim().to_ascii_lowercase())
840 .find(|os| !os.is_empty())
841}
842
843#[must_use]
845pub fn os_tag(os: &str) -> String {
846 format!("{OS_TAG_PREFIX}{}", os.trim().to_ascii_lowercase())
847}
848
849impl Default for WorkerConfig {
850 fn default() -> Self {
851 Self {
852 id: WorkerId::new("default-worker"),
853 host: "localhost".to_string(),
854 user: "user".to_string(),
855 identity_file: "~/.ssh/id_rsa".to_string(),
856 total_slots: 4,
857 priority: default_priority(),
858 tags: Vec::new(),
859 tools: Vec::new(),
860 }
861 }
862}
863
864#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)]
870pub struct WorkerCapabilities {
871 #[serde(default, skip_serializing_if = "Option::is_none")]
873 pub rustc_version: Option<String>,
874 #[serde(default, skip_serializing_if = "Vec::is_empty")]
877 pub rustup_toolchains: Vec<String>,
878 #[serde(default, skip_serializing_if = "Vec::is_empty")]
882 pub rustup_components: Vec<String>,
883 #[serde(default, skip_serializing_if = "Vec::is_empty")]
889 pub probe_warnings: Vec<String>,
890 #[serde(default, skip_serializing_if = "Option::is_none")]
892 pub bun_version: Option<String>,
893 #[serde(default, skip_serializing_if = "Option::is_none")]
895 pub node_version: Option<String>,
896 #[serde(default, skip_serializing_if = "Option::is_none")]
898 pub npm_version: Option<String>,
899 #[serde(default, skip_serializing_if = "Option::is_none")]
903 pub nix_version: Option<String>,
904 #[serde(default, skip_serializing_if = "Option::is_none")]
907 pub go_version: Option<String>,
908 #[serde(default, skip_serializing_if = "Option::is_none")]
912 pub zig_version: Option<String>,
913 #[serde(default, skip_serializing_if = "Option::is_none")]
917 pub cargo_zigbuild_version: Option<String>,
918 #[serde(default, skip_serializing_if = "Option::is_none")]
925 pub cpu_microarch_level: Option<u8>,
926 #[serde(default, skip_serializing_if = "Vec::is_empty")]
933 pub tools_present: Vec<String>,
934 #[serde(default, skip_serializing_if = "Vec::is_empty")]
940 pub tools_absent: Vec<String>,
941
942 #[serde(default, skip_serializing_if = "Option::is_none")]
945 pub num_cpus: Option<u32>,
946 #[serde(default, skip_serializing_if = "Option::is_none")]
948 pub load_avg_1: Option<f64>,
949 #[serde(default, skip_serializing_if = "Option::is_none")]
951 pub load_avg_5: Option<f64>,
952 #[serde(default, skip_serializing_if = "Option::is_none")]
954 pub load_avg_15: Option<f64>,
955 #[serde(default, skip_serializing_if = "Option::is_none")]
959 pub disk_free_gb: Option<f64>,
960 #[serde(default, skip_serializing_if = "Option::is_none")]
962 pub disk_total_gb: Option<f64>,
963 #[serde(default, skip_serializing_if = "Option::is_none")]
968 pub build_disk_free_gb: Option<f64>,
969 #[serde(default, skip_serializing_if = "Option::is_none")]
971 pub build_disk_total_gb: Option<f64>,
972 #[serde(default, skip_serializing_if = "Option::is_none")]
977 pub projects_root_ok: Option<bool>,
978 #[serde(default, skip_serializing_if = "Option::is_none")]
980 pub projects_root_issue: Option<String>,
981 #[serde(default, skip_serializing_if = "Option::is_none")]
983 pub projects_root_checked_at_unix_ms: Option<i64>,
984}
985
986impl WorkerCapabilities {
987 pub fn new() -> Self {
989 Self::default()
990 }
991
992 pub fn mock_with_rust() -> Self {
996 Self {
997 rustc_version: Some("1.85.0-nightly (mock)".to_string()),
998 projects_root_ok: Some(true),
999 ..Self::default()
1000 }
1001 }
1002
1003 pub fn is_pre_v3_x86(&self) -> bool {
1008 matches!(self.cpu_microarch_level, Some(1 | 2))
1009 }
1010
1011 pub fn has_bun(&self) -> bool {
1013 self.bun_version.is_some()
1014 }
1015
1016 pub fn has_node(&self) -> bool {
1018 self.node_version.is_some()
1019 }
1020
1021 pub fn has_rust(&self) -> bool {
1023 self.rustc_version.is_some()
1024 }
1025
1026 pub fn has_rustup_component(&self, toolchain: &str, component: &str) -> bool {
1033 let owner_prefix = format!("{toolchain}-");
1034 self.rustup_components.iter().any(|fact| {
1035 fact.split_once(':').is_some_and(|(owner, name)| {
1036 name == component && (owner == toolchain || owner.starts_with(&owner_prefix))
1037 })
1038 })
1039 }
1040
1041 pub fn has_nix(&self) -> bool {
1043 self.nix_version.is_some()
1044 }
1045
1046 pub fn has_go(&self) -> bool {
1048 self.go_version.is_some()
1049 }
1050
1051 pub fn has_zig(&self) -> bool {
1061 self.zig_version.is_some() && self.cargo_zigbuild_version.is_some()
1062 }
1063
1064 pub fn load_per_core(&self) -> Option<f64> {
1067 match (self.load_avg_1, self.num_cpus) {
1068 (Some(load), Some(cpus)) if cpus > 0 => Some(load / cpus as f64),
1069 _ => None,
1070 }
1071 }
1072
1073 pub fn is_high_load(&self, max_load_per_core: f64) -> Option<bool> {
1076 self.load_per_core().map(|lpc| lpc > max_load_per_core)
1077 }
1078
1079 pub fn build_disk_gb(&self) -> (Option<f64>, Option<f64>) {
1087 if self.build_disk_free_gb.is_some() {
1088 (self.build_disk_free_gb, self.build_disk_total_gb)
1089 } else {
1090 (self.disk_free_gb, self.disk_total_gb)
1091 }
1092 }
1093
1094 pub fn is_low_disk(&self, min_free_gb: f64) -> Option<bool> {
1097 self.build_disk_gb().0.map(|free| free < min_free_gb)
1098 }
1099
1100 pub fn is_topology_healthy(&self) -> Option<bool> {
1104 self.projects_root_ok
1105 }
1106}
1107
1108#[must_use]
1117pub fn x86_64_microarch_level_from_flags(flags: &str) -> u8 {
1118 let set: std::collections::HashSet<&str> = flags.split_whitespace().collect();
1119 let has_all = |required: &[&str]| required.iter().all(|flag| set.contains(flag));
1120 const V2: &[&str] = &["cx16", "lahf_lm", "popcnt", "sse4_1", "sse4_2", "ssse3"];
1122 const V3: &[&str] = &[
1124 "abm", "avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "movbe", "xsave",
1125 ];
1126 const V4: &[&str] = &["avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl"];
1128 if !has_all(V2) {
1129 return 1;
1130 }
1131 if !has_all(V3) {
1132 return 2;
1133 }
1134 if !has_all(V4) {
1135 return 3;
1136 }
1137 4
1138}
1139
1140#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1153pub struct PathTopologyConfig {
1154 #[serde(default)]
1157 pub canonical_root: Option<String>,
1158
1159 #[serde(default)]
1162 pub alias_root: Option<String>,
1163}
1164
1165impl PathTopologyConfig {
1166 pub fn to_policy(&self) -> crate::path_topology::PathTopologyPolicy {
1172 let canonical = self
1173 .canonical_root
1174 .as_deref()
1175 .filter(|s| !s.is_empty())
1176 .map(|s| shellexpand::tilde(s).into_owned())
1177 .unwrap_or_else(|| crate::path_topology::DEFAULT_CANONICAL_PROJECT_ROOT.to_string());
1178 let alias = self
1179 .alias_root
1180 .as_deref()
1181 .filter(|s| !s.is_empty())
1182 .map(|s| shellexpand::tilde(s).into_owned())
1183 .unwrap_or_else(|| crate::path_topology::DEFAULT_ALIAS_PROJECT_ROOT.to_string());
1184 crate::path_topology::PathTopologyPolicy::new(
1185 PathBuf::from(canonical),
1186 PathBuf::from(alias),
1187 )
1188 }
1189}
1190
1191#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1193pub struct RchConfig {
1194 #[serde(default)]
1195 pub general: GeneralConfig,
1196 #[serde(default)]
1197 pub compilation: CompilationConfig,
1198 #[serde(default)]
1199 pub transfer: TransferConfig,
1200 #[serde(default)]
1201 pub environment: EnvironmentConfig,
1202 #[serde(default)]
1203 pub circuit: CircuitBreakerConfig,
1204 #[serde(default)]
1205 pub output: OutputConfig,
1206 #[serde(default)]
1207 pub self_healing: SelfHealingConfig,
1208 #[serde(default)]
1209 pub self_test: SelfTestConfig,
1210 #[serde(default)]
1212 pub selection: SelectionConfig,
1213 #[serde(default)]
1215 pub execution: ExecutionConfig,
1216 #[serde(default)]
1218 pub alerts: AlertsConfig,
1219 #[serde(default)]
1221 pub fleet: FleetConfig,
1222 #[serde(default)]
1224 pub path_topology: PathTopologyConfig,
1225 #[serde(default)]
1227 pub doctor: DoctorConfig,
1228 #[serde(default)]
1233 pub remediation: crate::remediation_config::RemediationConfig,
1234 #[serde(default)]
1237 pub layer0: Layer0Config,
1238 #[serde(default)]
1241 pub api: ApiConfig,
1242 #[serde(default)]
1244 pub dashboard: DashboardConfig,
1245 #[serde(default)]
1247 pub jobs: JobsConfig,
1248}
1249
1250#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1252pub struct JobsConfig {
1253 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1263 pub required_tools: Vec<String>,
1264}
1265
1266#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1283#[serde(default)]
1284pub struct ApiConfig {
1285 pub bind: String,
1289 pub token: Option<String>,
1291 pub token_file: Option<String>,
1293 pub no_token: bool,
1297 pub allow_any_addr: bool,
1299}
1300
1301#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1303#[serde(default)]
1304pub struct DashboardConfig {
1305 pub url: Option<String>,
1307}
1308
1309#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1315pub struct DoctorConfig {
1316 #[serde(default)]
1318 pub webhooks: DoctorWebhooksConfig,
1319}
1320
1321#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1327pub struct DoctorWebhooksConfig {
1328 #[serde(default)]
1330 pub endpoints: Vec<DoctorWebhookEndpoint>,
1331 #[serde(default = "default_webhook_max_inflight")]
1336 pub max_inflight: usize,
1337}
1338
1339impl Default for DoctorWebhooksConfig {
1343 fn default() -> Self {
1344 Self {
1345 endpoints: Vec::new(),
1346 max_inflight: default_webhook_max_inflight(),
1347 }
1348 }
1349}
1350
1351#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1359pub struct DoctorWebhookEndpoint {
1360 pub name: String,
1362 pub url: String,
1364 #[serde(default)]
1366 pub format: DoctorWebhookFormat,
1367 #[serde(default)]
1372 pub on_transitions: Vec<String>,
1373 #[serde(default = "default_webhook_retry_max")]
1375 pub retry_max: u32,
1376 #[serde(default = "default_webhook_retry_backoff_ms")]
1378 pub retry_backoff_ms: u64,
1379 #[serde(default = "default_webhook_timeout_ms")]
1381 pub timeout_ms: u64,
1382 #[serde(default = "default_true")]
1384 pub enabled: bool,
1385 #[serde(default)]
1388 pub routing_key_env: Option<String>,
1389 #[serde(default)]
1392 pub bearer_token_env: Option<String>,
1393 #[serde(default)]
1397 pub signing_secret_env: Option<String>,
1398}
1399
1400#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
1402#[serde(rename_all = "snake_case")]
1403pub enum DoctorWebhookFormat {
1404 #[default]
1406 GenericJson,
1407 Slack,
1409 Pagerduty,
1411}
1412
1413fn default_webhook_max_inflight() -> usize {
1414 16
1415}
1416
1417fn default_webhook_retry_max() -> u32 {
1418 3
1419}
1420
1421fn default_webhook_retry_backoff_ms() -> u64 {
1422 200
1423}
1424
1425fn default_webhook_timeout_ms() -> u64 {
1426 5000
1427}
1428
1429#[derive(Debug, Clone, Serialize, Deserialize)]
1431pub struct AlertsConfig {
1432 #[serde(default = "default_true")]
1434 pub enabled: bool,
1435 #[serde(default = "default_alert_suppress_duplicates_secs")]
1437 pub suppress_duplicates_secs: u64,
1438 #[serde(default = "default_alert_cleared_retention_secs")]
1443 pub cleared_retention_secs: u64,
1444 #[serde(default)]
1446 pub webhook: Option<WebhookConfig>,
1447}
1448
1449#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1451pub struct WebhookConfig {
1452 pub url: Option<String>,
1454 #[serde(default)]
1456 pub secret: Option<String>,
1457 #[serde(default = "default_webhook_timeout_secs")]
1459 pub timeout_secs: u64,
1460 #[serde(default = "default_webhook_retry_count")]
1462 pub retry_count: u32,
1463 #[serde(default)]
1466 pub events: Vec<String>,
1467}
1468
1469fn default_webhook_timeout_secs() -> u64 {
1470 5
1471}
1472
1473fn default_webhook_retry_count() -> u32 {
1474 3
1475}
1476
1477fn default_alert_suppress_duplicates_secs() -> u64 {
1478 300
1479}
1480
1481fn default_alert_cleared_retention_secs() -> u64 {
1482 300
1483}
1484
1485impl Default for AlertsConfig {
1486 fn default() -> Self {
1487 Self {
1488 enabled: true,
1489 suppress_duplicates_secs: default_alert_suppress_duplicates_secs(),
1490 cleared_retention_secs: default_alert_cleared_retention_secs(),
1491 webhook: None,
1492 }
1493 }
1494}
1495
1496#[derive(Debug, Clone, Serialize, Deserialize)]
1501pub struct FleetConfig {
1502 #[serde(default = "default_fleet_ssh_connect_timeout_secs")]
1504 pub ssh_connect_timeout_secs: u64,
1505 #[serde(default = "default_fleet_ssh_command_timeout_secs")]
1507 pub ssh_command_timeout_secs: u64,
1508 #[serde(default = "default_fleet_min_disk_space_mb")]
1510 pub min_disk_space_mb: u64,
1511 #[serde(default = "default_fleet_max_load_average")]
1513 pub max_load_average: f64,
1514 #[serde(default = "default_fleet_max_concurrent_workers")]
1516 pub max_concurrent_workers: usize,
1517 #[serde(default = "default_fleet_retry_count")]
1519 pub retry_count: u32,
1520 #[serde(default = "default_fleet_retry_delay_ms")]
1522 pub retry_delay_ms: u64,
1523}
1524
1525fn default_fleet_ssh_connect_timeout_secs() -> u64 {
1526 10
1527}
1528
1529fn default_fleet_ssh_command_timeout_secs() -> u64 {
1530 30
1531}
1532
1533fn default_fleet_min_disk_space_mb() -> u64 {
1534 500
1535}
1536
1537fn default_fleet_max_load_average() -> f64 {
1538 10.0
1539}
1540
1541fn default_fleet_max_concurrent_workers() -> usize {
1542 10
1543}
1544
1545fn default_fleet_retry_count() -> u32 {
1546 2
1547}
1548
1549fn default_fleet_retry_delay_ms() -> u64 {
1550 1000
1551}
1552
1553impl Default for FleetConfig {
1554 fn default() -> Self {
1555 Self {
1556 ssh_connect_timeout_secs: default_fleet_ssh_connect_timeout_secs(),
1557 ssh_command_timeout_secs: default_fleet_ssh_command_timeout_secs(),
1558 min_disk_space_mb: default_fleet_min_disk_space_mb(),
1559 max_load_average: default_fleet_max_load_average(),
1560 max_concurrent_workers: default_fleet_max_concurrent_workers(),
1561 retry_count: default_fleet_retry_count(),
1562 retry_delay_ms: default_fleet_retry_delay_ms(),
1563 }
1564 }
1565}
1566
1567impl FleetConfig {
1568 pub fn ssh_connect_timeout(&self) -> std::time::Duration {
1570 std::time::Duration::from_secs(self.ssh_connect_timeout_secs)
1571 }
1572
1573 pub fn ssh_command_timeout(&self) -> std::time::Duration {
1575 std::time::Duration::from_secs(self.ssh_command_timeout_secs)
1576 }
1577
1578 pub fn retry_delay(&self) -> std::time::Duration {
1580 std::time::Duration::from_millis(self.retry_delay_ms)
1581 }
1582}
1583
1584#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
1587#[serde(rename_all = "lowercase")]
1588pub enum BoxRole {
1589 Dispatcher,
1594 Worker,
1597 #[default]
1599 Hybrid,
1600}
1601
1602impl BoxRole {
1603 #[must_use]
1605 pub fn as_str(self) -> &'static str {
1606 match self {
1607 BoxRole::Dispatcher => "dispatcher",
1608 BoxRole::Worker => "worker",
1609 BoxRole::Hybrid => "hybrid",
1610 }
1611 }
1612}
1613
1614#[derive(Debug, Clone, Serialize, Deserialize)]
1615pub struct GeneralConfig {
1616 #[serde(default = "default_true")]
1618 pub enabled: bool,
1619 #[serde(default)]
1622 pub role: BoxRole,
1623 #[serde(default)]
1627 pub force_local: bool,
1628 #[serde(default)]
1633 pub force_remote: bool,
1634 #[serde(default = "default_log_level")]
1636 pub log_level: String,
1637 #[serde(default = "default_socket_path")]
1639 pub socket_path: String,
1640}
1641
1642#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1644pub struct EnvironmentConfig {
1645 #[serde(default)]
1647 pub allowlist: Vec<String>,
1648}
1649
1650impl Default for GeneralConfig {
1651 fn default() -> Self {
1652 Self {
1653 enabled: true,
1654 role: BoxRole::default(),
1655 force_local: false,
1656 force_remote: false,
1657 log_level: "info".to_string(),
1658 socket_path: default_socket_path(),
1659 }
1660 }
1661}
1662
1663#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
1665#[serde(rename_all = "lowercase")]
1666pub enum OutputVisibility {
1667 #[default]
1668 None,
1669 Summary,
1670 Verbose,
1671}
1672
1673impl std::fmt::Display for OutputVisibility {
1674 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1675 let value = match self {
1676 OutputVisibility::None => "none",
1677 OutputVisibility::Summary => "summary",
1678 OutputVisibility::Verbose => "verbose",
1679 };
1680 write!(f, "{}", value)
1681 }
1682}
1683
1684impl std::str::FromStr for OutputVisibility {
1685 type Err = ();
1686
1687 fn from_str(s: &str) -> Result<Self, Self::Err> {
1688 match s.trim().to_lowercase().as_str() {
1689 "none" | "silent" | "quiet" => Ok(Self::None),
1690 "summary" | "short" => Ok(Self::Summary),
1691 "verbose" | "debug" => Ok(Self::Verbose),
1692 _ => Err(()),
1693 }
1694 }
1695}
1696
1697#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1702#[serde(rename_all = "lowercase")]
1703pub enum ColorMode {
1704 #[default]
1707 Always,
1708 Auto,
1710 Never,
1712}
1713
1714impl std::fmt::Display for ColorMode {
1715 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1716 let value = match self {
1717 ColorMode::Always => "always",
1718 ColorMode::Auto => "auto",
1719 ColorMode::Never => "never",
1720 };
1721 write!(f, "{}", value)
1722 }
1723}
1724
1725impl std::str::FromStr for ColorMode {
1726 type Err = ();
1727
1728 fn from_str(s: &str) -> Result<Self, Self::Err> {
1729 match s.trim().to_lowercase().as_str() {
1730 "always" | "force" | "yes" | "true" => Ok(Self::Always),
1731 "auto" | "detect" => Ok(Self::Auto),
1732 "never" | "none" | "no" | "false" => Ok(Self::Never),
1733 _ => Err(()),
1734 }
1735 }
1736}
1737
1738fn default_color_mode() -> ColorMode {
1739 ColorMode::default()
1740}
1741
1742#[derive(Debug, Clone, Serialize, Deserialize)]
1744pub struct OutputConfig {
1745 #[serde(default = "default_output_visibility")]
1747 pub visibility: OutputVisibility,
1748 #[serde(default)]
1750 pub first_run_complete: bool,
1751 #[serde(default = "default_color_mode")]
1754 pub color_mode: ColorMode,
1755}
1756
1757impl Default for OutputConfig {
1758 fn default() -> Self {
1759 Self {
1760 visibility: OutputVisibility::None,
1761 first_run_complete: false,
1762 color_mode: ColorMode::default(),
1763 }
1764 }
1765}
1766
1767fn default_autostart_cooldown_secs() -> u64 {
1772 30
1773}
1774
1775fn default_autostart_timeout_secs() -> u64 {
1776 3
1777}
1778
1779#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
1785#[serde(rename_all = "lowercase")]
1786pub enum SelfHealingLogLevel {
1787 Debug,
1788 #[default]
1789 Info,
1790 Warn,
1791 Error,
1792}
1793
1794impl SelfHealingLogLevel {
1795 pub fn from_env_str(s: &str) -> Option<Self> {
1797 match s.trim().to_ascii_lowercase().as_str() {
1798 "debug" => Some(Self::Debug),
1799 "info" => Some(Self::Info),
1800 "warn" | "warning" => Some(Self::Warn),
1801 "error" => Some(Self::Error),
1802 _ => None,
1803 }
1804 }
1805}
1806
1807#[derive(Debug, Clone, Serialize, Deserialize)]
1809pub struct SelfHealingConfig {
1810 #[serde(default = "default_true")]
1812 pub hook_starts_daemon: bool,
1813 #[serde(default = "default_true")]
1815 pub daemon_installs_hooks: bool,
1816 #[serde(default = "default_autostart_cooldown_secs")]
1818 pub auto_start_cooldown_secs: u64,
1819 #[serde(
1822 default = "default_autostart_timeout_secs",
1823 alias = "daemon_start_timeout"
1824 )]
1825 pub auto_start_timeout_secs: u64,
1826 #[serde(default)]
1829 pub self_healing_log_level: SelfHealingLogLevel,
1830}
1831
1832impl Default for SelfHealingConfig {
1833 fn default() -> Self {
1834 Self {
1835 hook_starts_daemon: default_true(),
1836 daemon_installs_hooks: default_true(),
1837 auto_start_cooldown_secs: default_autostart_cooldown_secs(),
1838 auto_start_timeout_secs: default_autostart_timeout_secs(),
1839 self_healing_log_level: SelfHealingLogLevel::default(),
1840 }
1841 }
1842}
1843
1844impl SelfHealingConfig {
1845 pub fn with_env_overrides(mut self) -> Self {
1854 if let Ok(val) = std::env::var("RCH_NO_SELF_HEALING")
1856 && (val == "1" || val.eq_ignore_ascii_case("true"))
1857 {
1858 self.hook_starts_daemon = false;
1859 self.daemon_installs_hooks = false;
1860 return self;
1861 }
1862
1863 if let Ok(val) = std::env::var("RCH_HOOK_STARTS_DAEMON") {
1865 self.hook_starts_daemon = val != "0" && !val.eq_ignore_ascii_case("false");
1866 }
1867 if let Ok(val) = std::env::var("RCH_DAEMON_INSTALLS_HOOKS") {
1868 self.daemon_installs_hooks = val != "0" && !val.eq_ignore_ascii_case("false");
1869 }
1870
1871 if let Ok(val) = std::env::var("RCH_AUTO_START_TIMEOUT_SECS")
1873 && let Ok(secs) = val.parse()
1874 {
1875 self.auto_start_timeout_secs = secs;
1876 }
1877 if let Ok(val) = std::env::var("RCH_AUTO_START_COOLDOWN_SECS")
1878 && let Ok(secs) = val.parse()
1879 {
1880 self.auto_start_cooldown_secs = secs;
1881 }
1882 if let Ok(val) = std::env::var("RCH_SELF_HEALING_LOG_LEVEL") {
1884 if let Some(level) = SelfHealingLogLevel::from_env_str(&val) {
1885 self.self_healing_log_level = level;
1886 } else {
1887 tracing::warn!(
1888 target: "rch::self_healing",
1889 invalid = %val,
1890 "RCH_SELF_HEALING_LOG_LEVEL has invalid value; using default=info"
1891 );
1892 }
1893 }
1894
1895 self
1896 }
1897}
1898
1899#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1905#[serde(rename_all = "snake_case")]
1906pub enum SelfTestFailureAction {
1907 #[default]
1909 Alert,
1910 DisableWorker,
1912 AlertAndDisable,
1914}
1915
1916#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1918#[serde(untagged)]
1919pub enum SelfTestWorkers {
1920 All(String),
1922 List(Vec<WorkerId>),
1924}
1925
1926impl Default for SelfTestWorkers {
1927 fn default() -> Self {
1928 Self::All("all".to_string())
1929 }
1930}
1931
1932impl SelfTestWorkers {
1933 pub fn resolve(&self) -> Option<Vec<WorkerId>> {
1935 match self {
1936 SelfTestWorkers::All(value) => {
1937 if value.eq_ignore_ascii_case("all") {
1938 None
1939 } else {
1940 Some(vec![WorkerId::new(value.clone())])
1941 }
1942 }
1943 SelfTestWorkers::List(list) => Some(list.clone()),
1944 }
1945 }
1946}
1947
1948fn default_self_test_retry_count() -> u32 {
1949 3
1950}
1951
1952fn default_self_test_retry_delay() -> String {
1953 "5m".to_string()
1954}
1955
1956fn default_self_test_enabled() -> bool {
1957 false
1958}
1959
1960#[derive(Debug, Clone, Serialize, Deserialize)]
1962pub struct SelfTestConfig {
1963 #[serde(default = "default_self_test_enabled")]
1965 pub enabled: bool,
1966 #[serde(default)]
1968 pub schedule: Option<String>,
1969 #[serde(default)]
1971 pub interval: Option<String>,
1972 #[serde(default)]
1974 pub workers: SelfTestWorkers,
1975 #[serde(default)]
1977 pub on_failure: SelfTestFailureAction,
1978 #[serde(default = "default_self_test_retry_count")]
1980 pub retry_count: u32,
1981 #[serde(default = "default_self_test_retry_delay")]
1983 pub retry_delay: String,
1984}
1985
1986impl Default for SelfTestConfig {
1987 fn default() -> Self {
1988 Self {
1989 enabled: default_self_test_enabled(),
1990 schedule: None,
1991 interval: None,
1992 workers: SelfTestWorkers::default(),
1993 on_failure: SelfTestFailureAction::default(),
1994 retry_count: default_self_test_retry_count(),
1995 retry_delay: default_self_test_retry_delay(),
1996 }
1997 }
1998}
1999
2000#[derive(Debug, Clone, Serialize, Deserialize)]
2002pub struct CircuitBreakerConfig {
2003 #[serde(default = "default_circuit_failure_threshold")]
2005 pub failure_threshold: u32,
2006 #[serde(default = "default_circuit_success_threshold")]
2008 pub success_threshold: u32,
2009 #[serde(default = "default_circuit_error_rate_threshold")]
2011 pub error_rate_threshold: f64,
2012 #[serde(default = "default_circuit_window_secs")]
2014 pub window_secs: u64,
2015 #[serde(default = "default_circuit_open_cooldown_secs")]
2017 pub open_cooldown_secs: u64,
2018 #[serde(default = "default_circuit_half_open_max_probes")]
2020 pub half_open_max_probes: u32,
2021 #[serde(default = "default_circuit_command_failure_threshold")]
2032 pub command_failure_threshold: u32,
2033 #[serde(default = "default_circuit_open_cooldown_max_secs")]
2041 pub open_cooldown_max_secs: u64,
2042}
2043
2044impl Default for CircuitBreakerConfig {
2045 fn default() -> Self {
2046 Self {
2047 failure_threshold: default_circuit_failure_threshold(),
2048 success_threshold: default_circuit_success_threshold(),
2049 error_rate_threshold: default_circuit_error_rate_threshold(),
2050 window_secs: default_circuit_window_secs(),
2051 open_cooldown_secs: default_circuit_open_cooldown_secs(),
2052 half_open_max_probes: default_circuit_half_open_max_probes(),
2053 command_failure_threshold: default_circuit_command_failure_threshold(),
2054 open_cooldown_max_secs: default_circuit_open_cooldown_max_secs(),
2055 }
2056 }
2057}
2058
2059impl CircuitBreakerConfig {
2060 pub fn cooldown_ms_for_streak(&self, reopen_streak: u32) -> u64 {
2064 let base_ms = self.open_cooldown_secs.saturating_mul(1000);
2065 let cap_ms = self
2066 .open_cooldown_max_secs
2067 .saturating_mul(1000)
2068 .max(base_ms);
2069 let exponent = reopen_streak.saturating_sub(1).min(16);
2071 base_ms.saturating_mul(1u64 << exponent).min(cap_ms)
2072 }
2073}
2074
2075#[derive(Debug, Clone, Default)]
2080pub struct CircuitStats {
2081 state: CircuitState,
2083 consecutive_failures: u32,
2085 consecutive_successes: u32,
2087 window_successes: u32,
2089 window_failures: u32,
2091 opened_at: Option<u64>,
2093 last_state_change: u64,
2095 active_probes: u32,
2097 consecutive_command_failures: u32,
2102 reopen_streak: u32,
2105 recent_results: Vec<bool>,
2108}
2109
2110const CIRCUIT_HISTORY_SIZE: usize = 10;
2112
2113impl CircuitStats {
2114 pub fn new() -> Self {
2116 Self {
2117 state: CircuitState::Closed,
2118 consecutive_failures: 0,
2119 consecutive_successes: 0,
2120 window_successes: 0,
2121 window_failures: 0,
2122 opened_at: None,
2123 last_state_change: Self::now_millis(),
2124 active_probes: 0,
2125 consecutive_command_failures: 0,
2126 reopen_streak: 0,
2127 recent_results: Vec::with_capacity(CIRCUIT_HISTORY_SIZE),
2128 }
2129 }
2130
2131 pub fn consecutive_command_failures(&self) -> u32 {
2134 self.consecutive_command_failures
2135 }
2136
2137 pub fn reopen_streak(&self) -> u32 {
2140 self.reopen_streak
2141 }
2142
2143 fn effective_cooldown_ms(&self, config: &CircuitBreakerConfig) -> u64 {
2145 config.cooldown_ms_for_streak(self.reopen_streak.max(1))
2146 }
2147
2148 pub fn state(&self) -> CircuitState {
2150 self.state
2151 }
2152
2153 pub fn opened_at(&self) -> Option<u64> {
2155 self.opened_at
2156 }
2157
2158 pub fn last_state_change(&self) -> u64 {
2160 self.last_state_change
2161 }
2162
2163 pub fn consecutive_failures(&self) -> u32 {
2165 self.consecutive_failures
2166 }
2167
2168 pub fn error_rate(&self) -> f64 {
2178 if self.recent_results.is_empty() {
2179 return 0.0;
2180 }
2181 let failures = self.recent_results.iter().filter(|&&ok| !ok).count();
2182 failures as f64 / self.recent_results.len() as f64
2183 }
2184
2185 pub fn record_success(&mut self) {
2190 self.window_successes += 1;
2191 self.consecutive_failures = 0;
2192 self.push_result(true);
2193
2194 if self.state == CircuitState::HalfOpen {
2195 self.consecutive_successes += 1;
2196 if self.active_probes > 0 {
2197 self.active_probes -= 1;
2198 }
2199 }
2200 }
2201
2202 pub fn record_failure(&mut self) {
2207 self.window_failures += 1;
2208 self.consecutive_failures += 1;
2209 self.push_result(false);
2210
2211 if self.state == CircuitState::HalfOpen {
2212 self.consecutive_successes = 0;
2213 if self.active_probes > 0 {
2214 self.active_probes -= 1;
2215 }
2216 }
2217 }
2218
2219 pub fn should_open(&self, config: &CircuitBreakerConfig) -> bool {
2225 if self.state != CircuitState::Closed {
2226 return false;
2227 }
2228
2229 if self.consecutive_failures >= config.failure_threshold {
2231 return true;
2232 }
2233
2234 if self.recent_results.len() >= 5 && self.error_rate() >= config.error_rate_threshold {
2238 return true;
2239 }
2240
2241 false
2242 }
2243
2244 pub fn should_half_open(&self, config: &CircuitBreakerConfig) -> bool {
2248 if self.state != CircuitState::Open {
2249 return false;
2250 }
2251
2252 let now = Self::now_millis();
2253 let cooldown_ms = self.effective_cooldown_ms(config);
2254
2255 if let Some(opened_at) = self.opened_at {
2256 now.saturating_sub(opened_at) >= cooldown_ms
2257 } else {
2258 false
2259 }
2260 }
2261
2262 pub fn should_close(&self, config: &CircuitBreakerConfig) -> bool {
2271 if self.state != CircuitState::HalfOpen {
2272 return false;
2273 }
2274
2275 self.consecutive_successes >= config.success_threshold
2276 && self.consecutive_command_failures == 0
2277 }
2278
2279 pub fn can_probe(&self, config: &CircuitBreakerConfig) -> bool {
2283 if self.state != CircuitState::HalfOpen {
2284 return false;
2285 }
2286
2287 self.active_probes < config.half_open_max_probes
2288 }
2289
2290 pub fn start_probe(&mut self, config: &CircuitBreakerConfig) -> bool {
2294 if !self.can_probe(config) {
2295 return false;
2296 }
2297 self.active_probes += 1;
2298 true
2299 }
2300
2301 pub fn open(&mut self) {
2303 if self.state != CircuitState::Open {
2304 self.state = CircuitState::Open;
2305 self.opened_at = Some(Self::now_millis());
2306 self.last_state_change = Self::now_millis();
2307 self.consecutive_successes = 0;
2308 self.active_probes = 0;
2309 self.reopen_streak = self.reopen_streak.saturating_add(1);
2312 }
2313 }
2314
2315 pub fn half_open(&mut self) {
2317 if self.state != CircuitState::HalfOpen {
2318 self.state = CircuitState::HalfOpen;
2319 self.last_state_change = Self::now_millis();
2320 self.consecutive_successes = 0;
2321 self.active_probes = 0;
2322 }
2323 }
2324
2325 pub fn close(&mut self) {
2327 if self.state != CircuitState::Closed {
2328 self.state = CircuitState::Closed;
2329 self.last_state_change = Self::now_millis();
2330 self.opened_at = None;
2331 self.consecutive_failures = 0;
2332 self.consecutive_successes = 0;
2333 self.active_probes = 0;
2334 self.consecutive_command_failures = 0;
2335 self.reopen_streak = 0;
2336 self.window_successes = 0;
2342 self.window_failures = 0;
2343 self.recent_results.clear();
2344 }
2345 }
2346
2347 pub fn reset_window(&mut self) {
2353 self.window_successes = 0;
2354 self.window_failures = 0;
2355 self.recent_results.clear();
2356 }
2357
2358 pub fn recent_results(&self) -> &[bool] {
2363 &self.recent_results
2364 }
2365
2366 pub fn recovery_remaining_secs(&self, config: &CircuitBreakerConfig) -> Option<u64> {
2370 if self.state != CircuitState::Open {
2371 return None;
2372 }
2373
2374 let now = Self::now_millis();
2375 let cooldown_ms = self.effective_cooldown_ms(config);
2376
2377 if let Some(opened_at) = self.opened_at {
2378 let elapsed_ms = now.saturating_sub(opened_at);
2379 if elapsed_ms < cooldown_ms {
2380 return Some((cooldown_ms - elapsed_ms) / 1000);
2381 }
2382 }
2383 None
2384 }
2385
2386 fn push_result(&mut self, success: bool) {
2388 if self.recent_results.len() >= CIRCUIT_HISTORY_SIZE {
2389 self.recent_results.remove(0);
2390 }
2391 self.recent_results.push(success);
2392 }
2393
2394 fn now_millis() -> u64 {
2395 std::time::SystemTime::now()
2396 .duration_since(std::time::UNIX_EPOCH)
2397 .map(|d| d.as_millis() as u64)
2398 .unwrap_or(0)
2399 }
2400
2401 pub fn apply_health_outcome(
2427 &mut self,
2428 healthy: bool,
2429 transient: bool,
2430 config: &CircuitBreakerConfig,
2431 ) -> CircuitState {
2432 if healthy {
2433 if self.state == CircuitState::Open && self.should_half_open(config) {
2437 self.half_open();
2438 }
2439 self.record_success();
2440 if self.should_close(config) {
2441 self.close();
2442 }
2443 } else {
2444 let was_half_open = self.state == CircuitState::HalfOpen;
2445 self.record_failure();
2446 if self.should_open(config) {
2447 self.open();
2448 } else if was_half_open && !transient {
2449 self.open();
2453 }
2454 }
2455
2456 if self.state == CircuitState::Open && self.should_half_open(config) {
2459 self.half_open();
2460 }
2461
2462 self.state
2463 }
2464
2465 pub fn apply_command_outcome(
2489 &mut self,
2490 success: bool,
2491 config: &CircuitBreakerConfig,
2492 ) -> CircuitState {
2493 if success {
2494 self.consecutive_command_failures = 0;
2495 return self.apply_health_outcome(true, false, config);
2496 }
2497
2498 self.consecutive_command_failures = self.consecutive_command_failures.saturating_add(1);
2499 let was_half_open = self.state == CircuitState::HalfOpen;
2500 self.record_failure();
2501 let threshold = config.command_failure_threshold.max(1);
2502 if was_half_open
2503 || self.should_open(config)
2504 || (self.state == CircuitState::Closed
2505 && self.consecutive_command_failures >= threshold)
2506 {
2507 self.open();
2508 }
2509
2510 if self.state == CircuitState::Open && self.should_half_open(config) {
2511 self.half_open();
2512 }
2513
2514 self.state
2515 }
2516}
2517
2518#[derive(Debug, Clone, Serialize, Deserialize)]
2519pub struct CompilationConfig {
2520 #[serde(default = "default_confidence")]
2522 pub confidence_threshold: f64,
2523 #[serde(default = "default_min_local_time")]
2526 pub min_local_time_ms: u64,
2527 #[serde(default = "default_speedup_threshold")]
2531 pub remote_speedup_threshold: f64,
2532 #[serde(default)]
2538 pub local_slots: u32,
2539 #[serde(default = "default_build_slots")]
2541 pub build_slots: u32,
2542 #[serde(default = "default_test_slots")]
2545 pub test_slots: u32,
2546 #[serde(default = "default_check_slots")]
2549 pub check_slots: u32,
2550 #[serde(default)]
2556 pub disk_headroom_gib: u32,
2557 #[serde(default = "default_build_timeout")]
2560 pub build_timeout_sec: u64,
2561 #[serde(default = "default_test_timeout")]
2564 pub test_timeout_sec: u64,
2565 #[serde(default = "default_bun_timeout")]
2569 pub bun_timeout_sec: u64,
2570 #[serde(default = "default_external_timeout_enabled")]
2574 pub external_timeout_enabled: bool,
2575 #[serde(default = "default_allow_local_fallback")]
2588 pub allow_local_fallback: bool,
2589 #[serde(default)]
2600 pub remote_build_jobs: RemoteBuildJobs,
2601}
2602
2603#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
2615pub enum RemoteBuildJobs {
2616 #[default]
2618 Auto,
2619 Off,
2621 Fixed(u32),
2623}
2624
2625impl RemoteBuildJobs {
2626 pub fn parse(value: &str) -> Result<Self, String> {
2628 let trimmed = value.trim();
2629 match trimmed.to_ascii_lowercase().as_str() {
2630 "auto" => Ok(Self::Auto),
2631 "off" | "none" | "disabled" => Ok(Self::Off),
2632 _ => match trimmed.parse::<u32>() {
2633 Ok(0) => Err(
2634 "remote_build_jobs must be >= 1 (cargo rejects `-j 0`); use `off` to disable"
2635 .to_string(),
2636 ),
2637 Ok(n) => Ok(Self::Fixed(n)),
2638 Err(_) => Err(format!(
2639 "'{trimmed}' is not a valid remote_build_jobs value (expected `auto`, `off`, or a positive integer)"
2640 )),
2641 },
2642 }
2643 }
2644}
2645
2646impl std::fmt::Display for RemoteBuildJobs {
2647 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2648 match self {
2649 Self::Auto => f.write_str("auto"),
2650 Self::Off => f.write_str("off"),
2651 Self::Fixed(n) => write!(f, "{n}"),
2652 }
2653 }
2654}
2655
2656impl std::str::FromStr for RemoteBuildJobs {
2657 type Err = String;
2658
2659 fn from_str(value: &str) -> Result<Self, Self::Err> {
2660 Self::parse(value)
2661 }
2662}
2663
2664impl Serialize for RemoteBuildJobs {
2665 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
2666 match self {
2667 Self::Auto => serializer.serialize_str("auto"),
2668 Self::Off => serializer.serialize_str("off"),
2669 Self::Fixed(n) => serializer.serialize_u32(*n),
2670 }
2671 }
2672}
2673
2674impl<'de> Deserialize<'de> for RemoteBuildJobs {
2675 fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
2676 #[derive(Deserialize)]
2677 #[serde(untagged)]
2678 enum Raw {
2679 Count(u64),
2680 Spelling(String),
2681 }
2682 match Raw::deserialize(deserializer)? {
2683 Raw::Count(n) => {
2684 let n = u32::try_from(n).map_err(|_| {
2685 serde::de::Error::custom(format!("remote_build_jobs {n} is out of range"))
2686 })?;
2687 Self::parse(&n.to_string()).map_err(serde::de::Error::custom)
2688 }
2689 Raw::Spelling(s) => Self::parse(&s).map_err(serde::de::Error::custom),
2690 }
2691 }
2692}
2693
2694impl Default for CompilationConfig {
2695 fn default() -> Self {
2696 Self {
2697 confidence_threshold: 0.85,
2698 min_local_time_ms: 2000,
2699 remote_speedup_threshold: default_speedup_threshold(),
2700 local_slots: 0,
2701 build_slots: default_build_slots(),
2702 test_slots: default_test_slots(),
2703 check_slots: default_check_slots(),
2704 disk_headroom_gib: 0,
2705 build_timeout_sec: default_build_timeout(),
2706 test_timeout_sec: default_test_timeout(),
2707 bun_timeout_sec: default_bun_timeout(),
2708 external_timeout_enabled: default_external_timeout_enabled(),
2709 allow_local_fallback: default_allow_local_fallback(),
2710 remote_build_jobs: RemoteBuildJobs::default(),
2711 }
2712 }
2713}
2714
2715fn default_build_slots() -> u32 {
2716 4
2717}
2718
2719fn default_test_slots() -> u32 {
2720 8
2721}
2722
2723fn default_check_slots() -> u32 {
2724 2
2725}
2726
2727fn default_build_timeout() -> u64 {
2730 300
2731}
2732
2733fn default_test_timeout() -> u64 {
2736 1800
2737}
2738
2739fn default_bun_timeout() -> u64 {
2743 600
2744}
2745
2746fn default_external_timeout_enabled() -> bool {
2748 true
2749}
2750
2751fn default_allow_local_fallback() -> bool {
2755 true
2756}
2757
2758impl CompilationConfig {
2759 pub fn timeout_for_kind(&self, kind: Option<CompilationKind>) -> std::time::Duration {
2765 let secs = match kind {
2766 Some(CompilationKind::BunTest) | Some(CompilationKind::BunTypecheck) => {
2768 self.bun_timeout_sec
2769 }
2770 Some(CompilationKind::CargoTest) | Some(CompilationKind::CargoNextest) => {
2772 self.test_timeout_sec
2773 }
2774 Some(CompilationKind::NixBuild) => self.test_timeout_sec,
2778 Some(CompilationKind::GoTest) => self.test_timeout_sec,
2781 _ => self.build_timeout_sec,
2783 };
2784 std::time::Duration::from_secs(secs)
2785 }
2786
2787 pub fn external_timeout_enabled(&self) -> bool {
2789 self.external_timeout_enabled
2790 }
2791}
2792
2793pub const LAYER0_PACK_VERSION: &str = "1";
2799
2800#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
2809#[serde(default)]
2810pub struct Layer0Config {
2811 pub enabled: bool,
2813 pub release_lto_thin: bool,
2815 pub release_codegen_units_1: bool,
2818}
2819
2820impl Layer0Config {
2821 #[must_use]
2827 pub fn active_env(&self) -> Vec<(&'static str, &'static str)> {
2828 if !self.enabled {
2829 return Vec::new();
2830 }
2831 let mut env = Vec::new();
2832 if self.release_lto_thin {
2833 env.push(("CARGO_PROFILE_RELEASE_LTO", "thin"));
2834 }
2835 if self.release_codegen_units_1 {
2836 env.push(("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"));
2837 }
2838 env
2839 }
2840}
2841
2842#[derive(Debug, Clone, Serialize, Deserialize)]
2843pub struct TransferConfig {
2844 #[serde(default, skip_serializing_if = "Option::is_none")]
2847 pub retrieval_staging_dir: Option<PathBuf>,
2848 #[serde(default)]
2851 pub retrieval_min_free_bytes: u64,
2852 #[serde(default = "default_compression")]
2854 pub compression_level: u32,
2855 #[serde(default = "default_excludes")]
2857 pub exclude_patterns: Vec<String>,
2858 #[serde(default, skip_serializing_if = "Option::is_none")]
2862 pub ssh_server_alive_interval_secs: Option<u64>,
2863 #[serde(default, skip_serializing_if = "Option::is_none")]
2868 pub ssh_control_persist_secs: Option<u64>,
2869 #[serde(default = "default_remote_base")]
2880 pub remote_base: String,
2881 #[serde(default, skip_serializing_if = "Option::is_none")]
2887 pub sync_timeout_ms: Option<u64>,
2888 #[serde(default = "default_source_sync_silence_timeout_secs")]
2900 pub source_sync_silence_timeout_secs: u64,
2901 #[serde(default)]
2903 pub retry: RetryConfig,
2904 #[serde(default)]
2910 pub verify_artifacts: bool,
2911 #[serde(default = "default_verify_max_size")]
2916 pub verify_max_size_bytes: u64,
2917
2918 #[serde(default, skip_serializing_if = "Option::is_none")]
2927 pub max_transfer_mb: Option<u64>,
2928
2929 #[serde(default, skip_serializing_if = "Option::is_none")]
2935 pub max_transfer_time_ms: Option<u64>,
2936
2937 #[serde(default, skip_serializing_if = "Option::is_none")]
2942 pub bwlimit_kbps: Option<u64>,
2943
2944 #[serde(default, skip_serializing_if = "Option::is_none")]
2953 pub rsync_bin: Option<String>,
2954
2955 #[serde(default, skip_serializing_if = "Option::is_none")]
2960 pub estimated_bandwidth_bps: Option<u64>,
2961
2962 #[serde(default)]
2975 pub adaptive_compression: bool,
2976
2977 #[serde(default = "default_min_compression")]
2982 pub min_compression_level: u32,
2983
2984 #[serde(default = "default_max_compression")]
2989 pub max_compression_level: u32,
2990}
2991
2992impl Default for TransferConfig {
2993 fn default() -> Self {
2994 Self {
2995 retrieval_staging_dir: None,
2996 retrieval_min_free_bytes: 0,
2997 compression_level: 3,
2998 exclude_patterns: default_excludes(),
2999 ssh_server_alive_interval_secs: None,
3000 ssh_control_persist_secs: None,
3001 remote_base: default_remote_base(),
3002 sync_timeout_ms: None,
3003 source_sync_silence_timeout_secs: default_source_sync_silence_timeout_secs(),
3004 retry: RetryConfig::default(),
3005 verify_artifacts: false,
3006 verify_max_size_bytes: default_verify_max_size(),
3007 max_transfer_mb: None,
3009 max_transfer_time_ms: None,
3010 bwlimit_kbps: None,
3011 rsync_bin: None,
3012 estimated_bandwidth_bps: None,
3013 adaptive_compression: false,
3015 min_compression_level: default_min_compression(),
3016 max_compression_level: default_max_compression(),
3017 }
3018 }
3019}
3020
3021impl TransferConfig {
3022 pub const MIN_SYNC_TIMEOUT_MS: u64 = 1_000;
3024 pub const MAX_SYNC_TIMEOUT_MS: u64 = 3_600_000;
3026 pub const DEFAULT_SYNC_TIMEOUT_FLOOR_MS: u64 = 30_000;
3028 const DEFAULT_SYNC_BYTES_PER_SECOND: u64 = 1024 * 1024;
3029
3030 #[must_use]
3032 pub fn valid_sync_timeout_ms(value: u64) -> bool {
3033 (Self::MIN_SYNC_TIMEOUT_MS..=Self::MAX_SYNC_TIMEOUT_MS).contains(&value)
3034 }
3035
3036 #[must_use]
3043 pub fn sync_timeout_for_payload(&self, payload_bytes: u64) -> std::time::Duration {
3044 let timeout_ms = self
3045 .sync_timeout_ms
3046 .filter(|value| Self::valid_sync_timeout_ms(*value))
3047 .unwrap_or_else(|| {
3048 let transfer_seconds = payload_bytes
3049 .saturating_add(Self::DEFAULT_SYNC_BYTES_PER_SECOND - 1)
3050 / Self::DEFAULT_SYNC_BYTES_PER_SECOND;
3051 Self::DEFAULT_SYNC_TIMEOUT_FLOOR_MS
3052 .saturating_add(transfer_seconds.saturating_mul(1_000))
3053 .min(Self::MAX_SYNC_TIMEOUT_MS)
3054 });
3055 std::time::Duration::from_millis(timeout_ms)
3056 }
3057
3058 pub fn select_compression_level(&self, estimated_bytes: Option<u64>) -> u32 {
3070 if !self.adaptive_compression {
3071 return self.compression_level;
3072 }
3073
3074 let Some(bytes) = estimated_bytes else {
3075 return self.compression_level;
3077 };
3078
3079 const SMALL_THRESHOLD: u64 = 10_000_000; const LARGE_THRESHOLD: u64 = 200_000_000; let level = if bytes < SMALL_THRESHOLD {
3084 1 } else if bytes < LARGE_THRESHOLD {
3086 3 } else {
3088 7 };
3090
3091 level.clamp(self.min_compression_level, self.max_compression_level)
3093 }
3094}
3095
3096#[rustfmt::skip]
3110pub const BUILTIN_EXECUTION_BASES: &[&str] = &[
3111 "cargo",
3113 "rustc",
3114 "nextest",
3116 "gcc",
3118 "g++",
3119 "clang",
3120 "clang++",
3121 "cc",
3122 "c++",
3123 "make",
3125 "cmake",
3126 "ninja",
3127 "meson",
3128 "bun",
3130 "nix",
3132 "go",
3134 "tsc",
3136];
3137
3138fn default_execution_allowlist() -> Vec<String> {
3139 BUILTIN_EXECUTION_BASES
3140 .iter()
3141 .map(|base| (*base).to_string())
3142 .collect()
3143}
3144
3145#[derive(Debug, Clone, Serialize, Deserialize)]
3150pub struct ExecutionConfig {
3151 #[serde(default = "default_execution_allowlist")]
3160 pub allowlist: Vec<String>,
3161}
3162
3163impl Default for ExecutionConfig {
3164 fn default() -> Self {
3165 Self {
3166 allowlist: default_execution_allowlist(),
3167 }
3168 }
3169}
3170
3171impl ExecutionConfig {
3172 pub fn is_allowed(&self, command_base: &str) -> bool {
3174 self.allowlist
3175 .iter()
3176 .any(|allowed| allowed.eq_ignore_ascii_case(command_base))
3177 }
3178
3179 pub fn missing_builtin_bases(&self) -> Vec<&'static str> {
3185 BUILTIN_EXECUTION_BASES
3186 .iter()
3187 .copied()
3188 .filter(|base| !self.is_allowed(base))
3189 .collect()
3190 }
3191}
3192
3193fn default_retry_max_attempts() -> u32 {
3198 3
3199}
3200
3201fn default_retry_base_delay_ms() -> u64 {
3202 100
3203}
3204
3205fn default_retry_max_delay_ms() -> u64 {
3206 5000
3207}
3208
3209fn default_retry_jitter_factor() -> f64 {
3210 0.1
3211}
3212
3213fn default_retry_total_timeout_ms() -> u64 {
3214 30000
3215}
3216
3217#[derive(Debug, Clone, Serialize, Deserialize)]
3222pub struct RetryConfig {
3223 #[serde(default = "default_retry_max_attempts")]
3225 pub max_attempts: u32,
3226
3227 #[serde(default = "default_retry_base_delay_ms")]
3229 pub base_delay_ms: u64,
3230
3231 #[serde(default = "default_retry_max_delay_ms")]
3233 pub max_delay_ms: u64,
3234
3235 #[serde(default = "default_retry_jitter_factor")]
3238 pub jitter_factor: f64,
3239
3240 #[serde(default = "default_retry_total_timeout_ms")]
3243 pub total_timeout_ms: u64,
3244}
3245
3246impl Default for RetryConfig {
3247 fn default() -> Self {
3248 Self {
3249 max_attempts: default_retry_max_attempts(),
3250 base_delay_ms: default_retry_base_delay_ms(),
3251 max_delay_ms: default_retry_max_delay_ms(),
3252 jitter_factor: default_retry_jitter_factor(),
3253 total_timeout_ms: default_retry_total_timeout_ms(),
3254 }
3255 }
3256}
3257
3258impl RetryConfig {
3259 pub fn no_retry() -> Self {
3261 Self {
3262 max_attempts: 1,
3263 ..Default::default()
3264 }
3265 }
3266
3267 pub fn delay_for_attempt(&self, attempt: u32) -> std::time::Duration {
3271 if attempt == 0 {
3272 return std::time::Duration::ZERO;
3273 }
3274
3275 let base = self.base_delay_ms as f64;
3277 let delay = base * (2.0_f64.powi(attempt as i32 - 1));
3278 let capped = delay.min(self.max_delay_ms as f64);
3279
3280 let jittered = if self.jitter_factor > 0.0 {
3282 let jitter_range = capped * self.jitter_factor;
3283 let jitter = (rand::rng().random::<f64>() * 2.0 - 1.0) * jitter_range;
3284 (capped + jitter).max(0.0)
3285 } else {
3286 capped
3287 };
3288
3289 std::time::Duration::from_millis(jittered as u64)
3290 }
3291
3292 pub fn should_retry(&self, attempt: u32, elapsed: std::time::Duration) -> bool {
3294 if attempt >= self.max_attempts {
3295 return false;
3296 }
3297 elapsed.as_millis() < self.total_timeout_ms as u128
3298 }
3299}
3300
3301fn default_source_sync_silence_timeout_secs() -> u64 {
3306 120
3307}
3308
3309pub fn default_remote_base() -> String {
3318 "/data/tmp/rch".to_string()
3319}
3320
3321pub fn validate_remote_base(path: &str) -> Result<String, String> {
3324 let expanded = shellexpand::tilde(path).into_owned();
3326
3327 if !expanded.starts_with('/') {
3329 return Err(format!(
3330 "remote_base must be an absolute path, got: {}",
3331 path
3332 ));
3333 }
3334
3335 if expanded.contains("..") {
3337 return Err(format!(
3338 "remote_base must not contain path traversal (..): {}",
3339 path
3340 ));
3341 }
3342
3343 let normalized = expanded.trim_end_matches('/').to_string();
3345
3346 if normalized.is_empty() || normalized == "/" {
3348 return Err("remote_base cannot be the root directory (safety restriction)".to_string());
3349 }
3350
3351 let components: Vec<&str> = normalized.split('/').filter(|c| !c.is_empty()).collect();
3355 if components.len() < 2 {
3356 return Err(format!(
3357 "remote_base must be at least 2 levels deep (e.g. /tmp/rch), got: {}",
3358 normalized
3359 ));
3360 }
3361
3362 Ok(normalized)
3363}
3364
3365fn default_circuit_failure_threshold() -> u32 {
3366 3
3367}
3368
3369fn default_circuit_success_threshold() -> u32 {
3370 2
3371}
3372
3373fn default_circuit_error_rate_threshold() -> f64 {
3374 0.5
3375}
3376
3377fn default_circuit_window_secs() -> u64 {
3378 60
3379}
3380
3381fn default_circuit_open_cooldown_secs() -> u64 {
3382 30
3383}
3384
3385fn default_circuit_half_open_max_probes() -> u32 {
3386 1
3387}
3388
3389fn default_circuit_command_failure_threshold() -> u32 {
3390 2
3391}
3392
3393fn default_circuit_open_cooldown_max_secs() -> u64 {
3394 480
3396}
3397
3398fn default_true() -> bool {
3399 true
3400}
3401
3402fn default_log_level() -> String {
3403 "info".to_string()
3404}
3405
3406pub fn default_socket_path() -> String {
3407 if let Ok(runtime_dir) = std::env::var("XDG_RUNTIME_DIR")
3409 && !runtime_dir.trim().is_empty()
3410 {
3411 let path = PathBuf::from(runtime_dir).join("rch.sock");
3412 return path.to_string_lossy().to_string();
3413 }
3414
3415 if let Some(cache_dir) = dirs::cache_dir() {
3417 let rch_cache = cache_dir.join("rch");
3418 let _ = std::fs::create_dir_all(&rch_cache);
3419 return rch_cache.join("rch.sock").to_string_lossy().to_string();
3420 }
3421
3422 "/tmp/rch.sock".to_string()
3424}
3425
3426#[allow(dead_code)]
3427fn default_output_visibility() -> OutputVisibility {
3428 OutputVisibility::None
3429}
3430
3431fn default_confidence() -> f64 {
3432 0.85
3433}
3434
3435fn default_min_local_time() -> u64 {
3436 2000
3437}
3438
3439fn default_speedup_threshold() -> f64 {
3443 1.2
3444}
3445
3446fn default_compression() -> u32 {
3447 3
3448}
3449
3450fn default_verify_max_size() -> u64 {
3452 100 * 1024 * 1024 }
3454
3455fn default_min_compression() -> u32 {
3457 1
3458}
3459
3460fn default_max_compression() -> u32 {
3462 9
3463}
3464
3465fn default_excludes() -> Vec<String> {
3466 vec![
3467 "target/".to_string(),
3469 "*.rlib".to_string(),
3470 "*.rmeta".to_string(),
3471 ".git/".to_string(),
3474 "node_modules/".to_string(),
3476 ".bun/".to_string(),
3478 ".npm/".to_string(),
3480 ".pnpm-store/".to_string(),
3481 "dist/".to_string(),
3483 "build/".to_string(),
3484 ".next/".to_string(),
3486 ".nuxt/".to_string(),
3487 ".turbo/".to_string(),
3488 ".parcel-cache/".to_string(),
3489 ".beads/".to_string(),
3491 ".doctor/".to_string(),
3493 "coverage/".to_string(),
3495 ".nyc_output/".to_string(),
3496 ".cargo/credentials".to_string(),
3498 ".cargo/credentials.toml".to_string(),
3499 ".env".to_string(),
3500 ".env.*".to_string(),
3501 "*.pem".to_string(),
3502 "*.key".to_string(),
3503 "credentials.json".to_string(),
3504 "secrets.json".to_string(),
3505 "secrets.yaml".to_string(),
3506 "secrets.yml".to_string(),
3507 "secrets.toml".to_string(),
3508 "secrets.env".to_string(),
3509 "secrets.txt".to_string(),
3510 "secrets.conf".to_string(),
3511 "secrets.cfg".to_string(),
3512 "secrets.properties".to_string(),
3513 "secrets.xml".to_string(),
3514 "id_rsa".to_string(),
3528 "id_ed25519".to_string(),
3529 "id_ecdsa".to_string(),
3530 "id_dsa".to_string(),
3531 "*.p12".to_string(),
3533 "*.pfx".to_string(),
3534 "*.keystore".to_string(),
3535 "*.jks".to_string(),
3536 ".netrc".to_string(),
3539 "_netrc".to_string(),
3540 ".aws/credentials".to_string(),
3541 ".docker/config.json".to_string(),
3542 ".npmrc".to_string(),
3546 ".pypirc".to_string(),
3547 "secring.*".to_string(),
3549 ]
3550}
3551
3552#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
3558#[serde(rename_all = "snake_case")]
3559pub enum BuildLocation {
3560 #[default]
3562 Local,
3563 Remote,
3565}
3566
3567#[derive(Debug, Clone, Serialize, Deserialize)]
3569pub struct BuildCancellationWorkerHealth {
3570 pub status: String,
3572 pub speed_score: f64,
3574 pub used_slots: u32,
3576 pub available_slots: u32,
3578 pub pressure_state: String,
3580 pub pressure_reason_code: String,
3582}
3583
3584#[derive(Debug, Clone, Serialize, Deserialize)]
3586pub struct BuildCancellationMetadata {
3587 pub operation_id: String,
3589 pub origin: String,
3591 pub reason_code: String,
3593 #[serde(default)]
3595 pub decision_path: Vec<String>,
3596 pub escalation_stage: String,
3598 pub escalation_count: u32,
3600 pub remote_kill_attempted: bool,
3602 pub cleanup_ok: bool,
3604 pub history_cancelled: bool,
3606 pub final_state: String,
3608 #[serde(default, skip_serializing_if = "Option::is_none")]
3610 pub worker_health: Option<BuildCancellationWorkerHealth>,
3611}
3612
3613#[derive(Debug, Clone, Serialize, Deserialize)]
3615pub struct BuildRecord {
3616 pub id: u64,
3618 pub started_at: String,
3620 pub completed_at: String,
3622 pub project_id: String,
3624 pub worker_id: Option<String>,
3626 pub command: String,
3628 pub exit_code: i32,
3630 pub duration_ms: u64,
3632 pub location: BuildLocation,
3634 pub bytes_transferred: Option<u64>,
3636 #[serde(default)]
3638 pub timing: Option<CommandTimingBreakdown>,
3639 #[serde(default, skip_serializing_if = "Option::is_none")]
3641 pub cancellation: Option<BuildCancellationMetadata>,
3642}
3643
3644#[derive(Debug, Clone, Serialize, Deserialize)]
3646pub struct BuildRecordInput {
3647 pub started_at: String,
3649 pub completed_at: String,
3651 pub project_id: String,
3653 pub worker_id: Option<String>,
3655 pub command: String,
3657 pub exit_code: i32,
3659 pub duration_ms: u64,
3661 pub location: BuildLocation,
3663 pub bytes_transferred: Option<u64>,
3665 #[serde(default)]
3667 pub timing: Option<CommandTimingBreakdown>,
3668 #[serde(default, skip_serializing_if = "Option::is_none")]
3670 pub cancellation: Option<BuildCancellationMetadata>,
3671}
3672
3673impl BuildRecordInput {
3674 pub fn into_record(self, id: u64) -> BuildRecord {
3676 BuildRecord {
3677 id,
3678 started_at: self.started_at,
3679 completed_at: self.completed_at,
3680 project_id: self.project_id,
3681 worker_id: self.worker_id,
3682 command: self.command,
3683 exit_code: self.exit_code,
3684 duration_ms: self.duration_ms,
3685 location: self.location,
3686 bytes_transferred: self.bytes_transferred,
3687 timing: self.timing,
3688 cancellation: self.cancellation,
3689 }
3690 }
3691}
3692
3693#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3698pub struct BuildStats {
3699 pub total_builds: usize,
3701 pub success_count: usize,
3703 pub failure_count: usize,
3705 pub remote_count: usize,
3707 pub local_count: usize,
3709 pub avg_duration_ms: u64,
3711}
3712
3713#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3720pub struct SavedTimeStats {
3721 pub total_remote_duration_ms: u64,
3723 pub estimated_local_duration_ms: u64,
3726 pub time_saved_ms: u64,
3729 pub builds_counted: usize,
3731 pub avg_speedup: f64,
3735 pub today_saved_ms: u64,
3737 pub week_saved_ms: u64,
3739 #[serde(default)]
3742 pub estimate_basis: String,
3743 #[serde(default)]
3745 pub local_baseline_builds: usize,
3746}
3747
3748use std::collections::VecDeque;
3753use std::time::{Duration, Instant};
3754
3755#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3757pub struct CompilationTimingBreakdown {
3758 #[serde(with = "duration_millis")]
3760 pub rsync_up: Duration,
3761 #[serde(with = "duration_millis")]
3763 pub remote_build: Duration,
3764 #[serde(with = "duration_millis")]
3766 pub rsync_down: Duration,
3767 #[serde(with = "duration_millis")]
3769 pub total: Duration,
3770}
3771
3772#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3776pub struct CommandTimingBreakdown {
3777 #[serde(with = "option_duration_millis")]
3779 pub classify: Option<Duration>,
3780 #[serde(with = "option_duration_millis")]
3782 pub select: Option<Duration>,
3783 #[serde(with = "option_duration_millis")]
3785 pub sync_up: Option<Duration>,
3786 #[serde(with = "option_duration_millis")]
3788 pub exec: Option<Duration>,
3789 #[serde(with = "option_duration_millis")]
3791 pub sync_down: Option<Duration>,
3792 #[serde(with = "option_duration_millis")]
3794 pub cleanup: Option<Duration>,
3795 #[serde(with = "option_duration_millis")]
3797 pub total: Option<Duration>,
3798}
3799
3800mod duration_millis {
3801 use serde::{Deserialize, Deserializer, Serialize, Serializer};
3802 use std::time::Duration;
3803
3804 pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
3805 where
3806 S: Serializer,
3807 {
3808 duration.as_millis().serialize(serializer)
3809 }
3810
3811 pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
3812 where
3813 D: Deserializer<'de>,
3814 {
3815 let millis = u64::deserialize(deserializer)?;
3816 Ok(Duration::from_millis(millis))
3817 }
3818}
3819
3820#[derive(Debug, Clone, Serialize, Deserialize)]
3822pub struct CompilationMetrics {
3823 pub project_id: String,
3825 pub worker_id: String,
3827 pub timestamp: chrono::DateTime<chrono::Utc>,
3829 pub timing: CompilationTimingBreakdown,
3831 #[serde(with = "option_duration_millis")]
3833 pub local_build_time: Option<Duration>,
3834 pub speedup: Option<f64>,
3836 pub files_synced: u64,
3838 pub bytes_transferred: u64,
3840 pub exit_code: i32,
3842 pub success: bool,
3844}
3845
3846mod option_duration_millis {
3847 use serde::{Deserialize, Deserializer, Serialize, Serializer};
3848 use std::time::Duration;
3849
3850 pub fn serialize<S>(duration: &Option<Duration>, serializer: S) -> Result<S::Ok, S::Error>
3851 where
3852 S: Serializer,
3853 {
3854 match duration {
3855 Some(d) => Some(d.as_millis() as u64).serialize(serializer),
3856 None => Option::<u64>::None.serialize(serializer),
3857 }
3858 }
3859
3860 pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Duration>, D::Error>
3861 where
3862 D: Deserializer<'de>,
3863 {
3864 let opt: Option<u64> = Option::deserialize(deserializer)?;
3865 Ok(opt.map(Duration::from_millis))
3866 }
3867}
3868
3869impl CompilationMetrics {
3870 pub fn calculate_speedup(&mut self) {
3872 if let Some(local) = self.local_build_time
3873 && self.timing.total.as_millis() > 0
3874 {
3875 self.speedup = Some(local.as_secs_f64() / self.timing.total.as_secs_f64());
3876 }
3877 }
3878
3879 pub fn is_beneficial(&self) -> bool {
3881 self.speedup.map(|s| s > 1.0).unwrap_or(false)
3882 }
3883}
3884
3885impl Default for CompilationMetrics {
3886 fn default() -> Self {
3887 Self {
3888 project_id: String::new(),
3889 worker_id: String::new(),
3890 timestamp: chrono::Utc::now(),
3891 timing: CompilationTimingBreakdown::default(),
3892 local_build_time: None,
3893 speedup: None,
3894 files_synced: 0,
3895 bytes_transferred: 0,
3896 exit_code: 0,
3897 success: true,
3898 }
3899 }
3900}
3901
3902#[derive(Debug)]
3918pub struct CompilationTimer {
3919 project_id: String,
3920 worker_id: String,
3921 start: Instant,
3922 phase_start: Instant,
3923 rsync_up: Option<Duration>,
3924 remote_build: Option<Duration>,
3925 rsync_down: Option<Duration>,
3926}
3927
3928impl CompilationTimer {
3929 pub fn new(project_id: &str, worker_id: &str) -> Self {
3931 let now = Instant::now();
3932 Self {
3933 project_id: project_id.to_string(),
3934 worker_id: worker_id.to_string(),
3935 start: now,
3936 phase_start: now,
3937 rsync_up: None,
3938 remote_build: None,
3939 rsync_down: None,
3940 }
3941 }
3942
3943 pub fn end_rsync_up(&mut self) {
3945 self.rsync_up = Some(self.phase_start.elapsed());
3946 self.phase_start = Instant::now();
3947 tracing::info!(
3948 rsync_up_ms = %self.rsync_up.unwrap().as_millis(),
3949 "TIMING: rsync_up completed"
3950 );
3951 }
3952
3953 pub fn end_remote_build(&mut self) {
3955 self.remote_build = Some(self.phase_start.elapsed());
3956 self.phase_start = Instant::now();
3957 tracing::info!(
3958 remote_build_ms = %self.remote_build.unwrap().as_millis(),
3959 "TIMING: remote_build completed"
3960 );
3961 }
3962
3963 pub fn end_rsync_down(&mut self) {
3965 self.rsync_down = Some(self.phase_start.elapsed());
3966 tracing::info!(
3967 rsync_down_ms = %self.rsync_down.unwrap().as_millis(),
3968 "TIMING: rsync_down completed"
3969 );
3970 }
3971
3972 pub fn finish(self, exit_code: i32, files: u64, bytes: u64) -> CompilationMetrics {
3974 let total = self.start.elapsed();
3975 tracing::info!(
3976 total_ms = %total.as_millis(),
3977 exit_code = %exit_code,
3978 "TIMING: compilation completed"
3979 );
3980
3981 CompilationMetrics {
3982 project_id: self.project_id,
3983 worker_id: self.worker_id,
3984 timestamp: chrono::Utc::now(),
3985 timing: CompilationTimingBreakdown {
3986 rsync_up: self.rsync_up.unwrap_or_default(),
3987 remote_build: self.remote_build.unwrap_or_default(),
3988 rsync_down: self.rsync_down.unwrap_or_default(),
3989 total,
3990 },
3991 local_build_time: None,
3992 speedup: None,
3993 files_synced: files,
3994 bytes_transferred: bytes,
3995 exit_code,
3996 success: exit_code == 0,
3997 }
3998 }
3999}
4000
4001#[derive(Debug)]
4003pub struct MetricsAggregator {
4004 history: VecDeque<CompilationMetrics>,
4005 max_history: usize,
4006}
4007
4008impl MetricsAggregator {
4009 pub fn new(max_history: usize) -> Self {
4011 Self {
4012 history: VecDeque::with_capacity(max_history),
4013 max_history,
4014 }
4015 }
4016
4017 pub fn record(&mut self, metrics: CompilationMetrics) {
4019 if self.history.len() >= self.max_history {
4020 self.history.pop_front();
4021 }
4022 self.history.push_back(metrics);
4023 }
4024
4025 pub fn average_speedup(&self) -> Option<f64> {
4027 let speedups: Vec<f64> = self.history.iter().filter_map(|m| m.speedup).collect();
4028
4029 if speedups.is_empty() {
4030 None
4031 } else {
4032 Some(speedups.iter().sum::<f64>() / speedups.len() as f64)
4033 }
4034 }
4035
4036 pub fn p50_total_time(&self) -> Option<Duration> {
4038 self.percentile_total_time(0.50)
4039 }
4040
4041 pub fn p95_total_time(&self) -> Option<Duration> {
4043 self.percentile_total_time(0.95)
4044 }
4045
4046 pub fn p99_total_time(&self) -> Option<Duration> {
4048 self.percentile_total_time(0.99)
4049 }
4050
4051 fn percentile_total_time(&self, percentile: f64) -> Option<Duration> {
4056 let mut times: Vec<_> = self.history.iter().map(|m| m.timing.total).collect();
4057
4058 if times.is_empty() {
4059 return None;
4060 }
4061
4062 times.sort();
4063 let idx = (((times.len() - 1) as f64 * percentile).round() as usize).min(times.len() - 1);
4067 Some(times[idx])
4068 }
4069
4070 pub fn success_rate(&self) -> f64 {
4072 if self.history.is_empty() {
4073 return 100.0;
4074 }
4075 let successes = self.history.iter().filter(|m| m.success).count();
4076 (successes as f64 / self.history.len() as f64) * 100.0
4077 }
4078
4079 pub fn count(&self) -> usize {
4081 self.history.len()
4082 }
4083
4084 pub fn metrics(&self) -> &VecDeque<CompilationMetrics> {
4086 &self.history
4087 }
4088
4089 pub fn clear(&mut self) {
4091 self.history.clear();
4092 }
4093}
4094
4095impl Default for MetricsAggregator {
4096 fn default() -> Self {
4097 Self::new(1000)
4098 }
4099}
4100
4101#[cfg(test)]
4102mod tests {
4103 use super::*;
4104 use crate::test_guard;
4105
4106 #[test]
4107 fn test_declared_os_reads_reserved_tag() {
4108 assert_eq!(
4109 declared_os(&["rust".to_string(), "os:windows".to_string()]),
4110 Some("windows".to_string())
4111 );
4112 assert_eq!(
4113 declared_os(&["os:WINDOWS".to_string()]),
4114 Some("windows".to_string())
4115 );
4116 }
4117
4118 #[test]
4119 fn test_declared_os_absent_for_ordinary_tags() {
4120 assert_eq!(declared_os(&[]), None);
4123 assert_eq!(
4124 declared_os(&["rust".to_string(), "bun".to_string(), "go".to_string()]),
4125 None
4126 );
4127 assert_eq!(declared_os(&["os:".to_string()]), None);
4129 assert_eq!(declared_os(&["os: ".to_string()]), None);
4130 }
4131
4132 #[test]
4133 fn test_os_tag_roundtrips_through_declared_os() {
4134 for os in ["windows", "Darwin", " linux "] {
4135 let tag = os_tag(os);
4136 assert_eq!(
4137 declared_os(std::slice::from_ref(&tag)).as_deref(),
4138 Some(os.trim().to_ascii_lowercase().as_str())
4139 );
4140 }
4141 }
4142
4143 #[test]
4148 fn test_no_inline_secret_fields() {
4149 let endpoint = DoctorWebhookEndpoint {
4150 name: "ops".to_string(),
4151 url: "https://example.com/hook".to_string(),
4152 format: DoctorWebhookFormat::Pagerduty,
4153 on_transitions: vec!["any_to_failing".to_string()],
4154 retry_max: 3,
4155 retry_backoff_ms: 200,
4156 timeout_ms: 5000,
4157 enabled: true,
4158 routing_key_env: Some("RCH_PAGERDUTY_ROUTING_KEY".to_string()),
4159 bearer_token_env: Some("RCH_WEBHOOK_BEARER".to_string()),
4160 signing_secret_env: Some("RCH_WEBHOOK_HMAC".to_string()),
4161 };
4162 let value = serde_json::to_value(&endpoint).unwrap();
4165 let obj = value.as_object().expect("endpoint serializes to an object");
4166 for forbidden in ["secret", "routing_key", "bearer_token", "password", "token"] {
4167 assert!(
4168 !obj.contains_key(forbidden),
4169 "endpoint config must not expose an inline `{forbidden}` field"
4170 );
4171 }
4172 assert!(obj.contains_key("routing_key_env"));
4173 assert!(obj.contains_key("bearer_token_env"));
4174 assert!(obj.contains_key("signing_secret_env"));
4175 }
4176
4177 #[test]
4178 fn test_webhook_format_toml_spelling() {
4179 #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)]
4184 struct FormatWrap {
4185 format: DoctorWebhookFormat,
4186 }
4187 let rendered = toml::to_string(&FormatWrap {
4188 format: DoctorWebhookFormat::GenericJson,
4189 })
4190 .unwrap();
4191 assert!(
4192 rendered.contains("format = \"generic_json\""),
4193 "unexpected TOML: {rendered}"
4194 );
4195 let slack: FormatWrap = toml::from_str("format = \"slack\"").unwrap();
4196 assert_eq!(slack.format, DoctorWebhookFormat::Slack);
4197 let pd: FormatWrap = toml::from_str("format = \"pagerduty\"").unwrap();
4198 assert_eq!(pd.format, DoctorWebhookFormat::Pagerduty);
4199 }
4200
4201 #[test]
4202 fn test_doctor_webhooks_defaults() {
4203 let cfg = DoctorWebhooksConfig::default();
4204 assert!(cfg.endpoints.is_empty());
4205 assert_eq!(cfg.max_inflight, 16);
4206 let rch: RchConfig = toml::from_str("").unwrap();
4209 assert!(rch.doctor.webhooks.endpoints.is_empty());
4210 }
4211
4212 #[test]
4213 fn test_doctor_webhook_endpoint_parses_from_toml() {
4214 let toml_src = r#"
4215[doctor.webhooks]
4216max_inflight = 8
4217
4218[[doctor.webhooks.endpoints]]
4219name = "ops-slack"
4220url = "https://hooks.slack.com/services/T0/B0/x"
4221format = "slack"
4222on_transitions = ["healthy_to_degraded", "any_to_failing"]
4223retry_max = 2
4224"#;
4225 let cfg: RchConfig = toml::from_str(toml_src).unwrap();
4226 assert_eq!(cfg.doctor.webhooks.max_inflight, 8);
4227 assert_eq!(cfg.doctor.webhooks.endpoints.len(), 1);
4228 let ep = &cfg.doctor.webhooks.endpoints[0];
4229 assert_eq!(ep.name, "ops-slack");
4230 assert_eq!(ep.format, DoctorWebhookFormat::Slack);
4231 assert_eq!(ep.on_transitions.len(), 2);
4232 assert_eq!(ep.retry_max, 2);
4233 assert_eq!(ep.retry_backoff_ms, 200);
4235 assert_eq!(ep.timeout_ms, 5000);
4236 assert!(ep.enabled);
4237 }
4238
4239 fn assert_golden_json(
4240 actual: serde_json::Value,
4241 expected: &serde_json::Value,
4242 fixture_path: &str,
4243 ) {
4244 if &actual == expected {
4245 return;
4246 }
4247
4248 let actual_json = serde_json::to_string_pretty(&actual).expect("actual JSON renders");
4249 let expected_json = serde_json::to_string_pretty(expected).expect("expected JSON renders");
4250 assert_eq!(
4251 &actual,
4252 expected,
4253 "golden mismatch in {fixture_path}\n\
4254 expected_blake3={}\n\
4255 actual_blake3={}\n\
4256 bless path: review the semantic diff, update the fixture expected_* field, \
4257 and record both hashes in the Beads closeout.\n\
4258 expected:\n{expected_json}\n\
4259 actual:\n{actual_json}",
4260 blake3::hash(expected_json.as_bytes()),
4261 blake3::hash(actual_json.as_bytes())
4262 );
4263 }
4264
4265 #[test]
4266 fn test_circuit_state_default() {
4267 let _guard = test_guard!();
4268 assert_eq!(CircuitState::default(), CircuitState::Closed);
4269 }
4270
4271 #[test]
4272 fn test_circuit_config_defaults() {
4273 let _guard = test_guard!();
4274 let config = CircuitBreakerConfig::default();
4275 assert_eq!(config.failure_threshold, 3);
4276 assert_eq!(config.success_threshold, 2);
4277 assert_eq!(config.error_rate_threshold, 0.5);
4278 assert_eq!(config.window_secs, 60);
4279 assert_eq!(config.open_cooldown_secs, 30);
4280 assert_eq!(config.half_open_max_probes, 1);
4281 }
4282
4283 #[test]
4284 fn test_rch_config_has_circuit_defaults() {
4285 let _guard = test_guard!();
4286 let config = RchConfig::default();
4287 assert_eq!(config.circuit.failure_threshold, 3);
4288 }
4289
4290 #[test]
4291 fn test_circuit_config_serde_roundtrip() {
4292 let _guard = test_guard!();
4293 let config = CircuitBreakerConfig {
4294 failure_threshold: 5,
4295 success_threshold: 3,
4296 error_rate_threshold: 0.75,
4297 window_secs: 120,
4298 open_cooldown_secs: 45,
4299 half_open_max_probes: 2,
4300 command_failure_threshold: 4,
4301 open_cooldown_max_secs: 900,
4302 };
4303 let json = serde_json::to_string(&config).unwrap();
4304 let parsed: CircuitBreakerConfig = serde_json::from_str(&json).unwrap();
4305 assert_eq!(parsed.command_failure_threshold, 4);
4306 assert_eq!(parsed.open_cooldown_max_secs, 900);
4307 assert_eq!(parsed.failure_threshold, 5);
4308 assert_eq!(parsed.success_threshold, 3);
4309 assert_eq!(parsed.error_rate_threshold, 0.75);
4310 assert_eq!(parsed.window_secs, 120);
4311 assert_eq!(parsed.open_cooldown_secs, 45);
4312 assert_eq!(parsed.half_open_max_probes, 2);
4313 }
4314
4315 #[test]
4320 fn test_worker_capabilities_default() {
4321 let _guard = test_guard!();
4322 let caps = WorkerCapabilities::default();
4323 assert!(caps.rustc_version.is_none());
4324 assert!(caps.rustup_toolchains.is_empty());
4325 assert!(caps.rustup_components.is_empty());
4326 assert!(caps.bun_version.is_none());
4327 assert!(caps.node_version.is_none());
4328 assert!(caps.npm_version.is_none());
4329 assert!(caps.projects_root_ok.is_none());
4330 assert!(caps.projects_root_issue.is_none());
4331 assert!(caps.projects_root_checked_at_unix_ms.is_none());
4332 }
4333
4334 #[test]
4335 fn test_worker_capabilities_new() {
4336 let _guard = test_guard!();
4337 let caps = WorkerCapabilities::new();
4338 assert!(caps.rustc_version.is_none());
4339 assert!(!caps.has_rust());
4340 assert!(!caps.has_bun());
4341 assert!(!caps.has_node());
4342 assert!(caps.is_topology_healthy().is_none());
4343 }
4344
4345 #[test]
4346 fn test_worker_capabilities_has_rust() {
4347 let _guard = test_guard!();
4348 let mut caps = WorkerCapabilities::new();
4349 assert!(!caps.has_rust());
4350
4351 caps.rustc_version = Some("rustc 1.76.0 (07dca489a 2024-02-04)".to_string());
4352 assert!(caps.has_rust());
4353 }
4354
4355 #[test]
4356 fn test_worker_capabilities_components_are_toolchain_scoped_and_exact() {
4357 let _guard = test_guard!();
4358 let caps = WorkerCapabilities {
4359 rustup_components: vec![
4360 "stable-x86_64-unknown-linux-gnu:clippy".to_string(),
4361 "nightly-2026-07-05-x86_64-unknown-linux-gnu:rustfmt".to_string(),
4362 "beta-x86_64-unknown-linux-gnu:clippy-preview".to_string(),
4363 ],
4364 ..Default::default()
4365 };
4366
4367 assert!(caps.has_rustup_component("stable", "clippy"));
4368 assert!(caps.has_rustup_component("nightly-2026-07-05", "rustfmt"));
4369 assert!(!caps.has_rustup_component("nightly-2026-07-05", "clippy"));
4370 assert!(!caps.has_rustup_component("beta", "clippy"));
4371 assert!(!caps.has_rustup_component("nightly-2026-07-0", "rustfmt"));
4372 }
4373
4374 #[test]
4375 fn test_worker_capabilities_has_bun() {
4376 let _guard = test_guard!();
4377 let mut caps = WorkerCapabilities::new();
4378 assert!(!caps.has_bun());
4379
4380 caps.bun_version = Some("1.0.25".to_string());
4381 assert!(caps.has_bun());
4382 }
4383
4384 #[test]
4385 fn test_worker_capabilities_has_nix() {
4386 let _guard = test_guard!();
4387 let mut caps = WorkerCapabilities::new();
4388 assert!(!caps.has_nix());
4389
4390 caps.nix_version = Some("nix (Nix) 2.24.9".to_string());
4391 assert!(caps.has_nix());
4392 }
4393
4394 #[test]
4395 fn test_worker_capabilities_has_zig_requires_both_halves() {
4396 let _guard = test_guard!();
4397 let mut caps = WorkerCapabilities::new();
4398 assert!(!caps.has_zig());
4399
4400 caps.zig_version = Some("0.14.1".to_string());
4403 assert!(!caps.has_zig());
4404
4405 caps.zig_version = None;
4407 caps.cargo_zigbuild_version = Some("cargo-zigbuild 0.23.0".to_string());
4408 assert!(!caps.has_zig());
4409
4410 caps.zig_version = Some("0.14.1".to_string());
4411 assert!(caps.has_zig());
4412 }
4413
4414 #[test]
4415 fn test_nix_build_uses_generous_timeout() {
4416 let _guard = test_guard!();
4417 let cfg = CompilationConfig::default();
4418 assert_eq!(
4420 cfg.timeout_for_kind(Some(CompilationKind::NixBuild)),
4421 std::time::Duration::from_secs(cfg.test_timeout_sec)
4422 );
4423 }
4424
4425 #[test]
4426 fn test_job_uses_build_timeout_bucket() {
4427 let _guard = test_guard!();
4428 let cfg = CompilationConfig::default();
4431 assert_eq!(
4432 cfg.timeout_for_kind(Some(CompilationKind::Job)),
4433 std::time::Duration::from_secs(cfg.build_timeout_sec)
4434 );
4435 }
4436
4437 #[test]
4438 fn test_layer0_disabled_by_default_and_env_gated() {
4439 let _guard = test_guard!();
4440 let cfg = Layer0Config::default();
4441 assert!(!cfg.enabled);
4442 let knobs_on = Layer0Config {
4444 enabled: false,
4445 release_lto_thin: true,
4446 release_codegen_units_1: true,
4447 };
4448 assert!(cfg.active_env().is_empty());
4449 assert!(knobs_on.active_env().is_empty());
4450 }
4451
4452 #[test]
4453 fn test_layer0_knobs_emit_stable_env_pairs() {
4454 let _guard = test_guard!();
4455 let lto_only = Layer0Config {
4456 enabled: true,
4457 release_lto_thin: true,
4458 release_codegen_units_1: false,
4459 };
4460 assert_eq!(
4461 lto_only.active_env(),
4462 vec![("CARGO_PROFILE_RELEASE_LTO", "thin")]
4463 );
4464 let both = Layer0Config {
4465 enabled: true,
4466 release_lto_thin: true,
4467 release_codegen_units_1: true,
4468 };
4469 assert_eq!(
4470 both.active_env(),
4471 vec![
4472 ("CARGO_PROFILE_RELEASE_LTO", "thin"),
4473 ("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"),
4474 ],
4475 "emission order must stay stable for deterministic baselines"
4476 );
4477 }
4478
4479 #[test]
4480 fn test_layer0_pack_version_is_recorded() {
4481 let _guard = test_guard!();
4482 assert!(!LAYER0_PACK_VERSION.is_empty());
4484 }
4485 #[test]
4486 fn test_default_allowlist_includes_nix() {
4487 let _guard = test_guard!();
4488 let cfg = ExecutionConfig::default();
4489 assert!(cfg.is_allowed("nix"));
4490 }
4491
4492 #[test]
4493 fn test_worker_capabilities_has_node() {
4494 let _guard = test_guard!();
4495 let mut caps = WorkerCapabilities::new();
4496 assert!(!caps.has_node());
4497
4498 caps.node_version = Some("v20.11.0".to_string());
4499 assert!(caps.has_node());
4500 }
4501
4502 #[test]
4503 fn test_worker_capabilities_multiple_runtimes() {
4504 let _guard = test_guard!();
4505 let caps = WorkerCapabilities {
4506 rustc_version: Some("rustc 1.76.0".to_string()),
4507 bun_version: Some("1.0.25".to_string()),
4508 node_version: Some("v20.11.0".to_string()),
4509 npm_version: Some("10.2.4".to_string()),
4510 ..Default::default()
4511 };
4512
4513 assert!(caps.has_rust());
4514 assert!(caps.has_bun());
4515 assert!(caps.has_node());
4516 }
4517
4518 #[test]
4519 fn test_worker_capabilities_serialization_empty() {
4520 let _guard = test_guard!();
4521 let caps = WorkerCapabilities::new();
4522 let json = serde_json::to_string(&caps).unwrap();
4523 assert_eq!(json, "{}");
4525
4526 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4528 assert!(!parsed.has_rust());
4529 assert!(!parsed.has_bun());
4530 assert!(!parsed.has_node());
4531 }
4532
4533 #[test]
4534 fn test_worker_capabilities_serialization_with_versions() {
4535 let _guard = test_guard!();
4536 let caps = WorkerCapabilities {
4537 rustc_version: Some("rustc 1.76.0-nightly".to_string()),
4538 bun_version: Some("1.0.25".to_string()),
4539 node_version: None,
4540 npm_version: None,
4541 ..Default::default()
4542 };
4543
4544 let json = serde_json::to_string(&caps).unwrap();
4545 assert!(json.contains("rustc_version"));
4546 assert!(json.contains("rustc 1.76.0-nightly"));
4547 assert!(json.contains("bun_version"));
4548 assert!(json.contains("1.0.25"));
4549 assert!(!json.contains("node_version"));
4551 assert!(!json.contains("npm_version"));
4552
4553 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4554 assert!(parsed.has_rust());
4555 assert!(parsed.has_bun());
4556 assert!(!parsed.has_node());
4557 }
4558
4559 #[test]
4560 fn test_worker_capabilities_deserialization_partial() {
4561 let _guard = test_guard!();
4562 let json = r#"{"bun_version": "1.0.0"}"#;
4564 let caps: WorkerCapabilities = serde_json::from_str(json).unwrap();
4565
4566 assert!(!caps.has_rust());
4567 assert!(caps.has_bun());
4568 assert!(!caps.has_node());
4569 assert_eq!(caps.bun_version, Some("1.0.0".to_string()));
4570 }
4571
4572 #[test]
4573 fn test_worker_capabilities_clone() {
4574 let _guard = test_guard!();
4575 let caps = WorkerCapabilities {
4576 rustc_version: Some("1.76.0".to_string()),
4577 bun_version: None,
4578 node_version: Some("v20".to_string()),
4579 npm_version: None,
4580 projects_root_ok: Some(false),
4581 projects_root_issue: Some("alias_missing".to_string()),
4582 projects_root_checked_at_unix_ms: Some(123),
4583 ..Default::default()
4584 };
4585
4586 let cloned = caps.clone();
4587 assert_eq!(cloned.rustc_version, caps.rustc_version);
4588 assert_eq!(cloned.bun_version, caps.bun_version);
4589 assert_eq!(cloned.node_version, caps.node_version);
4590 assert_eq!(cloned.npm_version, caps.npm_version);
4591 assert_eq!(cloned.projects_root_ok, caps.projects_root_ok);
4592 assert_eq!(cloned.projects_root_issue, caps.projects_root_issue);
4593 assert_eq!(
4594 cloned.projects_root_checked_at_unix_ms,
4595 caps.projects_root_checked_at_unix_ms
4596 );
4597 }
4598
4599 #[test]
4600 fn test_worker_capabilities_topology_health_helper() {
4601 let _guard = test_guard!();
4602 let mut caps = WorkerCapabilities::new();
4603 assert_eq!(caps.is_topology_healthy(), None);
4604
4605 caps.projects_root_ok = Some(true);
4606 assert_eq!(caps.is_topology_healthy(), Some(true));
4607
4608 caps.projects_root_ok = Some(false);
4609 assert_eq!(caps.is_topology_healthy(), Some(false));
4610 }
4611
4612 #[test]
4613 fn test_selection_reason_serialization() {
4614 let _guard = test_guard!();
4615 assert_eq!(
4617 serde_json::to_string(&SelectionReason::Success).unwrap(),
4618 "\"success\""
4619 );
4620 assert_eq!(
4621 serde_json::to_string(&SelectionReason::NoWorkersConfigured).unwrap(),
4622 "\"no_workers_configured\""
4623 );
4624 assert_eq!(
4625 serde_json::to_string(&SelectionReason::AllWorkersUnreachable).unwrap(),
4626 "\"all_workers_unreachable\""
4627 );
4628 assert_eq!(
4629 serde_json::to_string(&SelectionReason::AllCircuitsOpen).unwrap(),
4630 "\"all_circuits_open\""
4631 );
4632 assert_eq!(
4633 serde_json::to_string(&SelectionReason::AllWorkersBusy).unwrap(),
4634 "\"all_workers_busy\""
4635 );
4636 assert_eq!(
4637 serde_json::to_string(&SelectionReason::NoWorkersPassedHealth).unwrap(),
4638 "\"no_workers_passed_health\""
4639 );
4640 assert_eq!(
4641 serde_json::to_string(&SelectionReason::AllWorkersFailedPreflight).unwrap(),
4642 "\"all_workers_failed_preflight\""
4643 );
4644 assert_eq!(
4645 serde_json::to_string(&SelectionReason::AllWorkersFailedConvergence).unwrap(),
4646 "\"all_workers_failed_convergence\""
4647 );
4648 assert_eq!(
4649 serde_json::to_string(&SelectionReason::NoAdmissibleWorkers(
4650 "critical_pressure=1,insufficient_slots=1".to_string()
4651 ))
4652 .unwrap(),
4653 "{\"no_admissible_workers\":\"critical_pressure=1,insufficient_slots=1\"}"
4654 );
4655 assert_eq!(
4656 serde_json::to_string(&SelectionReason::NoMatchingWorkers).unwrap(),
4657 "\"no_matching_workers\""
4658 );
4659 }
4660
4661 #[test]
4662 fn test_selection_reason_with_error() {
4663 let _guard = test_guard!();
4664 let reason = SelectionReason::SelectionError("test error".to_string());
4665 let json = serde_json::to_string(&reason).unwrap();
4666 assert!(json.contains("selection_error"));
4667 assert!(json.contains("test error"));
4668 }
4669
4670 #[test]
4671 fn test_selection_reason_deserialization() {
4672 let _guard = test_guard!();
4673 assert_eq!(
4674 serde_json::from_str::<SelectionReason>("\"success\"").unwrap(),
4675 SelectionReason::Success
4676 );
4677 assert_eq!(
4678 serde_json::from_str::<SelectionReason>("\"all_workers_busy\"").unwrap(),
4679 SelectionReason::AllWorkersBusy
4680 );
4681 assert_eq!(
4682 serde_json::from_str::<SelectionReason>("\"no_workers_passed_health\"").unwrap(),
4683 SelectionReason::NoWorkersPassedHealth
4684 );
4685 assert_eq!(
4686 serde_json::from_str::<SelectionReason>("\"all_workers_failed_preflight\"").unwrap(),
4687 SelectionReason::AllWorkersFailedPreflight
4688 );
4689 assert_eq!(
4690 serde_json::from_str::<SelectionReason>("\"all_workers_failed_convergence\"").unwrap(),
4691 SelectionReason::AllWorkersFailedConvergence
4692 );
4693 assert_eq!(
4694 serde_json::from_str::<SelectionReason>(
4695 "{\"no_admissible_workers\":\"critical_pressure=1\"}"
4696 )
4697 .unwrap(),
4698 SelectionReason::NoAdmissibleWorkers("critical_pressure=1".to_string())
4699 );
4700 }
4701
4702 #[test]
4703 fn test_selection_reason_display() {
4704 let _guard = test_guard!();
4705 assert_eq!(
4706 SelectionReason::Success.to_string(),
4707 "worker assigned successfully"
4708 );
4709 assert_eq!(
4710 SelectionReason::NoWorkersConfigured.to_string(),
4711 "no workers configured"
4712 );
4713 assert_eq!(
4714 SelectionReason::AllWorkersUnreachable.to_string(),
4715 "all workers unreachable"
4716 );
4717 assert_eq!(
4718 SelectionReason::AllWorkersBusy.to_string(),
4719 "all workers at capacity"
4720 );
4721 assert_eq!(
4722 SelectionReason::NoWorkersPassedHealth.to_string(),
4723 "no workers passed health thresholds"
4724 );
4725 assert_eq!(
4726 SelectionReason::AllWorkersFailedPreflight.to_string(),
4727 "all workers failed preflight checks"
4728 );
4729 assert_eq!(
4730 SelectionReason::AllWorkersFailedConvergence.to_string(),
4731 "all workers failed repo convergence checks"
4732 );
4733 assert_eq!(
4734 SelectionReason::NoAdmissibleWorkers(
4735 "critical_pressure=1,insufficient_slots=1".to_string()
4736 )
4737 .to_string(),
4738 "no admissible workers: critical_pressure=1,insufficient_slots=1"
4739 );
4740 assert_eq!(
4741 SelectionReason::SelectionError("oops".to_string()).to_string(),
4742 "selection error: oops"
4743 );
4744 }
4745
4746 #[test]
4747 fn test_selection_response_with_worker() {
4748 let _guard = test_guard!();
4749 let response = SelectionResponse {
4750 worker: Some(SelectedWorker {
4751 id: WorkerId::new("test"),
4752 host: "localhost".to_string(),
4753 user: "user".to_string(),
4754 identity_file: "~/.ssh/id_rsa".to_string(),
4755 slots_available: 8,
4756 speed_score: 75.0,
4757 declared_os: None,
4758 }),
4759 reason: SelectionReason::Success,
4760 build_id: None,
4761 diagnostics: None,
4762 };
4763
4764 let json = serde_json::to_string(&response).unwrap();
4765 assert!(json.contains("\"reason\":\"success\""));
4766 assert!(json.contains("\"id\":\"test\""));
4767 }
4768
4769 #[test]
4770 fn test_selection_response_without_worker() {
4771 let _guard = test_guard!();
4772 let response = SelectionResponse {
4773 worker: None,
4774 reason: SelectionReason::AllWorkersBusy,
4775 build_id: None,
4776 diagnostics: None,
4777 };
4778
4779 let json = serde_json::to_string(&response).unwrap();
4780 assert!(json.contains("\"worker\":null"));
4781 assert!(json.contains("\"reason\":\"all_workers_busy\""));
4782 }
4783
4784 #[test]
4785 fn golden_selection_response_with_no_workers_passed_health_diagnostics() {
4786 let _guard = test_guard!();
4787 const FIXTURE_PATH: &str =
4788 "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json";
4789 let fixture: serde_json::Value = serde_json::from_str(include_str!(
4790 "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json"
4791 ))
4792 .expect("selection diagnostics golden fixture parses");
4793 assert_eq!(
4794 fixture["schema_version"],
4795 "rch.golden.selection_diagnostics.v1"
4796 );
4797
4798 let response = SelectionResponse {
4799 worker: None,
4800 reason: SelectionReason::NoWorkersPassedHealth,
4801 build_id: None,
4802 diagnostics: Some(SelectionDiagnostics {
4803 required_runtime: RequiredRuntime::Rust,
4804 estimated_cores: 2,
4805 min_success_rate: 0.8,
4806 fallback_min_success_rate: 0.5,
4807 active_project_exclusion_count: 0,
4808 workers: vec![WorkerSelectionDiagnostic {
4809 worker_id: WorkerId::new("fixture-worker"),
4810 status: "healthy".to_string(),
4811 circuit_state: "closed".to_string(),
4812 pressure_state: "healthy".to_string(),
4813 pressure_reason_code: "pressure.ok".to_string(),
4814 success_rate: Some(0.2),
4815 min_success_rate: 0.8,
4816 fallback_min_success_rate: 0.5,
4817 required_runtime: RequiredRuntime::Rust,
4818 runtime_available: true,
4819 available_slots: 8,
4820 total_slots: 8,
4821 estimated_cores: 2,
4822 active_project_excluded: false,
4823 final_decision: WorkerSelectionDiagnosticDecision::Deny,
4824 final_reason: "success_rate 0.20 < fallback 0.50".to_string(),
4825 reason_codes: vec![
4826 "health.below_min_success_rate".to_string(),
4827 "health.below_fallback_min_success_rate".to_string(),
4828 ],
4829 }],
4830 }),
4831 };
4832
4833 assert_golden_json(
4834 serde_json::to_value(&response).expect("response serializes"),
4835 &fixture["expected_response"],
4836 FIXTURE_PATH,
4837 );
4838 }
4839
4840 #[test]
4841 fn test_selection_response_roundtrip() {
4842 let _guard = test_guard!();
4843 let original = SelectionResponse {
4844 worker: Some(SelectedWorker {
4845 id: WorkerId::new("worker-1"),
4846 host: "192.168.1.100".to_string(),
4847 user: "ubuntu".to_string(),
4848 identity_file: "/path/to/key".to_string(),
4849 slots_available: 16,
4850 speed_score: 90.5,
4851 declared_os: None,
4852 }),
4853 reason: SelectionReason::Success,
4854 build_id: None,
4855 diagnostics: None,
4856 };
4857
4858 let json = serde_json::to_string(&original).unwrap();
4859 let parsed: SelectionResponse = serde_json::from_str(&json).unwrap();
4860
4861 assert!(parsed.worker.is_some());
4862 let worker = parsed.worker.unwrap();
4863 assert_eq!(worker.id.as_str(), "worker-1");
4864 assert_eq!(worker.host, "192.168.1.100");
4865 assert_eq!(worker.slots_available, 16);
4866 assert_eq!(parsed.reason, SelectionReason::Success);
4867 }
4868
4869 #[test]
4870 fn test_build_heartbeat_phase_roundtrip() {
4871 let _guard = test_guard!();
4872 let json = serde_json::to_string(&BuildHeartbeatPhase::SyncUp).unwrap();
4873 assert_eq!(json, "\"sync_up\"");
4874 let parsed: BuildHeartbeatPhase = serde_json::from_str(&json).unwrap();
4875 assert_eq!(parsed, BuildHeartbeatPhase::SyncUp);
4876 }
4877
4878 #[test]
4879 fn test_build_heartbeat_request_roundtrip() {
4880 let _guard = test_guard!();
4881 let request = BuildHeartbeatRequest {
4882 build_id: 42,
4883 worker_id: WorkerId::new("worker-a"),
4884 hook_pid: Some(12345),
4885 local_wrapper_id: Some("wrapper-abc123".to_string()),
4886 remote_pgid_file: Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string()),
4887 phase: BuildHeartbeatPhase::Execute,
4888 detail: Some("Compiling crates".to_string()),
4889 progress_counter: Some(7),
4890 progress_percent: Some(42.5),
4891 };
4892
4893 let json = serde_json::to_string(&request).unwrap();
4894 let parsed: BuildHeartbeatRequest = serde_json::from_str(&json).unwrap();
4895 assert_eq!(parsed.build_id, 42);
4896 assert_eq!(parsed.worker_id.as_str(), "worker-a");
4897 assert_eq!(
4898 parsed.remote_pgid_file,
4899 Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string())
4900 );
4901 assert_eq!(parsed.phase, BuildHeartbeatPhase::Execute);
4902 assert_eq!(parsed.progress_counter, Some(7));
4903 assert_eq!(parsed.progress_percent, Some(42.5));
4904 }
4905
4906 #[test]
4909 fn test_circuit_stats_new() {
4910 let _guard = test_guard!();
4911 let stats = CircuitStats::new();
4912 assert_eq!(stats.state(), CircuitState::Closed);
4913 assert_eq!(stats.consecutive_failures(), 0);
4914 assert_eq!(stats.error_rate(), 0.0);
4915 assert!(stats.opened_at().is_none());
4916 }
4917
4918 #[test]
4919 fn test_circuit_stats_record_success() {
4920 let _guard = test_guard!();
4921 let mut stats = CircuitStats::new();
4922 stats.record_success();
4923 stats.record_success();
4924 assert_eq!(stats.error_rate(), 0.0);
4925 assert_eq!(stats.consecutive_failures(), 0);
4926 }
4927
4928 #[test]
4929 fn test_circuit_stats_record_failure() {
4930 let _guard = test_guard!();
4931 let mut stats = CircuitStats::new();
4932 stats.record_failure();
4933 stats.record_failure();
4934 assert_eq!(stats.consecutive_failures(), 2);
4935 assert_eq!(stats.error_rate(), 1.0); }
4937
4938 #[test]
4939 fn test_circuit_stats_error_rate() {
4940 let _guard = test_guard!();
4941 let mut stats = CircuitStats::new();
4942 stats.record_success();
4943 stats.record_success();
4944 stats.record_failure();
4945 assert!((stats.error_rate() - 0.333).abs() < 0.01);
4947 }
4948
4949 #[test]
4950 fn test_circuit_stats_should_open_consecutive_failures() {
4951 let _guard = test_guard!();
4952 let mut stats = CircuitStats::new();
4953 let config = CircuitBreakerConfig::default(); stats.record_failure();
4956 assert!(!stats.should_open(&config));
4957
4958 stats.record_failure();
4959 assert!(!stats.should_open(&config));
4960
4961 stats.record_failure();
4962 assert!(stats.should_open(&config)); }
4964
4965 #[test]
4966 fn test_circuit_stats_should_open_error_rate() {
4967 let _guard = test_guard!();
4968 let mut stats = CircuitStats::new();
4969 let config = CircuitBreakerConfig {
4970 error_rate_threshold: 0.5,
4971 ..Default::default()
4972 };
4973
4974 stats.record_success();
4976 stats.record_success();
4977 stats.record_failure();
4978 stats.record_failure();
4979 assert!(!stats.should_open(&config)); stats.record_failure(); assert!(stats.should_open(&config));
4983 }
4984
4985 #[test]
4986 fn test_circuit_close_clears_recent_results_no_flap() {
4987 let _guard = test_guard!();
4988 let mut stats = CircuitStats::new();
4992 let config = CircuitBreakerConfig {
4993 error_rate_threshold: 0.5,
4994 ..Default::default()
4995 };
4996 for _ in 0..6 {
4997 stats.record_failure();
4998 }
4999 assert!(stats.should_open(&config));
5000 stats.open();
5001 stats.half_open();
5003 stats.record_success();
5004 stats.record_success();
5005 stats.close();
5006 assert_eq!(stats.error_rate(), 0.0);
5008 assert!(!stats.should_open(&config));
5009 }
5010
5011 #[test]
5012 fn test_apply_health_outcome_opens_on_failure_threshold() {
5013 let _guard = test_guard!();
5014 let config = CircuitBreakerConfig {
5016 failure_threshold: 3,
5017 open_cooldown_secs: 300,
5018 ..Default::default()
5019 };
5020 let mut stats = CircuitStats::new();
5021 assert_eq!(
5022 stats.apply_health_outcome(false, false, &config),
5023 CircuitState::Closed
5024 );
5025 assert_eq!(
5026 stats.apply_health_outcome(false, false, &config),
5027 CircuitState::Closed
5028 );
5029 assert_eq!(
5030 stats.apply_health_outcome(false, false, &config),
5031 CircuitState::Open
5032 );
5033 }
5034
5035 #[test]
5036 fn test_apply_health_outcome_half_open_two_probe_recovery() {
5037 let _guard = test_guard!();
5038 let config = CircuitBreakerConfig {
5044 failure_threshold: 3,
5045 success_threshold: 2,
5046 open_cooldown_secs: 0,
5047 ..Default::default()
5048 };
5049 let mut stats = CircuitStats::new();
5050 for _ in 0..3 {
5051 stats.apply_health_outcome(false, false, &config);
5052 }
5053 assert_eq!(stats.state(), CircuitState::HalfOpen);
5055
5056 let after_first = stats.apply_health_outcome(true, false, &config);
5058 assert_eq!(after_first, CircuitState::HalfOpen);
5059
5060 let after_second = stats.apply_health_outcome(true, false, &config);
5062 assert_eq!(after_second, CircuitState::Closed);
5063 }
5064
5065 #[test]
5066 fn test_apply_health_outcome_transient_half_open_does_not_reopen() {
5067 let _guard = test_guard!();
5068 let config = CircuitBreakerConfig {
5071 failure_threshold: 3,
5072 success_threshold: 2,
5073 open_cooldown_secs: 0,
5074 ..Default::default()
5075 };
5076 let mut stats = CircuitStats::new();
5077 for _ in 0..3 {
5078 stats.apply_health_outcome(false, false, &config);
5079 }
5080 assert_eq!(stats.state(), CircuitState::HalfOpen);
5082
5083 assert_eq!(
5085 stats.apply_health_outcome(true, false, &config),
5086 CircuitState::HalfOpen
5087 );
5088
5089 let after_transient = stats.apply_health_outcome(false, true, &config);
5095 assert_eq!(after_transient, CircuitState::HalfOpen);
5096
5097 assert_eq!(
5099 stats.apply_health_outcome(true, false, &config),
5100 CircuitState::HalfOpen
5101 );
5102 assert_eq!(
5103 stats.apply_health_outcome(true, false, &config),
5104 CircuitState::Closed
5105 );
5106 }
5107
5108 #[test]
5109 fn test_apply_health_outcome_real_vs_transient_half_open_failure() {
5110 let _guard = test_guard!();
5111 let config = CircuitBreakerConfig {
5119 failure_threshold: 3,
5120 success_threshold: 2,
5121 open_cooldown_secs: 300,
5122 ..Default::default()
5123 };
5124
5125 let mut real = CircuitStats::new();
5127 real.open();
5128 real.half_open();
5129 assert_eq!(real.state(), CircuitState::HalfOpen);
5130 let after_real = real.apply_health_outcome(false, false, &config);
5131 assert_eq!(
5132 after_real,
5133 CircuitState::Open,
5134 "a real half-open failure must reopen the circuit"
5135 );
5136
5137 let mut transient = CircuitStats::new();
5139 transient.open();
5140 transient.half_open();
5141 assert_eq!(transient.state(), CircuitState::HalfOpen);
5142 let after_transient = transient.apply_health_outcome(false, true, &config);
5143 assert_eq!(
5144 after_transient,
5145 CircuitState::HalfOpen,
5146 "a transient half-open failure must NOT reopen the circuit"
5147 );
5148 }
5149
5150 #[test]
5155 fn test_command_failure_blocks_half_open_close_on_liveness_successes() {
5156 let _guard = test_guard!();
5157 let config = CircuitBreakerConfig {
5158 failure_threshold: 3,
5159 success_threshold: 2,
5160 open_cooldown_secs: 0,
5161 command_failure_threshold: 2,
5162 ..Default::default()
5163 };
5164 let mut stats = CircuitStats::new();
5165 stats.open();
5166 stats.half_open();
5167
5168 assert_eq!(
5170 stats.apply_command_outcome(false, &config),
5171 CircuitState::HalfOpen,
5172 "cooldown 0: reopen then immediately half-open again"
5173 );
5174 assert_eq!(stats.consecutive_command_failures(), 1);
5175
5176 for _ in 0..10 {
5178 assert_eq!(
5179 stats.apply_health_outcome(true, false, &config),
5180 CircuitState::HalfOpen,
5181 "liveness successes must not close the circuit while a command failure is outstanding"
5182 );
5183 }
5184
5185 assert_eq!(
5187 stats.apply_command_outcome(true, &config),
5188 CircuitState::Closed
5189 );
5190 assert_eq!(stats.consecutive_command_failures(), 0);
5191 assert_eq!(stats.reopen_streak(), 0);
5192 }
5193
5194 #[test]
5197 fn test_command_failures_open_closed_circuit_despite_liveness_successes() {
5198 let _guard = test_guard!();
5199 let config = CircuitBreakerConfig {
5200 failure_threshold: 3,
5201 command_failure_threshold: 2,
5202 open_cooldown_secs: 300,
5203 ..Default::default()
5204 };
5205 let mut stats = CircuitStats::new();
5206
5207 assert_eq!(
5208 stats.apply_command_outcome(false, &config),
5209 CircuitState::Closed
5210 );
5211 stats.apply_health_outcome(true, false, &config);
5213 stats.apply_health_outcome(true, false, &config);
5214 assert_eq!(stats.consecutive_failures(), 0);
5215 assert_eq!(
5217 stats.apply_command_outcome(false, &config),
5218 CircuitState::Open
5219 );
5220 assert_eq!(stats.reopen_streak(), 1);
5221 }
5222
5223 #[test]
5227 fn test_command_failure_in_half_open_always_reopens() {
5228 let _guard = test_guard!();
5229 let config = CircuitBreakerConfig {
5230 open_cooldown_secs: 300,
5231 ..Default::default()
5232 };
5233 let mut stats = CircuitStats::new();
5234 stats.open();
5235 stats.half_open();
5236 assert_eq!(
5237 stats.apply_command_outcome(false, &config),
5238 CircuitState::Open
5239 );
5240 assert_eq!(stats.reopen_streak(), 2);
5241 }
5242
5243 #[test]
5247 fn test_reopen_streak_backs_off_cooldown_until_cap() {
5248 let _guard = test_guard!();
5249 let config = CircuitBreakerConfig {
5250 open_cooldown_secs: 30,
5251 open_cooldown_max_secs: 100,
5252 ..Default::default()
5253 };
5254 assert_eq!(config.cooldown_ms_for_streak(0), 30_000);
5255 assert_eq!(config.cooldown_ms_for_streak(1), 30_000);
5256 assert_eq!(config.cooldown_ms_for_streak(2), 60_000);
5257 assert_eq!(config.cooldown_ms_for_streak(3), 100_000, "capped");
5258 assert_eq!(config.cooldown_ms_for_streak(40), 100_000, "no overflow");
5259
5260 let inverted = CircuitBreakerConfig {
5262 open_cooldown_secs: 30,
5263 open_cooldown_max_secs: 5,
5264 ..Default::default()
5265 };
5266 assert_eq!(inverted.cooldown_ms_for_streak(3), 30_000);
5267
5268 let mut stats = CircuitStats::new();
5271 stats.open();
5272 stats.half_open();
5273 stats.open();
5274 assert_eq!(stats.reopen_streak(), 2);
5275 let remaining = stats.recovery_remaining_secs(&config).unwrap();
5276 assert!(
5277 remaining > 30 && remaining <= 60,
5278 "second open should wait ~60s, got {remaining}"
5279 );
5280
5281 stats.close();
5283 assert_eq!(stats.reopen_streak(), 0);
5284 stats.open();
5285 assert!(stats.recovery_remaining_secs(&config).unwrap() <= 30);
5286 }
5287
5288 #[test]
5289 fn test_circuit_stats_success_resets_consecutive_failures() {
5290 let _guard = test_guard!();
5291 let mut stats = CircuitStats::new();
5292 stats.record_failure();
5293 stats.record_failure();
5294 assert_eq!(stats.consecutive_failures(), 2);
5295
5296 stats.record_success();
5297 assert_eq!(stats.consecutive_failures(), 0);
5298 }
5299
5300 #[test]
5301 fn test_circuit_stats_open_transition() {
5302 let _guard = test_guard!();
5303 let mut stats = CircuitStats::new();
5304 stats.open();
5305
5306 assert_eq!(stats.state(), CircuitState::Open);
5307 assert!(stats.opened_at().is_some());
5308 }
5309
5310 #[test]
5311 fn test_circuit_stats_half_open_transition() {
5312 let _guard = test_guard!();
5313 let mut stats = CircuitStats::new();
5314 stats.open();
5315 stats.half_open();
5316
5317 assert_eq!(stats.state(), CircuitState::HalfOpen);
5318 }
5319
5320 #[test]
5321 fn test_circuit_stats_close_transition() {
5322 let _guard = test_guard!();
5323 let mut stats = CircuitStats::new();
5324 stats.open();
5325 stats.half_open();
5326 stats.close();
5327
5328 assert_eq!(stats.state(), CircuitState::Closed);
5329 assert!(stats.opened_at().is_none());
5330 assert_eq!(stats.consecutive_failures(), 0);
5331 }
5332
5333 #[test]
5334 fn test_circuit_stats_should_close() {
5335 let _guard = test_guard!();
5336 let mut stats = CircuitStats::new();
5337 let config = CircuitBreakerConfig {
5338 success_threshold: 2,
5339 ..Default::default()
5340 };
5341
5342 stats.open();
5343 stats.half_open();
5344
5345 assert!(!stats.should_close(&config)); stats.record_success();
5348 assert!(!stats.should_close(&config)); stats.record_success();
5351 assert!(stats.should_close(&config)); }
5353
5354 #[test]
5355 fn test_circuit_stats_can_probe() {
5356 let _guard = test_guard!();
5357 let mut stats = CircuitStats::new();
5358 let config = CircuitBreakerConfig {
5359 half_open_max_probes: 1,
5360 ..Default::default()
5361 };
5362
5363 assert!(!stats.can_probe(&config));
5365
5366 stats.open();
5367 assert!(!stats.can_probe(&config));
5369
5370 stats.half_open();
5371 assert!(stats.can_probe(&config));
5373
5374 assert!(stats.start_probe(&config));
5376 assert!(!stats.can_probe(&config));
5378 assert!(!stats.start_probe(&config));
5379 }
5380
5381 #[test]
5382 fn test_circuit_stats_probe_completion() {
5383 let _guard = test_guard!();
5384 let mut stats = CircuitStats::new();
5385 let config = CircuitBreakerConfig {
5386 half_open_max_probes: 1,
5387 ..Default::default()
5388 };
5389
5390 stats.open();
5391 stats.half_open();
5392 stats.start_probe(&config);
5393
5394 stats.record_success();
5396
5397 assert!(stats.can_probe(&config));
5399 }
5400
5401 #[test]
5402 fn test_circuit_stats_failure_in_half_open() {
5403 let _guard = test_guard!();
5404 let mut stats = CircuitStats::new();
5405 let config = CircuitBreakerConfig {
5406 success_threshold: 2,
5407 ..Default::default()
5408 };
5409
5410 stats.open();
5411 stats.half_open();
5412
5413 stats.record_success();
5414 assert_eq!(stats.consecutive_failures(), 0);
5415
5416 stats.record_failure();
5418 assert!(!stats.should_close(&config));
5419
5420 stats.record_success();
5422 stats.record_success();
5423 assert!(stats.should_close(&config));
5424 }
5425
5426 #[test]
5427 fn test_circuit_stats_reset_window() {
5428 let _guard = test_guard!();
5429 let mut stats = CircuitStats::new();
5430 stats.record_success();
5431 stats.record_failure();
5432
5433 assert!(stats.error_rate() > 0.0);
5434
5435 stats.reset_window();
5436 assert_eq!(stats.error_rate(), 0.0);
5437 }
5438
5439 #[test]
5440 fn test_circuit_state_transitions_are_deterministic() {
5441 let _guard = test_guard!();
5442 let config = CircuitBreakerConfig::default();
5443 let mut stats = CircuitStats::new();
5444
5445 assert_eq!(stats.state(), CircuitState::Closed);
5447
5448 for _ in 0..3 {
5450 stats.record_failure();
5451 }
5452 assert!(stats.should_open(&config));
5453 stats.open();
5454 assert_eq!(stats.state(), CircuitState::Open);
5455
5456 stats.half_open();
5458 assert_eq!(stats.state(), CircuitState::HalfOpen);
5459
5460 for _ in 0..2 {
5462 stats.record_success();
5463 }
5464 assert!(stats.should_close(&config));
5465 stats.close();
5466 assert_eq!(stats.state(), CircuitState::Closed);
5467 }
5468
5469 fn make_test_metrics(
5474 speedup: Option<f64>,
5475 total_secs: u64,
5476 success: bool,
5477 ) -> CompilationMetrics {
5478 CompilationMetrics {
5479 project_id: "test-project".to_string(),
5480 worker_id: "worker-1".to_string(),
5481 timestamp: chrono::Utc::now(),
5482 timing: CompilationTimingBreakdown {
5483 rsync_up: Duration::from_millis(100),
5484 remote_build: Duration::from_millis(800),
5485 rsync_down: Duration::from_millis(100),
5486 total: Duration::from_secs(total_secs),
5487 },
5488 local_build_time: speedup.map(|s| Duration::from_secs_f64(total_secs as f64 * s)),
5489 speedup,
5490 files_synced: 100,
5491 bytes_transferred: 1_000_000,
5492 exit_code: if success { 0 } else { 1 },
5493 success,
5494 }
5495 }
5496
5497 #[test]
5498 fn test_compilation_timing_breakdown_default() {
5499 let _guard = test_guard!();
5500 let timing = CompilationTimingBreakdown::default();
5501 assert_eq!(timing.rsync_up, Duration::ZERO);
5502 assert_eq!(timing.remote_build, Duration::ZERO);
5503 assert_eq!(timing.rsync_down, Duration::ZERO);
5504 assert_eq!(timing.total, Duration::ZERO);
5505 }
5506
5507 #[test]
5508 fn test_compilation_timing_breakdown_serialization() {
5509 let _guard = test_guard!();
5510 let timing = CompilationTimingBreakdown {
5511 rsync_up: Duration::from_millis(100),
5512 remote_build: Duration::from_millis(2000),
5513 rsync_down: Duration::from_millis(50),
5514 total: Duration::from_millis(2150),
5515 };
5516
5517 let json = serde_json::to_string(&timing).unwrap();
5518 assert!(json.contains("100")); assert!(json.contains("2000")); assert!(json.contains("50")); assert!(json.contains("2150")); let parsed: CompilationTimingBreakdown = serde_json::from_str(&json).unwrap();
5524 assert_eq!(parsed.rsync_up, timing.rsync_up);
5525 assert_eq!(parsed.remote_build, timing.remote_build);
5526 assert_eq!(parsed.rsync_down, timing.rsync_down);
5527 assert_eq!(parsed.total, timing.total);
5528 }
5529
5530 #[test]
5531 fn test_command_timing_breakdown_serialization() {
5532 let _guard = test_guard!();
5533 let timing = CommandTimingBreakdown {
5534 classify: Some(Duration::from_millis(2)),
5535 select: None,
5536 sync_up: Some(Duration::from_millis(120)),
5537 exec: Some(Duration::from_millis(900)),
5538 sync_down: None,
5539 cleanup: Some(Duration::from_millis(5)),
5540 total: Some(Duration::from_millis(1027)),
5541 };
5542
5543 let json = serde_json::to_string(&timing).unwrap();
5544 assert!(json.contains("\"classify\":2"));
5545 assert!(json.contains("\"select\":null"));
5546 assert!(json.contains("\"sync_up\":120"));
5547 assert!(json.contains("\"exec\":900"));
5548 assert!(json.contains("\"sync_down\":null"));
5549 assert!(json.contains("\"cleanup\":5"));
5550 assert!(json.contains("\"total\":1027"));
5551
5552 let parsed: CommandTimingBreakdown = serde_json::from_str(&json).unwrap();
5553 assert_eq!(parsed.classify, timing.classify);
5554 assert_eq!(parsed.select, timing.select);
5555 assert_eq!(parsed.sync_up, timing.sync_up);
5556 assert_eq!(parsed.exec, timing.exec);
5557 assert_eq!(parsed.sync_down, timing.sync_down);
5558 assert_eq!(parsed.cleanup, timing.cleanup);
5559 assert_eq!(parsed.total, timing.total);
5560 }
5561
5562 #[test]
5563 fn test_compilation_metrics_calculate_speedup() {
5564 let _guard = test_guard!();
5565 let mut metrics = CompilationMetrics {
5566 timing: CompilationTimingBreakdown {
5567 total: Duration::from_secs(10),
5568 ..Default::default()
5569 },
5570 local_build_time: Some(Duration::from_secs(30)),
5571 ..Default::default()
5572 };
5573
5574 metrics.calculate_speedup();
5575
5576 assert!(metrics.speedup.is_some());
5577 let speedup = metrics.speedup.unwrap();
5578 assert!(
5579 (speedup - 3.0).abs() < 0.01,
5580 "Expected 3.0x speedup, got {}",
5581 speedup
5582 );
5583 }
5584
5585 #[test]
5586 fn test_compilation_metrics_is_beneficial() {
5587 let _guard = test_guard!();
5588 let mut metrics = CompilationMetrics::default();
5589
5590 assert!(!metrics.is_beneficial());
5592
5593 metrics.speedup = Some(1.5);
5595 assert!(metrics.is_beneficial());
5596
5597 metrics.speedup = Some(0.8);
5599 assert!(!metrics.is_beneficial());
5600
5601 metrics.speedup = Some(1.0);
5603 assert!(!metrics.is_beneficial());
5604 }
5605
5606 #[test]
5607 fn test_compilation_metrics_serialization() {
5608 let _guard = test_guard!();
5609 let metrics = make_test_metrics(Some(2.5), 10, true);
5610 let json = serde_json::to_string(&metrics).unwrap();
5611
5612 assert!(json.contains("test-project"));
5613 assert!(json.contains("worker-1"));
5614 assert!(json.contains("2.5")); let parsed: CompilationMetrics = serde_json::from_str(&json).unwrap();
5617 assert_eq!(parsed.project_id, "test-project");
5618 assert_eq!(parsed.worker_id, "worker-1");
5619 assert!(parsed.success);
5620 }
5621
5622 #[test]
5623 fn test_compilation_timer_new() {
5624 let _guard = test_guard!();
5625 let timer = CompilationTimer::new("my-project", "my-worker");
5626 assert_eq!(timer.project_id, "my-project");
5627 assert_eq!(timer.worker_id, "my-worker");
5628 assert!(timer.rsync_up.is_none());
5629 assert!(timer.remote_build.is_none());
5630 assert!(timer.rsync_down.is_none());
5631 }
5632
5633 #[test]
5634 fn test_compilation_timer_phases() {
5635 let _guard = test_guard!();
5636 let mut timer = CompilationTimer::new("test", "worker");
5637
5638 std::thread::sleep(Duration::from_millis(5));
5640 timer.end_rsync_up();
5641 assert!(timer.rsync_up.is_some());
5642 assert!(timer.rsync_up.unwrap() >= Duration::from_millis(5));
5643
5644 std::thread::sleep(Duration::from_millis(10));
5646 timer.end_remote_build();
5647 assert!(timer.remote_build.is_some());
5648 assert!(timer.remote_build.unwrap() >= Duration::from_millis(10));
5649
5650 std::thread::sleep(Duration::from_millis(5));
5652 timer.end_rsync_down();
5653 assert!(timer.rsync_down.is_some());
5654 assert!(timer.rsync_down.unwrap() >= Duration::from_millis(5));
5655
5656 let metrics = timer.finish(0, 50, 500_000);
5658 assert_eq!(metrics.project_id, "test");
5659 assert_eq!(metrics.worker_id, "worker");
5660 assert!(metrics.success);
5661 assert!(metrics.timing.total >= Duration::from_millis(20));
5662 assert_eq!(metrics.files_synced, 50);
5663 assert_eq!(metrics.bytes_transferred, 500_000);
5664 }
5665
5666 #[test]
5667 fn test_metrics_aggregator_new() {
5668 let _guard = test_guard!();
5669 let agg = MetricsAggregator::new(100);
5670 assert_eq!(agg.count(), 0);
5671 assert!(agg.average_speedup().is_none());
5672 }
5673
5674 #[test]
5675 fn test_metrics_aggregator_record() {
5676 let _guard = test_guard!();
5677 let mut agg = MetricsAggregator::new(3);
5678
5679 agg.record(make_test_metrics(Some(2.0), 10, true));
5681 assert_eq!(agg.count(), 1);
5682
5683 agg.record(make_test_metrics(Some(3.0), 20, true));
5685 agg.record(make_test_metrics(Some(4.0), 30, true));
5686 assert_eq!(agg.count(), 3);
5687
5688 agg.record(make_test_metrics(Some(5.0), 40, true));
5690 assert_eq!(agg.count(), 3);
5691 }
5692
5693 #[test]
5694 fn test_metrics_aggregator_average_speedup() {
5695 let _guard = test_guard!();
5696 let mut agg = MetricsAggregator::new(100);
5697
5698 assert!(agg.average_speedup().is_none());
5700
5701 for i in 1..=5 {
5703 agg.record(make_test_metrics(Some(i as f64), 10, true));
5704 }
5705
5706 let avg = agg.average_speedup().unwrap();
5707 assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5708 }
5709
5710 #[test]
5711 fn test_metrics_aggregator_average_speedup_with_none() {
5712 let _guard = test_guard!();
5713 let mut agg = MetricsAggregator::new(100);
5714
5715 agg.record(make_test_metrics(Some(2.0), 10, true));
5717 agg.record(make_test_metrics(None, 10, true)); agg.record(make_test_metrics(Some(4.0), 10, true));
5719
5720 let avg = agg.average_speedup().unwrap();
5722 assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5723 }
5724
5725 #[test]
5726 fn test_metrics_aggregator_percentiles() {
5727 let _guard = test_guard!();
5728 let mut agg = MetricsAggregator::new(100);
5729
5730 for i in 1..=10 {
5732 agg.record(make_test_metrics(Some(1.0), i, true));
5733 }
5734
5735 let p50 = agg.p50_total_time().unwrap();
5737 assert!(p50 >= Duration::from_secs(5) && p50 <= Duration::from_secs(6));
5738
5739 let p95 = agg.p95_total_time().unwrap();
5741 assert!(p95 >= Duration::from_secs(9) && p95 <= Duration::from_secs(10));
5742
5743 let p99 = agg.p99_total_time().unwrap();
5745 assert!(p99 >= Duration::from_secs(9));
5746 }
5747
5748 #[test]
5749 fn test_metrics_aggregator_success_rate() {
5750 let _guard = test_guard!();
5751 let mut agg = MetricsAggregator::new(100);
5752
5753 assert_eq!(agg.success_rate(), 100.0);
5755
5756 agg.record(make_test_metrics(Some(1.0), 10, true));
5758 agg.record(make_test_metrics(Some(1.0), 10, true));
5759 assert_eq!(agg.success_rate(), 100.0);
5760
5761 agg.record(make_test_metrics(Some(1.0), 10, false));
5763 let rate = agg.success_rate();
5764 assert!((rate - 66.67).abs() < 1.0, "Expected ~66.67%, got {}", rate);
5765 }
5766
5767 #[test]
5768 fn test_metrics_aggregator_clear() {
5769 let _guard = test_guard!();
5770 let mut agg = MetricsAggregator::new(100);
5771 agg.record(make_test_metrics(Some(1.0), 10, true));
5772 agg.record(make_test_metrics(Some(2.0), 20, true));
5773
5774 assert_eq!(agg.count(), 2);
5775
5776 agg.clear();
5777 assert_eq!(agg.count(), 0);
5778 assert!(agg.average_speedup().is_none());
5779 }
5780
5781 #[test]
5784 fn test_remote_build_jobs_parse_display_and_serde() {
5785 let _guard = test_guard!();
5786 assert_eq!(RemoteBuildJobs::default(), RemoteBuildJobs::Auto);
5787 for (spelling, expected) in [
5788 ("auto", RemoteBuildJobs::Auto),
5789 (" AUTO ", RemoteBuildJobs::Auto),
5790 ("off", RemoteBuildJobs::Off),
5791 ("none", RemoteBuildJobs::Off),
5792 ("4", RemoteBuildJobs::Fixed(4)),
5793 ] {
5794 assert_eq!(
5795 RemoteBuildJobs::parse(spelling).unwrap(),
5796 expected,
5797 "{spelling}"
5798 );
5799 }
5800 assert!(RemoteBuildJobs::parse("0").is_err(), "cargo rejects -j 0");
5801 assert!(RemoteBuildJobs::parse("many").is_err());
5802 assert_eq!(RemoteBuildJobs::Fixed(4).to_string(), "4");
5803 assert_eq!(RemoteBuildJobs::Off.to_string(), "off");
5804
5805 #[derive(Deserialize)]
5808 struct Wrap {
5809 compilation: CompilationConfig,
5810 }
5811 let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = \"off\"\n").unwrap();
5812 assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Off);
5813 let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = 6\n").unwrap();
5814 assert_eq!(
5815 parsed.compilation.remote_build_jobs,
5816 RemoteBuildJobs::Fixed(6)
5817 );
5818 let parsed: Wrap = toml::from_str("[compilation]\nbuild_slots = 4\n").unwrap();
5819 assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Auto);
5820 assert!(toml::from_str::<Wrap>("[compilation]\nremote_build_jobs = 0\n").is_err());
5821
5822 let json = serde_json::to_string(&RemoteBuildJobs::Fixed(3)).unwrap();
5824 assert_eq!(json, "3");
5825 assert_eq!(
5826 serde_json::to_string(&RemoteBuildJobs::Auto).unwrap(),
5827 "\"auto\""
5828 );
5829 let back: RemoteBuildJobs = serde_json::from_str("\"auto\"").unwrap();
5830 assert_eq!(back, RemoteBuildJobs::Auto);
5831 }
5832
5833 #[test]
5834 fn test_compilation_config_default_timeouts() {
5835 let _guard = test_guard!();
5836 let config = CompilationConfig::default();
5837 assert_eq!(config.build_timeout_sec, 300);
5839 assert_eq!(config.test_timeout_sec, 1800);
5841 }
5842
5843 #[test]
5844 fn test_compilation_config_timeout_for_test_kinds() {
5845 let _guard = test_guard!();
5846 let config = CompilationConfig::default();
5847
5848 assert_eq!(
5850 config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5851 std::time::Duration::from_secs(1800)
5852 );
5853 assert_eq!(
5854 config.timeout_for_kind(Some(crate::CompilationKind::CargoNextest)),
5855 std::time::Duration::from_secs(1800)
5856 );
5857 assert_eq!(
5858 config.timeout_for_kind(Some(crate::CompilationKind::BunTest)),
5859 std::time::Duration::from_secs(config.bun_timeout_sec)
5860 );
5861 }
5862
5863 #[test]
5864 fn test_compilation_config_timeout_for_build_kinds() {
5865 let _guard = test_guard!();
5866 let config = CompilationConfig::default();
5867
5868 assert_eq!(
5870 config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5871 std::time::Duration::from_secs(300)
5872 );
5873 assert_eq!(
5874 config.timeout_for_kind(Some(crate::CompilationKind::CargoCheck)),
5875 std::time::Duration::from_secs(300)
5876 );
5877 assert_eq!(
5878 config.timeout_for_kind(Some(crate::CompilationKind::CargoClippy)),
5879 std::time::Duration::from_secs(300)
5880 );
5881 assert_eq!(
5882 config.timeout_for_kind(None),
5883 std::time::Duration::from_secs(300)
5884 );
5885 }
5886
5887 #[test]
5888 fn test_compilation_config_custom_timeouts() {
5889 let _guard = test_guard!();
5890 let config = CompilationConfig {
5891 build_timeout_sec: 600, test_timeout_sec: 3600, ..Default::default()
5894 };
5895
5896 assert_eq!(
5897 config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5898 std::time::Duration::from_secs(600)
5899 );
5900 assert_eq!(
5901 config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5902 std::time::Duration::from_secs(3600)
5903 );
5904 }
5905
5906 #[test]
5907 fn test_compilation_config_speedup_threshold_default() {
5908 let _guard = test_guard!();
5909 let config = CompilationConfig::default();
5910 assert!((config.remote_speedup_threshold - 1.2).abs() < 0.001);
5912 }
5913
5914 #[test]
5915 fn test_compilation_config_speedup_threshold_custom() {
5916 let _guard = test_guard!();
5917 let config = CompilationConfig {
5918 remote_speedup_threshold: 1.5, ..Default::default()
5920 };
5921 assert!((config.remote_speedup_threshold - 1.5).abs() < 0.001);
5922 }
5923
5924 #[test]
5925 fn test_compilation_config_speedup_threshold_no_minimum() {
5926 let _guard = test_guard!();
5927 let config = CompilationConfig {
5929 remote_speedup_threshold: 1.0,
5930 ..Default::default()
5931 };
5932 assert!((config.remote_speedup_threshold - 1.0).abs() < 0.001);
5933 }
5934
5935 #[test]
5936 fn test_compilation_config_min_local_time_ms_default() {
5937 let _guard = test_guard!();
5938 let config = CompilationConfig::default();
5939 assert_eq!(config.min_local_time_ms, 2000);
5941 }
5942
5943 #[test]
5944 fn test_compilation_config_serde_roundtrip_speedup_threshold() {
5945 let _guard = test_guard!();
5946 let config = CompilationConfig {
5947 remote_speedup_threshold: 2.5,
5948 min_local_time_ms: 5000,
5949 ..Default::default()
5950 };
5951 let json = serde_json::to_string(&config).unwrap();
5952 let parsed: CompilationConfig = serde_json::from_str(&json).unwrap();
5953 assert!((parsed.remote_speedup_threshold - 2.5).abs() < 0.001);
5954 assert_eq!(parsed.min_local_time_ms, 5000);
5955 }
5956
5957 #[test]
5962 fn test_validate_remote_base_absolute_path() {
5963 let _guard = test_guard!();
5964 assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5965 assert_eq!(
5966 validate_remote_base("/var/rch-builds").unwrap(),
5967 "/var/rch-builds"
5968 );
5969 assert_eq!(
5970 validate_remote_base("/home/builder/.rch").unwrap(),
5971 "/home/builder/.rch"
5972 );
5973 }
5974
5975 #[test]
5976 fn test_validate_remote_base_tilde_expansion() {
5977 let _guard = test_guard!();
5978 let result = validate_remote_base("~/rch");
5980 assert!(result.is_ok());
5981 let path = result.unwrap();
5982 assert!(
5983 path.starts_with('/'),
5984 "Path should be absolute after expansion: {}",
5985 path
5986 );
5987 assert!(!path.contains('~'), "Tilde should be expanded: {}", path);
5988 }
5989
5990 #[test]
5991 fn test_validate_remote_base_rejects_relative_path() {
5992 let _guard = test_guard!();
5993 let result = validate_remote_base("tmp/rch");
5994 assert!(result.is_err());
5995 assert!(result.unwrap_err().contains("absolute path"));
5996 }
5997
5998 #[test]
5999 fn test_validate_remote_base_rejects_path_traversal() {
6000 let _guard = test_guard!();
6001 let result = validate_remote_base("/tmp/../etc/rch");
6002 assert!(result.is_err());
6003 assert!(result.unwrap_err().contains("path traversal"));
6004
6005 let result = validate_remote_base("/tmp/rch/../other");
6006 assert!(result.is_err());
6007 assert!(result.unwrap_err().contains("path traversal"));
6008 }
6009
6010 #[test]
6011 fn test_validate_remote_base_normalizes_trailing_slash() {
6012 let _guard = test_guard!();
6013 assert_eq!(validate_remote_base("/tmp/rch/").unwrap(), "/tmp/rch");
6014 assert_eq!(validate_remote_base("/tmp/rch///").unwrap(), "/tmp/rch");
6015 }
6016
6017 #[test]
6018 fn test_validate_remote_base_root_path() {
6019 let _guard = test_guard!();
6020 let result = validate_remote_base("/");
6022 assert!(result.is_err());
6023 assert!(result.unwrap_err().contains("root directory"));
6024 }
6025
6026 #[test]
6027 fn test_validate_remote_base_rejects_top_level() {
6028 let _guard = test_guard!();
6029 let result = validate_remote_base("/tmp");
6031 assert!(result.is_err());
6032 assert!(result.unwrap_err().contains("at least 2 levels deep"));
6033
6034 let result = validate_remote_base("/home");
6035 assert!(result.is_err());
6036
6037 assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
6039 }
6040
6041 #[test]
6042 fn test_default_remote_base() {
6043 let _guard = test_guard!();
6044 assert_eq!(default_remote_base(), "/data/tmp/rch");
6045 }
6046
6047 #[test]
6048 fn test_transfer_config_default_has_remote_base() {
6049 let _guard = test_guard!();
6050 let config = TransferConfig::default();
6051 assert_eq!(config.remote_base, "/data/tmp/rch");
6052 }
6053
6054 #[test]
6055 fn test_source_sync_timeout_validation_boundaries() {
6056 let _guard = test_guard!();
6057 assert!(!TransferConfig::valid_sync_timeout_ms(0));
6058 assert!(!TransferConfig::valid_sync_timeout_ms(999));
6059 assert!(TransferConfig::valid_sync_timeout_ms(1_000));
6060 assert!(TransferConfig::valid_sync_timeout_ms(3_600_000));
6061 assert!(!TransferConfig::valid_sync_timeout_ms(3_600_001));
6062 }
6063
6064 #[test]
6065 fn test_source_sync_timeout_is_explicit_or_payload_aware() {
6066 let _guard = test_guard!();
6067 let default = TransferConfig::default();
6068 assert_eq!(
6069 default.sync_timeout_for_payload(0),
6070 std::time::Duration::from_millis(30_000)
6071 );
6072 assert_eq!(
6073 default.sync_timeout_for_payload(10 * 1024 * 1024),
6074 std::time::Duration::from_millis(40_000)
6075 );
6076 assert_eq!(
6077 default.sync_timeout_for_payload(u64::MAX),
6078 std::time::Duration::from_millis(3_600_000)
6079 );
6080
6081 let explicit = TransferConfig {
6082 sync_timeout_ms: Some(120_000),
6083 ..TransferConfig::default()
6084 };
6085 assert_eq!(
6086 explicit.sync_timeout_for_payload(u64::MAX),
6087 std::time::Duration::from_millis(120_000)
6088 );
6089 }
6090
6091 #[test]
6096 fn test_retry_config_default() {
6097 let _guard = test_guard!();
6098 let config = RetryConfig::default();
6099 assert_eq!(config.max_attempts, 3);
6100 assert_eq!(config.base_delay_ms, 100);
6101 assert_eq!(config.max_delay_ms, 5000);
6102 assert_eq!(config.jitter_factor, 0.1);
6103 assert_eq!(config.total_timeout_ms, 30000);
6104 }
6105
6106 #[test]
6107 fn test_retry_config_no_retry() {
6108 let _guard = test_guard!();
6109 let config = RetryConfig::no_retry();
6110 assert_eq!(config.max_attempts, 1);
6111 }
6112
6113 #[test]
6114 fn test_retry_config_delay_for_attempt_zero() {
6115 let _guard = test_guard!();
6116 let config = RetryConfig::default();
6117 assert_eq!(config.delay_for_attempt(0), std::time::Duration::ZERO);
6118 }
6119
6120 #[test]
6121 fn test_retry_config_delay_for_attempt_exponential() {
6122 let _guard = test_guard!();
6123 let config = RetryConfig {
6124 base_delay_ms: 100,
6125 max_delay_ms: 10000,
6126 jitter_factor: 0.0, ..Default::default()
6128 };
6129
6130 assert_eq!(
6132 config.delay_for_attempt(1),
6133 std::time::Duration::from_millis(100)
6134 );
6135 assert_eq!(
6137 config.delay_for_attempt(2),
6138 std::time::Duration::from_millis(200)
6139 );
6140 assert_eq!(
6142 config.delay_for_attempt(3),
6143 std::time::Duration::from_millis(400)
6144 );
6145 }
6146
6147 #[test]
6148 fn test_retry_config_delay_capped_at_max() {
6149 let _guard = test_guard!();
6150 let config = RetryConfig {
6151 base_delay_ms: 1000,
6152 max_delay_ms: 2000,
6153 jitter_factor: 0.0,
6154 ..Default::default()
6155 };
6156
6157 assert_eq!(
6159 config.delay_for_attempt(2),
6160 std::time::Duration::from_millis(2000)
6161 );
6162 assert_eq!(
6164 config.delay_for_attempt(3),
6165 std::time::Duration::from_millis(2000)
6166 );
6167 }
6168
6169 #[test]
6170 fn test_retry_config_random_jitter_stays_within_bounds() {
6171 let _guard = test_guard!();
6172 let config = RetryConfig {
6173 base_delay_ms: 100,
6174 max_delay_ms: 200,
6175 jitter_factor: 0.25,
6176 ..Default::default()
6177 };
6178
6179 for (attempt, minimum_ms, maximum_ms) in [(1, 75, 125), (3, 150, 250)] {
6182 for _ in 0..128 {
6183 let delay = config.delay_for_attempt(attempt);
6184 assert!(delay >= std::time::Duration::from_millis(minimum_ms));
6185 assert!(delay <= std::time::Duration::from_millis(maximum_ms));
6186 }
6187 }
6188 }
6189
6190 #[test]
6191 fn test_retry_config_should_retry() {
6192 let _guard = test_guard!();
6193 let config = RetryConfig {
6194 max_attempts: 3,
6195 total_timeout_ms: 1000,
6196 ..Default::default()
6197 };
6198
6199 assert!(config.should_retry(0, std::time::Duration::from_millis(0)));
6201 assert!(config.should_retry(1, std::time::Duration::from_millis(500)));
6202 assert!(config.should_retry(2, std::time::Duration::from_millis(900)));
6203
6204 assert!(!config.should_retry(3, std::time::Duration::from_millis(0)));
6206
6207 assert!(!config.should_retry(1, std::time::Duration::from_millis(1001)));
6209 }
6210
6211 #[test]
6212 fn test_retry_config_serde_roundtrip() {
6213 let _guard = test_guard!();
6214 let config = RetryConfig {
6215 max_attempts: 5,
6216 base_delay_ms: 200,
6217 max_delay_ms: 8000,
6218 jitter_factor: 0.2,
6219 total_timeout_ms: 60000,
6220 };
6221 let json = serde_json::to_string(&config).unwrap();
6222 let parsed: RetryConfig = serde_json::from_str(&json).unwrap();
6223 assert_eq!(parsed.max_attempts, 5);
6224 assert_eq!(parsed.base_delay_ms, 200);
6225 assert_eq!(parsed.max_delay_ms, 8000);
6226 assert_eq!(parsed.jitter_factor, 0.2);
6227 assert_eq!(parsed.total_timeout_ms, 60000);
6228 }
6229
6230 #[test]
6231 fn test_transfer_config_includes_retry() {
6232 let _guard = test_guard!();
6233 let config = TransferConfig::default();
6234 assert_eq!(config.retry.max_attempts, 3);
6235 }
6236
6237 #[test]
6242 fn test_execution_config_default_allowlist() {
6243 let _guard = test_guard!();
6244 let config = ExecutionConfig::default();
6245 assert!(config.is_allowed("cargo"));
6247 assert!(config.is_allowed("rustc"));
6248 assert!(config.is_allowed("gcc"));
6249 assert!(config.is_allowed("g++"));
6250 assert!(config.is_allowed("clang"));
6251 assert!(config.is_allowed("clang++"));
6252 assert!(config.is_allowed("make"));
6253 assert!(config.is_allowed("cmake"));
6254 assert!(config.is_allowed("ninja"));
6255 assert!(config.is_allowed("meson"));
6256 assert!(config.is_allowed("bun"));
6257 assert!(config.is_allowed("nextest"));
6258 assert!(config.is_allowed("cc"));
6259 assert!(config.is_allowed("c++"));
6260 }
6261
6262 #[test]
6263 fn test_execution_config_is_allowed_case_insensitive() {
6264 let _guard = test_guard!();
6265 let config = ExecutionConfig::default();
6266 assert!(config.is_allowed("CARGO"));
6268 assert!(config.is_allowed("Cargo"));
6269 assert!(config.is_allowed("GCC"));
6270 assert!(config.is_allowed("Gcc"));
6271 }
6272
6273 #[test]
6274 fn test_execution_config_is_not_allowed() {
6275 let _guard = test_guard!();
6276 let config = ExecutionConfig::default();
6277 assert!(!config.is_allowed("python"));
6279 assert!(!config.is_allowed("npm"));
6280 assert!(!config.is_allowed(""));
6281 assert!(config.is_allowed("go"));
6286 assert!(config.is_allowed("tsc"));
6287 }
6288
6289 #[test]
6290 fn test_execution_config_empty_allowlist() {
6291 let _guard = test_guard!();
6292 let config = ExecutionConfig { allowlist: vec![] };
6293 assert!(!config.is_allowed("cargo"));
6295 assert!(!config.is_allowed("gcc"));
6296 }
6297
6298 #[test]
6299 fn test_execution_config_custom_allowlist() {
6300 let _guard = test_guard!();
6301 let config = ExecutionConfig {
6302 allowlist: vec!["cargo".to_string(), "custom_tool".to_string()],
6303 };
6304 assert!(config.is_allowed("cargo"));
6305 assert!(config.is_allowed("custom_tool"));
6306 assert!(!config.is_allowed("gcc"));
6307 }
6308
6309 #[test]
6310 fn test_execution_config_serde() {
6311 let _guard = test_guard!();
6312 let config = ExecutionConfig {
6313 allowlist: vec!["cargo".to_string(), "rustc".to_string()],
6314 };
6315 let json = serde_json::to_string(&config).unwrap();
6316 let parsed: ExecutionConfig = serde_json::from_str(&json).unwrap();
6317 assert_eq!(parsed.allowlist, config.allowlist);
6318 }
6319
6320 #[test]
6321 fn test_rch_config_includes_execution() {
6322 let _guard = test_guard!();
6323 let config = RchConfig::default();
6324 assert!(config.execution.is_allowed("cargo"));
6326 }
6327
6328 #[test]
6333 fn test_adaptive_compression_disabled_by_default() {
6334 let _guard = test_guard!();
6335 let config = TransferConfig::default();
6336 assert!(!config.adaptive_compression);
6337 assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6339 assert_eq!(config.select_compression_level(Some(500_000_000)), 3);
6340 }
6341
6342 #[test]
6343 fn test_adaptive_compression_small_payload() {
6344 let _guard = test_guard!();
6345 let config = TransferConfig {
6346 adaptive_compression: true,
6347 ..Default::default()
6348 };
6349 assert_eq!(config.select_compression_level(Some(0)), 1);
6351 assert_eq!(config.select_compression_level(Some(1_000_000)), 1); assert_eq!(config.select_compression_level(Some(9_999_999)), 1); }
6354
6355 #[test]
6356 fn test_adaptive_compression_medium_payload() {
6357 let _guard = test_guard!();
6358 let config = TransferConfig {
6359 adaptive_compression: true,
6360 ..Default::default()
6361 };
6362 assert_eq!(config.select_compression_level(Some(10_000_000)), 3); assert_eq!(config.select_compression_level(Some(100_000_000)), 3); assert_eq!(config.select_compression_level(Some(199_999_999)), 3); }
6367
6368 #[test]
6369 fn test_adaptive_compression_large_payload() {
6370 let _guard = test_guard!();
6371 let config = TransferConfig {
6372 adaptive_compression: true,
6373 ..Default::default()
6374 };
6375 assert_eq!(config.select_compression_level(Some(200_000_000)), 7); assert_eq!(config.select_compression_level(Some(500_000_000)), 7); assert_eq!(config.select_compression_level(Some(1_000_000_000)), 7); }
6380
6381 #[test]
6382 fn test_adaptive_compression_no_estimate() {
6383 let _guard = test_guard!();
6384 let config = TransferConfig {
6385 adaptive_compression: true,
6386 compression_level: 5,
6387 ..Default::default()
6388 };
6389 assert_eq!(config.select_compression_level(None), 5);
6391 }
6392
6393 #[test]
6394 fn test_adaptive_compression_respects_min_level() {
6395 let _guard = test_guard!();
6396 let config = TransferConfig {
6397 adaptive_compression: true,
6398 min_compression_level: 3,
6399 ..Default::default()
6400 };
6401 assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6403 }
6404
6405 #[test]
6406 fn test_adaptive_compression_respects_max_level() {
6407 let _guard = test_guard!();
6408 let config = TransferConfig {
6409 adaptive_compression: true,
6410 max_compression_level: 5,
6411 ..Default::default()
6412 };
6413 assert_eq!(config.select_compression_level(Some(500_000_000)), 5);
6415 }
6416
6417 #[test]
6418 fn test_adaptive_compression_serde_roundtrip() {
6419 let _guard = test_guard!();
6420 let config = TransferConfig {
6421 adaptive_compression: true,
6422 min_compression_level: 2,
6423 max_compression_level: 8,
6424 ..Default::default()
6425 };
6426 let json = serde_json::to_string(&config).unwrap();
6427 let parsed: TransferConfig = serde_json::from_str(&json).unwrap();
6428 assert!(parsed.adaptive_compression);
6429 assert_eq!(parsed.min_compression_level, 2);
6430 assert_eq!(parsed.max_compression_level, 8);
6431 }
6432
6433 #[test]
6438 fn test_self_healing_config_defaults() {
6439 let _guard = test_guard!();
6440 let config = SelfHealingConfig::default();
6442
6443 assert!(
6445 config.hook_starts_daemon,
6446 "hook_starts_daemon should be true by default"
6447 );
6448 assert!(
6449 config.daemon_installs_hooks,
6450 "daemon_installs_hooks should be true by default"
6451 );
6452
6453 assert_eq!(
6455 config.auto_start_cooldown_secs, 30,
6456 "auto_start_cooldown_secs should default to 30"
6457 );
6458 assert_eq!(
6459 config.auto_start_timeout_secs, 3,
6460 "auto_start_timeout_secs should default to 3"
6461 );
6462 }
6464
6465 #[test]
6466 fn test_self_healing_config_serde_full() {
6467 let _guard = test_guard!();
6468 let config = SelfHealingConfig {
6470 hook_starts_daemon: false,
6471 daemon_installs_hooks: false,
6472 auto_start_cooldown_secs: 60,
6473 auto_start_timeout_secs: 10,
6474 self_healing_log_level: SelfHealingLogLevel::Debug,
6475 };
6476
6477 let json = serde_json::to_string(&config).unwrap();
6478 assert!(json.contains("\"hook_starts_daemon\":false"));
6479 assert!(json.contains("\"daemon_installs_hooks\":false"));
6480 assert!(json.contains("\"auto_start_cooldown_secs\":60"));
6481 assert!(json.contains("\"auto_start_timeout_secs\":10"));
6482 assert!(json.contains("\"self_healing_log_level\":\"debug\""));
6483
6484 let parsed: SelfHealingConfig = serde_json::from_str(&json).unwrap();
6485 assert!(!parsed.hook_starts_daemon);
6486 assert!(!parsed.daemon_installs_hooks);
6487 assert_eq!(parsed.auto_start_cooldown_secs, 60);
6488 assert_eq!(parsed.auto_start_timeout_secs, 10);
6489 assert_eq!(parsed.self_healing_log_level, SelfHealingLogLevel::Debug);
6490 }
6492
6493 #[test]
6496 fn test_self_healing_log_level_default_is_info() {
6497 let level = SelfHealingLogLevel::default();
6499 assert_eq!(level, SelfHealingLogLevel::Info);
6500 let config = SelfHealingConfig::default();
6502 assert_eq!(
6503 config.self_healing_log_level,
6504 SelfHealingLogLevel::Info,
6505 "config default should be Info"
6506 );
6507 }
6509
6510 #[test]
6511 fn test_self_healing_log_level_serializes_lowercase() {
6512 let json = serde_json::to_string(&SelfHealingLogLevel::Warn).unwrap();
6514 assert_eq!(json, "\"warn\"");
6515 let parsed: SelfHealingLogLevel = serde_json::from_str("\"error\"").unwrap();
6516 assert_eq!(parsed, SelfHealingLogLevel::Error);
6517 }
6519
6520 #[test]
6521 fn test_self_healing_log_level_from_env_str_accepts_known() {
6522 assert_eq!(
6524 SelfHealingLogLevel::from_env_str("debug"),
6525 Some(SelfHealingLogLevel::Debug)
6526 );
6527 assert_eq!(
6528 SelfHealingLogLevel::from_env_str("DEBUG"),
6529 Some(SelfHealingLogLevel::Debug)
6530 );
6531 assert_eq!(
6532 SelfHealingLogLevel::from_env_str("info"),
6533 Some(SelfHealingLogLevel::Info)
6534 );
6535 assert_eq!(
6536 SelfHealingLogLevel::from_env_str("warn"),
6537 Some(SelfHealingLogLevel::Warn)
6538 );
6539 assert_eq!(
6540 SelfHealingLogLevel::from_env_str("warning"),
6541 Some(SelfHealingLogLevel::Warn),
6542 "warning should map to Warn"
6543 );
6544 assert_eq!(
6545 SelfHealingLogLevel::from_env_str("error"),
6546 Some(SelfHealingLogLevel::Error)
6547 );
6548 assert_eq!(
6550 SelfHealingLogLevel::from_env_str(" debug "),
6551 Some(SelfHealingLogLevel::Debug)
6552 );
6553 }
6555
6556 #[test]
6557 fn test_self_healing_log_level_from_env_str_rejects_unknown() {
6558 assert!(SelfHealingLogLevel::from_env_str("banana").is_none());
6560 assert!(SelfHealingLogLevel::from_env_str("").is_none());
6561 assert!(SelfHealingLogLevel::from_env_str("trace").is_none());
6562 }
6564
6565 #[test]
6566 fn test_self_healing_config_serde_partial_uses_defaults() {
6567 let _guard = test_guard!();
6568 let json = r#"{"hook_starts_daemon": false}"#;
6570 let config: SelfHealingConfig = serde_json::from_str(json).unwrap();
6571
6572 assert!(
6573 !config.hook_starts_daemon,
6574 "Explicit false should be parsed"
6575 );
6576 assert!(
6577 config.daemon_installs_hooks,
6578 "Missing field should use default (true)"
6579 );
6580 assert_eq!(
6581 config.auto_start_cooldown_secs, 30,
6582 "Missing field should use default (30)"
6583 );
6584 assert_eq!(
6585 config.auto_start_timeout_secs, 3,
6586 "Missing field should use default (3)"
6587 );
6588 }
6590
6591 #[test]
6592 fn test_self_healing_config_toml_with_alias() {
6593 let _guard = test_guard!();
6594 let toml_str = r#"
6596 hook_starts_daemon = true
6597 daemon_installs_hooks = false
6598 auto_start_cooldown_secs = 45
6599 daemon_start_timeout = 7
6600 "#;
6601
6602 let config: SelfHealingConfig = toml::from_str(toml_str).unwrap();
6603 assert!(config.hook_starts_daemon);
6604 assert!(!config.daemon_installs_hooks);
6605 assert_eq!(config.auto_start_cooldown_secs, 45);
6606 assert_eq!(
6607 config.auto_start_timeout_secs, 7,
6608 "daemon_start_timeout alias should set auto_start_timeout_secs"
6609 );
6610 }
6612
6613 #[allow(unsafe_code)]
6614 mod self_healing_env_override_tests {
6615 use super::*;
6616 use crate::config::env_test_lock;
6617
6618 fn env_guard() -> std::sync::MutexGuard<'static, ()> {
6619 env_test_lock()
6620 }
6621
6622 fn set_env(key: &str, value: &str) {
6623 unsafe { std::env::set_var(key, value) };
6625 }
6626
6627 fn remove_env(key: &str) {
6628 unsafe { std::env::remove_var(key) };
6630 }
6631
6632 struct EnvVarGuard {
6633 key: &'static str,
6634 old: Option<String>,
6635 }
6636
6637 impl EnvVarGuard {
6638 fn set(key: &'static str, value: &str) -> Self {
6639 let old = std::env::var(key).ok();
6640 set_env(key, value);
6641 Self { key, old }
6642 }
6643 }
6644
6645 impl Drop for EnvVarGuard {
6646 fn drop(&mut self) {
6647 if let Some(old) = &self.old {
6648 set_env(self.key, old);
6649 } else {
6650 remove_env(self.key);
6651 }
6652 }
6653 }
6654
6655 #[test]
6656 fn test_self_healing_config_with_env_overrides_master_disable() {
6657 let _guard = test_guard!();
6658 let _guard = env_guard();
6659
6660 let _no_self_healing = EnvVarGuard::set("RCH_NO_SELF_HEALING", "1");
6661 let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "1");
6662 let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "1");
6663 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "99");
6664 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "99");
6665
6666 let config = SelfHealingConfig::default().with_env_overrides();
6667 assert!(
6668 !config.hook_starts_daemon,
6669 "RCH_NO_SELF_HEALING should disable hook auto-start"
6670 );
6671 assert!(
6672 !config.daemon_installs_hooks,
6673 "RCH_NO_SELF_HEALING should disable daemon hook installation"
6674 );
6675
6676 assert_eq!(config.auto_start_cooldown_secs, 30);
6678 assert_eq!(config.auto_start_timeout_secs, 3);
6679 }
6680
6681 #[test]
6682 fn test_self_healing_config_with_env_overrides_toggles_and_numbers() {
6683 let _guard = test_guard!();
6684 let _guard = env_guard();
6685
6686 let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "0");
6687 let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "false");
6688 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "45");
6689 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "7");
6690
6691 let config = SelfHealingConfig::default().with_env_overrides();
6692 assert!(!config.hook_starts_daemon);
6693 assert!(!config.daemon_installs_hooks);
6694 assert_eq!(config.auto_start_cooldown_secs, 45);
6695 assert_eq!(config.auto_start_timeout_secs, 7);
6696 }
6697
6698 #[test]
6699 fn test_self_healing_config_with_env_overrides_invalid_numbers_ignored() {
6700 let _guard = test_guard!();
6701 let _guard = env_guard();
6702
6703 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "not-a-number");
6704 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "nope");
6705
6706 let config = SelfHealingConfig::default().with_env_overrides();
6707 assert_eq!(config.auto_start_cooldown_secs, 30);
6708 assert_eq!(config.auto_start_timeout_secs, 3);
6709 }
6710 }
6711
6712 #[test]
6717 fn test_load_per_core_calculation() {
6718 let _guard = test_guard!();
6719 let mut caps = WorkerCapabilities::new();
6720 assert!(caps.load_per_core().is_none());
6722
6723 caps.load_avg_1 = Some(4.0);
6725 assert!(caps.load_per_core().is_none());
6726
6727 caps.num_cpus = Some(4);
6729 assert_eq!(caps.load_per_core(), Some(1.0)); caps.load_avg_1 = Some(16.0);
6733 assert_eq!(caps.load_per_core(), Some(4.0)); }
6735
6736 #[test]
6737 fn test_is_high_load() {
6738 let _guard = test_guard!();
6739 let mut caps = WorkerCapabilities::new();
6740 caps.load_avg_1 = Some(8.0);
6741 caps.num_cpus = Some(4);
6742 assert_eq!(caps.is_high_load(3.0), Some(false));
6746 assert_eq!(caps.is_high_load(2.0), Some(false)); assert_eq!(caps.is_high_load(1.5), Some(true));
6750
6751 caps.load_avg_1 = None;
6753 assert!(caps.is_high_load(1.0).is_none());
6754 }
6755
6756 #[test]
6757 fn test_is_low_disk() {
6758 let _guard = test_guard!();
6759 let mut caps = WorkerCapabilities::new();
6760 caps.disk_free_gb = Some(15.0);
6761
6762 assert_eq!(caps.is_low_disk(10.0), Some(false));
6764 assert_eq!(caps.is_low_disk(15.0), Some(false)); assert_eq!(caps.is_low_disk(20.0), Some(true));
6768
6769 caps.disk_free_gb = None;
6771 assert!(caps.is_low_disk(10.0).is_none());
6772 }
6773
6774 #[test]
6775 fn low_disk_judges_the_build_disk_not_a_small_tmpfs() {
6776 let _guard = test_guard!();
6777 let mut caps = WorkerCapabilities::new();
6780 caps.disk_free_gb = Some(8.0);
6781 caps.disk_total_gb = Some(15.5);
6782 caps.build_disk_free_gb = Some(1700.0);
6783 caps.build_disk_total_gb = Some(1900.0);
6784 assert_eq!(caps.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6785 assert_eq!(caps.is_low_disk(10.0), Some(false));
6786
6787 caps.build_disk_free_gb = Some(4.0);
6789 assert_eq!(caps.is_low_disk(10.0), Some(true));
6790
6791 caps.build_disk_free_gb = None;
6793 caps.build_disk_total_gb = None;
6794 assert_eq!(caps.build_disk_gb(), (Some(8.0), Some(15.5)));
6795 assert_eq!(caps.is_low_disk(10.0), Some(true));
6796 }
6797
6798 #[test]
6799 fn build_disk_sample_is_optional_on_the_wire() {
6800 let _guard = test_guard!();
6801 let old: WorkerCapabilities =
6802 serde_json::from_str(r#"{"disk_free_gb":13.9,"disk_total_gb":15.5}"#).unwrap();
6803 assert_eq!(old.build_disk_free_gb, None);
6804 assert_eq!(old.build_disk_gb(), (Some(13.9), Some(15.5)));
6805
6806 let new: WorkerCapabilities = serde_json::from_str(
6807 r#"{"disk_free_gb":13.9,"disk_total_gb":15.5,"build_disk_free_gb":1700.0,"build_disk_total_gb":1900.0}"#,
6808 )
6809 .unwrap();
6810 assert_eq!(new.build_disk_gb(), (Some(1700.0), Some(1900.0)));
6811 let json = serde_json::to_string(&new).unwrap();
6812 assert!(json.contains("\"build_disk_free_gb\":1700.0"), "{json}");
6813 }
6814
6815 #[test]
6816 fn test_selection_config_preflight_defaults() {
6817 let _guard = test_guard!();
6818 let config = SelectionConfig::default();
6819 assert_eq!(config.max_load_per_core, Some(2.0));
6821 assert_eq!(config.min_free_gb, Some(10.0));
6822 assert_eq!(config.disk_gb_per_slot, 10.0);
6823 }
6824
6825 #[test]
6826 fn test_selection_config_preflight_serde() {
6827 let _guard = test_guard!();
6828 let config = SelectionConfig {
6829 max_load_per_core: Some(3.5),
6830 min_free_gb: Some(25.0),
6831 disk_gb_per_slot: 0.83,
6832 ..Default::default()
6833 };
6834 let json = serde_json::to_string(&config).unwrap();
6835 let parsed: SelectionConfig = serde_json::from_str(&json).unwrap();
6836 assert_eq!(parsed.max_load_per_core, Some(3.5));
6837 assert_eq!(parsed.min_free_gb, Some(25.0));
6838 assert_eq!(parsed.disk_gb_per_slot, 0.83);
6839 }
6840
6841 #[test]
6842 fn test_selection_disk_slot_budget_validation() {
6843 let _guard = test_guard!();
6844 for value in ["0.0", "-1.0", "nan", "inf", "-inf"] {
6845 let result = toml::from_str::<SelectionConfig>(&format!("disk_gb_per_slot = {value}"));
6846 assert!(result.is_err(), "invalid disk slot budget {value}");
6847 }
6848 let default: SelectionConfig = toml::from_str("").unwrap();
6849 assert_eq!(default.disk_gb_per_slot, 10.0);
6850 }
6851
6852 #[test]
6853 fn test_selection_disk_weight_validation_and_roundtrip() {
6854 for value in ["-0.1", "1.1", "nan", "inf", "-inf"] {
6855 assert!(toml::from_str::<SelectionWeightConfig>(&format!("disk = {value}")).is_err());
6856 }
6857 for value in [0.0, 0.2, 1.0] {
6858 let weights: SelectionWeightConfig =
6859 toml::from_str(&format!("disk = {value}")).unwrap();
6860 assert_eq!(weights.disk, value);
6861 let roundtrip: SelectionWeightConfig =
6862 serde_json::from_str(&serde_json::to_string(&weights).unwrap()).unwrap();
6863 assert_eq!(roundtrip.disk, value);
6864 }
6865 let defaults: SelectionWeightConfig = toml::from_str("").unwrap();
6866 assert_eq!(defaults.disk, 0.2);
6867 }
6868
6869 #[test]
6870 fn test_worker_capabilities_health_metrics_serde() {
6871 let _guard = test_guard!();
6872 let caps = WorkerCapabilities {
6873 num_cpus: Some(8),
6874 load_avg_1: Some(1.5),
6875 load_avg_5: Some(2.0),
6876 load_avg_15: Some(1.8),
6877 disk_free_gb: Some(50.5),
6878 disk_total_gb: Some(100.0),
6879 ..Default::default()
6880 };
6881 let json = serde_json::to_string(&caps).unwrap();
6882 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
6883 assert_eq!(parsed.num_cpus, Some(8));
6884 assert_eq!(parsed.load_avg_1, Some(1.5));
6885 assert_eq!(parsed.disk_free_gb, Some(50.5));
6886 }
6887
6888 #[test]
6889 fn microarch_level_classifies_ivy_bridge_as_v2() {
6890 let ivy = "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov \
6893 pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx rdtscp lm \
6894 constant_tsc pni pclmulqdq ssse3 cx16 sse4_1 sse4_2 x2apic \
6895 popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm";
6896 assert_eq!(x86_64_microarch_level_from_flags(ivy), 2);
6897 }
6898
6899 #[test]
6900 fn microarch_level_classifies_v3_and_v4() {
6901 let v3 = "cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 abm avx avx2 bmi1 bmi2 \
6902 f16c fma movbe xsave";
6903 assert_eq!(x86_64_microarch_level_from_flags(v3), 3);
6904
6905 let v4 = format!("{v3} avx512f avx512bw avx512cd avx512dq avx512vl");
6906 assert_eq!(x86_64_microarch_level_from_flags(&v4), 4);
6907 }
6908
6909 #[test]
6910 fn microarch_level_v1_for_ancient_or_empty_flags() {
6911 assert_eq!(x86_64_microarch_level_from_flags(""), 1);
6912 assert_eq!(x86_64_microarch_level_from_flags("fpu mmx sse sse2"), 1);
6913 }
6914
6915 #[test]
6916 fn pre_v3_predicate_only_fires_on_affirmative_evidence() {
6917 let mut caps = WorkerCapabilities::new();
6918 assert!(!caps.is_pre_v3_x86(), "unknown must not deprioritize");
6919 caps.cpu_microarch_level = Some(2);
6920 assert!(caps.is_pre_v3_x86());
6921 caps.cpu_microarch_level = Some(3);
6922 assert!(!caps.is_pre_v3_x86());
6923 }
6924}