1use crate::{CompilationKind, toolchain::ToolchainInfo};
4use rand::RngExt;
5use schemars::JsonSchema;
6use serde::{Deserialize, Serialize};
7use std::path::PathBuf;
8
9#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
11pub struct WorkerId(pub String);
12
13impl WorkerId {
14 pub fn new(id: impl Into<String>) -> Self {
15 Self(id.into())
16 }
17
18 pub fn as_str(&self) -> &str {
19 &self.0
20 }
21}
22
23impl std::fmt::Display for WorkerId {
24 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
25 write!(f, "{}", self.0)
26 }
27}
28
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
46#[serde(rename_all = "snake_case")]
47pub enum WorkerStatus {
48 #[default]
50 Healthy,
51 Degraded,
53 Unreachable,
55 Draining,
57 Drained,
59 Disabled,
61}
62
63#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
65#[serde(rename_all = "snake_case")]
66pub enum CircuitState {
67 #[default]
69 Closed,
70 Open,
72 HalfOpen,
74}
75
76#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
78#[serde(rename_all = "lowercase")]
79pub enum RequiredRuntime {
80 #[default]
82 None,
83 Rust,
85 Bun,
87 Node,
89 Nix,
91 Go,
93 Zig,
95}
96
97#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
102#[serde(rename_all = "lowercase")]
103pub enum CommandPriority {
104 Low,
105 #[default]
106 Normal,
107 High,
108}
109
110impl std::fmt::Display for CommandPriority {
111 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112 let value = match self {
113 Self::Low => "low",
114 Self::Normal => "normal",
115 Self::High => "high",
116 };
117 write!(f, "{}", value)
118 }
119}
120
121impl std::str::FromStr for CommandPriority {
122 type Err = ();
123
124 fn from_str(s: &str) -> Result<Self, Self::Err> {
125 match s.trim().to_lowercase().as_str() {
126 "low" => Ok(Self::Low),
127 "normal" => Ok(Self::Normal),
128 "high" => Ok(Self::High),
129 _ => Err(()),
130 }
131 }
132}
133
134#[derive(Debug, Clone, Serialize, Deserialize)]
136pub struct SelectionRequest {
137 pub project: String,
139 #[serde(default, skip_serializing_if = "Option::is_none")]
141 pub command: Option<String>,
142 #[serde(default)]
144 pub command_priority: CommandPriority,
145 pub estimated_cores: u32,
147 #[serde(default)]
149 pub preferred_workers: Vec<WorkerId>,
150 #[serde(default)]
152 pub toolchain: Option<ToolchainInfo>,
153 #[serde(default)]
155 pub required_runtime: RequiredRuntime,
156 #[serde(default, skip_serializing_if = "Option::is_none")]
159 pub classification_duration_us: Option<u64>,
160 #[serde(default, skip_serializing_if = "std::ops::Not::not")]
166 pub job_mode: bool,
167 #[serde(default, skip_serializing_if = "Vec::is_empty")]
177 pub required_tools: Vec<String>,
178 #[serde(default, skip_serializing_if = "Option::is_none")]
180 pub hook_pid: Option<u32>,
181}
182
183#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
188#[serde(rename_all = "snake_case")]
189pub enum SelectionReason {
190 Success,
192 NoWorkersConfigured,
194 AllWorkersUnreachable,
196 AllCircuitsOpen,
198 AllWorkersBusy,
200 NoWorkersPassedHealth,
202 AllWorkersFailedPreflight,
204 AllWorkersFailedConvergence,
206 NoAdmissibleWorkers(String),
208 NoMatchingWorkers,
210 NoWorkersWithRuntime(String),
212 SelectionError(String),
214 AffinityPinned,
216 AffinityFallback,
218}
219
220pub const SELECTION_RESPONSE_PROTOCOL_VERSION: u64 = 1;
226
227impl std::fmt::Display for SelectionReason {
228 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
229 match self {
230 Self::Success => write!(f, "worker assigned successfully"),
231 Self::NoWorkersConfigured => write!(f, "no workers configured"),
232 Self::AllWorkersUnreachable => write!(f, "all workers unreachable"),
233 Self::AllCircuitsOpen => write!(f, "all worker circuits open"),
234 Self::AllWorkersBusy => write!(f, "all workers at capacity"),
235 Self::NoWorkersPassedHealth => write!(f, "no workers passed health thresholds"),
236 Self::AllWorkersFailedPreflight => write!(f, "all workers failed preflight checks"),
237 Self::AllWorkersFailedConvergence => {
238 write!(f, "all workers failed repo convergence checks")
239 }
240 Self::NoAdmissibleWorkers(summary) => {
241 write!(f, "no admissible workers: {}", summary)
242 }
243 Self::NoMatchingWorkers => write!(f, "no matching workers found"),
244 Self::NoWorkersWithRuntime(rt) => write!(f, "no workers with {} installed", rt),
245 Self::SelectionError(e) => write!(f, "selection error: {}", e),
246 Self::AffinityPinned => write!(f, "worker assigned via affinity pinning"),
247 Self::AffinityFallback => write!(f, "worker assigned via last-success fallback"),
248 }
249 }
250}
251
252#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
258#[serde(rename_all = "snake_case")]
259pub enum SelectionStrategy {
260 Priority,
263 Fastest,
266 #[default]
269 Balanced,
270 CacheAffinity,
273 FairFastest,
276}
277
278impl std::fmt::Display for SelectionStrategy {
279 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
280 let name = match self {
281 Self::Priority => "priority",
282 Self::Fastest => "fastest",
283 Self::Balanced => "balanced",
284 Self::CacheAffinity => "cache_affinity",
285 Self::FairFastest => "fair_fastest",
286 };
287 write!(f, "{}", name)
288 }
289}
290
291impl std::str::FromStr for SelectionStrategy {
292 type Err = String;
293
294 fn from_str(s: &str) -> Result<Self, Self::Err> {
295 match s.trim().to_lowercase().as_str() {
296 "priority" => Ok(Self::Priority),
297 "fastest" => Ok(Self::Fastest),
298 "balanced" => Ok(Self::Balanced),
299 "cache_affinity" | "cache-affinity" | "cacheaffinity" => Ok(Self::CacheAffinity),
300 "fair_fastest" | "fair-fastest" | "fairfastest" => Ok(Self::FairFastest),
301 _ => Err(format!(
302 "unknown selection strategy '{}', expected one of: priority, fastest, balanced, cache_affinity, fair_fastest",
303 s
304 )),
305 }
306 }
307}
308
309#[derive(Debug, Clone, Serialize, Deserialize)]
311pub struct SelectionConfig {
312 #[serde(default)]
314 pub strategy: SelectionStrategy,
315 #[serde(default = "default_min_success_rate")]
317 pub min_success_rate: f64,
318 #[serde(default)]
320 pub weights: SelectionWeightConfig,
321 #[serde(default)]
323 pub fairness: FairnessConfig,
324 #[serde(default)]
326 pub affinity: AffinityConfig,
327
328 #[serde(default = "default_max_load_per_core")]
332 pub max_load_per_core: Option<f64>,
333 #[serde(default = "default_min_free_gb")]
336 pub min_free_gb: Option<f64>,
337 #[serde(
340 default = "default_disk_gb_per_slot",
341 deserialize_with = "deserialize_disk_gb_per_slot"
342 )]
343 pub disk_gb_per_slot: f64,
344}
345
346impl Default for SelectionConfig {
347 fn default() -> Self {
348 Self {
349 strategy: SelectionStrategy::default(),
350 min_success_rate: default_min_success_rate(),
351 weights: SelectionWeightConfig::default(),
352 fairness: FairnessConfig::default(),
353 affinity: AffinityConfig::default(),
354 max_load_per_core: default_max_load_per_core(),
355 min_free_gb: default_min_free_gb(),
356 disk_gb_per_slot: default_disk_gb_per_slot(),
357 }
358 }
359}
360
361fn default_min_success_rate() -> f64 {
362 0.8
363}
364
365fn default_max_load_per_core() -> Option<f64> {
366 Some(2.0) }
368
369fn default_min_free_gb() -> Option<f64> {
370 Some(10.0) }
372
373fn default_disk_gb_per_slot() -> f64 {
374 10.0
375}
376
377fn deserialize_disk_gb_per_slot<'de, D>(deserializer: D) -> Result<f64, D::Error>
378where
379 D: serde::Deserializer<'de>,
380{
381 let budget = f64::deserialize(deserializer)?;
382 if budget.is_finite() && budget > 0.0 {
383 Ok(budget)
384 } else {
385 Err(serde::de::Error::custom(
386 "disk_gb_per_slot must be finite and greater than zero",
387 ))
388 }
389}
390
391#[derive(Debug, Clone, Serialize, Deserialize)]
393pub struct SelectionWeightConfig {
394 #[serde(default = "default_weight_speedscore")]
396 pub speedscore: f64,
397 #[serde(default = "default_weight_slots")]
399 pub slots: f64,
400 #[serde(default = "default_weight_health")]
402 pub health: f64,
403 #[serde(default = "default_weight_cache")]
405 pub cache: f64,
406 #[serde(default = "default_weight_network")]
408 pub network: f64,
409 #[serde(default = "default_weight_priority")]
411 pub priority: f64,
412 #[serde(
414 default = "default_weight_disk",
415 deserialize_with = "deserialize_weight_disk"
416 )]
417 pub disk: f64,
418 #[serde(default = "default_half_open_penalty")]
420 pub half_open_penalty: f64,
421}
422
423impl Default for SelectionWeightConfig {
424 fn default() -> Self {
425 Self {
426 speedscore: default_weight_speedscore(),
427 slots: default_weight_slots(),
428 health: default_weight_health(),
429 cache: default_weight_cache(),
430 network: default_weight_network(),
431 priority: default_weight_priority(),
432 disk: default_weight_disk(),
433 half_open_penalty: default_half_open_penalty(),
434 }
435 }
436}
437
438fn default_weight_speedscore() -> f64 {
439 0.5
440}
441fn default_weight_slots() -> f64 {
442 0.4
443}
444fn default_weight_health() -> f64 {
445 0.3
446}
447fn default_weight_cache() -> f64 {
448 0.2
449}
450fn default_weight_network() -> f64 {
451 0.1
452}
453fn default_weight_priority() -> f64 {
454 0.5
461}
462fn default_half_open_penalty() -> f64 {
463 0.5
464}
465
466fn default_weight_disk() -> f64 {
467 0.2
468}
469
470fn deserialize_weight_disk<'de, D>(deserializer: D) -> Result<f64, D::Error>
471where
472 D: serde::Deserializer<'de>,
473{
474 let weight = f64::deserialize(deserializer)?;
475 if weight.is_finite() && (0.0..=1.0).contains(&weight) {
476 Ok(weight)
477 } else {
478 Err(serde::de::Error::custom(
479 "disk weight must be finite and between zero and one",
480 ))
481 }
482}
483
484#[derive(Debug, Clone, Serialize, Deserialize)]
486pub struct FairnessConfig {
487 #[serde(default = "default_fairness_lookback_secs")]
489 pub lookback_secs: u64,
490 #[serde(default = "default_max_consecutive_selections")]
492 pub max_consecutive_selections: u32,
493}
494
495impl Default for FairnessConfig {
496 fn default() -> Self {
497 Self {
498 lookback_secs: default_fairness_lookback_secs(),
499 max_consecutive_selections: default_max_consecutive_selections(),
500 }
501 }
502}
503
504fn default_fairness_lookback_secs() -> u64 {
505 300 }
507
508fn default_max_consecutive_selections() -> u32 {
509 3
510}
511
512#[derive(Debug, Clone, Serialize, Deserialize)]
514pub struct AffinityConfig {
515 #[serde(default = "default_affinity_enabled")]
517 pub enabled: bool,
518 #[serde(default = "default_affinity_pin_minutes")]
521 pub pin_minutes: u64,
522 #[serde(default = "default_last_success_fallback")]
526 pub enable_last_success_fallback: bool,
527 #[serde(default = "default_fallback_min_success_rate")]
530 pub fallback_min_success_rate: f64,
531}
532
533impl Default for AffinityConfig {
534 fn default() -> Self {
535 Self {
536 enabled: default_affinity_enabled(),
537 pin_minutes: default_affinity_pin_minutes(),
538 enable_last_success_fallback: default_last_success_fallback(),
539 fallback_min_success_rate: default_fallback_min_success_rate(),
540 }
541 }
542}
543
544fn default_affinity_enabled() -> bool {
545 true
546}
547
548fn default_affinity_pin_minutes() -> u64 {
549 60 }
551
552fn default_last_success_fallback() -> bool {
553 true
554}
555
556fn default_fallback_min_success_rate() -> f64 {
557 0.5 }
559
560#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
562pub struct SelectedWorker {
563 pub id: WorkerId,
565 pub host: String,
567 pub user: String,
569 pub identity_file: String,
571 pub slots_available: u32,
573 pub speed_score: f64,
575 #[serde(default)]
581 pub declared_os: Option<String>,
582}
583
584#[derive(Debug, Clone, Serialize, Deserialize)]
586pub struct SelectionResponse {
587 pub worker: Option<SelectedWorker>,
589 pub reason: SelectionReason,
591 #[serde(default, skip_serializing_if = "Option::is_none")]
593 pub build_id: Option<u64>,
594 #[serde(default, skip_serializing_if = "Option::is_none")]
596 pub diagnostics: Option<SelectionDiagnostics>,
597}
598
599#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
601#[serde(rename_all = "snake_case")]
602pub enum WorkerSelectionDiagnosticDecision {
603 Allow,
605 FallbackCandidate,
607 Deny,
609}
610
611#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
613pub struct WorkerSelectionDiagnostic {
614 pub worker_id: WorkerId,
616 pub status: String,
618 pub circuit_state: String,
620 pub pressure_state: String,
622 pub pressure_reason_code: String,
624 #[serde(skip_serializing_if = "Option::is_none")]
626 pub success_rate: Option<f64>,
627 pub min_success_rate: f64,
629 pub fallback_min_success_rate: f64,
631 pub required_runtime: RequiredRuntime,
633 pub runtime_available: bool,
635 pub available_slots: u32,
637 pub total_slots: u32,
639 pub estimated_cores: u32,
641 pub active_project_excluded: bool,
643 pub final_decision: WorkerSelectionDiagnosticDecision,
645 pub final_reason: String,
647 pub reason_codes: Vec<String>,
649}
650
651#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
653pub struct SelectionDiagnostics {
654 pub required_runtime: RequiredRuntime,
656 pub estimated_cores: u32,
658 pub min_success_rate: f64,
660 pub fallback_min_success_rate: f64,
662 pub active_project_exclusion_count: usize,
664 pub workers: Vec<WorkerSelectionDiagnostic>,
666}
667
668#[derive(Debug, Clone, Serialize, Deserialize)]
670pub struct ReleaseRequest {
671 pub worker_id: WorkerId,
673 pub slots: u32,
675 #[serde(default, skip_serializing_if = "Option::is_none")]
677 pub build_id: Option<u64>,
678 #[serde(default, skip_serializing_if = "Option::is_none")]
680 pub local_wrapper_id: Option<String>,
681 #[serde(default, skip_serializing_if = "Option::is_none")]
683 pub exit_code: Option<i32>,
684 #[serde(default, skip_serializing_if = "Option::is_none")]
688 pub duration_ms: Option<u64>,
689 #[serde(default, skip_serializing_if = "Option::is_none")]
691 pub bytes_transferred: Option<u64>,
692 #[serde(default, skip_serializing_if = "Option::is_none")]
694 pub timing: Option<CommandTimingBreakdown>,
695}
696
697#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
699#[serde(rename_all = "snake_case")]
700pub enum BuildHeartbeatPhase {
701 SyncUp,
703 Execute,
705 SyncDown,
707 Finalize,
709}
710
711#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
713pub struct BuildHeartbeatRequest {
714 pub build_id: u64,
716 pub worker_id: WorkerId,
718 #[serde(default, skip_serializing_if = "Option::is_none")]
720 pub hook_pid: Option<u32>,
721 #[serde(default, skip_serializing_if = "Option::is_none")]
723 pub local_wrapper_id: Option<String>,
724 #[serde(default, skip_serializing_if = "Option::is_none")]
726 pub remote_pgid_file: Option<String>,
727 pub phase: BuildHeartbeatPhase,
729 #[serde(default, skip_serializing_if = "Option::is_none")]
731 pub detail: Option<String>,
732 #[serde(default, skip_serializing_if = "Option::is_none")]
737 pub progress_counter: Option<u64>,
738 #[serde(default, skip_serializing_if = "Option::is_none")]
740 pub progress_percent: Option<f64>,
741}
742
743#[derive(Debug, Clone, Serialize, Deserialize)]
745pub struct WorkerConfig {
746 pub id: WorkerId,
748 pub host: String,
750 pub user: String,
752 pub identity_file: String,
754 pub total_slots: u32,
756 #[serde(default = "default_priority")]
758 pub priority: u32,
759 #[serde(default)]
765 pub tags: Vec<String>,
766 #[serde(default, skip_serializing_if = "Vec::is_empty")]
775 pub tools: Vec<WorkerToolProbe>,
776}
777
778#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
784pub struct WorkerToolProbe {
785 pub name: String,
787 pub command: Vec<String>,
789}
790
791fn default_priority() -> u32 {
792 100
793}
794
795pub const OS_TAG_PREFIX: &str = "os:";
797
798#[must_use]
809pub fn declared_os(tags: &[String]) -> Option<String> {
810 tags.iter()
815 .filter_map(|tag| tag.strip_prefix(OS_TAG_PREFIX))
816 .map(|os| os.trim().to_ascii_lowercase())
817 .find(|os| !os.is_empty())
818}
819
820#[must_use]
822pub fn os_tag(os: &str) -> String {
823 format!("{OS_TAG_PREFIX}{}", os.trim().to_ascii_lowercase())
824}
825
826impl Default for WorkerConfig {
827 fn default() -> Self {
828 Self {
829 id: WorkerId::new("default-worker"),
830 host: "localhost".to_string(),
831 user: "user".to_string(),
832 identity_file: "~/.ssh/id_rsa".to_string(),
833 total_slots: 4,
834 priority: default_priority(),
835 tags: Vec::new(),
836 tools: Vec::new(),
837 }
838 }
839}
840
841#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)]
847pub struct WorkerCapabilities {
848 #[serde(default, skip_serializing_if = "Option::is_none")]
850 pub rustc_version: Option<String>,
851 #[serde(default, skip_serializing_if = "Vec::is_empty")]
854 pub rustup_toolchains: Vec<String>,
855 #[serde(default, skip_serializing_if = "Vec::is_empty")]
859 pub rustup_components: Vec<String>,
860 #[serde(default, skip_serializing_if = "Vec::is_empty")]
866 pub probe_warnings: Vec<String>,
867 #[serde(default, skip_serializing_if = "Option::is_none")]
869 pub bun_version: Option<String>,
870 #[serde(default, skip_serializing_if = "Option::is_none")]
872 pub node_version: Option<String>,
873 #[serde(default, skip_serializing_if = "Option::is_none")]
875 pub npm_version: Option<String>,
876 #[serde(default, skip_serializing_if = "Option::is_none")]
880 pub nix_version: Option<String>,
881 #[serde(default, skip_serializing_if = "Option::is_none")]
884 pub go_version: Option<String>,
885 #[serde(default, skip_serializing_if = "Option::is_none")]
889 pub zig_version: Option<String>,
890 #[serde(default, skip_serializing_if = "Option::is_none")]
894 pub cargo_zigbuild_version: Option<String>,
895 #[serde(default, skip_serializing_if = "Option::is_none")]
902 pub cpu_microarch_level: Option<u8>,
903 #[serde(default, skip_serializing_if = "Vec::is_empty")]
910 pub tools_present: Vec<String>,
911 #[serde(default, skip_serializing_if = "Vec::is_empty")]
917 pub tools_absent: Vec<String>,
918
919 #[serde(default, skip_serializing_if = "Option::is_none")]
922 pub num_cpus: Option<u32>,
923 #[serde(default, skip_serializing_if = "Option::is_none")]
925 pub load_avg_1: Option<f64>,
926 #[serde(default, skip_serializing_if = "Option::is_none")]
928 pub load_avg_5: Option<f64>,
929 #[serde(default, skip_serializing_if = "Option::is_none")]
931 pub load_avg_15: Option<f64>,
932 #[serde(default, skip_serializing_if = "Option::is_none")]
934 pub disk_free_gb: Option<f64>,
935 #[serde(default, skip_serializing_if = "Option::is_none")]
937 pub disk_total_gb: Option<f64>,
938 #[serde(default, skip_serializing_if = "Option::is_none")]
943 pub projects_root_ok: Option<bool>,
944 #[serde(default, skip_serializing_if = "Option::is_none")]
946 pub projects_root_issue: Option<String>,
947 #[serde(default, skip_serializing_if = "Option::is_none")]
949 pub projects_root_checked_at_unix_ms: Option<i64>,
950}
951
952impl WorkerCapabilities {
953 pub fn new() -> Self {
955 Self::default()
956 }
957
958 pub fn mock_with_rust() -> Self {
962 Self {
963 rustc_version: Some("1.85.0-nightly (mock)".to_string()),
964 projects_root_ok: Some(true),
965 ..Self::default()
966 }
967 }
968
969 pub fn is_pre_v3_x86(&self) -> bool {
974 matches!(self.cpu_microarch_level, Some(1 | 2))
975 }
976
977 pub fn has_bun(&self) -> bool {
979 self.bun_version.is_some()
980 }
981
982 pub fn has_node(&self) -> bool {
984 self.node_version.is_some()
985 }
986
987 pub fn has_rust(&self) -> bool {
989 self.rustc_version.is_some()
990 }
991
992 pub fn has_rustup_component(&self, toolchain: &str, component: &str) -> bool {
999 let owner_prefix = format!("{toolchain}-");
1000 self.rustup_components.iter().any(|fact| {
1001 fact.split_once(':').is_some_and(|(owner, name)| {
1002 name == component && (owner == toolchain || owner.starts_with(&owner_prefix))
1003 })
1004 })
1005 }
1006
1007 pub fn has_nix(&self) -> bool {
1009 self.nix_version.is_some()
1010 }
1011
1012 pub fn has_go(&self) -> bool {
1014 self.go_version.is_some()
1015 }
1016
1017 pub fn has_zig(&self) -> bool {
1027 self.zig_version.is_some() && self.cargo_zigbuild_version.is_some()
1028 }
1029
1030 pub fn load_per_core(&self) -> Option<f64> {
1033 match (self.load_avg_1, self.num_cpus) {
1034 (Some(load), Some(cpus)) if cpus > 0 => Some(load / cpus as f64),
1035 _ => None,
1036 }
1037 }
1038
1039 pub fn is_high_load(&self, max_load_per_core: f64) -> Option<bool> {
1042 self.load_per_core().map(|lpc| lpc > max_load_per_core)
1043 }
1044
1045 pub fn is_low_disk(&self, min_free_gb: f64) -> Option<bool> {
1048 self.disk_free_gb.map(|free| free < min_free_gb)
1049 }
1050
1051 pub fn is_topology_healthy(&self) -> Option<bool> {
1055 self.projects_root_ok
1056 }
1057}
1058
1059#[must_use]
1068pub fn x86_64_microarch_level_from_flags(flags: &str) -> u8 {
1069 let set: std::collections::HashSet<&str> = flags.split_whitespace().collect();
1070 let has_all = |required: &[&str]| required.iter().all(|flag| set.contains(flag));
1071 const V2: &[&str] = &["cx16", "lahf_lm", "popcnt", "sse4_1", "sse4_2", "ssse3"];
1073 const V3: &[&str] = &[
1075 "abm", "avx", "avx2", "bmi1", "bmi2", "f16c", "fma", "movbe", "xsave",
1076 ];
1077 const V4: &[&str] = &["avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl"];
1079 if !has_all(V2) {
1080 return 1;
1081 }
1082 if !has_all(V3) {
1083 return 2;
1084 }
1085 if !has_all(V4) {
1086 return 3;
1087 }
1088 4
1089}
1090
1091#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1104pub struct PathTopologyConfig {
1105 #[serde(default)]
1108 pub canonical_root: Option<String>,
1109
1110 #[serde(default)]
1113 pub alias_root: Option<String>,
1114}
1115
1116impl PathTopologyConfig {
1117 pub fn to_policy(&self) -> crate::path_topology::PathTopologyPolicy {
1123 let canonical = self
1124 .canonical_root
1125 .as_deref()
1126 .filter(|s| !s.is_empty())
1127 .map(|s| shellexpand::tilde(s).into_owned())
1128 .unwrap_or_else(|| crate::path_topology::DEFAULT_CANONICAL_PROJECT_ROOT.to_string());
1129 let alias = self
1130 .alias_root
1131 .as_deref()
1132 .filter(|s| !s.is_empty())
1133 .map(|s| shellexpand::tilde(s).into_owned())
1134 .unwrap_or_else(|| crate::path_topology::DEFAULT_ALIAS_PROJECT_ROOT.to_string());
1135 crate::path_topology::PathTopologyPolicy::new(
1136 PathBuf::from(canonical),
1137 PathBuf::from(alias),
1138 )
1139 }
1140}
1141
1142#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1144pub struct RchConfig {
1145 #[serde(default)]
1146 pub general: GeneralConfig,
1147 #[serde(default)]
1148 pub compilation: CompilationConfig,
1149 #[serde(default)]
1150 pub transfer: TransferConfig,
1151 #[serde(default)]
1152 pub environment: EnvironmentConfig,
1153 #[serde(default)]
1154 pub circuit: CircuitBreakerConfig,
1155 #[serde(default)]
1156 pub output: OutputConfig,
1157 #[serde(default)]
1158 pub self_healing: SelfHealingConfig,
1159 #[serde(default)]
1160 pub self_test: SelfTestConfig,
1161 #[serde(default)]
1163 pub selection: SelectionConfig,
1164 #[serde(default)]
1166 pub execution: ExecutionConfig,
1167 #[serde(default)]
1169 pub alerts: AlertsConfig,
1170 #[serde(default)]
1172 pub fleet: FleetConfig,
1173 #[serde(default)]
1175 pub path_topology: PathTopologyConfig,
1176 #[serde(default)]
1178 pub doctor: DoctorConfig,
1179 #[serde(default)]
1184 pub remediation: crate::remediation_config::RemediationConfig,
1185 #[serde(default)]
1188 pub layer0: Layer0Config,
1189 #[serde(default)]
1192 pub api: ApiConfig,
1193 #[serde(default)]
1195 pub dashboard: DashboardConfig,
1196 #[serde(default)]
1198 pub jobs: JobsConfig,
1199}
1200
1201#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1203pub struct JobsConfig {
1204 #[serde(default, skip_serializing_if = "Vec::is_empty")]
1214 pub required_tools: Vec<String>,
1215}
1216
1217#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1234#[serde(default)]
1235pub struct ApiConfig {
1236 pub bind: String,
1240 pub token: Option<String>,
1242 pub token_file: Option<String>,
1244 pub no_token: bool,
1248 pub allow_any_addr: bool,
1250}
1251
1252#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1254#[serde(default)]
1255pub struct DashboardConfig {
1256 pub url: Option<String>,
1258}
1259
1260#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
1266pub struct DoctorConfig {
1267 #[serde(default)]
1269 pub webhooks: DoctorWebhooksConfig,
1270}
1271
1272#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1278pub struct DoctorWebhooksConfig {
1279 #[serde(default)]
1281 pub endpoints: Vec<DoctorWebhookEndpoint>,
1282 #[serde(default = "default_webhook_max_inflight")]
1287 pub max_inflight: usize,
1288}
1289
1290impl Default for DoctorWebhooksConfig {
1294 fn default() -> Self {
1295 Self {
1296 endpoints: Vec::new(),
1297 max_inflight: default_webhook_max_inflight(),
1298 }
1299 }
1300}
1301
1302#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
1310pub struct DoctorWebhookEndpoint {
1311 pub name: String,
1313 pub url: String,
1315 #[serde(default)]
1317 pub format: DoctorWebhookFormat,
1318 #[serde(default)]
1323 pub on_transitions: Vec<String>,
1324 #[serde(default = "default_webhook_retry_max")]
1326 pub retry_max: u32,
1327 #[serde(default = "default_webhook_retry_backoff_ms")]
1329 pub retry_backoff_ms: u64,
1330 #[serde(default = "default_webhook_timeout_ms")]
1332 pub timeout_ms: u64,
1333 #[serde(default = "default_true")]
1335 pub enabled: bool,
1336 #[serde(default)]
1339 pub routing_key_env: Option<String>,
1340 #[serde(default)]
1343 pub bearer_token_env: Option<String>,
1344 #[serde(default)]
1348 pub signing_secret_env: Option<String>,
1349}
1350
1351#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
1353#[serde(rename_all = "snake_case")]
1354pub enum DoctorWebhookFormat {
1355 #[default]
1357 GenericJson,
1358 Slack,
1360 Pagerduty,
1362}
1363
1364fn default_webhook_max_inflight() -> usize {
1365 16
1366}
1367
1368fn default_webhook_retry_max() -> u32 {
1369 3
1370}
1371
1372fn default_webhook_retry_backoff_ms() -> u64 {
1373 200
1374}
1375
1376fn default_webhook_timeout_ms() -> u64 {
1377 5000
1378}
1379
1380#[derive(Debug, Clone, Serialize, Deserialize)]
1382pub struct AlertsConfig {
1383 #[serde(default = "default_true")]
1385 pub enabled: bool,
1386 #[serde(default = "default_alert_suppress_duplicates_secs")]
1388 pub suppress_duplicates_secs: u64,
1389 #[serde(default = "default_alert_cleared_retention_secs")]
1394 pub cleared_retention_secs: u64,
1395 #[serde(default)]
1397 pub webhook: Option<WebhookConfig>,
1398}
1399
1400#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1402pub struct WebhookConfig {
1403 pub url: Option<String>,
1405 #[serde(default)]
1407 pub secret: Option<String>,
1408 #[serde(default = "default_webhook_timeout_secs")]
1410 pub timeout_secs: u64,
1411 #[serde(default = "default_webhook_retry_count")]
1413 pub retry_count: u32,
1414 #[serde(default)]
1417 pub events: Vec<String>,
1418}
1419
1420fn default_webhook_timeout_secs() -> u64 {
1421 5
1422}
1423
1424fn default_webhook_retry_count() -> u32 {
1425 3
1426}
1427
1428fn default_alert_suppress_duplicates_secs() -> u64 {
1429 300
1430}
1431
1432fn default_alert_cleared_retention_secs() -> u64 {
1433 300
1434}
1435
1436impl Default for AlertsConfig {
1437 fn default() -> Self {
1438 Self {
1439 enabled: true,
1440 suppress_duplicates_secs: default_alert_suppress_duplicates_secs(),
1441 cleared_retention_secs: default_alert_cleared_retention_secs(),
1442 webhook: None,
1443 }
1444 }
1445}
1446
1447#[derive(Debug, Clone, Serialize, Deserialize)]
1452pub struct FleetConfig {
1453 #[serde(default = "default_fleet_ssh_connect_timeout_secs")]
1455 pub ssh_connect_timeout_secs: u64,
1456 #[serde(default = "default_fleet_ssh_command_timeout_secs")]
1458 pub ssh_command_timeout_secs: u64,
1459 #[serde(default = "default_fleet_min_disk_space_mb")]
1461 pub min_disk_space_mb: u64,
1462 #[serde(default = "default_fleet_max_load_average")]
1464 pub max_load_average: f64,
1465 #[serde(default = "default_fleet_max_concurrent_workers")]
1467 pub max_concurrent_workers: usize,
1468 #[serde(default = "default_fleet_retry_count")]
1470 pub retry_count: u32,
1471 #[serde(default = "default_fleet_retry_delay_ms")]
1473 pub retry_delay_ms: u64,
1474}
1475
1476fn default_fleet_ssh_connect_timeout_secs() -> u64 {
1477 10
1478}
1479
1480fn default_fleet_ssh_command_timeout_secs() -> u64 {
1481 30
1482}
1483
1484fn default_fleet_min_disk_space_mb() -> u64 {
1485 500
1486}
1487
1488fn default_fleet_max_load_average() -> f64 {
1489 10.0
1490}
1491
1492fn default_fleet_max_concurrent_workers() -> usize {
1493 10
1494}
1495
1496fn default_fleet_retry_count() -> u32 {
1497 2
1498}
1499
1500fn default_fleet_retry_delay_ms() -> u64 {
1501 1000
1502}
1503
1504impl Default for FleetConfig {
1505 fn default() -> Self {
1506 Self {
1507 ssh_connect_timeout_secs: default_fleet_ssh_connect_timeout_secs(),
1508 ssh_command_timeout_secs: default_fleet_ssh_command_timeout_secs(),
1509 min_disk_space_mb: default_fleet_min_disk_space_mb(),
1510 max_load_average: default_fleet_max_load_average(),
1511 max_concurrent_workers: default_fleet_max_concurrent_workers(),
1512 retry_count: default_fleet_retry_count(),
1513 retry_delay_ms: default_fleet_retry_delay_ms(),
1514 }
1515 }
1516}
1517
1518impl FleetConfig {
1519 pub fn ssh_connect_timeout(&self) -> std::time::Duration {
1521 std::time::Duration::from_secs(self.ssh_connect_timeout_secs)
1522 }
1523
1524 pub fn ssh_command_timeout(&self) -> std::time::Duration {
1526 std::time::Duration::from_secs(self.ssh_command_timeout_secs)
1527 }
1528
1529 pub fn retry_delay(&self) -> std::time::Duration {
1531 std::time::Duration::from_millis(self.retry_delay_ms)
1532 }
1533}
1534
1535#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
1538#[serde(rename_all = "lowercase")]
1539pub enum BoxRole {
1540 Dispatcher,
1545 Worker,
1548 #[default]
1550 Hybrid,
1551}
1552
1553impl BoxRole {
1554 #[must_use]
1556 pub fn as_str(self) -> &'static str {
1557 match self {
1558 BoxRole::Dispatcher => "dispatcher",
1559 BoxRole::Worker => "worker",
1560 BoxRole::Hybrid => "hybrid",
1561 }
1562 }
1563}
1564
1565#[derive(Debug, Clone, Serialize, Deserialize)]
1566pub struct GeneralConfig {
1567 #[serde(default = "default_true")]
1569 pub enabled: bool,
1570 #[serde(default)]
1573 pub role: BoxRole,
1574 #[serde(default)]
1578 pub force_local: bool,
1579 #[serde(default)]
1584 pub force_remote: bool,
1585 #[serde(default = "default_log_level")]
1587 pub log_level: String,
1588 #[serde(default = "default_socket_path")]
1590 pub socket_path: String,
1591}
1592
1593#[derive(Debug, Clone, Default, Serialize, Deserialize)]
1595pub struct EnvironmentConfig {
1596 #[serde(default)]
1598 pub allowlist: Vec<String>,
1599}
1600
1601impl Default for GeneralConfig {
1602 fn default() -> Self {
1603 Self {
1604 enabled: true,
1605 role: BoxRole::default(),
1606 force_local: false,
1607 force_remote: false,
1608 log_level: "info".to_string(),
1609 socket_path: default_socket_path(),
1610 }
1611 }
1612}
1613
1614#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, JsonSchema)]
1616#[serde(rename_all = "lowercase")]
1617pub enum OutputVisibility {
1618 #[default]
1619 None,
1620 Summary,
1621 Verbose,
1622}
1623
1624impl std::fmt::Display for OutputVisibility {
1625 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1626 let value = match self {
1627 OutputVisibility::None => "none",
1628 OutputVisibility::Summary => "summary",
1629 OutputVisibility::Verbose => "verbose",
1630 };
1631 write!(f, "{}", value)
1632 }
1633}
1634
1635impl std::str::FromStr for OutputVisibility {
1636 type Err = ();
1637
1638 fn from_str(s: &str) -> Result<Self, Self::Err> {
1639 match s.trim().to_lowercase().as_str() {
1640 "none" | "silent" | "quiet" => Ok(Self::None),
1641 "summary" | "short" => Ok(Self::Summary),
1642 "verbose" | "debug" => Ok(Self::Verbose),
1643 _ => Err(()),
1644 }
1645 }
1646}
1647
1648#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1653#[serde(rename_all = "lowercase")]
1654pub enum ColorMode {
1655 #[default]
1658 Always,
1659 Auto,
1661 Never,
1663}
1664
1665impl std::fmt::Display for ColorMode {
1666 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1667 let value = match self {
1668 ColorMode::Always => "always",
1669 ColorMode::Auto => "auto",
1670 ColorMode::Never => "never",
1671 };
1672 write!(f, "{}", value)
1673 }
1674}
1675
1676impl std::str::FromStr for ColorMode {
1677 type Err = ();
1678
1679 fn from_str(s: &str) -> Result<Self, Self::Err> {
1680 match s.trim().to_lowercase().as_str() {
1681 "always" | "force" | "yes" | "true" => Ok(Self::Always),
1682 "auto" | "detect" => Ok(Self::Auto),
1683 "never" | "none" | "no" | "false" => Ok(Self::Never),
1684 _ => Err(()),
1685 }
1686 }
1687}
1688
1689fn default_color_mode() -> ColorMode {
1690 ColorMode::default()
1691}
1692
1693#[derive(Debug, Clone, Serialize, Deserialize)]
1695pub struct OutputConfig {
1696 #[serde(default = "default_output_visibility")]
1698 pub visibility: OutputVisibility,
1699 #[serde(default)]
1701 pub first_run_complete: bool,
1702 #[serde(default = "default_color_mode")]
1705 pub color_mode: ColorMode,
1706}
1707
1708impl Default for OutputConfig {
1709 fn default() -> Self {
1710 Self {
1711 visibility: OutputVisibility::None,
1712 first_run_complete: false,
1713 color_mode: ColorMode::default(),
1714 }
1715 }
1716}
1717
1718fn default_autostart_cooldown_secs() -> u64 {
1723 30
1724}
1725
1726fn default_autostart_timeout_secs() -> u64 {
1727 3
1728}
1729
1730#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
1736#[serde(rename_all = "lowercase")]
1737pub enum SelfHealingLogLevel {
1738 Debug,
1739 #[default]
1740 Info,
1741 Warn,
1742 Error,
1743}
1744
1745impl SelfHealingLogLevel {
1746 pub fn from_env_str(s: &str) -> Option<Self> {
1748 match s.trim().to_ascii_lowercase().as_str() {
1749 "debug" => Some(Self::Debug),
1750 "info" => Some(Self::Info),
1751 "warn" | "warning" => Some(Self::Warn),
1752 "error" => Some(Self::Error),
1753 _ => None,
1754 }
1755 }
1756}
1757
1758#[derive(Debug, Clone, Serialize, Deserialize)]
1760pub struct SelfHealingConfig {
1761 #[serde(default = "default_true")]
1763 pub hook_starts_daemon: bool,
1764 #[serde(default = "default_true")]
1766 pub daemon_installs_hooks: bool,
1767 #[serde(default = "default_autostart_cooldown_secs")]
1769 pub auto_start_cooldown_secs: u64,
1770 #[serde(
1773 default = "default_autostart_timeout_secs",
1774 alias = "daemon_start_timeout"
1775 )]
1776 pub auto_start_timeout_secs: u64,
1777 #[serde(default)]
1780 pub self_healing_log_level: SelfHealingLogLevel,
1781}
1782
1783impl Default for SelfHealingConfig {
1784 fn default() -> Self {
1785 Self {
1786 hook_starts_daemon: default_true(),
1787 daemon_installs_hooks: default_true(),
1788 auto_start_cooldown_secs: default_autostart_cooldown_secs(),
1789 auto_start_timeout_secs: default_autostart_timeout_secs(),
1790 self_healing_log_level: SelfHealingLogLevel::default(),
1791 }
1792 }
1793}
1794
1795impl SelfHealingConfig {
1796 pub fn with_env_overrides(mut self) -> Self {
1805 if let Ok(val) = std::env::var("RCH_NO_SELF_HEALING")
1807 && (val == "1" || val.eq_ignore_ascii_case("true"))
1808 {
1809 self.hook_starts_daemon = false;
1810 self.daemon_installs_hooks = false;
1811 return self;
1812 }
1813
1814 if let Ok(val) = std::env::var("RCH_HOOK_STARTS_DAEMON") {
1816 self.hook_starts_daemon = val != "0" && !val.eq_ignore_ascii_case("false");
1817 }
1818 if let Ok(val) = std::env::var("RCH_DAEMON_INSTALLS_HOOKS") {
1819 self.daemon_installs_hooks = val != "0" && !val.eq_ignore_ascii_case("false");
1820 }
1821
1822 if let Ok(val) = std::env::var("RCH_AUTO_START_TIMEOUT_SECS")
1824 && let Ok(secs) = val.parse()
1825 {
1826 self.auto_start_timeout_secs = secs;
1827 }
1828 if let Ok(val) = std::env::var("RCH_AUTO_START_COOLDOWN_SECS")
1829 && let Ok(secs) = val.parse()
1830 {
1831 self.auto_start_cooldown_secs = secs;
1832 }
1833 if let Ok(val) = std::env::var("RCH_SELF_HEALING_LOG_LEVEL") {
1835 if let Some(level) = SelfHealingLogLevel::from_env_str(&val) {
1836 self.self_healing_log_level = level;
1837 } else {
1838 tracing::warn!(
1839 target: "rch::self_healing",
1840 invalid = %val,
1841 "RCH_SELF_HEALING_LOG_LEVEL has invalid value; using default=info"
1842 );
1843 }
1844 }
1845
1846 self
1847 }
1848}
1849
1850#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
1856#[serde(rename_all = "snake_case")]
1857pub enum SelfTestFailureAction {
1858 #[default]
1860 Alert,
1861 DisableWorker,
1863 AlertAndDisable,
1865}
1866
1867#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1869#[serde(untagged)]
1870pub enum SelfTestWorkers {
1871 All(String),
1873 List(Vec<WorkerId>),
1875}
1876
1877impl Default for SelfTestWorkers {
1878 fn default() -> Self {
1879 Self::All("all".to_string())
1880 }
1881}
1882
1883impl SelfTestWorkers {
1884 pub fn resolve(&self) -> Option<Vec<WorkerId>> {
1886 match self {
1887 SelfTestWorkers::All(value) => {
1888 if value.eq_ignore_ascii_case("all") {
1889 None
1890 } else {
1891 Some(vec![WorkerId::new(value.clone())])
1892 }
1893 }
1894 SelfTestWorkers::List(list) => Some(list.clone()),
1895 }
1896 }
1897}
1898
1899fn default_self_test_retry_count() -> u32 {
1900 3
1901}
1902
1903fn default_self_test_retry_delay() -> String {
1904 "5m".to_string()
1905}
1906
1907fn default_self_test_enabled() -> bool {
1908 false
1909}
1910
1911#[derive(Debug, Clone, Serialize, Deserialize)]
1913pub struct SelfTestConfig {
1914 #[serde(default = "default_self_test_enabled")]
1916 pub enabled: bool,
1917 #[serde(default)]
1919 pub schedule: Option<String>,
1920 #[serde(default)]
1922 pub interval: Option<String>,
1923 #[serde(default)]
1925 pub workers: SelfTestWorkers,
1926 #[serde(default)]
1928 pub on_failure: SelfTestFailureAction,
1929 #[serde(default = "default_self_test_retry_count")]
1931 pub retry_count: u32,
1932 #[serde(default = "default_self_test_retry_delay")]
1934 pub retry_delay: String,
1935}
1936
1937impl Default for SelfTestConfig {
1938 fn default() -> Self {
1939 Self {
1940 enabled: default_self_test_enabled(),
1941 schedule: None,
1942 interval: None,
1943 workers: SelfTestWorkers::default(),
1944 on_failure: SelfTestFailureAction::default(),
1945 retry_count: default_self_test_retry_count(),
1946 retry_delay: default_self_test_retry_delay(),
1947 }
1948 }
1949}
1950
1951#[derive(Debug, Clone, Serialize, Deserialize)]
1953pub struct CircuitBreakerConfig {
1954 #[serde(default = "default_circuit_failure_threshold")]
1956 pub failure_threshold: u32,
1957 #[serde(default = "default_circuit_success_threshold")]
1959 pub success_threshold: u32,
1960 #[serde(default = "default_circuit_error_rate_threshold")]
1962 pub error_rate_threshold: f64,
1963 #[serde(default = "default_circuit_window_secs")]
1965 pub window_secs: u64,
1966 #[serde(default = "default_circuit_open_cooldown_secs")]
1968 pub open_cooldown_secs: u64,
1969 #[serde(default = "default_circuit_half_open_max_probes")]
1971 pub half_open_max_probes: u32,
1972 #[serde(default = "default_circuit_command_failure_threshold")]
1983 pub command_failure_threshold: u32,
1984 #[serde(default = "default_circuit_open_cooldown_max_secs")]
1992 pub open_cooldown_max_secs: u64,
1993}
1994
1995impl Default for CircuitBreakerConfig {
1996 fn default() -> Self {
1997 Self {
1998 failure_threshold: default_circuit_failure_threshold(),
1999 success_threshold: default_circuit_success_threshold(),
2000 error_rate_threshold: default_circuit_error_rate_threshold(),
2001 window_secs: default_circuit_window_secs(),
2002 open_cooldown_secs: default_circuit_open_cooldown_secs(),
2003 half_open_max_probes: default_circuit_half_open_max_probes(),
2004 command_failure_threshold: default_circuit_command_failure_threshold(),
2005 open_cooldown_max_secs: default_circuit_open_cooldown_max_secs(),
2006 }
2007 }
2008}
2009
2010impl CircuitBreakerConfig {
2011 pub fn cooldown_ms_for_streak(&self, reopen_streak: u32) -> u64 {
2015 let base_ms = self.open_cooldown_secs.saturating_mul(1000);
2016 let cap_ms = self
2017 .open_cooldown_max_secs
2018 .saturating_mul(1000)
2019 .max(base_ms);
2020 let exponent = reopen_streak.saturating_sub(1).min(16);
2022 base_ms.saturating_mul(1u64 << exponent).min(cap_ms)
2023 }
2024}
2025
2026#[derive(Debug, Clone, Default)]
2031pub struct CircuitStats {
2032 state: CircuitState,
2034 consecutive_failures: u32,
2036 consecutive_successes: u32,
2038 window_successes: u32,
2040 window_failures: u32,
2042 opened_at: Option<u64>,
2044 last_state_change: u64,
2046 active_probes: u32,
2048 consecutive_command_failures: u32,
2053 reopen_streak: u32,
2056 recent_results: Vec<bool>,
2059}
2060
2061const CIRCUIT_HISTORY_SIZE: usize = 10;
2063
2064impl CircuitStats {
2065 pub fn new() -> Self {
2067 Self {
2068 state: CircuitState::Closed,
2069 consecutive_failures: 0,
2070 consecutive_successes: 0,
2071 window_successes: 0,
2072 window_failures: 0,
2073 opened_at: None,
2074 last_state_change: Self::now_millis(),
2075 active_probes: 0,
2076 consecutive_command_failures: 0,
2077 reopen_streak: 0,
2078 recent_results: Vec::with_capacity(CIRCUIT_HISTORY_SIZE),
2079 }
2080 }
2081
2082 pub fn consecutive_command_failures(&self) -> u32 {
2085 self.consecutive_command_failures
2086 }
2087
2088 pub fn reopen_streak(&self) -> u32 {
2091 self.reopen_streak
2092 }
2093
2094 fn effective_cooldown_ms(&self, config: &CircuitBreakerConfig) -> u64 {
2096 config.cooldown_ms_for_streak(self.reopen_streak.max(1))
2097 }
2098
2099 pub fn state(&self) -> CircuitState {
2101 self.state
2102 }
2103
2104 pub fn opened_at(&self) -> Option<u64> {
2106 self.opened_at
2107 }
2108
2109 pub fn last_state_change(&self) -> u64 {
2111 self.last_state_change
2112 }
2113
2114 pub fn consecutive_failures(&self) -> u32 {
2116 self.consecutive_failures
2117 }
2118
2119 pub fn error_rate(&self) -> f64 {
2129 if self.recent_results.is_empty() {
2130 return 0.0;
2131 }
2132 let failures = self.recent_results.iter().filter(|&&ok| !ok).count();
2133 failures as f64 / self.recent_results.len() as f64
2134 }
2135
2136 pub fn record_success(&mut self) {
2141 self.window_successes += 1;
2142 self.consecutive_failures = 0;
2143 self.push_result(true);
2144
2145 if self.state == CircuitState::HalfOpen {
2146 self.consecutive_successes += 1;
2147 if self.active_probes > 0 {
2148 self.active_probes -= 1;
2149 }
2150 }
2151 }
2152
2153 pub fn record_failure(&mut self) {
2158 self.window_failures += 1;
2159 self.consecutive_failures += 1;
2160 self.push_result(false);
2161
2162 if self.state == CircuitState::HalfOpen {
2163 self.consecutive_successes = 0;
2164 if self.active_probes > 0 {
2165 self.active_probes -= 1;
2166 }
2167 }
2168 }
2169
2170 pub fn should_open(&self, config: &CircuitBreakerConfig) -> bool {
2176 if self.state != CircuitState::Closed {
2177 return false;
2178 }
2179
2180 if self.consecutive_failures >= config.failure_threshold {
2182 return true;
2183 }
2184
2185 if self.recent_results.len() >= 5 && self.error_rate() >= config.error_rate_threshold {
2189 return true;
2190 }
2191
2192 false
2193 }
2194
2195 pub fn should_half_open(&self, config: &CircuitBreakerConfig) -> bool {
2199 if self.state != CircuitState::Open {
2200 return false;
2201 }
2202
2203 let now = Self::now_millis();
2204 let cooldown_ms = self.effective_cooldown_ms(config);
2205
2206 if let Some(opened_at) = self.opened_at {
2207 now.saturating_sub(opened_at) >= cooldown_ms
2208 } else {
2209 false
2210 }
2211 }
2212
2213 pub fn should_close(&self, config: &CircuitBreakerConfig) -> bool {
2222 if self.state != CircuitState::HalfOpen {
2223 return false;
2224 }
2225
2226 self.consecutive_successes >= config.success_threshold
2227 && self.consecutive_command_failures == 0
2228 }
2229
2230 pub fn can_probe(&self, config: &CircuitBreakerConfig) -> bool {
2234 if self.state != CircuitState::HalfOpen {
2235 return false;
2236 }
2237
2238 self.active_probes < config.half_open_max_probes
2239 }
2240
2241 pub fn start_probe(&mut self, config: &CircuitBreakerConfig) -> bool {
2245 if !self.can_probe(config) {
2246 return false;
2247 }
2248 self.active_probes += 1;
2249 true
2250 }
2251
2252 pub fn open(&mut self) {
2254 if self.state != CircuitState::Open {
2255 self.state = CircuitState::Open;
2256 self.opened_at = Some(Self::now_millis());
2257 self.last_state_change = Self::now_millis();
2258 self.consecutive_successes = 0;
2259 self.active_probes = 0;
2260 self.reopen_streak = self.reopen_streak.saturating_add(1);
2263 }
2264 }
2265
2266 pub fn half_open(&mut self) {
2268 if self.state != CircuitState::HalfOpen {
2269 self.state = CircuitState::HalfOpen;
2270 self.last_state_change = Self::now_millis();
2271 self.consecutive_successes = 0;
2272 self.active_probes = 0;
2273 }
2274 }
2275
2276 pub fn close(&mut self) {
2278 if self.state != CircuitState::Closed {
2279 self.state = CircuitState::Closed;
2280 self.last_state_change = Self::now_millis();
2281 self.opened_at = None;
2282 self.consecutive_failures = 0;
2283 self.consecutive_successes = 0;
2284 self.active_probes = 0;
2285 self.consecutive_command_failures = 0;
2286 self.reopen_streak = 0;
2287 self.window_successes = 0;
2293 self.window_failures = 0;
2294 self.recent_results.clear();
2295 }
2296 }
2297
2298 pub fn reset_window(&mut self) {
2304 self.window_successes = 0;
2305 self.window_failures = 0;
2306 self.recent_results.clear();
2307 }
2308
2309 pub fn recent_results(&self) -> &[bool] {
2314 &self.recent_results
2315 }
2316
2317 pub fn recovery_remaining_secs(&self, config: &CircuitBreakerConfig) -> Option<u64> {
2321 if self.state != CircuitState::Open {
2322 return None;
2323 }
2324
2325 let now = Self::now_millis();
2326 let cooldown_ms = self.effective_cooldown_ms(config);
2327
2328 if let Some(opened_at) = self.opened_at {
2329 let elapsed_ms = now.saturating_sub(opened_at);
2330 if elapsed_ms < cooldown_ms {
2331 return Some((cooldown_ms - elapsed_ms) / 1000);
2332 }
2333 }
2334 None
2335 }
2336
2337 fn push_result(&mut self, success: bool) {
2339 if self.recent_results.len() >= CIRCUIT_HISTORY_SIZE {
2340 self.recent_results.remove(0);
2341 }
2342 self.recent_results.push(success);
2343 }
2344
2345 fn now_millis() -> u64 {
2346 std::time::SystemTime::now()
2347 .duration_since(std::time::UNIX_EPOCH)
2348 .map(|d| d.as_millis() as u64)
2349 .unwrap_or(0)
2350 }
2351
2352 pub fn apply_health_outcome(
2378 &mut self,
2379 healthy: bool,
2380 transient: bool,
2381 config: &CircuitBreakerConfig,
2382 ) -> CircuitState {
2383 if healthy {
2384 if self.state == CircuitState::Open && self.should_half_open(config) {
2388 self.half_open();
2389 }
2390 self.record_success();
2391 if self.should_close(config) {
2392 self.close();
2393 }
2394 } else {
2395 let was_half_open = self.state == CircuitState::HalfOpen;
2396 self.record_failure();
2397 if self.should_open(config) {
2398 self.open();
2399 } else if was_half_open && !transient {
2400 self.open();
2404 }
2405 }
2406
2407 if self.state == CircuitState::Open && self.should_half_open(config) {
2410 self.half_open();
2411 }
2412
2413 self.state
2414 }
2415
2416 pub fn apply_command_outcome(
2440 &mut self,
2441 success: bool,
2442 config: &CircuitBreakerConfig,
2443 ) -> CircuitState {
2444 if success {
2445 self.consecutive_command_failures = 0;
2446 return self.apply_health_outcome(true, false, config);
2447 }
2448
2449 self.consecutive_command_failures = self.consecutive_command_failures.saturating_add(1);
2450 let was_half_open = self.state == CircuitState::HalfOpen;
2451 self.record_failure();
2452 let threshold = config.command_failure_threshold.max(1);
2453 if was_half_open
2454 || self.should_open(config)
2455 || (self.state == CircuitState::Closed
2456 && self.consecutive_command_failures >= threshold)
2457 {
2458 self.open();
2459 }
2460
2461 if self.state == CircuitState::Open && self.should_half_open(config) {
2462 self.half_open();
2463 }
2464
2465 self.state
2466 }
2467}
2468
2469#[derive(Debug, Clone, Serialize, Deserialize)]
2470pub struct CompilationConfig {
2471 #[serde(default = "default_confidence")]
2473 pub confidence_threshold: f64,
2474 #[serde(default = "default_min_local_time")]
2476 pub min_local_time_ms: u64,
2477 #[serde(default = "default_speedup_threshold")]
2481 pub remote_speedup_threshold: f64,
2482 #[serde(default = "default_build_slots")]
2484 pub build_slots: u32,
2485 #[serde(default = "default_test_slots")]
2488 pub test_slots: u32,
2489 #[serde(default = "default_check_slots")]
2492 pub check_slots: u32,
2493 #[serde(default = "default_build_timeout")]
2496 pub build_timeout_sec: u64,
2497 #[serde(default = "default_test_timeout")]
2500 pub test_timeout_sec: u64,
2501 #[serde(default = "default_bun_timeout")]
2505 pub bun_timeout_sec: u64,
2506 #[serde(default = "default_external_timeout_enabled")]
2510 pub external_timeout_enabled: bool,
2511 #[serde(default = "default_allow_local_fallback")]
2524 pub allow_local_fallback: bool,
2525 #[serde(default)]
2536 pub remote_build_jobs: RemoteBuildJobs,
2537}
2538
2539#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
2551pub enum RemoteBuildJobs {
2552 #[default]
2554 Auto,
2555 Off,
2557 Fixed(u32),
2559}
2560
2561impl RemoteBuildJobs {
2562 pub fn parse(value: &str) -> Result<Self, String> {
2564 let trimmed = value.trim();
2565 match trimmed.to_ascii_lowercase().as_str() {
2566 "auto" => Ok(Self::Auto),
2567 "off" | "none" | "disabled" => Ok(Self::Off),
2568 _ => match trimmed.parse::<u32>() {
2569 Ok(0) => Err(
2570 "remote_build_jobs must be >= 1 (cargo rejects `-j 0`); use `off` to disable"
2571 .to_string(),
2572 ),
2573 Ok(n) => Ok(Self::Fixed(n)),
2574 Err(_) => Err(format!(
2575 "'{trimmed}' is not a valid remote_build_jobs value (expected `auto`, `off`, or a positive integer)"
2576 )),
2577 },
2578 }
2579 }
2580}
2581
2582impl std::fmt::Display for RemoteBuildJobs {
2583 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2584 match self {
2585 Self::Auto => f.write_str("auto"),
2586 Self::Off => f.write_str("off"),
2587 Self::Fixed(n) => write!(f, "{n}"),
2588 }
2589 }
2590}
2591
2592impl std::str::FromStr for RemoteBuildJobs {
2593 type Err = String;
2594
2595 fn from_str(value: &str) -> Result<Self, Self::Err> {
2596 Self::parse(value)
2597 }
2598}
2599
2600impl Serialize for RemoteBuildJobs {
2601 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
2602 match self {
2603 Self::Auto => serializer.serialize_str("auto"),
2604 Self::Off => serializer.serialize_str("off"),
2605 Self::Fixed(n) => serializer.serialize_u32(*n),
2606 }
2607 }
2608}
2609
2610impl<'de> Deserialize<'de> for RemoteBuildJobs {
2611 fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
2612 #[derive(Deserialize)]
2613 #[serde(untagged)]
2614 enum Raw {
2615 Count(u64),
2616 Spelling(String),
2617 }
2618 match Raw::deserialize(deserializer)? {
2619 Raw::Count(n) => {
2620 let n = u32::try_from(n).map_err(|_| {
2621 serde::de::Error::custom(format!("remote_build_jobs {n} is out of range"))
2622 })?;
2623 Self::parse(&n.to_string()).map_err(serde::de::Error::custom)
2624 }
2625 Raw::Spelling(s) => Self::parse(&s).map_err(serde::de::Error::custom),
2626 }
2627 }
2628}
2629
2630impl Default for CompilationConfig {
2631 fn default() -> Self {
2632 Self {
2633 confidence_threshold: 0.85,
2634 min_local_time_ms: 2000,
2635 remote_speedup_threshold: default_speedup_threshold(),
2636 build_slots: default_build_slots(),
2637 test_slots: default_test_slots(),
2638 check_slots: default_check_slots(),
2639 build_timeout_sec: default_build_timeout(),
2640 test_timeout_sec: default_test_timeout(),
2641 bun_timeout_sec: default_bun_timeout(),
2642 external_timeout_enabled: default_external_timeout_enabled(),
2643 allow_local_fallback: default_allow_local_fallback(),
2644 remote_build_jobs: RemoteBuildJobs::default(),
2645 }
2646 }
2647}
2648
2649fn default_build_slots() -> u32 {
2650 4
2651}
2652
2653fn default_test_slots() -> u32 {
2654 8
2655}
2656
2657fn default_check_slots() -> u32 {
2658 2
2659}
2660
2661fn default_build_timeout() -> u64 {
2664 300
2665}
2666
2667fn default_test_timeout() -> u64 {
2670 1800
2671}
2672
2673fn default_bun_timeout() -> u64 {
2677 600
2678}
2679
2680fn default_external_timeout_enabled() -> bool {
2682 true
2683}
2684
2685fn default_allow_local_fallback() -> bool {
2689 true
2690}
2691
2692impl CompilationConfig {
2693 pub fn timeout_for_kind(&self, kind: Option<CompilationKind>) -> std::time::Duration {
2699 let secs = match kind {
2700 Some(CompilationKind::BunTest) | Some(CompilationKind::BunTypecheck) => {
2702 self.bun_timeout_sec
2703 }
2704 Some(CompilationKind::CargoTest) | Some(CompilationKind::CargoNextest) => {
2706 self.test_timeout_sec
2707 }
2708 Some(CompilationKind::NixBuild) => self.test_timeout_sec,
2712 Some(CompilationKind::GoTest) => self.test_timeout_sec,
2715 _ => self.build_timeout_sec,
2717 };
2718 std::time::Duration::from_secs(secs)
2719 }
2720
2721 pub fn external_timeout_enabled(&self) -> bool {
2723 self.external_timeout_enabled
2724 }
2725}
2726
2727pub const LAYER0_PACK_VERSION: &str = "1";
2733
2734#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
2743#[serde(default)]
2744pub struct Layer0Config {
2745 pub enabled: bool,
2747 pub release_lto_thin: bool,
2749 pub release_codegen_units_1: bool,
2752}
2753
2754impl Layer0Config {
2755 #[must_use]
2761 pub fn active_env(&self) -> Vec<(&'static str, &'static str)> {
2762 if !self.enabled {
2763 return Vec::new();
2764 }
2765 let mut env = Vec::new();
2766 if self.release_lto_thin {
2767 env.push(("CARGO_PROFILE_RELEASE_LTO", "thin"));
2768 }
2769 if self.release_codegen_units_1 {
2770 env.push(("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"));
2771 }
2772 env
2773 }
2774}
2775
2776#[derive(Debug, Clone, Serialize, Deserialize)]
2777pub struct TransferConfig {
2778 #[serde(default = "default_compression")]
2780 pub compression_level: u32,
2781 #[serde(default = "default_excludes")]
2783 pub exclude_patterns: Vec<String>,
2784 #[serde(default, skip_serializing_if = "Option::is_none")]
2788 pub ssh_server_alive_interval_secs: Option<u64>,
2789 #[serde(default, skip_serializing_if = "Option::is_none")]
2794 pub ssh_control_persist_secs: Option<u64>,
2795 #[serde(default = "default_remote_base")]
2806 pub remote_base: String,
2807 #[serde(default, skip_serializing_if = "Option::is_none")]
2813 pub sync_timeout_ms: Option<u64>,
2814 #[serde(default = "default_source_sync_silence_timeout_secs")]
2826 pub source_sync_silence_timeout_secs: u64,
2827 #[serde(default)]
2829 pub retry: RetryConfig,
2830 #[serde(default)]
2836 pub verify_artifacts: bool,
2837 #[serde(default = "default_verify_max_size")]
2842 pub verify_max_size_bytes: u64,
2843
2844 #[serde(default, skip_serializing_if = "Option::is_none")]
2853 pub max_transfer_mb: Option<u64>,
2854
2855 #[serde(default, skip_serializing_if = "Option::is_none")]
2861 pub max_transfer_time_ms: Option<u64>,
2862
2863 #[serde(default, skip_serializing_if = "Option::is_none")]
2868 pub bwlimit_kbps: Option<u64>,
2869
2870 #[serde(default, skip_serializing_if = "Option::is_none")]
2879 pub rsync_bin: Option<String>,
2880
2881 #[serde(default, skip_serializing_if = "Option::is_none")]
2886 pub estimated_bandwidth_bps: Option<u64>,
2887
2888 #[serde(default)]
2901 pub adaptive_compression: bool,
2902
2903 #[serde(default = "default_min_compression")]
2908 pub min_compression_level: u32,
2909
2910 #[serde(default = "default_max_compression")]
2915 pub max_compression_level: u32,
2916}
2917
2918impl Default for TransferConfig {
2919 fn default() -> Self {
2920 Self {
2921 compression_level: 3,
2922 exclude_patterns: default_excludes(),
2923 ssh_server_alive_interval_secs: None,
2924 ssh_control_persist_secs: None,
2925 remote_base: default_remote_base(),
2926 sync_timeout_ms: None,
2927 source_sync_silence_timeout_secs: default_source_sync_silence_timeout_secs(),
2928 retry: RetryConfig::default(),
2929 verify_artifacts: false,
2930 verify_max_size_bytes: default_verify_max_size(),
2931 max_transfer_mb: None,
2933 max_transfer_time_ms: None,
2934 bwlimit_kbps: None,
2935 rsync_bin: None,
2936 estimated_bandwidth_bps: None,
2937 adaptive_compression: false,
2939 min_compression_level: default_min_compression(),
2940 max_compression_level: default_max_compression(),
2941 }
2942 }
2943}
2944
2945impl TransferConfig {
2946 pub const MIN_SYNC_TIMEOUT_MS: u64 = 1_000;
2948 pub const MAX_SYNC_TIMEOUT_MS: u64 = 3_600_000;
2950 pub const DEFAULT_SYNC_TIMEOUT_FLOOR_MS: u64 = 30_000;
2952 const DEFAULT_SYNC_BYTES_PER_SECOND: u64 = 1024 * 1024;
2953
2954 #[must_use]
2956 pub fn valid_sync_timeout_ms(value: u64) -> bool {
2957 (Self::MIN_SYNC_TIMEOUT_MS..=Self::MAX_SYNC_TIMEOUT_MS).contains(&value)
2958 }
2959
2960 #[must_use]
2967 pub fn sync_timeout_for_payload(&self, payload_bytes: u64) -> std::time::Duration {
2968 let timeout_ms = self
2969 .sync_timeout_ms
2970 .filter(|value| Self::valid_sync_timeout_ms(*value))
2971 .unwrap_or_else(|| {
2972 let transfer_seconds = payload_bytes
2973 .saturating_add(Self::DEFAULT_SYNC_BYTES_PER_SECOND - 1)
2974 / Self::DEFAULT_SYNC_BYTES_PER_SECOND;
2975 Self::DEFAULT_SYNC_TIMEOUT_FLOOR_MS
2976 .saturating_add(transfer_seconds.saturating_mul(1_000))
2977 .min(Self::MAX_SYNC_TIMEOUT_MS)
2978 });
2979 std::time::Duration::from_millis(timeout_ms)
2980 }
2981
2982 pub fn select_compression_level(&self, estimated_bytes: Option<u64>) -> u32 {
2994 if !self.adaptive_compression {
2995 return self.compression_level;
2996 }
2997
2998 let Some(bytes) = estimated_bytes else {
2999 return self.compression_level;
3001 };
3002
3003 const SMALL_THRESHOLD: u64 = 10_000_000; const LARGE_THRESHOLD: u64 = 200_000_000; let level = if bytes < SMALL_THRESHOLD {
3008 1 } else if bytes < LARGE_THRESHOLD {
3010 3 } else {
3012 7 };
3014
3015 level.clamp(self.min_compression_level, self.max_compression_level)
3017 }
3018}
3019
3020#[rustfmt::skip]
3034pub const BUILTIN_EXECUTION_BASES: &[&str] = &[
3035 "cargo",
3037 "rustc",
3038 "nextest",
3040 "gcc",
3042 "g++",
3043 "clang",
3044 "clang++",
3045 "cc",
3046 "c++",
3047 "make",
3049 "cmake",
3050 "ninja",
3051 "meson",
3052 "bun",
3054 "nix",
3056 "go",
3058 "tsc",
3060];
3061
3062fn default_execution_allowlist() -> Vec<String> {
3063 BUILTIN_EXECUTION_BASES
3064 .iter()
3065 .map(|base| (*base).to_string())
3066 .collect()
3067}
3068
3069#[derive(Debug, Clone, Serialize, Deserialize)]
3074pub struct ExecutionConfig {
3075 #[serde(default = "default_execution_allowlist")]
3084 pub allowlist: Vec<String>,
3085}
3086
3087impl Default for ExecutionConfig {
3088 fn default() -> Self {
3089 Self {
3090 allowlist: default_execution_allowlist(),
3091 }
3092 }
3093}
3094
3095impl ExecutionConfig {
3096 pub fn is_allowed(&self, command_base: &str) -> bool {
3098 self.allowlist
3099 .iter()
3100 .any(|allowed| allowed.eq_ignore_ascii_case(command_base))
3101 }
3102
3103 pub fn missing_builtin_bases(&self) -> Vec<&'static str> {
3109 BUILTIN_EXECUTION_BASES
3110 .iter()
3111 .copied()
3112 .filter(|base| !self.is_allowed(base))
3113 .collect()
3114 }
3115}
3116
3117fn default_retry_max_attempts() -> u32 {
3122 3
3123}
3124
3125fn default_retry_base_delay_ms() -> u64 {
3126 100
3127}
3128
3129fn default_retry_max_delay_ms() -> u64 {
3130 5000
3131}
3132
3133fn default_retry_jitter_factor() -> f64 {
3134 0.1
3135}
3136
3137fn default_retry_total_timeout_ms() -> u64 {
3138 30000
3139}
3140
3141#[derive(Debug, Clone, Serialize, Deserialize)]
3146pub struct RetryConfig {
3147 #[serde(default = "default_retry_max_attempts")]
3149 pub max_attempts: u32,
3150
3151 #[serde(default = "default_retry_base_delay_ms")]
3153 pub base_delay_ms: u64,
3154
3155 #[serde(default = "default_retry_max_delay_ms")]
3157 pub max_delay_ms: u64,
3158
3159 #[serde(default = "default_retry_jitter_factor")]
3162 pub jitter_factor: f64,
3163
3164 #[serde(default = "default_retry_total_timeout_ms")]
3167 pub total_timeout_ms: u64,
3168}
3169
3170impl Default for RetryConfig {
3171 fn default() -> Self {
3172 Self {
3173 max_attempts: default_retry_max_attempts(),
3174 base_delay_ms: default_retry_base_delay_ms(),
3175 max_delay_ms: default_retry_max_delay_ms(),
3176 jitter_factor: default_retry_jitter_factor(),
3177 total_timeout_ms: default_retry_total_timeout_ms(),
3178 }
3179 }
3180}
3181
3182impl RetryConfig {
3183 pub fn no_retry() -> Self {
3185 Self {
3186 max_attempts: 1,
3187 ..Default::default()
3188 }
3189 }
3190
3191 pub fn delay_for_attempt(&self, attempt: u32) -> std::time::Duration {
3195 if attempt == 0 {
3196 return std::time::Duration::ZERO;
3197 }
3198
3199 let base = self.base_delay_ms as f64;
3201 let delay = base * (2.0_f64.powi(attempt as i32 - 1));
3202 let capped = delay.min(self.max_delay_ms as f64);
3203
3204 let jittered = if self.jitter_factor > 0.0 {
3206 let jitter_range = capped * self.jitter_factor;
3207 let jitter = (rand::rng().random::<f64>() * 2.0 - 1.0) * jitter_range;
3208 (capped + jitter).max(0.0)
3209 } else {
3210 capped
3211 };
3212
3213 std::time::Duration::from_millis(jittered as u64)
3214 }
3215
3216 pub fn should_retry(&self, attempt: u32, elapsed: std::time::Duration) -> bool {
3218 if attempt >= self.max_attempts {
3219 return false;
3220 }
3221 elapsed.as_millis() < self.total_timeout_ms as u128
3222 }
3223}
3224
3225fn default_source_sync_silence_timeout_secs() -> u64 {
3230 120
3231}
3232
3233pub fn default_remote_base() -> String {
3242 "/data/tmp/rch".to_string()
3243}
3244
3245pub fn validate_remote_base(path: &str) -> Result<String, String> {
3248 let expanded = shellexpand::tilde(path).into_owned();
3250
3251 if !expanded.starts_with('/') {
3253 return Err(format!(
3254 "remote_base must be an absolute path, got: {}",
3255 path
3256 ));
3257 }
3258
3259 if expanded.contains("..") {
3261 return Err(format!(
3262 "remote_base must not contain path traversal (..): {}",
3263 path
3264 ));
3265 }
3266
3267 let normalized = expanded.trim_end_matches('/').to_string();
3269
3270 if normalized.is_empty() || normalized == "/" {
3272 return Err("remote_base cannot be the root directory (safety restriction)".to_string());
3273 }
3274
3275 let components: Vec<&str> = normalized.split('/').filter(|c| !c.is_empty()).collect();
3279 if components.len() < 2 {
3280 return Err(format!(
3281 "remote_base must be at least 2 levels deep (e.g. /tmp/rch), got: {}",
3282 normalized
3283 ));
3284 }
3285
3286 Ok(normalized)
3287}
3288
3289fn default_circuit_failure_threshold() -> u32 {
3290 3
3291}
3292
3293fn default_circuit_success_threshold() -> u32 {
3294 2
3295}
3296
3297fn default_circuit_error_rate_threshold() -> f64 {
3298 0.5
3299}
3300
3301fn default_circuit_window_secs() -> u64 {
3302 60
3303}
3304
3305fn default_circuit_open_cooldown_secs() -> u64 {
3306 30
3307}
3308
3309fn default_circuit_half_open_max_probes() -> u32 {
3310 1
3311}
3312
3313fn default_circuit_command_failure_threshold() -> u32 {
3314 2
3315}
3316
3317fn default_circuit_open_cooldown_max_secs() -> u64 {
3318 480
3320}
3321
3322fn default_true() -> bool {
3323 true
3324}
3325
3326fn default_log_level() -> String {
3327 "info".to_string()
3328}
3329
3330pub fn default_socket_path() -> String {
3331 if let Ok(runtime_dir) = std::env::var("XDG_RUNTIME_DIR")
3333 && !runtime_dir.trim().is_empty()
3334 {
3335 let path = PathBuf::from(runtime_dir).join("rch.sock");
3336 return path.to_string_lossy().to_string();
3337 }
3338
3339 if let Some(cache_dir) = dirs::cache_dir() {
3341 let rch_cache = cache_dir.join("rch");
3342 let _ = std::fs::create_dir_all(&rch_cache);
3343 return rch_cache.join("rch.sock").to_string_lossy().to_string();
3344 }
3345
3346 "/tmp/rch.sock".to_string()
3348}
3349
3350#[allow(dead_code)]
3351fn default_output_visibility() -> OutputVisibility {
3352 OutputVisibility::None
3353}
3354
3355fn default_confidence() -> f64 {
3356 0.85
3357}
3358
3359fn default_min_local_time() -> u64 {
3360 2000
3361}
3362
3363fn default_speedup_threshold() -> f64 {
3367 1.2
3368}
3369
3370fn default_compression() -> u32 {
3371 3
3372}
3373
3374fn default_verify_max_size() -> u64 {
3376 100 * 1024 * 1024 }
3378
3379fn default_min_compression() -> u32 {
3381 1
3382}
3383
3384fn default_max_compression() -> u32 {
3386 9
3387}
3388
3389fn default_excludes() -> Vec<String> {
3390 vec![
3391 "target/".to_string(),
3393 "*.rlib".to_string(),
3394 "*.rmeta".to_string(),
3395 ".git/".to_string(),
3398 "node_modules/".to_string(),
3400 ".bun/".to_string(),
3402 ".npm/".to_string(),
3404 ".pnpm-store/".to_string(),
3405 "dist/".to_string(),
3407 "build/".to_string(),
3408 ".next/".to_string(),
3410 ".nuxt/".to_string(),
3411 ".turbo/".to_string(),
3412 ".parcel-cache/".to_string(),
3413 ".beads/".to_string(),
3415 ".doctor/".to_string(),
3417 "coverage/".to_string(),
3419 ".nyc_output/".to_string(),
3420 ".cargo/credentials".to_string(),
3422 ".cargo/credentials.toml".to_string(),
3423 ".env".to_string(),
3424 ".env.*".to_string(),
3425 "*.pem".to_string(),
3426 "*.key".to_string(),
3427 "credentials.json".to_string(),
3428 "secrets.json".to_string(),
3429 "secrets.yaml".to_string(),
3430 "secrets.yml".to_string(),
3431 "secrets.toml".to_string(),
3432 "secrets.env".to_string(),
3433 "secrets.txt".to_string(),
3434 "secrets.conf".to_string(),
3435 "secrets.cfg".to_string(),
3436 "secrets.properties".to_string(),
3437 "secrets.xml".to_string(),
3438 "id_rsa".to_string(),
3452 "id_ed25519".to_string(),
3453 "id_ecdsa".to_string(),
3454 "id_dsa".to_string(),
3455 "*.p12".to_string(),
3457 "*.pfx".to_string(),
3458 "*.keystore".to_string(),
3459 "*.jks".to_string(),
3460 ".netrc".to_string(),
3463 "_netrc".to_string(),
3464 ".aws/credentials".to_string(),
3465 ".docker/config.json".to_string(),
3466 ".npmrc".to_string(),
3470 ".pypirc".to_string(),
3471 "secring.*".to_string(),
3473 ]
3474}
3475
3476#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
3482#[serde(rename_all = "snake_case")]
3483pub enum BuildLocation {
3484 #[default]
3486 Local,
3487 Remote,
3489}
3490
3491#[derive(Debug, Clone, Serialize, Deserialize)]
3493pub struct BuildCancellationWorkerHealth {
3494 pub status: String,
3496 pub speed_score: f64,
3498 pub used_slots: u32,
3500 pub available_slots: u32,
3502 pub pressure_state: String,
3504 pub pressure_reason_code: String,
3506}
3507
3508#[derive(Debug, Clone, Serialize, Deserialize)]
3510pub struct BuildCancellationMetadata {
3511 pub operation_id: String,
3513 pub origin: String,
3515 pub reason_code: String,
3517 #[serde(default)]
3519 pub decision_path: Vec<String>,
3520 pub escalation_stage: String,
3522 pub escalation_count: u32,
3524 pub remote_kill_attempted: bool,
3526 pub cleanup_ok: bool,
3528 pub history_cancelled: bool,
3530 pub final_state: String,
3532 #[serde(default, skip_serializing_if = "Option::is_none")]
3534 pub worker_health: Option<BuildCancellationWorkerHealth>,
3535}
3536
3537#[derive(Debug, Clone, Serialize, Deserialize)]
3539pub struct BuildRecord {
3540 pub id: u64,
3542 pub started_at: String,
3544 pub completed_at: String,
3546 pub project_id: String,
3548 pub worker_id: Option<String>,
3550 pub command: String,
3552 pub exit_code: i32,
3554 pub duration_ms: u64,
3556 pub location: BuildLocation,
3558 pub bytes_transferred: Option<u64>,
3560 #[serde(default)]
3562 pub timing: Option<CommandTimingBreakdown>,
3563 #[serde(default, skip_serializing_if = "Option::is_none")]
3565 pub cancellation: Option<BuildCancellationMetadata>,
3566}
3567
3568#[derive(Debug, Clone, Serialize, Deserialize)]
3570pub struct BuildRecordInput {
3571 pub started_at: String,
3573 pub completed_at: String,
3575 pub project_id: String,
3577 pub worker_id: Option<String>,
3579 pub command: String,
3581 pub exit_code: i32,
3583 pub duration_ms: u64,
3585 pub location: BuildLocation,
3587 pub bytes_transferred: Option<u64>,
3589 #[serde(default)]
3591 pub timing: Option<CommandTimingBreakdown>,
3592 #[serde(default, skip_serializing_if = "Option::is_none")]
3594 pub cancellation: Option<BuildCancellationMetadata>,
3595}
3596
3597impl BuildRecordInput {
3598 pub fn into_record(self, id: u64) -> BuildRecord {
3600 BuildRecord {
3601 id,
3602 started_at: self.started_at,
3603 completed_at: self.completed_at,
3604 project_id: self.project_id,
3605 worker_id: self.worker_id,
3606 command: self.command,
3607 exit_code: self.exit_code,
3608 duration_ms: self.duration_ms,
3609 location: self.location,
3610 bytes_transferred: self.bytes_transferred,
3611 timing: self.timing,
3612 cancellation: self.cancellation,
3613 }
3614 }
3615}
3616
3617#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3622pub struct BuildStats {
3623 pub total_builds: usize,
3625 pub success_count: usize,
3627 pub failure_count: usize,
3629 pub remote_count: usize,
3631 pub local_count: usize,
3633 pub avg_duration_ms: u64,
3635}
3636
3637#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3644pub struct SavedTimeStats {
3645 pub total_remote_duration_ms: u64,
3647 pub estimated_local_duration_ms: u64,
3650 pub time_saved_ms: u64,
3653 pub builds_counted: usize,
3655 pub avg_speedup: f64,
3659 pub today_saved_ms: u64,
3661 pub week_saved_ms: u64,
3663 #[serde(default)]
3666 pub estimate_basis: String,
3667 #[serde(default)]
3669 pub local_baseline_builds: usize,
3670}
3671
3672use std::collections::VecDeque;
3677use std::time::{Duration, Instant};
3678
3679#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3681pub struct CompilationTimingBreakdown {
3682 #[serde(with = "duration_millis")]
3684 pub rsync_up: Duration,
3685 #[serde(with = "duration_millis")]
3687 pub remote_build: Duration,
3688 #[serde(with = "duration_millis")]
3690 pub rsync_down: Duration,
3691 #[serde(with = "duration_millis")]
3693 pub total: Duration,
3694}
3695
3696#[derive(Debug, Clone, Default, Serialize, Deserialize)]
3700pub struct CommandTimingBreakdown {
3701 #[serde(with = "option_duration_millis")]
3703 pub classify: Option<Duration>,
3704 #[serde(with = "option_duration_millis")]
3706 pub select: Option<Duration>,
3707 #[serde(with = "option_duration_millis")]
3709 pub sync_up: Option<Duration>,
3710 #[serde(with = "option_duration_millis")]
3712 pub exec: Option<Duration>,
3713 #[serde(with = "option_duration_millis")]
3715 pub sync_down: Option<Duration>,
3716 #[serde(with = "option_duration_millis")]
3718 pub cleanup: Option<Duration>,
3719 #[serde(with = "option_duration_millis")]
3721 pub total: Option<Duration>,
3722}
3723
3724mod duration_millis {
3725 use serde::{Deserialize, Deserializer, Serialize, Serializer};
3726 use std::time::Duration;
3727
3728 pub fn serialize<S>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error>
3729 where
3730 S: Serializer,
3731 {
3732 duration.as_millis().serialize(serializer)
3733 }
3734
3735 pub fn deserialize<'de, D>(deserializer: D) -> Result<Duration, D::Error>
3736 where
3737 D: Deserializer<'de>,
3738 {
3739 let millis = u64::deserialize(deserializer)?;
3740 Ok(Duration::from_millis(millis))
3741 }
3742}
3743
3744#[derive(Debug, Clone, Serialize, Deserialize)]
3746pub struct CompilationMetrics {
3747 pub project_id: String,
3749 pub worker_id: String,
3751 pub timestamp: chrono::DateTime<chrono::Utc>,
3753 pub timing: CompilationTimingBreakdown,
3755 #[serde(with = "option_duration_millis")]
3757 pub local_build_time: Option<Duration>,
3758 pub speedup: Option<f64>,
3760 pub files_synced: u64,
3762 pub bytes_transferred: u64,
3764 pub exit_code: i32,
3766 pub success: bool,
3768}
3769
3770mod option_duration_millis {
3771 use serde::{Deserialize, Deserializer, Serialize, Serializer};
3772 use std::time::Duration;
3773
3774 pub fn serialize<S>(duration: &Option<Duration>, serializer: S) -> Result<S::Ok, S::Error>
3775 where
3776 S: Serializer,
3777 {
3778 match duration {
3779 Some(d) => Some(d.as_millis() as u64).serialize(serializer),
3780 None => Option::<u64>::None.serialize(serializer),
3781 }
3782 }
3783
3784 pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<Duration>, D::Error>
3785 where
3786 D: Deserializer<'de>,
3787 {
3788 let opt: Option<u64> = Option::deserialize(deserializer)?;
3789 Ok(opt.map(Duration::from_millis))
3790 }
3791}
3792
3793impl CompilationMetrics {
3794 pub fn calculate_speedup(&mut self) {
3796 if let Some(local) = self.local_build_time
3797 && self.timing.total.as_millis() > 0
3798 {
3799 self.speedup = Some(local.as_secs_f64() / self.timing.total.as_secs_f64());
3800 }
3801 }
3802
3803 pub fn is_beneficial(&self) -> bool {
3805 self.speedup.map(|s| s > 1.0).unwrap_or(false)
3806 }
3807}
3808
3809impl Default for CompilationMetrics {
3810 fn default() -> Self {
3811 Self {
3812 project_id: String::new(),
3813 worker_id: String::new(),
3814 timestamp: chrono::Utc::now(),
3815 timing: CompilationTimingBreakdown::default(),
3816 local_build_time: None,
3817 speedup: None,
3818 files_synced: 0,
3819 bytes_transferred: 0,
3820 exit_code: 0,
3821 success: true,
3822 }
3823 }
3824}
3825
3826#[derive(Debug)]
3842pub struct CompilationTimer {
3843 project_id: String,
3844 worker_id: String,
3845 start: Instant,
3846 phase_start: Instant,
3847 rsync_up: Option<Duration>,
3848 remote_build: Option<Duration>,
3849 rsync_down: Option<Duration>,
3850}
3851
3852impl CompilationTimer {
3853 pub fn new(project_id: &str, worker_id: &str) -> Self {
3855 let now = Instant::now();
3856 Self {
3857 project_id: project_id.to_string(),
3858 worker_id: worker_id.to_string(),
3859 start: now,
3860 phase_start: now,
3861 rsync_up: None,
3862 remote_build: None,
3863 rsync_down: None,
3864 }
3865 }
3866
3867 pub fn end_rsync_up(&mut self) {
3869 self.rsync_up = Some(self.phase_start.elapsed());
3870 self.phase_start = Instant::now();
3871 tracing::info!(
3872 rsync_up_ms = %self.rsync_up.unwrap().as_millis(),
3873 "TIMING: rsync_up completed"
3874 );
3875 }
3876
3877 pub fn end_remote_build(&mut self) {
3879 self.remote_build = Some(self.phase_start.elapsed());
3880 self.phase_start = Instant::now();
3881 tracing::info!(
3882 remote_build_ms = %self.remote_build.unwrap().as_millis(),
3883 "TIMING: remote_build completed"
3884 );
3885 }
3886
3887 pub fn end_rsync_down(&mut self) {
3889 self.rsync_down = Some(self.phase_start.elapsed());
3890 tracing::info!(
3891 rsync_down_ms = %self.rsync_down.unwrap().as_millis(),
3892 "TIMING: rsync_down completed"
3893 );
3894 }
3895
3896 pub fn finish(self, exit_code: i32, files: u64, bytes: u64) -> CompilationMetrics {
3898 let total = self.start.elapsed();
3899 tracing::info!(
3900 total_ms = %total.as_millis(),
3901 exit_code = %exit_code,
3902 "TIMING: compilation completed"
3903 );
3904
3905 CompilationMetrics {
3906 project_id: self.project_id,
3907 worker_id: self.worker_id,
3908 timestamp: chrono::Utc::now(),
3909 timing: CompilationTimingBreakdown {
3910 rsync_up: self.rsync_up.unwrap_or_default(),
3911 remote_build: self.remote_build.unwrap_or_default(),
3912 rsync_down: self.rsync_down.unwrap_or_default(),
3913 total,
3914 },
3915 local_build_time: None,
3916 speedup: None,
3917 files_synced: files,
3918 bytes_transferred: bytes,
3919 exit_code,
3920 success: exit_code == 0,
3921 }
3922 }
3923}
3924
3925#[derive(Debug)]
3927pub struct MetricsAggregator {
3928 history: VecDeque<CompilationMetrics>,
3929 max_history: usize,
3930}
3931
3932impl MetricsAggregator {
3933 pub fn new(max_history: usize) -> Self {
3935 Self {
3936 history: VecDeque::with_capacity(max_history),
3937 max_history,
3938 }
3939 }
3940
3941 pub fn record(&mut self, metrics: CompilationMetrics) {
3943 if self.history.len() >= self.max_history {
3944 self.history.pop_front();
3945 }
3946 self.history.push_back(metrics);
3947 }
3948
3949 pub fn average_speedup(&self) -> Option<f64> {
3951 let speedups: Vec<f64> = self.history.iter().filter_map(|m| m.speedup).collect();
3952
3953 if speedups.is_empty() {
3954 None
3955 } else {
3956 Some(speedups.iter().sum::<f64>() / speedups.len() as f64)
3957 }
3958 }
3959
3960 pub fn p50_total_time(&self) -> Option<Duration> {
3962 self.percentile_total_time(0.50)
3963 }
3964
3965 pub fn p95_total_time(&self) -> Option<Duration> {
3967 self.percentile_total_time(0.95)
3968 }
3969
3970 pub fn p99_total_time(&self) -> Option<Duration> {
3972 self.percentile_total_time(0.99)
3973 }
3974
3975 fn percentile_total_time(&self, percentile: f64) -> Option<Duration> {
3980 let mut times: Vec<_> = self.history.iter().map(|m| m.timing.total).collect();
3981
3982 if times.is_empty() {
3983 return None;
3984 }
3985
3986 times.sort();
3987 let idx = (((times.len() - 1) as f64 * percentile).round() as usize).min(times.len() - 1);
3991 Some(times[idx])
3992 }
3993
3994 pub fn success_rate(&self) -> f64 {
3996 if self.history.is_empty() {
3997 return 100.0;
3998 }
3999 let successes = self.history.iter().filter(|m| m.success).count();
4000 (successes as f64 / self.history.len() as f64) * 100.0
4001 }
4002
4003 pub fn count(&self) -> usize {
4005 self.history.len()
4006 }
4007
4008 pub fn metrics(&self) -> &VecDeque<CompilationMetrics> {
4010 &self.history
4011 }
4012
4013 pub fn clear(&mut self) {
4015 self.history.clear();
4016 }
4017}
4018
4019impl Default for MetricsAggregator {
4020 fn default() -> Self {
4021 Self::new(1000)
4022 }
4023}
4024
4025#[cfg(test)]
4026mod tests {
4027 use super::*;
4028 use crate::test_guard;
4029
4030 #[test]
4031 fn test_declared_os_reads_reserved_tag() {
4032 assert_eq!(
4033 declared_os(&["rust".to_string(), "os:windows".to_string()]),
4034 Some("windows".to_string())
4035 );
4036 assert_eq!(
4037 declared_os(&["os:WINDOWS".to_string()]),
4038 Some("windows".to_string())
4039 );
4040 }
4041
4042 #[test]
4043 fn test_declared_os_absent_for_ordinary_tags() {
4044 assert_eq!(declared_os(&[]), None);
4047 assert_eq!(
4048 declared_os(&["rust".to_string(), "bun".to_string(), "go".to_string()]),
4049 None
4050 );
4051 assert_eq!(declared_os(&["os:".to_string()]), None);
4053 assert_eq!(declared_os(&["os: ".to_string()]), None);
4054 }
4055
4056 #[test]
4057 fn test_os_tag_roundtrips_through_declared_os() {
4058 for os in ["windows", "Darwin", " linux "] {
4059 let tag = os_tag(os);
4060 assert_eq!(
4061 declared_os(std::slice::from_ref(&tag)).as_deref(),
4062 Some(os.trim().to_ascii_lowercase().as_str())
4063 );
4064 }
4065 }
4066
4067 #[test]
4072 fn test_no_inline_secret_fields() {
4073 let endpoint = DoctorWebhookEndpoint {
4074 name: "ops".to_string(),
4075 url: "https://example.com/hook".to_string(),
4076 format: DoctorWebhookFormat::Pagerduty,
4077 on_transitions: vec!["any_to_failing".to_string()],
4078 retry_max: 3,
4079 retry_backoff_ms: 200,
4080 timeout_ms: 5000,
4081 enabled: true,
4082 routing_key_env: Some("RCH_PAGERDUTY_ROUTING_KEY".to_string()),
4083 bearer_token_env: Some("RCH_WEBHOOK_BEARER".to_string()),
4084 signing_secret_env: Some("RCH_WEBHOOK_HMAC".to_string()),
4085 };
4086 let value = serde_json::to_value(&endpoint).unwrap();
4089 let obj = value.as_object().expect("endpoint serializes to an object");
4090 for forbidden in ["secret", "routing_key", "bearer_token", "password", "token"] {
4091 assert!(
4092 !obj.contains_key(forbidden),
4093 "endpoint config must not expose an inline `{forbidden}` field"
4094 );
4095 }
4096 assert!(obj.contains_key("routing_key_env"));
4097 assert!(obj.contains_key("bearer_token_env"));
4098 assert!(obj.contains_key("signing_secret_env"));
4099 }
4100
4101 #[test]
4102 fn test_webhook_format_toml_spelling() {
4103 #[derive(serde::Serialize, serde::Deserialize, PartialEq, Debug)]
4108 struct FormatWrap {
4109 format: DoctorWebhookFormat,
4110 }
4111 let rendered = toml::to_string(&FormatWrap {
4112 format: DoctorWebhookFormat::GenericJson,
4113 })
4114 .unwrap();
4115 assert!(
4116 rendered.contains("format = \"generic_json\""),
4117 "unexpected TOML: {rendered}"
4118 );
4119 let slack: FormatWrap = toml::from_str("format = \"slack\"").unwrap();
4120 assert_eq!(slack.format, DoctorWebhookFormat::Slack);
4121 let pd: FormatWrap = toml::from_str("format = \"pagerduty\"").unwrap();
4122 assert_eq!(pd.format, DoctorWebhookFormat::Pagerduty);
4123 }
4124
4125 #[test]
4126 fn test_doctor_webhooks_defaults() {
4127 let cfg = DoctorWebhooksConfig::default();
4128 assert!(cfg.endpoints.is_empty());
4129 assert_eq!(cfg.max_inflight, 16);
4130 let rch: RchConfig = toml::from_str("").unwrap();
4133 assert!(rch.doctor.webhooks.endpoints.is_empty());
4134 }
4135
4136 #[test]
4137 fn test_doctor_webhook_endpoint_parses_from_toml() {
4138 let toml_src = r#"
4139[doctor.webhooks]
4140max_inflight = 8
4141
4142[[doctor.webhooks.endpoints]]
4143name = "ops-slack"
4144url = "https://hooks.slack.com/services/T0/B0/x"
4145format = "slack"
4146on_transitions = ["healthy_to_degraded", "any_to_failing"]
4147retry_max = 2
4148"#;
4149 let cfg: RchConfig = toml::from_str(toml_src).unwrap();
4150 assert_eq!(cfg.doctor.webhooks.max_inflight, 8);
4151 assert_eq!(cfg.doctor.webhooks.endpoints.len(), 1);
4152 let ep = &cfg.doctor.webhooks.endpoints[0];
4153 assert_eq!(ep.name, "ops-slack");
4154 assert_eq!(ep.format, DoctorWebhookFormat::Slack);
4155 assert_eq!(ep.on_transitions.len(), 2);
4156 assert_eq!(ep.retry_max, 2);
4157 assert_eq!(ep.retry_backoff_ms, 200);
4159 assert_eq!(ep.timeout_ms, 5000);
4160 assert!(ep.enabled);
4161 }
4162
4163 fn assert_golden_json(
4164 actual: serde_json::Value,
4165 expected: &serde_json::Value,
4166 fixture_path: &str,
4167 ) {
4168 if &actual == expected {
4169 return;
4170 }
4171
4172 let actual_json = serde_json::to_string_pretty(&actual).expect("actual JSON renders");
4173 let expected_json = serde_json::to_string_pretty(expected).expect("expected JSON renders");
4174 assert_eq!(
4175 &actual,
4176 expected,
4177 "golden mismatch in {fixture_path}\n\
4178 expected_blake3={}\n\
4179 actual_blake3={}\n\
4180 bless path: review the semantic diff, update the fixture expected_* field, \
4181 and record both hashes in the Beads closeout.\n\
4182 expected:\n{expected_json}\n\
4183 actual:\n{actual_json}",
4184 blake3::hash(expected_json.as_bytes()),
4185 blake3::hash(actual_json.as_bytes())
4186 );
4187 }
4188
4189 #[test]
4190 fn test_circuit_state_default() {
4191 let _guard = test_guard!();
4192 assert_eq!(CircuitState::default(), CircuitState::Closed);
4193 }
4194
4195 #[test]
4196 fn test_circuit_config_defaults() {
4197 let _guard = test_guard!();
4198 let config = CircuitBreakerConfig::default();
4199 assert_eq!(config.failure_threshold, 3);
4200 assert_eq!(config.success_threshold, 2);
4201 assert_eq!(config.error_rate_threshold, 0.5);
4202 assert_eq!(config.window_secs, 60);
4203 assert_eq!(config.open_cooldown_secs, 30);
4204 assert_eq!(config.half_open_max_probes, 1);
4205 }
4206
4207 #[test]
4208 fn test_rch_config_has_circuit_defaults() {
4209 let _guard = test_guard!();
4210 let config = RchConfig::default();
4211 assert_eq!(config.circuit.failure_threshold, 3);
4212 }
4213
4214 #[test]
4215 fn test_circuit_config_serde_roundtrip() {
4216 let _guard = test_guard!();
4217 let config = CircuitBreakerConfig {
4218 failure_threshold: 5,
4219 success_threshold: 3,
4220 error_rate_threshold: 0.75,
4221 window_secs: 120,
4222 open_cooldown_secs: 45,
4223 half_open_max_probes: 2,
4224 command_failure_threshold: 4,
4225 open_cooldown_max_secs: 900,
4226 };
4227 let json = serde_json::to_string(&config).unwrap();
4228 let parsed: CircuitBreakerConfig = serde_json::from_str(&json).unwrap();
4229 assert_eq!(parsed.command_failure_threshold, 4);
4230 assert_eq!(parsed.open_cooldown_max_secs, 900);
4231 assert_eq!(parsed.failure_threshold, 5);
4232 assert_eq!(parsed.success_threshold, 3);
4233 assert_eq!(parsed.error_rate_threshold, 0.75);
4234 assert_eq!(parsed.window_secs, 120);
4235 assert_eq!(parsed.open_cooldown_secs, 45);
4236 assert_eq!(parsed.half_open_max_probes, 2);
4237 }
4238
4239 #[test]
4244 fn test_worker_capabilities_default() {
4245 let _guard = test_guard!();
4246 let caps = WorkerCapabilities::default();
4247 assert!(caps.rustc_version.is_none());
4248 assert!(caps.rustup_toolchains.is_empty());
4249 assert!(caps.rustup_components.is_empty());
4250 assert!(caps.bun_version.is_none());
4251 assert!(caps.node_version.is_none());
4252 assert!(caps.npm_version.is_none());
4253 assert!(caps.projects_root_ok.is_none());
4254 assert!(caps.projects_root_issue.is_none());
4255 assert!(caps.projects_root_checked_at_unix_ms.is_none());
4256 }
4257
4258 #[test]
4259 fn test_worker_capabilities_new() {
4260 let _guard = test_guard!();
4261 let caps = WorkerCapabilities::new();
4262 assert!(caps.rustc_version.is_none());
4263 assert!(!caps.has_rust());
4264 assert!(!caps.has_bun());
4265 assert!(!caps.has_node());
4266 assert!(caps.is_topology_healthy().is_none());
4267 }
4268
4269 #[test]
4270 fn test_worker_capabilities_has_rust() {
4271 let _guard = test_guard!();
4272 let mut caps = WorkerCapabilities::new();
4273 assert!(!caps.has_rust());
4274
4275 caps.rustc_version = Some("rustc 1.76.0 (07dca489a 2024-02-04)".to_string());
4276 assert!(caps.has_rust());
4277 }
4278
4279 #[test]
4280 fn test_worker_capabilities_components_are_toolchain_scoped_and_exact() {
4281 let _guard = test_guard!();
4282 let caps = WorkerCapabilities {
4283 rustup_components: vec![
4284 "stable-x86_64-unknown-linux-gnu:clippy".to_string(),
4285 "nightly-2026-07-05-x86_64-unknown-linux-gnu:rustfmt".to_string(),
4286 "beta-x86_64-unknown-linux-gnu:clippy-preview".to_string(),
4287 ],
4288 ..Default::default()
4289 };
4290
4291 assert!(caps.has_rustup_component("stable", "clippy"));
4292 assert!(caps.has_rustup_component("nightly-2026-07-05", "rustfmt"));
4293 assert!(!caps.has_rustup_component("nightly-2026-07-05", "clippy"));
4294 assert!(!caps.has_rustup_component("beta", "clippy"));
4295 assert!(!caps.has_rustup_component("nightly-2026-07-0", "rustfmt"));
4296 }
4297
4298 #[test]
4299 fn test_worker_capabilities_has_bun() {
4300 let _guard = test_guard!();
4301 let mut caps = WorkerCapabilities::new();
4302 assert!(!caps.has_bun());
4303
4304 caps.bun_version = Some("1.0.25".to_string());
4305 assert!(caps.has_bun());
4306 }
4307
4308 #[test]
4309 fn test_worker_capabilities_has_nix() {
4310 let _guard = test_guard!();
4311 let mut caps = WorkerCapabilities::new();
4312 assert!(!caps.has_nix());
4313
4314 caps.nix_version = Some("nix (Nix) 2.24.9".to_string());
4315 assert!(caps.has_nix());
4316 }
4317
4318 #[test]
4319 fn test_worker_capabilities_has_zig_requires_both_halves() {
4320 let _guard = test_guard!();
4321 let mut caps = WorkerCapabilities::new();
4322 assert!(!caps.has_zig());
4323
4324 caps.zig_version = Some("0.14.1".to_string());
4327 assert!(!caps.has_zig());
4328
4329 caps.zig_version = None;
4331 caps.cargo_zigbuild_version = Some("cargo-zigbuild 0.23.0".to_string());
4332 assert!(!caps.has_zig());
4333
4334 caps.zig_version = Some("0.14.1".to_string());
4335 assert!(caps.has_zig());
4336 }
4337
4338 #[test]
4339 fn test_nix_build_uses_generous_timeout() {
4340 let _guard = test_guard!();
4341 let cfg = CompilationConfig::default();
4342 assert_eq!(
4344 cfg.timeout_for_kind(Some(CompilationKind::NixBuild)),
4345 std::time::Duration::from_secs(cfg.test_timeout_sec)
4346 );
4347 }
4348
4349 #[test]
4350 fn test_job_uses_build_timeout_bucket() {
4351 let _guard = test_guard!();
4352 let cfg = CompilationConfig::default();
4355 assert_eq!(
4356 cfg.timeout_for_kind(Some(CompilationKind::Job)),
4357 std::time::Duration::from_secs(cfg.build_timeout_sec)
4358 );
4359 }
4360
4361 #[test]
4362 fn test_layer0_disabled_by_default_and_env_gated() {
4363 let _guard = test_guard!();
4364 let cfg = Layer0Config::default();
4365 assert!(!cfg.enabled);
4366 let knobs_on = Layer0Config {
4368 enabled: false,
4369 release_lto_thin: true,
4370 release_codegen_units_1: true,
4371 };
4372 assert!(cfg.active_env().is_empty());
4373 assert!(knobs_on.active_env().is_empty());
4374 }
4375
4376 #[test]
4377 fn test_layer0_knobs_emit_stable_env_pairs() {
4378 let _guard = test_guard!();
4379 let lto_only = Layer0Config {
4380 enabled: true,
4381 release_lto_thin: true,
4382 release_codegen_units_1: false,
4383 };
4384 assert_eq!(
4385 lto_only.active_env(),
4386 vec![("CARGO_PROFILE_RELEASE_LTO", "thin")]
4387 );
4388 let both = Layer0Config {
4389 enabled: true,
4390 release_lto_thin: true,
4391 release_codegen_units_1: true,
4392 };
4393 assert_eq!(
4394 both.active_env(),
4395 vec![
4396 ("CARGO_PROFILE_RELEASE_LTO", "thin"),
4397 ("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1"),
4398 ],
4399 "emission order must stay stable for deterministic baselines"
4400 );
4401 }
4402
4403 #[test]
4404 fn test_layer0_pack_version_is_recorded() {
4405 let _guard = test_guard!();
4406 assert!(!LAYER0_PACK_VERSION.is_empty());
4408 }
4409 #[test]
4410 fn test_default_allowlist_includes_nix() {
4411 let _guard = test_guard!();
4412 let cfg = ExecutionConfig::default();
4413 assert!(cfg.is_allowed("nix"));
4414 }
4415
4416 #[test]
4417 fn test_worker_capabilities_has_node() {
4418 let _guard = test_guard!();
4419 let mut caps = WorkerCapabilities::new();
4420 assert!(!caps.has_node());
4421
4422 caps.node_version = Some("v20.11.0".to_string());
4423 assert!(caps.has_node());
4424 }
4425
4426 #[test]
4427 fn test_worker_capabilities_multiple_runtimes() {
4428 let _guard = test_guard!();
4429 let caps = WorkerCapabilities {
4430 rustc_version: Some("rustc 1.76.0".to_string()),
4431 bun_version: Some("1.0.25".to_string()),
4432 node_version: Some("v20.11.0".to_string()),
4433 npm_version: Some("10.2.4".to_string()),
4434 ..Default::default()
4435 };
4436
4437 assert!(caps.has_rust());
4438 assert!(caps.has_bun());
4439 assert!(caps.has_node());
4440 }
4441
4442 #[test]
4443 fn test_worker_capabilities_serialization_empty() {
4444 let _guard = test_guard!();
4445 let caps = WorkerCapabilities::new();
4446 let json = serde_json::to_string(&caps).unwrap();
4447 assert_eq!(json, "{}");
4449
4450 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4452 assert!(!parsed.has_rust());
4453 assert!(!parsed.has_bun());
4454 assert!(!parsed.has_node());
4455 }
4456
4457 #[test]
4458 fn test_worker_capabilities_serialization_with_versions() {
4459 let _guard = test_guard!();
4460 let caps = WorkerCapabilities {
4461 rustc_version: Some("rustc 1.76.0-nightly".to_string()),
4462 bun_version: Some("1.0.25".to_string()),
4463 node_version: None,
4464 npm_version: None,
4465 ..Default::default()
4466 };
4467
4468 let json = serde_json::to_string(&caps).unwrap();
4469 assert!(json.contains("rustc_version"));
4470 assert!(json.contains("rustc 1.76.0-nightly"));
4471 assert!(json.contains("bun_version"));
4472 assert!(json.contains("1.0.25"));
4473 assert!(!json.contains("node_version"));
4475 assert!(!json.contains("npm_version"));
4476
4477 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
4478 assert!(parsed.has_rust());
4479 assert!(parsed.has_bun());
4480 assert!(!parsed.has_node());
4481 }
4482
4483 #[test]
4484 fn test_worker_capabilities_deserialization_partial() {
4485 let _guard = test_guard!();
4486 let json = r#"{"bun_version": "1.0.0"}"#;
4488 let caps: WorkerCapabilities = serde_json::from_str(json).unwrap();
4489
4490 assert!(!caps.has_rust());
4491 assert!(caps.has_bun());
4492 assert!(!caps.has_node());
4493 assert_eq!(caps.bun_version, Some("1.0.0".to_string()));
4494 }
4495
4496 #[test]
4497 fn test_worker_capabilities_clone() {
4498 let _guard = test_guard!();
4499 let caps = WorkerCapabilities {
4500 rustc_version: Some("1.76.0".to_string()),
4501 bun_version: None,
4502 node_version: Some("v20".to_string()),
4503 npm_version: None,
4504 projects_root_ok: Some(false),
4505 projects_root_issue: Some("alias_missing".to_string()),
4506 projects_root_checked_at_unix_ms: Some(123),
4507 ..Default::default()
4508 };
4509
4510 let cloned = caps.clone();
4511 assert_eq!(cloned.rustc_version, caps.rustc_version);
4512 assert_eq!(cloned.bun_version, caps.bun_version);
4513 assert_eq!(cloned.node_version, caps.node_version);
4514 assert_eq!(cloned.npm_version, caps.npm_version);
4515 assert_eq!(cloned.projects_root_ok, caps.projects_root_ok);
4516 assert_eq!(cloned.projects_root_issue, caps.projects_root_issue);
4517 assert_eq!(
4518 cloned.projects_root_checked_at_unix_ms,
4519 caps.projects_root_checked_at_unix_ms
4520 );
4521 }
4522
4523 #[test]
4524 fn test_worker_capabilities_topology_health_helper() {
4525 let _guard = test_guard!();
4526 let mut caps = WorkerCapabilities::new();
4527 assert_eq!(caps.is_topology_healthy(), None);
4528
4529 caps.projects_root_ok = Some(true);
4530 assert_eq!(caps.is_topology_healthy(), Some(true));
4531
4532 caps.projects_root_ok = Some(false);
4533 assert_eq!(caps.is_topology_healthy(), Some(false));
4534 }
4535
4536 #[test]
4537 fn test_selection_reason_serialization() {
4538 let _guard = test_guard!();
4539 assert_eq!(
4541 serde_json::to_string(&SelectionReason::Success).unwrap(),
4542 "\"success\""
4543 );
4544 assert_eq!(
4545 serde_json::to_string(&SelectionReason::NoWorkersConfigured).unwrap(),
4546 "\"no_workers_configured\""
4547 );
4548 assert_eq!(
4549 serde_json::to_string(&SelectionReason::AllWorkersUnreachable).unwrap(),
4550 "\"all_workers_unreachable\""
4551 );
4552 assert_eq!(
4553 serde_json::to_string(&SelectionReason::AllCircuitsOpen).unwrap(),
4554 "\"all_circuits_open\""
4555 );
4556 assert_eq!(
4557 serde_json::to_string(&SelectionReason::AllWorkersBusy).unwrap(),
4558 "\"all_workers_busy\""
4559 );
4560 assert_eq!(
4561 serde_json::to_string(&SelectionReason::NoWorkersPassedHealth).unwrap(),
4562 "\"no_workers_passed_health\""
4563 );
4564 assert_eq!(
4565 serde_json::to_string(&SelectionReason::AllWorkersFailedPreflight).unwrap(),
4566 "\"all_workers_failed_preflight\""
4567 );
4568 assert_eq!(
4569 serde_json::to_string(&SelectionReason::AllWorkersFailedConvergence).unwrap(),
4570 "\"all_workers_failed_convergence\""
4571 );
4572 assert_eq!(
4573 serde_json::to_string(&SelectionReason::NoAdmissibleWorkers(
4574 "critical_pressure=1,insufficient_slots=1".to_string()
4575 ))
4576 .unwrap(),
4577 "{\"no_admissible_workers\":\"critical_pressure=1,insufficient_slots=1\"}"
4578 );
4579 assert_eq!(
4580 serde_json::to_string(&SelectionReason::NoMatchingWorkers).unwrap(),
4581 "\"no_matching_workers\""
4582 );
4583 }
4584
4585 #[test]
4586 fn test_selection_reason_with_error() {
4587 let _guard = test_guard!();
4588 let reason = SelectionReason::SelectionError("test error".to_string());
4589 let json = serde_json::to_string(&reason).unwrap();
4590 assert!(json.contains("selection_error"));
4591 assert!(json.contains("test error"));
4592 }
4593
4594 #[test]
4595 fn test_selection_reason_deserialization() {
4596 let _guard = test_guard!();
4597 assert_eq!(
4598 serde_json::from_str::<SelectionReason>("\"success\"").unwrap(),
4599 SelectionReason::Success
4600 );
4601 assert_eq!(
4602 serde_json::from_str::<SelectionReason>("\"all_workers_busy\"").unwrap(),
4603 SelectionReason::AllWorkersBusy
4604 );
4605 assert_eq!(
4606 serde_json::from_str::<SelectionReason>("\"no_workers_passed_health\"").unwrap(),
4607 SelectionReason::NoWorkersPassedHealth
4608 );
4609 assert_eq!(
4610 serde_json::from_str::<SelectionReason>("\"all_workers_failed_preflight\"").unwrap(),
4611 SelectionReason::AllWorkersFailedPreflight
4612 );
4613 assert_eq!(
4614 serde_json::from_str::<SelectionReason>("\"all_workers_failed_convergence\"").unwrap(),
4615 SelectionReason::AllWorkersFailedConvergence
4616 );
4617 assert_eq!(
4618 serde_json::from_str::<SelectionReason>(
4619 "{\"no_admissible_workers\":\"critical_pressure=1\"}"
4620 )
4621 .unwrap(),
4622 SelectionReason::NoAdmissibleWorkers("critical_pressure=1".to_string())
4623 );
4624 }
4625
4626 #[test]
4627 fn test_selection_reason_display() {
4628 let _guard = test_guard!();
4629 assert_eq!(
4630 SelectionReason::Success.to_string(),
4631 "worker assigned successfully"
4632 );
4633 assert_eq!(
4634 SelectionReason::NoWorkersConfigured.to_string(),
4635 "no workers configured"
4636 );
4637 assert_eq!(
4638 SelectionReason::AllWorkersUnreachable.to_string(),
4639 "all workers unreachable"
4640 );
4641 assert_eq!(
4642 SelectionReason::AllWorkersBusy.to_string(),
4643 "all workers at capacity"
4644 );
4645 assert_eq!(
4646 SelectionReason::NoWorkersPassedHealth.to_string(),
4647 "no workers passed health thresholds"
4648 );
4649 assert_eq!(
4650 SelectionReason::AllWorkersFailedPreflight.to_string(),
4651 "all workers failed preflight checks"
4652 );
4653 assert_eq!(
4654 SelectionReason::AllWorkersFailedConvergence.to_string(),
4655 "all workers failed repo convergence checks"
4656 );
4657 assert_eq!(
4658 SelectionReason::NoAdmissibleWorkers(
4659 "critical_pressure=1,insufficient_slots=1".to_string()
4660 )
4661 .to_string(),
4662 "no admissible workers: critical_pressure=1,insufficient_slots=1"
4663 );
4664 assert_eq!(
4665 SelectionReason::SelectionError("oops".to_string()).to_string(),
4666 "selection error: oops"
4667 );
4668 }
4669
4670 #[test]
4671 fn test_selection_response_with_worker() {
4672 let _guard = test_guard!();
4673 let response = SelectionResponse {
4674 worker: Some(SelectedWorker {
4675 id: WorkerId::new("test"),
4676 host: "localhost".to_string(),
4677 user: "user".to_string(),
4678 identity_file: "~/.ssh/id_rsa".to_string(),
4679 slots_available: 8,
4680 speed_score: 75.0,
4681 declared_os: None,
4682 }),
4683 reason: SelectionReason::Success,
4684 build_id: None,
4685 diagnostics: None,
4686 };
4687
4688 let json = serde_json::to_string(&response).unwrap();
4689 assert!(json.contains("\"reason\":\"success\""));
4690 assert!(json.contains("\"id\":\"test\""));
4691 }
4692
4693 #[test]
4694 fn test_selection_response_without_worker() {
4695 let _guard = test_guard!();
4696 let response = SelectionResponse {
4697 worker: None,
4698 reason: SelectionReason::AllWorkersBusy,
4699 build_id: None,
4700 diagnostics: None,
4701 };
4702
4703 let json = serde_json::to_string(&response).unwrap();
4704 assert!(json.contains("\"worker\":null"));
4705 assert!(json.contains("\"reason\":\"all_workers_busy\""));
4706 }
4707
4708 #[test]
4709 fn golden_selection_response_with_no_workers_passed_health_diagnostics() {
4710 let _guard = test_guard!();
4711 const FIXTURE_PATH: &str =
4712 "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json";
4713 let fixture: serde_json::Value = serde_json::from_str(include_str!(
4714 "../../tests/goldens/ft_4tp7g/selection_diagnostics_no_workers_passed_health.json"
4715 ))
4716 .expect("selection diagnostics golden fixture parses");
4717 assert_eq!(
4718 fixture["schema_version"],
4719 "rch.golden.selection_diagnostics.v1"
4720 );
4721
4722 let response = SelectionResponse {
4723 worker: None,
4724 reason: SelectionReason::NoWorkersPassedHealth,
4725 build_id: None,
4726 diagnostics: Some(SelectionDiagnostics {
4727 required_runtime: RequiredRuntime::Rust,
4728 estimated_cores: 2,
4729 min_success_rate: 0.8,
4730 fallback_min_success_rate: 0.5,
4731 active_project_exclusion_count: 0,
4732 workers: vec![WorkerSelectionDiagnostic {
4733 worker_id: WorkerId::new("fixture-worker"),
4734 status: "healthy".to_string(),
4735 circuit_state: "closed".to_string(),
4736 pressure_state: "healthy".to_string(),
4737 pressure_reason_code: "pressure.ok".to_string(),
4738 success_rate: Some(0.2),
4739 min_success_rate: 0.8,
4740 fallback_min_success_rate: 0.5,
4741 required_runtime: RequiredRuntime::Rust,
4742 runtime_available: true,
4743 available_slots: 8,
4744 total_slots: 8,
4745 estimated_cores: 2,
4746 active_project_excluded: false,
4747 final_decision: WorkerSelectionDiagnosticDecision::Deny,
4748 final_reason: "success_rate 0.20 < fallback 0.50".to_string(),
4749 reason_codes: vec![
4750 "health.below_min_success_rate".to_string(),
4751 "health.below_fallback_min_success_rate".to_string(),
4752 ],
4753 }],
4754 }),
4755 };
4756
4757 assert_golden_json(
4758 serde_json::to_value(&response).expect("response serializes"),
4759 &fixture["expected_response"],
4760 FIXTURE_PATH,
4761 );
4762 }
4763
4764 #[test]
4765 fn test_selection_response_roundtrip() {
4766 let _guard = test_guard!();
4767 let original = SelectionResponse {
4768 worker: Some(SelectedWorker {
4769 id: WorkerId::new("worker-1"),
4770 host: "192.168.1.100".to_string(),
4771 user: "ubuntu".to_string(),
4772 identity_file: "/path/to/key".to_string(),
4773 slots_available: 16,
4774 speed_score: 90.5,
4775 declared_os: None,
4776 }),
4777 reason: SelectionReason::Success,
4778 build_id: None,
4779 diagnostics: None,
4780 };
4781
4782 let json = serde_json::to_string(&original).unwrap();
4783 let parsed: SelectionResponse = serde_json::from_str(&json).unwrap();
4784
4785 assert!(parsed.worker.is_some());
4786 let worker = parsed.worker.unwrap();
4787 assert_eq!(worker.id.as_str(), "worker-1");
4788 assert_eq!(worker.host, "192.168.1.100");
4789 assert_eq!(worker.slots_available, 16);
4790 assert_eq!(parsed.reason, SelectionReason::Success);
4791 }
4792
4793 #[test]
4794 fn test_build_heartbeat_phase_roundtrip() {
4795 let _guard = test_guard!();
4796 let json = serde_json::to_string(&BuildHeartbeatPhase::SyncUp).unwrap();
4797 assert_eq!(json, "\"sync_up\"");
4798 let parsed: BuildHeartbeatPhase = serde_json::from_str(&json).unwrap();
4799 assert_eq!(parsed, BuildHeartbeatPhase::SyncUp);
4800 }
4801
4802 #[test]
4803 fn test_build_heartbeat_request_roundtrip() {
4804 let _guard = test_guard!();
4805 let request = BuildHeartbeatRequest {
4806 build_id: 42,
4807 worker_id: WorkerId::new("worker-a"),
4808 hook_pid: Some(12345),
4809 local_wrapper_id: Some("wrapper-abc123".to_string()),
4810 remote_pgid_file: Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string()),
4811 phase: BuildHeartbeatPhase::Execute,
4812 detail: Some("Compiling crates".to_string()),
4813 progress_counter: Some(7),
4814 progress_percent: Some(42.5),
4815 };
4816
4817 let json = serde_json::to_string(&request).unwrap();
4818 let parsed: BuildHeartbeatRequest = serde_json::from_str(&json).unwrap();
4819 assert_eq!(parsed.build_id, 42);
4820 assert_eq!(parsed.worker_id.as_str(), "worker-a");
4821 assert_eq!(
4822 parsed.remote_pgid_file,
4823 Some("/tmp/rch/project/hash/.rch-run/42.pgid".to_string())
4824 );
4825 assert_eq!(parsed.phase, BuildHeartbeatPhase::Execute);
4826 assert_eq!(parsed.progress_counter, Some(7));
4827 assert_eq!(parsed.progress_percent, Some(42.5));
4828 }
4829
4830 #[test]
4833 fn test_circuit_stats_new() {
4834 let _guard = test_guard!();
4835 let stats = CircuitStats::new();
4836 assert_eq!(stats.state(), CircuitState::Closed);
4837 assert_eq!(stats.consecutive_failures(), 0);
4838 assert_eq!(stats.error_rate(), 0.0);
4839 assert!(stats.opened_at().is_none());
4840 }
4841
4842 #[test]
4843 fn test_circuit_stats_record_success() {
4844 let _guard = test_guard!();
4845 let mut stats = CircuitStats::new();
4846 stats.record_success();
4847 stats.record_success();
4848 assert_eq!(stats.error_rate(), 0.0);
4849 assert_eq!(stats.consecutive_failures(), 0);
4850 }
4851
4852 #[test]
4853 fn test_circuit_stats_record_failure() {
4854 let _guard = test_guard!();
4855 let mut stats = CircuitStats::new();
4856 stats.record_failure();
4857 stats.record_failure();
4858 assert_eq!(stats.consecutive_failures(), 2);
4859 assert_eq!(stats.error_rate(), 1.0); }
4861
4862 #[test]
4863 fn test_circuit_stats_error_rate() {
4864 let _guard = test_guard!();
4865 let mut stats = CircuitStats::new();
4866 stats.record_success();
4867 stats.record_success();
4868 stats.record_failure();
4869 assert!((stats.error_rate() - 0.333).abs() < 0.01);
4871 }
4872
4873 #[test]
4874 fn test_circuit_stats_should_open_consecutive_failures() {
4875 let _guard = test_guard!();
4876 let mut stats = CircuitStats::new();
4877 let config = CircuitBreakerConfig::default(); stats.record_failure();
4880 assert!(!stats.should_open(&config));
4881
4882 stats.record_failure();
4883 assert!(!stats.should_open(&config));
4884
4885 stats.record_failure();
4886 assert!(stats.should_open(&config)); }
4888
4889 #[test]
4890 fn test_circuit_stats_should_open_error_rate() {
4891 let _guard = test_guard!();
4892 let mut stats = CircuitStats::new();
4893 let config = CircuitBreakerConfig {
4894 error_rate_threshold: 0.5,
4895 ..Default::default()
4896 };
4897
4898 stats.record_success();
4900 stats.record_success();
4901 stats.record_failure();
4902 stats.record_failure();
4903 assert!(!stats.should_open(&config)); stats.record_failure(); assert!(stats.should_open(&config));
4907 }
4908
4909 #[test]
4910 fn test_circuit_close_clears_recent_results_no_flap() {
4911 let _guard = test_guard!();
4912 let mut stats = CircuitStats::new();
4916 let config = CircuitBreakerConfig {
4917 error_rate_threshold: 0.5,
4918 ..Default::default()
4919 };
4920 for _ in 0..6 {
4921 stats.record_failure();
4922 }
4923 assert!(stats.should_open(&config));
4924 stats.open();
4925 stats.half_open();
4927 stats.record_success();
4928 stats.record_success();
4929 stats.close();
4930 assert_eq!(stats.error_rate(), 0.0);
4932 assert!(!stats.should_open(&config));
4933 }
4934
4935 #[test]
4936 fn test_apply_health_outcome_opens_on_failure_threshold() {
4937 let _guard = test_guard!();
4938 let config = CircuitBreakerConfig {
4940 failure_threshold: 3,
4941 open_cooldown_secs: 300,
4942 ..Default::default()
4943 };
4944 let mut stats = CircuitStats::new();
4945 assert_eq!(
4946 stats.apply_health_outcome(false, false, &config),
4947 CircuitState::Closed
4948 );
4949 assert_eq!(
4950 stats.apply_health_outcome(false, false, &config),
4951 CircuitState::Closed
4952 );
4953 assert_eq!(
4954 stats.apply_health_outcome(false, false, &config),
4955 CircuitState::Open
4956 );
4957 }
4958
4959 #[test]
4960 fn test_apply_health_outcome_half_open_two_probe_recovery() {
4961 let _guard = test_guard!();
4962 let config = CircuitBreakerConfig {
4968 failure_threshold: 3,
4969 success_threshold: 2,
4970 open_cooldown_secs: 0,
4971 ..Default::default()
4972 };
4973 let mut stats = CircuitStats::new();
4974 for _ in 0..3 {
4975 stats.apply_health_outcome(false, false, &config);
4976 }
4977 assert_eq!(stats.state(), CircuitState::HalfOpen);
4979
4980 let after_first = stats.apply_health_outcome(true, false, &config);
4982 assert_eq!(after_first, CircuitState::HalfOpen);
4983
4984 let after_second = stats.apply_health_outcome(true, false, &config);
4986 assert_eq!(after_second, CircuitState::Closed);
4987 }
4988
4989 #[test]
4990 fn test_apply_health_outcome_transient_half_open_does_not_reopen() {
4991 let _guard = test_guard!();
4992 let config = CircuitBreakerConfig {
4995 failure_threshold: 3,
4996 success_threshold: 2,
4997 open_cooldown_secs: 0,
4998 ..Default::default()
4999 };
5000 let mut stats = CircuitStats::new();
5001 for _ in 0..3 {
5002 stats.apply_health_outcome(false, false, &config);
5003 }
5004 assert_eq!(stats.state(), CircuitState::HalfOpen);
5006
5007 assert_eq!(
5009 stats.apply_health_outcome(true, false, &config),
5010 CircuitState::HalfOpen
5011 );
5012
5013 let after_transient = stats.apply_health_outcome(false, true, &config);
5019 assert_eq!(after_transient, CircuitState::HalfOpen);
5020
5021 assert_eq!(
5023 stats.apply_health_outcome(true, false, &config),
5024 CircuitState::HalfOpen
5025 );
5026 assert_eq!(
5027 stats.apply_health_outcome(true, false, &config),
5028 CircuitState::Closed
5029 );
5030 }
5031
5032 #[test]
5033 fn test_apply_health_outcome_real_vs_transient_half_open_failure() {
5034 let _guard = test_guard!();
5035 let config = CircuitBreakerConfig {
5043 failure_threshold: 3,
5044 success_threshold: 2,
5045 open_cooldown_secs: 300,
5046 ..Default::default()
5047 };
5048
5049 let mut real = CircuitStats::new();
5051 real.open();
5052 real.half_open();
5053 assert_eq!(real.state(), CircuitState::HalfOpen);
5054 let after_real = real.apply_health_outcome(false, false, &config);
5055 assert_eq!(
5056 after_real,
5057 CircuitState::Open,
5058 "a real half-open failure must reopen the circuit"
5059 );
5060
5061 let mut transient = CircuitStats::new();
5063 transient.open();
5064 transient.half_open();
5065 assert_eq!(transient.state(), CircuitState::HalfOpen);
5066 let after_transient = transient.apply_health_outcome(false, true, &config);
5067 assert_eq!(
5068 after_transient,
5069 CircuitState::HalfOpen,
5070 "a transient half-open failure must NOT reopen the circuit"
5071 );
5072 }
5073
5074 #[test]
5079 fn test_command_failure_blocks_half_open_close_on_liveness_successes() {
5080 let _guard = test_guard!();
5081 let config = CircuitBreakerConfig {
5082 failure_threshold: 3,
5083 success_threshold: 2,
5084 open_cooldown_secs: 0,
5085 command_failure_threshold: 2,
5086 ..Default::default()
5087 };
5088 let mut stats = CircuitStats::new();
5089 stats.open();
5090 stats.half_open();
5091
5092 assert_eq!(
5094 stats.apply_command_outcome(false, &config),
5095 CircuitState::HalfOpen,
5096 "cooldown 0: reopen then immediately half-open again"
5097 );
5098 assert_eq!(stats.consecutive_command_failures(), 1);
5099
5100 for _ in 0..10 {
5102 assert_eq!(
5103 stats.apply_health_outcome(true, false, &config),
5104 CircuitState::HalfOpen,
5105 "liveness successes must not close the circuit while a command failure is outstanding"
5106 );
5107 }
5108
5109 assert_eq!(
5111 stats.apply_command_outcome(true, &config),
5112 CircuitState::Closed
5113 );
5114 assert_eq!(stats.consecutive_command_failures(), 0);
5115 assert_eq!(stats.reopen_streak(), 0);
5116 }
5117
5118 #[test]
5121 fn test_command_failures_open_closed_circuit_despite_liveness_successes() {
5122 let _guard = test_guard!();
5123 let config = CircuitBreakerConfig {
5124 failure_threshold: 3,
5125 command_failure_threshold: 2,
5126 open_cooldown_secs: 300,
5127 ..Default::default()
5128 };
5129 let mut stats = CircuitStats::new();
5130
5131 assert_eq!(
5132 stats.apply_command_outcome(false, &config),
5133 CircuitState::Closed
5134 );
5135 stats.apply_health_outcome(true, false, &config);
5137 stats.apply_health_outcome(true, false, &config);
5138 assert_eq!(stats.consecutive_failures(), 0);
5139 assert_eq!(
5141 stats.apply_command_outcome(false, &config),
5142 CircuitState::Open
5143 );
5144 assert_eq!(stats.reopen_streak(), 1);
5145 }
5146
5147 #[test]
5151 fn test_command_failure_in_half_open_always_reopens() {
5152 let _guard = test_guard!();
5153 let config = CircuitBreakerConfig {
5154 open_cooldown_secs: 300,
5155 ..Default::default()
5156 };
5157 let mut stats = CircuitStats::new();
5158 stats.open();
5159 stats.half_open();
5160 assert_eq!(
5161 stats.apply_command_outcome(false, &config),
5162 CircuitState::Open
5163 );
5164 assert_eq!(stats.reopen_streak(), 2);
5165 }
5166
5167 #[test]
5171 fn test_reopen_streak_backs_off_cooldown_until_cap() {
5172 let _guard = test_guard!();
5173 let config = CircuitBreakerConfig {
5174 open_cooldown_secs: 30,
5175 open_cooldown_max_secs: 100,
5176 ..Default::default()
5177 };
5178 assert_eq!(config.cooldown_ms_for_streak(0), 30_000);
5179 assert_eq!(config.cooldown_ms_for_streak(1), 30_000);
5180 assert_eq!(config.cooldown_ms_for_streak(2), 60_000);
5181 assert_eq!(config.cooldown_ms_for_streak(3), 100_000, "capped");
5182 assert_eq!(config.cooldown_ms_for_streak(40), 100_000, "no overflow");
5183
5184 let inverted = CircuitBreakerConfig {
5186 open_cooldown_secs: 30,
5187 open_cooldown_max_secs: 5,
5188 ..Default::default()
5189 };
5190 assert_eq!(inverted.cooldown_ms_for_streak(3), 30_000);
5191
5192 let mut stats = CircuitStats::new();
5195 stats.open();
5196 stats.half_open();
5197 stats.open();
5198 assert_eq!(stats.reopen_streak(), 2);
5199 let remaining = stats.recovery_remaining_secs(&config).unwrap();
5200 assert!(
5201 remaining > 30 && remaining <= 60,
5202 "second open should wait ~60s, got {remaining}"
5203 );
5204
5205 stats.close();
5207 assert_eq!(stats.reopen_streak(), 0);
5208 stats.open();
5209 assert!(stats.recovery_remaining_secs(&config).unwrap() <= 30);
5210 }
5211
5212 #[test]
5213 fn test_circuit_stats_success_resets_consecutive_failures() {
5214 let _guard = test_guard!();
5215 let mut stats = CircuitStats::new();
5216 stats.record_failure();
5217 stats.record_failure();
5218 assert_eq!(stats.consecutive_failures(), 2);
5219
5220 stats.record_success();
5221 assert_eq!(stats.consecutive_failures(), 0);
5222 }
5223
5224 #[test]
5225 fn test_circuit_stats_open_transition() {
5226 let _guard = test_guard!();
5227 let mut stats = CircuitStats::new();
5228 stats.open();
5229
5230 assert_eq!(stats.state(), CircuitState::Open);
5231 assert!(stats.opened_at().is_some());
5232 }
5233
5234 #[test]
5235 fn test_circuit_stats_half_open_transition() {
5236 let _guard = test_guard!();
5237 let mut stats = CircuitStats::new();
5238 stats.open();
5239 stats.half_open();
5240
5241 assert_eq!(stats.state(), CircuitState::HalfOpen);
5242 }
5243
5244 #[test]
5245 fn test_circuit_stats_close_transition() {
5246 let _guard = test_guard!();
5247 let mut stats = CircuitStats::new();
5248 stats.open();
5249 stats.half_open();
5250 stats.close();
5251
5252 assert_eq!(stats.state(), CircuitState::Closed);
5253 assert!(stats.opened_at().is_none());
5254 assert_eq!(stats.consecutive_failures(), 0);
5255 }
5256
5257 #[test]
5258 fn test_circuit_stats_should_close() {
5259 let _guard = test_guard!();
5260 let mut stats = CircuitStats::new();
5261 let config = CircuitBreakerConfig {
5262 success_threshold: 2,
5263 ..Default::default()
5264 };
5265
5266 stats.open();
5267 stats.half_open();
5268
5269 assert!(!stats.should_close(&config)); stats.record_success();
5272 assert!(!stats.should_close(&config)); stats.record_success();
5275 assert!(stats.should_close(&config)); }
5277
5278 #[test]
5279 fn test_circuit_stats_can_probe() {
5280 let _guard = test_guard!();
5281 let mut stats = CircuitStats::new();
5282 let config = CircuitBreakerConfig {
5283 half_open_max_probes: 1,
5284 ..Default::default()
5285 };
5286
5287 assert!(!stats.can_probe(&config));
5289
5290 stats.open();
5291 assert!(!stats.can_probe(&config));
5293
5294 stats.half_open();
5295 assert!(stats.can_probe(&config));
5297
5298 assert!(stats.start_probe(&config));
5300 assert!(!stats.can_probe(&config));
5302 assert!(!stats.start_probe(&config));
5303 }
5304
5305 #[test]
5306 fn test_circuit_stats_probe_completion() {
5307 let _guard = test_guard!();
5308 let mut stats = CircuitStats::new();
5309 let config = CircuitBreakerConfig {
5310 half_open_max_probes: 1,
5311 ..Default::default()
5312 };
5313
5314 stats.open();
5315 stats.half_open();
5316 stats.start_probe(&config);
5317
5318 stats.record_success();
5320
5321 assert!(stats.can_probe(&config));
5323 }
5324
5325 #[test]
5326 fn test_circuit_stats_failure_in_half_open() {
5327 let _guard = test_guard!();
5328 let mut stats = CircuitStats::new();
5329 let config = CircuitBreakerConfig {
5330 success_threshold: 2,
5331 ..Default::default()
5332 };
5333
5334 stats.open();
5335 stats.half_open();
5336
5337 stats.record_success();
5338 assert_eq!(stats.consecutive_failures(), 0);
5339
5340 stats.record_failure();
5342 assert!(!stats.should_close(&config));
5343
5344 stats.record_success();
5346 stats.record_success();
5347 assert!(stats.should_close(&config));
5348 }
5349
5350 #[test]
5351 fn test_circuit_stats_reset_window() {
5352 let _guard = test_guard!();
5353 let mut stats = CircuitStats::new();
5354 stats.record_success();
5355 stats.record_failure();
5356
5357 assert!(stats.error_rate() > 0.0);
5358
5359 stats.reset_window();
5360 assert_eq!(stats.error_rate(), 0.0);
5361 }
5362
5363 #[test]
5364 fn test_circuit_state_transitions_are_deterministic() {
5365 let _guard = test_guard!();
5366 let config = CircuitBreakerConfig::default();
5367 let mut stats = CircuitStats::new();
5368
5369 assert_eq!(stats.state(), CircuitState::Closed);
5371
5372 for _ in 0..3 {
5374 stats.record_failure();
5375 }
5376 assert!(stats.should_open(&config));
5377 stats.open();
5378 assert_eq!(stats.state(), CircuitState::Open);
5379
5380 stats.half_open();
5382 assert_eq!(stats.state(), CircuitState::HalfOpen);
5383
5384 for _ in 0..2 {
5386 stats.record_success();
5387 }
5388 assert!(stats.should_close(&config));
5389 stats.close();
5390 assert_eq!(stats.state(), CircuitState::Closed);
5391 }
5392
5393 fn make_test_metrics(
5398 speedup: Option<f64>,
5399 total_secs: u64,
5400 success: bool,
5401 ) -> CompilationMetrics {
5402 CompilationMetrics {
5403 project_id: "test-project".to_string(),
5404 worker_id: "worker-1".to_string(),
5405 timestamp: chrono::Utc::now(),
5406 timing: CompilationTimingBreakdown {
5407 rsync_up: Duration::from_millis(100),
5408 remote_build: Duration::from_millis(800),
5409 rsync_down: Duration::from_millis(100),
5410 total: Duration::from_secs(total_secs),
5411 },
5412 local_build_time: speedup.map(|s| Duration::from_secs_f64(total_secs as f64 * s)),
5413 speedup,
5414 files_synced: 100,
5415 bytes_transferred: 1_000_000,
5416 exit_code: if success { 0 } else { 1 },
5417 success,
5418 }
5419 }
5420
5421 #[test]
5422 fn test_compilation_timing_breakdown_default() {
5423 let _guard = test_guard!();
5424 let timing = CompilationTimingBreakdown::default();
5425 assert_eq!(timing.rsync_up, Duration::ZERO);
5426 assert_eq!(timing.remote_build, Duration::ZERO);
5427 assert_eq!(timing.rsync_down, Duration::ZERO);
5428 assert_eq!(timing.total, Duration::ZERO);
5429 }
5430
5431 #[test]
5432 fn test_compilation_timing_breakdown_serialization() {
5433 let _guard = test_guard!();
5434 let timing = CompilationTimingBreakdown {
5435 rsync_up: Duration::from_millis(100),
5436 remote_build: Duration::from_millis(2000),
5437 rsync_down: Duration::from_millis(50),
5438 total: Duration::from_millis(2150),
5439 };
5440
5441 let json = serde_json::to_string(&timing).unwrap();
5442 assert!(json.contains("100")); assert!(json.contains("2000")); assert!(json.contains("50")); assert!(json.contains("2150")); let parsed: CompilationTimingBreakdown = serde_json::from_str(&json).unwrap();
5448 assert_eq!(parsed.rsync_up, timing.rsync_up);
5449 assert_eq!(parsed.remote_build, timing.remote_build);
5450 assert_eq!(parsed.rsync_down, timing.rsync_down);
5451 assert_eq!(parsed.total, timing.total);
5452 }
5453
5454 #[test]
5455 fn test_command_timing_breakdown_serialization() {
5456 let _guard = test_guard!();
5457 let timing = CommandTimingBreakdown {
5458 classify: Some(Duration::from_millis(2)),
5459 select: None,
5460 sync_up: Some(Duration::from_millis(120)),
5461 exec: Some(Duration::from_millis(900)),
5462 sync_down: None,
5463 cleanup: Some(Duration::from_millis(5)),
5464 total: Some(Duration::from_millis(1027)),
5465 };
5466
5467 let json = serde_json::to_string(&timing).unwrap();
5468 assert!(json.contains("\"classify\":2"));
5469 assert!(json.contains("\"select\":null"));
5470 assert!(json.contains("\"sync_up\":120"));
5471 assert!(json.contains("\"exec\":900"));
5472 assert!(json.contains("\"sync_down\":null"));
5473 assert!(json.contains("\"cleanup\":5"));
5474 assert!(json.contains("\"total\":1027"));
5475
5476 let parsed: CommandTimingBreakdown = serde_json::from_str(&json).unwrap();
5477 assert_eq!(parsed.classify, timing.classify);
5478 assert_eq!(parsed.select, timing.select);
5479 assert_eq!(parsed.sync_up, timing.sync_up);
5480 assert_eq!(parsed.exec, timing.exec);
5481 assert_eq!(parsed.sync_down, timing.sync_down);
5482 assert_eq!(parsed.cleanup, timing.cleanup);
5483 assert_eq!(parsed.total, timing.total);
5484 }
5485
5486 #[test]
5487 fn test_compilation_metrics_calculate_speedup() {
5488 let _guard = test_guard!();
5489 let mut metrics = CompilationMetrics {
5490 timing: CompilationTimingBreakdown {
5491 total: Duration::from_secs(10),
5492 ..Default::default()
5493 },
5494 local_build_time: Some(Duration::from_secs(30)),
5495 ..Default::default()
5496 };
5497
5498 metrics.calculate_speedup();
5499
5500 assert!(metrics.speedup.is_some());
5501 let speedup = metrics.speedup.unwrap();
5502 assert!(
5503 (speedup - 3.0).abs() < 0.01,
5504 "Expected 3.0x speedup, got {}",
5505 speedup
5506 );
5507 }
5508
5509 #[test]
5510 fn test_compilation_metrics_is_beneficial() {
5511 let _guard = test_guard!();
5512 let mut metrics = CompilationMetrics::default();
5513
5514 assert!(!metrics.is_beneficial());
5516
5517 metrics.speedup = Some(1.5);
5519 assert!(metrics.is_beneficial());
5520
5521 metrics.speedup = Some(0.8);
5523 assert!(!metrics.is_beneficial());
5524
5525 metrics.speedup = Some(1.0);
5527 assert!(!metrics.is_beneficial());
5528 }
5529
5530 #[test]
5531 fn test_compilation_metrics_serialization() {
5532 let _guard = test_guard!();
5533 let metrics = make_test_metrics(Some(2.5), 10, true);
5534 let json = serde_json::to_string(&metrics).unwrap();
5535
5536 assert!(json.contains("test-project"));
5537 assert!(json.contains("worker-1"));
5538 assert!(json.contains("2.5")); let parsed: CompilationMetrics = serde_json::from_str(&json).unwrap();
5541 assert_eq!(parsed.project_id, "test-project");
5542 assert_eq!(parsed.worker_id, "worker-1");
5543 assert!(parsed.success);
5544 }
5545
5546 #[test]
5547 fn test_compilation_timer_new() {
5548 let _guard = test_guard!();
5549 let timer = CompilationTimer::new("my-project", "my-worker");
5550 assert_eq!(timer.project_id, "my-project");
5551 assert_eq!(timer.worker_id, "my-worker");
5552 assert!(timer.rsync_up.is_none());
5553 assert!(timer.remote_build.is_none());
5554 assert!(timer.rsync_down.is_none());
5555 }
5556
5557 #[test]
5558 fn test_compilation_timer_phases() {
5559 let _guard = test_guard!();
5560 let mut timer = CompilationTimer::new("test", "worker");
5561
5562 std::thread::sleep(Duration::from_millis(5));
5564 timer.end_rsync_up();
5565 assert!(timer.rsync_up.is_some());
5566 assert!(timer.rsync_up.unwrap() >= Duration::from_millis(5));
5567
5568 std::thread::sleep(Duration::from_millis(10));
5570 timer.end_remote_build();
5571 assert!(timer.remote_build.is_some());
5572 assert!(timer.remote_build.unwrap() >= Duration::from_millis(10));
5573
5574 std::thread::sleep(Duration::from_millis(5));
5576 timer.end_rsync_down();
5577 assert!(timer.rsync_down.is_some());
5578 assert!(timer.rsync_down.unwrap() >= Duration::from_millis(5));
5579
5580 let metrics = timer.finish(0, 50, 500_000);
5582 assert_eq!(metrics.project_id, "test");
5583 assert_eq!(metrics.worker_id, "worker");
5584 assert!(metrics.success);
5585 assert!(metrics.timing.total >= Duration::from_millis(20));
5586 assert_eq!(metrics.files_synced, 50);
5587 assert_eq!(metrics.bytes_transferred, 500_000);
5588 }
5589
5590 #[test]
5591 fn test_metrics_aggregator_new() {
5592 let _guard = test_guard!();
5593 let agg = MetricsAggregator::new(100);
5594 assert_eq!(agg.count(), 0);
5595 assert!(agg.average_speedup().is_none());
5596 }
5597
5598 #[test]
5599 fn test_metrics_aggregator_record() {
5600 let _guard = test_guard!();
5601 let mut agg = MetricsAggregator::new(3);
5602
5603 agg.record(make_test_metrics(Some(2.0), 10, true));
5605 assert_eq!(agg.count(), 1);
5606
5607 agg.record(make_test_metrics(Some(3.0), 20, true));
5609 agg.record(make_test_metrics(Some(4.0), 30, true));
5610 assert_eq!(agg.count(), 3);
5611
5612 agg.record(make_test_metrics(Some(5.0), 40, true));
5614 assert_eq!(agg.count(), 3);
5615 }
5616
5617 #[test]
5618 fn test_metrics_aggregator_average_speedup() {
5619 let _guard = test_guard!();
5620 let mut agg = MetricsAggregator::new(100);
5621
5622 assert!(agg.average_speedup().is_none());
5624
5625 for i in 1..=5 {
5627 agg.record(make_test_metrics(Some(i as f64), 10, true));
5628 }
5629
5630 let avg = agg.average_speedup().unwrap();
5631 assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5632 }
5633
5634 #[test]
5635 fn test_metrics_aggregator_average_speedup_with_none() {
5636 let _guard = test_guard!();
5637 let mut agg = MetricsAggregator::new(100);
5638
5639 agg.record(make_test_metrics(Some(2.0), 10, true));
5641 agg.record(make_test_metrics(None, 10, true)); agg.record(make_test_metrics(Some(4.0), 10, true));
5643
5644 let avg = agg.average_speedup().unwrap();
5646 assert!((avg - 3.0).abs() < 0.01, "Expected 3.0, got {}", avg);
5647 }
5648
5649 #[test]
5650 fn test_metrics_aggregator_percentiles() {
5651 let _guard = test_guard!();
5652 let mut agg = MetricsAggregator::new(100);
5653
5654 for i in 1..=10 {
5656 agg.record(make_test_metrics(Some(1.0), i, true));
5657 }
5658
5659 let p50 = agg.p50_total_time().unwrap();
5661 assert!(p50 >= Duration::from_secs(5) && p50 <= Duration::from_secs(6));
5662
5663 let p95 = agg.p95_total_time().unwrap();
5665 assert!(p95 >= Duration::from_secs(9) && p95 <= Duration::from_secs(10));
5666
5667 let p99 = agg.p99_total_time().unwrap();
5669 assert!(p99 >= Duration::from_secs(9));
5670 }
5671
5672 #[test]
5673 fn test_metrics_aggregator_success_rate() {
5674 let _guard = test_guard!();
5675 let mut agg = MetricsAggregator::new(100);
5676
5677 assert_eq!(agg.success_rate(), 100.0);
5679
5680 agg.record(make_test_metrics(Some(1.0), 10, true));
5682 agg.record(make_test_metrics(Some(1.0), 10, true));
5683 assert_eq!(agg.success_rate(), 100.0);
5684
5685 agg.record(make_test_metrics(Some(1.0), 10, false));
5687 let rate = agg.success_rate();
5688 assert!((rate - 66.67).abs() < 1.0, "Expected ~66.67%, got {}", rate);
5689 }
5690
5691 #[test]
5692 fn test_metrics_aggregator_clear() {
5693 let _guard = test_guard!();
5694 let mut agg = MetricsAggregator::new(100);
5695 agg.record(make_test_metrics(Some(1.0), 10, true));
5696 agg.record(make_test_metrics(Some(2.0), 20, true));
5697
5698 assert_eq!(agg.count(), 2);
5699
5700 agg.clear();
5701 assert_eq!(agg.count(), 0);
5702 assert!(agg.average_speedup().is_none());
5703 }
5704
5705 #[test]
5708 fn test_remote_build_jobs_parse_display_and_serde() {
5709 let _guard = test_guard!();
5710 assert_eq!(RemoteBuildJobs::default(), RemoteBuildJobs::Auto);
5711 for (spelling, expected) in [
5712 ("auto", RemoteBuildJobs::Auto),
5713 (" AUTO ", RemoteBuildJobs::Auto),
5714 ("off", RemoteBuildJobs::Off),
5715 ("none", RemoteBuildJobs::Off),
5716 ("4", RemoteBuildJobs::Fixed(4)),
5717 ] {
5718 assert_eq!(
5719 RemoteBuildJobs::parse(spelling).unwrap(),
5720 expected,
5721 "{spelling}"
5722 );
5723 }
5724 assert!(RemoteBuildJobs::parse("0").is_err(), "cargo rejects -j 0");
5725 assert!(RemoteBuildJobs::parse("many").is_err());
5726 assert_eq!(RemoteBuildJobs::Fixed(4).to_string(), "4");
5727 assert_eq!(RemoteBuildJobs::Off.to_string(), "off");
5728
5729 #[derive(Deserialize)]
5732 struct Wrap {
5733 compilation: CompilationConfig,
5734 }
5735 let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = \"off\"\n").unwrap();
5736 assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Off);
5737 let parsed: Wrap = toml::from_str("[compilation]\nremote_build_jobs = 6\n").unwrap();
5738 assert_eq!(
5739 parsed.compilation.remote_build_jobs,
5740 RemoteBuildJobs::Fixed(6)
5741 );
5742 let parsed: Wrap = toml::from_str("[compilation]\nbuild_slots = 4\n").unwrap();
5743 assert_eq!(parsed.compilation.remote_build_jobs, RemoteBuildJobs::Auto);
5744 assert!(toml::from_str::<Wrap>("[compilation]\nremote_build_jobs = 0\n").is_err());
5745
5746 let json = serde_json::to_string(&RemoteBuildJobs::Fixed(3)).unwrap();
5748 assert_eq!(json, "3");
5749 assert_eq!(
5750 serde_json::to_string(&RemoteBuildJobs::Auto).unwrap(),
5751 "\"auto\""
5752 );
5753 let back: RemoteBuildJobs = serde_json::from_str("\"auto\"").unwrap();
5754 assert_eq!(back, RemoteBuildJobs::Auto);
5755 }
5756
5757 #[test]
5758 fn test_compilation_config_default_timeouts() {
5759 let _guard = test_guard!();
5760 let config = CompilationConfig::default();
5761 assert_eq!(config.build_timeout_sec, 300);
5763 assert_eq!(config.test_timeout_sec, 1800);
5765 }
5766
5767 #[test]
5768 fn test_compilation_config_timeout_for_test_kinds() {
5769 let _guard = test_guard!();
5770 let config = CompilationConfig::default();
5771
5772 assert_eq!(
5774 config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5775 std::time::Duration::from_secs(1800)
5776 );
5777 assert_eq!(
5778 config.timeout_for_kind(Some(crate::CompilationKind::CargoNextest)),
5779 std::time::Duration::from_secs(1800)
5780 );
5781 assert_eq!(
5782 config.timeout_for_kind(Some(crate::CompilationKind::BunTest)),
5783 std::time::Duration::from_secs(config.bun_timeout_sec)
5784 );
5785 }
5786
5787 #[test]
5788 fn test_compilation_config_timeout_for_build_kinds() {
5789 let _guard = test_guard!();
5790 let config = CompilationConfig::default();
5791
5792 assert_eq!(
5794 config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5795 std::time::Duration::from_secs(300)
5796 );
5797 assert_eq!(
5798 config.timeout_for_kind(Some(crate::CompilationKind::CargoCheck)),
5799 std::time::Duration::from_secs(300)
5800 );
5801 assert_eq!(
5802 config.timeout_for_kind(Some(crate::CompilationKind::CargoClippy)),
5803 std::time::Duration::from_secs(300)
5804 );
5805 assert_eq!(
5806 config.timeout_for_kind(None),
5807 std::time::Duration::from_secs(300)
5808 );
5809 }
5810
5811 #[test]
5812 fn test_compilation_config_custom_timeouts() {
5813 let _guard = test_guard!();
5814 let config = CompilationConfig {
5815 build_timeout_sec: 600, test_timeout_sec: 3600, ..Default::default()
5818 };
5819
5820 assert_eq!(
5821 config.timeout_for_kind(Some(crate::CompilationKind::CargoBuild)),
5822 std::time::Duration::from_secs(600)
5823 );
5824 assert_eq!(
5825 config.timeout_for_kind(Some(crate::CompilationKind::CargoTest)),
5826 std::time::Duration::from_secs(3600)
5827 );
5828 }
5829
5830 #[test]
5831 fn test_compilation_config_speedup_threshold_default() {
5832 let _guard = test_guard!();
5833 let config = CompilationConfig::default();
5834 assert!((config.remote_speedup_threshold - 1.2).abs() < 0.001);
5836 }
5837
5838 #[test]
5839 fn test_compilation_config_speedup_threshold_custom() {
5840 let _guard = test_guard!();
5841 let config = CompilationConfig {
5842 remote_speedup_threshold: 1.5, ..Default::default()
5844 };
5845 assert!((config.remote_speedup_threshold - 1.5).abs() < 0.001);
5846 }
5847
5848 #[test]
5849 fn test_compilation_config_speedup_threshold_no_minimum() {
5850 let _guard = test_guard!();
5851 let config = CompilationConfig {
5853 remote_speedup_threshold: 1.0,
5854 ..Default::default()
5855 };
5856 assert!((config.remote_speedup_threshold - 1.0).abs() < 0.001);
5857 }
5858
5859 #[test]
5860 fn test_compilation_config_min_local_time_ms_default() {
5861 let _guard = test_guard!();
5862 let config = CompilationConfig::default();
5863 assert_eq!(config.min_local_time_ms, 2000);
5865 }
5866
5867 #[test]
5868 fn test_compilation_config_serde_roundtrip_speedup_threshold() {
5869 let _guard = test_guard!();
5870 let config = CompilationConfig {
5871 remote_speedup_threshold: 2.5,
5872 min_local_time_ms: 5000,
5873 ..Default::default()
5874 };
5875 let json = serde_json::to_string(&config).unwrap();
5876 let parsed: CompilationConfig = serde_json::from_str(&json).unwrap();
5877 assert!((parsed.remote_speedup_threshold - 2.5).abs() < 0.001);
5878 assert_eq!(parsed.min_local_time_ms, 5000);
5879 }
5880
5881 #[test]
5886 fn test_validate_remote_base_absolute_path() {
5887 let _guard = test_guard!();
5888 assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5889 assert_eq!(
5890 validate_remote_base("/var/rch-builds").unwrap(),
5891 "/var/rch-builds"
5892 );
5893 assert_eq!(
5894 validate_remote_base("/home/builder/.rch").unwrap(),
5895 "/home/builder/.rch"
5896 );
5897 }
5898
5899 #[test]
5900 fn test_validate_remote_base_tilde_expansion() {
5901 let _guard = test_guard!();
5902 let result = validate_remote_base("~/rch");
5904 assert!(result.is_ok());
5905 let path = result.unwrap();
5906 assert!(
5907 path.starts_with('/'),
5908 "Path should be absolute after expansion: {}",
5909 path
5910 );
5911 assert!(!path.contains('~'), "Tilde should be expanded: {}", path);
5912 }
5913
5914 #[test]
5915 fn test_validate_remote_base_rejects_relative_path() {
5916 let _guard = test_guard!();
5917 let result = validate_remote_base("tmp/rch");
5918 assert!(result.is_err());
5919 assert!(result.unwrap_err().contains("absolute path"));
5920 }
5921
5922 #[test]
5923 fn test_validate_remote_base_rejects_path_traversal() {
5924 let _guard = test_guard!();
5925 let result = validate_remote_base("/tmp/../etc/rch");
5926 assert!(result.is_err());
5927 assert!(result.unwrap_err().contains("path traversal"));
5928
5929 let result = validate_remote_base("/tmp/rch/../other");
5930 assert!(result.is_err());
5931 assert!(result.unwrap_err().contains("path traversal"));
5932 }
5933
5934 #[test]
5935 fn test_validate_remote_base_normalizes_trailing_slash() {
5936 let _guard = test_guard!();
5937 assert_eq!(validate_remote_base("/tmp/rch/").unwrap(), "/tmp/rch");
5938 assert_eq!(validate_remote_base("/tmp/rch///").unwrap(), "/tmp/rch");
5939 }
5940
5941 #[test]
5942 fn test_validate_remote_base_root_path() {
5943 let _guard = test_guard!();
5944 let result = validate_remote_base("/");
5946 assert!(result.is_err());
5947 assert!(result.unwrap_err().contains("root directory"));
5948 }
5949
5950 #[test]
5951 fn test_validate_remote_base_rejects_top_level() {
5952 let _guard = test_guard!();
5953 let result = validate_remote_base("/tmp");
5955 assert!(result.is_err());
5956 assert!(result.unwrap_err().contains("at least 2 levels deep"));
5957
5958 let result = validate_remote_base("/home");
5959 assert!(result.is_err());
5960
5961 assert_eq!(validate_remote_base("/tmp/rch").unwrap(), "/tmp/rch");
5963 }
5964
5965 #[test]
5966 fn test_default_remote_base() {
5967 let _guard = test_guard!();
5968 assert_eq!(default_remote_base(), "/data/tmp/rch");
5969 }
5970
5971 #[test]
5972 fn test_transfer_config_default_has_remote_base() {
5973 let _guard = test_guard!();
5974 let config = TransferConfig::default();
5975 assert_eq!(config.remote_base, "/data/tmp/rch");
5976 }
5977
5978 #[test]
5979 fn test_source_sync_timeout_validation_boundaries() {
5980 let _guard = test_guard!();
5981 assert!(!TransferConfig::valid_sync_timeout_ms(0));
5982 assert!(!TransferConfig::valid_sync_timeout_ms(999));
5983 assert!(TransferConfig::valid_sync_timeout_ms(1_000));
5984 assert!(TransferConfig::valid_sync_timeout_ms(3_600_000));
5985 assert!(!TransferConfig::valid_sync_timeout_ms(3_600_001));
5986 }
5987
5988 #[test]
5989 fn test_source_sync_timeout_is_explicit_or_payload_aware() {
5990 let _guard = test_guard!();
5991 let default = TransferConfig::default();
5992 assert_eq!(
5993 default.sync_timeout_for_payload(0),
5994 std::time::Duration::from_millis(30_000)
5995 );
5996 assert_eq!(
5997 default.sync_timeout_for_payload(10 * 1024 * 1024),
5998 std::time::Duration::from_millis(40_000)
5999 );
6000 assert_eq!(
6001 default.sync_timeout_for_payload(u64::MAX),
6002 std::time::Duration::from_millis(3_600_000)
6003 );
6004
6005 let explicit = TransferConfig {
6006 sync_timeout_ms: Some(120_000),
6007 ..TransferConfig::default()
6008 };
6009 assert_eq!(
6010 explicit.sync_timeout_for_payload(u64::MAX),
6011 std::time::Duration::from_millis(120_000)
6012 );
6013 }
6014
6015 #[test]
6020 fn test_retry_config_default() {
6021 let _guard = test_guard!();
6022 let config = RetryConfig::default();
6023 assert_eq!(config.max_attempts, 3);
6024 assert_eq!(config.base_delay_ms, 100);
6025 assert_eq!(config.max_delay_ms, 5000);
6026 assert_eq!(config.jitter_factor, 0.1);
6027 assert_eq!(config.total_timeout_ms, 30000);
6028 }
6029
6030 #[test]
6031 fn test_retry_config_no_retry() {
6032 let _guard = test_guard!();
6033 let config = RetryConfig::no_retry();
6034 assert_eq!(config.max_attempts, 1);
6035 }
6036
6037 #[test]
6038 fn test_retry_config_delay_for_attempt_zero() {
6039 let _guard = test_guard!();
6040 let config = RetryConfig::default();
6041 assert_eq!(config.delay_for_attempt(0), std::time::Duration::ZERO);
6042 }
6043
6044 #[test]
6045 fn test_retry_config_delay_for_attempt_exponential() {
6046 let _guard = test_guard!();
6047 let config = RetryConfig {
6048 base_delay_ms: 100,
6049 max_delay_ms: 10000,
6050 jitter_factor: 0.0, ..Default::default()
6052 };
6053
6054 assert_eq!(
6056 config.delay_for_attempt(1),
6057 std::time::Duration::from_millis(100)
6058 );
6059 assert_eq!(
6061 config.delay_for_attempt(2),
6062 std::time::Duration::from_millis(200)
6063 );
6064 assert_eq!(
6066 config.delay_for_attempt(3),
6067 std::time::Duration::from_millis(400)
6068 );
6069 }
6070
6071 #[test]
6072 fn test_retry_config_delay_capped_at_max() {
6073 let _guard = test_guard!();
6074 let config = RetryConfig {
6075 base_delay_ms: 1000,
6076 max_delay_ms: 2000,
6077 jitter_factor: 0.0,
6078 ..Default::default()
6079 };
6080
6081 assert_eq!(
6083 config.delay_for_attempt(2),
6084 std::time::Duration::from_millis(2000)
6085 );
6086 assert_eq!(
6088 config.delay_for_attempt(3),
6089 std::time::Duration::from_millis(2000)
6090 );
6091 }
6092
6093 #[test]
6094 fn test_retry_config_random_jitter_stays_within_bounds() {
6095 let _guard = test_guard!();
6096 let config = RetryConfig {
6097 base_delay_ms: 100,
6098 max_delay_ms: 200,
6099 jitter_factor: 0.25,
6100 ..Default::default()
6101 };
6102
6103 for (attempt, minimum_ms, maximum_ms) in [(1, 75, 125), (3, 150, 250)] {
6106 for _ in 0..128 {
6107 let delay = config.delay_for_attempt(attempt);
6108 assert!(delay >= std::time::Duration::from_millis(minimum_ms));
6109 assert!(delay <= std::time::Duration::from_millis(maximum_ms));
6110 }
6111 }
6112 }
6113
6114 #[test]
6115 fn test_retry_config_should_retry() {
6116 let _guard = test_guard!();
6117 let config = RetryConfig {
6118 max_attempts: 3,
6119 total_timeout_ms: 1000,
6120 ..Default::default()
6121 };
6122
6123 assert!(config.should_retry(0, std::time::Duration::from_millis(0)));
6125 assert!(config.should_retry(1, std::time::Duration::from_millis(500)));
6126 assert!(config.should_retry(2, std::time::Duration::from_millis(900)));
6127
6128 assert!(!config.should_retry(3, std::time::Duration::from_millis(0)));
6130
6131 assert!(!config.should_retry(1, std::time::Duration::from_millis(1001)));
6133 }
6134
6135 #[test]
6136 fn test_retry_config_serde_roundtrip() {
6137 let _guard = test_guard!();
6138 let config = RetryConfig {
6139 max_attempts: 5,
6140 base_delay_ms: 200,
6141 max_delay_ms: 8000,
6142 jitter_factor: 0.2,
6143 total_timeout_ms: 60000,
6144 };
6145 let json = serde_json::to_string(&config).unwrap();
6146 let parsed: RetryConfig = serde_json::from_str(&json).unwrap();
6147 assert_eq!(parsed.max_attempts, 5);
6148 assert_eq!(parsed.base_delay_ms, 200);
6149 assert_eq!(parsed.max_delay_ms, 8000);
6150 assert_eq!(parsed.jitter_factor, 0.2);
6151 assert_eq!(parsed.total_timeout_ms, 60000);
6152 }
6153
6154 #[test]
6155 fn test_transfer_config_includes_retry() {
6156 let _guard = test_guard!();
6157 let config = TransferConfig::default();
6158 assert_eq!(config.retry.max_attempts, 3);
6159 }
6160
6161 #[test]
6166 fn test_execution_config_default_allowlist() {
6167 let _guard = test_guard!();
6168 let config = ExecutionConfig::default();
6169 assert!(config.is_allowed("cargo"));
6171 assert!(config.is_allowed("rustc"));
6172 assert!(config.is_allowed("gcc"));
6173 assert!(config.is_allowed("g++"));
6174 assert!(config.is_allowed("clang"));
6175 assert!(config.is_allowed("clang++"));
6176 assert!(config.is_allowed("make"));
6177 assert!(config.is_allowed("cmake"));
6178 assert!(config.is_allowed("ninja"));
6179 assert!(config.is_allowed("meson"));
6180 assert!(config.is_allowed("bun"));
6181 assert!(config.is_allowed("nextest"));
6182 assert!(config.is_allowed("cc"));
6183 assert!(config.is_allowed("c++"));
6184 }
6185
6186 #[test]
6187 fn test_execution_config_is_allowed_case_insensitive() {
6188 let _guard = test_guard!();
6189 let config = ExecutionConfig::default();
6190 assert!(config.is_allowed("CARGO"));
6192 assert!(config.is_allowed("Cargo"));
6193 assert!(config.is_allowed("GCC"));
6194 assert!(config.is_allowed("Gcc"));
6195 }
6196
6197 #[test]
6198 fn test_execution_config_is_not_allowed() {
6199 let _guard = test_guard!();
6200 let config = ExecutionConfig::default();
6201 assert!(!config.is_allowed("python"));
6203 assert!(!config.is_allowed("npm"));
6204 assert!(!config.is_allowed(""));
6205 assert!(config.is_allowed("go"));
6210 assert!(config.is_allowed("tsc"));
6211 }
6212
6213 #[test]
6214 fn test_execution_config_empty_allowlist() {
6215 let _guard = test_guard!();
6216 let config = ExecutionConfig { allowlist: vec![] };
6217 assert!(!config.is_allowed("cargo"));
6219 assert!(!config.is_allowed("gcc"));
6220 }
6221
6222 #[test]
6223 fn test_execution_config_custom_allowlist() {
6224 let _guard = test_guard!();
6225 let config = ExecutionConfig {
6226 allowlist: vec!["cargo".to_string(), "custom_tool".to_string()],
6227 };
6228 assert!(config.is_allowed("cargo"));
6229 assert!(config.is_allowed("custom_tool"));
6230 assert!(!config.is_allowed("gcc"));
6231 }
6232
6233 #[test]
6234 fn test_execution_config_serde() {
6235 let _guard = test_guard!();
6236 let config = ExecutionConfig {
6237 allowlist: vec!["cargo".to_string(), "rustc".to_string()],
6238 };
6239 let json = serde_json::to_string(&config).unwrap();
6240 let parsed: ExecutionConfig = serde_json::from_str(&json).unwrap();
6241 assert_eq!(parsed.allowlist, config.allowlist);
6242 }
6243
6244 #[test]
6245 fn test_rch_config_includes_execution() {
6246 let _guard = test_guard!();
6247 let config = RchConfig::default();
6248 assert!(config.execution.is_allowed("cargo"));
6250 }
6251
6252 #[test]
6257 fn test_adaptive_compression_disabled_by_default() {
6258 let _guard = test_guard!();
6259 let config = TransferConfig::default();
6260 assert!(!config.adaptive_compression);
6261 assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6263 assert_eq!(config.select_compression_level(Some(500_000_000)), 3);
6264 }
6265
6266 #[test]
6267 fn test_adaptive_compression_small_payload() {
6268 let _guard = test_guard!();
6269 let config = TransferConfig {
6270 adaptive_compression: true,
6271 ..Default::default()
6272 };
6273 assert_eq!(config.select_compression_level(Some(0)), 1);
6275 assert_eq!(config.select_compression_level(Some(1_000_000)), 1); assert_eq!(config.select_compression_level(Some(9_999_999)), 1); }
6278
6279 #[test]
6280 fn test_adaptive_compression_medium_payload() {
6281 let _guard = test_guard!();
6282 let config = TransferConfig {
6283 adaptive_compression: true,
6284 ..Default::default()
6285 };
6286 assert_eq!(config.select_compression_level(Some(10_000_000)), 3); assert_eq!(config.select_compression_level(Some(100_000_000)), 3); assert_eq!(config.select_compression_level(Some(199_999_999)), 3); }
6291
6292 #[test]
6293 fn test_adaptive_compression_large_payload() {
6294 let _guard = test_guard!();
6295 let config = TransferConfig {
6296 adaptive_compression: true,
6297 ..Default::default()
6298 };
6299 assert_eq!(config.select_compression_level(Some(200_000_000)), 7); assert_eq!(config.select_compression_level(Some(500_000_000)), 7); assert_eq!(config.select_compression_level(Some(1_000_000_000)), 7); }
6304
6305 #[test]
6306 fn test_adaptive_compression_no_estimate() {
6307 let _guard = test_guard!();
6308 let config = TransferConfig {
6309 adaptive_compression: true,
6310 compression_level: 5,
6311 ..Default::default()
6312 };
6313 assert_eq!(config.select_compression_level(None), 5);
6315 }
6316
6317 #[test]
6318 fn test_adaptive_compression_respects_min_level() {
6319 let _guard = test_guard!();
6320 let config = TransferConfig {
6321 adaptive_compression: true,
6322 min_compression_level: 3,
6323 ..Default::default()
6324 };
6325 assert_eq!(config.select_compression_level(Some(1_000_000)), 3);
6327 }
6328
6329 #[test]
6330 fn test_adaptive_compression_respects_max_level() {
6331 let _guard = test_guard!();
6332 let config = TransferConfig {
6333 adaptive_compression: true,
6334 max_compression_level: 5,
6335 ..Default::default()
6336 };
6337 assert_eq!(config.select_compression_level(Some(500_000_000)), 5);
6339 }
6340
6341 #[test]
6342 fn test_adaptive_compression_serde_roundtrip() {
6343 let _guard = test_guard!();
6344 let config = TransferConfig {
6345 adaptive_compression: true,
6346 min_compression_level: 2,
6347 max_compression_level: 8,
6348 ..Default::default()
6349 };
6350 let json = serde_json::to_string(&config).unwrap();
6351 let parsed: TransferConfig = serde_json::from_str(&json).unwrap();
6352 assert!(parsed.adaptive_compression);
6353 assert_eq!(parsed.min_compression_level, 2);
6354 assert_eq!(parsed.max_compression_level, 8);
6355 }
6356
6357 #[test]
6362 fn test_self_healing_config_defaults() {
6363 let _guard = test_guard!();
6364 let config = SelfHealingConfig::default();
6366
6367 assert!(
6369 config.hook_starts_daemon,
6370 "hook_starts_daemon should be true by default"
6371 );
6372 assert!(
6373 config.daemon_installs_hooks,
6374 "daemon_installs_hooks should be true by default"
6375 );
6376
6377 assert_eq!(
6379 config.auto_start_cooldown_secs, 30,
6380 "auto_start_cooldown_secs should default to 30"
6381 );
6382 assert_eq!(
6383 config.auto_start_timeout_secs, 3,
6384 "auto_start_timeout_secs should default to 3"
6385 );
6386 }
6388
6389 #[test]
6390 fn test_self_healing_config_serde_full() {
6391 let _guard = test_guard!();
6392 let config = SelfHealingConfig {
6394 hook_starts_daemon: false,
6395 daemon_installs_hooks: false,
6396 auto_start_cooldown_secs: 60,
6397 auto_start_timeout_secs: 10,
6398 self_healing_log_level: SelfHealingLogLevel::Debug,
6399 };
6400
6401 let json = serde_json::to_string(&config).unwrap();
6402 assert!(json.contains("\"hook_starts_daemon\":false"));
6403 assert!(json.contains("\"daemon_installs_hooks\":false"));
6404 assert!(json.contains("\"auto_start_cooldown_secs\":60"));
6405 assert!(json.contains("\"auto_start_timeout_secs\":10"));
6406 assert!(json.contains("\"self_healing_log_level\":\"debug\""));
6407
6408 let parsed: SelfHealingConfig = serde_json::from_str(&json).unwrap();
6409 assert!(!parsed.hook_starts_daemon);
6410 assert!(!parsed.daemon_installs_hooks);
6411 assert_eq!(parsed.auto_start_cooldown_secs, 60);
6412 assert_eq!(parsed.auto_start_timeout_secs, 10);
6413 assert_eq!(parsed.self_healing_log_level, SelfHealingLogLevel::Debug);
6414 }
6416
6417 #[test]
6420 fn test_self_healing_log_level_default_is_info() {
6421 let level = SelfHealingLogLevel::default();
6423 assert_eq!(level, SelfHealingLogLevel::Info);
6424 let config = SelfHealingConfig::default();
6426 assert_eq!(
6427 config.self_healing_log_level,
6428 SelfHealingLogLevel::Info,
6429 "config default should be Info"
6430 );
6431 }
6433
6434 #[test]
6435 fn test_self_healing_log_level_serializes_lowercase() {
6436 let json = serde_json::to_string(&SelfHealingLogLevel::Warn).unwrap();
6438 assert_eq!(json, "\"warn\"");
6439 let parsed: SelfHealingLogLevel = serde_json::from_str("\"error\"").unwrap();
6440 assert_eq!(parsed, SelfHealingLogLevel::Error);
6441 }
6443
6444 #[test]
6445 fn test_self_healing_log_level_from_env_str_accepts_known() {
6446 assert_eq!(
6448 SelfHealingLogLevel::from_env_str("debug"),
6449 Some(SelfHealingLogLevel::Debug)
6450 );
6451 assert_eq!(
6452 SelfHealingLogLevel::from_env_str("DEBUG"),
6453 Some(SelfHealingLogLevel::Debug)
6454 );
6455 assert_eq!(
6456 SelfHealingLogLevel::from_env_str("info"),
6457 Some(SelfHealingLogLevel::Info)
6458 );
6459 assert_eq!(
6460 SelfHealingLogLevel::from_env_str("warn"),
6461 Some(SelfHealingLogLevel::Warn)
6462 );
6463 assert_eq!(
6464 SelfHealingLogLevel::from_env_str("warning"),
6465 Some(SelfHealingLogLevel::Warn),
6466 "warning should map to Warn"
6467 );
6468 assert_eq!(
6469 SelfHealingLogLevel::from_env_str("error"),
6470 Some(SelfHealingLogLevel::Error)
6471 );
6472 assert_eq!(
6474 SelfHealingLogLevel::from_env_str(" debug "),
6475 Some(SelfHealingLogLevel::Debug)
6476 );
6477 }
6479
6480 #[test]
6481 fn test_self_healing_log_level_from_env_str_rejects_unknown() {
6482 assert!(SelfHealingLogLevel::from_env_str("banana").is_none());
6484 assert!(SelfHealingLogLevel::from_env_str("").is_none());
6485 assert!(SelfHealingLogLevel::from_env_str("trace").is_none());
6486 }
6488
6489 #[test]
6490 fn test_self_healing_config_serde_partial_uses_defaults() {
6491 let _guard = test_guard!();
6492 let json = r#"{"hook_starts_daemon": false}"#;
6494 let config: SelfHealingConfig = serde_json::from_str(json).unwrap();
6495
6496 assert!(
6497 !config.hook_starts_daemon,
6498 "Explicit false should be parsed"
6499 );
6500 assert!(
6501 config.daemon_installs_hooks,
6502 "Missing field should use default (true)"
6503 );
6504 assert_eq!(
6505 config.auto_start_cooldown_secs, 30,
6506 "Missing field should use default (30)"
6507 );
6508 assert_eq!(
6509 config.auto_start_timeout_secs, 3,
6510 "Missing field should use default (3)"
6511 );
6512 }
6514
6515 #[test]
6516 fn test_self_healing_config_toml_with_alias() {
6517 let _guard = test_guard!();
6518 let toml_str = r#"
6520 hook_starts_daemon = true
6521 daemon_installs_hooks = false
6522 auto_start_cooldown_secs = 45
6523 daemon_start_timeout = 7
6524 "#;
6525
6526 let config: SelfHealingConfig = toml::from_str(toml_str).unwrap();
6527 assert!(config.hook_starts_daemon);
6528 assert!(!config.daemon_installs_hooks);
6529 assert_eq!(config.auto_start_cooldown_secs, 45);
6530 assert_eq!(
6531 config.auto_start_timeout_secs, 7,
6532 "daemon_start_timeout alias should set auto_start_timeout_secs"
6533 );
6534 }
6536
6537 #[allow(unsafe_code)]
6538 mod self_healing_env_override_tests {
6539 use super::*;
6540 use crate::config::env_test_lock;
6541
6542 fn env_guard() -> std::sync::MutexGuard<'static, ()> {
6543 env_test_lock()
6544 }
6545
6546 fn set_env(key: &str, value: &str) {
6547 unsafe { std::env::set_var(key, value) };
6549 }
6550
6551 fn remove_env(key: &str) {
6552 unsafe { std::env::remove_var(key) };
6554 }
6555
6556 struct EnvVarGuard {
6557 key: &'static str,
6558 old: Option<String>,
6559 }
6560
6561 impl EnvVarGuard {
6562 fn set(key: &'static str, value: &str) -> Self {
6563 let old = std::env::var(key).ok();
6564 set_env(key, value);
6565 Self { key, old }
6566 }
6567 }
6568
6569 impl Drop for EnvVarGuard {
6570 fn drop(&mut self) {
6571 if let Some(old) = &self.old {
6572 set_env(self.key, old);
6573 } else {
6574 remove_env(self.key);
6575 }
6576 }
6577 }
6578
6579 #[test]
6580 fn test_self_healing_config_with_env_overrides_master_disable() {
6581 let _guard = test_guard!();
6582 let _guard = env_guard();
6583
6584 let _no_self_healing = EnvVarGuard::set("RCH_NO_SELF_HEALING", "1");
6585 let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "1");
6586 let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "1");
6587 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "99");
6588 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "99");
6589
6590 let config = SelfHealingConfig::default().with_env_overrides();
6591 assert!(
6592 !config.hook_starts_daemon,
6593 "RCH_NO_SELF_HEALING should disable hook auto-start"
6594 );
6595 assert!(
6596 !config.daemon_installs_hooks,
6597 "RCH_NO_SELF_HEALING should disable daemon hook installation"
6598 );
6599
6600 assert_eq!(config.auto_start_cooldown_secs, 30);
6602 assert_eq!(config.auto_start_timeout_secs, 3);
6603 }
6604
6605 #[test]
6606 fn test_self_healing_config_with_env_overrides_toggles_and_numbers() {
6607 let _guard = test_guard!();
6608 let _guard = env_guard();
6609
6610 let _hook_starts_daemon = EnvVarGuard::set("RCH_HOOK_STARTS_DAEMON", "0");
6611 let _daemon_installs_hooks = EnvVarGuard::set("RCH_DAEMON_INSTALLS_HOOKS", "false");
6612 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "45");
6613 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "7");
6614
6615 let config = SelfHealingConfig::default().with_env_overrides();
6616 assert!(!config.hook_starts_daemon);
6617 assert!(!config.daemon_installs_hooks);
6618 assert_eq!(config.auto_start_cooldown_secs, 45);
6619 assert_eq!(config.auto_start_timeout_secs, 7);
6620 }
6621
6622 #[test]
6623 fn test_self_healing_config_with_env_overrides_invalid_numbers_ignored() {
6624 let _guard = test_guard!();
6625 let _guard = env_guard();
6626
6627 let _cooldown = EnvVarGuard::set("RCH_AUTO_START_COOLDOWN_SECS", "not-a-number");
6628 let _timeout = EnvVarGuard::set("RCH_AUTO_START_TIMEOUT_SECS", "nope");
6629
6630 let config = SelfHealingConfig::default().with_env_overrides();
6631 assert_eq!(config.auto_start_cooldown_secs, 30);
6632 assert_eq!(config.auto_start_timeout_secs, 3);
6633 }
6634 }
6635
6636 #[test]
6641 fn test_load_per_core_calculation() {
6642 let _guard = test_guard!();
6643 let mut caps = WorkerCapabilities::new();
6644 assert!(caps.load_per_core().is_none());
6646
6647 caps.load_avg_1 = Some(4.0);
6649 assert!(caps.load_per_core().is_none());
6650
6651 caps.num_cpus = Some(4);
6653 assert_eq!(caps.load_per_core(), Some(1.0)); caps.load_avg_1 = Some(16.0);
6657 assert_eq!(caps.load_per_core(), Some(4.0)); }
6659
6660 #[test]
6661 fn test_is_high_load() {
6662 let _guard = test_guard!();
6663 let mut caps = WorkerCapabilities::new();
6664 caps.load_avg_1 = Some(8.0);
6665 caps.num_cpus = Some(4);
6666 assert_eq!(caps.is_high_load(3.0), Some(false));
6670 assert_eq!(caps.is_high_load(2.0), Some(false)); assert_eq!(caps.is_high_load(1.5), Some(true));
6674
6675 caps.load_avg_1 = None;
6677 assert!(caps.is_high_load(1.0).is_none());
6678 }
6679
6680 #[test]
6681 fn test_is_low_disk() {
6682 let _guard = test_guard!();
6683 let mut caps = WorkerCapabilities::new();
6684 caps.disk_free_gb = Some(15.0);
6685
6686 assert_eq!(caps.is_low_disk(10.0), Some(false));
6688 assert_eq!(caps.is_low_disk(15.0), Some(false)); assert_eq!(caps.is_low_disk(20.0), Some(true));
6692
6693 caps.disk_free_gb = None;
6695 assert!(caps.is_low_disk(10.0).is_none());
6696 }
6697
6698 #[test]
6699 fn test_selection_config_preflight_defaults() {
6700 let _guard = test_guard!();
6701 let config = SelectionConfig::default();
6702 assert_eq!(config.max_load_per_core, Some(2.0));
6704 assert_eq!(config.min_free_gb, Some(10.0));
6705 assert_eq!(config.disk_gb_per_slot, 10.0);
6706 }
6707
6708 #[test]
6709 fn test_selection_config_preflight_serde() {
6710 let _guard = test_guard!();
6711 let config = SelectionConfig {
6712 max_load_per_core: Some(3.5),
6713 min_free_gb: Some(25.0),
6714 disk_gb_per_slot: 0.83,
6715 ..Default::default()
6716 };
6717 let json = serde_json::to_string(&config).unwrap();
6718 let parsed: SelectionConfig = serde_json::from_str(&json).unwrap();
6719 assert_eq!(parsed.max_load_per_core, Some(3.5));
6720 assert_eq!(parsed.min_free_gb, Some(25.0));
6721 assert_eq!(parsed.disk_gb_per_slot, 0.83);
6722 }
6723
6724 #[test]
6725 fn test_selection_disk_slot_budget_validation() {
6726 let _guard = test_guard!();
6727 for value in ["0.0", "-1.0", "nan", "inf", "-inf"] {
6728 let result = toml::from_str::<SelectionConfig>(&format!("disk_gb_per_slot = {value}"));
6729 assert!(result.is_err(), "invalid disk slot budget {value}");
6730 }
6731 let default: SelectionConfig = toml::from_str("").unwrap();
6732 assert_eq!(default.disk_gb_per_slot, 10.0);
6733 }
6734
6735 #[test]
6736 fn test_selection_disk_weight_validation_and_roundtrip() {
6737 for value in ["-0.1", "1.1", "nan", "inf", "-inf"] {
6738 assert!(toml::from_str::<SelectionWeightConfig>(&format!("disk = {value}")).is_err());
6739 }
6740 for value in [0.0, 0.2, 1.0] {
6741 let weights: SelectionWeightConfig =
6742 toml::from_str(&format!("disk = {value}")).unwrap();
6743 assert_eq!(weights.disk, value);
6744 let roundtrip: SelectionWeightConfig =
6745 serde_json::from_str(&serde_json::to_string(&weights).unwrap()).unwrap();
6746 assert_eq!(roundtrip.disk, value);
6747 }
6748 let defaults: SelectionWeightConfig = toml::from_str("").unwrap();
6749 assert_eq!(defaults.disk, 0.2);
6750 }
6751
6752 #[test]
6753 fn test_worker_capabilities_health_metrics_serde() {
6754 let _guard = test_guard!();
6755 let caps = WorkerCapabilities {
6756 num_cpus: Some(8),
6757 load_avg_1: Some(1.5),
6758 load_avg_5: Some(2.0),
6759 load_avg_15: Some(1.8),
6760 disk_free_gb: Some(50.5),
6761 disk_total_gb: Some(100.0),
6762 ..Default::default()
6763 };
6764 let json = serde_json::to_string(&caps).unwrap();
6765 let parsed: WorkerCapabilities = serde_json::from_str(&json).unwrap();
6766 assert_eq!(parsed.num_cpus, Some(8));
6767 assert_eq!(parsed.load_avg_1, Some(1.5));
6768 assert_eq!(parsed.disk_free_gb, Some(50.5));
6769 }
6770
6771 #[test]
6772 fn microarch_level_classifies_ivy_bridge_as_v2() {
6773 let ivy = "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov \
6776 pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx rdtscp lm \
6777 constant_tsc pni pclmulqdq ssse3 cx16 sse4_1 sse4_2 x2apic \
6778 popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm";
6779 assert_eq!(x86_64_microarch_level_from_flags(ivy), 2);
6780 }
6781
6782 #[test]
6783 fn microarch_level_classifies_v3_and_v4() {
6784 let v3 = "cx16 lahf_lm popcnt sse4_1 sse4_2 ssse3 abm avx avx2 bmi1 bmi2 \
6785 f16c fma movbe xsave";
6786 assert_eq!(x86_64_microarch_level_from_flags(v3), 3);
6787
6788 let v4 = format!("{v3} avx512f avx512bw avx512cd avx512dq avx512vl");
6789 assert_eq!(x86_64_microarch_level_from_flags(&v4), 4);
6790 }
6791
6792 #[test]
6793 fn microarch_level_v1_for_ancient_or_empty_flags() {
6794 assert_eq!(x86_64_microarch_level_from_flags(""), 1);
6795 assert_eq!(x86_64_microarch_level_from_flags("fpu mmx sse sse2"), 1);
6796 }
6797
6798 #[test]
6799 fn pre_v3_predicate_only_fires_on_affirmative_evidence() {
6800 let mut caps = WorkerCapabilities::new();
6801 assert!(!caps.is_pre_v3_x86(), "unknown must not deprioritize");
6802 caps.cpu_microarch_level = Some(2);
6803 assert!(caps.is_pre_v3_x86());
6804 caps.cpu_microarch_level = Some(3);
6805 assert!(!caps.is_pre_v3_x86());
6806 }
6807}