rch_common/
transfer_hardening.rs

1//! Transfer hardening against volatile files and retryable transport failures
2//! (bd-session-history-remediation-ocv9i.8.2).
3//!
4//! rsync/SSH transfers race against a live working tree: `.git/index.lock`
5//! appears and vanishes, build artifacts are rewritten mid-flight, and the
6//! network blips. Treating every nonzero rsync exit as corruption causes
7//! spurious build failures. This module classifies a transfer outcome
8//! ([`classify_rsync_outcome`]) — separating a *vanished source file* (rsync
9//! exit 24) and *retryable transport* errors from genuine corruption / disk /
10//! permission faults — and yields a bounded [`RetryDecision`]. It also knows
11//! which paths are *ephemeral* ([`is_ephemeral_path`]) and should be excluded
12//! up front. Pure + deterministic.
13
14use serde::{Deserialize, Serialize};
15
16use crate::incident::IncidentReasonCode;
17use crate::ssh_utils::is_retryable_transport_error_text;
18
19/// Default bounded retry attempts for a hardenable transfer failure.
20pub const DEFAULT_MAX_TRANSFER_ATTEMPTS: u32 = 3;
21/// rsync exit code: partial transfer because source files vanished.
22pub const RSYNC_EXIT_VANISHED: i32 = 24;
23/// rsync exit code: partial transfer due to a real error.
24pub const RSYNC_EXIT_PARTIAL_ERROR: i32 = 23;
25
26/// Classification of a single rsync/transfer outcome.
27#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
28#[serde(rename_all = "snake_case")]
29pub enum RsyncFailureClass {
30    /// Transfer succeeded.
31    Success,
32    /// A source file vanished mid-transfer (exit 24 / "vanished") — benign;
33    /// retry/ignore.
34    VanishedFile,
35    /// A bounded, retryable SSH/network transport error.
36    RetryableTransport,
37    /// The destination disk is full.
38    DiskFull,
39    /// A permission error on source or destination.
40    PermissionDenied,
41    /// A partial transfer due to a real (non-vanished) error.
42    PartialCorruption,
43    /// A non-retryable, fatal error.
44    Fatal,
45}
46
47impl RsyncFailureClass {
48    /// Is this a success?
49    #[must_use]
50    pub const fn is_success(self) -> bool {
51        matches!(self, Self::Success)
52    }
53
54    /// The incident reason this class maps to, if any.
55    #[must_use]
56    pub const fn incident_reason(self) -> Option<IncidentReasonCode> {
57        match self {
58            Self::VanishedFile => Some(IncidentReasonCode::RsyncVanishedFile),
59            Self::DiskFull => Some(IncidentReasonCode::DiskFull),
60            _ => None,
61        }
62    }
63
64    /// Bounded retry decision for this class on `attempt` (1-based) of
65    /// `max_attempts`.
66    #[must_use]
67    pub fn retry_decision(self, attempt: u32, max_attempts: u32) -> RetryDecision {
68        let exhausted = attempt >= max_attempts;
69        match self {
70            Self::Success => RetryDecision::done("transfer succeeded"),
71            // Vanished files are benign: re-run (excluding the vanished path) up
72            // to the cap; treat exhaustion as ignorable, not a hard failure.
73            Self::VanishedFile if !exhausted => {
74                RetryDecision::retry(0, "source file vanished; retrying (ignoring vanished path)")
75            }
76            Self::VanishedFile => RetryDecision {
77                should_retry: false,
78                backoff_ms: 0,
79                ignorable: true,
80                reason: "vanished files persisted across retries; ignoring (benign)",
81            },
82            Self::RetryableTransport if !exhausted => RetryDecision::retry(
83                backoff_ms(attempt),
84                "retryable transport error; retrying with backoff",
85            ),
86            Self::PartialCorruption if !exhausted => {
87                RetryDecision::retry(backoff_ms(attempt), "partial transfer; retrying")
88            }
89            // Non-retryable or exhausted.
90            Self::DiskFull => RetryDecision::fatal("destination disk full"),
91            Self::PermissionDenied => RetryDecision::fatal("permission denied"),
92            Self::Fatal => RetryDecision::fatal("fatal transfer error"),
93            _ => RetryDecision::fatal("retry attempts exhausted"),
94        }
95    }
96}
97
98/// Exponential backoff (250ms base, doubling), capped.
99fn backoff_ms(attempt: u32) -> u64 {
100    let shift = attempt.saturating_sub(1).min(5);
101    250u64.saturating_mul(1u64 << shift)
102}
103
104/// What to do after a classified transfer failure.
105#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
106pub struct RetryDecision {
107    pub should_retry: bool,
108    pub backoff_ms: u64,
109    /// True when the failure is benign and may be ignored if retries run out
110    /// (e.g. vanished files).
111    pub ignorable: bool,
112    pub reason: &'static str,
113}
114
115impl RetryDecision {
116    const fn retry(backoff_ms: u64, reason: &'static str) -> Self {
117        Self {
118            should_retry: true,
119            backoff_ms,
120            ignorable: false,
121            reason,
122        }
123    }
124    const fn done(reason: &'static str) -> Self {
125        Self {
126            should_retry: false,
127            backoff_ms: 0,
128            ignorable: true,
129            reason,
130        }
131    }
132    const fn fatal(reason: &'static str) -> Self {
133        Self {
134            should_retry: false,
135            backoff_ms: 0,
136            ignorable: false,
137            reason,
138        }
139    }
140}
141
142/// Classify a transfer outcome from rsync's exit code and stderr. Order matters:
143/// vanished is detected before generic partial-error so a benign race is not
144/// mistaken for corruption.
145#[must_use]
146pub fn classify_rsync_outcome(exit_code: i32, stderr: &str) -> RsyncFailureClass {
147    if exit_code == 0 {
148        return RsyncFailureClass::Success;
149    }
150    let lower = stderr.to_ascii_lowercase();
151    if exit_code == RSYNC_EXIT_VANISHED || lower.contains("vanished") {
152        return RsyncFailureClass::VanishedFile;
153    }
154    if lower.contains("no space left") || lower.contains("disk quota exceeded") {
155        return RsyncFailureClass::DiskFull;
156    }
157    if lower.contains("permission denied") {
158        return RsyncFailureClass::PermissionDenied;
159    }
160    if is_retryable_transport_error_text(stderr) {
161        return RsyncFailureClass::RetryableTransport;
162    }
163    if exit_code == RSYNC_EXIT_PARTIAL_ERROR {
164        return RsyncFailureClass::PartialCorruption;
165    }
166    RsyncFailureClass::Fatal
167}
168
169/// Whether a path is an ephemeral / volatile store that should be excluded from
170/// transfer and convergence (it appears and vanishes during normal operation,
171/// so syncing it is pointless and race-prone).
172#[must_use]
173pub fn is_ephemeral_path(path: &str) -> bool {
174    const EPHEMERAL_SUFFIXES: &[&str] = &[
175        "/.git/index.lock",
176        "/.git/HEAD.lock",
177        "/.git/config.lock",
178        "/.git/packed-refs.lock",
179        "/.cargo-lock",
180        "/.rch.lock",
181    ];
182    const EPHEMERAL_SUBSTRINGS: &[&str] = &[
183        "/.git/objects/tmp_", // packing temp objects
184        "/.git/refs/",        // ref lock churn
185        "/incremental/",      // cargo incremental cruft
186    ];
187    let p = path.trim_end_matches('/');
188    if EPHEMERAL_SUFFIXES.iter().any(|s| p.ends_with(s)) {
189        return true;
190    }
191    // A `.lock` directly under any `.git` dir.
192    if p.ends_with(".lock") && p.contains("/.git/") {
193        return true;
194    }
195    EPHEMERAL_SUBSTRINGS.iter().any(|s| p.contains(s))
196}
197
198/// Accumulates the FIRST concrete failing path/command across a multi-attempt
199/// transfer, so selection/doctor output can name the original cause rather than
200/// the last (possibly less informative) retry.
201#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
202pub struct FirstFailure {
203    #[serde(default, skip_serializing_if = "Option::is_none")]
204    pub path: Option<String>,
205    #[serde(default, skip_serializing_if = "Option::is_none")]
206    pub command: Option<String>,
207    #[serde(default, skip_serializing_if = "Option::is_none")]
208    pub detail: Option<String>,
209}
210
211impl FirstFailure {
212    /// Record a failure, keeping only the first one seen.
213    pub fn record(&mut self, path: Option<&str>, command: Option<&str>, detail: &str) {
214        if self.detail.is_none() {
215            self.path = path.map(str::to_string);
216            self.command = command.map(str::to_string);
217            self.detail = Some(detail.to_string());
218        }
219    }
220
221    #[must_use]
222    pub fn is_set(&self) -> bool {
223        self.detail.is_some()
224    }
225}
226
227#[cfg(test)]
228mod tests {
229    use super::*;
230
231    #[test]
232    fn vanished_file_classified_from_exit_24() {
233        assert_eq!(
234            classify_rsync_outcome(24, ""),
235            RsyncFailureClass::VanishedFile
236        );
237    }
238
239    #[test]
240    fn vanished_file_classified_from_stderr() {
241        let class = classify_rsync_outcome(23, "file has vanished: \"/proj/.git/index.lock\"");
242        assert_eq!(class, RsyncFailureClass::VanishedFile);
243        assert_eq!(
244            class.incident_reason(),
245            Some(IncidentReasonCode::RsyncVanishedFile)
246        );
247    }
248
249    #[test]
250    fn success_disk_permission_and_fatal() {
251        assert_eq!(classify_rsync_outcome(0, ""), RsyncFailureClass::Success);
252        assert_eq!(
253            classify_rsync_outcome(11, "rsync: write failed: No space left on device (28)"),
254            RsyncFailureClass::DiskFull
255        );
256        assert_eq!(
257            classify_rsync_outcome(23, "rsync: mkdir failed: Permission denied (13)"),
258            RsyncFailureClass::PermissionDenied
259        );
260        assert_eq!(
261            classify_rsync_outcome(23, "some unrecognized partial error"),
262            RsyncFailureClass::PartialCorruption
263        );
264        assert_eq!(
265            classify_rsync_outcome(99, "totally unknown"),
266            RsyncFailureClass::Fatal
267        );
268    }
269
270    #[test]
271    fn retryable_transport_detected() {
272        // Reuses the shared transport-error classifier.
273        let class = classify_rsync_outcome(255, "ssh: connect to host: Connection timed out");
274        assert_eq!(class, RsyncFailureClass::RetryableTransport);
275    }
276
277    #[test]
278    fn vanished_file_retries_then_becomes_ignorable() {
279        let v = RsyncFailureClass::VanishedFile;
280        let d1 = v.retry_decision(1, 3);
281        assert!(d1.should_retry);
282        assert_eq!(d1.backoff_ms, 0); // vanished retries immediately
283        // Exhausted: not a hard failure — ignorable (benign).
284        let d3 = v.retry_decision(3, 3);
285        assert!(!d3.should_retry);
286        assert!(d3.ignorable);
287    }
288
289    #[test]
290    fn retryable_transport_backs_off_then_stops() {
291        let t = RsyncFailureClass::RetryableTransport;
292        let d1 = t.retry_decision(1, 3);
293        assert!(d1.should_retry && d1.backoff_ms == 250);
294        let d2 = t.retry_decision(2, 3);
295        assert!(d2.should_retry && d2.backoff_ms == 500);
296        let d3 = t.retry_decision(3, 3);
297        assert!(!d3.should_retry && !d3.ignorable);
298    }
299
300    #[test]
301    fn fatal_classes_never_retry() {
302        assert!(
303            !RsyncFailureClass::DiskFull
304                .retry_decision(1, 3)
305                .should_retry
306        );
307        assert!(
308            !RsyncFailureClass::PermissionDenied
309                .retry_decision(1, 3)
310                .should_retry
311        );
312        assert!(!RsyncFailureClass::Fatal.retry_decision(1, 3).should_retry);
313    }
314
315    #[test]
316    fn ephemeral_paths_detected() {
317        assert!(is_ephemeral_path("/proj/.git/index.lock"));
318        assert!(is_ephemeral_path("/proj/.git/refs/heads/main.lock"));
319        assert!(is_ephemeral_path("/proj/target/debug/incremental/foo"));
320        assert!(is_ephemeral_path("/proj/.git/objects/tmp_abc"));
321        // Real source files are not ephemeral.
322        assert!(!is_ephemeral_path("/proj/src/lib.rs"));
323        assert!(!is_ephemeral_path("/proj/Cargo.toml"));
324        assert!(!is_ephemeral_path("/proj/.gitignore"));
325    }
326
327    #[test]
328    fn first_failure_keeps_the_first() {
329        let mut f = FirstFailure::default();
330        assert!(!f.is_set());
331        f.record(Some("/a"), Some("rsync ..."), "first error");
332        f.record(Some("/b"), Some("retry ..."), "second error");
333        assert!(f.is_set());
334        assert_eq!(f.path.as_deref(), Some("/a"));
335        assert_eq!(f.detail.as_deref(), Some("first error"));
336    }
337
338    #[test]
339    fn serde_roundtrips() {
340        let class = RsyncFailureClass::VanishedFile;
341        let v = serde_json::to_value(class).unwrap();
342        assert_eq!(v, "vanished_file");
343        let mut f = FirstFailure::default();
344        f.record(Some("/a"), None, "boom");
345        let back: FirstFailure = serde_json::from_str(&serde_json::to_string(&f).unwrap()).unwrap();
346        assert_eq!(f, back);
347    }
348}
rch_common/transfer_hardening.rs

rch_common/
transfer_hardening.rs
