1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
//! What:
//! When a failed `git-lfs push` is tried again: the server-error classification, the delays and the notice.
//! Why:
//! The LFS server of a repository can answer HTTP 5xx for a moment, and `git-lfs` treats such an answer as fatal for the
//! run (`lfshttp/errors.go`: a status above 499 other than 501, 507 and 509 becomes a fatal error), so a push that was
//! fine a second later fails. Uploads are content addressed, so a second run never sends an object twice. The retry is
//! bounded twice over: a short fixed list of delays, and a classification from the program's own output that never
//! retries a refused credential, a missing local object, or any failure that is not a server error.
//!
//! In TS you'd write (pseudocode):
//! ```ts
//! const delay = isServerError(output) ? delays[retriesDone] : undefined; // undefined: stop and explain
//! ```
/// The invocation's environment values.
use super::child_environment::environment_value;
/// Owned OS text for the environment.
use std::ffi::OsString;
/// Delays.
use std::time::Duration;
/// How long to wait before each retry of a push that failed with a server error: two retries, so three runs at most.
pub const RETRY_DELAYS: [Duration; 2] = [Duration::from_secs(2), Duration::from_secs(5)];
/// The variable whose comma-separated millisecond list replaces [`RETRY_DELAYS`] (an empty value means no retry); it
/// exists so a test does not wait for the real delays.
pub const RETRY_DELAYS_VARIABLE: &str = "CLI_GIT_LFS_RETRY_DELAYS_MILLISECONDS";
/// The most retries a configured list may hold, so a mistyped list cannot make the step unbounded.
pub const MAX_CONFIGURED_RETRIES: usize = 5;
/// The longest single delay a configured list may hold.
pub const MAX_CONFIGURED_DELAY: Duration = Duration::from_secs(60);
/// Lines that name a failure retrying cannot fix: refused credentials, a missing object or repository, a refused request,
/// a corrupt or missing local object, locked files.
const PERMANENT_MARKERS: [&str; 7] = [
"Authorization error",
"Repository or object not found",
"Client error",
"Unprocessable entity",
"(missing)",
"(corrupt)",
"locked files",
];
/// What `git-lfs` puts before the message of every 5xx answer except 501, 507 and 509 (`errors/types.go`,
/// `NewFatalError`, used only by `lfshttp/errors.go` for those statuses), whatever the server's body says: a JSON
/// `message` in the body replaces the `Server error` wording but not this prefix.
const FATAL_PREFIX: &str = "Fatal error: ";
/// What the message of a 500 answer starts with (`lfshttp/errors.go`, `defaultError`).
const SERVER_ERROR_500: &str = "Server error: ";
/// What the message of any other retryable 5xx answer holds (`lfshttp/errors.go`, `defaultError`).
const SERVER_ERROR_OTHER: &str = "Server error ";
/// What precedes the status code in that message.
const FROM_HTTP: &str = " from HTTP ";
/// Whether the text after `from HTTP ` starts with a 5xx status.
fn starts_with_server_status(rest: &str) -> bool {
let digits: &str = rest.get(..3).unwrap_or_default();
return digits.len() == 3 && digits.starts_with('5') && digits.bytes().all(|byte| {
return byte.is_ascii_digit();
});
}
/// Whether one line is a server-error message of `git-lfs`.
fn line_is_server_error(line: &str) -> bool {
if line.contains(FATAL_PREFIX) || line.contains(SERVER_ERROR_500) {
return true;
}
return line.contains(SERVER_ERROR_OTHER)
&& line.split(FROM_HTTP).skip(1).any(starts_with_server_status);
}
/// What:
/// Whether the output of a failed `git-lfs push` says the LFS server answered with a retryable server error, and says
/// nothing that retrying cannot fix.
/// Why:
/// A positive match on the program's own words (`Server error: <url>` for 500, `Server error <url> from HTTP 5xx` for the
/// other retryable 5xx) keeps every unknown failure terminal. A line naming a refused credential, a missing object or a
/// locked file anywhere in the output makes the whole failure terminal, because a retry would meet it again.
/// The words are `git-lfs`'s English messages; a translated message is not retried, which fails toward the explanation.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// lines.some(isServerErrorLine) && !lines.some((line) => PERMANENT_MARKERS.some((marker) => line.includes(marker)))
/// ```
pub fn is_server_error(output: &str) -> bool {
let permanent: bool = output.lines().any(|line| {
return PERMANENT_MARKERS.iter().any(|marker| {
return line.contains(marker);
});
});
return !permanent && output.lines().any(line_is_server_error);
}
/// What:
/// The delays to use: the configured millisecond list when the variable is present and well formed, else
/// [`RETRY_DELAYS`].
/// Why:
/// A test injects short delays through the environment the wrapper was started with. A list is cut to
/// [`MAX_CONFIGURED_RETRIES`] entries of at most [`MAX_CONFIGURED_DELAY`] each, and a value that is not a list of
/// whole numbers is ignored, so the step stays bounded whatever the variable holds.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// function retryDelays(env): number[]; // milliseconds
/// ```
pub fn retry_delays(environment: &[(OsString, OsString)]) -> Vec<Duration> {
let Some(value) = environment_value(environment, RETRY_DELAYS_VARIABLE) else {
return RETRY_DELAYS.to_vec();
};
let text: String = value.to_string_lossy().trim().to_owned();
if text.is_empty() {
return Vec::new();
}
let parsed: Option<Vec<Duration>> = text
.split(',')
.map(|entry| {
return entry.trim().parse::<u64>().ok().map(Duration::from_millis);
})
.collect();
return parsed.map_or_else(
|| {
return RETRY_DELAYS.to_vec();
},
|delays| {
return delays
.into_iter()
.take(MAX_CONFIGURED_RETRIES)
.map(|delay| {
return delay.min(MAX_CONFIGURED_DELAY);
})
.collect();
},
);
}
/// What:
/// The delay before the next run of a push that just failed with `output`, or nothing when it is not to be retried.
/// Why:
/// `retries_done` runs have already been repeated; the list is the whole allowance.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// function nextDelay(output: string, retriesDone: number, delays: number[]): number | undefined;
/// ```
pub fn next_delay(output: &str, retries_done: usize, delays: &[Duration]) -> Option<Duration> {
let delay: Duration = delays.get(retries_done).copied()?;
return is_server_error(output).then_some(delay);
}
/// What:
/// The line printed before a retry.
/// Why:
/// The caller sees why the run is repeated, how long it waits and how many repeats are left.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// `cli-git: git-lfs push failed with a server error; trying again in ${seconds} seconds (retry ${n} of ${total}).`
/// ```
pub fn retry_notice(delay: Duration, retries_done: usize, total: usize) -> String {
return format!(
"cli-git: `git-lfs push` failed with a server error, so it is run again in {:.1} seconds (retry {} of {total}); \
objects the server already has are not sent twice.\n",
delay.as_secs_f64(),
retries_done + 1,
);
}
/// The classification and the delays are controlled in the test module.
#[cfg(test)]
#[path = "lfs_server_retry_tests.rs"]
mod tests;