git_policy_cli/
lfs_upload.rs

1//! What:
2//!  The parent's run of the Git LFS upload before a push: `git-lfs push <remote> <refs>` as one child it owns.
3//! Why:
4//!  Without an upload step a push publishes pointer files whose objects the LFS server never received.
5//!  The wrapper runs `git-lfs push` itself, so a failure is explained by the process that saw it and the push, which
6//!  then carries no hook, is never started. Nothing here starts the wrapper, writes shell text or injects Git
7//!  configuration (`doc/planning/cli-git-no-self-invocation.md`). What to upload is decided in `lfs_upload_plan`.
8//!  The child sees what the caller gave Git: the global `-c` options travel as `GIT_CONFIG_COUNT` pairs, `--git-dir`,
9//!  `--work-tree` and `--namespace` as their variables, `-C` as the working directory, and the real Git's directory
10//!  leads its `PATH` so a `git` it starts is never the wrapper while `git-lfs` itself is the one the caller's `PATH`
11//!  finds first.
12//!
13//! In TS you'd write (pseudocode):
14//! ```ts
15//! const code = await spawn(findProgram(env.PATH), plan.arguments(), { env: withPrefix(env, globalPrefix), stdio: 'inherit' });
16//! ```
17
18/// The attempt record, the coordinator's Git and its error type.
19use super::auto_push_single_flight::{PushAttempt, PushGit, SingleFlightError};
20/// The deadline, the grace and the push runner.
21use super::bounded_push::{STOPPED_EXIT_CODE, STOP_GRACE, budget_spent_message, push_deadline, run_bounded_push};
22/// The invocation's environment values.
23use super::child_environment::environment_value;
24/// How a child ended, as a status.
25use super::forwarding::{ChildOutcome, exit_code, outcome_of};
26/// The texts, the program lookup and the plan.
27use super::lfs_pre_push::{failed_message, find_program, missing_program_message};
28/// Hides the credentials in a URL of relayed text.
29use super::lfs_output_mask::{mask_userinfo, mask_userinfo_text};
30/// Which failures are run again, and after how long.
31use super::lfs_server_retry::{next_delay, retry_delays, retry_notice};
32use super::lfs_upload_plan::UploadPlan;
33/// The anonymous file the interactive upload's standard error goes to.
34use super::scratch_location::scratch_file;
35/// Relays signals to the child.
36use super::signal_relay::SignalGuard;
37/// Starts that ride out a short shortage of processes.
38use super::start_retry::RetryStart;
39/// Owned OS text for arguments.
40use std::ffi::OsString;
41/// The real Git whose directory leads `PATH`.
42use std::path::Path;
43/// The child and its standard error.
44use std::process::{Command, Stdio};
45/// Reading the captured standard error back.
46use std::io::{Read, Seek, SeekFrom};
47
48/// `PATH` with the directory of the real Git first, so a `git` that `git-lfs` starts is the real Git and never the
49///  wrapper (Git does the same for the hooks it starts).
50fn path_with_real_git_first(environment: &[(OsString, OsString)], real_git: &Path) -> OsString {
51    let existing: OsString = environment_value(environment, "PATH").unwrap_or_default();
52    let mut directories: Vec<std::path::PathBuf> = Vec::new();
53    if let Some(directory) = real_git.parent() {
54        directories.push(directory.to_path_buf());
55    }
56    directories.extend(std::env::split_paths(existing.as_os_str()));
57    return std::env::join_paths(directories).unwrap_or(existing);
58}
59
60/// What:
61///  The variables that carry the global options of a push into a child that reads Git's configuration.
62/// Why:
63///   `-c name=value` becomes `GIT_CONFIG_KEY_n`/`GIT_CONFIG_VALUE_n` after any pairs the environment already holds
64///  (a `-c` without `=` is the boolean true), `--git-dir`, `--work-tree` and `--namespace` become `GIT_DIR`,
65///  `GIT_WORK_TREE` and `GIT_NAMESPACE`; `-C` is the working directory and is applied by the caller.
66///  No shell is involved, and an `-c http.extraHeader=...` or `-c lfs.url=...` reaches `git-lfs` as it reached the
67///  Git that used to run the hook.
68///
69/// In TS you'd write (pseudocode):
70/// ```ts
71/// function prefixVariables(globalPrefix: string[], environment): [string, string][];
72/// ```
73pub fn prefix_variables(global_prefix: &[OsString], environment: &[(OsString, OsString)]) -> Vec<(OsString, OsString)> {
74    let mut pairs: Vec<(OsString, OsString)> = Vec::new();
75    let mut variables: Vec<(OsString, OsString)> = Vec::new();
76    let mut tokens = global_prefix.iter();
77    while let Some(token) = tokens.next() {
78        let text: String = token.to_string_lossy().into_owned();
79        if text == "-c" {
80            if let Some(setting) = tokens.next() {
81                let setting_text: String = setting.to_string_lossy().into_owned();
82                let (name, value) = setting_text.split_once('=').unwrap_or((setting_text.as_str(), "true"));
83                pairs.push((OsString::from(name), OsString::from(value)));
84            }
85            continue;
86        }
87        for (option, variable) in [("--git-dir", "GIT_DIR"), ("--work-tree", "GIT_WORK_TREE"), ("--namespace", "GIT_NAMESPACE")] {
88            if text == option {
89                if let Some(value) = tokens.next() {
90                    variables.push((OsString::from(variable), value.clone()));
91                }
92            } else if let Some(value) = text.strip_prefix(format!("{option}=").as_str()) {
93                variables.push((OsString::from(variable), OsString::from(value)));
94            }
95        }
96    }
97    let existing: usize = environment_value(environment, "GIT_CONFIG_COUNT")
98        .and_then(|value| {
99            return value.to_str().and_then(|count| {
100                return count.parse::<usize>().ok();
101            });
102        })
103        .unwrap_or(0);
104    if !pairs.is_empty() {
105        variables.push((OsString::from("GIT_CONFIG_COUNT"), OsString::from((existing + pairs.len()).to_string())));
106    }
107    for (offset, (name, value)) in pairs.into_iter().enumerate() {
108        variables.push((OsString::from(format!("GIT_CONFIG_KEY_{}", existing + offset)), name));
109        variables.push((OsString::from(format!("GIT_CONFIG_VALUE_{}", existing + offset)), value));
110    }
111    return variables;
112}
113
114/// Why a run with captured standard error did not produce an outcome.
115enum CaptureError {
116    /// The file for the standard error could not be made; the text says why.
117    Scratch(String),
118    /// The program could not be started.
119    Start,
120}
121
122/// The most standard error text one run keeps and relays; the rest is dropped, so a flood cannot fill memory.
123const MAX_RELAYED_BYTES: u64 = 4 * 1024 * 1024;
124
125/// What:
126///  Run the command with its standard error in an anonymous file and return how it ended with that text, the
127///  credentials of every URL in it hidden.
128/// Why:
129///   Standard output stays the caller's, so the progress meter is unchanged. The standard error is where `git-lfs`
130///  prints its errors, which the parent must read (to tell a server error from a refused credential) and which can
131///  carry the `lfs.url` token, so it is relayed only after its URLs are masked. A file needs no reader thread that a
132///  stuck helper could keep alive.
133///
134/// In TS you'd write (pseudocode):
135/// ```ts
136/// const { outcome, stderr } = await spawnCapturingStderr(command); return [outcome, maskUserinfo(stderr)];
137/// ```
138fn run_with_captured_stderr(
139    command: &mut Command,
140    environment: &[(OsString, OsString)],
141) -> Result<(ChildOutcome, Vec<u8>), CaptureError> {
142    let mut captured: std::fs::File = scratch_file(environment).map_err(|error| {
143        return CaptureError::Scratch(error.to_string());
144    })?;
145    let for_child: std::fs::File = captured.try_clone().map_err(|error| {
146        return CaptureError::Scratch(error.to_string());
147    })?;
148    command.stderr(Stdio::from(for_child));
149    let outcome: std::io::Result<ChildOutcome> = match SignalGuard::install() {
150        Ok(guard) => guard.run_child(command),
151        Err(_) => command.status_retrying().map(outcome_of),
152    };
153    let Ok(ended) = outcome else {
154        return Err(CaptureError::Start);
155    };
156    let mut bytes: Vec<u8> = Vec::new();
157    captured
158        .seek(SeekFrom::Start(0))
159        .and_then(|_| {
160            return captured.by_ref().take(MAX_RELAYED_BYTES).read_to_end(&mut bytes);
161        })
162        .map_err(|error| {
163            return CaptureError::Scratch(error.to_string());
164        })?;
165    return Ok((ended, mask_userinfo(bytes.as_slice())));
166}
167
168/// The refusal printed when the standard error of the upload cannot be captured.
169fn capture_failed_message(why: &str) -> String {
170    return format!(
171        "cli-git: this push was stopped before it sent anything. The wrapper reads the output of `git-lfs push` through \
172         a private temporary file, to run it again after a server error and to hide the credentials in the URLs it \
173         names, and that file could not be made ({why}). Ways forward: fix the cause named in the parentheses (the \
174         wrapper needs a writable `XDG_STATE_HOME`, or a `HOME`, for its private scratch directory) and push again; or \
175         skip the upload for this push with `GIT_LFS_SKIP_PUSH=1` or `git push --no-verify`, which publishes pointer \
176         files without their objects.\n"
177    );
178}
179
180/// What:
181///  Run the upload as a child with the caller's streams and return the status to stop with, or nothing when it succeeded.
182/// Why:
183///   The parent saw the child end, so it prints the explanation after the program's own output; the push is not started.
184///  A signal sent to the wrapper is relayed to the child, so stopping the wrapper stops the upload. A push that also
185///  pushes submodules gets a notice, because their objects are not part of this upload.
186///
187/// In TS you'd write (pseudocode):
188/// ```ts
189/// const code = await spawn('git-lfs', plan.arguments(), { stdio: 'inherit' }); if (code) { print(failedMessage(...)); return code; }
190/// ```
191pub fn run_upload_interactive(
192    plan: &UploadPlan,
193    global_prefix: &[OsString],
194    directory: &Path,
195    real_git: &Path,
196    environment: &[(OsString, OsString)],
197    overlay: &[(OsString, OsString)],
198    diagnostics: &mut dyn std::io::Write,
199) -> Option<i32> {
200    let Some(program) = find_program(environment_value(environment, "PATH").unwrap_or_default().as_os_str()) else {
201        // Terminal output error: the exit status already carries the refusal.
202        let _written = diagnostics.write_all(missing_program_message().as_bytes());
203        return Some(1);
204    };
205    if plan.submodules {
206        // Terminal output error: the notice is advisory.
207        let _written = diagnostics.write_all(submodule_notice().as_bytes());
208    }
209    for arguments in plan.remotes.iter().flat_map(|remote| { return plan.runs(remote.as_os_str()); }) {
210        let delays: Vec<std::time::Duration> = retry_delays(environment);
211        let mut retries_done: usize = 0;
212        loop {
213            let mut command: Command = Command::new(&program);
214            command
215                .current_dir(directory)
216                .args(arguments.as_slice())
217                .env("PATH", path_with_real_git_first(environment, real_git));
218            for (name, value) in prefix_variables(global_prefix, environment).into_iter().chain(overlay.iter().cloned()) {
219                command.env(name, value);
220            }
221            let (ended, relayed): (ChildOutcome, Vec<u8>) = match run_with_captured_stderr(&mut command, environment) {
222                Ok(finished) => finished,
223                Err(CaptureError::Scratch(why)) => {
224                    // Terminal output error: the exit status already carries the refusal.
225                    let _written = diagnostics.write_all(capture_failed_message(why.as_str()).as_bytes());
226                    return Some(1);
227                }
228                Err(CaptureError::Start) => {
229                    // Terminal output error: see above.
230                    let _written = diagnostics.write_all(missing_program_message().as_bytes());
231                    return Some(1);
232                }
233            };
234            // Terminal output error: the program's own words are advisory; the exit status carries the result.
235            let _relayed = diagnostics.write_all(relayed.as_slice());
236            let code: i32 = exit_code(ended);
237            if code == 0 {
238                break;
239            }
240            if matches!(ended, ChildOutcome::Exited(_))
241                && let Some(delay) = next_delay(String::from_utf8_lossy(relayed.as_slice()).as_ref(), retries_done, delays.as_slice())
242            {
243                // Terminal output error: see above.
244                let _written = diagnostics.write_all(retry_notice(delay, retries_done, delays.len()).as_bytes());
245                retries_done += 1;
246                std::thread::sleep(delay);
247                continue;
248            }
249            let description: String = match ended {
250                ChildOutcome::Exited(status) => format!("exited with status {status}"),
251                ChildOutcome::Signaled(signal) => format!("was ended by signal {signal}"),
252            };
253            // Terminal output error: see above.
254            let _written = diagnostics.write_all(failed_message(description.as_str()).as_bytes());
255            return Some(code);
256        }
257    }
258    return None;
259}
260
261/// What:
262///  The notice for a push that also pushes submodules.
263/// Why:
264///   Git pushes a submodule's commits itself; their LFS objects are not uploaded by the wrapper, and the pointer files
265///  would be published without them. Every way forward is named.
266///
267/// In TS you'd write (pseudocode):
268/// ```ts
269/// 'cli-git: this push also pushes submodules, ...'
270/// ```
271pub fn submodule_notice() -> String {
272    return String::from(
273        "cli-git: this push also pushes submodules (`--recurse-submodules`), and the wrapper uploads Git LFS objects only \\
274         for this repository's own refs. If a submodule uses Git LFS, upload its objects first by running \\
275         `git lfs push --all <remote>` inside it, or push the submodule on its own; otherwise its pointer files are \\
276         published without their objects.\\n",
277    );
278}
279
280/// What:
281///  Run the upload as a bounded child whose output is captured, and return the failed attempt to stop with, or nothing
282///  when it succeeded.
283/// Why:
284///   The auto-push records and shows a failed attempt's output; the explanation is part of it. A stop by deadline says
285///  so, and a step the holder's time budget left no time for is not started and says that instead.
286///
287/// In TS you'd write (pseudocode):
288/// ```ts
289/// const attempt = await runBoundedPush({ command: gitLfs(plan), deadlineMs }); return attempt.exitCode ? withExplanation(attempt) : undefined;
290/// ```
291pub fn run_upload_bounded(git: &PushGit, plan: &UploadPlan) -> Result<Option<PushAttempt>, SingleFlightError> {
292    if super::holder_budget::exhausted() {
293        return Ok(Some(PushAttempt { exit_code: STOPPED_EXIT_CODE, output: budget_spent_message() }));
294    }
295    let Some(program) = find_program(environment_value(git.environment.as_slice(), "PATH").unwrap_or_default().as_os_str())
296    else {
297        return Ok(Some(PushAttempt { exit_code: 1, output: missing_program_message().trim_end().to_owned() }));
298    };
299    for arguments in plan.remotes.iter().flat_map(|remote| { return plan.runs(remote.as_os_str()); }) {
300        let delays: Vec<std::time::Duration> = retry_delays(git.environment.as_slice());
301        let mut earlier: String = String::new();
302        let mut retries_done: usize = 0;
303        loop {
304            let mut command: Command = Command::new(&program);
305            command
306                .current_dir(&git.cwd)
307                .env_clear()
308                .envs(git.environment.iter().map(|(name, value)| {
309                    return (name, value);
310                }))
311                .env("PATH", path_with_real_git_first(git.environment.as_slice(), git.real_git.as_path()))
312                .args(arguments.as_slice());
313            let attempt: PushAttempt =
314                match run_bounded_push(command, push_deadline(git.environment.as_slice()), STOP_GRACE, git.environment.as_slice()) {
315                    Ok(finished) => finished,
316                    Err(SingleFlightError::Start(_)) => {
317                        return Ok(Some(PushAttempt { exit_code: 1, output: missing_program_message().trim_end().to_owned() }));
318                    }
319                    Err(other) => return Err(other),
320                };
321            if attempt.exit_code == 0 {
322                break;
323            }
324            let shown: String = mask_userinfo_text(attempt.output.as_str());
325            let delay: Option<std::time::Duration> = next_delay(shown.as_str(), retries_done, delays.as_slice())
326                .filter(|wait| {
327                    return attempt.exit_code != STOPPED_EXIT_CODE && super::holder_budget::remaining().is_none_or(|left| {
328                        return left > *wait;
329                    });
330                });
331            let Some(wait) = delay else {
332                let joined: String = [earlier.as_str(), shown.as_str()].concat();
333                return Ok(Some(explained(PushAttempt { exit_code: attempt.exit_code, output: joined })));
334            };
335            let notice: String = retry_notice(wait, retries_done, delays.len());
336            earlier.push_str(shown.as_str());
337            if !earlier.is_empty() && !earlier.ends_with('\n') {
338                earlier.push('\n');
339            }
340            earlier.push_str(notice.as_str());
341            retries_done += 1;
342            std::thread::sleep(wait);
343        }
344    }
345    return Ok(None);
346}
347
348/// The failed attempt with the explanation after the program's own output; a stop by deadline says so.
349fn explained(attempt: PushAttempt) -> PushAttempt {
350    let description: String = if attempt.exit_code == STOPPED_EXIT_CODE {
351        String::from("did not finish and was stopped")
352    } else {
353        format!("exited with status {}", attempt.exit_code)
354    };
355    let explanation: String = failed_message(description.as_str());
356    let shown: &str = explanation.trim_end();
357    let output: String = if attempt.output.is_empty() { shown.to_owned() } else { format!("{}\n{shown}", attempt.output) };
358    return PushAttempt { exit_code: attempt.exit_code, output };
359}
360
361/// The runs are controlled in the test module.
362#[cfg(test)]
363#[path = "lfs_upload_tests.rs"]
364mod tests;
git_policy_cli/lfs_upload.rs

git_policy_cli/
lfs_upload.rs
