monochromatic-git-policy-cli 0.5.7

//! Binary-level controls for the Git LFS upload against the real `git-lfs` and an LFS server written inline.
//! The server speaks the batch API and the basic transfer over a local socket and records, for every uploaded object,
//!  what the origin's `main` was at that moment, so the order "object first, ref second" is observed and not assumed.
//! Hosts without `git-lfs` on PATH report that and check nothing; the stand-in controls in
//!  `binary_lfs_pre_push_tests.rs` cover the same paths there.

use super::commit::remove_settled;
use super::support::{Fixture, Observed, REAL_GIT, decline_in_process_lfs_scan, direct, fixture, git, observe, repository, stderr_of};
use std::ffi::OsStr;
use std::io::{BufRead, BufReader, Read, Write};
use std::os::unix::fs::PermissionsExt;
use std::net::{TcpListener, TcpStream};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
use std::time::Duration;

/// What the server saw of one uploaded object.
#[derive(Clone, Debug)]
struct Upload {
    /// The object name the client sent.
    oid: String,
    /// The bytes it sent.
    bytes: Vec<u8>,
    /// The origin's `refs/heads/main` when the object arrived, or `absent`.
    origin_main: String,
}

/// The server's shared record.
#[derive(Default)]
struct ServerState {
    /// Every stored object.
    uploads: Vec<Upload>,
    /// Whether the batch endpoint refuses, as a server that rejects the credentials would.
    refuse_batch: bool,
    /// How many of the next batch requests are answered with HTTP 503, as the incident's server did.
    fail_batches: usize,
    /// Every batch request received, answered or not.
    batches: usize,
}

/// A running LFS server on a loopback port.
struct LfsServer {
    /// The port it listens on.
    port: u16,
    /// What it has seen.
    state: Arc<Mutex<ServerState>>,
    /// Set to end the accept loop.
    stop: Arc<AtomicBool>,
    /// The accept loop.
    thread: Option<JoinHandle<()>>,
    /// The repository's hooks directory, made read-only so `git-lfs` cannot write its shims into it again;
    ///  restored when the server ends so the fixture can be removed.
    hooks: Option<PathBuf>,
}

/// The origin's `main` for the server's record.
fn origin_main(origin: &Path) -> String {
    let output = std::process::Command::new(REAL_GIT)
        .arg(format!("--git-dir={}", origin.display()))
        .args(["rev-parse", "--verify", "--quiet", "refs/heads/main"])
        .output()
        .expect("rev-parse starts");
    let text: String = String::from_utf8_lossy(output.stdout.as_slice()).trim().to_owned();
    return if text.is_empty() { String::from("absent") } else { text };
}

/// Read one request: its request line, its headers' content length, and its body.
fn read_request(stream: &TcpStream) -> Option<(String, Vec<u8>)> {
    let mut reader: BufReader<&TcpStream> = BufReader::new(stream);
    let mut request_line: String = String::new();
    reader.read_line(&mut request_line).ok()?;
    let mut length: usize = 0;
    loop {
        let mut line: String = String::new();
        reader.read_line(&mut line).ok()?;
        if line == "\r\n" || line.is_empty() {
            break;
        }
        if let Some((name, value)) = line.split_once(':')
            && name.eq_ignore_ascii_case("content-length")
        {
            length = value.trim().parse().ok()?;
        }
    }
    let mut body: Vec<u8> = vec![0; length];
    reader.read_exact(body.as_mut_slice()).ok()?;
    return Some((request_line.trim_end().to_owned(), body));
}

/// Write one response with a body and close the connection.
fn respond(mut stream: &TcpStream, status: &str, body: &str) {
    let text: String = format!(
        "HTTP/1.1 {status}\r\nContent-Type: application/vnd.git-lfs+json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
        body.len()
    );
    // A client that went away is not a server failure.
    let _written = stream.write_all(text.as_bytes());
}

/// Answer one request of the batch API, the lock-verification endpoint or the basic upload.
fn serve(stream: &TcpStream, port: u16, origin: &Path, state: &Mutex<ServerState>) {
    let Some((request_line, body)) = read_request(stream) else {
        return;
    };
    if request_line.starts_with("POST /lfs/objects/batch ") {
        {
            let mut seen = state.lock().expect("state");
            seen.batches += 1;
            if seen.fail_batches > 0 {
                seen.fail_batches -= 1;
                drop(seen);
                respond(stream, "503 Service Unavailable", "{}");
                return;
            }
        }
        if state.lock().expect("state").refuse_batch {
            respond(stream, "403 Forbidden", "{\"message\":\"the fixture server refuses every batch\"}");
            return;
        }
        let asked: serde_json::Value = serde_json::from_slice(body.as_slice()).unwrap_or_default();
        let known: Vec<String> = state.lock().expect("state").uploads.iter().map(|upload| { return upload.oid.clone(); }).collect();
        // Built as text: the objects only carry hexadecimal names and whole numbers.
        let objects: Vec<String> = asked["objects"]
            .as_array()
            .cloned()
            .unwrap_or_default()
            .iter()
            .map(|object| {
                let oid: String = object["oid"].as_str().unwrap_or_default().to_owned();
                let size: u64 = object["size"].as_u64().unwrap_or_default();
                if known.contains(&oid) {
                    return format!("{{\"oid\":\"{oid}\",\"size\":{size}}}");
                }
                return format!(
                    "{{\"oid\":\"{oid}\",\"size\":{size},\"actions\":{{\"upload\":{{\"href\":\"http://127.0.0.1:{port}/lfs/objects/{oid}\"}}}}}}"
                );
            })
            .collect();
        respond(stream, "200 OK", format!("{{\"transfer\":\"basic\",\"objects\":[{}]}}", objects.join(",")).as_str());
    } else if request_line.starts_with("POST /lfs/locks/verify ") {
        respond(stream, "200 OK", "{\"ours\":[],\"theirs\":[]}");
    } else if let Some(rest) = request_line.strip_prefix("PUT /lfs/objects/") {
        let oid: String = rest.split(' ').next().unwrap_or_default().to_owned();
        let seen: String = origin_main(origin);
        state.lock().expect("state").uploads.push(Upload { oid, bytes: body, origin_main: seen });
        respond(stream, "200 OK", "{}");
    } else {
        respond(stream, "404 Not Found", "{\"message\":\"unknown endpoint\"}");
    }
}

impl LfsServer {
    /// Start the accept loop on an unused loopback port.
    fn start(origin: PathBuf) -> Self {
        let listener: TcpListener = TcpListener::bind("127.0.0.1:0").expect("bind loopback");
        let port: u16 = listener.local_addr().expect("address").port();
        let state: Arc<Mutex<ServerState>> = Arc::new(Mutex::new(ServerState::default()));
        let stop: Arc<AtomicBool> = Arc::new(AtomicBool::new(false));
        let thread_state: Arc<Mutex<ServerState>> = Arc::clone(&state);
        let thread_stop: Arc<AtomicBool> = Arc::clone(&stop);
        let thread: JoinHandle<()> = std::thread::spawn(move || {
            for connection in listener.incoming() {
                if thread_stop.load(Ordering::SeqCst) {
                    return;
                }
                let Ok(stream) = connection else {
                    continue;
                };
                // A client that stalls must not hold the loop for good.
                let _timeout = stream.set_read_timeout(Some(Duration::from_secs(20)));
                serve(&stream, port, origin.as_path(), &thread_state);
            }
        });
        return Self { port, state, stop, thread: Some(thread), hooks: None };
    }

    /// The uploads so far.
    fn uploads(&self) -> Vec<Upload> {
        return self.state.lock().expect("state").uploads.clone();
    }
}

/// The server ends with its test: the flag is set and a connection wakes the accept loop.
impl Drop for LfsServer {
    fn drop(&mut self) {
        self.stop.store(true, Ordering::SeqCst);
        if let Some(hooks) = self.hooks.as_deref() {
            // A directory that is already gone needs no restoring.
            let _restored = std::fs::set_permissions(hooks, std::fs::Permissions::from_mode(0o755));
        }
        // The wake-up connection only has to arrive; whether it is accepted does not matter.
        let _connected = TcpStream::connect(("127.0.0.1", self.port));
        if let Some(thread) = self.thread.take() {
            // A panic in the loop already failed the test that owned the server.
            let _joined = thread.join();
        }
    }
}

/// The real `git-lfs` on the test process's PATH, if any.
fn real_lfs_program() -> Option<PathBuf> {
    let path: std::ffi::OsString = std::env::var_os("PATH")?;
    return std::env::split_paths(&path)
        .map(|directory| { return directory.join("git-lfs"); })
        .find(|candidate| { return candidate.is_file(); });
}

/// A repository wired to the real `git-lfs` and the server, with one committed LFS file; `None` without `git-lfs`.
fn lfs_world(fixture: &Fixture) -> Option<(PathBuf, PathBuf, LfsServer, Vec<u8>)> {
    let Some(program) = real_lfs_program() else {
        eprintln!("skipped: git-lfs is not on PATH, so the real upload was not exercised");
        return None;
    };
    std::os::unix::fs::symlink(program, fixture.root.join("bin/git-lfs")).expect("git-lfs link");
    let repo: PathBuf = repository(fixture, OsStr::new("repo"));
    let origin: PathBuf = fixture.root.join("origin.git");
    git(fixture, fixture.root.as_path(), &["init", "--quiet", "--bare", "--initial-branch=main", "origin.git"]);
    git(fixture, repo.as_path(), &["remote", "add", "origin", origin.to_str().expect("UTF-8")]);
    let mut server: LfsServer = LfsServer::start(origin.clone());
    let settings: [(&str, String); 7] = [
        ("filter.lfs.clean", String::from("git-lfs clean -- %f")),
        ("filter.lfs.smudge", String::from("git-lfs smudge -- %f")),
        ("filter.lfs.process", String::from("git-lfs filter-process")),
        ("filter.lfs.required", String::from("true")),
        ("lfs.url", format!("http://127.0.0.1:{}/lfs", server.port)),
        ("lfs.transfer.maxretries", String::from("1")),
        ("maintenance.autoDetach", String::from("false")),
    ];
    for (key, value) in settings {
        git(fixture, repo.as_path(), &["config", key, value.as_str()]);
    }
    std::fs::write(repo.join(".gitattributes"), b"*.bin filter=lfs diff=lfs merge=lfs -text\n").expect("attributes");
    let content: Vec<u8> = (0..4096_u32).map(|index| { return u8::try_from(index % 251).expect("below 256"); }).collect();
    std::fs::write(repo.join("big.bin"), content.as_slice()).expect("big.bin");
    // `git add` runs the clean filter, which needs `git-lfs` on PATH; the direct helper's PATH has none.
    let added: Observed = observe(
        &mut super::commit::wrapped_commit(fixture, repo.as_path(), &["add", "--", ".gitattributes", "big.bin"]),
        b"",
    );
    assert_eq!(added.code, Some(0), "{}", stderr_of(&added));
    // Real Git commits, so no auto-push runs; its PATH names the fixture's `bin` so the filter finds `git-lfs`.
    let mut path: std::ffi::OsString = fixture.root.join("bin").into_os_string();
    path.push(":/usr/bin:/bin");
    let committed = direct(fixture)
        .current_dir(repo.as_path())
        .env("PATH", path)
        .args(["commit", "--quiet", "--message=lfs"])
        .output()
        .expect("commit starts");
    assert!(committed.status.success(), "{}", String::from_utf8_lossy(committed.stderr.as_slice()));
    // `git-lfs` writes its four hook shims into the repository whenever its filter runs, which would make the real
    //  program upload through a traditional hook and hide a missing wrapper hook. The shims go, and the directory
    //  stays read-only until the server is dropped, so the upload under test can only come from the wrapper.
    let hooks: PathBuf = repo.join(".git/hooks");
    for shim in ["pre-push", "post-checkout", "post-commit", "post-merge"] {
        // A shim that was never written needs no removal.
        let _removed = std::fs::remove_file(hooks.join(shim));
    }
    std::fs::set_permissions(&hooks, std::fs::Permissions::from_mode(0o555)).expect("read-only hooks directory");
    server.hooks = Some(hooks);
    return Some((repo, origin, server, content));
}

/// The object name a pointer file records.
fn pointer_oid(fixture: &Fixture, repo: &Path) -> String {
    let pointer: String = super::commit::git_text(fixture, repo, &["cat-file", "blob", "HEAD:big.bin"]);
    return pointer
        .lines()
        .find_map(|line| { return line.strip_prefix("oid sha256:"); })
        .expect("a pointer file has an oid line")
        .to_owned();
}

/// The origin's `main` seen from outside.
fn published_main(fixture: &Fixture, origin: &Path) -> String {
    let output = direct(fixture)
        .current_dir(origin)
        .args(["rev-parse", "--verify", "--quiet", "refs/heads/main"])
        .output()
        .expect("rev-parse starts");
    let text: String = String::from_utf8_lossy(output.stdout.as_slice()).trim().to_owned();
    return if text.is_empty() { String::from("absent") } else { text };
}

/// The object reaches the server before the ref reaches the origin, and the origin then holds the pointer commit.
#[test]
fn the_real_upload_delivers_the_object_before_the_ref() {
    let fixture: Fixture = fixture("lfs-real");
    let Some((repo, origin, server, content)) = lfs_world(&fixture) else {
        return;
    };
    let pushed: Observed = observe(
        &mut super::commit::wrapped_commit(&fixture, repo.as_path(), &["push", "origin", "main"]),
        b"",
    );
    let stderr: String = stderr_of(&pushed);
    assert_eq!(pushed.code, Some(0), "{stderr}");
    let uploads: Vec<Upload> = server.uploads();
    assert_eq!(uploads.len(), 1, "one object was uploaded: {stderr}");
    assert_eq!(uploads[0].oid, pointer_oid(&fixture, repo.as_path()));
    assert_eq!(uploads[0].bytes, content, "the server holds the file's bytes");
    assert_eq!(uploads[0].origin_main, "absent", "the object arrived before the ref moved");
    assert_ne!(published_main(&fixture, origin.as_path()), "absent", "the ref moved after the upload: {stderr}");
    drop(server);
    remove_settled(&fixture);
}

/// A server that refuses the upload stops the push: the origin gets no ref and the caller gets the explanation.
#[test]
fn a_refused_upload_publishes_no_pointer() {
    let fixture: Fixture = fixture("lfs-real-refused");
    let Some((repo, origin, server, _content)) = lfs_world(&fixture) else {
        return;
    };
    server.state.lock().expect("state").refuse_batch = true;
    let pushed: Observed = observe(
        &mut super::commit::wrapped_commit(&fixture, repo.as_path(), &["push", "origin", "main"]),
        b"",
    );
    let stderr: String = stderr_of(&pushed);
    assert_ne!(pushed.code, Some(0), "{stderr}");
    assert!(stderr.contains("`git-lfs push` exited with status"), "{stderr}");
    assert!(stderr.contains("nothing was published"), "{stderr}");
    assert_eq!(published_main(&fixture, origin.as_path()), "absent", "{stderr}");
    assert!(server.uploads().is_empty());
    assert_eq!(batches(&server), 1, "a refused credential is asked once, never retried: {stderr}");
    drop(server);
    remove_settled(&fixture);
}

/// The auto-push of a commit that adds an LFS file uploads the object first, with the real program.
#[test]
fn the_real_upload_precedes_the_auto_push_of_a_commit() {
    let fixture: Fixture = fixture("lfs-real-auto");
    let Some((repo, origin, server, content)) = lfs_world(&fixture) else {
        return;
    };
    std::fs::write(repo.join("second.bin"), [content.as_slice(), b"second"].concat()).expect("second.bin");
    let added: Observed = observe(
        &mut super::commit::wrapped_commit(&fixture, repo.as_path(), &["add", "--", "second.bin"]),
        b"",
    );
    assert_eq!(added.code, Some(0), "{}", stderr_of(&added));
    let landed: Observed = observe(
        &mut super::commit::wrapped_commit(&fixture, repo.as_path(), &["commit", "--quiet", "--message=second", "--", "second.bin"]),
        b"",
    );
    let stderr: String = stderr_of(&landed);
    assert_eq!(landed.code, Some(0), "{stderr}");
    assert!(!stderr.contains("auto-push to origin failed"), "{stderr}");
    let uploads: Vec<Upload> = server.uploads();
    assert_eq!(uploads.len(), 2, "both LFS objects of the branch were uploaded: {stderr}");
    assert!(uploads.iter().all(|upload| { return upload.origin_main == "absent"; }), "{uploads:?}");
    assert_ne!(published_main(&fixture, origin.as_path()), "absent", "{stderr}");
    drop(server);
    remove_settled(&fixture);
}

/// The retry delays the controls inject: a millisecond each, so no control waits for the real ones.
const FAST_RETRIES: (&str, &str) = (git_policy_cli::lfs_server_retry::RETRY_DELAYS_VARIABLE, "1,1");

/// How many batch requests the server has seen.
fn batches(server: &LfsServer) -> usize {
    return server.state.lock().expect("state").batches;
}

/// A wrapped `git push origin main` with extra environment variables.
fn push_main(fixture: &Fixture, repo: &Path, variables: &[(&str, &str)]) -> Observed {
    let mut command = super::commit::wrapped_commit(fixture, repo, &["push", "origin", "main"]);
    command.envs(variables.iter().copied());
    return observe(&mut command, b"");
}

/// A commit that adds no file, made by real Git so that no auto-push runs.
fn empty_commit(fixture: &Fixture, repo: &Path) {
    let mut path: std::ffi::OsString = fixture.root.join("bin").into_os_string();
    path.push(":/usr/bin:/bin");
    let committed = direct(fixture)
        .current_dir(repo)
        .env("PATH", path)
        .args(["commit", "--quiet", "--allow-empty", "--message=no new object"])
        .output()
        .expect("commit starts");
    assert!(committed.status.success(), "{}", String::from_utf8_lossy(committed.stderr.as_slice()));
}

/// Point the clone at the server through a URL that carries a user and a token, as the incident's `lfs.url` did.
fn use_credentialed_url(fixture: &Fixture, repo: &Path, server: &LfsServer) {
    let url: String = format!("http://user:secret@127.0.0.1:{}/lfs", server.port);
    git(fixture, repo, &["config", "lfs.url", url.as_str()]);
}

/// A push of a commit that adds no LFS object sends no batch request: git-lfs skips what the tracking refs reach.
#[test]
fn a_push_that_adds_no_object_sends_no_batch_request() {
    let fixture: Fixture = fixture("lfs-real-bounded");
    let Some((repo, origin, server, _content)) = lfs_world(&fixture) else {
        return;
    };
    let first: Observed = push_main(&fixture, repo.as_path(), &[]);
    assert_eq!(first.code, Some(0), "{}", stderr_of(&first));
    let after_first: usize = batches(&server);
    assert!(after_first >= 1, "the first push uploaded its object through the batch API");
    empty_commit(&fixture, repo.as_path());
    let second: Observed = push_main(&fixture, repo.as_path(), &[]);
    let stderr: String = stderr_of(&second);
    assert_eq!(second.code, Some(0), "{stderr}");
    assert_eq!(batches(&server), after_first, "the second push added no object, so no batch request: {stderr}");
    assert_eq!(server.uploads().len(), 1);
    assert_eq!(published_main(&fixture, origin.as_path()), super::commit::git_text(&fixture, repo.as_path(), &["rev-parse", "main"]).trim());
    drop(server);
    remove_settled(&fixture);
}

/// Positive control for the bound: a remote with a push URL keeps the full scan, which asks the server about every
///  object again, so the same push that sent nothing above sends one batch request here.
#[test]
fn a_remote_with_a_push_url_keeps_the_full_scan() {
    let fixture: Fixture = fixture("lfs-real-full-scan");
    let Some((repo, origin, server, _content)) = lfs_world(&fixture) else {
        return;
    };
    git(&fixture, repo.as_path(), &["config", "remote.origin.pushurl", origin.to_str().expect("UTF-8")]);
    let first: Observed = push_main(&fixture, repo.as_path(), &[]);
    assert_eq!(first.code, Some(0), "{}", stderr_of(&first));
    let after_first: usize = batches(&server);
    empty_commit(&fixture, repo.as_path());
    let second: Observed = push_main(&fixture, repo.as_path(), &[]);
    let stderr: String = stderr_of(&second);
    assert_eq!(second.code, Some(0), "{stderr}");
    assert_eq!(batches(&server), after_first + 1, "the full scan asked about the known object again: {stderr}");
    assert_eq!(server.uploads().len(), 1, "an object the server has is not sent twice");
    drop(server);
    remove_settled(&fixture);
}

/// A server that answers HTTP 503 twice and then works: the push is run again, the object arrives once, the pointer follows.
#[test]
fn a_brief_server_error_is_retried_and_the_push_succeeds() {
    let fixture: Fixture = fixture("lfs-real-retry");
    let Some((repo, origin, server, content)) = lfs_world(&fixture) else {
        return;
    };
    use_credentialed_url(&fixture, repo.as_path(), &server);
    server.state.lock().expect("state").fail_batches = 2;
    let pushed: Observed = push_main(&fixture, repo.as_path(), &[FAST_RETRIES]);
    let stderr: String = stderr_of(&pushed);
    assert_eq!(pushed.code, Some(0), "{stderr}");
    assert_eq!(batches(&server), 3, "two refused batch requests and the one that worked: {stderr}");
    let uploads: Vec<Upload> = server.uploads();
    assert_eq!(uploads.len(), 1, "{stderr}");
    assert_eq!(uploads[0].bytes, content);
    assert_eq!(uploads[0].origin_main, "absent");
    assert!(stderr.contains("retry 1 of 2") && stderr.contains("retry 2 of 2"), "{stderr}");
    assert!(stderr.contains("Fatal error: Server error http://<credentials>@127.0.0.1"), "git-lfs's own words are relayed, masked: {stderr}");
    assert!(stderr.contains("from HTTP 503"), "{stderr}");
    assert!(!stderr.contains("secret"), "the token never reaches the caller: {stderr}");
    assert_ne!(published_main(&fixture, origin.as_path()), "absent", "{stderr}");
    drop(server);
    remove_settled(&fixture);
}

/// A server error that outlasts the allowance stops the push after the first run and two retries, with the explanation.
#[test]
fn a_server_error_that_lasts_stops_after_two_retries() {
    let fixture: Fixture = fixture("lfs-real-retry-spent");
    let Some((repo, origin, server, _content)) = lfs_world(&fixture) else {
        return;
    };
    use_credentialed_url(&fixture, repo.as_path(), &server);
    server.state.lock().expect("state").fail_batches = 99;
    let pushed: Observed = push_main(&fixture, repo.as_path(), &[FAST_RETRIES]);
    let stderr: String = stderr_of(&pushed);
    assert_ne!(pushed.code, Some(0), "{stderr}");
    assert_eq!(batches(&server), 3, "{stderr}");
    assert!(stderr.contains("`git-lfs push` exited with status") && stderr.contains("nothing was published"), "{stderr}");
    assert!(!stderr.contains("secret"), "{stderr}");
    assert_eq!(published_main(&fixture, origin.as_path()), "absent", "{stderr}");
    assert!(server.uploads().is_empty());
    drop(server);
    remove_settled(&fixture);
}

/// The auto-push of a commit rides out a brief server error the same way, and what it records has no credentials.
#[test]
fn the_auto_push_retries_a_brief_server_error() {
    let fixture: Fixture = fixture("lfs-real-auto-retry");
    let Some((repo, origin, server, content)) = lfs_world(&fixture) else {
        return;
    };
    use_credentialed_url(&fixture, repo.as_path(), &server);
    std::fs::write(repo.join("second.bin"), [content.as_slice(), b"second"].concat()).expect("second.bin");
    let added: Observed = observe(
        &mut super::commit::wrapped_commit(&fixture, repo.as_path(), &["add", "--", "second.bin"]),
        b"",
    );
    assert_eq!(added.code, Some(0), "{}", stderr_of(&added));
    server.state.lock().expect("state").fail_batches = 1;
    let mut commit = super::commit::wrapped_commit(&fixture, repo.as_path(), &["commit", "--quiet", "--message=second", "--", "second.bin"]);
    commit.env(FAST_RETRIES.0, FAST_RETRIES.1);
    let landed: Observed = observe(&mut commit, b"");
    let stderr: String = stderr_of(&landed);
    assert_eq!(landed.code, Some(0), "{stderr}");
    assert!(!stderr.contains("auto-push to origin failed"), "{stderr}");
    assert_eq!(server.uploads().len(), 2, "{stderr}");
    assert_eq!(batches(&server), 2, "one refused batch request and the one that worked: {stderr}");
    assert!(!stderr.contains("secret"), "{stderr}");
    assert_ne!(published_main(&fixture, origin.as_path()), "absent", "{stderr}");
    drop(server);
    remove_settled(&fixture);
}

/// The auto-push whose server error lasts fails with the program's words and the explanation, credentials hidden.
#[test]
fn the_auto_push_reports_a_lasting_server_error_without_credentials() {
    let fixture: Fixture = fixture("lfs-real-auto-retry-spent");
    // The upload runs in the service's environment, so its retry timing is given there.
    super::support::give_service_environment(&fixture, FAST_RETRIES.0, FAST_RETRIES.1);
    let Some((repo, _origin, server, content)) = lfs_world(&fixture) else {
        return;
    };
    use_credentialed_url(&fixture, repo.as_path(), &server);
    std::fs::write(repo.join("second.bin"), [content.as_slice(), b"second"].concat()).expect("second.bin");
    let added: Observed = observe(
        &mut super::commit::wrapped_commit(&fixture, repo.as_path(), &["add", "--", "second.bin"]),
        b"",
    );
    assert_eq!(added.code, Some(0), "{}", stderr_of(&added));
    server.state.lock().expect("state").fail_batches = 99;
    let mut commit = super::commit::wrapped_commit(&fixture, repo.as_path(), &["commit", "--quiet", "--message=second", "--", "second.bin"]);
    commit.env(FAST_RETRIES.0, FAST_RETRIES.1);
    let landed: Observed = observe(&mut commit, b"");
    let stderr: String = stderr_of(&landed);
    assert_eq!(landed.code, Some(0), "the commit stays saved: {stderr}");
    let report: String = super::support::push_report(repo.as_path(), &stderr);
    assert_eq!(batches(&server), 3, "{report}");
    assert!(report.contains(super::support::push_failed_words()), "{report}");
    assert!(report.contains("http://<credentials>@127.0.0.1"), "{report}");
    assert!(!report.contains("secret"), "{report}");
    drop(server);
    remove_settled(&fixture);
}

/// Put a recording script in place of the `git-lfs` link: it appends its arguments to the returned file, then runs the
///  real program. The file does not exist until the wrapper starts `git-lfs`.
fn record_git_lfs(fixture: &Fixture) -> PathBuf {
    let link: PathBuf = fixture.root.join("bin/git-lfs");
    let real: PathBuf = std::fs::read_link(&link).expect("the link to the real program");
    std::fs::remove_file(&link).expect("link removed");
    let log: PathBuf = fixture.root.join("git-lfs-runs.txt");
    let script: String = format!("#!/bin/sh\nprintf '%s\\n' \"$*\" >> '{}'\nexec '{}' \"$@\"\n", log.display(), real.display());
    super::support::executable(link.as_path(), script.as_bytes());
    return log;
}

/// The arguments of every `git-lfs push` run, one run per line. Git's own `filter-process` runs of the same program are
///  not the upload and are left out.
fn recorded_runs(log: &Path) -> Vec<String> {
    return std::fs::read_to_string(log)
        .unwrap_or_default()
        .lines()
        .filter(|line| { return line.starts_with("push"); })
        .map(str::to_owned)
        .collect();
}

/// Add `name` with `content` as an LFS file through the wrapper (so the clean filter stores the object) and commit it with
///  real Git, so that no auto-push runs; the object name its pointer holds.
fn commit_lfs_file(fixture: &Fixture, repo: &Path, name: &str, content: &[u8]) -> String {
    std::fs::write(repo.join(name), content).expect("lfs file");
    let added: Observed = observe(&mut super::commit::wrapped_commit(fixture, repo, &["add", "--", name]), b"");
    assert_eq!(added.code, Some(0), "{}", stderr_of(&added));
    let mut path: std::ffi::OsString = fixture.root.join("bin").into_os_string();
    path.push(":/usr/bin:/bin");
    let committed = direct(fixture)
        .current_dir(repo)
        .env("PATH", path)
        .args(["commit", "--quiet", &format!("--message={name}"), "--", name])
        .output()
        .expect("commit starts");
    assert!(committed.status.success(), "{}", String::from_utf8_lossy(committed.stderr.as_slice()));
    let pointer: String = super::commit::git_text(fixture, repo, &["cat-file", "blob", &format!("HEAD:{name}")]);
    return pointer.lines().find_map(|line| { return line.strip_prefix("oid sha256:"); }).expect("an oid line").to_owned();
}

/// A push of commits that add no pointer starts no `git-lfs` at all and makes no request to the LFS server.
#[test]
fn a_push_without_a_new_pointer_starts_no_git_lfs() {
    let fixture: Fixture = fixture("lfs-real-none");
    let Some((repo, _origin, server, _content)) = lfs_world(&fixture) else {
        return;
    };
    let first: Observed = push_main(&fixture, repo.as_path(), &[]);
    assert_eq!(first.code, Some(0), "{}", stderr_of(&first));
    let log: PathBuf = record_git_lfs(&fixture);
    let after_first: usize = batches(&server);
    empty_commit(&fixture, repo.as_path());
    let second: Observed = push_main(&fixture, repo.as_path(), &[]);
    assert_eq!(second.code, Some(0), "{}", stderr_of(&second));
    assert_eq!(recorded_runs(log.as_path()), Vec::<String>::new(), "git-lfs was not started");
    assert_eq!(batches(&server), after_first, "and no request reached the server");
    drop(server);
    remove_settled(&fixture);
}

/// A push that adds one pointer uploads exactly that object by name, with one `git-lfs` run and one batch request.
#[test]
fn a_push_that_adds_one_pointer_uploads_exactly_that_object() {
    let fixture: Fixture = fixture("lfs-real-one");
    let Some((repo, origin, server, content)) = lfs_world(&fixture) else {
        return;
    };
    let first: Observed = push_main(&fixture, repo.as_path(), &[]);
    assert_eq!(first.code, Some(0), "{}", stderr_of(&first));
    let log: PathBuf = record_git_lfs(&fixture);
    let after_first: usize = batches(&server);
    let added: String = commit_lfs_file(&fixture, repo.as_path(), "second.bin", [content.as_slice(), b"second"].concat().as_slice());
    let second: Observed = push_main(&fixture, repo.as_path(), &[]);
    let stderr: String = stderr_of(&second);
    assert_eq!(second.code, Some(0), "{stderr}");
    assert_eq!(recorded_runs(log.as_path()), vec![format!("push --object-id origin {added}")], "{stderr}");
    assert_eq!(batches(&server), after_first + 1, "{stderr}");
    let uploads: Vec<Upload> = server.uploads();
    assert_eq!(uploads.len(), 2, "{stderr}");
    assert_eq!(uploads[1].oid, added);
    assert_ne!(published_main(&fixture, origin.as_path()), "absent");
    drop(server);
    remove_settled(&fixture);
}

/// A pointer that one commit adds and the next removes is still uploaded, because the commit that holds it is pushed.
#[test]
fn a_pointer_added_and_removed_between_pushes_is_still_uploaded() {
    let fixture: Fixture = fixture("lfs-real-transient");
    let Some((repo, _origin, server, content)) = lfs_world(&fixture) else {
        return;
    };
    let first: Observed = push_main(&fixture, repo.as_path(), &[]);
    assert_eq!(first.code, Some(0), "{}", stderr_of(&first));
    let log: PathBuf = record_git_lfs(&fixture);
    let transient: String = commit_lfs_file(&fixture, repo.as_path(), "transient.bin", [content.as_slice(), b"transient"].concat().as_slice());
    let mut path: std::ffi::OsString = fixture.root.join("bin").into_os_string();
    path.push(":/usr/bin:/bin");
    for arguments in [["rm", "--quiet", "--", "transient.bin"], ["commit", "--quiet", "--message=remove it again", ""]] {
        let kept: Vec<&str> = arguments.iter().copied().filter(|word| { return !word.is_empty(); }).collect();
        let ran = direct(&fixture).current_dir(repo.as_path()).env("PATH", &path).args(kept).output().expect("starts");
        assert!(ran.status.success(), "{}", String::from_utf8_lossy(ran.stderr.as_slice()));
    }
    let second: Observed = push_main(&fixture, repo.as_path(), &[]);
    let stderr: String = stderr_of(&second);
    assert_eq!(second.code, Some(0), "{stderr}");
    assert_eq!(recorded_runs(log.as_path()), vec![format!("push --object-id origin {transient}")], "{stderr}");
    assert!(server.uploads().iter().any(|upload| { return upload.oid == transient; }), "{stderr}");
    drop(server);
    remove_settled(&fixture);
}

/// The destination the tracking refs may not describe is scanned from the start: the object the first push sent is named
///  again (the server answers that it has it), and only that, by name.
#[test]
fn an_untrusted_destination_names_every_pointer_of_the_history() {
    let fixture: Fixture = fixture("lfs-real-untrusted");
    let Some((repo, origin, server, _content)) = lfs_world(&fixture) else {
        return;
    };
    git(&fixture, repo.as_path(), &["config", "remote.origin.pushurl", origin.to_str().expect("UTF-8")]);
    let first: Observed = push_main(&fixture, repo.as_path(), &[]);
    assert_eq!(first.code, Some(0), "{}", stderr_of(&first));
    let log: PathBuf = record_git_lfs(&fixture);
    empty_commit(&fixture, repo.as_path());
    let second: Observed = push_main(&fixture, repo.as_path(), &[]);
    let stderr: String = stderr_of(&second);
    assert_eq!(second.code, Some(0), "{stderr}");
    let runs: Vec<String> = recorded_runs(log.as_path());
    assert_eq!(runs.len(), 1, "{runs:?}");
    assert!(runs[0].starts_with("push --object-id origin ") && runs[0].len() == "push --object-id origin ".len() + 64, "{runs:?}");
    assert_eq!(server.uploads().len(), 1, "an object the server has is not sent twice");
    drop(server);
    remove_settled(&fixture);
}

/// When the readers decline the repository, `git-lfs` scans with the bounded form, as before.
#[test]
fn a_repository_the_readers_decline_is_scanned_by_git_lfs() {
    let fixture: Fixture = fixture("lfs-real-declined");
    let Some((repo, _origin, server, _content)) = lfs_world(&fixture) else {
        return;
    };
    let first: Observed = push_main(&fixture, repo.as_path(), &[]);
    assert_eq!(first.code, Some(0), "{}", stderr_of(&first));
    let log: PathBuf = record_git_lfs(&fixture);
    decline_in_process_lfs_scan(&fixture, repo.as_path());
    empty_commit(&fixture, repo.as_path());
    let second: Observed = push_main(&fixture, repo.as_path(), &[]);
    assert_eq!(second.code, Some(0), "{}", stderr_of(&second));
    assert_eq!(recorded_runs(log.as_path()), vec![String::from("push origin main")]);
    drop(server);
    remove_settled(&fixture);
}

/// A push that recurses into submodules and adds no pointer of its own starts no `git-lfs`, and still prints the notice
///  that a submodule's LFS objects are not covered.
#[test]
fn a_submodule_push_without_a_new_pointer_still_gets_the_notice() {
    let fixture: Fixture = fixture("lfs-real-submodule-notice");
    let Some((repo, _origin, server, _content)) = lfs_world(&fixture) else {
        return;
    };
    let first: Observed = push_main(&fixture, repo.as_path(), &[]);
    assert_eq!(first.code, Some(0), "{}", stderr_of(&first));
    let log: PathBuf = record_git_lfs(&fixture);
    empty_commit(&fixture, repo.as_path());
    let mut command = super::commit::wrapped_commit(&fixture, repo.as_path(), &["push", "--recurse-submodules=on-demand", "origin", "main"]);
    let second: Observed = observe(&mut command, b"");
    let stderr: String = stderr_of(&second);
    assert_eq!(second.code, Some(0), "{stderr}");
    assert!(stderr.contains("this push also pushes submodules"), "{stderr}");
    assert_eq!(recorded_runs(log.as_path()), Vec::<String>::new(), "git-lfs was not started");
    drop(server);
    remove_settled(&fixture);
}