gunnar-server 4.0.0

//! **A one-commit thin fetch, on every engine this build carries, checked by
//! stock git** (lane GUN-PERF-FETCH, 2026-10-31).
//!
//! The fetch path hands the engine a voucher — what the client already holds —
//! and the engine may then send a delta against a base it does not put in the
//! pack. Get the voucher wrong in one direction and the pack is fatter; get it
//! wrong in the other and the client cannot resolve it. So the answer is judged
//! by the receiver git ships:
//!
//! * `git index-pack --stdin --fix-thin --strict` takes the pack, and appends
//!   the one base the client supplied (the pack really was thin);
//! * `git verify-pack` and `git fsck --full --strict` are clean afterwards, with
//!   the client's `main` moved to the new commit.
//!
//! The fixture is built so that the new commit's file is stored as a delta
//! against the previous version: the new version is smaller, and
//! `pack-objects` deltifies the smaller object against the larger one.

use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};

use gunnar_server::storage::{Engine, ObjectSet};
use gunnar_store::git_storage_trait::RefUpdate;

fn git(dir: &Path, args: &[&str]) -> String {
    let out = Command::new("git")
        .args(args)
        .current_dir(dir)
        .env("GIT_CONFIG_NOSYSTEM", "1")
        .env("GIT_CONFIG_GLOBAL", "/dev/null")
        .output()
        .expect("git is on PATH");
    assert!(
        out.status.success(),
        "git {args:?} in {}: {}",
        dir.display(),
        String::from_utf8_lossy(&out.stderr)
    );
    String::from_utf8(out.stdout)
        .expect("utf8")
        .trim()
        .to_owned()
}

/// `git <args>` with `stdin` piped in, returning (stdout, stderr).
fn git_in(dir: &Path, args: &[&str], stdin: &[u8]) -> (Vec<u8>, String) {
    let mut child = Command::new("git")
        .args(args)
        .current_dir(dir)
        .env("GIT_CONFIG_NOSYSTEM", "1")
        .env("GIT_CONFIG_GLOBAL", "/dev/null")
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()
        .expect("git is on PATH");
    child
        .stdin
        .take()
        .expect("stdin")
        .write_all(stdin)
        .expect("feed git");
    let out = child.wait_with_output().expect("git output");
    let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
    assert!(out.status.success(), "git {args:?}: {stderr}");
    (out.stdout, stderr)
}

/// The tests here share the process-wide store cache, and each ends with
/// `storage::shutdown()`, which releases every cached store: one at a time.
static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(());

fn serial() -> std::sync::MutexGuard<'static, ()> {
    SERIAL
        .lock()
        .unwrap_or_else(std::sync::PoisonError::into_inner)
}

fn pkt(line: &str) -> Vec<u8> {
    format!("{:04x}{line}", line.len() + 4).into_bytes()
}

/// The packfile out of a v2 `fetch` response: everything on band 1 after the
/// `packfile` section header.
fn pack_from_response(mut bytes: &[u8]) -> Vec<u8> {
    let mut pack = Vec::new();
    let mut in_pack = false;
    while bytes.len() >= 4 {
        let len = usize::from_str_radix(std::str::from_utf8(&bytes[..4]).expect("hex"), 16)
            .expect("a pkt-line length");
        if len < 4 {
            bytes = &bytes[4..];
            if in_pack && len == 0 {
                break;
            }
            continue;
        }
        let payload = &bytes[4..len];
        bytes = &bytes[len..];
        if in_pack {
            match payload[0] {
                1 => pack.extend_from_slice(&payload[1..]),
                2 => {}
                band => panic!(
                    "band {band} from the server: {}",
                    String::from_utf8_lossy(&payload[1..])
                ),
            }
        } else if payload == b"packfile\n" {
            in_pack = true;
        }
    }
    assert!(!pack.is_empty(), "the response carried no packfile");
    pack
}

/// A source repository with history worth negotiating over: one file of 400
/// lines changed one line per commit, a tag on every fifth commit (every tenth
/// annotated) and a side branch.
fn source(work: &Path) -> PathBuf {
    let src = work.join("src");
    std::fs::create_dir_all(&src).expect("src dir");
    git(&src, &["init", "-q", "--initial-branch=main", "."]);
    git(&src, &["config", "user.email", "t@t.invalid"]);
    git(&src, &["config", "user.name", "t"]);
    let mut lines: Vec<String> = (0..400)
        .map(|i| format!("line {i} of a file that is long enough to be worth a delta"))
        .collect();
    for i in 0..30 {
        lines[i * 7 % 400] = format!("line {} changed by commit {i}", i * 7 % 400);
        std::fs::write(src.join("big.txt"), lines.join("\n") + "\n").expect("write");
        std::fs::write(src.join(format!("small-{i}.txt")), format!("{i}\n")).expect("write");
        git(&src, &["add", "."]);
        git(&src, &["commit", "-qm", &format!("commit {i}")]);
        if i % 10 == 0 {
            git(&src, &["tag", "-a", "-m", "release", &format!("v{i}")]);
        } else if i % 5 == 0 {
            git(&src, &["tag", &format!("t{i}")]);
        }
        if i == 12 {
            git(&src, &["branch", "side"]);
        }
    }
    src
}

fn refs_of(dir: &Path) -> Vec<(String, String)> {
    git(dir, &["for-each-ref", "--format=%(objectname) %(refname)"])
        .lines()
        .map(|line| {
            let (oid, name) = line.split_once(' ').expect("oid and name");
            (oid.to_owned(), name.to_owned())
        })
        .collect()
}

fn thin_fetch_on(engine: Engine) {
    let dir = tempfile::tempdir().expect("tempdir");
    let src = source(dir.path());

    // The client: everything as it stands now.
    let client = dir.path().join("client.git");
    git(
        dir.path(),
        &[
            "clone",
            "-q",
            "--mirror",
            "--no-local",
            src.to_str().expect("utf8"),
            client.to_str().expect("utf8"),
        ],
    );
    let held_refs = refs_of(&client);

    // One new commit on main whose file SHRINKS, so it is stored as a delta
    // against the version the client already has.
    let text = std::fs::read_to_string(src.join("big.txt")).expect("read");
    let shorter: Vec<&str> = text.lines().take(350).collect();
    std::fs::write(src.join("big.txt"), shorter.join("\n") + "\n").expect("write");
    git(&src, &["commit", "-qam", "drop the last fifty lines"]);
    let new = git(&src, &["rev-parse", "HEAD"]);

    // The server: the whole history in one pack, as one push would bring it.
    let git_dir = dir.path().join("served.git");
    let kind = gix_hash::Kind::Sha1;
    let repo = engine
        .open_repo(&git_dir, kind)
        .expect("open the repository");
    let (pack, _) = git_in(
        &src,
        &[
            "pack-objects",
            "--all",
            "--revs",
            "--stdout",
            "-q",
            // What `send-pack` asks for from a server that advertises
            // `ofs-delta`, which every engine here does.
            "--delta-base-offset",
        ],
        b"",
    );
    engine
        .ingest_push(
            &git_dir,
            kind,
            &mut pack.as_slice(),
            ObjectSet::NotNeeded,
            None,
            None,
        )
        .expect("ingest the history");
    let updates: Vec<RefUpdate> = refs_of(&src)
        .into_iter()
        .map(|(oid, name)| RefUpdate::set(name, oid))
        .collect();
    repo.serve().put_refs(&updates).expect("install the refs");
    repo.serve().set_head("refs/heads/main").expect("HEAD");
    // The served state is the INDEXED one: the znippy arm folds a pushed pack
    // in the background, and a fetch that meets the fold mid-way is another
    // test's subject.
    repo.serve().index_now().expect("index the pushed pack");

    let pack = thin_fetch(repo.serve(), kind, &new, &held_refs)
        .unwrap_or_else(|e| panic!("{engine}: serve the fetch: {e}"));
    taken_thin_by_stock_git(&engine.to_string(), &client, &pack, &new);
    gunnar_server::storage::shutdown();
}

/// The fetch: want the new tip, have every ref the client holds — what `git
/// fetch` offers first — and the packfile that came back.
fn thin_fetch(
    serve: &dyn gunnar_store::git_storage_trait::GitServe,
    kind: gix_hash::Kind,
    new: &str,
    held_refs: &[(String, String)],
) -> Result<Vec<u8>, String> {
    let mut request = Vec::new();
    request.extend(pkt("command=fetch\n"));
    request.extend(pkt("object-format=sha1\n"));
    request.extend_from_slice(b"0001");
    request.extend(pkt("thin-pack\n"));
    request.extend(pkt("ofs-delta\n"));
    request.extend(pkt("no-progress\n"));
    request.extend(pkt(&format!("want {new}\n")));
    for (oid, _) in held_refs {
        request.extend(pkt(&format!("have {oid}\n")));
    }
    request.extend(pkt("done\n"));
    request.extend_from_slice(b"0000");
    let upload = gunnar_wire::UploadPack::new(serve, kind);
    let mut out = Vec::new();
    upload
        .serve_session(
            gunnar_wire::ProtocolVersion::V2,
            gunnar_wire::Session::StatelessRpc,
            &mut request.as_slice(),
            &mut out,
        )
        .map_err(|e| format!("{e:#}"))?;
    Ok(pack_from_response(&out))
}

/// Stock git takes `pack` into `client`, completing the thin delta from its own
/// objects, and the repository is clean with `main` at `new` afterwards.
fn taken_thin_by_stock_git(what: &str, client: &Path, pack: &[u8], new: &str) {
    let sent = u32::from_be_bytes(pack[8..12].try_into().expect("a pack header"));
    assert_eq!(
        sent, 3,
        "{what}: one commit, one tree and one blob: the pack must carry exactly those"
    );
    let (stdout, _) = git_in(
        client,
        &["index-pack", "--stdin", "--fix-thin", "--strict"],
        pack,
    );
    let name = String::from_utf8(stdout).expect("utf8");
    let hash = name
        .trim()
        .strip_prefix("pack\t")
        .unwrap_or(name.trim())
        .to_owned();
    let idx = client.join(format!("objects/pack/pack-{hash}.idx"));
    let (listed, _) = git_in(client, &["show-index"], &std::fs::read(&idx).expect("idx"));
    let indexed = String::from_utf8(listed).expect("utf8").lines().count();
    assert_eq!(
        indexed, 4,
        "{what}: `index-pack --fix-thin` must have appended the one base the client held (the \
         previous big.txt); {indexed} objects means the pack was not thin and the voucher was \
         lost"
    );
    git(client, &["verify-pack", idx.to_str().expect("utf8")]);
    git(client, &["update-ref", "refs/heads/main", new]);
    git(client, &["fsck", "--full", "--strict", "--no-dangling"]);
}

/// 64 MiB a 64-bit LCG makes, which deflate cannot shrink: what keeps the
/// znippy arm's fold busy long enough for a fetch to land inside it.
fn incompressible(seed: u64, len: usize) -> Vec<u8> {
    let mut state = seed | 1;
    let mut out = Vec::with_capacity(len);
    while out.len() < len {
        state = state
            .wrapping_mul(6364136223846793005)
            .wrapping_add(1442695040888963407);
        out.extend_from_slice(&state.to_le_bytes());
    }
    out.truncate(len);
    out
}

/// **A thin fetch that lands while the znippy arm is folding a pushed pack is
/// served, and is still thin** (lane GUN-PERF-FETCH, znippy-plugin-git 0.2.4).
///
/// The voucher is closed over by the engine's reachability. Until 0.2.4 that
/// refused with `index in progress` while the indexer held the gate, and gunnar
/// re-emitted the pack without the voucher (GF-4, 71023a43) — a fatter pack,
/// every boundary delta rebuilt whole. 0.2.4's `client_bases` answers from the
/// last committed fold instead. So: the history with one new commit indexed
/// (its big.txt a stored delta on the client's version), then 64 MiB of
/// incompressible blobs pushed on another branch, and a fetch of the new tip
/// while that pack folds.
/// Another request is folding that pack (`index_now` on a second thread) when
/// the fetch arrives; a fetch counts as *during* the fold only when the index
/// reports the same pack active both before and after it.
#[cfg(feature = "engine-znippy")]
#[test]
fn a_thin_fetch_that_meets_an_index_build_is_served_thin() {
    let _serial = serial();
    let engine = Engine::Znippy;
    let dir = tempfile::tempdir().expect("tempdir");
    let src = source(dir.path());
    let client = dir.path().join("client.git");
    git(
        dir.path(),
        &[
            "clone",
            "-q",
            "--mirror",
            "--no-local",
            src.to_str().expect("utf8"),
            client.to_str().expect("utf8"),
        ],
    );
    let held_refs = refs_of(&client);
    let old = git(&src, &["rev-parse", "HEAD"]);
    // One new commit on main whose file SHRINKS, so the history pack stores it
    // as a delta against the version the client already has.
    let text = std::fs::read_to_string(src.join("big.txt")).expect("read");
    let shorter: Vec<&str> = text.lines().take(350).collect();
    std::fs::write(src.join("big.txt"), shorter.join("\n") + "\n").expect("write");
    git(&src, &["commit", "-qam", "drop the last fifty lines"]);
    let new = git(&src, &["rev-parse", "HEAD"]);

    // The server holds the history with the new commit, indexed.
    let git_dir = dir.path().join("served.git");
    let kind = gix_hash::Kind::Sha1;
    let repo = engine
        .open_repo(&git_dir, kind)
        .expect("open the repository");
    let (history, _) = git_in(
        &src,
        &[
            "pack-objects",
            "--all",
            "--revs",
            "--stdout",
            "-q",
            "--delta-base-offset",
        ],
        b"",
    );
    engine
        .ingest_push(
            &git_dir,
            kind,
            &mut history.as_slice(),
            ObjectSet::NotNeeded,
            None,
            None,
        )
        .expect("ingest the history");
    let updates: Vec<RefUpdate> = refs_of(&src)
        .into_iter()
        .map(|(oid, name)| RefUpdate::set(name, oid))
        .collect();
    repo.serve().put_refs(&updates).expect("install the refs");
    repo.serve().set_head("refs/heads/main").expect("HEAD");
    repo.serve().index_now().expect("index the history");

    // The bulk branch to fold, off the commit the client holds.
    git(&src, &["checkout", "-q", "-b", "bulk", &old]);
    std::fs::create_dir_all(src.join("bulk")).expect("bulk dir");
    for i in 0..64u64 {
        std::fs::write(
            src.join(format!("bulk/blob-{i}")),
            incompressible(i + 1, 1 << 20),
        )
        .expect("write a bulk blob");
    }
    git(&src, &["add", "bulk"]);
    git(&src, &["commit", "-qm", "bulk"]);
    let bulk = git(&src, &["rev-parse", "HEAD"]);
    git(&src, &["checkout", "-q", "main"]);
    // The push the fetch lands in: the bulk branch, stored whole
    // (`--window=0`: incompressible blobs are not worth a delta search).
    let (push, _) = git_in(
        &src,
        &[
            "pack-objects",
            "--revs",
            "--stdout",
            "-q",
            "--delta-base-offset",
            "--window=0",
        ],
        format!("{bulk}\n^{old}\n").as_bytes(),
    );

    let folding = || -> Option<u64> {
        engine
            .index_progress()
            .into_iter()
            .find(|(d, s)| d == &git_dir && s.active && s.phase != "paused")
            .map(|(_, s)| s.pack_id)
    };

    let mut met = 0;
    // Each round pushes the bulk pack again (a fresh copy is a fresh fold), so
    // a fast box gets more than one chance to land a fetch inside one; the
    // first fetch that does is checked by stock git and ends the test.
    for round in 0..8 {
        engine
            .ingest_push(
                &git_dir,
                kind,
                &mut push.as_slice(),
                ObjectSet::NotNeeded,
                None,
                None,
            )
            .expect("ingest the push");
        if round == 0 {
            repo.serve()
                .put_refs(&[RefUpdate::set("refs/heads/bulk", bulk.clone())])
                .expect("install bulk");
        }
        // The fold is another request's: one thread indexes (it holds the
        // gate for the whole pack), and the fetch arrives while it does —
        // what two clients of one busy repository look like.
        std::thread::scope(|scope| {
            let indexer = scope.spawn(|| repo.serve().index_now());
            let mut before = folding();
            while before.is_none() && !indexer.is_finished() {
                std::thread::yield_now();
                before = folding();
            }
            if before.is_some() {
                let served = thin_fetch(repo.serve(), kind, &new, &held_refs);
                let after = folding();
                if before == after {
                    let pack = served.unwrap_or_else(|e| {
                        panic!("a thin fetch during an index build was REFUSED: {e}")
                    });
                    taken_thin_by_stock_git("znippy, during a fold", &client, &pack, &new);
                    met += 1;
                } else if let Err(e) = served {
                    panic!("a thin fetch was refused: {e}");
                }
            }
            indexer
                .join()
                .expect("the indexing thread")
                .expect("index the push");
        });
        if met > 0 {
            break;
        }
    }
    assert!(
        met > 0,
        "no fetch landed inside a fold in eight pushes of 64 MiB; this box folds too fast for \
         the test to say anything"
    );
    repo.serve().index_now().expect("finish the index");
    gunnar_server::storage::shutdown();
}

#[test]
fn a_one_commit_thin_fetch_is_what_stock_git_expects_on_every_engine() {
    let _serial = serial();
    for engine in Engine::COMPILED {
        thin_fetch_on(*engine);
    }
}