ldeflate 0.1.2

//! **The acceptance test that decides whether this crate is worth anything.**
//!
//! A parallel compressor that emits something only we can read is worthless for
//! the wire — stock `git` is the client in nearly every case. So this file does
//! not check "our decoder reads our encoder". It hands the bytes to the real
//! `git` binary on this machine and makes `git` judge them, two ways:
//!
//! 1. **Loose objects.** A repository is built whose *entire object store* this
//!    crate wrote. `git fsck --full --strict` then validates every object, and
//!    `git cat-file` reads each one back for a byte comparison.
//! 2. **A packfile.** A real pack-v2 file is assembled from our streams and put
//!    through `git index-pack --strict`, which inflates every object, re-hashes
//!    it, and verifies the trailer.
//!
//! Both arms run the **split** path as well as the whole-object path, because
//! the split path is the one that hand-writes a zlib header and a combined
//! Adler-32 trailer — precisely the bytes a subtly-wrong implementation gets
//! wrong in a way only a real decoder notices.
//!
//! The tests skip (loudly, on stderr) if `git` is not installed, rather than
//! passing vacuously.

use std::path::{Path, PathBuf};
use std::process::Command;

use ldeflate::{Config, compress_objects};
use sha1::{Digest, Sha1};

fn have_git() -> bool {
    Command::new("git")
        .arg("--version")
        .output()
        .map(|o| o.status.success())
        .unwrap_or(false)
}

fn scratch(name: &str) -> PathBuf {
    // Build scratch lives on md1, never /tmp (which is RAM on this box).
    let base = std::env::var("LDEFLATE_TEST_SCRATCH")
        .unwrap_or_else(|_| "/home/rickard/scratch/cargo/zoomies-deflate/gittest".to_string());
    let dir = PathBuf::from(base).join(format!("{name}-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&dir);
    std::fs::create_dir_all(&dir).expect("create scratch dir");
    dir
}

fn git(dir: &Path, args: &[&str]) -> (bool, String, String) {
    let o = Command::new("git")
        .current_dir(dir)
        .args(args)
        .output()
        .unwrap_or_else(|e| panic!("git {args:?}: {e}"));
    (
        o.status.success(),
        String::from_utf8_lossy(&o.stdout).into_owned(),
        String::from_utf8_lossy(&o.stderr).into_owned(),
    )
}

/// The corpus: sizes deliberately spanning all three planner queues, with
/// content that is compressible but distinct per object, so a mis-assembled
/// stream cannot decode to the right bytes by luck.
fn corpus() -> Vec<Vec<u8>> {
    let mut out = Vec::new();
    // The git-shaped bulk: hundreds of tiny objects (the coalesced queue).
    for i in 0..400usize {
        let mut v = Vec::new();
        let tag = format!("blob-{i} ");
        while v.len() < 40 + (i * 13) % 900 {
            v.extend_from_slice(tag.as_bytes());
            v.extend_from_slice(b"fn main() { let x = 1; }\n");
        }
        out.push(v);
    }
    // Mid-sized (the whole-object queue).
    for i in 0..8usize {
        let mut v = Vec::new();
        let tag = format!("mid-{i} ");
        while v.len() < 300_000 {
            v.extend_from_slice(tag.as_bytes());
            v.extend_from_slice(b"use std::collections::HashMap; // padding padding\n");
        }
        out.push(v);
    }
    // Large (the split queue) — this is the one that exercises the hand-written
    // header + combined Adler-32.
    for i in 0..2usize {
        let mut v = Vec::new();
        let tag = format!("big-{i} ");
        while v.len() < 12 << 20 {
            v.extend_from_slice(tag.as_bytes());
            v.extend_from_slice(b"the quick brown fox jumps over the lazy dog 0123456789\n");
        }
        out.push(v);
    }
    // Incompressible, large — the LFS-shaped case, and the one where a stream
    // can legitimately grow. Still must be a valid zlib stream.
    let mut noise = Vec::with_capacity(4 << 20);
    let mut s: u64 = 0x243F6A8885A308D3;
    while noise.len() < 4 << 20 {
        s ^= s << 13;
        s ^= s >> 7;
        s ^= s << 17;
        noise.extend_from_slice(&s.to_le_bytes());
    }
    out.push(noise);
    // Empty object — git has a real empty blob and it must round-trip.
    out.push(Vec::new());
    out
}

fn blob_oid(content: &[u8]) -> [u8; 20] {
    let mut h = Sha1::new();
    h.update(format!("blob {}\0", content.len()).as_bytes());
    h.update(content);
    h.finalize().into()
}

fn hex(b: &[u8]) -> String {
    b.iter().map(|x| format!("{x:02x}")).collect()
}

/// ARM 1 — a repository whose whole object store this crate wrote, validated by
/// `git fsck --full --strict` and read back with `git cat-file`.
#[test]
fn stock_git_reads_a_loose_object_store_we_wrote() {
    if !have_git() {
        eprintln!("SKIP stock_git loose-object arm: `git` not on PATH");
        return;
    }
    let dir = scratch("loose");
    let (ok, _, err) = git(&dir, &["init", "--quiet", "."]);
    assert!(ok, "git init failed: {err}");

    let data = corpus();
    // The *loose object body* is `"<type> <len>\0" ++ content`, zlib-compressed.
    let bodies: Vec<Vec<u8>> = data
        .iter()
        .map(|c| {
            let mut b = format!("blob {}\0", c.len()).into_bytes();
            b.extend_from_slice(c);
            b
        })
        .collect();
    let refs: Vec<&[u8]> = bodies.iter().map(|b| b.as_slice()).collect();

    // Compress the whole store in ONE parallel pass — every queue at once.
    let streams = compress_objects(&refs, &Config::default()).expect("compress");
    assert_eq!(streams.len(), data.len());

    let mut oids = Vec::new();
    for (content, z) in data.iter().zip(&streams) {
        let oid = hex(&blob_oid(content));
        let sub = dir.join(".git/objects").join(&oid[..2]);
        std::fs::create_dir_all(&sub).unwrap();
        std::fs::write(sub.join(&oid[2..]), z).unwrap();
        oids.push(oid);
    }

    // ── git's own verdict ───────────────────────────────────────────────────
    let (ok, out, err) = git(&dir, &["fsck", "--full", "--strict", "--no-progress"]);
    assert!(
        ok,
        "git fsck REJECTED an object store written by ldeflate.\nstdout:\n{out}\nstderr:\n{err}"
    );
    // fsck prints `dangling blob <oid>` for every unreferenced blob (all of
    // ours are — there are no refs) and a `notice: No default references`.
    // Anything else is a real complaint.
    let mut dangling = 0usize;
    for line in out.lines().chain(err.lines()) {
        let l = line.trim();
        if l.is_empty() || l.starts_with("Checking") || l.starts_with("notice:") {
            continue;
        }
        if l.starts_with("dangling ") {
            dangling += 1;
            continue;
        }
        panic!("git fsck complained: {l}");
    }
    // ...and fsck must have SEEN them. A silent fsck that walked an empty store
    // would otherwise "pass" this arm without inflating a single byte.
    assert_eq!(
        dangling,
        data.len(),
        "git fsck reported {dangling} dangling objects but we wrote {} — fsck did \
         not actually visit the store, so its clean verdict means nothing",
        data.len()
    );

    // ── byte-for-byte read-back through git's own inflate ───────────────────
    for (i, (oid, content)) in oids.iter().zip(&data).enumerate() {
        let o = Command::new("git")
            .current_dir(&dir)
            .args(["cat-file", "blob", oid])
            .output()
            .unwrap();
        assert!(
            o.status.success(),
            "git cat-file failed for object {i} ({oid})"
        );
        assert_eq!(
            o.stdout.len(),
            content.len(),
            "object {i} ({oid}, {} B): git read back a different length",
            content.len()
        );
        assert_eq!(
            &o.stdout, content,
            "object {i} ({oid}): git read back different bytes"
        );
    }
    let _ = std::fs::remove_dir_all(&dir);
}

/// ARM 2 — a real pack-v2 file assembled from our streams, judged by
/// `git index-pack --strict`, which inflates every entry, re-hashes it against
/// the object id, and verifies the pack trailer.
#[test]
fn stock_git_index_pack_accepts_a_pack_we_wrote() {
    if !have_git() {
        eprintln!("SKIP stock_git pack arm: `git` not on PATH");
        return;
    }
    let dir = scratch("pack");
    let (ok, _, err) = git(&dir, &["init", "--quiet", "."]);
    assert!(ok, "git init failed: {err}");

    let data = corpus();
    let refs: Vec<&[u8]> = data.iter().map(|b| b.as_slice()).collect();
    // In a pack the entry payload is the RAW object content — the type and size
    // live in the entry header, not in a prefix inside the zlib stream.
    let streams = compress_objects(&refs, &Config::default()).expect("compress");

    let mut pack: Vec<u8> = Vec::new();
    pack.extend_from_slice(b"PACK");
    pack.extend_from_slice(&2u32.to_be_bytes());
    pack.extend_from_slice(&(data.len() as u32).to_be_bytes());
    for (content, z) in data.iter().zip(&streams) {
        pack.extend_from_slice(&pack_entry_header(3 /* OBJ_BLOB */, content.len()));
        pack.extend_from_slice(z);
    }
    let mut h = Sha1::new();
    h.update(&pack);
    let trailer: [u8; 20] = h.finalize().into();
    pack.extend_from_slice(&trailer);

    let pack_path = dir.join("ours.pack");
    std::fs::write(&pack_path, &pack).unwrap();

    let o = Command::new("git")
        .current_dir(&dir)
        .args([
            "index-pack",
            "--strict",
            "-o",
            "ours.idx",
            pack_path.to_str().unwrap(),
        ])
        .output()
        .unwrap();
    assert!(
        o.status.success(),
        "git index-pack --strict REJECTED a pack written by ldeflate.\nstdout: {}\nstderr: {}",
        String::from_utf8_lossy(&o.stdout),
        String::from_utf8_lossy(&o.stderr)
    );

    // index-pack prints the pack's own SHA-1; a successful run means every entry
    // inflated and every object id matched. Verify the objects are now readable
    // through the pack by asking git for one of the large (split-path) ones.
    let (ok, _, err) = git(&dir, &["verify-pack", "-v", "ours.idx"]);
    assert!(ok, "git verify-pack failed on our pack: {err}");
    let _ = std::fs::remove_dir_all(&dir);
}

/// RED-when-broken: `git index-pack --strict` must actually reject a corrupt
/// stream. Without this, ARM 2 would pass even if index-pack were ignoring the
/// payload entirely — the "check output, not exit code" trap.
#[test]
fn index_pack_actually_rejects_a_corrupted_stream() {
    if !have_git() {
        eprintln!("SKIP index-pack negative control: `git` not on PATH");
        return;
    }
    let dir = scratch("packneg");
    let (ok, _, err) = git(&dir, &["init", "--quiet", "."]);
    assert!(ok, "git init failed: {err}");

    let content = b"the quick brown fox jumps over the lazy dog".repeat(500);
    let refs: Vec<&[u8]> = vec![&content];
    let mut streams = compress_objects(&refs, &Config::default()).unwrap();
    // Flip a byte in the middle of the compressed payload.
    let mid = streams[0].len() / 2;
    streams[0][mid] ^= 0x5a;

    let mut pack: Vec<u8> = Vec::new();
    pack.extend_from_slice(b"PACK");
    pack.extend_from_slice(&2u32.to_be_bytes());
    pack.extend_from_slice(&1u32.to_be_bytes());
    pack.extend_from_slice(&pack_entry_header(3, content.len()));
    pack.extend_from_slice(&streams[0]);
    let mut h = Sha1::new();
    h.update(&pack);
    let trailer: [u8; 20] = h.finalize().into();
    pack.extend_from_slice(&trailer);

    let pack_path = dir.join("bad.pack");
    std::fs::write(&pack_path, &pack).unwrap();
    let o = Command::new("git")
        .current_dir(&dir)
        .args([
            "index-pack",
            "--strict",
            "-o",
            "bad.idx",
            pack_path.to_str().unwrap(),
        ])
        .output()
        .unwrap();
    assert!(
        !o.status.success(),
        "git index-pack ACCEPTED a pack with a corrupted zlib payload — the \
         acceptance test above is not actually validating anything"
    );
    let _ = std::fs::remove_dir_all(&dir);
}

/// Pack-v2 per-object header: 3 type bits and a variable-length size, low bits
/// first, continuation flag in bit 7.
fn pack_entry_header(obj_type: u8, mut size: usize) -> Vec<u8> {
    let mut out = Vec::with_capacity(8);
    let mut byte = ((obj_type & 0x7) << 4) | (size as u8 & 0x0f);
    size >>= 4;
    while size > 0 {
        out.push(byte | 0x80);
        byte = (size as u8) & 0x7f;
        size >>= 7;
    }
    out.push(byte);
    out
}

/// The header encoder is hand-written, so pin it against the values git's own
/// `pack-objects` produces for known sizes. A wrong header makes index-pack
/// fail in a way that looks like a compressor bug.
#[test]
fn pack_entry_header_known_values() {
    // blob, size 0  -> type 3 << 4 | 0 = 0x30
    assert_eq!(pack_entry_header(3, 0), vec![0x30]);
    // blob, size 15 -> 0x3f (fits in the 4 low bits, no continuation)
    assert_eq!(pack_entry_header(3, 15), vec![0x3f]);
    // blob, size 16 -> low nibble 0, continue, then 1
    assert_eq!(pack_entry_header(3, 16), vec![0xb0, 0x01]);
    // blob, size 2048 -> 2048 = 0x800; low nibble 0, rest 0x80 -> 0x00|cont, 0x01
    assert_eq!(pack_entry_header(3, 2048), vec![0xb0, 0x80, 0x01]);
    // commit (type 1), size 1
    assert_eq!(pack_entry_header(1, 1), vec![0x11]);
}

/// RED-when-broken control for ARM 1: `git fsck --strict` must actually reject
/// a loose object whose zlib payload is corrupt. Proves the clean verdict in
/// `stock_git_reads_a_loose_object_store_we_wrote` is a real inflate, not a
/// directory listing.
#[test]
fn fsck_actually_rejects_a_corrupted_loose_object() {
    if !have_git() {
        eprintln!("SKIP fsck negative control: `git` not on PATH");
        return;
    }
    let dir = scratch("looseneg");
    let (ok, _, err) = git(&dir, &["init", "--quiet", "."]);
    assert!(ok, "git init failed: {err}");

    let content = b"the quick brown fox jumps over the lazy dog\n".repeat(400);
    let mut body = format!("blob {}\0", content.len()).into_bytes();
    body.extend_from_slice(&content);
    let refs: Vec<&[u8]> = vec![&body];
    let mut z = compress_objects(&refs, &Config::default())
        .unwrap()
        .pop()
        .unwrap();
    let mid = z.len() / 2;
    z[mid] ^= 0x5a;

    let oid = hex(&blob_oid(&content));
    let sub = dir.join(".git/objects").join(&oid[..2]);
    std::fs::create_dir_all(&sub).unwrap();
    std::fs::write(sub.join(&oid[2..]), &z).unwrap();

    let (ok, out, err) = git(&dir, &["fsck", "--full", "--strict", "--no-progress"]);
    assert!(
        !ok || out.contains("error") || err.contains("error") || err.contains("corrupt"),
        "git fsck ACCEPTED a loose object with a corrupted zlib payload — the \
         acceptance test above proves nothing.\nstdout:{out}\nstderr:{err}"
    );
    let _ = std::fs::remove_dir_all(&dir);
}