znippy-plugin-git 0.1.1

//! **A clone must not hold the pack it is sending.** Measured, in bytes, by
//! counting every allocation the process makes while it serves one.
//!
//! # The defect this exists to keep dead
//!
//! [`EmitEntry::stored`](znippy_plugin_git::pack_walk::EmitEntry::stored) was a
//! `Vec<u8>` until 2026-08-14. `GitStore::emit_set` filled it with a `pread`
//! into a fresh `Vec::with_capacity(len)` — one syscall, one allocation, one
//! kernel→user copy **per object** — and every one of those `Vec`s stayed live
//! at once, because the whole entry set is built before `emit_pack` writes a
//! byte. On a `linux.git` clone: ~13.8 M syscalls, ~13.8 M payload allocations,
//! 6.4 GB copied and held, and a peak RSS of **2314 MB** against gitea's 122 MB
//! on the identical clone (`gunnar/.nornir/forge-bakeoff-benchmarks.md`,
//! `vs_forge_clone`). 39 `perf` captures of that clone put ~20 % of cycles in
//! the copy and the reads feeding it.
//!
//! An entry is an *address* now — 16 bytes — resolved against one read-only
//! mapping of the append-only blob file at emit time.
//!
//! # Why a counting global allocator and not `/proc/self/statm`
//!
//! This test began as an RSS measurement inside the unit suite and that
//! measurement was **blind**: run against the defective version, which really
//! does allocate and fill 5 653 270 bytes per clone, resident set size did not
//! move by a single page — the allocator was handing back arena pages the test
//! process already had. A guard that cannot tell the defect from the fix is not
//! a weak guard, it is no guard. Counting `alloc`/`dealloc` sees the bytes
//! whatever the allocator does with its arenas afterwards, and it deliberately
//! does **not** see the mapping — an `mmap` is not a heap allocation, which is
//! the whole point: those bytes are page cache the kernel already had, shared
//! with every other clone of the same repository, and reclaimable under
//! pressure. Held heap is none of those things.
//!
//! Its sibling `resolve_peak_memory.rs` does the same for the absorb path.

use std::alloc::{GlobalAlloc, Layout, System};
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};

use znippy_plugin_git::git_ops::{GitOps, GitStore};
use znippy_plugin_git::serve::{Caps, GitServe};

// ── the counting allocator ──────────────────────────────────────────────────

static LIVE: AtomicUsize = AtomicUsize::new(0);
static PEAK: AtomicUsize = AtomicUsize::new(0);

struct Counting;

impl Counting {
    fn grew(by: usize) {
        let now = LIVE.fetch_add(by, Ordering::Relaxed).saturating_add(by);
        // A plain max: the peak only ever rises, so a lost race costs at most
        // one sample and never reports a peak that did not happen.
        PEAK.fetch_max(now, Ordering::Relaxed);
    }
    /// **Saturating, and that is not pedantry.** `resolve_peak_memory.rs`'s copy
    /// of this harness subtracts plainly and panics `attempt to add with
    /// overflow` once `LIVE` wraps — which it does the moment anything frees an
    /// allocation this counter never saw (a thread's stack-adjacent bookkeeping,
    /// an allocation made before `main`). A memory *measurement* that aborts the
    /// run it is measuring reports nothing at all.
    fn shrank(by: usize) {
        let _ = LIVE.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |v| {
            Some(v.saturating_sub(by))
        });
    }
}

unsafe impl GlobalAlloc for Counting {
    unsafe fn alloc(&self, l: Layout) -> *mut u8 {
        let p = unsafe { System.alloc(l) };
        if !p.is_null() {
            Self::grew(l.size());
        }
        p
    }
    unsafe fn dealloc(&self, p: *mut u8, l: Layout) {
        Self::shrank(l.size());
        unsafe { System.dealloc(p, l) }
    }
    unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 {
        let q = unsafe { System.realloc(p, l, new) };
        if !q.is_null() {
            if new >= l.size() {
                Self::grew(new - l.size());
            } else {
                Self::shrank(l.size() - new);
            }
        }
        q
    }
    unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 {
        let p = unsafe { System.alloc_zeroed(l) };
        if !p.is_null() {
            Self::grew(l.size());
        }
        p
    }
}

#[global_allocator]
static A: Counting = Counting;

/// A writer that keeps the byte count and throws the bytes away.
///
/// The pack must not be accumulated anywhere, or this test would be measuring
/// its own sink. `emit_pack` streams, so a counter is a legitimate destination
/// — and one that allocates nothing, so it cannot pollute the measurement.
struct Sink(u64);

impl std::io::Write for Sink {
    fn write(&mut self, b: &[u8]) -> std::io::Result<usize> {
        self.0 += b.len() as u64;
        Ok(b.len())
    }
    fn flush(&mut self) -> std::io::Result<()> {
        Ok(())
    }
}

/// The biggest real pack under 32 MB on this machine.
///
/// Real git data and not a fixture, for the same reason `resolve_peak_memory`
/// gives: what is being measured is a ratio against a pack's payload, and a
/// synthetic pack's delta shape is not a repository's.
fn real_pack() -> Option<(PathBuf, Vec<u8>)> {
    let mut best: Option<(u64, PathBuf)> = None;
    for root in [
        "/home/rickard/git",
        "/home/rickard/scratch/gunnar-bench-fixtures",
    ] {
        let Ok(repos) = std::fs::read_dir(root) else {
            continue;
        };
        for repo in repos.flatten() {
            for sub in [".git/objects/pack", "objects/pack"] {
                let Ok(files) = std::fs::read_dir(repo.path().join(sub)) else {
                    continue;
                };
                for f in files.flatten() {
                    let p = f.path();
                    if p.extension().is_some_and(|e| e == "pack") {
                        let len = f.metadata().map(|m| m.len()).unwrap_or(0);
                        if len < (32 << 20) && best.as_ref().is_none_or(|(b, _)| len > *b) {
                            best = Some((len, p));
                        }
                    }
                }
            }
        }
    }
    let (_, p) = best?;
    let bytes = std::fs::read(&p).ok()?;
    Some((p, bytes))
}

fn tmpdir(tag: &str) -> PathBuf {
    let d = std::env::temp_dir().join(format!(
        "znippy-emit-peak-{tag}-{}-{}",
        std::process::id(),
        std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .map(|d| d.as_nanos())
            .unwrap_or(0)
    ));
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    d
}

/// 🔴 **Serving a whole-repository clone must not allocate the pack.**
///
/// # The bound is per OBJECT, and that is the whole claim
///
/// The first draft of this test bounded peak heap against the pack's **bytes**,
/// and that is the wrong axis: what a correct emitter still holds is one entry
/// per object — a slot, an oid, an index row — which is `O(objects)` and has
/// nothing to do with how large the objects are. Against pack bytes the same
/// correct emitter reads `0.29x` on a corpus of 848-byte objects and would read
/// `0.01x` on one of 64 KB blobs, so a ratio bound is a bound on the *corpus*.
///
/// The defect had a per-object cost too, and a much larger one: it held every
/// object's compressed payload, so its peak was `entry overhead + mean object
/// size` per object against `entry overhead` alone. That is the difference this
/// asserts, on the axis it actually lives on.
///
/// # Measured, oden 2026-08-14 (loadavg ~9 — a shared box, see LAW 7)
///
/// | | peak heap | per object | vs pack bytes |
/// |---|---:|---:|---:|
/// | extents (this change) | 7 135 852 | **246 B** | 0.29x |
/// | owned payloads (before) | 31 650 217 | **1 095 B** | 1.29x |
///
/// on `forge-year-8a-240b-12c`'s 28 893-object, 24 522 797-byte pack. **4.4x
/// less heap**, and the residual is the entry vector rather than the pack. The
/// difference — 849 B per object — is that pack's mean compressed object size,
/// which is exactly what the defect was holding.
///
/// **Seen RED** by reverting `emit_set`'s base-inside arm to
/// `EntryBytes::Owned(self.read_extent(row.offset, row.len)?)`, verbatim what
/// the line was before this change:
///
/// ```text
/// serving 28893 objects held a peak of 31650217 live heap bytes — 1095 B per
/// object against a 512 B ceiling. An entry that owns its payload holds the
/// pack; an entry that addresses it does not.
/// ```
///
/// # What it does not prove
///
/// It measures one process serving one clone of a ~24 MB pack, so it says
/// nothing directly about the 2314 MB peak of a concurrent `linux.git` clone —
/// that was measured with `perf` and `/proc` on the real server. And 246 B per
/// object is *not* nothing: at `linux.git`'s ~13.8 M objects it is still ~3.4 GB
/// of entry vector, which this change does not fix and does not claim to. The
/// payload copy is gone; the per-entry slot and oid are next.
///
/// Both figures were taken while another tenant had oden at loadavg ~9–14.
/// Allocation *counts* are not load-sensitive the way a wall clock is, so the
/// ratio stands as measured; no timing was taken, and none should be trusted
/// off a box in that state.
#[test]
fn serving_a_clone_does_not_allocate_the_pack_it_sends() {
    let Some((path, pack)) = real_pack() else {
        eprintln!("no real pack on this machine; nothing to measure");
        return;
    };

    let dir = tmpdir("clone");
    let store = GitStore::open(&dir, "rickard").expect("opening a store");
    store.put_pack(&pack).expect("pushing the corpus pack");
    store.absorb_pending().expect("absorbing it");

    let oids = store.index().oids_in_order().expect("the store's own oids");
    assert!(
        oids.len() > 500,
        "{} holds only {} objects — too few to tell a held pack from an addressed one",
        path.display(),
        oids.len()
    );
    let want: Vec<&[u8]> = oids.iter().map(Vec::as_slice).collect();

    // Warm everything that is not the emission: one throwaway clone, so the
    // measured pass allocates nothing for a lazily-built index, a mapping that
    // has not been taken yet, or a `§14` table that has not been opened. Without
    // this the peak would be measuring the store's construction.
    let mut warm = Sink(0);
    store
        .emit_pack(&want, &[], &Caps::modern(), &mut warm)
        .expect("the warm-up clone");

    // ── the measurement ──────────────────────────────────────────────────────
    let before = LIVE.load(Ordering::Acquire);
    PEAK.store(before, Ordering::Release);
    let mut sink = Sink(0);
    let stats = store
        .emit_pack(&want, &[], &Caps::modern(), &mut sink)
        .expect("serving the clone");
    let peak = PEAK.load(Ordering::Acquire).saturating_sub(before);

    assert_eq!(stats.bytes, sink.0, "the receipt must be what was written");
    assert_eq!(
        stats.objects,
        oids.len() as u64,
        "a whole-repository clone must send every object"
    );
    assert_eq!(
        stats.recompressed, 0,
        "a whole-repository clone re-deflates nothing, so every entry is a copy and every copy \
         must have come from an extent"
    );

    /// Heap an emitter may hold per object. The entry slot is 80 B, its oid
    /// 20 B plus the allocator's bucket, and the index row and ordering maps
    /// add their own — call it ~250 B measured, and bound it at twice that so
    /// an unrelated bookkeeping change is not a failure. The defect it has to
    /// exclude is a *whole compressed object* per entry, which no real corpus
    /// gets under 512 B on average.
    const PER_OBJECT_CEILING: u64 = 512;

    let per_object = peak as u64 / stats.objects;
    let ratio = peak as f64 / stats.bytes as f64;
    eprintln!(
        "{}: {} objects, {} pack bytes, peak {peak} live heap bytes — {per_object} B per object, \
         {ratio:.2}x the pack. The mapping is NOT counted: it is page cache, not heap.",
        path.display(),
        stats.objects,
        stats.bytes,
    );
    assert!(
        per_object < PER_OBJECT_CEILING,
        "serving {} objects held a peak of {peak} live heap bytes — {per_object} B per object \
         against a {PER_OBJECT_CEILING} B ceiling. An entry that owns its payload holds the pack; \
         an entry that addresses it does not.",
        stats.objects
    );

    let _ = std::fs::remove_dir_all(&dir);
}