Skip to main content

znippy_plugin_git/
store.rs

1//! The storage functions. Nothing else is permitted here.
2//!
3//! **Twelve, then thirteen.** The count moved on 2026-08-10 when
4//! [`GitOps::put_refs_cas`] landed — atomic batch compare-and-swap, the
5//! `git push --atomic` primitive, which was previously *unrepresentable*:
6//! `put_refs` is a batch without CAS, `update_ref` is CAS without a batch, and
7//! `--atomic` is defined as both at once. It is a storage concern and it belongs
8//! here. What does **not** belong here is the *reading* contract — decoded
9//! reads, `HEAD`, negotiation, pack emission — which is [`crate::serve`].
10//!
11//! **ZNIPPY-GIT APACHE ARROW IPC IS LAW.** Storage is Arrow IPC in a znippy
12//! archive. Not a packfile directory, not loose objects, not `gix-odb`, not a
13//! filesystem layout borrowed from somebody else's model. gix is a codec for
14//! framing bytes onto the wire and nothing more: it never owns a handle, never
15//! decides where a byte lives, and never appears in a signature below.
16//!
17//! ---
18//!
19//! The twelve are the methods of [`GitOps`], implemented once, for
20//! [`GitStore`] — the handle and the trait live in [`crate::git_ops`], and
21//! everything each method reaches for was already built and already measured:
22//!
23//! | this file calls | which is | measured in |
24//! |---|---|---|
25//! | `PushPath` → `SafeWriter` | blob fsync, then the journal row | `archive_write` |
26//! | `ObjectReadStack` | stree → Arrow → redb tail | `read_stack` |
27//! | `RefLog` | one Arrow IPC frame per push | `refs` / `pushlog` |
28//! | `pack_walk` + `resolve` | the split and the oids | `pack_walk` / `resolve` |
29//! | `NewGeneration` → `compact_archive` | base znippy's compaction | `gc` |
30//!
31//! No method below invents storage, an index, a durability contract or a
32//! concurrency mechanism. If one looks like it does, that is the bug.
33
34use std::path::Path;
35
36use anyhow::{anyhow, bail, Context, Result};
37use znippy_common::ReservedSection;
38
39use crate::gc::GcReport;
40use crate::git_ops::{lookup_path, GitOps, GitStore, LookupPath, RefRow, Stored, TxId};
41use crate::index_layout::ObjectIndex;
42use crate::pack_walk::walk;
43use crate::refs::RefUpdate;
44use git_storage_trait::{Observed, RefCas, RefRejection, RefTarget};
45
46/// An object id, borrowed.
47pub type Oid<'a> = &'a [u8];
48
49/// A byte range inside the archive: `(offset, len)`.
50pub type Extent = (u64, u64);
51
52impl<S: ObjectIndex + 'static> GitOps for GitStore<S> {
53    // ── STORE ───────────────────────────────────────────────────────────────
54
55    /// One push, and the order inside it is the contract.
56    ///
57    /// 1. the pack's bytes, durable — [`put_pack`](GitOps::put_pack)
58    /// 2. **then** the refs that point into them — [`put_refs`](GitOps::put_refs)
59    ///
60    /// Not interchangeable, and it is the same argument `SafeWriter` makes one
61    /// level down about the blob and its journal row: a crash between the two
62    /// leaves objects nobody points at, which a GC reclaims. The reverse order
63    /// leaves a **ref pointing at objects that are not there**, which is a
64    /// corrupt repository that no later pass can repair.
65    fn put(&self, pack: &[u8], refs: &[RefUpdate]) -> Result<TxId> {
66        let mut tx = if pack.is_empty() {
67            TxId::default()
68        } else {
69            self.put_pack(pack)?
70        };
71        if !refs.is_empty() {
72            tx.push_seq = self.put_refs(refs)?.push_seq;
73        }
74        Ok(tx)
75    }
76
77    /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
78    ///
79    /// **The ack path.** Walk, check, store verbatim, ack, then queue the index.
80    ///
81    /// 1. **Walk every entry** ([`crate::pack_walk`]). Not optional and not an
82    ///    extra pass: a pack entry has no length field, so finding the entry
83    ///    boundaries *is* how the pack gets split at all.
84    /// 2. **The closure check, free from that walk** (§13.7). Every `OFS_DELTA`
85    ///    base must land on an entry boundary in this pack — answered from the
86    ///    boundary set the walk just produced, consulting no index. Only a
87    ///    `REF_DELTA` base that is *outside* the pack is looked up, and that one
88    ///    reads `objects.oid` and nothing else, exactly as §13's table says
89    ///    receive-pack does. A pack that fails is refused **before** a byte is
90    ///    stored.
91    /// 3. **The bytes, verbatim.** `SafeWriter::append` writes the caller's
92    ///    buffer at its own address — no re-compression, no re-encoding, no
93    ///    re-framing, not one copy in userspace. The client's own deflate and
94    ///    every delta chain survive, which is what makes a later clone a
95    ///    byte-range copy instead of a re-pack (§14).
96    /// 4. **Durable before returning**: the blob is fsynced, *then* the journal
97    ///    row that references it, then that is fsynced. znippy's `hot.rs`
98    ///    ordering, so a crash between the two leaves orphan bytes nobody points
99    ///    at rather than a dangling reference.
100    /// 5. **Then** the index job goes on the channel and this returns. Nothing
101    ///    that could be done later is done here: no oid is computed, no table is
102    ///    built, no chain is resolved.
103    fn put_pack(&self, bytes: &[u8]) -> Result<TxId> {
104        if bytes.is_empty() {
105            bail!("an empty push is not a pack");
106        }
107        // (1) + (2) — one walk, and the check falls out of it.
108        let walked = walk(bytes, self.hash_kind().oid_len())
109            .context("the pushed pack could not be split")?;
110        self.external_bases_exist(bytes, &walked)?;
111
112        // (3) + (4) — verbatim, blob fsync, journal row, journal fsync.
113        // (5) — and the pack-level index job onto the account's channel.
114        let (pack_id, extent) = self
115            .push_path()
116            .push_pack(self.account(), bytes)
117            .context("storing the pack verbatim")?;
118
119        // The object-level index work: 24 bytes of extent, queued, drained by
120        // `absorb_pending` off this path. Until it runs the pack is un-indexed
121        // and every read falls back rather than answering absent (§13.12).
122        self.queue(pack_id, extent)?;
123
124        Ok(TxId {
125            pack_id: Some(pack_id),
126            extent: Some(extent),
127            push_seq: None,
128        })
129    }
130
131    /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
132    ///
133    /// One ref transaction: one Arrow IPC frame, fsynced.
134    ///
135    /// The frame boundary *is* the transaction (see [`crate::pushlog`]) — three
136    /// branches in one push are three rows in one batch, and they either all land
137    /// or none do. There is no lock file and no second journal.
138    ///
139    /// **Every target is checked against the index first.** A ref that points at
140    /// an object the repository does not have is a corrupt repository, and it is
141    /// refused here rather than written and discovered later. A deletion has no
142    /// target and is not checked.
143    fn put_refs(&self, updates: &[RefUpdate]) -> Result<TxId> {
144        if updates.is_empty() {
145            bail!("an empty ref update is not a transaction");
146        }
147        for u in updates {
148            for (what, oid_hex) in [("target", &u.target), ("peeled", &u.peeled)] {
149                let Some(hex_oid) = oid_hex else { continue };
150                let raw = hex::decode(hex_oid)
151                    .map_err(|e| anyhow!("{}'s {what} `{hex_oid}` is not hex: {e}", u.name))?;
152                if !self.has(&raw)? {
153                    bail!(
154                        "{} would point at {hex_oid}, which this repository does not have — the \
155                         ref update is refused rather than left dangling",
156                        u.name
157                    );
158                }
159            }
160        }
161        let push_seq = self.ref_log().push(updates)?;
162        Ok(TxId {
163            pack_id: None,
164            extent: None,
165            push_seq: Some(push_seq),
166        })
167    }
168
169    // ── READ ────────────────────────────────────────────────────────────────
170
171    /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
172    ///
173    /// The object's **stored** bytes: one index lookup and one `pread` of the
174    /// extent, with no decode of any kind in between.
175    ///
176    /// What comes back is the pack entry exactly as the client sent it, which for
177    /// a delta is a delta. [`Stored::obj_type`] says which, so the bytes cannot
178    /// be mistaken for the object's content — §14 makes the verbatim bytes the
179    /// truth and the resolved object a derived cache, and that cache does not
180    /// exist yet, so handing back delta bytes labelled "the object" would be the
181    /// one thing this crate refuses everywhere: a wrong answer where an honest
182    /// one was available.
183    fn get(&self, oid: Oid<'_>) -> Result<Option<Stored>> {
184        let Some(row) = self.lookup_one(oid)? else {
185            return Ok(None);
186        };
187        let bytes = self.read_extent(row.offset, row.len)?;
188        Ok(Some(Stored {
189            obj_type: row.obj_type,
190            uncompressed_size: row.uncompressed_size,
191            extent: (row.offset, row.len),
192            bytes,
193        }))
194    }
195
196    /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
197    ///
198    /// The negotiation call: `have` sends up to a thousand of these and **most of
199    /// them miss**. It reads `objects.oid` and no other column.
200    ///
201    /// It takes the **serial** path, not `lookup_batch` with one element: that is
202    /// 1.7× slower (818 ns against 482), and [`lookup_path`] is where that
203    /// decision is written down.
204    ///
205    /// # Why this returns `Result<bool>` and not `bool`
206    ///
207    /// The signature changed deliberately. A store with a pack whose bytes are
208    /// durable but whose objects are not indexed yet cannot answer "no" — the
209    /// object may be in that pack. So it absorbs the pack first and, if that
210    /// fails, **says so**. A `bool` could only have lied, and a wrong "absent"
211    /// during negotiation makes a client send nothing and lose data.
212    fn has(&self, oid: Oid<'_>) -> Result<bool> {
213        Ok(self.lookup_one(oid)?.is_some())
214    }
215
216    /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
217    ///
218    /// The object's **post-resolution** size — what it inflates to once its delta
219    /// chain is applied. That is the fact git's own `.idx` and `.rev` together
220    /// cannot answer, and it is why the quota gate is index-only here.
221    fn size(&self, oid: Oid<'_>) -> Result<Option<u64>> {
222        Ok(self.lookup_one(oid)?.map(|r| r.uncompressed_size))
223    }
224
225    /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
226    ///
227    /// **The wire path**, and batch by construction: for each oid the two facts a
228    /// clone needs — `offset` and `len` — and no others, so a byte-range copy out
229    /// of the verbatim pack can start immediately.
230    ///
231    /// `out[i]` answers `oids[i]`. One oid takes the serial path
232    /// ([`lookup_path`]); the batch path saturates at
233    /// [`BATCH_SATURATES_AT`](crate::git_ops::BATCH_SATURATES_AT) oids, so a
234    /// larger batch is passed through whole rather than split — splitting would
235    /// cost a pass and buy nothing.
236    fn extents(&self, oids: &[Oid<'_>]) -> Result<Vec<Option<Extent>>> {
237        match lookup_path(oids.len()) {
238            LookupPath::Serial => Ok(vec![self.lookup_one(oids[0])?.map(|r| (r.offset, r.len))]),
239            LookupPath::Batch => {
240                if self.unindexed_packs() > 0 {
241                    self.absorb_pending()?;
242                }
243                Ok(self.index().extents_batch(oids))
244            }
245        }
246    }
247
248    // ── REFS ────────────────────────────────────────────────────────────────
249
250    /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
251    ///
252    /// The whole ref namespace, tags peeled — the ref advertisement, and
253    /// `ls-refs` after a prefix filter. Name-sorted, because the log folds into a
254    /// `BTreeMap` and `ls-refs` wants a prefix scan.
255    ///
256    /// The cost is a scan of a structure sized by **pushes**, not by repository
257    /// size.
258    ///
259    /// # `HEAD` is not in here, and that is the contract
260    ///
261    /// **Changed 2026-08-10, and it is a fix rather than a preference.** The
262    /// other backend's `iter()` *"walks `refs/` (loose and packed) and
263    /// deliberately excludes the pseudo-refs such as `HEAD`, which is exactly the
264    /// contract every other backend honours"*. This one did not, so the two arms
265    /// disagreed about whether `HEAD` is a row — and the conformance suite could
266    /// not see it, because it never exercised `HEAD` at all.
267    ///
268    /// The reason `HEAD` cannot be a row is a type constraint, not taste: a name
269    /// type that admits `HEAD` also admits `MERGE_HEAD` and `FETCH_HEAD`, so a
270    /// row stream carrying pseudo-refs means either widening the name type or
271    /// filtering at every consumer. It gets an accessor pair instead —
272    /// [`GitServe::head`](crate::serve::GitServe::head) and
273    /// [`set_head`](crate::serve::GitServe::set_head) — and this is the one
274    /// filter, in the one place.
275    ///
276    /// Nothing else changes: the ref log still *stores* `HEAD` (a push writes it
277    /// like any other row), `live_set` still reaches it because it reads
278    /// `ref_state` directly, and the filter is one `!=`.
279    fn refs(&self) -> Result<Vec<RefRow>> {
280        let mut out = Vec::new();
281        for (name, state) in self
282            .ref_state()?
283            .into_iter()
284            .filter(|(name, _)| name != crate::serve::HEAD)
285        {
286            let decode = |h: &Option<String>| -> Result<Option<Vec<u8>>> {
287                match h {
288                    Some(h) => {
289                        Ok(Some(hex::decode(h).map_err(|e| {
290                            anyhow!("ref {name}: `{h}` is not a hex oid: {e}")
291                        })?))
292                    }
293                    None => Ok(None),
294                }
295            };
296            out.push(RefRow {
297                oid: decode(&state.target)?,
298                peeled: decode(&state.peeled)?,
299                symref_target: state.symref_target.clone(),
300                name,
301            });
302        }
303        Ok(out)
304    }
305
306    /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
307    ///
308    /// **Compare-and-swap**, in the same log a push writes its refs to — one
309    /// mechanism, not a second one for single updates.
310    ///
311    /// `old` is what the caller believes the ref is: `None` means *it must not
312    /// exist* (a create), `Some(oid)` means *it must be exactly this*. `new` of
313    /// `None` deletes. A mismatch names both values and **writes nothing**.
314    ///
315    /// The read-compare-append is serialised on the store's ref gate. Without it
316    /// two CAS calls could both read the old value and both append, and the
317    /// second would silently overwrite an update it had compared against
318    /// successfully — the classic lost update, and the only thing a CAS is for.
319    fn update_ref(&self, name: &str, old: Option<Oid<'_>>, new: Option<Oid<'_>>) -> Result<TxId> {
320        let _gate = self
321            .ref_gate()
322            .lock()
323            .map_err(|_| anyhow!("the ref gate is poisoned"))?;
324
325        let current = self.ref_state()?;
326        let actual: Option<Vec<u8>> = match current.get(name).and_then(|s| s.target.as_deref()) {
327            Some(h) => Some(
328                hex::decode(h)
329                    .map_err(|e| anyhow!("ref {name} holds `{h}`, not a hex oid: {e}"))?,
330            ),
331            None => None,
332        };
333        if actual.as_deref() != old {
334            // ── TYPED, like `put_refs_cas` three hundred lines down ─────────
335            //
336            // This raised a bare `bail!` while its own sibling raised
337            // `RefRejection::Cas`, so a caller had a string and no way to tell
338            // a lost race from a broken disk. `gunnar-wire`'s receive-pack
339            // says so at the call site: "A LOST COMPARE-AND-SWAP AND A BROKEN
340            // DISK ARE THE SAME STRING HERE … Nothing may recover the
341            // distinction by matching on this text; the fix is a typed
342            // rejection on the contract."
343            //
344            // The contract already had one. It was this method that did not
345            // use it. The `Display` text is unchanged — `RefRejection::Cas`
346            // renders the same sentence — so nothing that reads the message
347            // moves, and a caller that downcasts now gets `name`, `expected`
348            // and `actual` as values instead of parsing prose.
349            return Err(anyhow::Error::new(RefRejection::Cas {
350                name: name.to_string(),
351                expected: match old {
352                    None => Observed::Nothing,
353                    Some(o) => Observed::oid(o),
354                },
355                actual: match actual.as_deref() {
356                    None => Observed::Nothing,
357                    Some(a) => Observed::oid(a),
358                },
359            }));
360        }
361
362        let update = match new {
363            Some(n) => RefUpdate::set(name, hex::encode(n)),
364            None => RefUpdate::delete(name),
365        };
366        // Through `put_refs`, so the existence check and the log format are the
367        // same code a push uses (LAW 5) — a CAS cannot create a dangling ref that
368        // a push would have been refused for.
369        if new.is_some() {
370            self.put_refs(&[update])
371        } else {
372            let push_seq = self.ref_log().push(&[update])?;
373            Ok(TxId {
374                pack_id: None,
375                extent: None,
376                push_seq: Some(push_seq),
377            })
378        }
379    }
380
381    /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
382    ///
383    /// **`git push --atomic`: every edit or none**, in one Arrow IPC frame.
384    ///
385    /// The two halves come from different places and neither is reimplemented:
386    ///
387    /// * **the batch atomicity is the frame.** [`crate::pushlog`] makes the frame
388    ///   boundary the transaction — three branches in one push are three rows in
389    ///   one batch and they either all land or none do. There is no lock file and
390    ///   no second journal, so there is no partial-apply state to recover from;
391    /// * **the compare half is checked here, against one snapshot**, before a
392    ///   single [`RefUpdate`] is built.
393    ///
394    /// It goes out through [`put_refs`](GitOps::put_refs) rather than straight to
395    /// the log, so the dangling-target refusal and the frame format are the same
396    /// code a push uses (LAW 5): an atomic batch cannot create a ref pointing at
397    /// an object the repository does not have, which a second writer here would
398    /// eventually have allowed.
399    ///
400    /// # `S-023`, and why the check is a whole pass of its own
401    ///
402    /// Every expectation is evaluated **before any of them is applied**, and the
403    /// loop deliberately does not fuse with the one that builds the updates. The
404    /// defect it exists for is gix's: a backend that short-circuits an edge whose
405    /// new value already equals the current one never evaluates the expectation
406    /// the caller wrote, so an `old: None` — *must not exist* — becomes a silent
407    /// success and *"exactly one creator wins"* stops being true. This log has no
408    /// such short-circuit, but the ordering is what makes that irrelevant rather
409    /// than lucky, and a future optimisation that adds one cannot break it from
410    /// here.
411    ///
412    /// # What this arm cannot raise, stated rather than hidden
413    ///
414    /// **[`RefRejection::Locked`] never comes out of this backend.** The
415    /// read-compare-append is serialised on the store's ref gate, which a second
416    /// writer *blocks* on rather than failing against — so contention here is a
417    /// wait, never a rejection. The variant is in the contract because the gix
418    /// arm, which takes real per-ref lock files, raises it. A poisoned gate stays
419    /// an ordinary error: it is a fault, not the transient thing a caller retries.
420    fn put_refs_cas(&self, edits: &[RefCas<'_>]) -> Result<TxId> {
421        // Not an error. A deletions-free push that had nothing to apply calls
422        // this, and `put_refs` refuses an empty batch — rightly, since an empty
423        // ref update is not a transaction. An empty *atomic* batch is a no-op
424        // that succeeded, and saying so here is what keeps the special case out
425        // of every call site.
426        if edits.is_empty() {
427            return Ok(TxId::default());
428        }
429
430        let _gate = self
431            .ref_gate()
432            .lock()
433            .map_err(|_| anyhow!("the ref gate is poisoned"))?;
434
435        // ONE read of the namespace for the whole check, so every expectation is
436        // compared against one instant rather than against a namespace that may
437        // move between them.
438        let current = self.ref_state()?;
439        let observed = |name: &str| -> Observed {
440            match current.get(name) {
441                None => Observed::Nothing,
442                Some(s) => match (&s.symref_target, &s.target) {
443                    (Some(points_to), _) => {
444                        Observed::Value(RefTarget::Symbolic(points_to.clone()))
445                    }
446                    (None, Some(h)) => match hex::decode(h) {
447                        Ok(raw) => Observed::Value(RefTarget::Object(raw)),
448                        // Not dropped into `Nothing`: a ref that exists and
449                        // cannot be read is not a ref that was absent, and
450                        // reporting it as absent tells the pushing client its
451                        // create is free when it is not.
452                        Err(e) => Observed::Unreadable(format!("`{h}` is not a hex oid: {e}")),
453                    },
454                    (None, None) => Observed::Unreadable(
455                        "the ref log holds a row naming neither an object nor another ref".into(),
456                    ),
457                },
458            }
459        };
460
461        // S-023: the whole check, before anything is applied.
462        for e in edits {
463            let expected = match e.old {
464                None => Observed::Nothing,
465                Some(o) => Observed::oid(o),
466            };
467            let actual = observed(&e.name);
468            if actual != expected {
469                return Err(anyhow::Error::new(RefRejection::Cas {
470                    name: e.name.clone(),
471                    expected,
472                    actual,
473                }));
474            }
475        }
476
477        let updates: Vec<RefUpdate> = edits
478            .iter()
479            .map(|e| match e.new {
480                Some(n) => RefUpdate::set(e.name.clone(), hex::encode(n)),
481                None => RefUpdate::delete(e.name.clone()),
482            })
483            .collect();
484        self.put_refs(&updates)
485    }
486
487    // ── GRAPH ───────────────────────────────────────────────────────────────
488
489    /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
490    ///
491    /// **Selection: `want` minus `have`, as an `andnot` of roaring bitmaps.**
492    ///
493    /// Every returned oid is an object — commit, tree *and* blob — because the
494    /// bitmaps are built by [`crate::reach::build_reach`], which walks the trees.
495    /// A `want` that names a commit contributes that commit's whole closure; a
496    /// `want` that names a tag or a blob contributes itself.
497    ///
498    /// A `have` this repository does not know contributes nothing: the safe
499    /// direction is to send more, never less.
500    ///
501    /// The bitmaps are over the **store's own ordinal space**, rebuilt by the same
502    /// fold that rebuilds them, never over the Arrow projection's ordinals — an
503    /// [`crate::index_layout::IndexRow::ordinal`] is a row address within one
504    /// projection generation, so a bitmap over those would address different
505    /// objects after any rebuild, silently.
506    ///
507    /// # The per-oid `Vec` is paid HERE and nowhere else on the serving path
508    ///
509    /// The answer is computed flat (see
510    /// [`GitStore::reachable_raw`](crate::git_ops::GitStore::reachable_raw));
511    /// this splits it back out because the eleven say `Vec<Vec<u8>>` and this
512    /// method's callers are maintenance ones — a GC live set, a conformance
513    /// harness — not the serving path. `crate::serve`'s `select` and
514    /// `emit_pack` take the flat form directly.
515    fn reachable(&self, want: &[Oid<'_>], have: &[Oid<'_>]) -> Result<Vec<Vec<u8>>> {
516        Ok(self
517            .reachable_raw(want, have)?
518            .iter()
519            .map(<[u8]>::to_vec)
520            .collect())
521    }
522
523    // ── MAINT ───────────────────────────────────────────────────────────────
524
525    /// Garbage collection, in the order §13.20 fixes:
526    ///
527    /// 1. **compute reachability** — every object reachable from every ref, over
528    ///    the same bitmaps [`reachable`](GitOps::reachable) uses;
529    /// 2. **drop the dead rows from the index** — the one operation that is not
530    ///    append-only, and the projection is rebuilt inside it;
531    /// 3. **then** base znippy's compaction, through [`crate::gc::NewGeneration`]
532    ///    (link, compact, verify, rename, unlink last).
533    ///
534    /// That order is why **base znippy needs no new method**: by the time
535    /// `compact_archive` runs, "live" already means what git means.
536    ///
537    /// A repository with no ref pointing at anything is refused rather than
538    /// emptied.
539    ///
540    /// **Appended, not reworded: step 1b, the journal.** Between the live set and
541    /// the drop there is now one more durable act. §13.12's `indexed` bit is
542    /// derived on open as *extent in the journal, rows not in the index*, and
543    /// dropping **every** row of a pack produces exactly that state — so before
544    /// this existed the next open re-queued the pack and every object this GC had
545    /// just decided was dead came back.
546    /// [`GitStore::retire_dead_packs`](crate::git_ops::GitStore::retire_dead_packs)
547    /// appends a tombstone naming each all-dead pack, and it runs **before**
548    /// `drop_dead_rows` on purpose: killed before it, the rows are still there
549    /// and the GC simply did not happen; killed after it, the pack can never be
550    /// re-queued whether the rows went or not. A partly dead pack keeps rows and
551    /// is never tombstoned, so nothing about it changes.
552    ///
553    /// Step 2 also refolds what the two tables feed — the commit graph, the tree
554    /// payloads, the ordinal space, the bitmaps — because a derivation that still
555    /// names a dropped oid is wrong rather than stale. That is inside
556    /// `drop_dead_rows`, so any caller of it gets it.
557    fn gc(&self) -> Result<GcReport> {
558        self.absorb_pending()?;
559        let live = self.live_set()?;
560        let retired = self.retire_dead_packs(&live)?;
561        let before = self.index().len() as u64;
562        let dropped = self.drop_dead_rows(&live)?;
563        let after = self.index().len() as u64;
564        if before.saturating_sub(after) != dropped || after > before {
565            bail!(
566                "the index dropped {dropped} rows but went from {before} to {after} — refusing to \
567                 compact an archive whose index does not agree with itself"
568            );
569        }
570        let mut report = self.gc_arm().run(self.archive_path()).with_context(|| {
571            format!(
572                "compacting {} after dropping {dropped} dead rows",
573                self.archive_path().display()
574            )
575        })?;
576        report.retired_packs = retired.len() as u64;
577        Ok(report)
578    }
579}
580
581impl<S: ObjectIndex + 'static> GitStore<S> {
582    /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
583    ///
584    /// Fold the live logs into the reserved Arrow sections an archive carries:
585    /// `__gunnar_refs__`, `__gunnar_graph__`, `__gunnar_reach__`.
586    ///
587    /// **The twelfth method, and the one that is NOT on [`GitOps`]:** it returns
588    /// `znippy_common::ReservedSection` (Arrow `RecordBatch` payloads), which a
589    /// gix backend has no analog for — so it is inherent on the concrete store,
590    /// which is how gunnar already calls it (never through the trait).
591    ///
592    /// It absorbs first, because a section built while a pack is still
593    /// un-indexed would be **silently incomplete**, and a silently incomplete
594    /// index is worse than none (the same argument [`crate::lib`] makes about not
595    /// wiring `GitIndexBuilder` into the CLI). Every generation number in the
596    /// graph section is recomputed by that fold.
597    ///
598    /// **…and then it writes the archive.** Until 2026-08-08 it only *returned*
599    /// the sections and nothing on any path created
600    /// [`archive_path`](GitStore::archive_path) at all — so `gc()`'s last step
601    /// died on `stat repository.znippy: No such file or directory`, having
602    /// already done its first four. Both [`Gc`](crate::gc::Gc) implementations
603    /// compact an archive that exists; **this** is what makes one exist.
604    /// [`seal_generation_zero`](crate::archive_write::seal_generation_zero) has
605    /// the layout argument.
606    ///
607    /// The sections are still returned, and they are the same values that were
608    /// sealed rather than a second derivation of them — `ReservedSection` is
609    /// `Clone` for exactly that reason.
610    pub fn seal(&self) -> Result<Vec<ReservedSection>> {
611        self.absorb_pending()?;
612        self.index().rebuild()?;
613        let sections = self.reserved_sections()?;
614        self.seal_archive(sections.clone()).with_context(|| {
615            format!("sealing generation 0 at {}", self.archive_path().display())
616        })?;
617        Ok(sections)
618    }
619}
620
621/// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
622///
623/// Compact `src` into a **named** destination, leaving `src` untouched.
624///
625/// The file-level primitive behind [`GitOps::gc`]'s third step, for a caller that
626/// wants to choose the name rather than take
627/// [`crate::gc::next_generation`]'s. Three existing calls and no fourth
628/// mechanism:
629///
630/// 1. `hard_link(src, dst)` — a second name for the same inode. **Not one byte is
631///    copied**, and `src` keeps pointing at the original for the whole run.
632/// 2. `compact_archive(dst)` — base znippy's own compaction, verbatim, against
633///    the new name.
634/// 3. read every entry back through the ordinary reader — the same verification
635///    [`crate::gc::NewGeneration`] gates on, so a `dst` that does not read back
636///    is removed and the error says so.
637///
638/// `dst` must not exist. Silently compacting over a file is how a generation gets
639/// lost.
640pub fn compact(src: &Path, dst: &Path) -> Result<()> {
641    if dst.exists() {
642        bail!(
643            "{} already exists — refusing to compact over it",
644            dst.display()
645        );
646    }
647    std::fs::hard_link(src, dst).with_context(|| {
648        format!(
649            "hard-linking {} to {} — compaction runs against a second name for the same inode, \
650             which needs both on one filesystem",
651            src.display(),
652            dst.display()
653        )
654    })?;
655    if let Err(e) = znippy_common::compact_archive(dst) {
656        let _ = std::fs::remove_file(dst);
657        return Err(e.context(format!(
658            "compacting {} into {}",
659            src.display(),
660            dst.display()
661        )));
662    }
663    if let Err(e) = crate::gc::read_back_every_entry(dst) {
664        let _ = std::fs::remove_file(dst);
665        return Err(e.context(format!(
666            "{} did not read back after compaction — it was removed and {} is untouched",
667            dst.display(),
668            src.display()
669        )));
670    }
671    Ok(())
672}
673
674#[cfg(test)]
675pub(crate) mod tests {
676    use super::*;
677    use crate::git_ops::BATCH_SATURATES_AT;
678    use crate::index_layout::ObjType;
679    use crate::object::{canonical, GitHashKind, GitObjectKind};
680    use crate::resolve::{resolve, NoBases};
681    use std::path::PathBuf;
682    use std::time::Instant;
683
684    /// A store directory for one test.
685    ///
686    /// It returns a `PathBuf` and not a `tempfile::TempDir`, so **nothing
687    /// deletes it when the test ends** — 82 call sites take it as a path and
688    /// several deliberately outlive a store to reopen it, which is what the
689    /// borrowed form would forbid. The directories are therefore swept on the
690    /// way in instead of dropped on the way out; see [`sweep_dead_runs`].
691    pub(crate) fn tmpdir(tag: &str) -> PathBuf {
692        sweep_dead_runs();
693        let d = std::env::temp_dir().join(format!(
694            "znippy-git-store-{tag}-{}-{}",
695            std::process::id(),
696            std::time::SystemTime::now()
697                .duration_since(std::time::UNIX_EPOCH)
698                .map(|d| d.as_nanos())
699                .unwrap_or(0)
700        ));
701        std::fs::create_dir_all(&d).unwrap();
702        d
703    }
704
705    /// **Delete the store directories left behind by test runs that are over.**
706    ///
707    /// MEASURED on oden 2026-08-11: **1 088 of these directories, 96 GB**, and
708    /// `std::env::temp_dir()` on that box is `/tmp`, which is a `tmpfs` — so
709    /// that was 96 GB of *RAM* held by test runs that had exited days earlier,
710    /// on the shared machine every performance figure in this crate is taken
711    /// on. Each one is a whole store, and one with an exploded table is ~180 MB.
712    ///
713    /// Two conditions, both required, because the only way this can do harm is
714    /// by deleting a directory a running test still wants:
715    ///
716    /// 1. **The pid in the name is not a live process.** The name has carried
717    ///    the pid since it was written; nothing consulted it until now.
718    /// 2. **The directory has not been touched for an hour.** Redundant against
719    ///    a correct pid check, and there precisely because pids are reused: a
720    ///    fresh run that inherited a dead run's pid has a fresh mtime.
721    ///
722    /// Once per process, not once per call: 82 call sites would otherwise scan
723    /// the whole of `/tmp` 82 times.
724    fn sweep_dead_runs() {
725        static ONCE: std::sync::Once = std::sync::Once::new();
726        ONCE.call_once(|| {
727            let Ok(entries) = std::fs::read_dir(std::env::temp_dir()) else {
728                return;
729            };
730            for e in entries.flatten() {
731                let name = e.file_name();
732                let Some(rest) = name
733                    .to_str()
734                    .and_then(|n| n.strip_prefix("znippy-git-store-"))
735                else {
736                    continue;
737                };
738                // `<tag>-<pid>-<nanos>`, and a tag may contain `-`, so the pid
739                // is the second field from the right.
740                let Some(pid) = rest.rsplit('-').nth(1).and_then(|p| p.parse::<u32>().ok()) else {
741                    continue;
742                };
743                if pid == std::process::id() || Path::new(&format!("/proc/{pid}")).exists() {
744                    continue;
745                }
746                let stale = e
747                    .metadata()
748                    .and_then(|m| m.modified())
749                    .ok()
750                    .and_then(|t| t.elapsed().ok())
751                    .is_some_and(|age| age > std::time::Duration::from_secs(3600));
752                if stale {
753                    let _ = std::fs::remove_dir_all(e.path());
754                }
755            }
756        });
757    }
758
759    /// A real pack from this machine, with its objects already resolved so a test
760    /// knows what oids to ask for.
761    pub(crate) fn real_pack() -> (Vec<u8>, Vec<crate::resolve::Resolved>) {
762        let root = Path::new("/home/rickard/git");
763        for repo in std::fs::read_dir(root).unwrap().flatten() {
764            let dir = repo.path().join(".git/objects/pack");
765            let Ok(files) = std::fs::read_dir(&dir) else {
766                continue;
767            };
768            for f in files.flatten() {
769                let p = f.path();
770                if p.extension().is_some_and(|e| e == "pack")
771                    && f.metadata().map(|m| m.len() < 8 << 20).unwrap_or(false)
772                {
773                    let bytes = std::fs::read(&p).unwrap();
774                    // Thin packs cannot be resolved standalone (§14); skip them.
775                    if let Ok(rows) = resolve(&bytes, GitHashKind::Sha1, 0, &NoBases) {
776                        if rows.len() > 100 {
777                            return (bytes, rows);
778                        }
779                    }
780                }
781            }
782        }
783        panic!("no self-contained real pack under /home/rickard/git");
784    }
785
786    /// A tiny pack built here, so a test can push something that is not tied to
787    /// this machine. One blob, whole.
788    pub(crate) fn one_blob_pack(body: &[u8]) -> (Vec<u8>, Vec<u8>) {
789        use std::io::Write;
790        let mut pack = b"PACK".to_vec();
791        pack.extend_from_slice(&2u32.to_be_bytes());
792        pack.extend_from_slice(&1u32.to_be_bytes());
793        let mut size = body.len() as u64;
794        let mut header = vec![(3u8 << 4) | (size as u8 & 0x0f)];
795        size >>= 4;
796        while size > 0 {
797            let last = header.len() - 1;
798            header[last] |= 0x80;
799            header.push((size & 0x7f) as u8);
800            size >>= 7;
801        }
802        pack.extend_from_slice(&header);
803        let mut e = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
804        e.write_all(body).unwrap();
805        pack.extend_from_slice(&e.finish().unwrap());
806        pack.extend_from_slice(&[0u8; 20]);
807        let oid = GitHashKind::Sha1.oid_of(&canonical(GitObjectKind::Blob, body));
808        (pack, oid)
809    }
810
811    /// **The bytes come back byte for byte.** A real repository's pack is pushed,
812    /// absorbed, and then every object is fetched by oid and compared against the
813    /// exact slice of the pushed bytes it came from.
814    ///
815    /// This is the verbatim contract asserted as applied output — on the file on
816    /// disk, not on what `put` returned. A store that re-compressed, re-framed or
817    /// re-encoded anything cannot pass it.
818    ///
819    /// Seen RED by making `put_pack` store `&bytes[..bytes.len() - 1]`: "the
820    /// extent covers the whole pack — left: 5653301, right: 5653302". One byte
821    /// short of verbatim and the extent no longer covers the push.
822    #[test]
823    fn every_stored_object_reads_back_as_the_exact_bytes_that_were_pushed() {
824        let dir = tmpdir("verbatim");
825        let store = GitStore::open(&dir, "rickard").unwrap();
826        let (pack, rows) = real_pack();
827
828        // The drain is parked in front of its absorb for as long as this lives,
829        // so "queued, not run" below is a **state** and not a timing window. It
830        // used to be neither: `absorb_pending() == 1` raced the drain for the
831        // gate and lost whenever the drain got there first, which is a wrong
832        // expectation rather than a wrong store — both callers end in the same
833        // rows and the count only says which one paid.
834        let gate = store.hold_absorb_gate();
835        let tx = store.put(&pack, &[]).unwrap();
836        let (offset, len) = tx.extent.expect("a pack push records its extent");
837        assert_eq!(len, pack.len() as u64, "the extent covers the whole pack");
838
839        // The file on disk holds the pushed bytes, unchanged.
840        let on_disk = std::fs::read(store.blobs_path()).unwrap();
841        assert_eq!(
842            &on_disk[offset as usize..(offset + len) as usize],
843            &pack[..],
844            "the archive does not hold the pack verbatim"
845        );
846
847        assert_eq!(
848            store.unindexed_packs(),
849            1,
850            "the index job is queued, not run"
851        );
852        assert_eq!(
853            store.object_count(),
854            0,
855            "a row landed while the gate was held"
856        );
857        drop(gate);
858        store.wait_indexed();
859        assert_eq!(
860            store.absorb_pending().unwrap(),
861            0,
862            "the drain left index work"
863        );
864        assert_eq!(store.object_count(), rows.len(), "every object is indexed");
865
866        for (i, r) in rows.iter().enumerate() {
867            let got = store
868                .get(&r.oid)
869                .unwrap()
870                .unwrap_or_else(|| panic!("object {i} {} is missing", hex::encode(&r.oid)));
871            let from = (offset + r.offset) as usize;
872            assert_eq!(
873                got.bytes,
874                &pack[from..from + r.len as usize],
875                "object {i} {}: the stored bytes are not the pushed bytes",
876                hex::encode(&r.oid)
877            );
878            assert_eq!(got.extent, (offset + r.offset, r.len));
879            assert_eq!(got.uncompressed_size, r.uncompressed_size);
880            assert_eq!(got.obj_type, r.stored_type);
881            assert_eq!(store.size(&r.oid).unwrap(), Some(r.uncompressed_size));
882            assert!(store.has(&r.oid).unwrap());
883        }
884
885        // A delta really is labelled a delta, so nobody can mistake its bytes for
886        // content. (If this pack had none, the assertion above would be vacuous.)
887        let deltas = rows
888            .iter()
889            .filter(|r| matches!(r.stored_type, ObjType::OfsDelta | ObjType::RefDelta))
890            .count();
891        assert!(deltas > 0, "the fixture pack carries no deltas");
892        let d = rows
893            .iter()
894            .find(|r| matches!(r.stored_type, ObjType::OfsDelta | ObjType::RefDelta))
895            .unwrap();
896        let got = store.get(&d.oid).unwrap().unwrap();
897        assert!(matches!(
898            got.obj_type,
899            ObjType::OfsDelta | ObjType::RefDelta
900        ));
901        assert_ne!(
902            got.bytes.len() as u64,
903            got.uncompressed_size,
904            "a delta's stored bytes are not its resolved size"
905        );
906
907        // An oid nothing pushed is absent, and that is an answer, not a guess.
908        assert!(!store.has(&[0xab; 20]).unwrap());
909        assert!(store.get(&[0xab; 20]).unwrap().is_none());
910        assert_eq!(store.size(&[0xab; 20]).unwrap(), None);
911    }
912
913    /// **Durability, in the order that matters.** `put_pack` must not return
914    /// before the bytes are on the device *and* a journal row on the device
915    /// points at them — and if the machine dies between the two, what is left
916    /// must be orphan bytes, never a row pointing into a hole.
917    ///
918    /// Asserted by reading the two files back from disk after a fault injected
919    /// into the **real** `SafeWriter::append` (LAW 5: the fault goes into the one
920    /// writer, rather than a hand-rolled twin of it being tested).
921    ///
922    /// Seen RED by moving the blob `fsync` in `SafeWriter::append` to *after* the
923    /// journal row and its `fsync`, with the injected crash between them — i.e.
924    /// the reference is made durable before the bytes it references: "the journal
925    /// claims 1 extent(s) after a crash before the blob was durable — that is a
926    /// dangling reference, which is the failure this ordering exists to prevent".
927    /// Restored.
928    ///
929    /// **And the honest limit of this guard, found by trying a weaker mutation
930    /// first.** Moving the blob `fsync` to the end *without* moving the fault did
931    /// **not** turn it red: the fault still fires before the journal row, so the
932    /// on-disk state is still orphan bytes, and an `fsync` that did not happen is
933    /// unobservable from inside the process — the bytes read back out of the page
934    /// cache either way. What this guard proves is therefore the **ordering of the
935    /// reference against the bytes**, which is the part a crash can expose. It
936    /// cannot prove an `fsync` reached the platter; nothing short of cutting power
937    /// can.
938    #[test]
939    fn a_pack_is_durable_before_put_returns_and_the_fsyncs_are_ordered() {
940        use crate::archive_write::{read_journal, ArchiveWrite, Faults, SafeWriter};
941
942        let dir = tmpdir("durable");
943        let (pack, _) = one_blob_pack(b"durability is an ordering property");
944
945        // 1. The ordinary path: after `put_pack` returns, both files hold it.
946        let store = GitStore::open(&dir, "rickard").unwrap();
947        let tx = store.put_pack(&pack).unwrap();
948        let (offset, len) = tx.extent.unwrap();
949        let journal = SafeWriter::journal_path(store.blobs_path());
950        assert_eq!(
951            read_journal(&journal).unwrap(),
952            vec![(offset, len)],
953            "the journal row that claims the extent is not there when put_pack returned"
954        );
955        assert_eq!(
956            std::fs::metadata(store.blobs_path()).unwrap().len(),
957            offset + len,
958            "the blob file does not end where the journal says the pack does"
959        );
960
961        // 2. A crash between the two fsyncs leaves ORPHAN BYTES, not a dangling
962        //    reference. Same `append`, one fault flipped.
963        let dir2 = tmpdir("crash");
964        let blobs = dir2.join("objects.pack");
965        let w = SafeWriter::create_with_faults(
966            &blobs,
967            Faults {
968                die_between_fsyncs: true,
969                ..Default::default()
970            },
971        )
972        .unwrap();
973        assert!(
974            w.append(&pack).is_err(),
975            "the injected crash must not return Ok"
976        );
977        drop(w);
978        let orphan = std::fs::read(&blobs).unwrap();
979        assert_eq!(orphan, pack, "the blob bytes were fsynced before the crash");
980        assert!(
981            read_journal(&SafeWriter::journal_path(&blobs))
982                .unwrap()
983                .is_empty(),
984            "the journal claims {} extent(s) after a crash before the blob was durable — that is a \
985             dangling reference, which is the failure this ordering exists to prevent",
986            read_journal(&SafeWriter::journal_path(&blobs)).unwrap().len()
987        );
988
989        // 3. And a store reopened over the orphan file answers for nothing,
990        //    rather than for bytes nobody claimed.
991        let reopened = GitStore::open(&dir2, "rickard").unwrap();
992        assert_eq!(reopened.object_count(), 0);
993    }
994
995    /// **The closure check refuses a pack before storing it**, and it does so
996    /// without consulting the index for anything except a genuinely external
997    /// base.
998    ///
999    /// Asserted on applied output: the blob file does not grow, and no journal
1000    /// row appears.
1001    ///
1002    /// Seen RED by moving `external_bases_exist` to *after* `push_pack`: "the
1003    /// refused pack was stored anyway: 0 bytes became 65".
1004    #[test]
1005    fn a_pack_that_fails_the_closure_check_is_refused_before_a_byte_is_stored() {
1006        let dir = tmpdir("closure");
1007        let store = GitStore::open(&dir, "rickard").unwrap();
1008
1009        // A thin pack: one ref-delta against an oid this repository has never
1010        // seen.
1011        let mut thin = b"PACK".to_vec();
1012        thin.extend_from_slice(&2u32.to_be_bytes());
1013        thin.extend_from_slice(&1u32.to_be_bytes());
1014        thin.push(0x74); // type 7, size 4
1015        thin.extend_from_slice(&[0x11; 20]);
1016        {
1017            use std::io::Write;
1018            let mut e = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
1019            e.write_all(&[0x04, 0x04, 0x90, 0x04]).unwrap();
1020            thin.extend_from_slice(&e.finish().unwrap());
1021        }
1022        thin.extend_from_slice(&[0u8; 20]);
1023
1024        let before = std::fs::metadata(store.blobs_path()).unwrap().len();
1025        let err = store
1026            .put_pack(&thin)
1027            .expect_err("a dangling base is refused");
1028        let msg = err.to_string();
1029        assert!(
1030            msg.contains(&hex::encode([0x11; 20])),
1031            "names the base: {msg}"
1032        );
1033        let after = std::fs::metadata(store.blobs_path()).unwrap().len();
1034        assert_eq!(
1035            before, after,
1036            "the refused pack was stored anyway: {before} bytes became {after}"
1037        );
1038        assert_eq!(store.unindexed_packs(), 0, "and no index job was queued");
1039
1040        // A corrupt pack — a delta base that lands nowhere — is refused too, and
1041        // as corruption rather than as thinness.
1042        let (mut corrupt, _) = one_blob_pack(b"x");
1043        corrupt[11] = 2; // claim two objects, provide one
1044        assert!(store.put_pack(&corrupt).is_err());
1045        assert_eq!(
1046            std::fs::metadata(store.blobs_path()).unwrap().len(),
1047            before,
1048            "a corrupt pack was stored"
1049        );
1050    }
1051
1052    /// **CAS: a mismatch moves nothing.** The ref namespace is read back from the
1053    /// log after every attempt, so what is asserted is the ref's applied state
1054    /// and not the call's return value.
1055    ///
1056    /// Seen RED by comparing `actual.is_some() != old.is_some()` — existence
1057    /// only, the CAS bug that looks right: "stale old value: TxId { pack_id: None,
1058    /// extent: None, push_seq: Some(1) }", i.e. the swap from a wrong old value
1059    /// was accepted and written.
1060    #[test]
1061    fn a_compare_and_swap_that_loses_the_race_moves_no_ref() {
1062        let dir = tmpdir("cas");
1063        let store = GitStore::open(&dir, "rickard").unwrap();
1064        let (pack_a, oid_a) = one_blob_pack(b"first");
1065        let (pack_b, oid_b) = one_blob_pack(b"second");
1066        store.put_pack(&pack_a).unwrap();
1067        store.put_pack(&pack_b).unwrap();
1068        store.absorb_pending().unwrap();
1069
1070        let name = "refs/heads/main";
1071        let target = |s: &GitStore| -> Option<Vec<u8>> {
1072            s.refs()
1073                .unwrap()
1074                .into_iter()
1075                .find(|r| r.name == name)
1076                .and_then(|r| r.oid)
1077        };
1078
1079        // ★ A LOST CAS IS TYPED, not a string a caller has to parse.
1080        //
1081        // `update_ref` raised a bare `bail!` while its own sibling
1082        // `put_refs_cas` raised `RefRejection::Cas`, so `gunnar-wire`'s
1083        // receive-pack could not tell a lost race from a broken disk and said
1084        // so in a comment at the call site. The contract had the type all
1085        // along; this method did not use it.
1086        //
1087        // Seen RED by restoring the `bail!`: "a lost CAS must carry
1088        // RefRejection::Cas, not a string: compare-and-swap on refs/heads/main
1089        // failed: it is absent, the caller expected 0101…" — the message was
1090        // right and the TYPE was missing, which is exactly the failure a
1091        // string-shaped error hides.
1092        let err = store
1093            .update_ref(name, Some(&oid_a), Some(&oid_b))
1094            .expect_err("a create that claims a previous value must fail");
1095        match err.downcast_ref::<git_storage_trait::RefRejection>() {
1096            Some(git_storage_trait::RefRejection::Cas {
1097                name: n,
1098                expected,
1099                actual,
1100            }) => {
1101                assert_eq!(n, name, "the rejection names the wrong ref");
1102                assert_eq!(
1103                    *actual,
1104                    git_storage_trait::Observed::Nothing,
1105                    "the ref did not exist, so `actual` must say so"
1106                );
1107                assert_ne!(
1108                    *expected,
1109                    git_storage_trait::Observed::Nothing,
1110                    "the caller DID claim a previous value; `expected` must carry it"
1111                );
1112            }
1113            other => panic!(
1114                "a lost CAS must carry RefRejection::Cas, not a string: {err:#} (downcast: {})",
1115                if other.is_some() { "wrong variant" } else { "none" }
1116            ),
1117        }
1118        assert_eq!(target(&store), None, "the failed create wrote something");
1119
1120        let tx = store.update_ref(name, None, Some(&oid_a)).unwrap();
1121        assert!(tx.push_seq.is_some());
1122        assert_eq!(target(&store).as_deref(), Some(&oid_a[..]));
1123
1124        // A swap from the WRONG old value: refused, and nothing moves.
1125        let err = store
1126            .update_ref(name, Some(&oid_b), Some(&oid_b))
1127            .expect_err("stale old value");
1128        assert!(err.to_string().contains("compare-and-swap"), "{err}");
1129        assert_eq!(
1130            target(&store).as_deref(),
1131            Some(&oid_a[..]),
1132            "refs/heads/main moved on a failed CAS"
1133        );
1134
1135        // The right old value: it moves.
1136        store.update_ref(name, Some(&oid_a), Some(&oid_b)).unwrap();
1137        assert_eq!(target(&store).as_deref(), Some(&oid_b[..]));
1138
1139        // A ref may not point at an object we do not have.
1140        let err = store
1141            .update_ref(name, Some(&oid_b), Some(&[0xcd; 20]))
1142            .expect_err("dangling target");
1143        assert!(err.to_string().contains("does not have"), "{err}");
1144        assert_eq!(target(&store).as_deref(), Some(&oid_b[..]));
1145
1146        // Delete: gone from the namespace, and the log is the history.
1147        store.update_ref(name, Some(&oid_b), None).unwrap();
1148        assert_eq!(target(&store), None);
1149        assert!(store.refs().unwrap().iter().all(|r| r.name != name));
1150    }
1151
1152    /// **The bounded walk answers EXACTLY what a full bitmap table answers** —
1153    /// under-send and over-send both named, over every commit in the fixture.
1154    ///
1155    /// # What is actually at stake here
1156    ///
1157    /// Until 2026-08-14 the live table bitmapped **every** commit
1158    /// (`ReachPolicy { max_commits: usize::MAX }`), not as a choice but because
1159    /// `reachable_oids` had no fallback: a `want` with no bitmap contributed
1160    /// itself and nothing else. MEASURED on oden — 530 218 of a 4-object fetch's
1161    /// 575 349 allocations were that build, thrown away again by the next push's
1162    /// `refold`. [`crate::reach::accumulate`] is the walk that makes a sampled
1163    /// table legal, and `LIVE_REACH_COMMITS` is now 512.
1164    ///
1165    /// **A walk that stops too early UNDER-SENDS, and a clone that under-sends
1166    /// is silent data loss**: the pack indexes, `fsck`s and applies, the client
1167    /// exits zero, and the repository is missing objects it will not discover
1168    /// for days. `select` refusing was a correct-but-expensive answer to exactly
1169    /// this hazard, and this test is what replaces it.
1170    ///
1171    /// # Three arms over one store, and why three
1172    ///
1173    /// | cap | table | what runs |
1174    /// |---|---|---|
1175    /// | `usize::MAX` | every commit | no walk at all — **the reference** |
1176    /// | `0` | empty | the walk does *everything* |
1177    /// | `2` | two commits | the mixed case, where phase 1's stopping rule actually fires |
1178    ///
1179    /// Two arms would not be enough. `0` never exercises the stop-at-a-bitmap
1180    /// rule, which is the one thing that could OR in a *closed* set at the wrong
1181    /// moment, and `usize::MAX` never exercises the walk. The middle arm is the
1182    /// only one where both halves meet.
1183    ///
1184    /// # Both directions are named, and that is the anti-hollow half
1185    ///
1186    /// The comparison is a **symmetric difference**, not a length check and not
1187    /// a subset check. An over-send passes every downstream test we own — the
1188    /// pack indexes, `--check-self-contained-and-connected` accepts it, `fsck`
1189    /// is clean — and today's gix defect was exactly that shape. So `missing`
1190    /// and `extra` are computed and printed separately: a walk that stops short
1191    /// names the objects it lost, and a walk that ORs too much names the objects
1192    /// it invented.
1193    ///
1194    /// # Seen RED, 2026-08-14, three ways — and the directions are instructive
1195    ///
1196    /// * **stop one commit short** (`continue` before pushing parents in phase
1197    ///   1) — *"cap 0 vs the full table disagree for want 75c9e5c3…: MISSING 0
1198    ///   object(s) …; EXTRA 2 object(s) the full table did not select, first
1199    ///   Some(\"75c9e5c3d1d08dd92cc913c70ad10288a19eea4e\")"*.
1200    /// * **lose objects in the tree walk** (skip every seventh ordinal in
1201    ///   `accumulate_tree`) — *"cap 0 …: MISSING 2 object(s) the full table
1202    ///   selected, first Some(\"28b3c3658111299869d1f63f49a8cfb2635c0cfe\");
1203    ///   EXTRA 0"*.
1204    /// * **OR in an unrelated commit's bitmap** — *"cap 2 …: MISSING 13
1205    ///   object(s) the full table selected, first
1206    ///   Some(\"078f18e68253d6fc56e8cba230ef9f32dec2e2ab\"); EXTRA 0"*.
1207    ///
1208    /// 🔴 **Note which direction each one came out.** Truncating the walk
1209    /// produced an *over*-send and over-collecting produced an *under*-send —
1210    /// the opposite of the intuition in both cases — because the identical walk
1211    /// serves the `have` side, and an error there enters the answer through
1212    /// `union − had` with its sign flipped. That is precisely why the comparison
1213    /// is a symmetric difference and reports **both** counts: a guard that
1214    /// checked only for missing objects would have passed the first break, and a
1215    /// guard that checked only lengths would have named nothing.
1216    ///
1217    /// The premise is guarded too: with the arms wired to the same policy this
1218    /// test passes trivially, so the table size at each cap is asserted before
1219    /// any answer is compared.
1220    #[test]
1221    fn the_bounded_walk_answers_exactly_what_a_full_bitmap_table_answers() {
1222        use crate::reach::ReachPolicy;
1223
1224        let dir = tmpdir("reach-walk");
1225        let store = GitStore::open(&dir, "rickard").unwrap();
1226        let (pack, _) = real_pack();
1227        store.put(&pack, &[]).unwrap();
1228        store.absorb_pending().unwrap();
1229
1230        let commits: Vec<String> = store.graph_snapshot().iter().map(|c| c.oid.clone()).collect();
1231        assert!(
1232            commits.len() > 2,
1233            "premise: the fixture needs more commits than the smallest cap under test, or every \
1234             arm bitmaps everything and the walk never runs — found {}",
1235            commits.len()
1236        );
1237
1238        let full = ReachPolicy {
1239            max_commits: usize::MAX,
1240        };
1241        let none = ReachPolicy { max_commits: 0 };
1242        let some = ReachPolicy { max_commits: 2 };
1243
1244        // The premise, measured off the tables themselves rather than assumed.
1245        // A cap that did not actually shrink the table would make every
1246        // comparison below a comparison of one code path with itself.
1247        let n_full = store.reach_bitmaps_with(full, false).unwrap().len();
1248        let n_none = store.reach_bitmaps_with(none, false).unwrap().len();
1249        let n_some = store.reach_bitmaps_with(some, false).unwrap().len();
1250        assert_eq!(
1251            n_full,
1252            commits.len(),
1253            "premise: the uncapped table must bitmap every commit"
1254        );
1255        assert_eq!(
1256            n_none, 0,
1257            "premise: the zero cap must produce NO bitmaps, or the walk arm is not a walk arm"
1258        );
1259        assert!(
1260            n_some > 0 && n_some < commits.len(),
1261            "premise: the middle cap must bitmap SOME commits and not all ({n_some} of {}), or \
1262             the stop-at-a-bitmap rule is never reached",
1263            commits.len()
1264        );
1265
1266        // A stride through the graph as `want`, each against its successor as
1267        // `have` — so the walk is asked about tips, roots and the middle.
1268        //
1269        // A stride and not every commit: `reachable_oids_with` rebuilds the
1270        // table on every call here (`cache: false`, and it must be — see that
1271        // method), so the exhaustive form is quadratic and ran for 334 s against
1272        // this fixture. Twelve probes spread across the history cost seconds and
1273        // cover the same three positions; the arms that matter are the three
1274        // caps, not the commit count.
1275        let stride = commits.len().div_ceil(12).max(1);
1276        let probes: Vec<usize> = (0..commits.len())
1277            .step_by(stride)
1278            .chain([commits.len() - 1])
1279            .collect();
1280        let mut walked_any = false;
1281        for i in probes {
1282            let w = &commits[i];
1283            let w_raw = hex::decode(w).unwrap();
1284            let h_raw = (i + 1 < commits.len()).then(|| hex::decode(&commits[i + 1]).unwrap());
1285            let haves: Vec<&[u8]> = h_raw.iter().map(|v| v.as_slice()).collect();
1286
1287            let reference: std::collections::BTreeSet<String> = store
1288                .reachable_oids_with(&[&w_raw], &haves, full, false)
1289                .unwrap()
1290                .into_iter()
1291                .collect();
1292
1293            for (label, policy) in [("cap 0", none), ("cap 2", some)] {
1294                let got: std::collections::BTreeSet<String> = store
1295                    .reachable_oids_with(&[&w_raw], &haves, policy, false)
1296                    .unwrap()
1297                    .into_iter()
1298                    .collect();
1299                let missing: Vec<&String> = reference.difference(&got).collect();
1300                let extra: Vec<&String> = got.difference(&reference).collect();
1301                assert!(
1302                    missing.is_empty() && extra.is_empty(),
1303                    "{label} vs the full table disagree for want {w}: MISSING {} object(s) the \
1304                     full table selected, first {:?}; EXTRA {} object(s) the full table did not \
1305                     select, first {:?}",
1306                    missing.len(),
1307                    missing.first(),
1308                    extra.len(),
1309                    extra.first()
1310                );
1311            }
1312            walked_any = true;
1313        }
1314        assert!(walked_any, "no commit was compared");
1315
1316        // And the answers are not vacuously equal because they are all empty.
1317        let tip = hex::decode(&commits[0]).unwrap();
1318        let n = store
1319            .reachable_oids_with(&[&tip], &[], none, false)
1320            .unwrap()
1321            .len();
1322        assert!(
1323            n > 1,
1324            "premise: a commit must reach more than itself for the comparison above to have any \
1325             content — got {n}"
1326        );
1327    }
1328
1329    /// **`want` minus `have` is an `andnot`, over objects and not just commits.**
1330    ///
1331    /// Built on a real repository's pack: the second commit's closure minus the
1332    /// first's must be exactly the objects the second commit introduced, and
1333    /// every returned oid must be one the store really holds.
1334    ///
1335    /// Seen RED by returning `union` instead of `union - had`: "removing the
1336    /// parent's closure removed nothing: 8 vs 8".
1337    ///
1338    /// MEASURED on the fixture: the child commit's closure is 8 objects, the
1339    /// parent's is 5, and the difference is 3 — the objects that commit
1340    /// introduced.
1341    #[test]
1342    fn reachable_is_want_minus_have_over_objects() {
1343        let dir = tmpdir("reach");
1344        let store = GitStore::open(&dir, "rickard").unwrap();
1345        let (pack, _) = real_pack();
1346        store.put(&pack, &[]).unwrap();
1347        store.absorb_pending().unwrap();
1348        assert!(store.commit_count() > 1, "the fixture has no history");
1349
1350        // Two commits in parent→child order: the graph is folded that way.
1351        let (child, parent) = {
1352            let mut pair = None;
1353            for c in store.graph_snapshot() {
1354                if let Some(p) = c.parents.first() {
1355                    pair = Some((c.oid.clone(), p.clone()));
1356                    break;
1357                }
1358            }
1359            pair.expect("a commit with a parent")
1360        };
1361        let child_raw = hex::decode(&child).unwrap();
1362        let parent_raw = hex::decode(&parent).unwrap();
1363
1364        let all = store.reachable(&[&child_raw], &[]).unwrap();
1365        let delta = store.reachable(&[&child_raw], &[&parent_raw]).unwrap();
1366        let had = store.reachable(&[&parent_raw], &[]).unwrap();
1367
1368        assert!(
1369            !all.is_empty(),
1370            "a commit reaches at least itself and its tree"
1371        );
1372        assert!(
1373            delta.len() < all.len(),
1374            "removing the parent's closure removed nothing: {} vs {}",
1375            delta.len(),
1376            all.len()
1377        );
1378        assert_eq!(
1379            all.len(),
1380            delta.len() + had.iter().filter(|h| all.contains(h)).count(),
1381            "the three sets do not add up — want minus have is not an andnot"
1382        );
1383        for oid in delta.iter().chain(all.iter()) {
1384            assert!(
1385                store.has(oid).unwrap(),
1386                "reachable named {} which the store does not hold",
1387                hex::encode(oid)
1388            );
1389        }
1390        // The commit itself is in its own closure; its parent is not in the delta.
1391        assert!(all.contains(&child_raw));
1392        assert!(!delta.contains(&parent_raw));
1393        eprintln!(
1394            "closure {} objects, minus the parent's {} leaves {}",
1395            all.len(),
1396            had.len(),
1397            delta.len()
1398        );
1399    }
1400
1401    /// **The batch of one really is slower**, measured here rather than asserted
1402    /// from the plan — and the serial path is what `extents` takes for it.
1403    ///
1404    /// The ratio is machine- and load-dependent, so the guard on it is loose (it
1405    /// requires the batch path to be no *faster*, which is the direction the
1406    /// dispatch depends on) while the number is printed for the record. The
1407    /// dispatch itself is asserted exactly.
1408    ///
1409    /// MEASURED, release, oden, loadavg 4.94, 2687 objects: `lookup` 628 ns,
1410    /// `lookup_batch` of one 639 ns (**1.02x worse**, not the 1.7x the bare Arrow
1411    /// arms show), `lookup_batch` of 100 **161 ns/oid** — 3.9x better than the
1412    /// serial path. See [`lookup_path`] for why both numbers are recorded.
1413    ///
1414    /// Seen RED by making `lookup_path` return `Batch` for every n:
1415    /// "assertion `left == right` failed — left: Batch, right: Serial".
1416    ///
1417    /// The honest limit: a *timing* assertion cannot catch `extents` ignoring the
1418    /// dispatch, because the two paths differ by 2% here. That is exactly why the
1419    /// dispatch is a named function with an exact assertion on it rather than a
1420    /// comment above an `if`.
1421    #[test]
1422    fn the_batch_of_one_really_is_slower_than_the_serial_path() {
1423        assert_eq!(lookup_path(1), LookupPath::Serial);
1424        assert_eq!(lookup_path(2), LookupPath::Batch);
1425        assert_eq!(lookup_path(0), LookupPath::Batch);
1426        assert_eq!(BATCH_SATURATES_AT, 100);
1427
1428        let dir = tmpdir("batch");
1429        let store = GitStore::open(&dir, "rickard").unwrap();
1430        let (pack, rows) = real_pack();
1431        store.put(&pack, &[]).unwrap();
1432        store.absorb_pending().unwrap();
1433        let oids: Vec<&[u8]> = rows.iter().map(|r| r.oid.as_slice()).collect();
1434
1435        // Both paths answer identically for one oid — the dispatch is an
1436        // optimisation, never a difference in the answer.
1437        for oid in oids.iter().take(64) {
1438            assert_eq!(
1439                store.extents(&[*oid]).unwrap(),
1440                store.index().extents_batch(&[*oid]),
1441                "the serial and batch paths disagree for {}",
1442                hex::encode(oid)
1443            );
1444        }
1445
1446        let n = 20_000usize;
1447        let serial = {
1448            let t = Instant::now();
1449            for i in 0..n {
1450                let _ = store.index().lookup(oids[i % oids.len()]);
1451            }
1452            t.elapsed().as_nanos() as f64 / n as f64
1453        };
1454        let batched = {
1455            let t = Instant::now();
1456            for i in 0..n {
1457                let _ = store.index().lookup_batch(&[oids[i % oids.len()]]);
1458            }
1459            t.elapsed().as_nanos() as f64 / n as f64
1460        };
1461        // What the batch path is actually for: 100 oids in one call, which is
1462        // where it saturates.
1463        let hundred: Vec<&[u8]> = oids.iter().take(BATCH_SATURATES_AT).copied().collect();
1464        let per_oid_at_100 = {
1465            let rounds = n / BATCH_SATURATES_AT;
1466            let t = Instant::now();
1467            for _ in 0..rounds {
1468                let _ = store.index().lookup_batch(&hundred);
1469            }
1470            t.elapsed().as_nanos() as f64 / (rounds * hundred.len()) as f64
1471        };
1472        eprintln!(
1473            "load {}; {} objects: lookup {serial:.0} ns, lookup_batch[1] {batched:.0} ns \
1474             ({:.2}x), lookup_batch[{}] {per_oid_at_100:.0} ns/oid",
1475            std::fs::read_to_string("/proc/loadavg")
1476                .unwrap_or_default()
1477                .trim(),
1478            store.index().len(),
1479            batched / serial,
1480            hundred.len(),
1481        );
1482        // 2026-08-10: the fused batch walk (no sort phase) collapsed the old
1483        // 1.02x batch-of-one penalty to a measured TIE — 604-620 ns both sides,
1484        // winner decided by noise, seen flipping run-to-run on an idle box. The
1485        // `Serial` dispatch for n=1 stays correct (equal time, two fewer Vec
1486        // allocations), so the guard keeps only the direction that would make
1487        // it WRONG: a batch of one materially faster than serial. 10% is
1488        // outside the tie's observed 0.3% jitter and inside any real win.
1489        assert!(
1490            batched >= serial * 0.9,
1491            "a batch of one measured MATERIALLY faster ({batched:.0} ns vs {serial:.0} ns) — \
1492             if that is repeatable the dispatch in `lookup_path` is wrong and must be \
1493             changed, not kept"
1494        );
1495    }
1496
1497    /// **`seal` folds the live logs into reserved Arrow sections**, and the
1498    /// generation numbers in the graph section are recomputed by that fold.
1499    ///
1500    /// Asserted by decoding the sections back out of their Arrow IPC bytes.
1501    ///
1502    /// Seen RED by dropping the `assign_generations` call from `refold`:
1503    /// "e58a5032ac3b36f15f08c759121efc8f08680e2b has generation 0".
1504    #[test]
1505    fn seal_emits_the_reserved_sections_and_recomputes_every_generation() {
1506        use znippy_common::{GUNNAR_GRAPH_MODULE, GUNNAR_REACH_MODULE, GUNNAR_REFS_MODULE};
1507
1508        let dir = tmpdir("seal");
1509        let store = GitStore::open(&dir, "rickard").unwrap();
1510        let (pack, _) = real_pack();
1511        store.put(&pack, &[]).unwrap();
1512        store.absorb_pending().unwrap();
1513        let tip = store
1514            .graph_snapshot()
1515            .iter()
1516            .max_by_key(|c| c.generation)
1517            .expect("a graph")
1518            .clone();
1519        store
1520            .update_ref(
1521                "refs/heads/main",
1522                None,
1523                Some(&hex::decode(&tip.oid).unwrap()),
1524            )
1525            .unwrap();
1526
1527        let sections = store.seal().unwrap();
1528        let names: Vec<&str> = sections.iter().map(|s| s.module_name.as_str()).collect();
1529        for want in [GUNNAR_REFS_MODULE, GUNNAR_GRAPH_MODULE, GUNNAR_REACH_MODULE] {
1530            assert!(names.contains(&want), "{want} is not sealed: {names:?}");
1531        }
1532
1533        // Generations: a root is 1, a child is 1 + max(parents), and the graph
1534        // this store folded must say so.
1535        let graph = store.graph_snapshot();
1536        let by_oid: std::collections::HashMap<&str, &crate::graph::CommitNode> =
1537            graph.iter().map(|c| (c.oid.as_str(), c)).collect();
1538        let mut with_parents = 0usize;
1539        for c in &graph {
1540            assert!(c.generation >= 1, "{} has generation 0", c.oid);
1541            let known: Vec<&crate::graph::CommitNode> = c
1542                .parents
1543                .iter()
1544                .filter_map(|p| by_oid.get(p.as_str()).copied())
1545                .collect();
1546            if known.is_empty() {
1547                assert_eq!(c.generation, 1, "{} is a root here", c.oid);
1548            } else {
1549                with_parents += 1;
1550                let expect = 1 + known.iter().map(|p| p.generation).max().unwrap();
1551                assert_eq!(
1552                    c.generation, expect,
1553                    "{} must have generation {expect}",
1554                    c.oid
1555                );
1556                assert!(
1557                    c.generation > 1,
1558                    "a commit with a parent must have generation > 1"
1559                );
1560            }
1561        }
1562        assert!(with_parents > 0, "no commit in the fixture has a parent");
1563        eprintln!(
1564            "{} commits sealed, {with_parents} with parents",
1565            graph.len()
1566        );
1567    }
1568
1569    /// **`seal` writes generation 0, and `gc` then has something to compact.**
1570    ///
1571    /// This is the hole the four working steps of `gc()` used to fall into:
1572    /// absorb, live set, retire, drop all ran, and then
1573    /// `NewGeneration::run(archive_path())` died on
1574    /// `stat …/repository.znippy: No such file or directory` because **nothing
1575    /// ever created generation 0**. Every other test in this file that reached
1576    /// step 5 hand-built the archive with `znippy_common::create_archive`, which
1577    /// is why the gap survived: the fixture was doing the store's job.
1578    ///
1579    /// Asserted on **applied output**, through znippy's own reader and nothing
1580    /// else:
1581    ///
1582    /// * the live pack comes back out of the archive **byte for byte**, via
1583    ///   `extract_file_verified`, which reconstructs the entry and blake3-checks
1584    ///   it against the index — so a wrong checksum, a wrong extent or a wrong
1585    ///   `compressed` flag all fail here rather than being written and believed;
1586    /// * the tombstoned pack has **no** entry, because a row for it is exactly
1587    ///   what would stop the compaction reclaiming its bytes;
1588    /// * the three reserved sections are addressable **out of the file**, not
1589    ///   merely present in the returned vector;
1590    /// * `gc()` then runs end to end and `NewGeneration` produces
1591    ///   `repository.g1.znippy`, verified, with the old generation gone.
1592    ///
1593    /// # Seen RED, and one of the reds was in this guard
1594    ///
1595    /// Every mutation below was applied to the real code, run, and reverted.
1596    ///
1597    /// 1. **The hole itself.** With `seal` returning the sections and writing
1598    ///    nothing — the code as it stood — this fails at
1599    ///    `seal did not create …/repository.znippy`, and `gc()` fails with the
1600    ///    original `compacting …/repository.znippy after dropping 2683 dead
1601    ///    rows: stat …/repository.znippy: No such file or directory`.
1602    /// 2. **A dropped row** — `.take(packs.len() - 1)` on the row loop:
1603    ///    `objects.pack.2 is not in the sealed archive: ["objects.pack.0"]`.
1604    ///    **This is the mutation that first found a hollow guard, and the guard
1605    ///    was mine.** Against the two-pack fixture this test started with, the
1606    ///    row dropped was the tombstoned one, which is skipped anyway — the
1607    ///    mutation stayed GREEN. The fixture now pushes three packs and
1608    ///    tombstones the middle one, so there are two live rows and dropping
1609    ///    either is visible.
1610    /// 3. **No tombstone honoured** — the `retired.contains` skip disabled:
1611    ///    `the tombstoned pack got an index row anyway … ["objects.pack.0",
1612    ///    "objects.pack.2", "objects.pack.1"]`. A row for a retired pack is what
1613    ///    would keep its payload alive through every future compaction.
1614    /// 4. **A corrupted extent** — `blob_offset: offset + 1`. It writes, lists
1615    ///    and opens perfectly; it dies at the blake3 gate inside
1616    ///    `extract_file_verified`: `checksum mismatch for objects.pack.0 at
1617    ///    fdata_offset 0`.
1618    /// 5. **The checksum domain** — hashing the *path* instead of the bytes,
1619    ///    which is precisely the mistake that produces an archive that writes
1620    ///    cleanly and verifies wrong. Same gate, same message. Together 4 and 5
1621    ///    are what establish that the checksum is blake3 over the bytes at
1622    ///    `blob_offset`, and that this test can tell.
1623    /// 6. **The `compressed` flag** — `true` on bytes that were stored raw:
1624    ///    `OpenZL getDecompressedSize: ZL_getDecompressedSize failed`. The flag
1625    ///    is load-bearing, not decorative.
1626    /// 7. **A lying report** — `packs_retired: 0` in the `SealReport` literal:
1627    ///    `the report miscounts the tombstoned packs`. The report is read back
1628    ///    by a second seal, which also pins that a seal is a snapshot: taking it
1629    ///    twice yields the same entries, not a second generation.
1630    /// 8. **No reserved builder attached** — the sections are still *returned*,
1631    ///    so a guard that only inspected the return value would pass:
1632    ///    `__gunnar_refs__ is not in the sealed archive's manifest`. That is why
1633    ///    the sections are read back out of the file by module name.
1634    ///
1635    #[test]
1636    fn seal_writes_generation_zero_that_gc_can_compact() {
1637        use znippy_common::{
1638            read_reserved_section_bytes, ZnippyArchive, ZnippyReader, GUNNAR_GRAPH_MODULE,
1639            GUNNAR_REACH_MODULE, GUNNAR_REFS_MODULE,
1640        };
1641
1642        let dir = tmpdir("seal-g0");
1643        let store = GitStore::open(&dir, "rickard").unwrap();
1644        // THREE packs, and the middle one is tombstoned before the seal. Two
1645        // live rows rather than one is not decoration: with a single live entry,
1646        // "dropped a row" and "wrote no rows at all" are the same failure, and a
1647        // `.take(packs.len() - 1)` mutation of the row loop stayed GREEN against
1648        // an earlier two-pack version of this fixture. Three packs with a gap in
1649        // the middle also pin the ordinal: it is the position among the
1650        // journal's `Pack` rows, so the survivors are 0 and **2**, not 0 and 1.
1651        let (pack, _) = real_pack();
1652        let (doomed, doomed_oid) = one_blob_pack(b"this pack is retired before the seal");
1653        let (kept, kept_oid) = one_blob_pack(b"this pack is unreferenced but not tombstoned");
1654        store.put(&pack, &[]).unwrap();
1655        let doomed_tx = store.put_pack(&doomed).unwrap();
1656        store.put_pack(&kept).unwrap();
1657        store.absorb_pending().unwrap();
1658        for (what, oid) in [("doomed", &doomed_oid), ("kept", &kept_oid)] {
1659            assert!(
1660                store.has(oid).unwrap(),
1661                "the {what} blob was never stored, so this fixture proves nothing"
1662            );
1663        }
1664
1665        // A ref, so the live set is not empty and `gc` is not refused.
1666        let root = store
1667            .graph_snapshot()
1668            .into_iter()
1669            .find(|c| c.generation == 1)
1670            .expect("a root commit");
1671        let root_raw = hex::decode(&root.oid).unwrap();
1672        store
1673            .update_ref("refs/heads/root", None, Some(&root_raw))
1674            .unwrap();
1675
1676        // Exactly one tombstone, written the way `gc` writes them. Named
1677        // explicitly rather than via `retire_dead_packs`, which would tombstone
1678        // BOTH one-blob packs and leave nothing to prove the ordinal with.
1679        crate::archive_write::retire_packs(
1680            &crate::archive_write::SafeWriter::journal_path(store.blobs_path()),
1681            &[doomed_tx.extent.unwrap().0],
1682        )
1683        .unwrap();
1684
1685        assert!(
1686            !store.archive_path().exists(),
1687            "something created the archive before the seal ran"
1688        );
1689        let sections = store.seal().unwrap();
1690
1691        // ── the archive exists and reads back through znippy's own reader ────
1692
1693        assert!(
1694            store.archive_path().exists(),
1695            "seal did not create {}",
1696            store.archive_path().display()
1697        );
1698        let ar = ZnippyArchive::open(store.archive_path()).unwrap();
1699        let listed = ar.list_files().unwrap();
1700        assert!(
1701            listed.contains(&"objects.pack.0".to_string()),
1702            "objects.pack.0 is not in the sealed archive: {listed:?}"
1703        );
1704        assert!(
1705            listed.contains(&"objects.pack.2".to_string()),
1706            "objects.pack.2 is not in the sealed archive: {listed:?}"
1707        );
1708        assert!(
1709            !listed.contains(&"objects.pack.1".to_string()),
1710            "the tombstoned pack got an index row anyway, so the compaction can never \
1711             reclaim its bytes: {listed:?}"
1712        );
1713        assert_eq!(
1714            listed.len(),
1715            2,
1716            "the seal wrote entries for packs the journal never acked as live: {listed:?}"
1717        );
1718
1719        // The bytes, reconstructed and blake3-checked by the reader itself.
1720        assert_eq!(
1721            ar.extract_file_verified("objects.pack.0").unwrap(),
1722            pack,
1723            "objects.pack.0 does not read back as the pack that was pushed"
1724        );
1725        assert_eq!(
1726            ar.extract_file_verified("objects.pack.2").unwrap(),
1727            kept,
1728            "objects.pack.2 does not read back as the pack that was pushed"
1729        );
1730
1731        // ── the reserved sections, addressed out of the file ─────────────────
1732        let names: Vec<&str> = sections.iter().map(|s| s.module_name.as_str()).collect();
1733        for want in [GUNNAR_REFS_MODULE, GUNNAR_GRAPH_MODULE, GUNNAR_REACH_MODULE] {
1734            assert!(names.contains(&want), "{want} was not returned: {names:?}");
1735            let bytes = read_reserved_section_bytes(store.archive_path(), want)
1736                .unwrap()
1737                .unwrap_or_else(|| panic!("{want} is not in the sealed archive's manifest"));
1738            assert!(!bytes.is_empty(), "{want} is sealed as zero bytes");
1739        }
1740
1741        // ── the report, and that a second seal is not a different archive ───
1742        //
1743        // A count nobody asserts on is a count that can be wrong for ever, so
1744        // the report is read here — and reading it means sealing again, which
1745        // pins the other half: a seal is a snapshot and re-taking it produces
1746        // the same entries, not a second generation.
1747        let again = store
1748            .seal_archive(store.reserved_sections().unwrap())
1749            .unwrap();
1750        assert_eq!(
1751            again.packs_sealed, 2,
1752            "the report miscounts the sealed packs"
1753        );
1754        assert_eq!(
1755            again.packs_retired, 1,
1756            "the report miscounts the tombstoned packs"
1757        );
1758        assert_eq!(
1759            again.packs_after_copy, 0,
1760            "nothing was pushed during this seal, so nothing can have raced it"
1761        );
1762        let mut once = listed.clone();
1763        once.sort();
1764        let mut twice = ZnippyArchive::open(store.archive_path())
1765            .unwrap()
1766            .list_files()
1767            .unwrap();
1768        twice.sort();
1769        // Sorted on both sides: `list_files` does not promise an order, and it
1770        // was observed returning the same two entries the other way round.
1771        assert_eq!(
1772            twice, once,
1773            "sealing twice produced a different set of entries"
1774        );
1775
1776        // ── and now the fifth step of `gc` has something to compact ──────────
1777        let report = store.gc().unwrap();
1778        assert_eq!(report.strategy, "NewGeneration");
1779        assert!(report.verified, "the new generation was not read back");
1780        assert_eq!(
1781            report.archive,
1782            store.archive_path().with_file_name("repository.g1.znippy"),
1783            "NewGeneration did not produce generation 1"
1784        );
1785        assert!(report.archive.exists(), "the new generation is not on disk");
1786        assert!(
1787            !store.archive_path().exists(),
1788            "the old generation was not retired"
1789        );
1790        assert!(
1791            report.bytes_after < report.bytes_before,
1792            "the compaction reclaimed nothing: {} → {} bytes — the retired pack's payload \
1793             should have gone",
1794            report.bytes_before,
1795            report.bytes_after
1796        );
1797
1798        // The live pack survives the compaction; the retired one's row never
1799        // existed, so its bytes are what got reclaimed.
1800        let g1 = ZnippyArchive::open(&report.archive).unwrap();
1801        assert_eq!(
1802            g1.extract_file_verified("objects.pack.0").unwrap(),
1803            pack,
1804            "generation 1 does not carry the live pack"
1805        );
1806        eprintln!(
1807            "seal: {} bytes, gc: {} → {} bytes, {}",
1808            std::fs::metadata(&report.archive)
1809                .map(|m| m.len())
1810                .unwrap_or(0),
1811            report.bytes_before,
1812            report.bytes_after,
1813            report.archive.display()
1814        );
1815    }
1816
1817    /// **GC in the order §13.20 fixes**: reachability, then the dead index rows,
1818    /// then base znippy's compaction.
1819    ///
1820    /// The archive is a real znippy archive built the way `gc`'s own fixture
1821    /// builds one, so step 3 really runs `compact_archive` and really produces a
1822    /// new generation. Steps 1 and 2 are asserted as applied output: the
1823    /// unreachable oid stops resolving and the reachable one still does.
1824    ///
1825    /// Seen RED by having `gc` drop rows *before* computing the live set — the
1826    /// order §13.20 fixes, inverted: "gc dropped the live object too". Seen RED a
1827    /// second time by making `drop_dead_rows` return `Ok(0)` without removing
1828    /// anything: "the dead object still resolves after gc".
1829    ///
1830    /// MEASURED on the fixture: a ref on the root commit leaves **5** live objects
1831    /// of 2687, and all 2682 dead rows go.
1832    #[test]
1833    fn gc_computes_reachability_drops_the_dead_rows_then_compacts() {
1834        let dir = tmpdir("gc");
1835        let store = GitStore::open(&dir, "rickard").unwrap();
1836        let (pack, _) = real_pack();
1837        store.put(&pack, &[]).unwrap();
1838        store.absorb_pending().unwrap();
1839
1840        // A ref on one commit, and an object that commit cannot reach.
1841        let graph = store.graph_snapshot();
1842        let root = graph
1843            .iter()
1844            .find(|c| c.generation == 1)
1845            .expect("a root commit")
1846            .clone();
1847        let root_raw = hex::decode(&root.oid).unwrap();
1848        let live_from_root: std::collections::HashSet<Vec<u8>> = store
1849            .reachable(&[&root_raw], &[])
1850            .unwrap()
1851            .into_iter()
1852            .collect();
1853        let all: Vec<Vec<u8>> = store.index().oids_in_order().unwrap();
1854        let dead = all
1855            .iter()
1856            .find(|o| !live_from_root.contains(*o))
1857            .expect("the root does not reach everything in a real repository")
1858            .clone();
1859
1860        store
1861            .update_ref("refs/heads/root", None, Some(&root_raw))
1862            .unwrap();
1863
1864        // A real znippy archive for step 3 to compact.
1865        let files = vec![
1866            ("pack-0.pack".to_string(), pack.clone()),
1867            ("pack-1.pack".to_string(), pack.clone()),
1868        ];
1869        znippy_common::create_archive(store.archive_path(), &files, 3).unwrap();
1870
1871        let before = store.index().len();
1872        let report = store.gc().unwrap();
1873
1874        assert!(
1875            store.has(&root_raw).unwrap(),
1876            "gc dropped the live object too"
1877        );
1878        assert!(
1879            !store.has(&dead).unwrap(),
1880            "the dead object still resolves after gc"
1881        );
1882        assert!(
1883            store.index().len() < before,
1884            "gc dropped nothing: {} rows before and after",
1885            before
1886        );
1887        assert_eq!(
1888            store.index().len(),
1889            live_from_root.len(),
1890            "exactly the live set survives"
1891        );
1892        assert_eq!(report.strategy, "NewGeneration");
1893        assert!(report.verified, "the new generation was not read back");
1894        assert!(report.archive.exists(), "the new generation is not on disk");
1895        assert!(
1896            !store.archive_path().exists(),
1897            "the old generation was not retired"
1898        );
1899        eprintln!(
1900            "gc: {before} rows → {} live, {} → {} bytes, {}",
1901            store.index().len(),
1902            report.bytes_before,
1903            report.bytes_after,
1904            report.archive.display()
1905        );
1906
1907        // And a repository with no refs at all is refused rather than emptied.
1908        let dir2 = tmpdir("gc-norefs");
1909        let empty = GitStore::open(&dir2, "rickard").unwrap();
1910        empty.put(&pack, &[]).unwrap();
1911        empty.absorb_pending().unwrap();
1912        let n = empty.index().len();
1913        assert!(empty.gc().is_err(), "a GC that would delete everything ran");
1914        assert_eq!(empty.index().len(), n, "the refused GC dropped rows anyway");
1915    }
1916
1917    // ── the GC's journal half: a pack that is *wholly* dead ──────────────────
1918
1919    /// What [`one_live_pack_and_one_doomed_pack`] hands back: an open store, and
1920    /// the three oids the guards ask about.
1921    struct DoomedFixture {
1922        dir: PathBuf,
1923        store: GitStore,
1924        /// The only object of the pack that dies whole.
1925        doomed: Vec<u8>,
1926        /// The commit a ref points at.
1927        root: Vec<u8>,
1928        /// That commit's tree — a live object that is not the ref itself.
1929        tree: Vec<u8>,
1930    }
1931
1932    /// One repository with two packs: a real one that a ref reaches into, and a
1933    /// one-blob pack nothing will ever point at.
1934    ///
1935    /// The second pack is what makes the difference visible: after a GC it has
1936    /// **no rows at all**, which is the state a pack that was in flight when the
1937    /// machine died also leaves behind.
1938    ///
1939    /// The store is returned **open**, so a guard runs its first GC in the same
1940    /// process lifetime that pushed. The reopen is the thing under test and it
1941    /// belongs in the guard, not in the fixture.
1942    fn one_live_pack_and_one_doomed_pack(tag: &str) -> DoomedFixture {
1943        let dir = tmpdir(tag);
1944        let (pack, _) = real_pack();
1945        let (doomed, doomed_oid) = one_blob_pack(b"no ref will ever reach this blob");
1946        let store = GitStore::open(&dir, "rickard").unwrap();
1947        store.put(&pack, &[]).unwrap();
1948        store.put_pack(&doomed).unwrap();
1949        store.absorb_pending().unwrap();
1950        assert!(
1951            store.has(&doomed_oid).unwrap(),
1952            "the doomed blob was never stored, so this fixture proves nothing"
1953        );
1954
1955        let root = store
1956            .graph_snapshot()
1957            .into_iter()
1958            .find(|c| c.generation == 1)
1959            .expect("a root commit");
1960        let root_raw = hex::decode(&root.oid).unwrap();
1961        store
1962            .update_ref("refs/heads/root", None, Some(&root_raw))
1963            .unwrap();
1964        let tree = hex::decode(root.tree.as_ref().expect("the root commit names a tree")).unwrap();
1965        assert!(store.has(&tree).unwrap(), "the root's tree is not indexed");
1966
1967        // A real znippy archive, so the compaction step has something to compact.
1968        let files = vec![
1969            ("pack-0.pack".to_string(), pack.clone()),
1970            ("pack-1.pack".to_string(), pack.clone()),
1971        ];
1972        znippy_common::create_archive(store.archive_path(), &files, 3).unwrap();
1973        DoomedFixture {
1974            dir,
1975            store,
1976            doomed: doomed_oid,
1977            root: root_raw,
1978            tree,
1979        }
1980    }
1981
1982    /// The journal's rows, split into packs and tombstones, read off the disk.
1983    fn journal_state(dir: &Path) -> (Vec<(u64, u64)>, Vec<u64>) {
1984        use crate::archive_write::{acked_packs, read_journal, retired_offsets, SafeWriter};
1985        let rows = read_journal(&SafeWriter::journal_path(&dir.join("objects.pack"))).unwrap();
1986        let mut retired: Vec<u64> = retired_offsets(&rows).into_iter().collect();
1987        retired.sort_unstable();
1988        (acked_packs(&rows), retired)
1989    }
1990
1991    /// **THE BUG: a pack whose every object a GC found dead came back on the
1992    /// next open.**
1993    ///
1994    /// The `indexed` bit is derived as *extent in the journal, rows not in the
1995    /// index*, and a GC that drops every row of a pack produces exactly that
1996    /// state — so the reopen re-queued the pack and re-absorbed the objects the
1997    /// GC had just decided were dead. A partly dead pack keeps rows and was never
1998    /// affected, which is why this needs a pack that dies **whole**.
1999    ///
2000    /// Asserted on applied output on both sides of a process-lifetime boundary:
2001    /// the tombstone rows on disk, `has()` on the dead oid, and the object count
2002    /// after the reopen's drain has been waited for — a resurrection puts the row
2003    /// back, so counting is not a proxy for it, it *is* it.
2004    ///
2005    /// Seen RED by making `Absorber::adopt_journal` ignore the tombstones
2006    /// (`else if retired.contains(&extent.0)` → `else if false`), which is
2007    /// exactly the reader as it stood before this fix: "a dead object came back
2008    /// the instant the store was reopened". Restored.
2009    ///
2010    /// Seen RED a second time by not retiring anything in `GitOps::gc` (`let
2011    /// retired: Vec<u64> = Vec::new();` in place of the `retire_dead_packs`
2012    /// call — the writer as it stood before this fix): "gc retired 0 pack(s), not
2013    /// the one whose objects all died — left: 0, right: 1". Restored.
2014    ///
2015    /// Seen RED a third time by computing deadness the way a retirement written
2016    /// *after* the drop would have to (`occupied[i]` → `!occupied[i]` in
2017    /// `retire_dead_packs`): the same "gc retired 0 pack(s)" — after the drop an
2018    /// all-dead pack is indistinguishable from an unabsorbed one, which is the
2019    /// same confusion this whole change is about, one level up. Restored.
2020    ///
2021    /// And the derived half, seen RED by removing the `refold()` from
2022    /// `drop_dead_rows`: "the graph still holds 551 commits after gc dropped
2023    /// every dead one — left: 551, right: 1", with `reachable()` still selecting
2024    /// objects the index can no longer serve. Restored.
2025    #[test]
2026    fn a_wholly_dead_pack_does_not_come_back_when_the_store_is_reopened() {
2027        let DoomedFixture {
2028            dir,
2029            store,
2030            doomed: doomed_oid,
2031            root: root_raw,
2032            tree,
2033        } = one_live_pack_and_one_doomed_pack("gc-wholly-dead");
2034
2035        let (rows_after_gc, retired_by_gc) = {
2036            let report = store.gc().unwrap();
2037            assert_eq!(
2038                report.retired_packs, 1,
2039                "gc retired {} pack(s), not the one whose objects all died",
2040                report.retired_packs
2041            );
2042            assert!(
2043                !store.has(&doomed_oid).unwrap(),
2044                "the dead object still resolves in the store that dropped it"
2045            );
2046            assert!(store.has(&root_raw).unwrap(), "gc dropped the live commit");
2047
2048            // **Everything the indexer derived is dropped with the rows, in the
2049            // same process** — not merely rebuilt by the next open. The graph
2050            // holds one commit, and a selection over the bitmaps and the ordinal
2051            // space cannot name a dead object.
2052            assert_eq!(
2053                store.commit_count(),
2054                1,
2055                "the graph still holds {} commits after gc dropped every dead one",
2056                store.commit_count()
2057            );
2058            let selected = store.reachable(&[&root_raw], &[]).unwrap();
2059            assert!(
2060                !selected.contains(&doomed_oid),
2061                "a selection after gc still names a dead object — the bitmaps or the ordinal \
2062                 space were not refolded"
2063            );
2064            assert_eq!(
2065                selected.len(),
2066                store.index().len(),
2067                "a selection after gc names {} objects and the index holds {}",
2068                selected.len(),
2069                store.index().len()
2070            );
2071            (store.index().len(), report.retired_packs)
2072        };
2073        drop(store);
2074
2075        // The tombstone is on disk, and it names the doomed pack's offset — not
2076        // the live one's.
2077        let (packs, retired) = journal_state(&dir);
2078        assert_eq!(packs.len(), 2, "both packs are still acked: {packs:?}");
2079        assert_eq!(
2080            retired,
2081            vec![packs[1].0],
2082            "the journal retired {retired:?}, and the doomed pack starts at {}",
2083            packs[1].0
2084        );
2085        assert_eq!(retired.len() as u64, retired_by_gc);
2086
2087        // ── the reopen: the whole point ──────────────────────────────────────
2088        let store = GitStore::open(&dir, "rickard").unwrap();
2089        assert!(
2090            !store.has(&doomed_oid).unwrap(),
2091            "a dead object came back the instant the store was reopened"
2092        );
2093        store.wait_indexed();
2094        assert!(
2095            !store.has(&doomed_oid).unwrap(),
2096            "the reopen re-queued the wholly dead pack and its objects came back"
2097        );
2098        assert_eq!(
2099            store.index().len(),
2100            rows_after_gc,
2101            "the dead objects came back on the reopen: the store that dropped them holds \
2102             {rows_after_gc} rows, the reopened one holds {}",
2103            store.index().len()
2104        );
2105        assert_eq!(
2106            store.unindexed_packs(),
2107            0,
2108            "the retired pack is queued as work"
2109        );
2110        assert_eq!(
2111            store.absorb_pending().unwrap(),
2112            0,
2113            "a read falling back would re-absorb the retired pack"
2114        );
2115        // And the live side of the same repository is untouched by all of it.
2116        assert!(store.has(&root_raw).unwrap(), "the live commit is gone");
2117        assert!(store.has(&tree).unwrap(), "the live tree is gone");
2118        assert_eq!(
2119            store.commit_count(),
2120            1,
2121            "the graph after gc + reopen holds {} commits, not the one live one",
2122            store.commit_count()
2123        );
2124    }
2125
2126    /// **A partly dead pack is not retired, and comes through a GC and a reopen
2127    /// with its live objects.**
2128    ///
2129    /// This is the case that already worked and must keep working: the pack still
2130    /// has rows, so the crash-recovery diff calls it absorbed and nothing about
2131    /// it changes. Asserted on applied output: **no** tombstone appears in the
2132    /// journal, the live objects read back after the reopen, and the dead ones
2133    /// stay dead.
2134    ///
2135    /// Seen RED by retiring on `has_live[i]` instead of `!has_live[i]` in
2136    /// `retire_dead_packs` — i.e. tombstoning the packs that are *alive*: "gc
2137    /// retired the partly dead pack — its live objects are one reopen from a
2138    /// fallback that will never come — left: 1, right: 0", and the journal
2139    /// assertion below holds the same finding in on-disk bytes. The live objects
2140    /// still read back under that mutation, because a pack with rows is adopted
2141    /// as absorbed whatever the journal says about it — which is exactly why this
2142    /// guard asserts on the journal and not only on the reads. Restored.
2143    #[test]
2144    fn a_partly_dead_pack_is_never_retired_and_survives_a_reopen() {
2145        let dir = tmpdir("gc-partly-dead");
2146        let (pack, _) = real_pack();
2147        let store = GitStore::open(&dir, "rickard").unwrap();
2148        store.put(&pack, &[]).unwrap();
2149        store.absorb_pending().unwrap();
2150
2151        let root = store
2152            .graph_snapshot()
2153            .into_iter()
2154            .find(|c| c.generation == 1)
2155            .expect("a root commit");
2156        let root_raw = hex::decode(&root.oid).unwrap();
2157        let live: std::collections::HashSet<Vec<u8>> = store
2158            .reachable(&[&root_raw], &[])
2159            .unwrap()
2160            .into_iter()
2161            .collect();
2162        let all: Vec<Vec<u8>> = store.index().oids_in_order().unwrap();
2163        let dead = all
2164            .iter()
2165            .find(|o| !live.contains(*o))
2166            .expect("a real pack holds more than one commit's closure")
2167            .clone();
2168        assert!(
2169            live.len() < all.len(),
2170            "the fixture pack is not partly dead: {} live of {}",
2171            live.len(),
2172            all.len()
2173        );
2174        store
2175            .update_ref("refs/heads/root", None, Some(&root_raw))
2176            .unwrap();
2177        let files = vec![("pack-0.pack".to_string(), pack.clone())];
2178        znippy_common::create_archive(store.archive_path(), &files, 3).unwrap();
2179
2180        let report = store.gc().unwrap();
2181        assert_eq!(
2182            report.retired_packs, 0,
2183            "gc retired the partly dead pack — its live objects are one reopen from a fallback \
2184             that will never come"
2185        );
2186        let rows_after_gc = store.index().len();
2187        drop(store);
2188
2189        let (packs, retired) = journal_state(&dir);
2190        assert_eq!(packs.len(), 1);
2191        assert!(
2192            retired.is_empty(),
2193            "gc tombstoned a partly dead pack: {retired:?}"
2194        );
2195
2196        let store = GitStore::open(&dir, "rickard").unwrap();
2197        store.wait_indexed();
2198        assert_eq!(
2199            store.index().len(),
2200            rows_after_gc,
2201            "the reopen changed the row count of a partly dead pack's repository"
2202        );
2203        for oid in &live {
2204            assert!(
2205                store.has(oid).unwrap(),
2206                "the live object {} did not survive gc + reopen",
2207                hex::encode(oid)
2208            );
2209        }
2210        assert!(
2211            !store.has(&dead).unwrap(),
2212            "a dead object came back on the reopen of a partly dead pack"
2213        );
2214        // The pack was adopted as absorbed rather than re-queued: no work is
2215        // owed, and a read falling back would re-absorb it.
2216        assert_eq!(
2217            store.unindexed_packs(),
2218            0,
2219            "the surviving pack was re-queued"
2220        );
2221        assert_eq!(store.absorb_pending().unwrap(), 0);
2222    }
2223
2224    /// **An interruption at each step of the retirement leaves a readable
2225    /// archive and loses nothing.**
2226    ///
2227    /// The two steps are run as `GitOps::gc` runs them and the process boundary
2228    /// is a real one — the store is dropped and reopened between them, which is
2229    /// what a `kill -9` there would leave, because everything each step writes is
2230    /// fsynced before it returns.
2231    ///
2232    /// | killed after | on disk | what must be true |
2233    /// |---|---|---|
2234    /// | the tombstone | rows still there | **nothing is lost** — every object still reads, dead ones included |
2235    /// | the drop | rows gone, no compaction | the dead objects are gone and **stay** gone |
2236    /// | the compaction | a new generation | `gc.rs`'s own `an_interruption_at_every_step_leaves_a_readable_archive` |
2237    ///
2238    /// The first row is why the tombstone goes **first**: a GC killed there is a
2239    /// GC that did not happen, and the repository is exactly as it was. The
2240    /// reverse order has a window where the rows are gone and the journal still
2241    /// claims an unabsorbed pack, which is the resurrection this whole change is
2242    /// about.
2243    ///
2244    /// The archive itself is read back through the ordinary reader at every
2245    /// checkpoint, so "readable" is the whole file decoding, not a stat.
2246    ///
2247    /// Seen RED by computing deadness from `!occupied[i]` in `retire_dead_packs`
2248    /// — the shape a retirement written after the drop needs, which cannot see a
2249    /// pack that still has its rows: "the doomed pack was not retired: [] —
2250    /// left: 0, right: 1", at the first checkpoint, with every row still in the
2251    /// index. Restored.
2252    #[test]
2253    fn an_interruption_at_each_step_of_the_retirement_loses_nothing() {
2254        let DoomedFixture {
2255            dir,
2256            store,
2257            doomed: doomed_oid,
2258            root: root_raw,
2259            tree,
2260        } = one_live_pack_and_one_doomed_pack("gc-interrupt");
2261        let archive = dir.join("repository.znippy");
2262        let readable = |where_: &str| {
2263            crate::gc::read_back_every_entry(&archive)
2264                .unwrap_or_else(|e| panic!("the archive does not read back {where_}: {e:?}"));
2265        };
2266        readable("before anything ran");
2267
2268        // ── killed right after the tombstone, before any row was dropped ─────
2269        //
2270        // The live set is computed once and carried, which is what `gc()` does
2271        // with it too: it is an input to both steps, and computing it twice would
2272        // make this guard depend on a reachability fold rather than on the two
2273        // mutations it is here to interrupt.
2274        let (live, rows_before) = {
2275            let live = store.live_set().unwrap();
2276            let retired = store.retire_dead_packs(&live).unwrap();
2277            assert_eq!(
2278                retired.len(),
2279                1,
2280                "the doomed pack was not retired: {retired:?}"
2281            );
2282            let n = store.index().len();
2283            assert!(
2284                store.has(&doomed_oid).unwrap(),
2285                "retiring the journal row dropped an index row — the two steps are not separable"
2286            );
2287            (live, n)
2288        };
2289        drop(store);
2290        let (packs, on_disk) = journal_state(&dir);
2291        assert_eq!(
2292            on_disk,
2293            vec![packs[1].0],
2294            "the tombstone is not on disk before a single row was dropped: {on_disk:?}"
2295        );
2296        readable("after the tombstone");
2297
2298        {
2299            // The reopen a crash there produces: everything is still here.
2300            let store = GitStore::open(&dir, "rickard").unwrap();
2301            store.wait_indexed();
2302            assert_eq!(
2303                store.index().len(),
2304                rows_before,
2305                "a crash between the tombstone and the drop lost rows"
2306            );
2307            assert!(
2308                store.has(&doomed_oid).unwrap(),
2309                "a crash between the tombstone and the drop lost the objects of the pack it \
2310                 retired — the GC had not decided anything yet"
2311            );
2312            assert!(store.has(&root_raw).unwrap() && store.has(&tree).unwrap());
2313            // And resuming is idempotent: the pack is already retired.
2314            assert!(
2315                store.retire_dead_packs(&live).unwrap().is_empty(),
2316                "the resumed GC retired the same pack twice"
2317            );
2318        }
2319
2320        // ── killed after the drop, before the compaction ─────────────────────
2321        let rows_after_drop = {
2322            let store = GitStore::open(&dir, "rickard").unwrap();
2323            let dropped = store.drop_dead_rows(&live).unwrap();
2324            assert!(dropped > 0, "the drop dropped nothing");
2325            assert!(!store.has(&doomed_oid).unwrap());
2326            store.index().len()
2327        };
2328        readable("after the drop, before the compaction");
2329        let (_, still_retired) = journal_state(&dir);
2330        assert_eq!(still_retired.len(), 1, "the drop lost the tombstone");
2331
2332        {
2333            let store = GitStore::open(&dir, "rickard").unwrap();
2334            store.wait_indexed();
2335            assert!(
2336                !store.has(&doomed_oid).unwrap(),
2337                "a crash between the drop and the compaction resurrected the dead objects"
2338            );
2339            assert_eq!(store.index().len(), rows_after_drop);
2340            assert!(store.has(&root_raw).unwrap() && store.has(&tree).unwrap());
2341
2342            // ── and the compaction still completes on the resumed store ──────
2343            let report = store.gc().unwrap();
2344            assert_eq!(
2345                report.retired_packs, 0,
2346                "the resumed gc retired an already-retired pack"
2347            );
2348            assert!(
2349                report.verified,
2350                "the resumed gc did not verify its generation"
2351            );
2352            assert!(report.archive.exists());
2353            crate::gc::read_back_every_entry(&report.archive)
2354                .expect("the new generation does not read back");
2355            assert!(!store.has(&doomed_oid).unwrap());
2356            assert!(store.has(&root_raw).unwrap());
2357        }
2358    }
2359
2360    /// `compact` leaves its source untouched and refuses to overwrite.
2361    ///
2362    /// Seen RED by removing the `dst.exists()` refusal: "the refusal must be the
2363    /// explicit one and must name the file: hard-linking …/repo.znippy to
2364    /// …/repo.g1.znippy — compaction runs against a second name for the same
2365    /// inode …". The first version of this guard only asserted *that* it failed,
2366    /// and that version stayed green under the same mutation, because `hard_link`
2367    /// refuses EEXIST as well. It now asserts which mechanism refused.
2368    #[test]
2369    fn compact_names_its_destination_and_never_overwrites() {
2370        let dir = tmpdir("compact");
2371        let src = dir.join("repo.znippy");
2372        let dst = dir.join("repo.g1.znippy");
2373        let files = vec![
2374            ("pack-0.pack".to_string(), vec![7u8; 400_000]),
2375            ("pack-1.pack".to_string(), vec![9u8; 400_000]),
2376        ];
2377        znippy_common::create_archive(&src, &files, 3).unwrap();
2378        let before = std::fs::read(&src).unwrap();
2379
2380        compact(&src, &dst).unwrap();
2381        assert!(dst.exists(), "compact produced no destination");
2382        assert_eq!(
2383            std::fs::read(&src).unwrap(),
2384            before,
2385            "compact modified its source"
2386        );
2387
2388        let err = compact(&src, &dst).expect_err("dst exists");
2389        // Asserted on WHICH mechanism refused, not merely that something did:
2390        // `hard_link` would fail with EEXIST too, so a guard that only checked
2391        // for an error could not tell the explicit refusal from the syscall's —
2392        // and the difference is whether a filesystem operation is attempted
2393        // against a file that must not be touched at all.
2394        assert!(
2395            err.to_string().contains("refusing to compact over it")
2396                && err.to_string().contains(&dst.display().to_string()),
2397            "the refusal must be the explicit one and must name the file: {err}"
2398        );
2399        assert_eq!(
2400            std::fs::read(&src).unwrap(),
2401            before,
2402            "the refused second compaction touched the source"
2403        );
2404    }
2405}
2406
2407#[cfg(test)]
2408mod emit_set_tests {
2409    use crate::index_layout::ObjectIndex as _;
2410    use crate::pack_walk::topological_order;
2411    use crate::store::tests::{real_pack, tmpdir};
2412    use crate::{GitOps, GitStore};
2413
2414    /// **A SUBSET emits a pack holding EXACTLY the subset, and stock git accepts
2415    /// it.**
2416    ///
2417    /// Emitting a whole pack proves the encoder; it cannot prove anything about
2418    /// a request whose delta bases fall outside it, because every base is
2419    /// present by construction. A subset is that case.
2420    ///
2421    /// # What this asserted until 2026-08-11, and why it inverted
2422    ///
2423    /// It asserted `entries.len() >= half.len()` under the heading *"closure
2424    /// pulled in nothing"* — the set was expected to come back **larger**,
2425    /// because `emit_set` added the delta bases. That is the defect: a base
2426    /// pulled into a narrowed request is an object the client did not ask for,
2427    /// and a tree pulled in that way owes children the pack does not carry. See
2428    /// [`GitStore::emit_set`] for the production failure and the counts.
2429    ///
2430    /// So the assertion is now an equality, and it is the strong direction: the
2431    /// emitted set is the requested set, entry for entry. The delta whose base
2432    /// was left out is emitted **whole**, which is what `pack-objects` does and
2433    /// what keeps the pack connected.
2434    #[test]
2435    fn a_subset_emits_exactly_the_subset_and_stock_git_accepts_it() {
2436        if std::process::Command::new("git")
2437            .arg("--version")
2438            .output()
2439            .is_err()
2440        {
2441            eprintln!("skipping: no git on PATH");
2442            return;
2443        }
2444        let dir = tmpdir("emit-subset");
2445        let store = GitStore::open(&dir, "rickard").unwrap();
2446        let (pack, _) = real_pack();
2447        store.put_pack(&pack).unwrap();
2448        store.absorb_pending().unwrap();
2449
2450        let all = store.index().oids_in_order().unwrap();
2451        assert!(all.len() > 8, "corpus too small to take a subset of");
2452        // Every other object: enough deltas land with their base excluded that
2453        // the boundary rule has real work to do.
2454        let half: Vec<&[u8]> = all.iter().step_by(2).map(Vec::as_slice).collect();
2455        let rows = store.index().lookup_batch(&half);
2456        let inside: std::collections::HashSet<u64> =
2457            rows.iter().flatten().map(|r| r.offset).collect();
2458        let boundary = rows
2459            .iter()
2460            .flatten()
2461            .filter(|r| r.delta_base != 0 && !inside.contains(&r.delta_base))
2462            .count();
2463        assert!(
2464            boundary > 0,
2465            "this subset excludes no delta base, so it cannot tell an exact emission from a \
2466             closed one and the test proves nothing"
2467        );
2468
2469        let entries = store.emit_set(&half, true, None).unwrap();
2470        assert_eq!(
2471            entries.len(),
2472            half.len(),
2473            "the emitted set must be the requested set — {} asked, {} emitted",
2474            half.len(),
2475            entries.len()
2476        );
2477        assert_eq!(
2478            entries.iter().filter(|e| e.recompressed).count(),
2479            boundary,
2480            "exactly the entries whose base was excluded must be rebuilt whole"
2481        );
2482
2483        let (ordered, missing) = topological_order(entries);
2484        assert!(
2485            missing.is_empty(),
2486            "no entry may still name a base that is not here: {missing:?}"
2487        );
2488
2489        // Walk what we emitted, before handing it to git. If the header count
2490        // and the body disagree, that is the shape that crashes an indexer.
2491        {
2492            let mut b = Vec::new();
2493            store.emit_ordered(&ordered, &mut b).unwrap();
2494            let declared = u32::from_be_bytes([b[8], b[9], b[10], b[11]]);
2495            match crate::pack_walk::walk(&b, store.hash_kind().oid_len()) {
2496                Ok(w) => assert_eq!(
2497                    w.entries.len(),
2498                    declared as usize,
2499                    "the pack header declares {declared} entries and the body walks to {}",
2500                    w.entries.len()
2501                ),
2502                Err(e) => panic!("our own walk cannot read what we emitted: {e:#}"),
2503            }
2504        }
2505
2506        let mut bytes = Vec::new();
2507        let report = store.emit_ordered(&ordered, &mut bytes).unwrap();
2508        assert_eq!(
2509            report.copied + report.recompressed,
2510            report.written,
2511            "every entry is either copied or rebuilt, and the receipt must add up"
2512        );
2513        assert_eq!(
2514            report.recompressed as usize, boundary,
2515            "only the excluded-base entries may be rebuilt"
2516        );
2517
2518        // **No `--strict` here, and that is deliberate.** An arbitrary every-other
2519        // subset is not reachability-closed — a tree in it will name a blob that
2520        // is not — so `--strict`'s connectivity walk would be judging the
2521        // *caller's* selection, not this emitter. What is being asserted is
2522        // SELF-CONTAINMENT: that every delta in the pack resolves inside it,
2523        // which plain `index-pack` answers with `pack has N unresolved deltas`.
2524        // Connectivity has its own test, on a request that is closed:
2525        // `a_narrowed_clone_is_served_a_connected_pack_not_its_delta_bases`.
2526        let out_dir = tmpdir("emit-subset-idx");
2527        crate::git_oracle::assert_git_accepts(
2528            &out_dir,
2529            "dst.git",
2530            &bytes,
2531            crate::git_oracle::Strictness::SelfContained,
2532        );
2533    }
2534
2535    /// 🔴 **An oid this repository does not hold is REFUSED by name — it is not
2536    /// quietly left out of the pack.**
2537    ///
2538    /// `emit_set` used to `continue` past a `lookup_batch` miss. That is the
2539    /// silent-under-send shape: the request asks for N objects, the pack carries
2540    /// N-1, `PackStats` reports success and the *client* is the first to notice.
2541    /// It also made the two engines behind one contract disagree — gunnar's
2542    /// in-memory arm has always refused with *"emit_pack was asked for X, which
2543    /// this store does not hold"* — so a benchmark across the two was comparing
2544    /// two different contracts.
2545    ///
2546    /// # Why the assertions are shaped this way
2547    ///
2548    /// **RED before the change** on the first one: `emit_set` returned `Ok` with
2549    /// two entries for three asked oids, so `expect_err` panicked.
2550    ///
2551    /// The error must **name the oid**, because an operator reading
2552    /// `git.upload_pack.failed` needs the object, not the fact that something
2553    /// was missing; asserted on the rendered message.
2554    ///
2555    /// And the refusal must be *conditional*, or a function that errored
2556    /// unconditionally would pass the first two: the same store, the same two
2557    /// real oids and no absent one, must still emit.
2558    #[test]
2559    fn an_oid_the_store_does_not_hold_is_refused_by_name_not_skipped() {
2560        let dir = tmpdir("emit-missing");
2561        let store = GitStore::open(&dir, "rickard").unwrap();
2562        let (pack, _) = real_pack();
2563        store.put_pack(&pack).unwrap();
2564        store.absorb_pending().unwrap();
2565
2566        let all = store.index().oids_in_order().unwrap();
2567        assert!(all.len() > 2, "corpus too small");
2568        // A well-formed oid of the right width that this store cannot hold: it
2569        // is not in the index, and asking for it is what a caller with a stale
2570        // set does.
2571        let absent = vec![0xABu8; store.hash_kind().oid_len()];
2572        assert!(
2573            store.index().lookup(&absent).is_none(),
2574            "the fixture oid must really be absent or this test asserts nothing"
2575        );
2576
2577        let asked: Vec<&[u8]> = vec![all[0].as_slice(), absent.as_slice(), all[1].as_slice()];
2578        let err = store
2579            .emit_set(&asked, true, None)
2580            .expect_err("a pack for an object the store lacks must be refused, not shortened");
2581        let msg = format!("{err:#}");
2582        assert!(
2583            msg.contains(&hex::encode(&absent)),
2584            "the refusal must name the oid it could not find; got: {msg}"
2585        );
2586
2587        // Conditional, not unconditional: drop the absent one and the same call
2588        // emits both objects.
2589        let present: Vec<&[u8]> = vec![all[0].as_slice(), all[1].as_slice()];
2590        let entries = store
2591            .emit_set(&present, true, None)
2592            .expect("two objects this store does hold must still emit");
2593        assert_eq!(entries.len(), 2);
2594    }
2595}
2596
2597/// **The zero-copy emit path: the same bytes, and none of the memory.**
2598///
2599/// [`crate::pack_walk::EntryBytes`] turned `EmitEntry.stored` from a `Vec<u8>`
2600/// that owned a `pread`ed copy of the entry into a 16-byte address resolved
2601/// against a mapping of the archive. That is a change to *where the bytes live*
2602/// and to **nothing else**, so the guards here are of two kinds: three that
2603/// require the emitted bytes to be unchanged, and one that requires the memory
2604/// to be gone. Either kind alone would pass on a broken change — an emitter that
2605/// held nothing and wrote garbage, or one that wrote perfectly and still held
2606/// the repository.
2607#[cfg(test)]
2608mod zero_copy_tests {
2609    use crate::index_layout::{ObjType, ObjectIndex as _};
2610    use crate::pack_walk::{EmitEntry, EntryBytes, topological_order};
2611    use crate::store::tests::{real_pack, tmpdir};
2612    use crate::{GitOps, GitStore};
2613
2614    /// A store holding the whole corpus pack, and every oid in it.
2615    fn corpus(name: &str) -> (std::path::PathBuf, GitStore, Vec<Vec<u8>>) {
2616        let dir = tmpdir(name);
2617        let store = GitStore::open(&dir, "rickard").unwrap();
2618        let (pack, _) = real_pack();
2619        store.put_pack(&pack).unwrap();
2620        store.absorb_pending().unwrap();
2621        let all = store.index().oids_in_order().unwrap();
2622        assert!(all.len() > 100, "the corpus is too small to prove anything");
2623        (dir, store, all)
2624    }
2625
2626    /// The **old** shape of an emit set: every entry's bytes `pread` into an
2627    /// owned `Vec`, exactly as `emit_set` built them before 2026-08-14. The
2628    /// differential guards below emit this and the extent form and require the
2629    /// two packs to be identical.
2630    ///
2631    /// It reads through [`GitStore::read_extent`] — the `pread` path — so it
2632    /// shares no code with the mapping it is being compared against. A helper
2633    /// that resolved through `Mapped::get` would be comparing the change with
2634    /// itself.
2635    fn materialised(store: &GitStore, entries: &[EmitEntry]) -> Vec<EmitEntry> {
2636        entries
2637            .iter()
2638            .map(|e| {
2639                let mut e = e.clone();
2640                if let EntryBytes::Extent { offset, len } = e.stored {
2641                    e.stored = EntryBytes::Owned(store.read_extent(offset, len).unwrap());
2642                }
2643                e
2644            })
2645            .collect()
2646    }
2647
2648    /// 🔴 **A full clone's pack is byte-identical before and after the change.**
2649    ///
2650    /// The strongest guard available, and the one the whole change stands on: a
2651    /// clone is built twice out of the same selection — once with every entry an
2652    /// `EntryBytes::Extent` resolved through the mapping, once with every entry
2653    /// `pread` into an owned `Vec` the way `emit_set` used to build it — and the
2654    /// two packs must be the same bytes. Not the same length, not the same
2655    /// object count: the same **bytes**, trailer included, which for a pack
2656    /// means the same headers, the same `OFS_DELTA` distances and the same sha1.
2657    ///
2658    /// A whole-repository selection, so `recompressed` is 0 and every one of the
2659    /// entries really is an extent — asserted, because a run in which they were
2660    /// all `Owned` would compare the old path with itself and pass vacuously.
2661    ///
2662    /// Seen RED by returning `snap.get(offset, len - 1)` from
2663    /// [`GitStore::extent`] — a mapping that hands back a *short* slice rather
2664    /// than `None`, which is the single most plausible way to get this wrong:
2665    /// "the mapped emit and the `pread` emit disagree at byte 1944: 0xe8 vs
2666    /// 0x3a (mapped 5651349 bytes, pread 5654037 bytes)".
2667    ///
2668    /// Seen RED a second way, by slicing `snap.get(offset + 1, len)` — an
2669    /// extent addressed one byte off: "resolving the emit set's stored bytes:
2670    /// resolving the stored bytes of 00065363b6d5f3edb20c8c17a5938503fcdcc941
2671    /// for emission: object type code 0 is not one git writes — refusing to
2672    /// guess it". That is phase 1's header parse refusing before phase 2 had
2673    /// written a byte of the pack, which is why the parse is there.
2674    #[test]
2675    fn a_full_clone_emits_the_identical_pack_from_extents_and_from_owned_bytes() {
2676        let (_dir, store, all) = corpus("zc-identical");
2677        let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
2678
2679        let entries = store.emit_set(&oids, true, None).unwrap();
2680        let extents = entries
2681            .iter()
2682            .filter(|e| matches!(e.stored, EntryBytes::Extent { .. }))
2683            .count();
2684        assert_eq!(
2685            extents,
2686            entries.len(),
2687            "a whole-repository selection must be ALL extents — {} of {} were owned, so this \
2688             comparison would be the old path against itself",
2689            entries.len() - extents,
2690            entries.len()
2691        );
2692
2693        let owned = materialised(&store, &entries);
2694        let (ordered_a, missing) = topological_order(entries);
2695        assert!(missing.is_empty(), "a full clone is closed: {missing:?}");
2696        let (ordered_b, _) = topological_order(owned);
2697
2698        let mut mapped_pack = Vec::new();
2699        let report = store.emit_ordered(&ordered_a, &mut mapped_pack).unwrap();
2700        let mut pread_pack = Vec::new();
2701        store.emit_ordered(&ordered_b, &mut pread_pack).unwrap();
2702
2703        assert_eq!(report.recompressed, 0, "a full clone re-deflates nothing");
2704        assert_eq!(report.copied, report.written);
2705        assert!(mapped_pack.len() > 1_000_000, "the corpus pack is tiny?");
2706        if mapped_pack != pread_pack {
2707            let at = mapped_pack
2708                .iter()
2709                .zip(pread_pack.iter())
2710                .position(|(a, b)| a != b);
2711            let i = at.unwrap_or(mapped_pack.len().min(pread_pack.len()));
2712            panic!(
2713                "the mapped emit and the `pread` emit disagree at byte {i}: {:#04x} vs {:#04x} \
2714                 (mapped {} bytes, pread {} bytes)",
2715                mapped_pack.get(i).copied().unwrap_or(0),
2716                pread_pack.get(i).copied().unwrap_or(0),
2717                mapped_pack.len(),
2718                pread_pack.len()
2719            );
2720        }
2721    }
2722
2723    /// 🔴 **Every `OFS_DELTA`'s base distance still lands on its base.**
2724    ///
2725    /// The one thing phase 2 may not be parallelised around, asserted as applied
2726    /// output rather than as a receipt: the emitted pack is walked back, and for
2727    /// every `OfsDelta` entry the base its distance names must be an entry
2728    /// boundary in the same pack — which `PackWalk::closure` answers, and which
2729    /// no amount of correct-looking header encoding can satisfy by accident.
2730    /// `git index-pack` agrees separately in the tests beside this one; this is
2731    /// the guard that says *which* thing broke when it stops agreeing.
2732    ///
2733    /// The count of `OfsDelta` entries is asserted first. A pack with none would
2734    /// pass this vacuously, and a corpus that happened to hold none would make
2735    /// the whole of phase 2's serial argument untestable.
2736    ///
2737    /// Seen RED by encoding the distance as `here - base_at + 1` in
2738    /// `emit_pack`: "1345 of 1345 ofs-deltas name a base that is not an entry
2739    /// boundary — first at output offset 11". Seen RED a second, sharper way by
2740    /// emitting `emit_set`'s entries in their input order rather than
2741    /// `topological_order`'s: "entry at 3414738 deltas against archive offset
2742    /// 3414009, which is not in this pack — the set is not closed and was not
2743    /// ordered by `topological_order`".
2744    #[test]
2745    fn an_ofs_delta_still_names_its_base_by_the_right_distance() {
2746        let (_dir, store, all) = corpus("zc-ofs");
2747        let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
2748        let (ordered, _) = topological_order(store.emit_set(&oids, true, None).unwrap());
2749
2750        let mut pack = Vec::new();
2751        store.emit_ordered(&ordered, &mut pack).unwrap();
2752
2753        let walk = crate::pack_walk::walk(&pack, store.hash_kind().oid_len())
2754            .expect("our own walk must read what we emitted");
2755        let deltas = walk
2756            .entries
2757            .iter()
2758            .filter(|e| e.obj_type == ObjType::OfsDelta)
2759            .count();
2760        assert!(
2761            deltas > 0,
2762            "the corpus emitted no ofs-delta, so this proves nothing about distances"
2763        );
2764
2765        let closure = walk.closure();
2766        assert!(
2767            closure.broken_offsets.is_empty(),
2768            "{} of {deltas} ofs-deltas name a base that is not an entry boundary — first at \
2769             output offset {}",
2770            closure.broken_offsets.len(),
2771            closure.broken_offsets.first().copied().unwrap_or(0)
2772        );
2773        assert!(
2774            closure.external_refs.is_empty(),
2775            "a full clone must name no external base"
2776        );
2777    }
2778
2779    /// 🔴 **The `Owned` exception still carries a rebuilt delta's bytes.**
2780    ///
2781    /// `EntryBytes::Owned` is not vestigial: a delta whose base the request does
2782    /// not carry genuinely computes bytes that no extent addresses, and dropping
2783    /// the variant would have meant either shipping a delta the client cannot
2784    /// resolve or widening the request — the two failures
2785    /// `GitStore::emit_set`'s doc comment is written about.
2786    ///
2787    /// Asserted on applied output at both ends: the boundary entries are `Owned`
2788    /// and every other entry is an `Extent` (a mixed set, so the emitter is
2789    /// proved to handle both in one pack), and the resulting pack walks with
2790    /// its closure intact. The premise — that this subset really does cut delta
2791    /// chains — is asserted first.
2792    ///
2793    /// Seen RED by making the whole-rebuild arm of `emit_set` push
2794    /// `EntryBytes::Extent { offset: row.offset, len: row.len }` instead of the
2795    /// bytes it just built: "312 boundary entries are recompressed but 202 carry
2796    /// owned bytes — a rebuilt entry that points back at its stored extent ships
2797    /// the delta it was rebuilt to avoid". 202 and not 0, because the re-delta
2798    /// arm above it was untouched — which is what makes the count, and not a
2799    /// bare `is_some()`, the thing worth asserting.
2800    #[test]
2801    fn a_rebuilt_delta_still_carries_its_own_bytes_and_emits_beside_extents() {
2802        let (_dir, store, all) = corpus("zc-owned");
2803        let half: Vec<&[u8]> = all.iter().step_by(2).map(Vec::as_slice).collect();
2804
2805        let rows = store.index().lookup_batch(&half);
2806        let inside: std::collections::HashSet<u64> =
2807            rows.iter().flatten().map(|r| r.offset).collect();
2808        let boundary = rows
2809            .iter()
2810            .flatten()
2811            .filter(|r| r.delta_base != 0 && !inside.contains(&r.delta_base))
2812            .count();
2813        assert!(
2814            boundary > 0,
2815            "this subset cuts no delta chain, so the Owned path is never reached"
2816        );
2817
2818        let entries = store.emit_set(&half, true, None).unwrap();
2819        let rebuilt: Vec<&EmitEntry> = entries.iter().filter(|e| e.recompressed).collect();
2820        let with_bytes = rebuilt
2821            .iter()
2822            .filter(|e| e.stored.owned().is_some_and(|b| !b.is_empty()))
2823            .count();
2824        assert_eq!(
2825            with_bytes,
2826            rebuilt.len(),
2827            "{} boundary entries are recompressed but {with_bytes} carry owned bytes — a rebuilt \
2828             entry that points back at its stored extent ships the delta it was rebuilt to avoid",
2829            rebuilt.len()
2830        );
2831        assert_eq!(rebuilt.len(), boundary);
2832
2833        // The other half of the point: this is a MIXED pack, so the emitter is
2834        // resolving both variants in one pass.
2835        let as_extent = entries
2836            .iter()
2837            .filter(|e| matches!(e.stored, EntryBytes::Extent { .. }))
2838            .count();
2839        assert_eq!(
2840            as_extent,
2841            entries.len() - rebuilt.len(),
2842            "every entry that was NOT rebuilt must still be an extent"
2843        );
2844        assert!(
2845            as_extent > 0 && !rebuilt.is_empty(),
2846            "the set must be mixed"
2847        );
2848
2849        let (ordered, missing) = topological_order(entries);
2850        assert!(missing.is_empty(), "the emitted set is closed: {missing:?}");
2851        let mut pack = Vec::new();
2852        let report = store.emit_ordered(&ordered, &mut pack).unwrap();
2853        assert_eq!(report.recompressed as usize, boundary);
2854        let walk = crate::pack_walk::walk(&pack, store.hash_kind().oid_len())
2855            .expect("a mixed pack must still walk");
2856        assert!(
2857            walk.closure().broken_offsets.is_empty(),
2858            "a pack mixing extents and rebuilt bytes must still be self-contained"
2859        );
2860    }
2861
2862    /// 🔴 **An extent past the mapping falls back to `pread` and returns the
2863    /// SAME bytes.**
2864    ///
2865    /// `Mapped::get` answers `None` for an extent past the mapped end, which
2866    /// means *"go and pread it"* and never *"there are no bytes"*. The case is
2867    /// real and not hypothetical: a snapshot is taken once per emission and the
2868    /// blob file grows whenever a push lands, so any clone running across a push
2869    /// reads its tail this way.
2870    ///
2871    /// Constructed deliberately rather than raced: a snapshot is taken, a second
2872    /// pack is pushed, and the objects of that second pack are then emitted
2873    /// against the **stale** snapshot. Every one of their extents is past its
2874    /// end, so every one takes the fallback — asserted by `Mapped::get` refusing
2875    /// them, so a run where the snapshot happened to cover them cannot pass
2876    /// quietly.
2877    ///
2878    /// Seen RED by making [`GitStore::extent`]'s `None` arm
2879    /// `Ok(Cow::Owned(Vec::new()))` — the "absent means no bytes" misreading
2880    /// [`crate::archive_map::Mapped::get`]'s own doc comment forbids: "the
2881    /// fallback returned 0 bytes for the extent at (5653314, 43), which the
2882    /// mapping does not cover; `None` means pread, not empty".
2883    #[test]
2884    fn an_extent_past_the_mapping_falls_back_and_returns_the_same_bytes() {
2885        let (_dir, store, _all) = corpus("zc-fallback");
2886
2887        // The snapshot is taken NOW, before the second push.
2888        let stale = store.archive_snapshot().unwrap();
2889        let before = stale.len();
2890
2891        let (second, _) = crate::store::tests::one_blob_pack(b"a blob pushed after the snapshot\n");
2892        store.put_pack(&second).unwrap();
2893        store.absorb_pending().unwrap();
2894
2895        let after = store.archive_snapshot().unwrap().len();
2896        assert!(
2897            after > before,
2898            "the second push did not grow the blob file ({before} → {after}), so nothing is past \
2899             the stale mapping and this test proves nothing"
2900        );
2901
2902        // Every object of the second pack sits past the stale mapping's end.
2903        let fresh: Vec<Vec<u8>> = store
2904            .index()
2905            .oids_in_order()
2906            .unwrap()
2907            .into_iter()
2908            .filter(|oid| {
2909                store
2910                    .index()
2911                    .lookup(oid)
2912                    .is_some_and(|r| r.offset >= before)
2913            })
2914            .collect();
2915        assert!(
2916            !fresh.is_empty(),
2917            "no object landed past the stale mapping's end"
2918        );
2919
2920        for oid in &fresh {
2921            let row = store.index().lookup(oid).unwrap();
2922            assert!(
2923                stale.get(row.offset, row.len).is_none(),
2924                "the stale mapping must NOT cover ({}, {}) or the fallback is never taken",
2925                row.offset,
2926                row.len
2927            );
2928            let fell_back = store.extent(&stale, row.offset, row.len).unwrap();
2929            let preaded = store.read_extent(row.offset, row.len).unwrap();
2930            assert_eq!(
2931                fell_back.len(),
2932                preaded.len(),
2933                "the fallback returned {} bytes for the extent at ({}, {}), which the mapping does \
2934                 not cover; `None` means pread, not empty",
2935                fell_back.len(),
2936                row.offset,
2937                row.len
2938            );
2939            assert_eq!(
2940                fell_back.as_ref(),
2941                preaded.as_slice(),
2942                "the fallback bytes differ from the pread bytes at ({}, {})",
2943                row.offset,
2944                row.len
2945            );
2946            // A FRESH snapshot covers it, and agrees with both.
2947            let fresh_snap = store.archive_snapshot().unwrap();
2948            assert_eq!(
2949                fresh_snap.get(row.offset, row.len).unwrap(),
2950                preaded.as_slice(),
2951                "the remapped snapshot disagrees with the pread at ({}, {})",
2952                row.offset,
2953                row.len
2954            );
2955        }
2956        eprintln!(
2957            "{} object(s) resolved past a stale {before}-byte mapping and matched the pread",
2958            fresh.len()
2959        );
2960    }
2961
2962    /// 🔴 **The gatling fan-out resolves the identical bytes the serial pass
2963    /// does** — LAW 3's primitive, over data with no lock on it.
2964    ///
2965    /// Phase 1 is the one place the serving tier fans out, and the claim that
2966    /// makes it safe is a claim about the *data*: the blob file is append-only
2967    /// and `gc` truncates nothing, so N workers reading a snapshot of it share
2968    /// no mutable state and need no synchronisation. This asserts the observable
2969    /// consequence — the same set resolved on one thread and on four gives the
2970    /// same slices, in the same order, pointing at the same bytes.
2971    ///
2972    /// It calls
2973    /// [`resolve_with`](GitStore::resolve_with) directly rather than going
2974    /// through `resolve_emit_payloads`, because the production threshold is 4096
2975    /// entries and this corpus holds 2687: through the front door the gatling
2976    /// arm would never run and this test would be asserting the serial path
2977    /// against itself. The order matters as much as the content —
2978    /// `gatling_for_each` self-dispatches with no barrier, so a worker finishing
2979    /// unit 9 before unit 3 is normal, and an implementation that collected in
2980    /// **completion** order would produce a pack whose entries are shuffled and
2981    /// whose `OFS_DELTA` distances therefore point at the wrong objects.
2982    ///
2983    /// Seen RED by having the parallel arm return
2984    /// `gatling_for_each(n, workers, one).into_iter().rev().collect()` — the
2985    /// cheapest stand-in for "results not in index order": "the fan-out resolved
2986    /// entry 0 to different bytes than the serial pass did (2 workers): 46 bytes
2987    /// vs 1933, first differing at Some(0)".
2988    #[test]
2989    fn the_fan_out_resolves_the_identical_bytes_the_serial_pass_does() {
2990        let (_dir, store, all) = corpus("zc-gatling");
2991        let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
2992        let (ordered, _) = topological_order(store.emit_set(&oids, true, None).unwrap());
2993        assert!(
2994            ordered.len() > 1000,
2995            "too few entries to spread over workers"
2996        );
2997
2998        let snap = store.archive_snapshot().unwrap();
2999        let serial = store.resolve_with(&ordered, &snap, 1).unwrap();
3000        for workers in [2usize, 4, 8] {
3001            let parallel = store.resolve_with(&ordered, &snap, workers).unwrap();
3002            assert_eq!(
3003                parallel.len(),
3004                serial.len(),
3005                "the fan-out resolved {} entries against the serial pass's {} ({workers} workers)",
3006                parallel.len(),
3007                serial.len()
3008            );
3009            // Compared by hand rather than with `assert_eq!` on the slices: an
3010            // entry is kilobytes, and a failure that dumps two of them is a
3011            // failure nobody reads.
3012            for (i, (p, s)) in parallel.iter().zip(serial.iter()).enumerate() {
3013                if p.as_ref() != s.as_ref() {
3014                    let at = p.iter().zip(s.iter()).position(|(a, b)| a != b);
3015                    panic!(
3016                        "the fan-out resolved entry {i} to different bytes than the serial pass \
3017                         did ({workers} workers): {} bytes vs {}, first differing at {at:?}",
3018                        p.len(),
3019                        s.len()
3020                    );
3021                }
3022            }
3023        }
3024
3025        // And the pack itself: emitted off the fan-out, walked back, closed.
3026        let mut pack = Vec::new();
3027        store.emit_ordered(&ordered, &mut pack).unwrap();
3028        let walk = crate::pack_walk::walk(&pack, store.hash_kind().oid_len()).unwrap();
3029        assert_eq!(walk.entries.len(), ordered.len());
3030        assert!(walk.closure().broken_offsets.is_empty());
3031    }
3032
3033    /// 🔴 **The pack is never materialised: the whole emit set's heap is a
3034    /// small fraction of the pack it will write.**
3035    ///
3036    /// # What this asserts, and how completely
3037    ///
3038    /// The emit set's heap is **accounted exactly**, not sampled: an
3039    /// [`EmitEntry`] holds a `Vec<EmitEntry>` slot, an oid, and — for the
3040    /// `Owned` exception only — payload bytes. There is nothing else it can
3041    /// hold, so `slots + oids + owned` is the complete cost of the set, and it
3042    /// is compared against the bytes that set will emit. Before 2026-08-14 the
3043    /// `owned` term alone *was* the pack: every entry's bytes `pread` into a
3044    /// `Vec`, all of them live at once, because the set is built in full before
3045    /// [`crate::pack_walk::emit_pack`] writes a byte.
3046    ///
3047    /// Seen RED by reverting `emit_set`'s base-inside arm to
3048    /// `EntryBytes::Owned(self.read_extent(row.offset, row.len)?)`: "the emit
3049    /// set owns 5653270 bytes of payload — 100.0 % of the 5653270 bytes it will
3050    /// emit — which is the whole pack held in memory before the first byte goes
3051    /// out".
3052    ///
3053    /// # What it does NOT prove, stated plainly
3054    ///
3055    /// **It is not an RSS test, and an RSS test here would be theatre.** This
3056    /// began as one, asserting that `/proc/self/statm` grows by less than the
3057    /// pack across `emit_set`. It was **blind**: run against the broken version
3058    /// above, which really does allocate and fill 5 653 270 bytes, resident set
3059    /// size did not move by a single page — 203 366 400 before and after, and
3060    /// 203 182 080 → 203 182 080 in the green run. The allocator was handing
3061    /// back arena pages this test process already had resident. A guard that
3062    /// cannot tell the defect from the fix is not a weak guard, it is no guard,
3063    /// and it is exactly what LAW 2 says to expect of one's own new guards. It
3064    /// is gone; the accounting above replaces it.
3065    ///
3066    /// **It says nothing about the peak of a real clone.** This corpus is 5.65 MB
3067    /// against `linux.git`'s ~6.4 GB, and the 2314 MB peak this change is aimed
3068    /// at was measured with `perf` and `/proc`, not with a unit test.
3069    ///
3070    /// **The residual it exposes is real and is printed rather than hidden.**
3071    /// The `slots + oids` term is `O(objects)` and survives this change: every
3072    /// entry still carries its own `Vec<u8>` oid, one heap allocation each. The
3073    /// printed projection to 13.8 M objects is the honest size of what is left
3074    /// to do, and it is not small.
3075    #[test]
3076    fn an_emit_set_holds_no_payload_bytes_of_the_pack_it_will_write() {
3077        let (_dir, store, all) = corpus("zc-memory");
3078        let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
3079
3080        let entries = store.emit_set(&oids, true, None).unwrap();
3081
3082        let owned: u64 = entries
3083            .iter()
3084            .map(|e| e.stored.owned().map_or(0, |b| b.len() as u64))
3085            .sum();
3086        let will_emit: u64 = entries.iter().map(|e| e.stored.len()).sum();
3087        assert!(
3088            will_emit > 1_000_000,
3089            "the corpus emits {will_emit} bytes, too few to distinguish held from addressed"
3090        );
3091        assert_eq!(
3092            owned,
3093            0,
3094            "the emit set owns {owned} bytes of payload — {:.1} % of the {will_emit} bytes it will \
3095             emit — which is the whole pack held in memory before the first byte goes out",
3096            100.0 * owned as f64 / will_emit as f64
3097        );
3098
3099        // The complete heap of the set, term by term.
3100        let slots = (entries.len() * std::mem::size_of::<EmitEntry>()) as u64;
3101        let oid_bytes: u64 = entries.iter().map(|e| e.oid.len() as u64).sum();
3102        let held = slots + oid_bytes + owned;
3103        assert!(
3104            held < will_emit / 8,
3105            "the emit set holds {held} bytes ({:.1} % of the {will_emit}-byte pack): {slots} of \
3106             slots, {oid_bytes} of oids, {owned} of payload",
3107            100.0 * held as f64 / will_emit as f64
3108        );
3109        eprintln!(
3110            "{} entries: {slots} B slots + {oid_bytes} B oids + {owned} B payload = {held} B held \
3111             for a {will_emit} B pack ({:.1} %). EmitEntry is {} B. Projected to linux.git's \
3112             ~13.8 M objects that residual is ~{:.2} GB — the payload copy is gone, the per-entry \
3113             slot and oid are NOT.",
3114            entries.len(),
3115            100.0 * held as f64 / will_emit as f64,
3116            std::mem::size_of::<EmitEntry>(),
3117            13.8e6 * (std::mem::size_of::<EmitEntry>() as f64 + 20.0) / 1e9,
3118        );
3119    }
3120}
3121
3122/// **A `REF_DELTA` whose base is inside the same pack**, on both sides of the
3123/// store: ingesting one, and serving one.
3124///
3125/// # Why this shape and no other
3126///
3127/// It is not exotic and it is not hand-rolled. `git index-pack --fix-thin`
3128/// completes a pushed **thin** pack by *appending the base object to the pack*
3129/// and leaving the delta naming that base by oid — so the base lands **after**
3130/// its dependant, in the same pack, named the one way a pack can name something
3131/// that is not a position. Every repository that has been pushed to more than
3132/// once and not `gc`'d holds packs of this shape, which is to say: the ordinary
3133/// case, not a corner.
3134///
3135/// The fixture is therefore built by **git itself** (`pack-objects --thin` fed
3136/// to `index-pack --fix-thin`) rather than assembled here, because a hand-built
3137/// pack could only ever prove that this crate agrees with this crate.
3138///
3139/// Two defects met on it, one per direction:
3140///
3141/// | direction | defect | fix |
3142/// |---|---|---|
3143/// | in | `external_bases_exist` demanded every ref base be in the **store**, so a pack carrying its own base was refused | ask the pack too, and only when about to refuse |
3144/// | out | `emit_set` copied the stored `REF_DELTA` through, and **gitoxide cannot read one whose base is in the pack** | re-head it as an `OFS_DELTA` naming the same base by distance |
3145#[cfg(test)]
3146mod in_pack_ref_delta_tests {
3147    use crate::index_layout::ObjType;
3148    use crate::pack_walk::{DeltaBase, topological_order};
3149    use crate::store::tests::tmpdir;
3150    use crate::{GitOps, GitStore};
3151    use std::path::Path;
3152    use std::process::Command;
3153
3154    fn git(dir: &Path, args: &[&str]) -> String {
3155        let out = Command::new("git")
3156            .current_dir(dir)
3157            .args(args)
3158            .env("GIT_AUTHOR_NAME", "t")
3159            .env("GIT_AUTHOR_EMAIL", "t@t")
3160            .env("GIT_COMMITTER_NAME", "t")
3161            .env("GIT_COMMITTER_EMAIL", "t@t")
3162            .env("GIT_AUTHOR_DATE", "2020-01-01T00:00:00Z")
3163            .env("GIT_COMMITTER_DATE", "2020-01-01T00:00:00Z")
3164            .output()
3165            .unwrap_or_else(|e| panic!("running git {args:?}: {e}"));
3166        assert!(
3167            out.status.success(),
3168            "git {args:?} failed: {}",
3169            String::from_utf8_lossy(&out.stderr)
3170        );
3171        String::from_utf8_lossy(&out.stdout).trim().to_string()
3172    }
3173
3174    /// A pack **git wrote** that contains a `REF_DELTA` whose base is an entry
3175    /// of the same pack.
3176    ///
3177    /// `None` when there is no `git` on `PATH`, so the suite still runs where
3178    /// the oracle is absent rather than failing for the wrong reason.
3179    fn fix_thin_pack(scratch: &Path) -> Option<Vec<u8>> {
3180        if Command::new("git").arg("--version").output().is_err() {
3181            eprintln!("skipping: no git on PATH");
3182            return None;
3183        }
3184        let repo = scratch.join("src");
3185        std::fs::create_dir_all(&repo).unwrap();
3186        git(&repo, &["init", "-q", "--initial-branch=main"]);
3187
3188        // Big enough, and changed little enough, that `pack-objects` really does
3189        // deltify the second version against the first. A two-line file would be
3190        // stored whole and this fixture would prove nothing — which the premise
3191        // assertion below refuses to let happen silently.
3192        let lines: Vec<String> = (0..400)
3193            .map(|i| format!("line {i} {}", "x".repeat(20)))
3194            .collect();
3195        std::fs::write(repo.join("f.txt"), lines.join("\n")).unwrap();
3196        git(&repo, &["add", "f.txt"]);
3197        git(&repo, &["commit", "-q", "-m", "c1"]);
3198
3199        let mut changed = lines.clone();
3200        changed[10] = "CHANGED".to_string();
3201        changed.push("appended line".to_string());
3202        std::fs::write(repo.join("f.txt"), changed.join("\n")).unwrap();
3203        git(&repo, &["commit", "-q", "-a", "-m", "c2"]);
3204
3205        // A THIN pack: c2's objects only, deltified against c1's, which it does
3206        // not carry.
3207        let head = git(&repo, &["rev-parse", "HEAD"]);
3208        let parent = git(&repo, &["rev-parse", "HEAD~1"]);
3209        let thin = {
3210            use std::io::Write as _;
3211            let mut child = Command::new("git")
3212                .current_dir(&repo)
3213                .args(["pack-objects", "--thin", "--revs", "--stdout"])
3214                .stdin(std::process::Stdio::piped())
3215                .stdout(std::process::Stdio::piped())
3216                .stderr(std::process::Stdio::piped())
3217                .spawn()
3218                .unwrap();
3219            write!(child.stdin.take().unwrap(), "{head}\n^{parent}\n").unwrap();
3220            let out = child.wait_with_output().unwrap();
3221            assert!(out.status.success(), "git pack-objects --thin failed");
3222            out.stdout
3223        };
3224
3225        // `--fix-thin` appends the bases INTO the pack. The ref-deltas keep
3226        // naming them by oid, and now those oids ARE in the pack.
3227        let idx = scratch.join("fixed.idx");
3228        {
3229            use std::io::Write as _;
3230            let mut child = Command::new("git")
3231                .current_dir(&repo)
3232                .args([
3233                    "index-pack",
3234                    "--fix-thin",
3235                    "--stdin",
3236                    "-o",
3237                    idx.to_str().unwrap(),
3238                ])
3239                .stdin(std::process::Stdio::piped())
3240                .stdout(std::process::Stdio::piped())
3241                .stderr(std::process::Stdio::piped())
3242                .spawn()
3243                .unwrap();
3244            child.stdin.take().unwrap().write_all(&thin).unwrap();
3245            let out = child.wait_with_output().unwrap();
3246            assert!(
3247                out.status.success(),
3248                "git index-pack --fix-thin failed: {}",
3249                String::from_utf8_lossy(&out.stderr)
3250            );
3251        }
3252        let packdir = repo.join(".git/objects/pack");
3253        let pack = std::fs::read_dir(&packdir)
3254            .unwrap()
3255            .flatten()
3256            .map(|e| e.path())
3257            .find(|p| p.extension().is_some_and(|x| x == "pack"))
3258            .expect("git index-pack --fix-thin wrote no pack");
3259        Some(std::fs::read(pack).unwrap())
3260    }
3261
3262    /// The fixture is what it claims to be: at least one `REF_DELTA`, and every
3263    /// delta base resolvable **inside the pack alone**.
3264    ///
3265    /// Without this the two tests below could pass on a pack with no ref-delta
3266    /// in it at all, which is the vacuous green LAW 2 is about.
3267    fn assert_carries_its_own_ref_delta_base(pack: &[u8]) {
3268        let w = crate::pack_walk::walk(pack, 20).expect("git's own pack must walk");
3269        let refs = w
3270            .entries
3271            .iter()
3272            .filter(|e| e.obj_type == ObjType::RefDelta)
3273            .count();
3274        assert!(
3275            refs > 0,
3276            "the fixture has no ref-delta at all, so it cannot exercise either defect"
3277        );
3278        // Resolvable with NO external base source at all: every base a delta in
3279        // here names is carried in here.
3280        crate::resolve::resolve(
3281            pack,
3282            crate::GitHashKind::Sha1,
3283            0,
3284            &crate::resolve::NoBases,
3285        )
3286        .expect("the fixture must be self-contained, or `put_pack` is right to refuse it");
3287    }
3288
3289    /// 🔴 **znippy ingests a pack git wrote.**
3290    ///
3291    /// `pack_walk::closure()` reports every `REF_DELTA` base on `external_refs`
3292    /// — it cannot do otherwise, a walk knows no oids — and
3293    /// `external_bases_exist` demanded each one already be in the store. A pack
3294    /// carrying its own base was therefore refused, which is `git index-pack
3295    /// --fix-thin`'s ordinary output and `gunnar.multi_pack_serve`'s failure.
3296    ///
3297    /// Asserted on applied output, not on `Ok`: every object of the pack is
3298    /// afterwards *in the store's index*, so a `put_pack` that returned success
3299    /// having stored nothing would still be red.
3300    ///
3301    /// Seen RED by restoring the old check (refuse any ref base not already in
3302    /// the store):
3303    /// *"this pack deltas against 93c4207c152fa94c3978c75b136ff5a530ca16b6,
3304    /// which this repository does not have — the push is refused rather than
3305    /// stored with a dangling base"*.
3306    #[test]
3307    fn a_pack_carrying_its_own_ref_delta_base_is_ingested_not_refused() {
3308        let dir = tmpdir("inpack-put");
3309        let Some(pack) = fix_thin_pack(&dir) else {
3310            return;
3311        };
3312        assert_carries_its_own_ref_delta_base(&pack);
3313
3314        let store = GitStore::open(&dir.join("store"), "rickard").unwrap();
3315        store
3316            .put_pack(&pack)
3317            .expect("a self-contained pack git itself wrote must be accepted");
3318        store.absorb_pending().unwrap();
3319
3320        let walked = crate::pack_walk::walk(&pack, 20).unwrap();
3321        let stored = store.index().oids_in_order().unwrap();
3322        assert_eq!(
3323            stored.len(),
3324            walked.entries.len(),
3325            "the pack has {} entries and the store indexed {}",
3326            walked.entries.len(),
3327            stored.len()
3328        );
3329        // And specifically the base that used to be the refusal.
3330        let base = walked
3331            .entries
3332            .iter()
3333            .find_map(|e| match &e.delta_base {
3334                DeltaBase::Ref(oid) => Some(oid.clone()),
3335                _ => None,
3336            })
3337            .expect("the premise asserted there is one");
3338        assert!(
3339            store.has(&base).unwrap(),
3340            "the in-pack ref-delta base {} is not in the store after the push",
3341            hex::encode(&base)
3342        );
3343    }
3344
3345    /// 🔴 **A pack this server emits is consumed by `gunnar_client`.**
3346    ///
3347    /// `gunnar_client` resolves a fetch with gitoxide, and
3348    /// `gix_pack::data::input::LookupRefDeltaObjectsIter` reads every
3349    /// `OBJ_REF_DELTA` as naming an object *the receiver already has*: it
3350    /// consults the local object database and the bases it has already spliced
3351    /// in, and never the pack it is reading. A clone's target holds nothing, so
3352    /// one stored ref-delta copied through is fatal for the **whole** pack.
3353    ///
3354    /// The receiver here is `gix_object::find::Never` — an object database that
3355    /// holds nothing, which is exactly `open_or_init_bare` at the moment a clone
3356    /// starts — so this is the client's own code answering, not a paraphrase of
3357    /// it.
3358    ///
3359    /// Three assertions, and all three are needed:
3360    ///
3361    ///  1. **gitoxide accepts it**, with an empty base source. This is the bug.
3362    ///  2. **stock git still accepts it.** git accepted the *broken* pack too,
3363    ///     so a fix that satisfied gix and broke git would otherwise ship green.
3364    ///  3. **the object set is exactly what was asked for**, read back out of
3365    ///     stock git by oid. A re-head that dropped, duplicated or corrupted an
3366    ///     entry passes 1 and 2 — `index-pack` files an entry under the oid it
3367    ///     computes from that entry's own bytes, so a wrong object is a
3368    ///     *different* oid and only comparing the sets can see it.
3369    ///
3370    /// Seen RED, each separately — see the commit message for the exact texts.
3371    #[test]
3372    fn an_emitted_pack_names_no_base_inside_itself_by_oid() {
3373        let dir = tmpdir("inpack-emit");
3374        let Some(pack) = fix_thin_pack(&dir) else {
3375            return;
3376        };
3377        assert_carries_its_own_ref_delta_base(&pack);
3378
3379        let store = GitStore::open(&dir.join("store"), "rickard").unwrap();
3380        store.put_pack(&pack).unwrap();
3381        store.absorb_pending().unwrap();
3382
3383        let all = store.index().oids_in_order().unwrap();
3384        let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
3385        let (ordered, missing) = topological_order(store.emit_set(&oids, true, None).unwrap());
3386        assert!(
3387            missing.is_empty(),
3388            "an entry still names a base that is not here: {missing:?}"
3389        );
3390        let mut emitted = Vec::new();
3391        store.emit_ordered(&ordered, &mut emitted).unwrap();
3392
3393        // ── the premise: the emitted pack really does carry a delta, or nothing
3394        //    below distinguishes a fix from a pack of whole objects ────────────
3395        let w = crate::pack_walk::walk(&emitted, 20).expect("our own walk reads what we emitted");
3396        let deltas = w
3397            .entries
3398            .iter()
3399            .filter(|e| !matches!(e.delta_base, DeltaBase::None))
3400            .count();
3401        assert!(
3402            deltas > 0,
3403            "the emitted pack has no delta at all, so it cannot show how a base is named"
3404        );
3405
3406        // ── 1. THE BUG: no base may be named by oid ───────────────────────────
3407        let named_by_oid = w.closure().external_refs;
3408        assert!(
3409            named_by_oid.is_empty(),
3410            "a clone's pack names {} base(s) by oid — gitoxide resolves those against the \
3411             receiver's store and a clone's receiver is empty: {:?}",
3412            named_by_oid.len(),
3413            named_by_oid.iter().map(hex::encode).collect::<Vec<_>>()
3414        );
3415
3416        // …and the client's own code says so, not only our reading of it.
3417        let out = dir.join("gix-out");
3418        std::fs::create_dir_all(&out).unwrap();
3419        let stop = std::sync::atomic::AtomicBool::new(false);
3420        let mut cur = std::io::Cursor::new(emitted.as_slice());
3421        let outcome = gix_pack::Bundle::write_to_directory(
3422            &mut cur,
3423            Some(&out),
3424            &mut gix_features::progress::Discard,
3425            &stop,
3426            // An object database that holds nothing — a fresh clone target.
3427            Some(gix_object::find::Never),
3428            gix_pack::bundle::write::Options {
3429                object_hash: gix_hash::Kind::Sha1,
3430                ..Default::default()
3431            },
3432        );
3433        let outcome = match outcome {
3434            Ok(o) => o,
3435            Err(e) => panic!("gitoxide refused a pack this store emitted: {}", chain(&e)),
3436        };
3437        assert_eq!(
3438            outcome.index.num_objects as usize,
3439            oids.len(),
3440            "gitoxide indexed {} objects for a request of {}",
3441            outcome.index.num_objects,
3442            oids.len()
3443        );
3444
3445        // ── 2. stock git must still accept it ─────────────────────────────────
3446        crate::git_oracle::assert_git_accepts(
3447            &dir,
3448            "stock.git",
3449            &emitted,
3450            crate::git_oracle::Strictness::SelfContained,
3451        );
3452
3453        // ── 3. exactly the object set that was asked for, by oid, read back out
3454        //       of stock git ───────────────────────────────────────────────────
3455        let read_back = crate::git_oracle::git_reads_back(&dir, "readback.git", &emitted).unwrap();
3456        assert_eq!(
3457            read_back.len(),
3458            oids.len(),
3459            "the pack carries {} objects, the request asked for {}",
3460            read_back.len(),
3461            oids.len()
3462        );
3463        let got: std::collections::BTreeSet<String> =
3464            read_back.iter().map(|(oid, _)| oid.clone()).collect();
3465        let want: std::collections::BTreeSet<String> = oids.iter().map(hex::encode).collect();
3466        assert_eq!(
3467            got, want,
3468            "the emitted pack is not the requested set — missing {:?}, unexpected {:?}",
3469            want.difference(&got).collect::<Vec<_>>(),
3470            got.difference(&want).collect::<Vec<_>>()
3471        );
3472    }
3473
3474    /// gix errors nest their real cause; the top line alone names the operation
3475    /// and nothing about why. The same walk `gunnar-client::error::chain` does,
3476    /// and here for the same reason: without it the red above reads *"Failed to
3477    /// write pack"* and diagnoses nothing.
3478    fn chain(err: &(dyn std::error::Error + 'static)) -> String {
3479        let mut out = err.to_string();
3480        let mut cursor = err.source();
3481        for _ in 0..16 {
3482            let Some(next) = cursor else { break };
3483            let text = next.to_string();
3484            if !out.ends_with(&text) {
3485                out.push_str(": ");
3486                out.push_str(&text);
3487            }
3488            cursor = next.source();
3489        }
3490        out
3491    }
3492}