znippy_plugin_git/store.rs
1//! The storage functions. Nothing else is permitted here.
2//!
3//! **Twelve, then thirteen.** The count moved on 2026-08-10 when
4//! [`GitOps::put_refs_cas`] landed — atomic batch compare-and-swap, the
5//! `git push --atomic` primitive, which was previously *unrepresentable*:
6//! `put_refs` is a batch without CAS, `update_ref` is CAS without a batch, and
7//! `--atomic` is defined as both at once. It is a storage concern and it belongs
8//! here. What does **not** belong here is the *reading* contract — decoded
9//! reads, `HEAD`, negotiation, pack emission — which is [`crate::serve`].
10//!
11//! **ZNIPPY-GIT APACHE ARROW IPC IS LAW.** Storage is Arrow IPC in a znippy
12//! archive. Not a packfile directory, not loose objects, not `gix-odb`, not a
13//! filesystem layout borrowed from somebody else's model. gix is a codec for
14//! framing bytes onto the wire and nothing more: it never owns a handle, never
15//! decides where a byte lives, and never appears in a signature below.
16//!
17//! ---
18//!
19//! The twelve are the methods of [`GitOps`], implemented once, for
20//! [`GitStore`] — the handle and the trait live in [`crate::git_ops`], and
21//! everything each method reaches for was already built and already measured:
22//!
23//! | this file calls | which is | measured in |
24//! |---|---|---|
25//! | `PushPath` → `SafeWriter` | blob fsync, then the journal row | `archive_write` |
26//! | `ObjectReadStack` | stree → Arrow → redb tail | `read_stack` |
27//! | `RefLog` | one Arrow IPC frame per push | `refs` / `pushlog` |
28//! | `pack_walk` + `resolve` | the split and the oids | `pack_walk` / `resolve` |
29//! | `NewGeneration` → `compact_archive` | base znippy's compaction | `gc` |
30//!
31//! No method below invents storage, an index, a durability contract or a
32//! concurrency mechanism. If one looks like it does, that is the bug.
33
34use std::path::Path;
35
36use anyhow::{anyhow, bail, Context, Result};
37use znippy_common::ReservedSection;
38
39use crate::gc::GcReport;
40use crate::git_ops::{lookup_path, GitOps, GitStore, LookupPath, RefRow, Stored, TxId};
41use crate::index_layout::ObjectIndex;
42use crate::pack_walk::walk;
43use crate::refs::RefUpdate;
44use git_storage_trait::{Observed, RefCas, RefRejection, RefTarget};
45
46/// An object id, borrowed.
47pub type Oid<'a> = &'a [u8];
48
49/// A byte range inside the archive: `(offset, len)`.
50pub type Extent = (u64, u64);
51
52impl<S: ObjectIndex + 'static> GitOps for GitStore<S> {
53 // ── STORE ───────────────────────────────────────────────────────────────
54
55 /// One push, and the order inside it is the contract.
56 ///
57 /// 1. the pack's bytes, durable — [`put_pack`](GitOps::put_pack)
58 /// 2. **then** the refs that point into them — [`put_refs`](GitOps::put_refs)
59 ///
60 /// Not interchangeable, and it is the same argument `SafeWriter` makes one
61 /// level down about the blob and its journal row: a crash between the two
62 /// leaves objects nobody points at, which a GC reclaims. The reverse order
63 /// leaves a **ref pointing at objects that are not there**, which is a
64 /// corrupt repository that no later pass can repair.
65 fn put(&self, pack: &[u8], refs: &[RefUpdate]) -> Result<TxId> {
66 let mut tx = if pack.is_empty() {
67 TxId::default()
68 } else {
69 self.put_pack(pack)?
70 };
71 if !refs.is_empty() {
72 tx.push_seq = self.put_refs(refs)?.push_seq;
73 }
74 Ok(tx)
75 }
76
77 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
78 ///
79 /// **The ack path.** Walk, check, store verbatim, ack, then queue the index.
80 ///
81 /// 1. **Walk every entry** ([`crate::pack_walk`]). Not optional and not an
82 /// extra pass: a pack entry has no length field, so finding the entry
83 /// boundaries *is* how the pack gets split at all.
84 /// 2. **The closure check, free from that walk** (§13.7). Every `OFS_DELTA`
85 /// base must land on an entry boundary in this pack — answered from the
86 /// boundary set the walk just produced, consulting no index. Only a
87 /// `REF_DELTA` base that is *outside* the pack is looked up, and that one
88 /// reads `objects.oid` and nothing else, exactly as §13's table says
89 /// receive-pack does. A pack that fails is refused **before** a byte is
90 /// stored.
91 /// 3. **The bytes, verbatim.** `SafeWriter::append` writes the caller's
92 /// buffer at its own address — no re-compression, no re-encoding, no
93 /// re-framing, not one copy in userspace. The client's own deflate and
94 /// every delta chain survive, which is what makes a later clone a
95 /// byte-range copy instead of a re-pack (§14).
96 /// 4. **Durable before returning**: the blob is fsynced, *then* the journal
97 /// row that references it, then that is fsynced. znippy's `hot.rs`
98 /// ordering, so a crash between the two leaves orphan bytes nobody points
99 /// at rather than a dangling reference.
100 /// 5. **Then** the index job goes on the channel and this returns. Nothing
101 /// that could be done later is done here: no oid is computed, no table is
102 /// built, no chain is resolved.
103 fn put_pack(&self, bytes: &[u8]) -> Result<TxId> {
104 if bytes.is_empty() {
105 bail!("an empty push is not a pack");
106 }
107 // (1) + (2) — one walk, and the check falls out of it.
108 let walked = walk(bytes, self.hash_kind().oid_len())
109 .context("the pushed pack could not be split")?;
110 self.external_bases_exist(bytes, &walked)?;
111
112 // (3) + (4) — verbatim, blob fsync, journal row, journal fsync.
113 // (5) — and the pack-level index job onto the account's channel.
114 let (pack_id, extent) = self
115 .push_path()
116 .push_pack(self.account(), bytes)
117 .context("storing the pack verbatim")?;
118
119 // The object-level index work: 24 bytes of extent, queued, drained by
120 // `absorb_pending` off this path. Until it runs the pack is un-indexed
121 // and every read falls back rather than answering absent (§13.12).
122 self.queue(pack_id, extent)?;
123
124 Ok(TxId {
125 pack_id: Some(pack_id),
126 extent: Some(extent),
127 push_seq: None,
128 })
129 }
130
131 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
132 ///
133 /// One ref transaction: one Arrow IPC frame, fsynced.
134 ///
135 /// The frame boundary *is* the transaction (see [`crate::pushlog`]) — three
136 /// branches in one push are three rows in one batch, and they either all land
137 /// or none do. There is no lock file and no second journal.
138 ///
139 /// **Every target is checked against the index first.** A ref that points at
140 /// an object the repository does not have is a corrupt repository, and it is
141 /// refused here rather than written and discovered later. A deletion has no
142 /// target and is not checked.
143 fn put_refs(&self, updates: &[RefUpdate]) -> Result<TxId> {
144 if updates.is_empty() {
145 bail!("an empty ref update is not a transaction");
146 }
147 for u in updates {
148 for (what, oid_hex) in [("target", &u.target), ("peeled", &u.peeled)] {
149 let Some(hex_oid) = oid_hex else { continue };
150 let raw = hex::decode(hex_oid)
151 .map_err(|e| anyhow!("{}'s {what} `{hex_oid}` is not hex: {e}", u.name))?;
152 if !self.has(&raw)? {
153 bail!(
154 "{} would point at {hex_oid}, which this repository does not have — the \
155 ref update is refused rather than left dangling",
156 u.name
157 );
158 }
159 }
160 }
161 let push_seq = self.ref_log().push(updates)?;
162 Ok(TxId {
163 pack_id: None,
164 extent: None,
165 push_seq: Some(push_seq),
166 })
167 }
168
169 // ── READ ────────────────────────────────────────────────────────────────
170
171 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
172 ///
173 /// The object's **stored** bytes: one index lookup and one `pread` of the
174 /// extent, with no decode of any kind in between.
175 ///
176 /// What comes back is the pack entry exactly as the client sent it, which for
177 /// a delta is a delta. [`Stored::obj_type`] says which, so the bytes cannot
178 /// be mistaken for the object's content — §14 makes the verbatim bytes the
179 /// truth and the resolved object a derived cache, and that cache does not
180 /// exist yet, so handing back delta bytes labelled "the object" would be the
181 /// one thing this crate refuses everywhere: a wrong answer where an honest
182 /// one was available.
183 fn get(&self, oid: Oid<'_>) -> Result<Option<Stored>> {
184 let Some(row) = self.lookup_one(oid)? else {
185 return Ok(None);
186 };
187 let bytes = self.read_extent(row.offset, row.len)?;
188 Ok(Some(Stored {
189 obj_type: row.obj_type,
190 uncompressed_size: row.uncompressed_size,
191 extent: (row.offset, row.len),
192 bytes,
193 }))
194 }
195
196 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
197 ///
198 /// The negotiation call: `have` sends up to a thousand of these and **most of
199 /// them miss**. It reads `objects.oid` and no other column.
200 ///
201 /// It takes the **serial** path, not `lookup_batch` with one element: that is
202 /// 1.7× slower (818 ns against 482), and [`lookup_path`] is where that
203 /// decision is written down.
204 ///
205 /// # Why this returns `Result<bool>` and not `bool`
206 ///
207 /// The signature changed deliberately. A store with a pack whose bytes are
208 /// durable but whose objects are not indexed yet cannot answer "no" — the
209 /// object may be in that pack. So it absorbs the pack first and, if that
210 /// fails, **says so**. A `bool` could only have lied, and a wrong "absent"
211 /// during negotiation makes a client send nothing and lose data.
212 fn has(&self, oid: Oid<'_>) -> Result<bool> {
213 Ok(self.lookup_one(oid)?.is_some())
214 }
215
216 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
217 ///
218 /// The object's **post-resolution** size — what it inflates to once its delta
219 /// chain is applied. That is the fact git's own `.idx` and `.rev` together
220 /// cannot answer, and it is why the quota gate is index-only here.
221 fn size(&self, oid: Oid<'_>) -> Result<Option<u64>> {
222 Ok(self.lookup_one(oid)?.map(|r| r.uncompressed_size))
223 }
224
225 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
226 ///
227 /// **The wire path**, and batch by construction: for each oid the two facts a
228 /// clone needs — `offset` and `len` — and no others, so a byte-range copy out
229 /// of the verbatim pack can start immediately.
230 ///
231 /// `out[i]` answers `oids[i]`. One oid takes the serial path
232 /// ([`lookup_path`]); the batch path saturates at
233 /// [`BATCH_SATURATES_AT`](crate::git_ops::BATCH_SATURATES_AT) oids, so a
234 /// larger batch is passed through whole rather than split — splitting would
235 /// cost a pass and buy nothing.
236 fn extents(&self, oids: &[Oid<'_>]) -> Result<Vec<Option<Extent>>> {
237 match lookup_path(oids.len()) {
238 LookupPath::Serial => Ok(vec![self.lookup_one(oids[0])?.map(|r| (r.offset, r.len))]),
239 LookupPath::Batch => {
240 if self.unindexed_packs() > 0 {
241 self.absorb_pending()?;
242 }
243 Ok(self.index().extents_batch(oids))
244 }
245 }
246 }
247
248 // ── REFS ────────────────────────────────────────────────────────────────
249
250 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
251 ///
252 /// The whole ref namespace, tags peeled — the ref advertisement, and
253 /// `ls-refs` after a prefix filter. Name-sorted, because the log folds into a
254 /// `BTreeMap` and `ls-refs` wants a prefix scan.
255 ///
256 /// The cost is a scan of a structure sized by **pushes**, not by repository
257 /// size.
258 ///
259 /// # `HEAD` is not in here, and that is the contract
260 ///
261 /// **Changed 2026-08-10, and it is a fix rather than a preference.** The
262 /// other backend's `iter()` *"walks `refs/` (loose and packed) and
263 /// deliberately excludes the pseudo-refs such as `HEAD`, which is exactly the
264 /// contract every other backend honours"*. This one did not, so the two arms
265 /// disagreed about whether `HEAD` is a row — and the conformance suite could
266 /// not see it, because it never exercised `HEAD` at all.
267 ///
268 /// The reason `HEAD` cannot be a row is a type constraint, not taste: a name
269 /// type that admits `HEAD` also admits `MERGE_HEAD` and `FETCH_HEAD`, so a
270 /// row stream carrying pseudo-refs means either widening the name type or
271 /// filtering at every consumer. It gets an accessor pair instead —
272 /// [`GitServe::head`](crate::serve::GitServe::head) and
273 /// [`set_head`](crate::serve::GitServe::set_head) — and this is the one
274 /// filter, in the one place.
275 ///
276 /// Nothing else changes: the ref log still *stores* `HEAD` (a push writes it
277 /// like any other row), `live_set` still reaches it because it reads
278 /// `ref_state` directly, and the filter is one `!=`.
279 fn refs(&self) -> Result<Vec<RefRow>> {
280 let mut out = Vec::new();
281 for (name, state) in self
282 .ref_state()?
283 .into_iter()
284 .filter(|(name, _)| name != crate::serve::HEAD)
285 {
286 let decode = |h: &Option<String>| -> Result<Option<Vec<u8>>> {
287 match h {
288 Some(h) => {
289 Ok(Some(hex::decode(h).map_err(|e| {
290 anyhow!("ref {name}: `{h}` is not a hex oid: {e}")
291 })?))
292 }
293 None => Ok(None),
294 }
295 };
296 out.push(RefRow {
297 oid: decode(&state.target)?,
298 peeled: decode(&state.peeled)?,
299 symref_target: state.symref_target.clone(),
300 name,
301 });
302 }
303 Ok(out)
304 }
305
306 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
307 ///
308 /// **Compare-and-swap**, in the same log a push writes its refs to — one
309 /// mechanism, not a second one for single updates.
310 ///
311 /// `old` is what the caller believes the ref is: `None` means *it must not
312 /// exist* (a create), `Some(oid)` means *it must be exactly this*. `new` of
313 /// `None` deletes. A mismatch names both values and **writes nothing**.
314 ///
315 /// The read-compare-append is serialised on the store's ref gate. Without it
316 /// two CAS calls could both read the old value and both append, and the
317 /// second would silently overwrite an update it had compared against
318 /// successfully — the classic lost update, and the only thing a CAS is for.
319 fn update_ref(&self, name: &str, old: Option<Oid<'_>>, new: Option<Oid<'_>>) -> Result<TxId> {
320 let _gate = self
321 .ref_gate()
322 .lock()
323 .map_err(|_| anyhow!("the ref gate is poisoned"))?;
324
325 let current = self.ref_state()?;
326 let actual: Option<Vec<u8>> = match current.get(name).and_then(|s| s.target.as_deref()) {
327 Some(h) => Some(
328 hex::decode(h)
329 .map_err(|e| anyhow!("ref {name} holds `{h}`, not a hex oid: {e}"))?,
330 ),
331 None => None,
332 };
333 if actual.as_deref() != old {
334 // ── TYPED, like `put_refs_cas` three hundred lines down ─────────
335 //
336 // This raised a bare `bail!` while its own sibling raised
337 // `RefRejection::Cas`, so a caller had a string and no way to tell
338 // a lost race from a broken disk. `gunnar-wire`'s receive-pack
339 // says so at the call site: "A LOST COMPARE-AND-SWAP AND A BROKEN
340 // DISK ARE THE SAME STRING HERE … Nothing may recover the
341 // distinction by matching on this text; the fix is a typed
342 // rejection on the contract."
343 //
344 // The contract already had one. It was this method that did not
345 // use it. The `Display` text is unchanged — `RefRejection::Cas`
346 // renders the same sentence — so nothing that reads the message
347 // moves, and a caller that downcasts now gets `name`, `expected`
348 // and `actual` as values instead of parsing prose.
349 return Err(anyhow::Error::new(RefRejection::Cas {
350 name: name.to_string(),
351 expected: match old {
352 None => Observed::Nothing,
353 Some(o) => Observed::oid(o),
354 },
355 actual: match actual.as_deref() {
356 None => Observed::Nothing,
357 Some(a) => Observed::oid(a),
358 },
359 }));
360 }
361
362 let update = match new {
363 Some(n) => RefUpdate::set(name, hex::encode(n)),
364 None => RefUpdate::delete(name),
365 };
366 // Through `put_refs`, so the existence check and the log format are the
367 // same code a push uses (LAW 5) — a CAS cannot create a dangling ref that
368 // a push would have been refused for.
369 if new.is_some() {
370 self.put_refs(&[update])
371 } else {
372 let push_seq = self.ref_log().push(&[update])?;
373 Ok(TxId {
374 pack_id: None,
375 extent: None,
376 push_seq: Some(push_seq),
377 })
378 }
379 }
380
381 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
382 ///
383 /// **`git push --atomic`: every edit or none**, in one Arrow IPC frame.
384 ///
385 /// The two halves come from different places and neither is reimplemented:
386 ///
387 /// * **the batch atomicity is the frame.** [`crate::pushlog`] makes the frame
388 /// boundary the transaction — three branches in one push are three rows in
389 /// one batch and they either all land or none do. There is no lock file and
390 /// no second journal, so there is no partial-apply state to recover from;
391 /// * **the compare half is checked here, against one snapshot**, before a
392 /// single [`RefUpdate`] is built.
393 ///
394 /// It goes out through [`put_refs`](GitOps::put_refs) rather than straight to
395 /// the log, so the dangling-target refusal and the frame format are the same
396 /// code a push uses (LAW 5): an atomic batch cannot create a ref pointing at
397 /// an object the repository does not have, which a second writer here would
398 /// eventually have allowed.
399 ///
400 /// # `S-023`, and why the check is a whole pass of its own
401 ///
402 /// Every expectation is evaluated **before any of them is applied**, and the
403 /// loop deliberately does not fuse with the one that builds the updates. The
404 /// defect it exists for is gix's: a backend that short-circuits an edge whose
405 /// new value already equals the current one never evaluates the expectation
406 /// the caller wrote, so an `old: None` — *must not exist* — becomes a silent
407 /// success and *"exactly one creator wins"* stops being true. This log has no
408 /// such short-circuit, but the ordering is what makes that irrelevant rather
409 /// than lucky, and a future optimisation that adds one cannot break it from
410 /// here.
411 ///
412 /// # What this arm cannot raise, stated rather than hidden
413 ///
414 /// **[`RefRejection::Locked`] never comes out of this backend.** The
415 /// read-compare-append is serialised on the store's ref gate, which a second
416 /// writer *blocks* on rather than failing against — so contention here is a
417 /// wait, never a rejection. The variant is in the contract because the gix
418 /// arm, which takes real per-ref lock files, raises it. A poisoned gate stays
419 /// an ordinary error: it is a fault, not the transient thing a caller retries.
420 fn put_refs_cas(&self, edits: &[RefCas<'_>]) -> Result<TxId> {
421 // Not an error. A deletions-free push that had nothing to apply calls
422 // this, and `put_refs` refuses an empty batch — rightly, since an empty
423 // ref update is not a transaction. An empty *atomic* batch is a no-op
424 // that succeeded, and saying so here is what keeps the special case out
425 // of every call site.
426 if edits.is_empty() {
427 return Ok(TxId::default());
428 }
429
430 let _gate = self
431 .ref_gate()
432 .lock()
433 .map_err(|_| anyhow!("the ref gate is poisoned"))?;
434
435 // ONE read of the namespace for the whole check, so every expectation is
436 // compared against one instant rather than against a namespace that may
437 // move between them.
438 let current = self.ref_state()?;
439 let observed = |name: &str| -> Observed {
440 match current.get(name) {
441 None => Observed::Nothing,
442 Some(s) => match (&s.symref_target, &s.target) {
443 (Some(points_to), _) => {
444 Observed::Value(RefTarget::Symbolic(points_to.clone()))
445 }
446 (None, Some(h)) => match hex::decode(h) {
447 Ok(raw) => Observed::Value(RefTarget::Object(raw)),
448 // Not dropped into `Nothing`: a ref that exists and
449 // cannot be read is not a ref that was absent, and
450 // reporting it as absent tells the pushing client its
451 // create is free when it is not.
452 Err(e) => Observed::Unreadable(format!("`{h}` is not a hex oid: {e}")),
453 },
454 (None, None) => Observed::Unreadable(
455 "the ref log holds a row naming neither an object nor another ref".into(),
456 ),
457 },
458 }
459 };
460
461 // S-023: the whole check, before anything is applied.
462 for e in edits {
463 let expected = match e.old {
464 None => Observed::Nothing,
465 Some(o) => Observed::oid(o),
466 };
467 let actual = observed(&e.name);
468 if actual != expected {
469 return Err(anyhow::Error::new(RefRejection::Cas {
470 name: e.name.clone(),
471 expected,
472 actual,
473 }));
474 }
475 }
476
477 let updates: Vec<RefUpdate> = edits
478 .iter()
479 .map(|e| match e.new {
480 Some(n) => RefUpdate::set(e.name.clone(), hex::encode(n)),
481 None => RefUpdate::delete(e.name.clone()),
482 })
483 .collect();
484 self.put_refs(&updates)
485 }
486
487 // ── GRAPH ───────────────────────────────────────────────────────────────
488
489 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
490 ///
491 /// **Selection: `want` minus `have`, as an `andnot` of roaring bitmaps.**
492 ///
493 /// Every returned oid is an object — commit, tree *and* blob — because the
494 /// bitmaps are built by [`crate::reach::build_reach`], which walks the trees.
495 /// A `want` that names a commit contributes that commit's whole closure; a
496 /// `want` that names a tag or a blob contributes itself.
497 ///
498 /// A `have` this repository does not know contributes nothing: the safe
499 /// direction is to send more, never less.
500 ///
501 /// The bitmaps are over the **store's own ordinal space**, rebuilt by the same
502 /// fold that rebuilds them, never over the Arrow projection's ordinals — an
503 /// [`crate::index_layout::IndexRow::ordinal`] is a row address within one
504 /// projection generation, so a bitmap over those would address different
505 /// objects after any rebuild, silently.
506 ///
507 /// # The per-oid `Vec` is paid HERE and nowhere else on the serving path
508 ///
509 /// The answer is computed flat (see
510 /// [`GitStore::reachable_raw`](crate::git_ops::GitStore::reachable_raw));
511 /// this splits it back out because the eleven say `Vec<Vec<u8>>` and this
512 /// method's callers are maintenance ones — a GC live set, a conformance
513 /// harness — not the serving path. `crate::serve`'s `select` and
514 /// `emit_pack` take the flat form directly.
515 fn reachable(&self, want: &[Oid<'_>], have: &[Oid<'_>]) -> Result<Vec<Vec<u8>>> {
516 Ok(self
517 .reachable_raw(want, have)?
518 .iter()
519 .map(<[u8]>::to_vec)
520 .collect())
521 }
522
523 // ── MAINT ───────────────────────────────────────────────────────────────
524
525 /// Garbage collection, in the order §13.20 fixes:
526 ///
527 /// 1. **compute reachability** — every object reachable from every ref, over
528 /// the same bitmaps [`reachable`](GitOps::reachable) uses;
529 /// 2. **drop the dead rows from the index** — the one operation that is not
530 /// append-only, and the projection is rebuilt inside it;
531 /// 3. **then** base znippy's compaction, through [`crate::gc::NewGeneration`]
532 /// (link, compact, verify, rename, unlink last).
533 ///
534 /// That order is why **base znippy needs no new method**: by the time
535 /// `compact_archive` runs, "live" already means what git means.
536 ///
537 /// A repository with no ref pointing at anything is refused rather than
538 /// emptied.
539 ///
540 /// **Appended, not reworded: step 1b, the journal.** Between the live set and
541 /// the drop there is now one more durable act. §13.12's `indexed` bit is
542 /// derived on open as *extent in the journal, rows not in the index*, and
543 /// dropping **every** row of a pack produces exactly that state — so before
544 /// this existed the next open re-queued the pack and every object this GC had
545 /// just decided was dead came back.
546 /// [`GitStore::retire_dead_packs`](crate::git_ops::GitStore::retire_dead_packs)
547 /// appends a tombstone naming each all-dead pack, and it runs **before**
548 /// `drop_dead_rows` on purpose: killed before it, the rows are still there
549 /// and the GC simply did not happen; killed after it, the pack can never be
550 /// re-queued whether the rows went or not. A partly dead pack keeps rows and
551 /// is never tombstoned, so nothing about it changes.
552 ///
553 /// Step 2 also refolds what the two tables feed — the commit graph, the tree
554 /// payloads, the ordinal space, the bitmaps — because a derivation that still
555 /// names a dropped oid is wrong rather than stale. That is inside
556 /// `drop_dead_rows`, so any caller of it gets it.
557 fn gc(&self) -> Result<GcReport> {
558 self.absorb_pending()?;
559 let live = self.live_set()?;
560 let retired = self.retire_dead_packs(&live)?;
561 let before = self.index().len() as u64;
562 let dropped = self.drop_dead_rows(&live)?;
563 let after = self.index().len() as u64;
564 if before.saturating_sub(after) != dropped || after > before {
565 bail!(
566 "the index dropped {dropped} rows but went from {before} to {after} — refusing to \
567 compact an archive whose index does not agree with itself"
568 );
569 }
570 let mut report = self.gc_arm().run(self.archive_path()).with_context(|| {
571 format!(
572 "compacting {} after dropping {dropped} dead rows",
573 self.archive_path().display()
574 )
575 })?;
576 report.retired_packs = retired.len() as u64;
577 Ok(report)
578 }
579}
580
581impl<S: ObjectIndex + 'static> GitStore<S> {
582 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
583 ///
584 /// Fold the live logs into the reserved Arrow sections an archive carries:
585 /// `__gunnar_refs__`, `__gunnar_graph__`, `__gunnar_reach__`.
586 ///
587 /// **The twelfth method, and the one that is NOT on [`GitOps`]:** it returns
588 /// `znippy_common::ReservedSection` (Arrow `RecordBatch` payloads), which a
589 /// gix backend has no analog for — so it is inherent on the concrete store,
590 /// which is how gunnar already calls it (never through the trait).
591 ///
592 /// It absorbs first, because a section built while a pack is still
593 /// un-indexed would be **silently incomplete**, and a silently incomplete
594 /// index is worse than none (the same argument [`crate::lib`] makes about not
595 /// wiring `GitIndexBuilder` into the CLI). Every generation number in the
596 /// graph section is recomputed by that fold.
597 ///
598 /// **…and then it writes the archive.** Until 2026-08-08 it only *returned*
599 /// the sections and nothing on any path created
600 /// [`archive_path`](GitStore::archive_path) at all — so `gc()`'s last step
601 /// died on `stat repository.znippy: No such file or directory`, having
602 /// already done its first four. Both [`Gc`](crate::gc::Gc) implementations
603 /// compact an archive that exists; **this** is what makes one exist.
604 /// [`seal_generation_zero`](crate::archive_write::seal_generation_zero) has
605 /// the layout argument.
606 ///
607 /// The sections are still returned, and they are the same values that were
608 /// sealed rather than a second derivation of them — `ReservedSection` is
609 /// `Clone` for exactly that reason.
610 pub fn seal(&self) -> Result<Vec<ReservedSection>> {
611 self.absorb_pending()?;
612 self.index().rebuild()?;
613 let sections = self.reserved_sections()?;
614 self.seal_archive(sections.clone()).with_context(|| {
615 format!("sealing generation 0 at {}", self.archive_path().display())
616 })?;
617 Ok(sections)
618 }
619}
620
621/// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
622///
623/// Compact `src` into a **named** destination, leaving `src` untouched.
624///
625/// The file-level primitive behind [`GitOps::gc`]'s third step, for a caller that
626/// wants to choose the name rather than take
627/// [`crate::gc::next_generation`]'s. Three existing calls and no fourth
628/// mechanism:
629///
630/// 1. `hard_link(src, dst)` — a second name for the same inode. **Not one byte is
631/// copied**, and `src` keeps pointing at the original for the whole run.
632/// 2. `compact_archive(dst)` — base znippy's own compaction, verbatim, against
633/// the new name.
634/// 3. read every entry back through the ordinary reader — the same verification
635/// [`crate::gc::NewGeneration`] gates on, so a `dst` that does not read back
636/// is removed and the error says so.
637///
638/// `dst` must not exist. Silently compacting over a file is how a generation gets
639/// lost.
640pub fn compact(src: &Path, dst: &Path) -> Result<()> {
641 if dst.exists() {
642 bail!(
643 "{} already exists — refusing to compact over it",
644 dst.display()
645 );
646 }
647 std::fs::hard_link(src, dst).with_context(|| {
648 format!(
649 "hard-linking {} to {} — compaction runs against a second name for the same inode, \
650 which needs both on one filesystem",
651 src.display(),
652 dst.display()
653 )
654 })?;
655 if let Err(e) = znippy_common::compact_archive(dst) {
656 let _ = std::fs::remove_file(dst);
657 return Err(e.context(format!(
658 "compacting {} into {}",
659 src.display(),
660 dst.display()
661 )));
662 }
663 if let Err(e) = crate::gc::read_back_every_entry(dst) {
664 let _ = std::fs::remove_file(dst);
665 return Err(e.context(format!(
666 "{} did not read back after compaction — it was removed and {} is untouched",
667 dst.display(),
668 src.display()
669 )));
670 }
671 Ok(())
672}
673
674#[cfg(test)]
675pub(crate) mod tests {
676 use super::*;
677 use crate::git_ops::BATCH_SATURATES_AT;
678 use crate::index_layout::ObjType;
679 use crate::object::{canonical, GitHashKind, GitObjectKind};
680 use crate::resolve::{resolve, NoBases};
681 use std::path::PathBuf;
682 use std::time::Instant;
683
684 /// A store directory for one test.
685 ///
686 /// It returns a `PathBuf` and not a `tempfile::TempDir`, so **nothing
687 /// deletes it when the test ends** — 82 call sites take it as a path and
688 /// several deliberately outlive a store to reopen it, which is what the
689 /// borrowed form would forbid. The directories are therefore swept on the
690 /// way in instead of dropped on the way out; see [`sweep_dead_runs`].
691 pub(crate) fn tmpdir(tag: &str) -> PathBuf {
692 sweep_dead_runs();
693 let d = std::env::temp_dir().join(format!(
694 "znippy-git-store-{tag}-{}-{}",
695 std::process::id(),
696 std::time::SystemTime::now()
697 .duration_since(std::time::UNIX_EPOCH)
698 .map(|d| d.as_nanos())
699 .unwrap_or(0)
700 ));
701 std::fs::create_dir_all(&d).unwrap();
702 d
703 }
704
705 /// **Delete the store directories left behind by test runs that are over.**
706 ///
707 /// MEASURED on oden 2026-08-11: **1 088 of these directories, 96 GB**, and
708 /// `std::env::temp_dir()` on that box is `/tmp`, which is a `tmpfs` — so
709 /// that was 96 GB of *RAM* held by test runs that had exited days earlier,
710 /// on the shared machine every performance figure in this crate is taken
711 /// on. Each one is a whole store, and one with an exploded table is ~180 MB.
712 ///
713 /// Two conditions, both required, because the only way this can do harm is
714 /// by deleting a directory a running test still wants:
715 ///
716 /// 1. **The pid in the name is not a live process.** The name has carried
717 /// the pid since it was written; nothing consulted it until now.
718 /// 2. **The directory has not been touched for an hour.** Redundant against
719 /// a correct pid check, and there precisely because pids are reused: a
720 /// fresh run that inherited a dead run's pid has a fresh mtime.
721 ///
722 /// Once per process, not once per call: 82 call sites would otherwise scan
723 /// the whole of `/tmp` 82 times.
724 fn sweep_dead_runs() {
725 static ONCE: std::sync::Once = std::sync::Once::new();
726 ONCE.call_once(|| {
727 let Ok(entries) = std::fs::read_dir(std::env::temp_dir()) else {
728 return;
729 };
730 for e in entries.flatten() {
731 let name = e.file_name();
732 let Some(rest) = name
733 .to_str()
734 .and_then(|n| n.strip_prefix("znippy-git-store-"))
735 else {
736 continue;
737 };
738 // `<tag>-<pid>-<nanos>`, and a tag may contain `-`, so the pid
739 // is the second field from the right.
740 let Some(pid) = rest.rsplit('-').nth(1).and_then(|p| p.parse::<u32>().ok()) else {
741 continue;
742 };
743 if pid == std::process::id() || Path::new(&format!("/proc/{pid}")).exists() {
744 continue;
745 }
746 let stale = e
747 .metadata()
748 .and_then(|m| m.modified())
749 .ok()
750 .and_then(|t| t.elapsed().ok())
751 .is_some_and(|age| age > std::time::Duration::from_secs(3600));
752 if stale {
753 let _ = std::fs::remove_dir_all(e.path());
754 }
755 }
756 });
757 }
758
759 /// A real pack from this machine, with its objects already resolved so a test
760 /// knows what oids to ask for.
761 pub(crate) fn real_pack() -> (Vec<u8>, Vec<crate::resolve::Resolved>) {
762 let root = Path::new("/home/rickard/git");
763 for repo in std::fs::read_dir(root).unwrap().flatten() {
764 let dir = repo.path().join(".git/objects/pack");
765 let Ok(files) = std::fs::read_dir(&dir) else {
766 continue;
767 };
768 for f in files.flatten() {
769 let p = f.path();
770 if p.extension().is_some_and(|e| e == "pack")
771 && f.metadata().map(|m| m.len() < 8 << 20).unwrap_or(false)
772 {
773 let bytes = std::fs::read(&p).unwrap();
774 // Thin packs cannot be resolved standalone (§14); skip them.
775 if let Ok(rows) = resolve(&bytes, GitHashKind::Sha1, 0, &NoBases) {
776 if rows.len() > 100 {
777 return (bytes, rows);
778 }
779 }
780 }
781 }
782 }
783 panic!("no self-contained real pack under /home/rickard/git");
784 }
785
786 /// A tiny pack built here, so a test can push something that is not tied to
787 /// this machine. One blob, whole.
788 pub(crate) fn one_blob_pack(body: &[u8]) -> (Vec<u8>, Vec<u8>) {
789 use std::io::Write;
790 let mut pack = b"PACK".to_vec();
791 pack.extend_from_slice(&2u32.to_be_bytes());
792 pack.extend_from_slice(&1u32.to_be_bytes());
793 let mut size = body.len() as u64;
794 let mut header = vec![(3u8 << 4) | (size as u8 & 0x0f)];
795 size >>= 4;
796 while size > 0 {
797 let last = header.len() - 1;
798 header[last] |= 0x80;
799 header.push((size & 0x7f) as u8);
800 size >>= 7;
801 }
802 pack.extend_from_slice(&header);
803 let mut e = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
804 e.write_all(body).unwrap();
805 pack.extend_from_slice(&e.finish().unwrap());
806 pack.extend_from_slice(&[0u8; 20]);
807 let oid = GitHashKind::Sha1.oid_of(&canonical(GitObjectKind::Blob, body));
808 (pack, oid)
809 }
810
811 /// **The bytes come back byte for byte.** A real repository's pack is pushed,
812 /// absorbed, and then every object is fetched by oid and compared against the
813 /// exact slice of the pushed bytes it came from.
814 ///
815 /// This is the verbatim contract asserted as applied output — on the file on
816 /// disk, not on what `put` returned. A store that re-compressed, re-framed or
817 /// re-encoded anything cannot pass it.
818 ///
819 /// Seen RED by making `put_pack` store `&bytes[..bytes.len() - 1]`: "the
820 /// extent covers the whole pack — left: 5653301, right: 5653302". One byte
821 /// short of verbatim and the extent no longer covers the push.
822 #[test]
823 fn every_stored_object_reads_back_as_the_exact_bytes_that_were_pushed() {
824 let dir = tmpdir("verbatim");
825 let store = GitStore::open(&dir, "rickard").unwrap();
826 let (pack, rows) = real_pack();
827
828 // The drain is parked in front of its absorb for as long as this lives,
829 // so "queued, not run" below is a **state** and not a timing window. It
830 // used to be neither: `absorb_pending() == 1` raced the drain for the
831 // gate and lost whenever the drain got there first, which is a wrong
832 // expectation rather than a wrong store — both callers end in the same
833 // rows and the count only says which one paid.
834 let gate = store.hold_absorb_gate();
835 let tx = store.put(&pack, &[]).unwrap();
836 let (offset, len) = tx.extent.expect("a pack push records its extent");
837 assert_eq!(len, pack.len() as u64, "the extent covers the whole pack");
838
839 // The file on disk holds the pushed bytes, unchanged.
840 let on_disk = std::fs::read(store.blobs_path()).unwrap();
841 assert_eq!(
842 &on_disk[offset as usize..(offset + len) as usize],
843 &pack[..],
844 "the archive does not hold the pack verbatim"
845 );
846
847 assert_eq!(
848 store.unindexed_packs(),
849 1,
850 "the index job is queued, not run"
851 );
852 assert_eq!(
853 store.object_count(),
854 0,
855 "a row landed while the gate was held"
856 );
857 drop(gate);
858 store.wait_indexed();
859 assert_eq!(
860 store.absorb_pending().unwrap(),
861 0,
862 "the drain left index work"
863 );
864 assert_eq!(store.object_count(), rows.len(), "every object is indexed");
865
866 for (i, r) in rows.iter().enumerate() {
867 let got = store
868 .get(&r.oid)
869 .unwrap()
870 .unwrap_or_else(|| panic!("object {i} {} is missing", hex::encode(&r.oid)));
871 let from = (offset + r.offset) as usize;
872 assert_eq!(
873 got.bytes,
874 &pack[from..from + r.len as usize],
875 "object {i} {}: the stored bytes are not the pushed bytes",
876 hex::encode(&r.oid)
877 );
878 assert_eq!(got.extent, (offset + r.offset, r.len));
879 assert_eq!(got.uncompressed_size, r.uncompressed_size);
880 assert_eq!(got.obj_type, r.stored_type);
881 assert_eq!(store.size(&r.oid).unwrap(), Some(r.uncompressed_size));
882 assert!(store.has(&r.oid).unwrap());
883 }
884
885 // A delta really is labelled a delta, so nobody can mistake its bytes for
886 // content. (If this pack had none, the assertion above would be vacuous.)
887 let deltas = rows
888 .iter()
889 .filter(|r| matches!(r.stored_type, ObjType::OfsDelta | ObjType::RefDelta))
890 .count();
891 assert!(deltas > 0, "the fixture pack carries no deltas");
892 let d = rows
893 .iter()
894 .find(|r| matches!(r.stored_type, ObjType::OfsDelta | ObjType::RefDelta))
895 .unwrap();
896 let got = store.get(&d.oid).unwrap().unwrap();
897 assert!(matches!(
898 got.obj_type,
899 ObjType::OfsDelta | ObjType::RefDelta
900 ));
901 assert_ne!(
902 got.bytes.len() as u64,
903 got.uncompressed_size,
904 "a delta's stored bytes are not its resolved size"
905 );
906
907 // An oid nothing pushed is absent, and that is an answer, not a guess.
908 assert!(!store.has(&[0xab; 20]).unwrap());
909 assert!(store.get(&[0xab; 20]).unwrap().is_none());
910 assert_eq!(store.size(&[0xab; 20]).unwrap(), None);
911 }
912
913 /// **Durability, in the order that matters.** `put_pack` must not return
914 /// before the bytes are on the device *and* a journal row on the device
915 /// points at them — and if the machine dies between the two, what is left
916 /// must be orphan bytes, never a row pointing into a hole.
917 ///
918 /// Asserted by reading the two files back from disk after a fault injected
919 /// into the **real** `SafeWriter::append` (LAW 5: the fault goes into the one
920 /// writer, rather than a hand-rolled twin of it being tested).
921 ///
922 /// Seen RED by moving the blob `fsync` in `SafeWriter::append` to *after* the
923 /// journal row and its `fsync`, with the injected crash between them — i.e.
924 /// the reference is made durable before the bytes it references: "the journal
925 /// claims 1 extent(s) after a crash before the blob was durable — that is a
926 /// dangling reference, which is the failure this ordering exists to prevent".
927 /// Restored.
928 ///
929 /// **And the honest limit of this guard, found by trying a weaker mutation
930 /// first.** Moving the blob `fsync` to the end *without* moving the fault did
931 /// **not** turn it red: the fault still fires before the journal row, so the
932 /// on-disk state is still orphan bytes, and an `fsync` that did not happen is
933 /// unobservable from inside the process — the bytes read back out of the page
934 /// cache either way. What this guard proves is therefore the **ordering of the
935 /// reference against the bytes**, which is the part a crash can expose. It
936 /// cannot prove an `fsync` reached the platter; nothing short of cutting power
937 /// can.
938 #[test]
939 fn a_pack_is_durable_before_put_returns_and_the_fsyncs_are_ordered() {
940 use crate::archive_write::{read_journal, ArchiveWrite, Faults, SafeWriter};
941
942 let dir = tmpdir("durable");
943 let (pack, _) = one_blob_pack(b"durability is an ordering property");
944
945 // 1. The ordinary path: after `put_pack` returns, both files hold it.
946 let store = GitStore::open(&dir, "rickard").unwrap();
947 let tx = store.put_pack(&pack).unwrap();
948 let (offset, len) = tx.extent.unwrap();
949 let journal = SafeWriter::journal_path(store.blobs_path());
950 assert_eq!(
951 read_journal(&journal).unwrap(),
952 vec![(offset, len)],
953 "the journal row that claims the extent is not there when put_pack returned"
954 );
955 assert_eq!(
956 std::fs::metadata(store.blobs_path()).unwrap().len(),
957 offset + len,
958 "the blob file does not end where the journal says the pack does"
959 );
960
961 // 2. A crash between the two fsyncs leaves ORPHAN BYTES, not a dangling
962 // reference. Same `append`, one fault flipped.
963 let dir2 = tmpdir("crash");
964 let blobs = dir2.join("objects.pack");
965 let w = SafeWriter::create_with_faults(
966 &blobs,
967 Faults {
968 die_between_fsyncs: true,
969 ..Default::default()
970 },
971 )
972 .unwrap();
973 assert!(
974 w.append(&pack).is_err(),
975 "the injected crash must not return Ok"
976 );
977 drop(w);
978 let orphan = std::fs::read(&blobs).unwrap();
979 assert_eq!(orphan, pack, "the blob bytes were fsynced before the crash");
980 assert!(
981 read_journal(&SafeWriter::journal_path(&blobs))
982 .unwrap()
983 .is_empty(),
984 "the journal claims {} extent(s) after a crash before the blob was durable — that is a \
985 dangling reference, which is the failure this ordering exists to prevent",
986 read_journal(&SafeWriter::journal_path(&blobs)).unwrap().len()
987 );
988
989 // 3. And a store reopened over the orphan file answers for nothing,
990 // rather than for bytes nobody claimed.
991 let reopened = GitStore::open(&dir2, "rickard").unwrap();
992 assert_eq!(reopened.object_count(), 0);
993 }
994
995 /// **The closure check refuses a pack before storing it**, and it does so
996 /// without consulting the index for anything except a genuinely external
997 /// base.
998 ///
999 /// Asserted on applied output: the blob file does not grow, and no journal
1000 /// row appears.
1001 ///
1002 /// Seen RED by moving `external_bases_exist` to *after* `push_pack`: "the
1003 /// refused pack was stored anyway: 0 bytes became 65".
1004 #[test]
1005 fn a_pack_that_fails_the_closure_check_is_refused_before_a_byte_is_stored() {
1006 let dir = tmpdir("closure");
1007 let store = GitStore::open(&dir, "rickard").unwrap();
1008
1009 // A thin pack: one ref-delta against an oid this repository has never
1010 // seen.
1011 let mut thin = b"PACK".to_vec();
1012 thin.extend_from_slice(&2u32.to_be_bytes());
1013 thin.extend_from_slice(&1u32.to_be_bytes());
1014 thin.push(0x74); // type 7, size 4
1015 thin.extend_from_slice(&[0x11; 20]);
1016 {
1017 use std::io::Write;
1018 let mut e = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
1019 e.write_all(&[0x04, 0x04, 0x90, 0x04]).unwrap();
1020 thin.extend_from_slice(&e.finish().unwrap());
1021 }
1022 thin.extend_from_slice(&[0u8; 20]);
1023
1024 let before = std::fs::metadata(store.blobs_path()).unwrap().len();
1025 let err = store
1026 .put_pack(&thin)
1027 .expect_err("a dangling base is refused");
1028 let msg = err.to_string();
1029 assert!(
1030 msg.contains(&hex::encode([0x11; 20])),
1031 "names the base: {msg}"
1032 );
1033 let after = std::fs::metadata(store.blobs_path()).unwrap().len();
1034 assert_eq!(
1035 before, after,
1036 "the refused pack was stored anyway: {before} bytes became {after}"
1037 );
1038 assert_eq!(store.unindexed_packs(), 0, "and no index job was queued");
1039
1040 // A corrupt pack — a delta base that lands nowhere — is refused too, and
1041 // as corruption rather than as thinness.
1042 let (mut corrupt, _) = one_blob_pack(b"x");
1043 corrupt[11] = 2; // claim two objects, provide one
1044 assert!(store.put_pack(&corrupt).is_err());
1045 assert_eq!(
1046 std::fs::metadata(store.blobs_path()).unwrap().len(),
1047 before,
1048 "a corrupt pack was stored"
1049 );
1050 }
1051
1052 /// **CAS: a mismatch moves nothing.** The ref namespace is read back from the
1053 /// log after every attempt, so what is asserted is the ref's applied state
1054 /// and not the call's return value.
1055 ///
1056 /// Seen RED by comparing `actual.is_some() != old.is_some()` — existence
1057 /// only, the CAS bug that looks right: "stale old value: TxId { pack_id: None,
1058 /// extent: None, push_seq: Some(1) }", i.e. the swap from a wrong old value
1059 /// was accepted and written.
1060 #[test]
1061 fn a_compare_and_swap_that_loses_the_race_moves_no_ref() {
1062 let dir = tmpdir("cas");
1063 let store = GitStore::open(&dir, "rickard").unwrap();
1064 let (pack_a, oid_a) = one_blob_pack(b"first");
1065 let (pack_b, oid_b) = one_blob_pack(b"second");
1066 store.put_pack(&pack_a).unwrap();
1067 store.put_pack(&pack_b).unwrap();
1068 store.absorb_pending().unwrap();
1069
1070 let name = "refs/heads/main";
1071 let target = |s: &GitStore| -> Option<Vec<u8>> {
1072 s.refs()
1073 .unwrap()
1074 .into_iter()
1075 .find(|r| r.name == name)
1076 .and_then(|r| r.oid)
1077 };
1078
1079 // ★ A LOST CAS IS TYPED, not a string a caller has to parse.
1080 //
1081 // `update_ref` raised a bare `bail!` while its own sibling
1082 // `put_refs_cas` raised `RefRejection::Cas`, so `gunnar-wire`'s
1083 // receive-pack could not tell a lost race from a broken disk and said
1084 // so in a comment at the call site. The contract had the type all
1085 // along; this method did not use it.
1086 //
1087 // Seen RED by restoring the `bail!`: "a lost CAS must carry
1088 // RefRejection::Cas, not a string: compare-and-swap on refs/heads/main
1089 // failed: it is absent, the caller expected 0101…" — the message was
1090 // right and the TYPE was missing, which is exactly the failure a
1091 // string-shaped error hides.
1092 let err = store
1093 .update_ref(name, Some(&oid_a), Some(&oid_b))
1094 .expect_err("a create that claims a previous value must fail");
1095 match err.downcast_ref::<git_storage_trait::RefRejection>() {
1096 Some(git_storage_trait::RefRejection::Cas {
1097 name: n,
1098 expected,
1099 actual,
1100 }) => {
1101 assert_eq!(n, name, "the rejection names the wrong ref");
1102 assert_eq!(
1103 *actual,
1104 git_storage_trait::Observed::Nothing,
1105 "the ref did not exist, so `actual` must say so"
1106 );
1107 assert_ne!(
1108 *expected,
1109 git_storage_trait::Observed::Nothing,
1110 "the caller DID claim a previous value; `expected` must carry it"
1111 );
1112 }
1113 other => panic!(
1114 "a lost CAS must carry RefRejection::Cas, not a string: {err:#} (downcast: {})",
1115 if other.is_some() { "wrong variant" } else { "none" }
1116 ),
1117 }
1118 assert_eq!(target(&store), None, "the failed create wrote something");
1119
1120 let tx = store.update_ref(name, None, Some(&oid_a)).unwrap();
1121 assert!(tx.push_seq.is_some());
1122 assert_eq!(target(&store).as_deref(), Some(&oid_a[..]));
1123
1124 // A swap from the WRONG old value: refused, and nothing moves.
1125 let err = store
1126 .update_ref(name, Some(&oid_b), Some(&oid_b))
1127 .expect_err("stale old value");
1128 assert!(err.to_string().contains("compare-and-swap"), "{err}");
1129 assert_eq!(
1130 target(&store).as_deref(),
1131 Some(&oid_a[..]),
1132 "refs/heads/main moved on a failed CAS"
1133 );
1134
1135 // The right old value: it moves.
1136 store.update_ref(name, Some(&oid_a), Some(&oid_b)).unwrap();
1137 assert_eq!(target(&store).as_deref(), Some(&oid_b[..]));
1138
1139 // A ref may not point at an object we do not have.
1140 let err = store
1141 .update_ref(name, Some(&oid_b), Some(&[0xcd; 20]))
1142 .expect_err("dangling target");
1143 assert!(err.to_string().contains("does not have"), "{err}");
1144 assert_eq!(target(&store).as_deref(), Some(&oid_b[..]));
1145
1146 // Delete: gone from the namespace, and the log is the history.
1147 store.update_ref(name, Some(&oid_b), None).unwrap();
1148 assert_eq!(target(&store), None);
1149 assert!(store.refs().unwrap().iter().all(|r| r.name != name));
1150 }
1151
1152 /// **The bounded walk answers EXACTLY what a full bitmap table answers** —
1153 /// under-send and over-send both named, over every commit in the fixture.
1154 ///
1155 /// # What is actually at stake here
1156 ///
1157 /// Until 2026-08-14 the live table bitmapped **every** commit
1158 /// (`ReachPolicy { max_commits: usize::MAX }`), not as a choice but because
1159 /// `reachable_oids` had no fallback: a `want` with no bitmap contributed
1160 /// itself and nothing else. MEASURED on oden — 530 218 of a 4-object fetch's
1161 /// 575 349 allocations were that build, thrown away again by the next push's
1162 /// `refold`. [`crate::reach::accumulate`] is the walk that makes a sampled
1163 /// table legal, and `LIVE_REACH_COMMITS` is now 512.
1164 ///
1165 /// **A walk that stops too early UNDER-SENDS, and a clone that under-sends
1166 /// is silent data loss**: the pack indexes, `fsck`s and applies, the client
1167 /// exits zero, and the repository is missing objects it will not discover
1168 /// for days. `select` refusing was a correct-but-expensive answer to exactly
1169 /// this hazard, and this test is what replaces it.
1170 ///
1171 /// # Three arms over one store, and why three
1172 ///
1173 /// | cap | table | what runs |
1174 /// |---|---|---|
1175 /// | `usize::MAX` | every commit | no walk at all — **the reference** |
1176 /// | `0` | empty | the walk does *everything* |
1177 /// | `2` | two commits | the mixed case, where phase 1's stopping rule actually fires |
1178 ///
1179 /// Two arms would not be enough. `0` never exercises the stop-at-a-bitmap
1180 /// rule, which is the one thing that could OR in a *closed* set at the wrong
1181 /// moment, and `usize::MAX` never exercises the walk. The middle arm is the
1182 /// only one where both halves meet.
1183 ///
1184 /// # Both directions are named, and that is the anti-hollow half
1185 ///
1186 /// The comparison is a **symmetric difference**, not a length check and not
1187 /// a subset check. An over-send passes every downstream test we own — the
1188 /// pack indexes, `--check-self-contained-and-connected` accepts it, `fsck`
1189 /// is clean — and today's gix defect was exactly that shape. So `missing`
1190 /// and `extra` are computed and printed separately: a walk that stops short
1191 /// names the objects it lost, and a walk that ORs too much names the objects
1192 /// it invented.
1193 ///
1194 /// # Seen RED, 2026-08-14, three ways — and the directions are instructive
1195 ///
1196 /// * **stop one commit short** (`continue` before pushing parents in phase
1197 /// 1) — *"cap 0 vs the full table disagree for want 75c9e5c3…: MISSING 0
1198 /// object(s) …; EXTRA 2 object(s) the full table did not select, first
1199 /// Some(\"75c9e5c3d1d08dd92cc913c70ad10288a19eea4e\")"*.
1200 /// * **lose objects in the tree walk** (skip every seventh ordinal in
1201 /// `accumulate_tree`) — *"cap 0 …: MISSING 2 object(s) the full table
1202 /// selected, first Some(\"28b3c3658111299869d1f63f49a8cfb2635c0cfe\");
1203 /// EXTRA 0"*.
1204 /// * **OR in an unrelated commit's bitmap** — *"cap 2 …: MISSING 13
1205 /// object(s) the full table selected, first
1206 /// Some(\"078f18e68253d6fc56e8cba230ef9f32dec2e2ab\"); EXTRA 0"*.
1207 ///
1208 /// 🔴 **Note which direction each one came out.** Truncating the walk
1209 /// produced an *over*-send and over-collecting produced an *under*-send —
1210 /// the opposite of the intuition in both cases — because the identical walk
1211 /// serves the `have` side, and an error there enters the answer through
1212 /// `union − had` with its sign flipped. That is precisely why the comparison
1213 /// is a symmetric difference and reports **both** counts: a guard that
1214 /// checked only for missing objects would have passed the first break, and a
1215 /// guard that checked only lengths would have named nothing.
1216 ///
1217 /// The premise is guarded too: with the arms wired to the same policy this
1218 /// test passes trivially, so the table size at each cap is asserted before
1219 /// any answer is compared.
1220 #[test]
1221 fn the_bounded_walk_answers_exactly_what_a_full_bitmap_table_answers() {
1222 use crate::reach::ReachPolicy;
1223
1224 let dir = tmpdir("reach-walk");
1225 let store = GitStore::open(&dir, "rickard").unwrap();
1226 let (pack, _) = real_pack();
1227 store.put(&pack, &[]).unwrap();
1228 store.absorb_pending().unwrap();
1229
1230 let commits: Vec<String> = store.graph_snapshot().iter().map(|c| c.oid.clone()).collect();
1231 assert!(
1232 commits.len() > 2,
1233 "premise: the fixture needs more commits than the smallest cap under test, or every \
1234 arm bitmaps everything and the walk never runs — found {}",
1235 commits.len()
1236 );
1237
1238 let full = ReachPolicy {
1239 max_commits: usize::MAX,
1240 };
1241 let none = ReachPolicy { max_commits: 0 };
1242 let some = ReachPolicy { max_commits: 2 };
1243
1244 // The premise, measured off the tables themselves rather than assumed.
1245 // A cap that did not actually shrink the table would make every
1246 // comparison below a comparison of one code path with itself.
1247 let n_full = store.reach_bitmaps_with(full, false).unwrap().len();
1248 let n_none = store.reach_bitmaps_with(none, false).unwrap().len();
1249 let n_some = store.reach_bitmaps_with(some, false).unwrap().len();
1250 assert_eq!(
1251 n_full,
1252 commits.len(),
1253 "premise: the uncapped table must bitmap every commit"
1254 );
1255 assert_eq!(
1256 n_none, 0,
1257 "premise: the zero cap must produce NO bitmaps, or the walk arm is not a walk arm"
1258 );
1259 assert!(
1260 n_some > 0 && n_some < commits.len(),
1261 "premise: the middle cap must bitmap SOME commits and not all ({n_some} of {}), or \
1262 the stop-at-a-bitmap rule is never reached",
1263 commits.len()
1264 );
1265
1266 // A stride through the graph as `want`, each against its successor as
1267 // `have` — so the walk is asked about tips, roots and the middle.
1268 //
1269 // A stride and not every commit: `reachable_oids_with` rebuilds the
1270 // table on every call here (`cache: false`, and it must be — see that
1271 // method), so the exhaustive form is quadratic and ran for 334 s against
1272 // this fixture. Twelve probes spread across the history cost seconds and
1273 // cover the same three positions; the arms that matter are the three
1274 // caps, not the commit count.
1275 let stride = commits.len().div_ceil(12).max(1);
1276 let probes: Vec<usize> = (0..commits.len())
1277 .step_by(stride)
1278 .chain([commits.len() - 1])
1279 .collect();
1280 let mut walked_any = false;
1281 for i in probes {
1282 let w = &commits[i];
1283 let w_raw = hex::decode(w).unwrap();
1284 let h_raw = (i + 1 < commits.len()).then(|| hex::decode(&commits[i + 1]).unwrap());
1285 let haves: Vec<&[u8]> = h_raw.iter().map(|v| v.as_slice()).collect();
1286
1287 let reference: std::collections::BTreeSet<String> = store
1288 .reachable_oids_with(&[&w_raw], &haves, full, false)
1289 .unwrap()
1290 .into_iter()
1291 .collect();
1292
1293 for (label, policy) in [("cap 0", none), ("cap 2", some)] {
1294 let got: std::collections::BTreeSet<String> = store
1295 .reachable_oids_with(&[&w_raw], &haves, policy, false)
1296 .unwrap()
1297 .into_iter()
1298 .collect();
1299 let missing: Vec<&String> = reference.difference(&got).collect();
1300 let extra: Vec<&String> = got.difference(&reference).collect();
1301 assert!(
1302 missing.is_empty() && extra.is_empty(),
1303 "{label} vs the full table disagree for want {w}: MISSING {} object(s) the \
1304 full table selected, first {:?}; EXTRA {} object(s) the full table did not \
1305 select, first {:?}",
1306 missing.len(),
1307 missing.first(),
1308 extra.len(),
1309 extra.first()
1310 );
1311 }
1312 walked_any = true;
1313 }
1314 assert!(walked_any, "no commit was compared");
1315
1316 // And the answers are not vacuously equal because they are all empty.
1317 let tip = hex::decode(&commits[0]).unwrap();
1318 let n = store
1319 .reachable_oids_with(&[&tip], &[], none, false)
1320 .unwrap()
1321 .len();
1322 assert!(
1323 n > 1,
1324 "premise: a commit must reach more than itself for the comparison above to have any \
1325 content — got {n}"
1326 );
1327 }
1328
1329 /// **`want` minus `have` is an `andnot`, over objects and not just commits.**
1330 ///
1331 /// Built on a real repository's pack: the second commit's closure minus the
1332 /// first's must be exactly the objects the second commit introduced, and
1333 /// every returned oid must be one the store really holds.
1334 ///
1335 /// Seen RED by returning `union` instead of `union - had`: "removing the
1336 /// parent's closure removed nothing: 8 vs 8".
1337 ///
1338 /// MEASURED on the fixture: the child commit's closure is 8 objects, the
1339 /// parent's is 5, and the difference is 3 — the objects that commit
1340 /// introduced.
1341 #[test]
1342 fn reachable_is_want_minus_have_over_objects() {
1343 let dir = tmpdir("reach");
1344 let store = GitStore::open(&dir, "rickard").unwrap();
1345 let (pack, _) = real_pack();
1346 store.put(&pack, &[]).unwrap();
1347 store.absorb_pending().unwrap();
1348 assert!(store.commit_count() > 1, "the fixture has no history");
1349
1350 // Two commits in parent→child order: the graph is folded that way.
1351 let (child, parent) = {
1352 let mut pair = None;
1353 for c in store.graph_snapshot() {
1354 if let Some(p) = c.parents.first() {
1355 pair = Some((c.oid.clone(), p.clone()));
1356 break;
1357 }
1358 }
1359 pair.expect("a commit with a parent")
1360 };
1361 let child_raw = hex::decode(&child).unwrap();
1362 let parent_raw = hex::decode(&parent).unwrap();
1363
1364 let all = store.reachable(&[&child_raw], &[]).unwrap();
1365 let delta = store.reachable(&[&child_raw], &[&parent_raw]).unwrap();
1366 let had = store.reachable(&[&parent_raw], &[]).unwrap();
1367
1368 assert!(
1369 !all.is_empty(),
1370 "a commit reaches at least itself and its tree"
1371 );
1372 assert!(
1373 delta.len() < all.len(),
1374 "removing the parent's closure removed nothing: {} vs {}",
1375 delta.len(),
1376 all.len()
1377 );
1378 assert_eq!(
1379 all.len(),
1380 delta.len() + had.iter().filter(|h| all.contains(h)).count(),
1381 "the three sets do not add up — want minus have is not an andnot"
1382 );
1383 for oid in delta.iter().chain(all.iter()) {
1384 assert!(
1385 store.has(oid).unwrap(),
1386 "reachable named {} which the store does not hold",
1387 hex::encode(oid)
1388 );
1389 }
1390 // The commit itself is in its own closure; its parent is not in the delta.
1391 assert!(all.contains(&child_raw));
1392 assert!(!delta.contains(&parent_raw));
1393 eprintln!(
1394 "closure {} objects, minus the parent's {} leaves {}",
1395 all.len(),
1396 had.len(),
1397 delta.len()
1398 );
1399 }
1400
1401 /// **The batch of one really is slower**, measured here rather than asserted
1402 /// from the plan — and the serial path is what `extents` takes for it.
1403 ///
1404 /// The ratio is machine- and load-dependent, so the guard on it is loose (it
1405 /// requires the batch path to be no *faster*, which is the direction the
1406 /// dispatch depends on) while the number is printed for the record. The
1407 /// dispatch itself is asserted exactly.
1408 ///
1409 /// MEASURED, release, oden, loadavg 4.94, 2687 objects: `lookup` 628 ns,
1410 /// `lookup_batch` of one 639 ns (**1.02x worse**, not the 1.7x the bare Arrow
1411 /// arms show), `lookup_batch` of 100 **161 ns/oid** — 3.9x better than the
1412 /// serial path. See [`lookup_path`] for why both numbers are recorded.
1413 ///
1414 /// Seen RED by making `lookup_path` return `Batch` for every n:
1415 /// "assertion `left == right` failed — left: Batch, right: Serial".
1416 ///
1417 /// The honest limit: a *timing* assertion cannot catch `extents` ignoring the
1418 /// dispatch, because the two paths differ by 2% here. That is exactly why the
1419 /// dispatch is a named function with an exact assertion on it rather than a
1420 /// comment above an `if`.
1421 #[test]
1422 fn the_batch_of_one_really_is_slower_than_the_serial_path() {
1423 assert_eq!(lookup_path(1), LookupPath::Serial);
1424 assert_eq!(lookup_path(2), LookupPath::Batch);
1425 assert_eq!(lookup_path(0), LookupPath::Batch);
1426 assert_eq!(BATCH_SATURATES_AT, 100);
1427
1428 let dir = tmpdir("batch");
1429 let store = GitStore::open(&dir, "rickard").unwrap();
1430 let (pack, rows) = real_pack();
1431 store.put(&pack, &[]).unwrap();
1432 store.absorb_pending().unwrap();
1433 let oids: Vec<&[u8]> = rows.iter().map(|r| r.oid.as_slice()).collect();
1434
1435 // Both paths answer identically for one oid — the dispatch is an
1436 // optimisation, never a difference in the answer.
1437 for oid in oids.iter().take(64) {
1438 assert_eq!(
1439 store.extents(&[*oid]).unwrap(),
1440 store.index().extents_batch(&[*oid]),
1441 "the serial and batch paths disagree for {}",
1442 hex::encode(oid)
1443 );
1444 }
1445
1446 let n = 20_000usize;
1447 let serial = {
1448 let t = Instant::now();
1449 for i in 0..n {
1450 let _ = store.index().lookup(oids[i % oids.len()]);
1451 }
1452 t.elapsed().as_nanos() as f64 / n as f64
1453 };
1454 let batched = {
1455 let t = Instant::now();
1456 for i in 0..n {
1457 let _ = store.index().lookup_batch(&[oids[i % oids.len()]]);
1458 }
1459 t.elapsed().as_nanos() as f64 / n as f64
1460 };
1461 // What the batch path is actually for: 100 oids in one call, which is
1462 // where it saturates.
1463 let hundred: Vec<&[u8]> = oids.iter().take(BATCH_SATURATES_AT).copied().collect();
1464 let per_oid_at_100 = {
1465 let rounds = n / BATCH_SATURATES_AT;
1466 let t = Instant::now();
1467 for _ in 0..rounds {
1468 let _ = store.index().lookup_batch(&hundred);
1469 }
1470 t.elapsed().as_nanos() as f64 / (rounds * hundred.len()) as f64
1471 };
1472 eprintln!(
1473 "load {}; {} objects: lookup {serial:.0} ns, lookup_batch[1] {batched:.0} ns \
1474 ({:.2}x), lookup_batch[{}] {per_oid_at_100:.0} ns/oid",
1475 std::fs::read_to_string("/proc/loadavg")
1476 .unwrap_or_default()
1477 .trim(),
1478 store.index().len(),
1479 batched / serial,
1480 hundred.len(),
1481 );
1482 // 2026-08-10: the fused batch walk (no sort phase) collapsed the old
1483 // 1.02x batch-of-one penalty to a measured TIE — 604-620 ns both sides,
1484 // winner decided by noise, seen flipping run-to-run on an idle box. The
1485 // `Serial` dispatch for n=1 stays correct (equal time, two fewer Vec
1486 // allocations), so the guard keeps only the direction that would make
1487 // it WRONG: a batch of one materially faster than serial. 10% is
1488 // outside the tie's observed 0.3% jitter and inside any real win.
1489 assert!(
1490 batched >= serial * 0.9,
1491 "a batch of one measured MATERIALLY faster ({batched:.0} ns vs {serial:.0} ns) — \
1492 if that is repeatable the dispatch in `lookup_path` is wrong and must be \
1493 changed, not kept"
1494 );
1495 }
1496
1497 /// **`seal` folds the live logs into reserved Arrow sections**, and the
1498 /// generation numbers in the graph section are recomputed by that fold.
1499 ///
1500 /// Asserted by decoding the sections back out of their Arrow IPC bytes.
1501 ///
1502 /// Seen RED by dropping the `assign_generations` call from `refold`:
1503 /// "e58a5032ac3b36f15f08c759121efc8f08680e2b has generation 0".
1504 #[test]
1505 fn seal_emits_the_reserved_sections_and_recomputes_every_generation() {
1506 use znippy_common::{GUNNAR_GRAPH_MODULE, GUNNAR_REACH_MODULE, GUNNAR_REFS_MODULE};
1507
1508 let dir = tmpdir("seal");
1509 let store = GitStore::open(&dir, "rickard").unwrap();
1510 let (pack, _) = real_pack();
1511 store.put(&pack, &[]).unwrap();
1512 store.absorb_pending().unwrap();
1513 let tip = store
1514 .graph_snapshot()
1515 .iter()
1516 .max_by_key(|c| c.generation)
1517 .expect("a graph")
1518 .clone();
1519 store
1520 .update_ref(
1521 "refs/heads/main",
1522 None,
1523 Some(&hex::decode(&tip.oid).unwrap()),
1524 )
1525 .unwrap();
1526
1527 let sections = store.seal().unwrap();
1528 let names: Vec<&str> = sections.iter().map(|s| s.module_name.as_str()).collect();
1529 for want in [GUNNAR_REFS_MODULE, GUNNAR_GRAPH_MODULE, GUNNAR_REACH_MODULE] {
1530 assert!(names.contains(&want), "{want} is not sealed: {names:?}");
1531 }
1532
1533 // Generations: a root is 1, a child is 1 + max(parents), and the graph
1534 // this store folded must say so.
1535 let graph = store.graph_snapshot();
1536 let by_oid: std::collections::HashMap<&str, &crate::graph::CommitNode> =
1537 graph.iter().map(|c| (c.oid.as_str(), c)).collect();
1538 let mut with_parents = 0usize;
1539 for c in &graph {
1540 assert!(c.generation >= 1, "{} has generation 0", c.oid);
1541 let known: Vec<&crate::graph::CommitNode> = c
1542 .parents
1543 .iter()
1544 .filter_map(|p| by_oid.get(p.as_str()).copied())
1545 .collect();
1546 if known.is_empty() {
1547 assert_eq!(c.generation, 1, "{} is a root here", c.oid);
1548 } else {
1549 with_parents += 1;
1550 let expect = 1 + known.iter().map(|p| p.generation).max().unwrap();
1551 assert_eq!(
1552 c.generation, expect,
1553 "{} must have generation {expect}",
1554 c.oid
1555 );
1556 assert!(
1557 c.generation > 1,
1558 "a commit with a parent must have generation > 1"
1559 );
1560 }
1561 }
1562 assert!(with_parents > 0, "no commit in the fixture has a parent");
1563 eprintln!(
1564 "{} commits sealed, {with_parents} with parents",
1565 graph.len()
1566 );
1567 }
1568
1569 /// **`seal` writes generation 0, and `gc` then has something to compact.**
1570 ///
1571 /// This is the hole the four working steps of `gc()` used to fall into:
1572 /// absorb, live set, retire, drop all ran, and then
1573 /// `NewGeneration::run(archive_path())` died on
1574 /// `stat …/repository.znippy: No such file or directory` because **nothing
1575 /// ever created generation 0**. Every other test in this file that reached
1576 /// step 5 hand-built the archive with `znippy_common::create_archive`, which
1577 /// is why the gap survived: the fixture was doing the store's job.
1578 ///
1579 /// Asserted on **applied output**, through znippy's own reader and nothing
1580 /// else:
1581 ///
1582 /// * the live pack comes back out of the archive **byte for byte**, via
1583 /// `extract_file_verified`, which reconstructs the entry and blake3-checks
1584 /// it against the index — so a wrong checksum, a wrong extent or a wrong
1585 /// `compressed` flag all fail here rather than being written and believed;
1586 /// * the tombstoned pack has **no** entry, because a row for it is exactly
1587 /// what would stop the compaction reclaiming its bytes;
1588 /// * the three reserved sections are addressable **out of the file**, not
1589 /// merely present in the returned vector;
1590 /// * `gc()` then runs end to end and `NewGeneration` produces
1591 /// `repository.g1.znippy`, verified, with the old generation gone.
1592 ///
1593 /// # Seen RED, and one of the reds was in this guard
1594 ///
1595 /// Every mutation below was applied to the real code, run, and reverted.
1596 ///
1597 /// 1. **The hole itself.** With `seal` returning the sections and writing
1598 /// nothing — the code as it stood — this fails at
1599 /// `seal did not create …/repository.znippy`, and `gc()` fails with the
1600 /// original `compacting …/repository.znippy after dropping 2683 dead
1601 /// rows: stat …/repository.znippy: No such file or directory`.
1602 /// 2. **A dropped row** — `.take(packs.len() - 1)` on the row loop:
1603 /// `objects.pack.2 is not in the sealed archive: ["objects.pack.0"]`.
1604 /// **This is the mutation that first found a hollow guard, and the guard
1605 /// was mine.** Against the two-pack fixture this test started with, the
1606 /// row dropped was the tombstoned one, which is skipped anyway — the
1607 /// mutation stayed GREEN. The fixture now pushes three packs and
1608 /// tombstones the middle one, so there are two live rows and dropping
1609 /// either is visible.
1610 /// 3. **No tombstone honoured** — the `retired.contains` skip disabled:
1611 /// `the tombstoned pack got an index row anyway … ["objects.pack.0",
1612 /// "objects.pack.2", "objects.pack.1"]`. A row for a retired pack is what
1613 /// would keep its payload alive through every future compaction.
1614 /// 4. **A corrupted extent** — `blob_offset: offset + 1`. It writes, lists
1615 /// and opens perfectly; it dies at the blake3 gate inside
1616 /// `extract_file_verified`: `checksum mismatch for objects.pack.0 at
1617 /// fdata_offset 0`.
1618 /// 5. **The checksum domain** — hashing the *path* instead of the bytes,
1619 /// which is precisely the mistake that produces an archive that writes
1620 /// cleanly and verifies wrong. Same gate, same message. Together 4 and 5
1621 /// are what establish that the checksum is blake3 over the bytes at
1622 /// `blob_offset`, and that this test can tell.
1623 /// 6. **The `compressed` flag** — `true` on bytes that were stored raw:
1624 /// `OpenZL getDecompressedSize: ZL_getDecompressedSize failed`. The flag
1625 /// is load-bearing, not decorative.
1626 /// 7. **A lying report** — `packs_retired: 0` in the `SealReport` literal:
1627 /// `the report miscounts the tombstoned packs`. The report is read back
1628 /// by a second seal, which also pins that a seal is a snapshot: taking it
1629 /// twice yields the same entries, not a second generation.
1630 /// 8. **No reserved builder attached** — the sections are still *returned*,
1631 /// so a guard that only inspected the return value would pass:
1632 /// `__gunnar_refs__ is not in the sealed archive's manifest`. That is why
1633 /// the sections are read back out of the file by module name.
1634 ///
1635 #[test]
1636 fn seal_writes_generation_zero_that_gc_can_compact() {
1637 use znippy_common::{
1638 read_reserved_section_bytes, ZnippyArchive, ZnippyReader, GUNNAR_GRAPH_MODULE,
1639 GUNNAR_REACH_MODULE, GUNNAR_REFS_MODULE,
1640 };
1641
1642 let dir = tmpdir("seal-g0");
1643 let store = GitStore::open(&dir, "rickard").unwrap();
1644 // THREE packs, and the middle one is tombstoned before the seal. Two
1645 // live rows rather than one is not decoration: with a single live entry,
1646 // "dropped a row" and "wrote no rows at all" are the same failure, and a
1647 // `.take(packs.len() - 1)` mutation of the row loop stayed GREEN against
1648 // an earlier two-pack version of this fixture. Three packs with a gap in
1649 // the middle also pin the ordinal: it is the position among the
1650 // journal's `Pack` rows, so the survivors are 0 and **2**, not 0 and 1.
1651 let (pack, _) = real_pack();
1652 let (doomed, doomed_oid) = one_blob_pack(b"this pack is retired before the seal");
1653 let (kept, kept_oid) = one_blob_pack(b"this pack is unreferenced but not tombstoned");
1654 store.put(&pack, &[]).unwrap();
1655 let doomed_tx = store.put_pack(&doomed).unwrap();
1656 store.put_pack(&kept).unwrap();
1657 store.absorb_pending().unwrap();
1658 for (what, oid) in [("doomed", &doomed_oid), ("kept", &kept_oid)] {
1659 assert!(
1660 store.has(oid).unwrap(),
1661 "the {what} blob was never stored, so this fixture proves nothing"
1662 );
1663 }
1664
1665 // A ref, so the live set is not empty and `gc` is not refused.
1666 let root = store
1667 .graph_snapshot()
1668 .into_iter()
1669 .find(|c| c.generation == 1)
1670 .expect("a root commit");
1671 let root_raw = hex::decode(&root.oid).unwrap();
1672 store
1673 .update_ref("refs/heads/root", None, Some(&root_raw))
1674 .unwrap();
1675
1676 // Exactly one tombstone, written the way `gc` writes them. Named
1677 // explicitly rather than via `retire_dead_packs`, which would tombstone
1678 // BOTH one-blob packs and leave nothing to prove the ordinal with.
1679 crate::archive_write::retire_packs(
1680 &crate::archive_write::SafeWriter::journal_path(store.blobs_path()),
1681 &[doomed_tx.extent.unwrap().0],
1682 )
1683 .unwrap();
1684
1685 assert!(
1686 !store.archive_path().exists(),
1687 "something created the archive before the seal ran"
1688 );
1689 let sections = store.seal().unwrap();
1690
1691 // ── the archive exists and reads back through znippy's own reader ────
1692
1693 assert!(
1694 store.archive_path().exists(),
1695 "seal did not create {}",
1696 store.archive_path().display()
1697 );
1698 let ar = ZnippyArchive::open(store.archive_path()).unwrap();
1699 let listed = ar.list_files().unwrap();
1700 assert!(
1701 listed.contains(&"objects.pack.0".to_string()),
1702 "objects.pack.0 is not in the sealed archive: {listed:?}"
1703 );
1704 assert!(
1705 listed.contains(&"objects.pack.2".to_string()),
1706 "objects.pack.2 is not in the sealed archive: {listed:?}"
1707 );
1708 assert!(
1709 !listed.contains(&"objects.pack.1".to_string()),
1710 "the tombstoned pack got an index row anyway, so the compaction can never \
1711 reclaim its bytes: {listed:?}"
1712 );
1713 assert_eq!(
1714 listed.len(),
1715 2,
1716 "the seal wrote entries for packs the journal never acked as live: {listed:?}"
1717 );
1718
1719 // The bytes, reconstructed and blake3-checked by the reader itself.
1720 assert_eq!(
1721 ar.extract_file_verified("objects.pack.0").unwrap(),
1722 pack,
1723 "objects.pack.0 does not read back as the pack that was pushed"
1724 );
1725 assert_eq!(
1726 ar.extract_file_verified("objects.pack.2").unwrap(),
1727 kept,
1728 "objects.pack.2 does not read back as the pack that was pushed"
1729 );
1730
1731 // ── the reserved sections, addressed out of the file ─────────────────
1732 let names: Vec<&str> = sections.iter().map(|s| s.module_name.as_str()).collect();
1733 for want in [GUNNAR_REFS_MODULE, GUNNAR_GRAPH_MODULE, GUNNAR_REACH_MODULE] {
1734 assert!(names.contains(&want), "{want} was not returned: {names:?}");
1735 let bytes = read_reserved_section_bytes(store.archive_path(), want)
1736 .unwrap()
1737 .unwrap_or_else(|| panic!("{want} is not in the sealed archive's manifest"));
1738 assert!(!bytes.is_empty(), "{want} is sealed as zero bytes");
1739 }
1740
1741 // ── the report, and that a second seal is not a different archive ───
1742 //
1743 // A count nobody asserts on is a count that can be wrong for ever, so
1744 // the report is read here — and reading it means sealing again, which
1745 // pins the other half: a seal is a snapshot and re-taking it produces
1746 // the same entries, not a second generation.
1747 let again = store
1748 .seal_archive(store.reserved_sections().unwrap())
1749 .unwrap();
1750 assert_eq!(
1751 again.packs_sealed, 2,
1752 "the report miscounts the sealed packs"
1753 );
1754 assert_eq!(
1755 again.packs_retired, 1,
1756 "the report miscounts the tombstoned packs"
1757 );
1758 assert_eq!(
1759 again.packs_after_copy, 0,
1760 "nothing was pushed during this seal, so nothing can have raced it"
1761 );
1762 let mut once = listed.clone();
1763 once.sort();
1764 let mut twice = ZnippyArchive::open(store.archive_path())
1765 .unwrap()
1766 .list_files()
1767 .unwrap();
1768 twice.sort();
1769 // Sorted on both sides: `list_files` does not promise an order, and it
1770 // was observed returning the same two entries the other way round.
1771 assert_eq!(
1772 twice, once,
1773 "sealing twice produced a different set of entries"
1774 );
1775
1776 // ── and now the fifth step of `gc` has something to compact ──────────
1777 let report = store.gc().unwrap();
1778 assert_eq!(report.strategy, "NewGeneration");
1779 assert!(report.verified, "the new generation was not read back");
1780 assert_eq!(
1781 report.archive,
1782 store.archive_path().with_file_name("repository.g1.znippy"),
1783 "NewGeneration did not produce generation 1"
1784 );
1785 assert!(report.archive.exists(), "the new generation is not on disk");
1786 assert!(
1787 !store.archive_path().exists(),
1788 "the old generation was not retired"
1789 );
1790 assert!(
1791 report.bytes_after < report.bytes_before,
1792 "the compaction reclaimed nothing: {} → {} bytes — the retired pack's payload \
1793 should have gone",
1794 report.bytes_before,
1795 report.bytes_after
1796 );
1797
1798 // The live pack survives the compaction; the retired one's row never
1799 // existed, so its bytes are what got reclaimed.
1800 let g1 = ZnippyArchive::open(&report.archive).unwrap();
1801 assert_eq!(
1802 g1.extract_file_verified("objects.pack.0").unwrap(),
1803 pack,
1804 "generation 1 does not carry the live pack"
1805 );
1806 eprintln!(
1807 "seal: {} bytes, gc: {} → {} bytes, {}",
1808 std::fs::metadata(&report.archive)
1809 .map(|m| m.len())
1810 .unwrap_or(0),
1811 report.bytes_before,
1812 report.bytes_after,
1813 report.archive.display()
1814 );
1815 }
1816
1817 /// **GC in the order §13.20 fixes**: reachability, then the dead index rows,
1818 /// then base znippy's compaction.
1819 ///
1820 /// The archive is a real znippy archive built the way `gc`'s own fixture
1821 /// builds one, so step 3 really runs `compact_archive` and really produces a
1822 /// new generation. Steps 1 and 2 are asserted as applied output: the
1823 /// unreachable oid stops resolving and the reachable one still does.
1824 ///
1825 /// Seen RED by having `gc` drop rows *before* computing the live set — the
1826 /// order §13.20 fixes, inverted: "gc dropped the live object too". Seen RED a
1827 /// second time by making `drop_dead_rows` return `Ok(0)` without removing
1828 /// anything: "the dead object still resolves after gc".
1829 ///
1830 /// MEASURED on the fixture: a ref on the root commit leaves **5** live objects
1831 /// of 2687, and all 2682 dead rows go.
1832 #[test]
1833 fn gc_computes_reachability_drops_the_dead_rows_then_compacts() {
1834 let dir = tmpdir("gc");
1835 let store = GitStore::open(&dir, "rickard").unwrap();
1836 let (pack, _) = real_pack();
1837 store.put(&pack, &[]).unwrap();
1838 store.absorb_pending().unwrap();
1839
1840 // A ref on one commit, and an object that commit cannot reach.
1841 let graph = store.graph_snapshot();
1842 let root = graph
1843 .iter()
1844 .find(|c| c.generation == 1)
1845 .expect("a root commit")
1846 .clone();
1847 let root_raw = hex::decode(&root.oid).unwrap();
1848 let live_from_root: std::collections::HashSet<Vec<u8>> = store
1849 .reachable(&[&root_raw], &[])
1850 .unwrap()
1851 .into_iter()
1852 .collect();
1853 let all: Vec<Vec<u8>> = store.index().oids_in_order().unwrap();
1854 let dead = all
1855 .iter()
1856 .find(|o| !live_from_root.contains(*o))
1857 .expect("the root does not reach everything in a real repository")
1858 .clone();
1859
1860 store
1861 .update_ref("refs/heads/root", None, Some(&root_raw))
1862 .unwrap();
1863
1864 // A real znippy archive for step 3 to compact.
1865 let files = vec![
1866 ("pack-0.pack".to_string(), pack.clone()),
1867 ("pack-1.pack".to_string(), pack.clone()),
1868 ];
1869 znippy_common::create_archive(store.archive_path(), &files, 3).unwrap();
1870
1871 let before = store.index().len();
1872 let report = store.gc().unwrap();
1873
1874 assert!(
1875 store.has(&root_raw).unwrap(),
1876 "gc dropped the live object too"
1877 );
1878 assert!(
1879 !store.has(&dead).unwrap(),
1880 "the dead object still resolves after gc"
1881 );
1882 assert!(
1883 store.index().len() < before,
1884 "gc dropped nothing: {} rows before and after",
1885 before
1886 );
1887 assert_eq!(
1888 store.index().len(),
1889 live_from_root.len(),
1890 "exactly the live set survives"
1891 );
1892 assert_eq!(report.strategy, "NewGeneration");
1893 assert!(report.verified, "the new generation was not read back");
1894 assert!(report.archive.exists(), "the new generation is not on disk");
1895 assert!(
1896 !store.archive_path().exists(),
1897 "the old generation was not retired"
1898 );
1899 eprintln!(
1900 "gc: {before} rows → {} live, {} → {} bytes, {}",
1901 store.index().len(),
1902 report.bytes_before,
1903 report.bytes_after,
1904 report.archive.display()
1905 );
1906
1907 // And a repository with no refs at all is refused rather than emptied.
1908 let dir2 = tmpdir("gc-norefs");
1909 let empty = GitStore::open(&dir2, "rickard").unwrap();
1910 empty.put(&pack, &[]).unwrap();
1911 empty.absorb_pending().unwrap();
1912 let n = empty.index().len();
1913 assert!(empty.gc().is_err(), "a GC that would delete everything ran");
1914 assert_eq!(empty.index().len(), n, "the refused GC dropped rows anyway");
1915 }
1916
1917 // ── the GC's journal half: a pack that is *wholly* dead ──────────────────
1918
1919 /// What [`one_live_pack_and_one_doomed_pack`] hands back: an open store, and
1920 /// the three oids the guards ask about.
1921 struct DoomedFixture {
1922 dir: PathBuf,
1923 store: GitStore,
1924 /// The only object of the pack that dies whole.
1925 doomed: Vec<u8>,
1926 /// The commit a ref points at.
1927 root: Vec<u8>,
1928 /// That commit's tree — a live object that is not the ref itself.
1929 tree: Vec<u8>,
1930 }
1931
1932 /// One repository with two packs: a real one that a ref reaches into, and a
1933 /// one-blob pack nothing will ever point at.
1934 ///
1935 /// The second pack is what makes the difference visible: after a GC it has
1936 /// **no rows at all**, which is the state a pack that was in flight when the
1937 /// machine died also leaves behind.
1938 ///
1939 /// The store is returned **open**, so a guard runs its first GC in the same
1940 /// process lifetime that pushed. The reopen is the thing under test and it
1941 /// belongs in the guard, not in the fixture.
1942 fn one_live_pack_and_one_doomed_pack(tag: &str) -> DoomedFixture {
1943 let dir = tmpdir(tag);
1944 let (pack, _) = real_pack();
1945 let (doomed, doomed_oid) = one_blob_pack(b"no ref will ever reach this blob");
1946 let store = GitStore::open(&dir, "rickard").unwrap();
1947 store.put(&pack, &[]).unwrap();
1948 store.put_pack(&doomed).unwrap();
1949 store.absorb_pending().unwrap();
1950 assert!(
1951 store.has(&doomed_oid).unwrap(),
1952 "the doomed blob was never stored, so this fixture proves nothing"
1953 );
1954
1955 let root = store
1956 .graph_snapshot()
1957 .into_iter()
1958 .find(|c| c.generation == 1)
1959 .expect("a root commit");
1960 let root_raw = hex::decode(&root.oid).unwrap();
1961 store
1962 .update_ref("refs/heads/root", None, Some(&root_raw))
1963 .unwrap();
1964 let tree = hex::decode(root.tree.as_ref().expect("the root commit names a tree")).unwrap();
1965 assert!(store.has(&tree).unwrap(), "the root's tree is not indexed");
1966
1967 // A real znippy archive, so the compaction step has something to compact.
1968 let files = vec![
1969 ("pack-0.pack".to_string(), pack.clone()),
1970 ("pack-1.pack".to_string(), pack.clone()),
1971 ];
1972 znippy_common::create_archive(store.archive_path(), &files, 3).unwrap();
1973 DoomedFixture {
1974 dir,
1975 store,
1976 doomed: doomed_oid,
1977 root: root_raw,
1978 tree,
1979 }
1980 }
1981
1982 /// The journal's rows, split into packs and tombstones, read off the disk.
1983 fn journal_state(dir: &Path) -> (Vec<(u64, u64)>, Vec<u64>) {
1984 use crate::archive_write::{acked_packs, read_journal, retired_offsets, SafeWriter};
1985 let rows = read_journal(&SafeWriter::journal_path(&dir.join("objects.pack"))).unwrap();
1986 let mut retired: Vec<u64> = retired_offsets(&rows).into_iter().collect();
1987 retired.sort_unstable();
1988 (acked_packs(&rows), retired)
1989 }
1990
1991 /// **THE BUG: a pack whose every object a GC found dead came back on the
1992 /// next open.**
1993 ///
1994 /// The `indexed` bit is derived as *extent in the journal, rows not in the
1995 /// index*, and a GC that drops every row of a pack produces exactly that
1996 /// state — so the reopen re-queued the pack and re-absorbed the objects the
1997 /// GC had just decided were dead. A partly dead pack keeps rows and was never
1998 /// affected, which is why this needs a pack that dies **whole**.
1999 ///
2000 /// Asserted on applied output on both sides of a process-lifetime boundary:
2001 /// the tombstone rows on disk, `has()` on the dead oid, and the object count
2002 /// after the reopen's drain has been waited for — a resurrection puts the row
2003 /// back, so counting is not a proxy for it, it *is* it.
2004 ///
2005 /// Seen RED by making `Absorber::adopt_journal` ignore the tombstones
2006 /// (`else if retired.contains(&extent.0)` → `else if false`), which is
2007 /// exactly the reader as it stood before this fix: "a dead object came back
2008 /// the instant the store was reopened". Restored.
2009 ///
2010 /// Seen RED a second time by not retiring anything in `GitOps::gc` (`let
2011 /// retired: Vec<u64> = Vec::new();` in place of the `retire_dead_packs`
2012 /// call — the writer as it stood before this fix): "gc retired 0 pack(s), not
2013 /// the one whose objects all died — left: 0, right: 1". Restored.
2014 ///
2015 /// Seen RED a third time by computing deadness the way a retirement written
2016 /// *after* the drop would have to (`occupied[i]` → `!occupied[i]` in
2017 /// `retire_dead_packs`): the same "gc retired 0 pack(s)" — after the drop an
2018 /// all-dead pack is indistinguishable from an unabsorbed one, which is the
2019 /// same confusion this whole change is about, one level up. Restored.
2020 ///
2021 /// And the derived half, seen RED by removing the `refold()` from
2022 /// `drop_dead_rows`: "the graph still holds 551 commits after gc dropped
2023 /// every dead one — left: 551, right: 1", with `reachable()` still selecting
2024 /// objects the index can no longer serve. Restored.
2025 #[test]
2026 fn a_wholly_dead_pack_does_not_come_back_when_the_store_is_reopened() {
2027 let DoomedFixture {
2028 dir,
2029 store,
2030 doomed: doomed_oid,
2031 root: root_raw,
2032 tree,
2033 } = one_live_pack_and_one_doomed_pack("gc-wholly-dead");
2034
2035 let (rows_after_gc, retired_by_gc) = {
2036 let report = store.gc().unwrap();
2037 assert_eq!(
2038 report.retired_packs, 1,
2039 "gc retired {} pack(s), not the one whose objects all died",
2040 report.retired_packs
2041 );
2042 assert!(
2043 !store.has(&doomed_oid).unwrap(),
2044 "the dead object still resolves in the store that dropped it"
2045 );
2046 assert!(store.has(&root_raw).unwrap(), "gc dropped the live commit");
2047
2048 // **Everything the indexer derived is dropped with the rows, in the
2049 // same process** — not merely rebuilt by the next open. The graph
2050 // holds one commit, and a selection over the bitmaps and the ordinal
2051 // space cannot name a dead object.
2052 assert_eq!(
2053 store.commit_count(),
2054 1,
2055 "the graph still holds {} commits after gc dropped every dead one",
2056 store.commit_count()
2057 );
2058 let selected = store.reachable(&[&root_raw], &[]).unwrap();
2059 assert!(
2060 !selected.contains(&doomed_oid),
2061 "a selection after gc still names a dead object — the bitmaps or the ordinal \
2062 space were not refolded"
2063 );
2064 assert_eq!(
2065 selected.len(),
2066 store.index().len(),
2067 "a selection after gc names {} objects and the index holds {}",
2068 selected.len(),
2069 store.index().len()
2070 );
2071 (store.index().len(), report.retired_packs)
2072 };
2073 drop(store);
2074
2075 // The tombstone is on disk, and it names the doomed pack's offset — not
2076 // the live one's.
2077 let (packs, retired) = journal_state(&dir);
2078 assert_eq!(packs.len(), 2, "both packs are still acked: {packs:?}");
2079 assert_eq!(
2080 retired,
2081 vec![packs[1].0],
2082 "the journal retired {retired:?}, and the doomed pack starts at {}",
2083 packs[1].0
2084 );
2085 assert_eq!(retired.len() as u64, retired_by_gc);
2086
2087 // ── the reopen: the whole point ──────────────────────────────────────
2088 let store = GitStore::open(&dir, "rickard").unwrap();
2089 assert!(
2090 !store.has(&doomed_oid).unwrap(),
2091 "a dead object came back the instant the store was reopened"
2092 );
2093 store.wait_indexed();
2094 assert!(
2095 !store.has(&doomed_oid).unwrap(),
2096 "the reopen re-queued the wholly dead pack and its objects came back"
2097 );
2098 assert_eq!(
2099 store.index().len(),
2100 rows_after_gc,
2101 "the dead objects came back on the reopen: the store that dropped them holds \
2102 {rows_after_gc} rows, the reopened one holds {}",
2103 store.index().len()
2104 );
2105 assert_eq!(
2106 store.unindexed_packs(),
2107 0,
2108 "the retired pack is queued as work"
2109 );
2110 assert_eq!(
2111 store.absorb_pending().unwrap(),
2112 0,
2113 "a read falling back would re-absorb the retired pack"
2114 );
2115 // And the live side of the same repository is untouched by all of it.
2116 assert!(store.has(&root_raw).unwrap(), "the live commit is gone");
2117 assert!(store.has(&tree).unwrap(), "the live tree is gone");
2118 assert_eq!(
2119 store.commit_count(),
2120 1,
2121 "the graph after gc + reopen holds {} commits, not the one live one",
2122 store.commit_count()
2123 );
2124 }
2125
2126 /// **A partly dead pack is not retired, and comes through a GC and a reopen
2127 /// with its live objects.**
2128 ///
2129 /// This is the case that already worked and must keep working: the pack still
2130 /// has rows, so the crash-recovery diff calls it absorbed and nothing about
2131 /// it changes. Asserted on applied output: **no** tombstone appears in the
2132 /// journal, the live objects read back after the reopen, and the dead ones
2133 /// stay dead.
2134 ///
2135 /// Seen RED by retiring on `has_live[i]` instead of `!has_live[i]` in
2136 /// `retire_dead_packs` — i.e. tombstoning the packs that are *alive*: "gc
2137 /// retired the partly dead pack — its live objects are one reopen from a
2138 /// fallback that will never come — left: 1, right: 0", and the journal
2139 /// assertion below holds the same finding in on-disk bytes. The live objects
2140 /// still read back under that mutation, because a pack with rows is adopted
2141 /// as absorbed whatever the journal says about it — which is exactly why this
2142 /// guard asserts on the journal and not only on the reads. Restored.
2143 #[test]
2144 fn a_partly_dead_pack_is_never_retired_and_survives_a_reopen() {
2145 let dir = tmpdir("gc-partly-dead");
2146 let (pack, _) = real_pack();
2147 let store = GitStore::open(&dir, "rickard").unwrap();
2148 store.put(&pack, &[]).unwrap();
2149 store.absorb_pending().unwrap();
2150
2151 let root = store
2152 .graph_snapshot()
2153 .into_iter()
2154 .find(|c| c.generation == 1)
2155 .expect("a root commit");
2156 let root_raw = hex::decode(&root.oid).unwrap();
2157 let live: std::collections::HashSet<Vec<u8>> = store
2158 .reachable(&[&root_raw], &[])
2159 .unwrap()
2160 .into_iter()
2161 .collect();
2162 let all: Vec<Vec<u8>> = store.index().oids_in_order().unwrap();
2163 let dead = all
2164 .iter()
2165 .find(|o| !live.contains(*o))
2166 .expect("a real pack holds more than one commit's closure")
2167 .clone();
2168 assert!(
2169 live.len() < all.len(),
2170 "the fixture pack is not partly dead: {} live of {}",
2171 live.len(),
2172 all.len()
2173 );
2174 store
2175 .update_ref("refs/heads/root", None, Some(&root_raw))
2176 .unwrap();
2177 let files = vec![("pack-0.pack".to_string(), pack.clone())];
2178 znippy_common::create_archive(store.archive_path(), &files, 3).unwrap();
2179
2180 let report = store.gc().unwrap();
2181 assert_eq!(
2182 report.retired_packs, 0,
2183 "gc retired the partly dead pack — its live objects are one reopen from a fallback \
2184 that will never come"
2185 );
2186 let rows_after_gc = store.index().len();
2187 drop(store);
2188
2189 let (packs, retired) = journal_state(&dir);
2190 assert_eq!(packs.len(), 1);
2191 assert!(
2192 retired.is_empty(),
2193 "gc tombstoned a partly dead pack: {retired:?}"
2194 );
2195
2196 let store = GitStore::open(&dir, "rickard").unwrap();
2197 store.wait_indexed();
2198 assert_eq!(
2199 store.index().len(),
2200 rows_after_gc,
2201 "the reopen changed the row count of a partly dead pack's repository"
2202 );
2203 for oid in &live {
2204 assert!(
2205 store.has(oid).unwrap(),
2206 "the live object {} did not survive gc + reopen",
2207 hex::encode(oid)
2208 );
2209 }
2210 assert!(
2211 !store.has(&dead).unwrap(),
2212 "a dead object came back on the reopen of a partly dead pack"
2213 );
2214 // The pack was adopted as absorbed rather than re-queued: no work is
2215 // owed, and a read falling back would re-absorb it.
2216 assert_eq!(
2217 store.unindexed_packs(),
2218 0,
2219 "the surviving pack was re-queued"
2220 );
2221 assert_eq!(store.absorb_pending().unwrap(), 0);
2222 }
2223
2224 /// **An interruption at each step of the retirement leaves a readable
2225 /// archive and loses nothing.**
2226 ///
2227 /// The two steps are run as `GitOps::gc` runs them and the process boundary
2228 /// is a real one — the store is dropped and reopened between them, which is
2229 /// what a `kill -9` there would leave, because everything each step writes is
2230 /// fsynced before it returns.
2231 ///
2232 /// | killed after | on disk | what must be true |
2233 /// |---|---|---|
2234 /// | the tombstone | rows still there | **nothing is lost** — every object still reads, dead ones included |
2235 /// | the drop | rows gone, no compaction | the dead objects are gone and **stay** gone |
2236 /// | the compaction | a new generation | `gc.rs`'s own `an_interruption_at_every_step_leaves_a_readable_archive` |
2237 ///
2238 /// The first row is why the tombstone goes **first**: a GC killed there is a
2239 /// GC that did not happen, and the repository is exactly as it was. The
2240 /// reverse order has a window where the rows are gone and the journal still
2241 /// claims an unabsorbed pack, which is the resurrection this whole change is
2242 /// about.
2243 ///
2244 /// The archive itself is read back through the ordinary reader at every
2245 /// checkpoint, so "readable" is the whole file decoding, not a stat.
2246 ///
2247 /// Seen RED by computing deadness from `!occupied[i]` in `retire_dead_packs`
2248 /// — the shape a retirement written after the drop needs, which cannot see a
2249 /// pack that still has its rows: "the doomed pack was not retired: [] —
2250 /// left: 0, right: 1", at the first checkpoint, with every row still in the
2251 /// index. Restored.
2252 #[test]
2253 fn an_interruption_at_each_step_of_the_retirement_loses_nothing() {
2254 let DoomedFixture {
2255 dir,
2256 store,
2257 doomed: doomed_oid,
2258 root: root_raw,
2259 tree,
2260 } = one_live_pack_and_one_doomed_pack("gc-interrupt");
2261 let archive = dir.join("repository.znippy");
2262 let readable = |where_: &str| {
2263 crate::gc::read_back_every_entry(&archive)
2264 .unwrap_or_else(|e| panic!("the archive does not read back {where_}: {e:?}"));
2265 };
2266 readable("before anything ran");
2267
2268 // ── killed right after the tombstone, before any row was dropped ─────
2269 //
2270 // The live set is computed once and carried, which is what `gc()` does
2271 // with it too: it is an input to both steps, and computing it twice would
2272 // make this guard depend on a reachability fold rather than on the two
2273 // mutations it is here to interrupt.
2274 let (live, rows_before) = {
2275 let live = store.live_set().unwrap();
2276 let retired = store.retire_dead_packs(&live).unwrap();
2277 assert_eq!(
2278 retired.len(),
2279 1,
2280 "the doomed pack was not retired: {retired:?}"
2281 );
2282 let n = store.index().len();
2283 assert!(
2284 store.has(&doomed_oid).unwrap(),
2285 "retiring the journal row dropped an index row — the two steps are not separable"
2286 );
2287 (live, n)
2288 };
2289 drop(store);
2290 let (packs, on_disk) = journal_state(&dir);
2291 assert_eq!(
2292 on_disk,
2293 vec![packs[1].0],
2294 "the tombstone is not on disk before a single row was dropped: {on_disk:?}"
2295 );
2296 readable("after the tombstone");
2297
2298 {
2299 // The reopen a crash there produces: everything is still here.
2300 let store = GitStore::open(&dir, "rickard").unwrap();
2301 store.wait_indexed();
2302 assert_eq!(
2303 store.index().len(),
2304 rows_before,
2305 "a crash between the tombstone and the drop lost rows"
2306 );
2307 assert!(
2308 store.has(&doomed_oid).unwrap(),
2309 "a crash between the tombstone and the drop lost the objects of the pack it \
2310 retired — the GC had not decided anything yet"
2311 );
2312 assert!(store.has(&root_raw).unwrap() && store.has(&tree).unwrap());
2313 // And resuming is idempotent: the pack is already retired.
2314 assert!(
2315 store.retire_dead_packs(&live).unwrap().is_empty(),
2316 "the resumed GC retired the same pack twice"
2317 );
2318 }
2319
2320 // ── killed after the drop, before the compaction ─────────────────────
2321 let rows_after_drop = {
2322 let store = GitStore::open(&dir, "rickard").unwrap();
2323 let dropped = store.drop_dead_rows(&live).unwrap();
2324 assert!(dropped > 0, "the drop dropped nothing");
2325 assert!(!store.has(&doomed_oid).unwrap());
2326 store.index().len()
2327 };
2328 readable("after the drop, before the compaction");
2329 let (_, still_retired) = journal_state(&dir);
2330 assert_eq!(still_retired.len(), 1, "the drop lost the tombstone");
2331
2332 {
2333 let store = GitStore::open(&dir, "rickard").unwrap();
2334 store.wait_indexed();
2335 assert!(
2336 !store.has(&doomed_oid).unwrap(),
2337 "a crash between the drop and the compaction resurrected the dead objects"
2338 );
2339 assert_eq!(store.index().len(), rows_after_drop);
2340 assert!(store.has(&root_raw).unwrap() && store.has(&tree).unwrap());
2341
2342 // ── and the compaction still completes on the resumed store ──────
2343 let report = store.gc().unwrap();
2344 assert_eq!(
2345 report.retired_packs, 0,
2346 "the resumed gc retired an already-retired pack"
2347 );
2348 assert!(
2349 report.verified,
2350 "the resumed gc did not verify its generation"
2351 );
2352 assert!(report.archive.exists());
2353 crate::gc::read_back_every_entry(&report.archive)
2354 .expect("the new generation does not read back");
2355 assert!(!store.has(&doomed_oid).unwrap());
2356 assert!(store.has(&root_raw).unwrap());
2357 }
2358 }
2359
2360 /// `compact` leaves its source untouched and refuses to overwrite.
2361 ///
2362 /// Seen RED by removing the `dst.exists()` refusal: "the refusal must be the
2363 /// explicit one and must name the file: hard-linking …/repo.znippy to
2364 /// …/repo.g1.znippy — compaction runs against a second name for the same
2365 /// inode …". The first version of this guard only asserted *that* it failed,
2366 /// and that version stayed green under the same mutation, because `hard_link`
2367 /// refuses EEXIST as well. It now asserts which mechanism refused.
2368 #[test]
2369 fn compact_names_its_destination_and_never_overwrites() {
2370 let dir = tmpdir("compact");
2371 let src = dir.join("repo.znippy");
2372 let dst = dir.join("repo.g1.znippy");
2373 let files = vec![
2374 ("pack-0.pack".to_string(), vec![7u8; 400_000]),
2375 ("pack-1.pack".to_string(), vec![9u8; 400_000]),
2376 ];
2377 znippy_common::create_archive(&src, &files, 3).unwrap();
2378 let before = std::fs::read(&src).unwrap();
2379
2380 compact(&src, &dst).unwrap();
2381 assert!(dst.exists(), "compact produced no destination");
2382 assert_eq!(
2383 std::fs::read(&src).unwrap(),
2384 before,
2385 "compact modified its source"
2386 );
2387
2388 let err = compact(&src, &dst).expect_err("dst exists");
2389 // Asserted on WHICH mechanism refused, not merely that something did:
2390 // `hard_link` would fail with EEXIST too, so a guard that only checked
2391 // for an error could not tell the explicit refusal from the syscall's —
2392 // and the difference is whether a filesystem operation is attempted
2393 // against a file that must not be touched at all.
2394 assert!(
2395 err.to_string().contains("refusing to compact over it")
2396 && err.to_string().contains(&dst.display().to_string()),
2397 "the refusal must be the explicit one and must name the file: {err}"
2398 );
2399 assert_eq!(
2400 std::fs::read(&src).unwrap(),
2401 before,
2402 "the refused second compaction touched the source"
2403 );
2404 }
2405}
2406
2407#[cfg(test)]
2408mod emit_set_tests {
2409 use crate::index_layout::ObjectIndex as _;
2410 use crate::pack_walk::topological_order;
2411 use crate::store::tests::{real_pack, tmpdir};
2412 use crate::{GitOps, GitStore};
2413
2414 /// **A SUBSET emits a pack holding EXACTLY the subset, and stock git accepts
2415 /// it.**
2416 ///
2417 /// Emitting a whole pack proves the encoder; it cannot prove anything about
2418 /// a request whose delta bases fall outside it, because every base is
2419 /// present by construction. A subset is that case.
2420 ///
2421 /// # What this asserted until 2026-08-11, and why it inverted
2422 ///
2423 /// It asserted `entries.len() >= half.len()` under the heading *"closure
2424 /// pulled in nothing"* — the set was expected to come back **larger**,
2425 /// because `emit_set` added the delta bases. That is the defect: a base
2426 /// pulled into a narrowed request is an object the client did not ask for,
2427 /// and a tree pulled in that way owes children the pack does not carry. See
2428 /// [`GitStore::emit_set`] for the production failure and the counts.
2429 ///
2430 /// So the assertion is now an equality, and it is the strong direction: the
2431 /// emitted set is the requested set, entry for entry. The delta whose base
2432 /// was left out is emitted **whole**, which is what `pack-objects` does and
2433 /// what keeps the pack connected.
2434 #[test]
2435 fn a_subset_emits_exactly_the_subset_and_stock_git_accepts_it() {
2436 if std::process::Command::new("git")
2437 .arg("--version")
2438 .output()
2439 .is_err()
2440 {
2441 eprintln!("skipping: no git on PATH");
2442 return;
2443 }
2444 let dir = tmpdir("emit-subset");
2445 let store = GitStore::open(&dir, "rickard").unwrap();
2446 let (pack, _) = real_pack();
2447 store.put_pack(&pack).unwrap();
2448 store.absorb_pending().unwrap();
2449
2450 let all = store.index().oids_in_order().unwrap();
2451 assert!(all.len() > 8, "corpus too small to take a subset of");
2452 // Every other object: enough deltas land with their base excluded that
2453 // the boundary rule has real work to do.
2454 let half: Vec<&[u8]> = all.iter().step_by(2).map(Vec::as_slice).collect();
2455 let rows = store.index().lookup_batch(&half);
2456 let inside: std::collections::HashSet<u64> =
2457 rows.iter().flatten().map(|r| r.offset).collect();
2458 let boundary = rows
2459 .iter()
2460 .flatten()
2461 .filter(|r| r.delta_base != 0 && !inside.contains(&r.delta_base))
2462 .count();
2463 assert!(
2464 boundary > 0,
2465 "this subset excludes no delta base, so it cannot tell an exact emission from a \
2466 closed one and the test proves nothing"
2467 );
2468
2469 let entries = store.emit_set(&half, true, None).unwrap();
2470 assert_eq!(
2471 entries.len(),
2472 half.len(),
2473 "the emitted set must be the requested set — {} asked, {} emitted",
2474 half.len(),
2475 entries.len()
2476 );
2477 assert_eq!(
2478 entries.iter().filter(|e| e.recompressed).count(),
2479 boundary,
2480 "exactly the entries whose base was excluded must be rebuilt whole"
2481 );
2482
2483 let (ordered, missing) = topological_order(entries);
2484 assert!(
2485 missing.is_empty(),
2486 "no entry may still name a base that is not here: {missing:?}"
2487 );
2488
2489 // Walk what we emitted, before handing it to git. If the header count
2490 // and the body disagree, that is the shape that crashes an indexer.
2491 {
2492 let mut b = Vec::new();
2493 store.emit_ordered(&ordered, &mut b).unwrap();
2494 let declared = u32::from_be_bytes([b[8], b[9], b[10], b[11]]);
2495 match crate::pack_walk::walk(&b, store.hash_kind().oid_len()) {
2496 Ok(w) => assert_eq!(
2497 w.entries.len(),
2498 declared as usize,
2499 "the pack header declares {declared} entries and the body walks to {}",
2500 w.entries.len()
2501 ),
2502 Err(e) => panic!("our own walk cannot read what we emitted: {e:#}"),
2503 }
2504 }
2505
2506 let mut bytes = Vec::new();
2507 let report = store.emit_ordered(&ordered, &mut bytes).unwrap();
2508 assert_eq!(
2509 report.copied + report.recompressed,
2510 report.written,
2511 "every entry is either copied or rebuilt, and the receipt must add up"
2512 );
2513 assert_eq!(
2514 report.recompressed as usize, boundary,
2515 "only the excluded-base entries may be rebuilt"
2516 );
2517
2518 // **No `--strict` here, and that is deliberate.** An arbitrary every-other
2519 // subset is not reachability-closed — a tree in it will name a blob that
2520 // is not — so `--strict`'s connectivity walk would be judging the
2521 // *caller's* selection, not this emitter. What is being asserted is
2522 // SELF-CONTAINMENT: that every delta in the pack resolves inside it,
2523 // which plain `index-pack` answers with `pack has N unresolved deltas`.
2524 // Connectivity has its own test, on a request that is closed:
2525 // `a_narrowed_clone_is_served_a_connected_pack_not_its_delta_bases`.
2526 let out_dir = tmpdir("emit-subset-idx");
2527 crate::git_oracle::assert_git_accepts(
2528 &out_dir,
2529 "dst.git",
2530 &bytes,
2531 crate::git_oracle::Strictness::SelfContained,
2532 );
2533 }
2534
2535 /// 🔴 **An oid this repository does not hold is REFUSED by name — it is not
2536 /// quietly left out of the pack.**
2537 ///
2538 /// `emit_set` used to `continue` past a `lookup_batch` miss. That is the
2539 /// silent-under-send shape: the request asks for N objects, the pack carries
2540 /// N-1, `PackStats` reports success and the *client* is the first to notice.
2541 /// It also made the two engines behind one contract disagree — gunnar's
2542 /// in-memory arm has always refused with *"emit_pack was asked for X, which
2543 /// this store does not hold"* — so a benchmark across the two was comparing
2544 /// two different contracts.
2545 ///
2546 /// # Why the assertions are shaped this way
2547 ///
2548 /// **RED before the change** on the first one: `emit_set` returned `Ok` with
2549 /// two entries for three asked oids, so `expect_err` panicked.
2550 ///
2551 /// The error must **name the oid**, because an operator reading
2552 /// `git.upload_pack.failed` needs the object, not the fact that something
2553 /// was missing; asserted on the rendered message.
2554 ///
2555 /// And the refusal must be *conditional*, or a function that errored
2556 /// unconditionally would pass the first two: the same store, the same two
2557 /// real oids and no absent one, must still emit.
2558 #[test]
2559 fn an_oid_the_store_does_not_hold_is_refused_by_name_not_skipped() {
2560 let dir = tmpdir("emit-missing");
2561 let store = GitStore::open(&dir, "rickard").unwrap();
2562 let (pack, _) = real_pack();
2563 store.put_pack(&pack).unwrap();
2564 store.absorb_pending().unwrap();
2565
2566 let all = store.index().oids_in_order().unwrap();
2567 assert!(all.len() > 2, "corpus too small");
2568 // A well-formed oid of the right width that this store cannot hold: it
2569 // is not in the index, and asking for it is what a caller with a stale
2570 // set does.
2571 let absent = vec![0xABu8; store.hash_kind().oid_len()];
2572 assert!(
2573 store.index().lookup(&absent).is_none(),
2574 "the fixture oid must really be absent or this test asserts nothing"
2575 );
2576
2577 let asked: Vec<&[u8]> = vec![all[0].as_slice(), absent.as_slice(), all[1].as_slice()];
2578 let err = store
2579 .emit_set(&asked, true, None)
2580 .expect_err("a pack for an object the store lacks must be refused, not shortened");
2581 let msg = format!("{err:#}");
2582 assert!(
2583 msg.contains(&hex::encode(&absent)),
2584 "the refusal must name the oid it could not find; got: {msg}"
2585 );
2586
2587 // Conditional, not unconditional: drop the absent one and the same call
2588 // emits both objects.
2589 let present: Vec<&[u8]> = vec![all[0].as_slice(), all[1].as_slice()];
2590 let entries = store
2591 .emit_set(&present, true, None)
2592 .expect("two objects this store does hold must still emit");
2593 assert_eq!(entries.len(), 2);
2594 }
2595}
2596
2597/// **The zero-copy emit path: the same bytes, and none of the memory.**
2598///
2599/// [`crate::pack_walk::EntryBytes`] turned `EmitEntry.stored` from a `Vec<u8>`
2600/// that owned a `pread`ed copy of the entry into a 16-byte address resolved
2601/// against a mapping of the archive. That is a change to *where the bytes live*
2602/// and to **nothing else**, so the guards here are of two kinds: three that
2603/// require the emitted bytes to be unchanged, and one that requires the memory
2604/// to be gone. Either kind alone would pass on a broken change — an emitter that
2605/// held nothing and wrote garbage, or one that wrote perfectly and still held
2606/// the repository.
2607#[cfg(test)]
2608mod zero_copy_tests {
2609 use crate::index_layout::{ObjType, ObjectIndex as _};
2610 use crate::pack_walk::{EmitEntry, EntryBytes, topological_order};
2611 use crate::store::tests::{real_pack, tmpdir};
2612 use crate::{GitOps, GitStore};
2613
2614 /// A store holding the whole corpus pack, and every oid in it.
2615 fn corpus(name: &str) -> (std::path::PathBuf, GitStore, Vec<Vec<u8>>) {
2616 let dir = tmpdir(name);
2617 let store = GitStore::open(&dir, "rickard").unwrap();
2618 let (pack, _) = real_pack();
2619 store.put_pack(&pack).unwrap();
2620 store.absorb_pending().unwrap();
2621 let all = store.index().oids_in_order().unwrap();
2622 assert!(all.len() > 100, "the corpus is too small to prove anything");
2623 (dir, store, all)
2624 }
2625
2626 /// The **old** shape of an emit set: every entry's bytes `pread` into an
2627 /// owned `Vec`, exactly as `emit_set` built them before 2026-08-14. The
2628 /// differential guards below emit this and the extent form and require the
2629 /// two packs to be identical.
2630 ///
2631 /// It reads through [`GitStore::read_extent`] — the `pread` path — so it
2632 /// shares no code with the mapping it is being compared against. A helper
2633 /// that resolved through `Mapped::get` would be comparing the change with
2634 /// itself.
2635 fn materialised(store: &GitStore, entries: &[EmitEntry]) -> Vec<EmitEntry> {
2636 entries
2637 .iter()
2638 .map(|e| {
2639 let mut e = e.clone();
2640 if let EntryBytes::Extent { offset, len } = e.stored {
2641 e.stored = EntryBytes::Owned(store.read_extent(offset, len).unwrap());
2642 }
2643 e
2644 })
2645 .collect()
2646 }
2647
2648 /// 🔴 **A full clone's pack is byte-identical before and after the change.**
2649 ///
2650 /// The strongest guard available, and the one the whole change stands on: a
2651 /// clone is built twice out of the same selection — once with every entry an
2652 /// `EntryBytes::Extent` resolved through the mapping, once with every entry
2653 /// `pread` into an owned `Vec` the way `emit_set` used to build it — and the
2654 /// two packs must be the same bytes. Not the same length, not the same
2655 /// object count: the same **bytes**, trailer included, which for a pack
2656 /// means the same headers, the same `OFS_DELTA` distances and the same sha1.
2657 ///
2658 /// A whole-repository selection, so `recompressed` is 0 and every one of the
2659 /// entries really is an extent — asserted, because a run in which they were
2660 /// all `Owned` would compare the old path with itself and pass vacuously.
2661 ///
2662 /// Seen RED by returning `snap.get(offset, len - 1)` from
2663 /// [`GitStore::extent`] — a mapping that hands back a *short* slice rather
2664 /// than `None`, which is the single most plausible way to get this wrong:
2665 /// "the mapped emit and the `pread` emit disagree at byte 1944: 0xe8 vs
2666 /// 0x3a (mapped 5651349 bytes, pread 5654037 bytes)".
2667 ///
2668 /// Seen RED a second way, by slicing `snap.get(offset + 1, len)` — an
2669 /// extent addressed one byte off: "resolving the emit set's stored bytes:
2670 /// resolving the stored bytes of 00065363b6d5f3edb20c8c17a5938503fcdcc941
2671 /// for emission: object type code 0 is not one git writes — refusing to
2672 /// guess it". That is phase 1's header parse refusing before phase 2 had
2673 /// written a byte of the pack, which is why the parse is there.
2674 #[test]
2675 fn a_full_clone_emits_the_identical_pack_from_extents_and_from_owned_bytes() {
2676 let (_dir, store, all) = corpus("zc-identical");
2677 let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
2678
2679 let entries = store.emit_set(&oids, true, None).unwrap();
2680 let extents = entries
2681 .iter()
2682 .filter(|e| matches!(e.stored, EntryBytes::Extent { .. }))
2683 .count();
2684 assert_eq!(
2685 extents,
2686 entries.len(),
2687 "a whole-repository selection must be ALL extents — {} of {} were owned, so this \
2688 comparison would be the old path against itself",
2689 entries.len() - extents,
2690 entries.len()
2691 );
2692
2693 let owned = materialised(&store, &entries);
2694 let (ordered_a, missing) = topological_order(entries);
2695 assert!(missing.is_empty(), "a full clone is closed: {missing:?}");
2696 let (ordered_b, _) = topological_order(owned);
2697
2698 let mut mapped_pack = Vec::new();
2699 let report = store.emit_ordered(&ordered_a, &mut mapped_pack).unwrap();
2700 let mut pread_pack = Vec::new();
2701 store.emit_ordered(&ordered_b, &mut pread_pack).unwrap();
2702
2703 assert_eq!(report.recompressed, 0, "a full clone re-deflates nothing");
2704 assert_eq!(report.copied, report.written);
2705 assert!(mapped_pack.len() > 1_000_000, "the corpus pack is tiny?");
2706 if mapped_pack != pread_pack {
2707 let at = mapped_pack
2708 .iter()
2709 .zip(pread_pack.iter())
2710 .position(|(a, b)| a != b);
2711 let i = at.unwrap_or(mapped_pack.len().min(pread_pack.len()));
2712 panic!(
2713 "the mapped emit and the `pread` emit disagree at byte {i}: {:#04x} vs {:#04x} \
2714 (mapped {} bytes, pread {} bytes)",
2715 mapped_pack.get(i).copied().unwrap_or(0),
2716 pread_pack.get(i).copied().unwrap_or(0),
2717 mapped_pack.len(),
2718 pread_pack.len()
2719 );
2720 }
2721 }
2722
2723 /// 🔴 **Every `OFS_DELTA`'s base distance still lands on its base.**
2724 ///
2725 /// The one thing phase 2 may not be parallelised around, asserted as applied
2726 /// output rather than as a receipt: the emitted pack is walked back, and for
2727 /// every `OfsDelta` entry the base its distance names must be an entry
2728 /// boundary in the same pack — which `PackWalk::closure` answers, and which
2729 /// no amount of correct-looking header encoding can satisfy by accident.
2730 /// `git index-pack` agrees separately in the tests beside this one; this is
2731 /// the guard that says *which* thing broke when it stops agreeing.
2732 ///
2733 /// The count of `OfsDelta` entries is asserted first. A pack with none would
2734 /// pass this vacuously, and a corpus that happened to hold none would make
2735 /// the whole of phase 2's serial argument untestable.
2736 ///
2737 /// Seen RED by encoding the distance as `here - base_at + 1` in
2738 /// `emit_pack`: "1345 of 1345 ofs-deltas name a base that is not an entry
2739 /// boundary — first at output offset 11". Seen RED a second, sharper way by
2740 /// emitting `emit_set`'s entries in their input order rather than
2741 /// `topological_order`'s: "entry at 3414738 deltas against archive offset
2742 /// 3414009, which is not in this pack — the set is not closed and was not
2743 /// ordered by `topological_order`".
2744 #[test]
2745 fn an_ofs_delta_still_names_its_base_by_the_right_distance() {
2746 let (_dir, store, all) = corpus("zc-ofs");
2747 let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
2748 let (ordered, _) = topological_order(store.emit_set(&oids, true, None).unwrap());
2749
2750 let mut pack = Vec::new();
2751 store.emit_ordered(&ordered, &mut pack).unwrap();
2752
2753 let walk = crate::pack_walk::walk(&pack, store.hash_kind().oid_len())
2754 .expect("our own walk must read what we emitted");
2755 let deltas = walk
2756 .entries
2757 .iter()
2758 .filter(|e| e.obj_type == ObjType::OfsDelta)
2759 .count();
2760 assert!(
2761 deltas > 0,
2762 "the corpus emitted no ofs-delta, so this proves nothing about distances"
2763 );
2764
2765 let closure = walk.closure();
2766 assert!(
2767 closure.broken_offsets.is_empty(),
2768 "{} of {deltas} ofs-deltas name a base that is not an entry boundary — first at \
2769 output offset {}",
2770 closure.broken_offsets.len(),
2771 closure.broken_offsets.first().copied().unwrap_or(0)
2772 );
2773 assert!(
2774 closure.external_refs.is_empty(),
2775 "a full clone must name no external base"
2776 );
2777 }
2778
2779 /// 🔴 **The `Owned` exception still carries a rebuilt delta's bytes.**
2780 ///
2781 /// `EntryBytes::Owned` is not vestigial: a delta whose base the request does
2782 /// not carry genuinely computes bytes that no extent addresses, and dropping
2783 /// the variant would have meant either shipping a delta the client cannot
2784 /// resolve or widening the request — the two failures
2785 /// `GitStore::emit_set`'s doc comment is written about.
2786 ///
2787 /// Asserted on applied output at both ends: the boundary entries are `Owned`
2788 /// and every other entry is an `Extent` (a mixed set, so the emitter is
2789 /// proved to handle both in one pack), and the resulting pack walks with
2790 /// its closure intact. The premise — that this subset really does cut delta
2791 /// chains — is asserted first.
2792 ///
2793 /// Seen RED by making the whole-rebuild arm of `emit_set` push
2794 /// `EntryBytes::Extent { offset: row.offset, len: row.len }` instead of the
2795 /// bytes it just built: "312 boundary entries are recompressed but 202 carry
2796 /// owned bytes — a rebuilt entry that points back at its stored extent ships
2797 /// the delta it was rebuilt to avoid". 202 and not 0, because the re-delta
2798 /// arm above it was untouched — which is what makes the count, and not a
2799 /// bare `is_some()`, the thing worth asserting.
2800 #[test]
2801 fn a_rebuilt_delta_still_carries_its_own_bytes_and_emits_beside_extents() {
2802 let (_dir, store, all) = corpus("zc-owned");
2803 let half: Vec<&[u8]> = all.iter().step_by(2).map(Vec::as_slice).collect();
2804
2805 let rows = store.index().lookup_batch(&half);
2806 let inside: std::collections::HashSet<u64> =
2807 rows.iter().flatten().map(|r| r.offset).collect();
2808 let boundary = rows
2809 .iter()
2810 .flatten()
2811 .filter(|r| r.delta_base != 0 && !inside.contains(&r.delta_base))
2812 .count();
2813 assert!(
2814 boundary > 0,
2815 "this subset cuts no delta chain, so the Owned path is never reached"
2816 );
2817
2818 let entries = store.emit_set(&half, true, None).unwrap();
2819 let rebuilt: Vec<&EmitEntry> = entries.iter().filter(|e| e.recompressed).collect();
2820 let with_bytes = rebuilt
2821 .iter()
2822 .filter(|e| e.stored.owned().is_some_and(|b| !b.is_empty()))
2823 .count();
2824 assert_eq!(
2825 with_bytes,
2826 rebuilt.len(),
2827 "{} boundary entries are recompressed but {with_bytes} carry owned bytes — a rebuilt \
2828 entry that points back at its stored extent ships the delta it was rebuilt to avoid",
2829 rebuilt.len()
2830 );
2831 assert_eq!(rebuilt.len(), boundary);
2832
2833 // The other half of the point: this is a MIXED pack, so the emitter is
2834 // resolving both variants in one pass.
2835 let as_extent = entries
2836 .iter()
2837 .filter(|e| matches!(e.stored, EntryBytes::Extent { .. }))
2838 .count();
2839 assert_eq!(
2840 as_extent,
2841 entries.len() - rebuilt.len(),
2842 "every entry that was NOT rebuilt must still be an extent"
2843 );
2844 assert!(
2845 as_extent > 0 && !rebuilt.is_empty(),
2846 "the set must be mixed"
2847 );
2848
2849 let (ordered, missing) = topological_order(entries);
2850 assert!(missing.is_empty(), "the emitted set is closed: {missing:?}");
2851 let mut pack = Vec::new();
2852 let report = store.emit_ordered(&ordered, &mut pack).unwrap();
2853 assert_eq!(report.recompressed as usize, boundary);
2854 let walk = crate::pack_walk::walk(&pack, store.hash_kind().oid_len())
2855 .expect("a mixed pack must still walk");
2856 assert!(
2857 walk.closure().broken_offsets.is_empty(),
2858 "a pack mixing extents and rebuilt bytes must still be self-contained"
2859 );
2860 }
2861
2862 /// 🔴 **An extent past the mapping falls back to `pread` and returns the
2863 /// SAME bytes.**
2864 ///
2865 /// `Mapped::get` answers `None` for an extent past the mapped end, which
2866 /// means *"go and pread it"* and never *"there are no bytes"*. The case is
2867 /// real and not hypothetical: a snapshot is taken once per emission and the
2868 /// blob file grows whenever a push lands, so any clone running across a push
2869 /// reads its tail this way.
2870 ///
2871 /// Constructed deliberately rather than raced: a snapshot is taken, a second
2872 /// pack is pushed, and the objects of that second pack are then emitted
2873 /// against the **stale** snapshot. Every one of their extents is past its
2874 /// end, so every one takes the fallback — asserted by `Mapped::get` refusing
2875 /// them, so a run where the snapshot happened to cover them cannot pass
2876 /// quietly.
2877 ///
2878 /// Seen RED by making [`GitStore::extent`]'s `None` arm
2879 /// `Ok(Cow::Owned(Vec::new()))` — the "absent means no bytes" misreading
2880 /// [`crate::archive_map::Mapped::get`]'s own doc comment forbids: "the
2881 /// fallback returned 0 bytes for the extent at (5653314, 43), which the
2882 /// mapping does not cover; `None` means pread, not empty".
2883 #[test]
2884 fn an_extent_past_the_mapping_falls_back_and_returns_the_same_bytes() {
2885 let (_dir, store, _all) = corpus("zc-fallback");
2886
2887 // The snapshot is taken NOW, before the second push.
2888 let stale = store.archive_snapshot().unwrap();
2889 let before = stale.len();
2890
2891 let (second, _) = crate::store::tests::one_blob_pack(b"a blob pushed after the snapshot\n");
2892 store.put_pack(&second).unwrap();
2893 store.absorb_pending().unwrap();
2894
2895 let after = store.archive_snapshot().unwrap().len();
2896 assert!(
2897 after > before,
2898 "the second push did not grow the blob file ({before} → {after}), so nothing is past \
2899 the stale mapping and this test proves nothing"
2900 );
2901
2902 // Every object of the second pack sits past the stale mapping's end.
2903 let fresh: Vec<Vec<u8>> = store
2904 .index()
2905 .oids_in_order()
2906 .unwrap()
2907 .into_iter()
2908 .filter(|oid| {
2909 store
2910 .index()
2911 .lookup(oid)
2912 .is_some_and(|r| r.offset >= before)
2913 })
2914 .collect();
2915 assert!(
2916 !fresh.is_empty(),
2917 "no object landed past the stale mapping's end"
2918 );
2919
2920 for oid in &fresh {
2921 let row = store.index().lookup(oid).unwrap();
2922 assert!(
2923 stale.get(row.offset, row.len).is_none(),
2924 "the stale mapping must NOT cover ({}, {}) or the fallback is never taken",
2925 row.offset,
2926 row.len
2927 );
2928 let fell_back = store.extent(&stale, row.offset, row.len).unwrap();
2929 let preaded = store.read_extent(row.offset, row.len).unwrap();
2930 assert_eq!(
2931 fell_back.len(),
2932 preaded.len(),
2933 "the fallback returned {} bytes for the extent at ({}, {}), which the mapping does \
2934 not cover; `None` means pread, not empty",
2935 fell_back.len(),
2936 row.offset,
2937 row.len
2938 );
2939 assert_eq!(
2940 fell_back.as_ref(),
2941 preaded.as_slice(),
2942 "the fallback bytes differ from the pread bytes at ({}, {})",
2943 row.offset,
2944 row.len
2945 );
2946 // A FRESH snapshot covers it, and agrees with both.
2947 let fresh_snap = store.archive_snapshot().unwrap();
2948 assert_eq!(
2949 fresh_snap.get(row.offset, row.len).unwrap(),
2950 preaded.as_slice(),
2951 "the remapped snapshot disagrees with the pread at ({}, {})",
2952 row.offset,
2953 row.len
2954 );
2955 }
2956 eprintln!(
2957 "{} object(s) resolved past a stale {before}-byte mapping and matched the pread",
2958 fresh.len()
2959 );
2960 }
2961
2962 /// 🔴 **The gatling fan-out resolves the identical bytes the serial pass
2963 /// does** — LAW 3's primitive, over data with no lock on it.
2964 ///
2965 /// Phase 1 is the one place the serving tier fans out, and the claim that
2966 /// makes it safe is a claim about the *data*: the blob file is append-only
2967 /// and `gc` truncates nothing, so N workers reading a snapshot of it share
2968 /// no mutable state and need no synchronisation. This asserts the observable
2969 /// consequence — the same set resolved on one thread and on four gives the
2970 /// same slices, in the same order, pointing at the same bytes.
2971 ///
2972 /// It calls
2973 /// [`resolve_with`](GitStore::resolve_with) directly rather than going
2974 /// through `resolve_emit_payloads`, because the production threshold is 4096
2975 /// entries and this corpus holds 2687: through the front door the gatling
2976 /// arm would never run and this test would be asserting the serial path
2977 /// against itself. The order matters as much as the content —
2978 /// `gatling_for_each` self-dispatches with no barrier, so a worker finishing
2979 /// unit 9 before unit 3 is normal, and an implementation that collected in
2980 /// **completion** order would produce a pack whose entries are shuffled and
2981 /// whose `OFS_DELTA` distances therefore point at the wrong objects.
2982 ///
2983 /// Seen RED by having the parallel arm return
2984 /// `gatling_for_each(n, workers, one).into_iter().rev().collect()` — the
2985 /// cheapest stand-in for "results not in index order": "the fan-out resolved
2986 /// entry 0 to different bytes than the serial pass did (2 workers): 46 bytes
2987 /// vs 1933, first differing at Some(0)".
2988 #[test]
2989 fn the_fan_out_resolves_the_identical_bytes_the_serial_pass_does() {
2990 let (_dir, store, all) = corpus("zc-gatling");
2991 let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
2992 let (ordered, _) = topological_order(store.emit_set(&oids, true, None).unwrap());
2993 assert!(
2994 ordered.len() > 1000,
2995 "too few entries to spread over workers"
2996 );
2997
2998 let snap = store.archive_snapshot().unwrap();
2999 let serial = store.resolve_with(&ordered, &snap, 1).unwrap();
3000 for workers in [2usize, 4, 8] {
3001 let parallel = store.resolve_with(&ordered, &snap, workers).unwrap();
3002 assert_eq!(
3003 parallel.len(),
3004 serial.len(),
3005 "the fan-out resolved {} entries against the serial pass's {} ({workers} workers)",
3006 parallel.len(),
3007 serial.len()
3008 );
3009 // Compared by hand rather than with `assert_eq!` on the slices: an
3010 // entry is kilobytes, and a failure that dumps two of them is a
3011 // failure nobody reads.
3012 for (i, (p, s)) in parallel.iter().zip(serial.iter()).enumerate() {
3013 if p.as_ref() != s.as_ref() {
3014 let at = p.iter().zip(s.iter()).position(|(a, b)| a != b);
3015 panic!(
3016 "the fan-out resolved entry {i} to different bytes than the serial pass \
3017 did ({workers} workers): {} bytes vs {}, first differing at {at:?}",
3018 p.len(),
3019 s.len()
3020 );
3021 }
3022 }
3023 }
3024
3025 // And the pack itself: emitted off the fan-out, walked back, closed.
3026 let mut pack = Vec::new();
3027 store.emit_ordered(&ordered, &mut pack).unwrap();
3028 let walk = crate::pack_walk::walk(&pack, store.hash_kind().oid_len()).unwrap();
3029 assert_eq!(walk.entries.len(), ordered.len());
3030 assert!(walk.closure().broken_offsets.is_empty());
3031 }
3032
3033 /// 🔴 **The pack is never materialised: the whole emit set's heap is a
3034 /// small fraction of the pack it will write.**
3035 ///
3036 /// # What this asserts, and how completely
3037 ///
3038 /// The emit set's heap is **accounted exactly**, not sampled: an
3039 /// [`EmitEntry`] holds a `Vec<EmitEntry>` slot, an oid, and — for the
3040 /// `Owned` exception only — payload bytes. There is nothing else it can
3041 /// hold, so `slots + oids + owned` is the complete cost of the set, and it
3042 /// is compared against the bytes that set will emit. Before 2026-08-14 the
3043 /// `owned` term alone *was* the pack: every entry's bytes `pread` into a
3044 /// `Vec`, all of them live at once, because the set is built in full before
3045 /// [`crate::pack_walk::emit_pack`] writes a byte.
3046 ///
3047 /// Seen RED by reverting `emit_set`'s base-inside arm to
3048 /// `EntryBytes::Owned(self.read_extent(row.offset, row.len)?)`: "the emit
3049 /// set owns 5653270 bytes of payload — 100.0 % of the 5653270 bytes it will
3050 /// emit — which is the whole pack held in memory before the first byte goes
3051 /// out".
3052 ///
3053 /// # What it does NOT prove, stated plainly
3054 ///
3055 /// **It is not an RSS test, and an RSS test here would be theatre.** This
3056 /// began as one, asserting that `/proc/self/statm` grows by less than the
3057 /// pack across `emit_set`. It was **blind**: run against the broken version
3058 /// above, which really does allocate and fill 5 653 270 bytes, resident set
3059 /// size did not move by a single page — 203 366 400 before and after, and
3060 /// 203 182 080 → 203 182 080 in the green run. The allocator was handing
3061 /// back arena pages this test process already had resident. A guard that
3062 /// cannot tell the defect from the fix is not a weak guard, it is no guard,
3063 /// and it is exactly what LAW 2 says to expect of one's own new guards. It
3064 /// is gone; the accounting above replaces it.
3065 ///
3066 /// **It says nothing about the peak of a real clone.** This corpus is 5.65 MB
3067 /// against `linux.git`'s ~6.4 GB, and the 2314 MB peak this change is aimed
3068 /// at was measured with `perf` and `/proc`, not with a unit test.
3069 ///
3070 /// **The residual it exposes is real and is printed rather than hidden.**
3071 /// The `slots + oids` term is `O(objects)` and survives this change: every
3072 /// entry still carries its own `Vec<u8>` oid, one heap allocation each. The
3073 /// printed projection to 13.8 M objects is the honest size of what is left
3074 /// to do, and it is not small.
3075 #[test]
3076 fn an_emit_set_holds_no_payload_bytes_of_the_pack_it_will_write() {
3077 let (_dir, store, all) = corpus("zc-memory");
3078 let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
3079
3080 let entries = store.emit_set(&oids, true, None).unwrap();
3081
3082 let owned: u64 = entries
3083 .iter()
3084 .map(|e| e.stored.owned().map_or(0, |b| b.len() as u64))
3085 .sum();
3086 let will_emit: u64 = entries.iter().map(|e| e.stored.len()).sum();
3087 assert!(
3088 will_emit > 1_000_000,
3089 "the corpus emits {will_emit} bytes, too few to distinguish held from addressed"
3090 );
3091 assert_eq!(
3092 owned,
3093 0,
3094 "the emit set owns {owned} bytes of payload — {:.1} % of the {will_emit} bytes it will \
3095 emit — which is the whole pack held in memory before the first byte goes out",
3096 100.0 * owned as f64 / will_emit as f64
3097 );
3098
3099 // The complete heap of the set, term by term.
3100 let slots = (entries.len() * std::mem::size_of::<EmitEntry>()) as u64;
3101 let oid_bytes: u64 = entries.iter().map(|e| e.oid.len() as u64).sum();
3102 let held = slots + oid_bytes + owned;
3103 assert!(
3104 held < will_emit / 8,
3105 "the emit set holds {held} bytes ({:.1} % of the {will_emit}-byte pack): {slots} of \
3106 slots, {oid_bytes} of oids, {owned} of payload",
3107 100.0 * held as f64 / will_emit as f64
3108 );
3109 eprintln!(
3110 "{} entries: {slots} B slots + {oid_bytes} B oids + {owned} B payload = {held} B held \
3111 for a {will_emit} B pack ({:.1} %). EmitEntry is {} B. Projected to linux.git's \
3112 ~13.8 M objects that residual is ~{:.2} GB — the payload copy is gone, the per-entry \
3113 slot and oid are NOT.",
3114 entries.len(),
3115 100.0 * held as f64 / will_emit as f64,
3116 std::mem::size_of::<EmitEntry>(),
3117 13.8e6 * (std::mem::size_of::<EmitEntry>() as f64 + 20.0) / 1e9,
3118 );
3119 }
3120}
3121
3122/// **A `REF_DELTA` whose base is inside the same pack**, on both sides of the
3123/// store: ingesting one, and serving one.
3124///
3125/// # Why this shape and no other
3126///
3127/// It is not exotic and it is not hand-rolled. `git index-pack --fix-thin`
3128/// completes a pushed **thin** pack by *appending the base object to the pack*
3129/// and leaving the delta naming that base by oid — so the base lands **after**
3130/// its dependant, in the same pack, named the one way a pack can name something
3131/// that is not a position. Every repository that has been pushed to more than
3132/// once and not `gc`'d holds packs of this shape, which is to say: the ordinary
3133/// case, not a corner.
3134///
3135/// The fixture is therefore built by **git itself** (`pack-objects --thin` fed
3136/// to `index-pack --fix-thin`) rather than assembled here, because a hand-built
3137/// pack could only ever prove that this crate agrees with this crate.
3138///
3139/// Two defects met on it, one per direction:
3140///
3141/// | direction | defect | fix |
3142/// |---|---|---|
3143/// | in | `external_bases_exist` demanded every ref base be in the **store**, so a pack carrying its own base was refused | ask the pack too, and only when about to refuse |
3144/// | out | `emit_set` copied the stored `REF_DELTA` through, and **gitoxide cannot read one whose base is in the pack** | re-head it as an `OFS_DELTA` naming the same base by distance |
3145#[cfg(test)]
3146mod in_pack_ref_delta_tests {
3147 use crate::index_layout::ObjType;
3148 use crate::pack_walk::{DeltaBase, topological_order};
3149 use crate::store::tests::tmpdir;
3150 use crate::{GitOps, GitStore};
3151 use std::path::Path;
3152 use std::process::Command;
3153
3154 fn git(dir: &Path, args: &[&str]) -> String {
3155 let out = Command::new("git")
3156 .current_dir(dir)
3157 .args(args)
3158 .env("GIT_AUTHOR_NAME", "t")
3159 .env("GIT_AUTHOR_EMAIL", "t@t")
3160 .env("GIT_COMMITTER_NAME", "t")
3161 .env("GIT_COMMITTER_EMAIL", "t@t")
3162 .env("GIT_AUTHOR_DATE", "2020-01-01T00:00:00Z")
3163 .env("GIT_COMMITTER_DATE", "2020-01-01T00:00:00Z")
3164 .output()
3165 .unwrap_or_else(|e| panic!("running git {args:?}: {e}"));
3166 assert!(
3167 out.status.success(),
3168 "git {args:?} failed: {}",
3169 String::from_utf8_lossy(&out.stderr)
3170 );
3171 String::from_utf8_lossy(&out.stdout).trim().to_string()
3172 }
3173
3174 /// A pack **git wrote** that contains a `REF_DELTA` whose base is an entry
3175 /// of the same pack.
3176 ///
3177 /// `None` when there is no `git` on `PATH`, so the suite still runs where
3178 /// the oracle is absent rather than failing for the wrong reason.
3179 fn fix_thin_pack(scratch: &Path) -> Option<Vec<u8>> {
3180 if Command::new("git").arg("--version").output().is_err() {
3181 eprintln!("skipping: no git on PATH");
3182 return None;
3183 }
3184 let repo = scratch.join("src");
3185 std::fs::create_dir_all(&repo).unwrap();
3186 git(&repo, &["init", "-q", "--initial-branch=main"]);
3187
3188 // Big enough, and changed little enough, that `pack-objects` really does
3189 // deltify the second version against the first. A two-line file would be
3190 // stored whole and this fixture would prove nothing — which the premise
3191 // assertion below refuses to let happen silently.
3192 let lines: Vec<String> = (0..400)
3193 .map(|i| format!("line {i} {}", "x".repeat(20)))
3194 .collect();
3195 std::fs::write(repo.join("f.txt"), lines.join("\n")).unwrap();
3196 git(&repo, &["add", "f.txt"]);
3197 git(&repo, &["commit", "-q", "-m", "c1"]);
3198
3199 let mut changed = lines.clone();
3200 changed[10] = "CHANGED".to_string();
3201 changed.push("appended line".to_string());
3202 std::fs::write(repo.join("f.txt"), changed.join("\n")).unwrap();
3203 git(&repo, &["commit", "-q", "-a", "-m", "c2"]);
3204
3205 // A THIN pack: c2's objects only, deltified against c1's, which it does
3206 // not carry.
3207 let head = git(&repo, &["rev-parse", "HEAD"]);
3208 let parent = git(&repo, &["rev-parse", "HEAD~1"]);
3209 let thin = {
3210 use std::io::Write as _;
3211 let mut child = Command::new("git")
3212 .current_dir(&repo)
3213 .args(["pack-objects", "--thin", "--revs", "--stdout"])
3214 .stdin(std::process::Stdio::piped())
3215 .stdout(std::process::Stdio::piped())
3216 .stderr(std::process::Stdio::piped())
3217 .spawn()
3218 .unwrap();
3219 write!(child.stdin.take().unwrap(), "{head}\n^{parent}\n").unwrap();
3220 let out = child.wait_with_output().unwrap();
3221 assert!(out.status.success(), "git pack-objects --thin failed");
3222 out.stdout
3223 };
3224
3225 // `--fix-thin` appends the bases INTO the pack. The ref-deltas keep
3226 // naming them by oid, and now those oids ARE in the pack.
3227 let idx = scratch.join("fixed.idx");
3228 {
3229 use std::io::Write as _;
3230 let mut child = Command::new("git")
3231 .current_dir(&repo)
3232 .args([
3233 "index-pack",
3234 "--fix-thin",
3235 "--stdin",
3236 "-o",
3237 idx.to_str().unwrap(),
3238 ])
3239 .stdin(std::process::Stdio::piped())
3240 .stdout(std::process::Stdio::piped())
3241 .stderr(std::process::Stdio::piped())
3242 .spawn()
3243 .unwrap();
3244 child.stdin.take().unwrap().write_all(&thin).unwrap();
3245 let out = child.wait_with_output().unwrap();
3246 assert!(
3247 out.status.success(),
3248 "git index-pack --fix-thin failed: {}",
3249 String::from_utf8_lossy(&out.stderr)
3250 );
3251 }
3252 let packdir = repo.join(".git/objects/pack");
3253 let pack = std::fs::read_dir(&packdir)
3254 .unwrap()
3255 .flatten()
3256 .map(|e| e.path())
3257 .find(|p| p.extension().is_some_and(|x| x == "pack"))
3258 .expect("git index-pack --fix-thin wrote no pack");
3259 Some(std::fs::read(pack).unwrap())
3260 }
3261
3262 /// The fixture is what it claims to be: at least one `REF_DELTA`, and every
3263 /// delta base resolvable **inside the pack alone**.
3264 ///
3265 /// Without this the two tests below could pass on a pack with no ref-delta
3266 /// in it at all, which is the vacuous green LAW 2 is about.
3267 fn assert_carries_its_own_ref_delta_base(pack: &[u8]) {
3268 let w = crate::pack_walk::walk(pack, 20).expect("git's own pack must walk");
3269 let refs = w
3270 .entries
3271 .iter()
3272 .filter(|e| e.obj_type == ObjType::RefDelta)
3273 .count();
3274 assert!(
3275 refs > 0,
3276 "the fixture has no ref-delta at all, so it cannot exercise either defect"
3277 );
3278 // Resolvable with NO external base source at all: every base a delta in
3279 // here names is carried in here.
3280 crate::resolve::resolve(
3281 pack,
3282 crate::GitHashKind::Sha1,
3283 0,
3284 &crate::resolve::NoBases,
3285 )
3286 .expect("the fixture must be self-contained, or `put_pack` is right to refuse it");
3287 }
3288
3289 /// 🔴 **znippy ingests a pack git wrote.**
3290 ///
3291 /// `pack_walk::closure()` reports every `REF_DELTA` base on `external_refs`
3292 /// — it cannot do otherwise, a walk knows no oids — and
3293 /// `external_bases_exist` demanded each one already be in the store. A pack
3294 /// carrying its own base was therefore refused, which is `git index-pack
3295 /// --fix-thin`'s ordinary output and `gunnar.multi_pack_serve`'s failure.
3296 ///
3297 /// Asserted on applied output, not on `Ok`: every object of the pack is
3298 /// afterwards *in the store's index*, so a `put_pack` that returned success
3299 /// having stored nothing would still be red.
3300 ///
3301 /// Seen RED by restoring the old check (refuse any ref base not already in
3302 /// the store):
3303 /// *"this pack deltas against 93c4207c152fa94c3978c75b136ff5a530ca16b6,
3304 /// which this repository does not have — the push is refused rather than
3305 /// stored with a dangling base"*.
3306 #[test]
3307 fn a_pack_carrying_its_own_ref_delta_base_is_ingested_not_refused() {
3308 let dir = tmpdir("inpack-put");
3309 let Some(pack) = fix_thin_pack(&dir) else {
3310 return;
3311 };
3312 assert_carries_its_own_ref_delta_base(&pack);
3313
3314 let store = GitStore::open(&dir.join("store"), "rickard").unwrap();
3315 store
3316 .put_pack(&pack)
3317 .expect("a self-contained pack git itself wrote must be accepted");
3318 store.absorb_pending().unwrap();
3319
3320 let walked = crate::pack_walk::walk(&pack, 20).unwrap();
3321 let stored = store.index().oids_in_order().unwrap();
3322 assert_eq!(
3323 stored.len(),
3324 walked.entries.len(),
3325 "the pack has {} entries and the store indexed {}",
3326 walked.entries.len(),
3327 stored.len()
3328 );
3329 // And specifically the base that used to be the refusal.
3330 let base = walked
3331 .entries
3332 .iter()
3333 .find_map(|e| match &e.delta_base {
3334 DeltaBase::Ref(oid) => Some(oid.clone()),
3335 _ => None,
3336 })
3337 .expect("the premise asserted there is one");
3338 assert!(
3339 store.has(&base).unwrap(),
3340 "the in-pack ref-delta base {} is not in the store after the push",
3341 hex::encode(&base)
3342 );
3343 }
3344
3345 /// 🔴 **A pack this server emits is consumed by `gunnar_client`.**
3346 ///
3347 /// `gunnar_client` resolves a fetch with gitoxide, and
3348 /// `gix_pack::data::input::LookupRefDeltaObjectsIter` reads every
3349 /// `OBJ_REF_DELTA` as naming an object *the receiver already has*: it
3350 /// consults the local object database and the bases it has already spliced
3351 /// in, and never the pack it is reading. A clone's target holds nothing, so
3352 /// one stored ref-delta copied through is fatal for the **whole** pack.
3353 ///
3354 /// The receiver here is `gix_object::find::Never` — an object database that
3355 /// holds nothing, which is exactly `open_or_init_bare` at the moment a clone
3356 /// starts — so this is the client's own code answering, not a paraphrase of
3357 /// it.
3358 ///
3359 /// Three assertions, and all three are needed:
3360 ///
3361 /// 1. **gitoxide accepts it**, with an empty base source. This is the bug.
3362 /// 2. **stock git still accepts it.** git accepted the *broken* pack too,
3363 /// so a fix that satisfied gix and broke git would otherwise ship green.
3364 /// 3. **the object set is exactly what was asked for**, read back out of
3365 /// stock git by oid. A re-head that dropped, duplicated or corrupted an
3366 /// entry passes 1 and 2 — `index-pack` files an entry under the oid it
3367 /// computes from that entry's own bytes, so a wrong object is a
3368 /// *different* oid and only comparing the sets can see it.
3369 ///
3370 /// Seen RED, each separately — see the commit message for the exact texts.
3371 #[test]
3372 fn an_emitted_pack_names_no_base_inside_itself_by_oid() {
3373 let dir = tmpdir("inpack-emit");
3374 let Some(pack) = fix_thin_pack(&dir) else {
3375 return;
3376 };
3377 assert_carries_its_own_ref_delta_base(&pack);
3378
3379 let store = GitStore::open(&dir.join("store"), "rickard").unwrap();
3380 store.put_pack(&pack).unwrap();
3381 store.absorb_pending().unwrap();
3382
3383 let all = store.index().oids_in_order().unwrap();
3384 let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
3385 let (ordered, missing) = topological_order(store.emit_set(&oids, true, None).unwrap());
3386 assert!(
3387 missing.is_empty(),
3388 "an entry still names a base that is not here: {missing:?}"
3389 );
3390 let mut emitted = Vec::new();
3391 store.emit_ordered(&ordered, &mut emitted).unwrap();
3392
3393 // ── the premise: the emitted pack really does carry a delta, or nothing
3394 // below distinguishes a fix from a pack of whole objects ────────────
3395 let w = crate::pack_walk::walk(&emitted, 20).expect("our own walk reads what we emitted");
3396 let deltas = w
3397 .entries
3398 .iter()
3399 .filter(|e| !matches!(e.delta_base, DeltaBase::None))
3400 .count();
3401 assert!(
3402 deltas > 0,
3403 "the emitted pack has no delta at all, so it cannot show how a base is named"
3404 );
3405
3406 // ── 1. THE BUG: no base may be named by oid ───────────────────────────
3407 let named_by_oid = w.closure().external_refs;
3408 assert!(
3409 named_by_oid.is_empty(),
3410 "a clone's pack names {} base(s) by oid — gitoxide resolves those against the \
3411 receiver's store and a clone's receiver is empty: {:?}",
3412 named_by_oid.len(),
3413 named_by_oid.iter().map(hex::encode).collect::<Vec<_>>()
3414 );
3415
3416 // …and the client's own code says so, not only our reading of it.
3417 let out = dir.join("gix-out");
3418 std::fs::create_dir_all(&out).unwrap();
3419 let stop = std::sync::atomic::AtomicBool::new(false);
3420 let mut cur = std::io::Cursor::new(emitted.as_slice());
3421 let outcome = gix_pack::Bundle::write_to_directory(
3422 &mut cur,
3423 Some(&out),
3424 &mut gix_features::progress::Discard,
3425 &stop,
3426 // An object database that holds nothing — a fresh clone target.
3427 Some(gix_object::find::Never),
3428 gix_pack::bundle::write::Options {
3429 object_hash: gix_hash::Kind::Sha1,
3430 ..Default::default()
3431 },
3432 );
3433 let outcome = match outcome {
3434 Ok(o) => o,
3435 Err(e) => panic!("gitoxide refused a pack this store emitted: {}", chain(&e)),
3436 };
3437 assert_eq!(
3438 outcome.index.num_objects as usize,
3439 oids.len(),
3440 "gitoxide indexed {} objects for a request of {}",
3441 outcome.index.num_objects,
3442 oids.len()
3443 );
3444
3445 // ── 2. stock git must still accept it ─────────────────────────────────
3446 crate::git_oracle::assert_git_accepts(
3447 &dir,
3448 "stock.git",
3449 &emitted,
3450 crate::git_oracle::Strictness::SelfContained,
3451 );
3452
3453 // ── 3. exactly the object set that was asked for, by oid, read back out
3454 // of stock git ───────────────────────────────────────────────────
3455 let read_back = crate::git_oracle::git_reads_back(&dir, "readback.git", &emitted).unwrap();
3456 assert_eq!(
3457 read_back.len(),
3458 oids.len(),
3459 "the pack carries {} objects, the request asked for {}",
3460 read_back.len(),
3461 oids.len()
3462 );
3463 let got: std::collections::BTreeSet<String> =
3464 read_back.iter().map(|(oid, _)| oid.clone()).collect();
3465 let want: std::collections::BTreeSet<String> = oids.iter().map(hex::encode).collect();
3466 assert_eq!(
3467 got, want,
3468 "the emitted pack is not the requested set — missing {:?}, unexpected {:?}",
3469 want.difference(&got).collect::<Vec<_>>(),
3470 got.difference(&want).collect::<Vec<_>>()
3471 );
3472 }
3473
3474 /// gix errors nest their real cause; the top line alone names the operation
3475 /// and nothing about why. The same walk `gunnar-client::error::chain` does,
3476 /// and here for the same reason: without it the red above reads *"Failed to
3477 /// write pack"* and diagnoses nothing.
3478 fn chain(err: &(dyn std::error::Error + 'static)) -> String {
3479 let mut out = err.to_string();
3480 let mut cursor = err.source();
3481 for _ in 0..16 {
3482 let Some(next) = cursor else { break };
3483 let text = next.to_string();
3484 if !out.ends_with(&text) {
3485 out.push_str(": ");
3486 out.push_str(&text);
3487 }
3488 cursor = next.source();
3489 }
3490 out
3491 }
3492}