znippy_plugin_git/store.rs
1//! The storage functions. Nothing else is permitted here.
2//!
3//! **Twelve, then thirteen.** The count moved on 2026-08-10 when
4//! [`GitOps::put_refs_cas`] landed — atomic batch compare-and-swap, the
5//! `git push --atomic` primitive, which was previously *unrepresentable*:
6//! `put_refs` is a batch without CAS, `update_ref` is CAS without a batch, and
7//! `--atomic` is defined as both at once. It is a storage concern and it belongs
8//! here. What does **not** belong here is the *reading* contract — decoded
9//! reads, `HEAD`, negotiation, pack emission — which is [`crate::serve`].
10//!
11//! **ZNIPPY-GIT APACHE ARROW IPC IS LAW.** Storage is Arrow IPC in a znippy
12//! archive. Not a packfile directory, not loose objects, not `gix-odb`, not a
13//! filesystem layout borrowed from somebody else's model. gix is a codec for
14//! framing bytes onto the wire and nothing more: it never owns a handle, never
15//! decides where a byte lives, and never appears in a signature below.
16//!
17//! ---
18//!
19//! The twelve are the methods of [`GitOps`], implemented once, for
20//! [`GitStore`] — the handle and the trait live in [`crate::git_ops`], and
21//! everything each method reaches for was already built and already measured:
22//!
23//! | this file calls | which is | measured in |
24//! |---|---|---|
25//! | `PushPath` → `SafeWriter` | blob fsync, then the journal row | `archive_write` |
26//! | `ObjectReadStack` | stree → Arrow → redb tail | `read_stack` |
27//! | `RefLog` | one Arrow IPC frame per push | `refs` / `pushlog` |
28//! | `pack_walk` + `resolve` | the split and the oids | `pack_walk` / `resolve` |
29//! | `NewGeneration` → `compact_archive` | base znippy's compaction | `gc` |
30//!
31//! No method below invents storage, an index, a durability contract or a
32//! concurrency mechanism. If one looks like it does, that is the bug.
33
34use std::path::Path;
35
36use anyhow::{anyhow, bail, Context, Result};
37use znippy_common::ReservedSection;
38
39use crate::gc::GcReport;
40use crate::git_ops::{lookup_path, GitOps, GitStore, LookupPath, RefRow, Stored, TxId};
41use crate::index_layout::ObjectIndex;
42use crate::pack_walk::walk;
43use crate::refs::RefUpdate;
44use git_storage_trait::{Observed, RefCas, RefRejection, RefTarget};
45
46/// An object id, borrowed.
47pub type Oid<'a> = &'a [u8];
48
49/// A byte range inside the archive: `(offset, len)`.
50pub type Extent = (u64, u64);
51
52impl<S: ObjectIndex + 'static> GitStore<S> {
53 /// **The one body behind [`GitOps::put_pack`] and
54 /// [`GitOps::put_pack_observed`].**
55 ///
56 /// Private and inherent rather than a third trait method: the difference
57 /// between the two is one `Option`, and two copies of the ack path — the
58 /// closure check, the verbatim append, the journal ordering, the queued
59 /// index job — is exactly the twinning that ends with one of them quietly
60 /// missing a step. The steps are numbered on [`GitOps::put_pack`]'s doc,
61 /// which is where a reader looking for *what a push does* will arrive.
62 fn put_pack_watching(
63 &self,
64 bytes: &[u8],
65 progress: Option<&dyn git_storage_trait::PackProgress>,
66 ) -> Result<TxId> {
67 if bytes.is_empty() {
68 bail!("an empty push is not a pack");
69 }
70 // (1) + (2) — one walk, and the check falls out of it. The listener,
71 // when there is one, rides that same walk: `walk` is `walk_observed`
72 // with a no-op, so these two arms are one loop and not two.
73 let walked = match progress {
74 Some(sink) => crate::pack_walk::walk_observed(
75 bytes,
76 self.hash_kind().oid_len(),
77 &mut |done, total| sink.indexed(done, total),
78 ),
79 None => walk(bytes, self.hash_kind().oid_len()),
80 }
81 .context("the pushed pack could not be split")?;
82 self.external_bases_exist(bytes, &walked)?;
83
84 // (3) + (4) — verbatim, blob fsync, journal row, journal fsync.
85 // (5) — and the pack-level index job onto the account's channel.
86 let (pack_id, extent) = self
87 .push_path()
88 .push_pack(self.account(), bytes)
89 .context("storing the pack verbatim")?;
90
91 // The object-level index work: 24 bytes of extent, queued, drained by
92 // `absorb_pending` off this path. Until it runs the pack is un-indexed
93 // and every read falls back rather than answering absent (§13.12).
94 self.queue(pack_id, extent)?;
95
96 Ok(TxId {
97 pack_id: Some(pack_id),
98 extent: Some(extent),
99 push_seq: None,
100 })
101 }
102}
103
104impl<S: ObjectIndex + 'static> GitOps for GitStore<S> {
105 // ── STORE ───────────────────────────────────────────────────────────────
106
107 /// One push, and the order inside it is the contract.
108 ///
109 /// 1. the pack's bytes, durable — [`put_pack`](GitOps::put_pack)
110 /// 2. **then** the refs that point into them — [`put_refs`](GitOps::put_refs)
111 ///
112 /// Not interchangeable, and it is the same argument `SafeWriter` makes one
113 /// level down about the blob and its journal row: a crash between the two
114 /// leaves objects nobody points at, which a GC reclaims. The reverse order
115 /// leaves a **ref pointing at objects that are not there**, which is a
116 /// corrupt repository that no later pass can repair.
117 fn put(&self, pack: &[u8], refs: &[RefUpdate]) -> Result<TxId> {
118 let mut tx = if pack.is_empty() {
119 TxId::default()
120 } else {
121 self.put_pack(pack)?
122 };
123 if !refs.is_empty() {
124 tx.push_seq = self.put_refs(refs)?.push_seq;
125 }
126 Ok(tx)
127 }
128
129 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
130 ///
131 /// **The ack path.** Walk, check, store verbatim, ack, then queue the index.
132 ///
133 /// 1. **Walk every entry** ([`crate::pack_walk`]). Not optional and not an
134 /// extra pass: a pack entry has no length field, so finding the entry
135 /// boundaries *is* how the pack gets split at all.
136 /// 2. **The closure check, free from that walk** (§13.7). Every `OFS_DELTA`
137 /// base must land on an entry boundary in this pack — answered from the
138 /// boundary set the walk just produced, consulting no index. Only a
139 /// `REF_DELTA` base that is *outside* the pack is looked up, and that one
140 /// reads `objects.oid` and nothing else, exactly as §13's table says
141 /// receive-pack does. A pack that fails is refused **before** a byte is
142 /// stored.
143 /// 3. **The bytes, verbatim.** `SafeWriter::append` writes the caller's
144 /// buffer at its own address — no re-compression, no re-encoding, no
145 /// re-framing, not one copy in userspace. The client's own deflate and
146 /// every delta chain survive, which is what makes a later clone a
147 /// byte-range copy instead of a re-pack (§14).
148 /// 4. **Durable before returning**: the blob is fsynced, *then* the journal
149 /// row that references it, then that is fsynced. znippy's `hot.rs`
150 /// ordering, so a crash between the two leaves orphan bytes nobody points
151 /// at rather than a dangling reference.
152 /// 5. **Then** the index job goes on the channel and this returns. Nothing
153 /// that could be done later is done here: no oid is computed, no table is
154 /// built, no chain is resolved.
155 fn put_pack(&self, bytes: &[u8]) -> Result<TxId> {
156 self.put_pack_watching(bytes, None)
157 }
158
159 /// [`put_pack`](GitOps::put_pack), publishing the entry count as step (1)
160 /// produces it.
161 ///
162 /// **The count comes out of the walk that was already happening.** Step (1)
163 /// above is not optional and not an extra pass — it is how the pack gets
164 /// split — so the number of entries taken in is a fact this method already
165 /// has, once per object, and until now it simply had nobody to tell. That
166 /// is the whole of this change: no second decoder, no second inflate, and
167 /// the `put_pack` above is this method with no listener, so there is still
168 /// one implementation of the walk-check-store sequence (LAW 5).
169 ///
170 /// # What the numbers mean, and where they stop
171 ///
172 /// `done` reaches `total` when the **walk** finishes, which is before the
173 /// bytes are durable and long before the indexer has absorbed them. That is
174 /// the honest boundary: this store's ack path is deliberately "walk, check,
175 /// append, queue" (§13.6-8), so there is no later per-object event to count.
176 /// A caller wanting "how much is on disk" is asking for
177 /// [`GitOps::put_pack`]'s return, which is what it has always been.
178 fn put_pack_observed(
179 &self,
180 bytes: &[u8],
181 progress: &dyn git_storage_trait::PackProgress,
182 ) -> Result<TxId> {
183 self.put_pack_watching(bytes, Some(progress))
184 }
185
186 /// This store's answer is `true`, and the conformance suite holds it to it.
187 fn publishes_pack_progress(&self) -> bool {
188 true
189 }
190
191 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
192 ///
193 /// One ref transaction: one Arrow IPC frame, fsynced.
194 ///
195 /// The frame boundary *is* the transaction (see [`crate::pushlog`]) — three
196 /// branches in one push are three rows in one batch, and they either all land
197 /// or none do. There is no lock file and no second journal.
198 ///
199 /// **Every target is checked against the index first.** A ref that points at
200 /// an object the repository does not have is a corrupt repository, and it is
201 /// refused here rather than written and discovered later. A deletion has no
202 /// target and is not checked.
203 fn put_refs(&self, updates: &[RefUpdate]) -> Result<TxId> {
204 if updates.is_empty() {
205 bail!("an empty ref update is not a transaction");
206 }
207 for u in updates {
208 for (what, oid_hex) in [("target", &u.target), ("peeled", &u.peeled)] {
209 let Some(hex_oid) = oid_hex else { continue };
210 let raw = hex::decode(hex_oid)
211 .map_err(|e| anyhow!("{}'s {what} `{hex_oid}` is not hex: {e}", u.name))?;
212 if !self.has(&raw)? {
213 bail!(
214 "{} would point at {hex_oid}, which this repository does not have — the \
215 ref update is refused rather than left dangling",
216 u.name
217 );
218 }
219 }
220 }
221 let push_seq = self.ref_log().push(updates)?;
222 Ok(TxId {
223 pack_id: None,
224 extent: None,
225 push_seq: Some(push_seq),
226 })
227 }
228
229 // ── READ ────────────────────────────────────────────────────────────────
230
231 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
232 ///
233 /// The object's **stored** bytes: one index lookup and one `pread` of the
234 /// extent, with no decode of any kind in between.
235 ///
236 /// What comes back is the pack entry exactly as the client sent it, which for
237 /// a delta is a delta. [`Stored::obj_type`] says which, so the bytes cannot
238 /// be mistaken for the object's content — §14 makes the verbatim bytes the
239 /// truth and the resolved object a derived cache, and that cache does not
240 /// exist yet, so handing back delta bytes labelled "the object" would be the
241 /// one thing this crate refuses everywhere: a wrong answer where an honest
242 /// one was available.
243 fn get(&self, oid: Oid<'_>) -> Result<Option<Stored>> {
244 let Some(row) = self.lookup_one(oid)? else {
245 return Ok(None);
246 };
247 let bytes = self.read_extent(row.offset, row.len)?;
248 Ok(Some(Stored {
249 obj_type: row.obj_type,
250 uncompressed_size: row.uncompressed_size,
251 extent: (row.offset, row.len),
252 bytes,
253 }))
254 }
255
256 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
257 ///
258 /// The negotiation call: `have` sends up to a thousand of these and **most of
259 /// them miss**. It reads `objects.oid` and no other column.
260 ///
261 /// It takes the **serial** path, not `lookup_batch` with one element: that is
262 /// 1.7× slower (818 ns against 482), and [`lookup_path`] is where that
263 /// decision is written down.
264 ///
265 /// # Why this returns `Result<bool>` and not `bool`
266 ///
267 /// The signature changed deliberately. A store with a pack whose bytes are
268 /// durable but whose objects are not indexed yet cannot answer "no" — the
269 /// object may be in that pack. So it absorbs the pack first and, if that
270 /// fails, **says so**. A `bool` could only have lied, and a wrong "absent"
271 /// during negotiation makes a client send nothing and lose data.
272 fn has(&self, oid: Oid<'_>) -> Result<bool> {
273 Ok(self.lookup_one(oid)?.is_some())
274 }
275
276 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
277 ///
278 /// The object's **post-resolution** size — what it inflates to once its delta
279 /// chain is applied. That is the fact git's own `.idx` and `.rev` together
280 /// cannot answer, and it is why the quota gate is index-only here.
281 fn size(&self, oid: Oid<'_>) -> Result<Option<u64>> {
282 Ok(self.lookup_one(oid)?.map(|r| r.uncompressed_size))
283 }
284
285 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
286 ///
287 /// **The wire path**, and batch by construction: for each oid the two facts a
288 /// clone needs — `offset` and `len` — and no others, so a byte-range copy out
289 /// of the verbatim pack can start immediately.
290 ///
291 /// `out[i]` answers `oids[i]`. One oid takes the serial path
292 /// ([`lookup_path`]); the batch path saturates at
293 /// [`BATCH_SATURATES_AT`](crate::git_ops::BATCH_SATURATES_AT) oids, so a
294 /// larger batch is passed through whole rather than split — splitting would
295 /// cost a pass and buy nothing.
296 fn extents(&self, oids: &[Oid<'_>]) -> Result<Vec<Option<Extent>>> {
297 match lookup_path(oids.len()) {
298 LookupPath::Serial => Ok(vec![self.lookup_one(oids[0])?.map(|r| (r.offset, r.len))]),
299 LookupPath::Batch => Ok(self
300 .lookup_batch_never_wait(oids)?
301 .into_iter()
302 .map(|r| r.map(|r| (r.offset, r.len)))
303 .collect()),
304 }
305 }
306
307 // ── REFS ────────────────────────────────────────────────────────────────
308
309 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
310 ///
311 /// The whole ref namespace, tags peeled — the ref advertisement, and
312 /// `ls-refs` after a prefix filter. Name-sorted, because the log folds into a
313 /// `BTreeMap` and `ls-refs` wants a prefix scan.
314 ///
315 /// The cost is a scan of a structure sized by **pushes**, not by repository
316 /// size.
317 ///
318 /// # `HEAD` is not in here, and that is the contract
319 ///
320 /// **Changed 2026-08-10, and it is a fix rather than a preference.** The
321 /// other backend's `iter()` *"walks `refs/` (loose and packed) and
322 /// deliberately excludes the pseudo-refs such as `HEAD`, which is exactly the
323 /// contract every other backend honours"*. This one did not, so the two arms
324 /// disagreed about whether `HEAD` is a row — and the conformance suite could
325 /// not see it, because it never exercised `HEAD` at all.
326 ///
327 /// The reason `HEAD` cannot be a row is a type constraint, not taste: a name
328 /// type that admits `HEAD` also admits `MERGE_HEAD` and `FETCH_HEAD`, so a
329 /// row stream carrying pseudo-refs means either widening the name type or
330 /// filtering at every consumer. It gets an accessor pair instead —
331 /// [`GitServe::head`](crate::serve::GitServe::head) and
332 /// [`set_head`](crate::serve::GitServe::set_head) — and this is the one
333 /// filter, in the one place.
334 ///
335 /// Nothing else changes: the ref log still *stores* `HEAD` (a push writes it
336 /// like any other row), `live_set` still reaches it because it reads
337 /// `ref_state` directly, and the filter is one `!=`.
338 fn refs(&self) -> Result<Vec<RefRow>> {
339 let mut out = Vec::new();
340 for (name, state) in self
341 .ref_state()?
342 .into_iter()
343 .filter(|(name, _)| name != crate::serve::HEAD)
344 {
345 let decode = |h: &Option<String>| -> Result<Option<Vec<u8>>> {
346 match h {
347 Some(h) => {
348 Ok(Some(hex::decode(h).map_err(|e| {
349 anyhow!("ref {name}: `{h}` is not a hex oid: {e}")
350 })?))
351 }
352 None => Ok(None),
353 }
354 };
355 out.push(RefRow {
356 oid: decode(&state.target)?,
357 peeled: decode(&state.peeled)?,
358 symref_target: state.symref_target.clone(),
359 name,
360 });
361 }
362 Ok(out)
363 }
364
365 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
366 ///
367 /// **Compare-and-swap**, in the same log a push writes its refs to — one
368 /// mechanism, not a second one for single updates.
369 ///
370 /// `old` is what the caller believes the ref is: `None` means *it must not
371 /// exist* (a create), `Some(oid)` means *it must be exactly this*. `new` of
372 /// `None` deletes. A mismatch names both values and **writes nothing**.
373 ///
374 /// The read-compare-append is serialised on the store's ref gate. Without it
375 /// two CAS calls could both read the old value and both append, and the
376 /// second would silently overwrite an update it had compared against
377 /// successfully — the classic lost update, and the only thing a CAS is for.
378 fn update_ref(&self, name: &str, old: Option<Oid<'_>>, new: Option<Oid<'_>>) -> Result<TxId> {
379 let _gate = self
380 .ref_gate()
381 .lock()
382 .map_err(|_| anyhow!("the ref gate is poisoned"))?;
383
384 // One keyed read under the gate, not a clone of the whole namespace:
385 // this CAS compares and swaps exactly `name`.
386 let current = self.ref_get(name)?;
387 let actual: Option<Vec<u8>> = match current.as_ref().and_then(|s| s.target.as_deref()) {
388 Some(h) => Some(
389 hex::decode(h)
390 .map_err(|e| anyhow!("ref {name} holds `{h}`, not a hex oid: {e}"))?,
391 ),
392 None => None,
393 };
394 if actual.as_deref() != old {
395 // ── TYPED, like `put_refs_cas` three hundred lines down ─────────
396 //
397 // This raised a bare `bail!` while its own sibling raised
398 // `RefRejection::Cas`, so a caller had a string and no way to tell
399 // a lost race from a broken disk. `gunnar-wire`'s receive-pack
400 // says so at the call site: "A LOST COMPARE-AND-SWAP AND A BROKEN
401 // DISK ARE THE SAME STRING HERE … Nothing may recover the
402 // distinction by matching on this text; the fix is a typed
403 // rejection on the contract."
404 //
405 // The contract already had one. It was this method that did not
406 // use it. The `Display` text is unchanged — `RefRejection::Cas`
407 // renders the same sentence — so nothing that reads the message
408 // moves, and a caller that downcasts now gets `name`, `expected`
409 // and `actual` as values instead of parsing prose.
410 return Err(anyhow::Error::new(RefRejection::Cas {
411 name: name.to_string(),
412 expected: match old {
413 None => Observed::Nothing,
414 Some(o) => Observed::oid(o),
415 },
416 actual: match actual.as_deref() {
417 None => Observed::Nothing,
418 Some(a) => Observed::oid(a),
419 },
420 }));
421 }
422
423 let update = match new {
424 Some(n) => RefUpdate::set(name, hex::encode(n)),
425 None => RefUpdate::delete(name),
426 };
427 // Through `put_refs`, so the existence check and the log format are the
428 // same code a push uses (LAW 5) — a CAS cannot create a dangling ref that
429 // a push would have been refused for.
430 if new.is_some() {
431 self.put_refs(&[update])
432 } else {
433 let push_seq = self.ref_log().push(&[update])?;
434 Ok(TxId {
435 pack_id: None,
436 extent: None,
437 push_seq: Some(push_seq),
438 })
439 }
440 }
441
442 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
443 ///
444 /// **`git push --atomic`: every edit or none**, in one Arrow IPC frame.
445 ///
446 /// The two halves come from different places and neither is reimplemented:
447 ///
448 /// * **the batch atomicity is the frame.** [`crate::pushlog`] makes the frame
449 /// boundary the transaction — three branches in one push are three rows in
450 /// one batch and they either all land or none do. There is no lock file and
451 /// no second journal, so there is no partial-apply state to recover from;
452 /// * **the compare half is checked here, against one snapshot**, before a
453 /// single [`RefUpdate`] is built.
454 ///
455 /// It goes out through [`put_refs`](GitOps::put_refs) rather than straight to
456 /// the log, so the dangling-target refusal and the frame format are the same
457 /// code a push uses (LAW 5): an atomic batch cannot create a ref pointing at
458 /// an object the repository does not have, which a second writer here would
459 /// eventually have allowed.
460 ///
461 /// # `S-023`, and why the check is a whole pass of its own
462 ///
463 /// Every expectation is evaluated **before any of them is applied**, and the
464 /// loop deliberately does not fuse with the one that builds the updates. The
465 /// defect it exists for is gix's: a backend that short-circuits an edge whose
466 /// new value already equals the current one never evaluates the expectation
467 /// the caller wrote, so an `old: None` — *must not exist* — becomes a silent
468 /// success and *"exactly one creator wins"* stops being true. This log has no
469 /// such short-circuit, but the ordering is what makes that irrelevant rather
470 /// than lucky, and a future optimisation that adds one cannot break it from
471 /// here.
472 ///
473 /// # What this arm cannot raise, stated rather than hidden
474 ///
475 /// **[`RefRejection::Locked`] never comes out of this backend.** The
476 /// read-compare-append is serialised on the store's ref gate, which a second
477 /// writer *blocks* on rather than failing against — so contention here is a
478 /// wait, never a rejection. The variant is in the contract because the gix
479 /// arm, which takes real per-ref lock files, raises it. A poisoned gate stays
480 /// an ordinary error: it is a fault, not the transient thing a caller retries.
481 fn put_refs_cas(&self, edits: &[RefCas<'_>]) -> Result<TxId> {
482 // Not an error. A deletions-free push that had nothing to apply calls
483 // this, and `put_refs` refuses an empty batch — rightly, since an empty
484 // ref update is not a transaction. An empty *atomic* batch is a no-op
485 // that succeeded, and saying so here is what keeps the special case out
486 // of every call site.
487 if edits.is_empty() {
488 return Ok(TxId::default());
489 }
490
491 let _gate = self
492 .ref_gate()
493 .lock()
494 .map_err(|_| anyhow!("the ref gate is poisoned"))?;
495
496 // ONE read of the namespace for the whole check, so every expectation is
497 // compared against one instant rather than against a namespace that may
498 // move between them.
499 let current = self.ref_state()?;
500 let observed = |name: &str| -> Observed {
501 match current.get(name) {
502 None => Observed::Nothing,
503 Some(s) => match (&s.symref_target, &s.target) {
504 (Some(points_to), _) => {
505 Observed::Value(RefTarget::Symbolic(points_to.clone()))
506 }
507 (None, Some(h)) => match hex::decode(h) {
508 Ok(raw) => Observed::Value(RefTarget::Object(raw)),
509 // Not dropped into `Nothing`: a ref that exists and
510 // cannot be read is not a ref that was absent, and
511 // reporting it as absent tells the pushing client its
512 // create is free when it is not.
513 Err(e) => Observed::Unreadable(format!("`{h}` is not a hex oid: {e}")),
514 },
515 (None, None) => Observed::Unreadable(
516 "the ref log holds a row naming neither an object nor another ref".into(),
517 ),
518 },
519 }
520 };
521
522 // S-023: the whole check, before anything is applied.
523 for e in edits {
524 let expected = match e.old {
525 None => Observed::Nothing,
526 Some(o) => Observed::oid(o),
527 };
528 let actual = observed(&e.name);
529 if actual != expected {
530 return Err(anyhow::Error::new(RefRejection::Cas {
531 name: e.name.clone(),
532 expected,
533 actual,
534 }));
535 }
536 }
537
538 let updates: Vec<RefUpdate> = edits
539 .iter()
540 .map(|e| match e.new {
541 // The peel rides through `put_refs` into the same ref row, so a
542 // tag ingested this way advertises with zero object reads
543 // (`RefCas::peeled`, gunnar S-080). It is carried only on a set;
544 // a delete has no row to peel.
545 Some(n) => {
546 let update = RefUpdate::set(e.name.clone(), hex::encode(n));
547 match &e.peeled {
548 Some(p) => update.with_peeled(hex::encode(p)),
549 None => update,
550 }
551 }
552 None => RefUpdate::delete(e.name.clone()),
553 })
554 .collect();
555 self.put_refs(&updates)
556 }
557
558 // ── GRAPH ───────────────────────────────────────────────────────────────
559
560 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
561 ///
562 /// **Selection: `want` minus `have`, as an `andnot` of roaring bitmaps.**
563 ///
564 /// Every returned oid is an object — commit, tree *and* blob — because the
565 /// bitmaps are built by [`crate::reach::build_reach`], which walks the trees.
566 /// A `want` that names a commit contributes that commit's whole closure; a
567 /// `want` that names a tag or a blob contributes itself.
568 ///
569 /// A `have` this repository does not know contributes nothing: the safe
570 /// direction is to send more, never less.
571 ///
572 /// The bitmaps are over the **store's own ordinal space**, rebuilt by the same
573 /// fold that rebuilds them, never over the Arrow projection's ordinals — an
574 /// [`crate::index_layout::IndexRow::ordinal`] is a row address within one
575 /// projection generation, so a bitmap over those would address different
576 /// objects after any rebuild, silently.
577 ///
578 /// # The per-oid `Vec` is paid HERE and nowhere else on the serving path
579 ///
580 /// The answer is computed flat (see
581 /// [`GitStore::reachable_raw`](crate::git_ops::GitStore::reachable_raw));
582 /// this splits it back out because the eleven say `Vec<Vec<u8>>` and this
583 /// method's callers are maintenance ones — a GC live set, a conformance
584 /// harness — not the serving path. `crate::serve`'s `select` and
585 /// `emit_pack` take the flat form directly.
586 fn reachable(&self, want: &[Oid<'_>], have: &[Oid<'_>]) -> Result<Vec<Vec<u8>>> {
587 Ok(self
588 .reachable_raw(want, have)?
589 .iter()
590 .map(<[u8]>::to_vec)
591 .collect())
592 }
593
594 // ── MAINT ───────────────────────────────────────────────────────────────
595
596 /// Garbage collection, in the order §13.20 fixes:
597 ///
598 /// 1. **compute reachability** — every object reachable from every ref, over
599 /// the same bitmaps [`reachable`](GitOps::reachable) uses;
600 /// 2. **drop the dead rows from the index** — the one operation that is not
601 /// append-only, and the projection is rebuilt inside it;
602 /// 3. **then** base znippy's compaction, through [`crate::gc::NewGeneration`]
603 /// (link, compact, verify, rename, unlink last).
604 ///
605 /// That order is why **base znippy needs no new method**: by the time
606 /// `compact_archive` runs, "live" already means what git means.
607 ///
608 /// A repository with no ref pointing at anything is refused rather than
609 /// emptied.
610 ///
611 /// **Appended, not reworded: step 1b, the journal.** Between the live set and
612 /// the drop there is now one more durable act. §13.12's `indexed` bit is
613 /// derived on open as *extent in the journal, rows not in the index*, and
614 /// dropping **every** row of a pack produces exactly that state — so before
615 /// this existed the next open re-queued the pack and every object this GC had
616 /// just decided was dead came back.
617 /// [`GitStore::retire_dead_packs`](crate::git_ops::GitStore::retire_dead_packs)
618 /// appends a tombstone naming each all-dead pack, and it runs **before**
619 /// `drop_dead_rows` on purpose: killed before it, the rows are still there
620 /// and the GC simply did not happen; killed after it, the pack can never be
621 /// re-queued whether the rows went or not. A partly dead pack keeps rows and
622 /// is never tombstoned, so nothing about it changes.
623 ///
624 /// Step 2 also refolds what the two tables feed — the commit graph, the tree
625 /// payloads, the ordinal space, the bitmaps — because a derivation that still
626 /// names a dropped oid is wrong rather than stale. That is inside
627 /// `drop_dead_rows`, so any caller of it gets it.
628 fn gc(&self) -> Result<GcReport> {
629 self.absorb_pending()?;
630 // **What the GC may touch is fixed BEFORE reachability is computed.**
631 // `live_set` walks from the refs as they are now; a push that lands
632 // after this point appends rows and a pack that no ref names yet, and
633 // to `live` those objects are simply unknown. The old code dropped
634 // what was not live — so it dropped them. Two snapshots taken first:
635 // the rows that exist (`seen`) and the journal's end (`acked_end`).
636 // Dead is `seen − live`; a pack is retirable only if it was acked
637 // before `acked_end`. A row or pack that arrives during the GC is in
638 // neither and survives, whatever the refs say about it yet.
639 let seen = self.index().oids_in_order()?;
640 let acked_end = self.acked_end()?;
641 let live = self.live_set()?;
642 let dead = crate::git_ops::dead_oids(seen, &live);
643 let retired = self.retire_dead_packs(&live, acked_end)?;
644 let before = self.index().len() as u64;
645 let dropped = self.drop_dead_rows(&dead)?;
646 let after = self.index().len() as u64;
647 if before.saturating_sub(after) != dropped || after > before {
648 bail!(
649 "the index dropped {dropped} rows but went from {before} to {after} — refusing to \
650 compact an archive whose index does not agree with itself"
651 );
652 }
653 let mut report = self.gc_arm().run(self.archive_path()).with_context(|| {
654 format!(
655 "compacting {} after dropping {dropped} dead rows",
656 self.archive_path().display()
657 )
658 })?;
659 report.retired_packs = retired.len() as u64;
660 Ok(report)
661 }
662}
663
664impl<S: ObjectIndex + 'static> GitStore<S> {
665 /// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
666 ///
667 /// Fold the live logs into the reserved Arrow sections an archive carries:
668 /// `__gunnar_refs__`, `__gunnar_graph__`, `__gunnar_reach__`.
669 ///
670 /// **The twelfth method, and the one that is NOT on [`GitOps`]:** it returns
671 /// `znippy_common::ReservedSection` (Arrow `RecordBatch` payloads), which a
672 /// gix backend has no analog for — so it is inherent on the concrete store,
673 /// which is how gunnar already calls it (never through the trait).
674 ///
675 /// It absorbs first, because a section built while a pack is still
676 /// un-indexed would be **silently incomplete**, and a silently incomplete
677 /// index is worse than none (the same argument [`crate::lib`] makes about not
678 /// wiring `GitIndexBuilder` into the CLI). Every generation number in the
679 /// graph section is recomputed by that fold.
680 ///
681 /// **…and then it writes the archive.** Until 2026-08-08 it only *returned*
682 /// the sections and nothing on any path created
683 /// [`archive_path`](GitStore::archive_path) at all — so `gc()`'s last step
684 /// died on `stat repository.znippy: No such file or directory`, having
685 /// already done its first four. Both [`Gc`](crate::gc::Gc) implementations
686 /// compact an archive that exists; **this** is what makes one exist.
687 /// [`seal_generation_zero`](crate::archive_write::seal_generation_zero) has
688 /// the layout argument.
689 ///
690 /// The sections are still returned, and they are the same values that were
691 /// sealed rather than a second derivation of them — `ReservedSection` is
692 /// `Clone` for exactly that reason.
693 pub fn seal(&self) -> Result<Vec<ReservedSection>> {
694 self.absorb_pending()?;
695 self.index().rebuild()?;
696 let sections = self.reserved_sections()?;
697 self.seal_archive(sections.clone()).with_context(|| {
698 format!("sealing generation 0 at {}", self.archive_path().display())
699 })?;
700 Ok(sections)
701 }
702}
703
704/// CLAUDE FUCKED UP AND MUST NOT FUCK UP AGAIN. ZNIPP-GIT APACHE ARROR IPC IS LAW
705///
706/// Compact `src` into a **named** destination, leaving `src` untouched.
707///
708/// The file-level primitive behind [`GitOps::gc`]'s third step, for a caller that
709/// wants to choose the name rather than take
710/// [`crate::gc::next_generation`]'s. Three existing calls and no fourth
711/// mechanism:
712///
713/// 1. `hard_link(src, dst)` — a second name for the same inode. **Not one byte is
714/// copied**, and `src` keeps pointing at the original for the whole run.
715/// 2. `compact_archive(dst)` — base znippy's own compaction, verbatim, against
716/// the new name.
717/// 3. read every entry back through the ordinary reader — the same verification
718/// [`crate::gc::NewGeneration`] gates on, so a `dst` that does not read back
719/// is removed and the error says so.
720///
721/// `dst` must not exist. Silently compacting over a file is how a generation gets
722/// lost.
723pub fn compact(src: &Path, dst: &Path) -> Result<()> {
724 if dst.exists() {
725 bail!(
726 "{} already exists — refusing to compact over it",
727 dst.display()
728 );
729 }
730 std::fs::hard_link(src, dst).with_context(|| {
731 format!(
732 "hard-linking {} to {} — compaction runs against a second name for the same inode, \
733 which needs both on one filesystem",
734 src.display(),
735 dst.display()
736 )
737 })?;
738 if let Err(e) = znippy_common::compact_archive(dst) {
739 let _ = std::fs::remove_file(dst);
740 return Err(e.context(format!(
741 "compacting {} into {}",
742 src.display(),
743 dst.display()
744 )));
745 }
746 if let Err(e) = crate::gc::read_back_every_entry(dst) {
747 let _ = std::fs::remove_file(dst);
748 return Err(e.context(format!(
749 "{} did not read back after compaction — it was removed and {} is untouched",
750 dst.display(),
751 src.display()
752 )));
753 }
754 Ok(())
755}
756
757#[cfg(test)]
758pub(crate) mod tests {
759 use super::*;
760 use crate::git_ops::BATCH_SATURATES_AT;
761 use crate::index_layout::ObjType;
762 use crate::object::{canonical, GitHashKind, GitObjectKind};
763 use crate::resolve::{resolve, NoBases};
764 use std::path::PathBuf;
765 use std::time::Instant;
766
767 /// A store directory for one test.
768 ///
769 /// It returns a `PathBuf` and not a `tempfile::TempDir`, so **nothing
770 /// deletes it when the test ends** — 82 call sites take it as a path and
771 /// several deliberately outlive a store to reopen it, which is what the
772 /// borrowed form would forbid. The directories are therefore swept on the
773 /// way in instead of dropped on the way out; see [`sweep_dead_runs`].
774 pub(crate) fn tmpdir(tag: &str) -> PathBuf {
775 sweep_dead_runs();
776 let d = std::env::temp_dir().join(format!(
777 "znippy-git-store-{tag}-{}-{}",
778 std::process::id(),
779 std::time::SystemTime::now()
780 .duration_since(std::time::UNIX_EPOCH)
781 .map(|d| d.as_nanos())
782 .unwrap_or(0)
783 ));
784 std::fs::create_dir_all(&d).unwrap();
785 d
786 }
787
788 /// **Delete the store directories left behind by test runs that are over.**
789 ///
790 /// MEASURED on oden 2026-08-11: **1 088 of these directories, 96 GB**, and
791 /// `std::env::temp_dir()` on that box is `/tmp`, which is a `tmpfs` — so
792 /// that was 96 GB of *RAM* held by test runs that had exited days earlier,
793 /// on the shared machine every performance figure in this crate is taken
794 /// on. Each one is a whole store, and one with an exploded table is ~180 MB.
795 ///
796 /// Two conditions, both required, because the only way this can do harm is
797 /// by deleting a directory a running test still wants:
798 ///
799 /// 1. **The pid in the name is not a live process.** The name has carried
800 /// the pid since it was written; nothing consulted it until now.
801 /// 2. **The directory has not been touched for an hour.** Redundant against
802 /// a correct pid check, and there precisely because pids are reused: a
803 /// fresh run that inherited a dead run's pid has a fresh mtime.
804 ///
805 /// Once per process, not once per call: 82 call sites would otherwise scan
806 /// the whole of `/tmp` 82 times.
807 fn sweep_dead_runs() {
808 static ONCE: std::sync::Once = std::sync::Once::new();
809 ONCE.call_once(|| {
810 let Ok(entries) = std::fs::read_dir(std::env::temp_dir()) else {
811 return;
812 };
813 for e in entries.flatten() {
814 let name = e.file_name();
815 let Some(rest) = name
816 .to_str()
817 .and_then(|n| n.strip_prefix("znippy-git-store-"))
818 else {
819 continue;
820 };
821 // `<tag>-<pid>-<nanos>`, and a tag may contain `-`, so the pid
822 // is the second field from the right.
823 let Some(pid) = rest.rsplit('-').nth(1).and_then(|p| p.parse::<u32>().ok()) else {
824 continue;
825 };
826 if pid == std::process::id() || Path::new(&format!("/proc/{pid}")).exists() {
827 continue;
828 }
829 let stale = e
830 .metadata()
831 .and_then(|m| m.modified())
832 .ok()
833 .and_then(|t| t.elapsed().ok())
834 .is_some_and(|age| age > std::time::Duration::from_secs(3600));
835 if stale {
836 let _ = std::fs::remove_dir_all(e.path());
837 }
838 }
839 });
840 }
841
842 /// **The corpus pack, with its objects already resolved so a test knows what
843 /// oids to ask for.**
844 ///
845 /// # 🔴 What this was, and why it is not that any more
846 ///
847 /// Until 2026-08-25 this scanned `~/git` and returned **the
848 /// first `.pack` under 8 MiB, in `read_dir` order, that resolved standalone
849 /// to more than 100 objects**. Thirty-five tests take their corpus from
850 /// here, so the subject of all thirty-five was whichever repository the
851 /// filesystem happened to name first on one developer's box — and this box
852 /// had seven agents cloning and removing repositories beside it.
853 ///
854 /// It had drifted. `cargo test -p znippy-plugin-git --lib` on `3358bc3`,
855 /// clean tree: **197 passed, 8 failed**, every failure a property of the
856 /// found pack (it was not reachability-closed, it carried `REF_DELTA`s, and
857 /// it was 933 KB where a bound wanted over 1 MB). And with the corpus root
858 /// simply absent — a CI box, or anyone else's machine — all thirty-five died
859 /// in the fixture: *"panicked at src/store.rs:763: Os { code: 2, kind:
860 /// NotFound }"*.
861 ///
862 /// It is now [`crate::pack_fixture::corpus`]: a linear history generated in
863 /// Rust from constants that module states, touching no filesystem. See that
864 /// module for the eight failures in full and for why generating beat
865 /// committing a `.pack`.
866 ///
867 /// **Built once per test binary.** The corpus is deterministic and
868 /// immutable, and thirty-five call sites were each re-reading and
869 /// re-resolving a pack.
870 pub(crate) fn corpus_pack() -> (Vec<u8>, Vec<crate::resolve::Resolved>) {
871 static ONCE: std::sync::OnceLock<(Vec<u8>, Vec<crate::resolve::Resolved>)> =
872 std::sync::OnceLock::new();
873 ONCE.get_or_init(|| {
874 let pack = crate::pack_fixture::corpus().pack.clone();
875 let rows = resolve(&pack, GitHashKind::Sha1, 0, &NoBases)
876 .expect("the generated corpus resolves standalone");
877 (pack, rows)
878 })
879 .clone()
880 }
881
882 /// A tiny pack built here, so a test can push something that is not tied to
883 /// this machine. One object, whole, of whatever type is asked for.
884 ///
885 /// One writer for the framing: `one_blob_pack` and `one_tag_pack` differ by
886 /// the three bits git spends on an object type and by nothing else, and two
887 /// copies of a pack header is exactly the kind of twin LAW 5 is about.
888 pub(crate) fn one_object_pack(kind: GitObjectKind, body: &[u8]) -> (Vec<u8>, Vec<u8>) {
889 use std::io::Write;
890 let type_bits: u8 = match kind {
891 GitObjectKind::Commit => 1,
892 GitObjectKind::Tree => 2,
893 GitObjectKind::Blob => 3,
894 GitObjectKind::Tag => 4,
895 };
896 let mut pack = b"PACK".to_vec();
897 pack.extend_from_slice(&2u32.to_be_bytes());
898 pack.extend_from_slice(&1u32.to_be_bytes());
899 let mut size = body.len() as u64;
900 let mut header = vec![(type_bits << 4) | (size as u8 & 0x0f)];
901 size >>= 4;
902 while size > 0 {
903 let last = header.len() - 1;
904 header[last] |= 0x80;
905 header.push((size & 0x7f) as u8);
906 size >>= 7;
907 }
908 pack.extend_from_slice(&header);
909 let mut e = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
910 e.write_all(body).unwrap();
911 pack.extend_from_slice(&e.finish().unwrap());
912 pack.extend_from_slice(&[0u8; 20]);
913 let oid = GitHashKind::Sha1.oid_of(&canonical(kind, body));
914 (pack, oid)
915 }
916
917 /// One blob, whole.
918 pub(crate) fn one_blob_pack(body: &[u8]) -> (Vec<u8>, Vec<u8>) {
919 one_object_pack(GitObjectKind::Blob, body)
920 }
921
922 /// **One ANNOTATED TAG, pointing at `target_hex`.**
923 ///
924 /// The object format is the point of the fixture: a tag's payload opens
925 /// `object <hex>\n`, and that line is the only thing that connects a tag to
926 /// the history it names. A reachability walk that does not read it answers
927 /// with the tag and nothing under it.
928 pub(crate) fn one_tag_pack(target_hex: &str, name: &str) -> (Vec<u8>, Vec<u8>) {
929 let body = format!(
930 "object {target_hex}\ntype commit\ntag {name}\ntagger t <t@t> 0 +0000\n\nannotated\n"
931 );
932 one_object_pack(GitObjectKind::Tag, body.as_bytes())
933 }
934
935 /// **The bytes come back byte for byte.** A real repository's pack is pushed,
936 /// absorbed, and then every object is fetched by oid and compared against the
937 /// exact slice of the pushed bytes it came from.
938 ///
939 /// This is the verbatim contract asserted as applied output — on the file on
940 /// disk, not on what `put` returned. A store that re-compressed, re-framed or
941 /// re-encoded anything cannot pass it.
942 ///
943 /// Seen RED by making `put_pack` store `&bytes[..bytes.len() - 1]`: "the
944 /// extent covers the whole pack — left: 5653301, right: 5653302". One byte
945 /// short of verbatim and the extent no longer covers the push.
946 #[test]
947 fn every_stored_object_reads_back_as_the_exact_bytes_that_were_pushed() {
948 let dir = tmpdir("verbatim");
949 let store = GitStore::open(&dir, "alice").unwrap();
950 let (pack, rows) = corpus_pack();
951
952 // The drain is parked in front of its absorb for as long as this lives,
953 // so "queued, not run" below is a **state** and not a timing window. It
954 // used to be neither: `absorb_pending() == 1` raced the drain for the
955 // gate and lost whenever the drain got there first, which is a wrong
956 // expectation rather than a wrong store — both callers end in the same
957 // rows and the count only says which one paid.
958 let gate = store.hold_absorb_gate();
959 let tx = store.put(&pack, &[]).unwrap();
960 let (offset, len) = tx.extent.expect("a pack push records its extent");
961 assert_eq!(len, pack.len() as u64, "the extent covers the whole pack");
962
963 // The file on disk holds the pushed bytes, unchanged.
964 let on_disk = std::fs::read(store.blobs_path()).unwrap();
965 assert_eq!(
966 &on_disk[offset as usize..(offset + len) as usize],
967 &pack[..],
968 "the archive does not hold the pack verbatim"
969 );
970
971 assert_eq!(
972 store.unindexed_packs(),
973 1,
974 "the index job is queued, not run"
975 );
976 assert_eq!(
977 store.object_count(),
978 0,
979 "a row landed while the gate was held"
980 );
981 drop(gate);
982 store.wait_indexed();
983 assert_eq!(
984 store.absorb_pending().unwrap(),
985 0,
986 "the drain left index work"
987 );
988 assert_eq!(store.object_count(), rows.len(), "every object is indexed");
989
990 for (i, r) in rows.iter().enumerate() {
991 let got = store
992 .get(&r.oid)
993 .unwrap()
994 .unwrap_or_else(|| panic!("object {i} {} is missing", hex::encode(&r.oid)));
995 let from = (offset + r.offset) as usize;
996 assert_eq!(
997 got.bytes,
998 &pack[from..from + r.len as usize],
999 "object {i} {}: the stored bytes are not the pushed bytes",
1000 hex::encode(&r.oid)
1001 );
1002 assert_eq!(got.extent, (offset + r.offset, r.len));
1003 assert_eq!(got.uncompressed_size, r.uncompressed_size);
1004 assert_eq!(got.obj_type, r.stored_type);
1005 assert_eq!(store.size(&r.oid).unwrap(), Some(r.uncompressed_size));
1006 assert!(store.has(&r.oid).unwrap());
1007 }
1008
1009 // A delta really is labelled a delta, so nobody can mistake its bytes for
1010 // content. (If this pack had none, the assertion above would be vacuous.)
1011 let deltas = rows
1012 .iter()
1013 .filter(|r| matches!(r.stored_type, ObjType::OfsDelta | ObjType::RefDelta))
1014 .count();
1015 assert!(deltas > 0, "the fixture pack carries no deltas");
1016 let d = rows
1017 .iter()
1018 .find(|r| matches!(r.stored_type, ObjType::OfsDelta | ObjType::RefDelta))
1019 .unwrap();
1020 let got = store.get(&d.oid).unwrap().unwrap();
1021 assert!(matches!(
1022 got.obj_type,
1023 ObjType::OfsDelta | ObjType::RefDelta
1024 ));
1025 assert_ne!(
1026 got.bytes.len() as u64,
1027 got.uncompressed_size,
1028 "a delta's stored bytes are not its resolved size"
1029 );
1030
1031 // An oid nothing pushed is absent, and that is an answer, not a guess.
1032 assert!(!store.has(&[0xab; 20]).unwrap());
1033 assert!(store.get(&[0xab; 20]).unwrap().is_none());
1034 assert_eq!(store.size(&[0xab; 20]).unwrap(), None);
1035 }
1036
1037 /// **Durability, in the order that matters.** `put_pack` must not return
1038 /// before the bytes are on the device *and* a journal row on the device
1039 /// points at them — and if the machine dies between the two, what is left
1040 /// must be orphan bytes, never a row pointing into a hole.
1041 ///
1042 /// Asserted by reading the two files back from disk after a fault injected
1043 /// into the **real** `SafeWriter::append` (LAW 5: the fault goes into the one
1044 /// writer, rather than a hand-rolled twin of it being tested).
1045 ///
1046 /// Seen RED by moving the blob `fsync` in `SafeWriter::append` to *after* the
1047 /// journal row and its `fsync`, with the injected crash between them — i.e.
1048 /// the reference is made durable before the bytes it references: "the journal
1049 /// claims 1 extent(s) after a crash before the blob was durable — that is a
1050 /// dangling reference, which is the failure this ordering exists to prevent".
1051 /// Restored.
1052 ///
1053 /// **And the honest limit of this guard, found by trying a weaker mutation
1054 /// first.** Moving the blob `fsync` to the end *without* moving the fault did
1055 /// **not** turn it red: the fault still fires before the journal row, so the
1056 /// on-disk state is still orphan bytes, and an `fsync` that did not happen is
1057 /// unobservable from inside the process — the bytes read back out of the page
1058 /// cache either way. What this guard proves is therefore the **ordering of the
1059 /// reference against the bytes**, which is the part a crash can expose. It
1060 /// cannot prove an `fsync` reached the platter; nothing short of cutting power
1061 /// can.
1062 #[test]
1063 fn a_pack_is_durable_before_put_returns_and_the_fsyncs_are_ordered() {
1064 use crate::archive_write::{read_journal, ArchiveWrite, Faults, SafeWriter};
1065
1066 let dir = tmpdir("durable");
1067 let (pack, _) = one_blob_pack(b"durability is an ordering property");
1068
1069 // 1. The ordinary path: after `put_pack` returns, both files hold it.
1070 let store = GitStore::open(&dir, "alice").unwrap();
1071 let tx = store.put_pack(&pack).unwrap();
1072 let (offset, len) = tx.extent.unwrap();
1073 let journal = SafeWriter::journal_path(store.blobs_path());
1074 assert_eq!(
1075 read_journal(&journal).unwrap(),
1076 vec![(offset, len)],
1077 "the journal row that claims the extent is not there when put_pack returned"
1078 );
1079 assert_eq!(
1080 std::fs::metadata(store.blobs_path()).unwrap().len(),
1081 offset + len,
1082 "the blob file does not end where the journal says the pack does"
1083 );
1084
1085 // 2. A crash between the two fsyncs leaves ORPHAN BYTES, not a dangling
1086 // reference. Same `append`, one fault flipped.
1087 let dir2 = tmpdir("crash");
1088 let blobs = dir2.join("objects.pack");
1089 let w = SafeWriter::create_with_faults(
1090 &blobs,
1091 Faults {
1092 die_between_fsyncs: true,
1093 ..Default::default()
1094 },
1095 )
1096 .unwrap();
1097 assert!(
1098 w.append(&pack).is_err(),
1099 "the injected crash must not return Ok"
1100 );
1101 drop(w);
1102 let orphan = std::fs::read(&blobs).unwrap();
1103 assert_eq!(orphan, pack, "the blob bytes were fsynced before the crash");
1104 assert!(
1105 read_journal(&SafeWriter::journal_path(&blobs))
1106 .unwrap()
1107 .is_empty(),
1108 "the journal claims {} extent(s) after a crash before the blob was durable — that is a \
1109 dangling reference, which is the failure this ordering exists to prevent",
1110 read_journal(&SafeWriter::journal_path(&blobs)).unwrap().len()
1111 );
1112
1113 // 3. And a store reopened over the orphan file answers for nothing,
1114 // rather than for bytes nobody claimed.
1115 let reopened = GitStore::open(&dir2, "alice").unwrap();
1116 assert_eq!(reopened.object_count(), 0);
1117 }
1118
1119 /// **The closure check refuses a pack before storing it**, and it does so
1120 /// without consulting the index for anything except a genuinely external
1121 /// base.
1122 ///
1123 /// Asserted on applied output: the blob file does not grow, and no journal
1124 /// row appears.
1125 ///
1126 /// Seen RED by moving `external_bases_exist` to *after* `push_pack`: "the
1127 /// refused pack was stored anyway: 0 bytes became 65".
1128 #[test]
1129 fn a_pack_that_fails_the_closure_check_is_refused_before_a_byte_is_stored() {
1130 let dir = tmpdir("closure");
1131 let store = GitStore::open(&dir, "alice").unwrap();
1132
1133 // A thin pack: one ref-delta against an oid this repository has never
1134 // seen.
1135 let mut thin = b"PACK".to_vec();
1136 thin.extend_from_slice(&2u32.to_be_bytes());
1137 thin.extend_from_slice(&1u32.to_be_bytes());
1138 thin.push(0x74); // type 7, size 4
1139 thin.extend_from_slice(&[0x11; 20]);
1140 {
1141 use std::io::Write;
1142 let mut e = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
1143 e.write_all(&[0x04, 0x04, 0x90, 0x04]).unwrap();
1144 thin.extend_from_slice(&e.finish().unwrap());
1145 }
1146 thin.extend_from_slice(&[0u8; 20]);
1147
1148 let before = std::fs::metadata(store.blobs_path()).unwrap().len();
1149 let err = store
1150 .put_pack(&thin)
1151 .expect_err("a dangling base is refused");
1152 let msg = err.to_string();
1153 assert!(
1154 msg.contains(&hex::encode([0x11; 20])),
1155 "names the base: {msg}"
1156 );
1157 let after = std::fs::metadata(store.blobs_path()).unwrap().len();
1158 assert_eq!(
1159 before, after,
1160 "the refused pack was stored anyway: {before} bytes became {after}"
1161 );
1162 assert_eq!(store.unindexed_packs(), 0, "and no index job was queued");
1163
1164 // A corrupt pack — a delta base that lands nowhere — is refused too, and
1165 // as corruption rather than as thinness.
1166 let (mut corrupt, _) = one_blob_pack(b"x");
1167 corrupt[11] = 2; // claim two objects, provide one
1168 assert!(store.put_pack(&corrupt).is_err());
1169 assert_eq!(
1170 std::fs::metadata(store.blobs_path()).unwrap().len(),
1171 before,
1172 "a corrupt pack was stored"
1173 );
1174 }
1175
1176 /// **CAS: a mismatch moves nothing.** The ref namespace is read back from the
1177 /// log after every attempt, so what is asserted is the ref's applied state
1178 /// and not the call's return value.
1179 ///
1180 /// Seen RED by comparing `actual.is_some() != old.is_some()` — existence
1181 /// only, the CAS bug that looks right: "stale old value: TxId { pack_id: None,
1182 /// extent: None, push_seq: Some(1) }", i.e. the swap from a wrong old value
1183 /// was accepted and written.
1184 #[test]
1185 fn a_compare_and_swap_that_loses_the_race_moves_no_ref() {
1186 let dir = tmpdir("cas");
1187 let store = GitStore::open(&dir, "alice").unwrap();
1188 let (pack_a, oid_a) = one_blob_pack(b"first");
1189 let (pack_b, oid_b) = one_blob_pack(b"second");
1190 store.put_pack(&pack_a).unwrap();
1191 store.put_pack(&pack_b).unwrap();
1192 store.absorb_pending().unwrap();
1193
1194 let name = "refs/heads/main";
1195 let target = |s: &GitStore| -> Option<Vec<u8>> {
1196 s.refs()
1197 .unwrap()
1198 .into_iter()
1199 .find(|r| r.name == name)
1200 .and_then(|r| r.oid)
1201 };
1202
1203 // ★ A LOST CAS IS TYPED, not a string a caller has to parse.
1204 //
1205 // `update_ref` raised a bare `bail!` while its own sibling
1206 // `put_refs_cas` raised `RefRejection::Cas`, so `gunnar-wire`'s
1207 // receive-pack could not tell a lost race from a broken disk and said
1208 // so in a comment at the call site. The contract had the type all
1209 // along; this method did not use it.
1210 //
1211 // Seen RED by restoring the `bail!`: "a lost CAS must carry
1212 // RefRejection::Cas, not a string: compare-and-swap on refs/heads/main
1213 // failed: it is absent, the caller expected 0101…" — the message was
1214 // right and the TYPE was missing, which is exactly the failure a
1215 // string-shaped error hides.
1216 let err = store
1217 .update_ref(name, Some(&oid_a), Some(&oid_b))
1218 .expect_err("a create that claims a previous value must fail");
1219 match err.downcast_ref::<git_storage_trait::RefRejection>() {
1220 Some(git_storage_trait::RefRejection::Cas {
1221 name: n,
1222 expected,
1223 actual,
1224 }) => {
1225 assert_eq!(n, name, "the rejection names the wrong ref");
1226 assert_eq!(
1227 *actual,
1228 git_storage_trait::Observed::Nothing,
1229 "the ref did not exist, so `actual` must say so"
1230 );
1231 assert_ne!(
1232 *expected,
1233 git_storage_trait::Observed::Nothing,
1234 "the caller DID claim a previous value; `expected` must carry it"
1235 );
1236 }
1237 other => panic!(
1238 "a lost CAS must carry RefRejection::Cas, not a string: {err:#} (downcast: {})",
1239 if other.is_some() { "wrong variant" } else { "none" }
1240 ),
1241 }
1242 assert_eq!(target(&store), None, "the failed create wrote something");
1243
1244 let tx = store.update_ref(name, None, Some(&oid_a)).unwrap();
1245 assert!(tx.push_seq.is_some());
1246 assert_eq!(target(&store).as_deref(), Some(&oid_a[..]));
1247
1248 // A swap from the WRONG old value: refused, and nothing moves.
1249 let err = store
1250 .update_ref(name, Some(&oid_b), Some(&oid_b))
1251 .expect_err("stale old value");
1252 assert!(err.to_string().contains("compare-and-swap"), "{err}");
1253 assert_eq!(
1254 target(&store).as_deref(),
1255 Some(&oid_a[..]),
1256 "refs/heads/main moved on a failed CAS"
1257 );
1258
1259 // The right old value: it moves.
1260 store.update_ref(name, Some(&oid_a), Some(&oid_b)).unwrap();
1261 assert_eq!(target(&store).as_deref(), Some(&oid_b[..]));
1262
1263 // A ref may not point at an object we do not have.
1264 let err = store
1265 .update_ref(name, Some(&oid_b), Some(&[0xcd; 20]))
1266 .expect_err("dangling target");
1267 assert!(err.to_string().contains("does not have"), "{err}");
1268 assert_eq!(target(&store).as_deref(), Some(&oid_b[..]));
1269
1270 // Delete: gone from the namespace, and the log is the history.
1271 store.update_ref(name, Some(&oid_b), None).unwrap();
1272 assert_eq!(target(&store), None);
1273 assert!(store.refs().unwrap().iter().all(|r| r.name != name));
1274 }
1275
1276 /// **`put_refs_cas` persists a `RefCas`'s peeled target into the ref row.**
1277 ///
1278 /// This is the write half of gunnar S-080: the advertisement re-peels every
1279 /// annotated tag on every clone/fetch/`ls-remote` for want of a stored
1280 /// peel, and the store is where the peel has to be kept. A tag ref applied
1281 /// through the atomic CAS path must land its peeled column, exactly as one
1282 /// applied through [`GitOps::put_refs`] does — otherwise a tag pushed with
1283 /// `git push --atomic` would advertise unpeeled while the same tag pushed
1284 /// without `--atomic` advertised peeled.
1285 ///
1286 /// Seen RED by dropping `e.peeled` in `put_refs_cas`'s `RefUpdate::set`
1287 /// arm — the ref lands, `oid` is right, and `peeled` comes back `None`.
1288 #[test]
1289 fn put_refs_cas_persists_a_tags_peeled_target() {
1290 let dir = tmpdir("cas-peel");
1291 let store = GitStore::open(&dir, "alice").unwrap();
1292 // Two real objects: one stands in for the tag object the ref names, the
1293 // other for the commit it peels to. The store only requires the ref's
1294 // own target to be present; the peel rides through as data.
1295 let (pack_tag, oid_tag) = one_blob_pack(b"tag-object");
1296 let (pack_commit, oid_commit) = one_blob_pack(b"peeled-commit");
1297 store.put_pack(&pack_tag).unwrap();
1298 store.put_pack(&pack_commit).unwrap();
1299 store.absorb_pending().unwrap();
1300
1301 store
1302 .put_refs_cas(&[RefCas {
1303 name: "refs/tags/v9".into(),
1304 old: None,
1305 new: Some(&oid_tag),
1306 peeled: Some(oid_commit.clone()),
1307 }])
1308 .unwrap();
1309
1310 let row = store
1311 .refs()
1312 .unwrap()
1313 .into_iter()
1314 .find(|r| r.name == "refs/tags/v9")
1315 .expect("the tag ref was applied");
1316 assert_eq!(row.oid.as_deref(), Some(&oid_tag[..]), "the ref target");
1317 assert_eq!(
1318 row.peeled.as_deref(),
1319 Some(&oid_commit[..]),
1320 "the CAS path dropped the peeled target the RefCas carried"
1321 );
1322
1323 // A plain (non-tag) create through the same path carries no peel and so
1324 // stores none — the branch-head case that must stay `None`.
1325 store
1326 .put_refs_cas(&[RefCas {
1327 name: "refs/heads/plain".into(),
1328 old: None,
1329 new: Some(&oid_commit),
1330 peeled: None,
1331 }])
1332 .unwrap();
1333 let branch = store
1334 .refs()
1335 .unwrap()
1336 .into_iter()
1337 .find(|r| r.name == "refs/heads/plain")
1338 .expect("the branch was applied");
1339 assert_eq!(
1340 branch.peeled, None,
1341 "a non-tag create must not invent a peeled target"
1342 );
1343 }
1344
1345 /// **The bounded walk answers EXACTLY what a full bitmap table answers** —
1346 /// under-send and over-send both named, over every commit in the fixture.
1347 ///
1348 /// # What is actually at stake here
1349 ///
1350 /// Until 2026-08-14 the live table bitmapped **every** commit
1351 /// (`ReachPolicy { max_commits: usize::MAX }`), not as a choice but because
1352 /// `reachable_oids` had no fallback: a `want` with no bitmap contributed
1353 /// itself and nothing else. MEASURED on oden — 530 218 of a 4-object fetch's
1354 /// 575 349 allocations were that build, thrown away again by the next push's
1355 /// `refold`. [`crate::reach::accumulate`] is the walk that makes a sampled
1356 /// table legal, and `LIVE_REACH_COMMITS` is now 512.
1357 ///
1358 /// **A walk that stops too early UNDER-SENDS, and a clone that under-sends
1359 /// is silent data loss**: the pack indexes, `fsck`s and applies, the client
1360 /// exits zero, and the repository is missing objects it will not discover
1361 /// for days. `select` refusing was a correct-but-expensive answer to exactly
1362 /// this hazard, and this test is what replaces it.
1363 ///
1364 /// # Three arms over one store, and why three
1365 ///
1366 /// | cap | table | what runs |
1367 /// |---|---|---|
1368 /// | `usize::MAX` | every commit | no walk at all — **the reference** |
1369 /// | `0` | empty | the walk does *everything* |
1370 /// | `2` | two commits | the mixed case, where phase 1's stopping rule actually fires |
1371 ///
1372 /// Two arms would not be enough. `0` never exercises the stop-at-a-bitmap
1373 /// rule, which is the one thing that could OR in a *closed* set at the wrong
1374 /// moment, and `usize::MAX` never exercises the walk. The middle arm is the
1375 /// only one where both halves meet.
1376 ///
1377 /// # Both directions are named, and that is the anti-hollow half
1378 ///
1379 /// The comparison is a **symmetric difference**, not a length check and not
1380 /// a subset check. An over-send passes every downstream test we own — the
1381 /// pack indexes, `--check-self-contained-and-connected` accepts it, `fsck`
1382 /// is clean — and today's gix defect was exactly that shape. So `missing`
1383 /// and `extra` are computed and printed separately: a walk that stops short
1384 /// names the objects it lost, and a walk that ORs too much names the objects
1385 /// it invented.
1386 ///
1387 /// # Seen RED, 2026-08-14, three ways — and the directions are instructive
1388 ///
1389 /// * **stop one commit short** (`continue` before pushing parents in phase
1390 /// 1) — *"cap 0 vs the full table disagree for want 75c9e5c3…: MISSING 0
1391 /// object(s) …; EXTRA 2 object(s) the full table did not select, first
1392 /// Some(\"75c9e5c3d1d08dd92cc913c70ad10288a19eea4e\")"*.
1393 /// * **lose objects in the tree walk** (skip every seventh ordinal in
1394 /// `accumulate_tree`) — *"cap 0 …: MISSING 2 object(s) the full table
1395 /// selected, first Some(\"28b3c3658111299869d1f63f49a8cfb2635c0cfe\");
1396 /// EXTRA 0"*.
1397 /// * **OR in an unrelated commit's bitmap** — *"cap 2 …: MISSING 13
1398 /// object(s) the full table selected, first
1399 /// Some(\"078f18e68253d6fc56e8cba230ef9f32dec2e2ab\"); EXTRA 0"*.
1400 ///
1401 /// 🔴 **Note which direction each one came out.** Truncating the walk
1402 /// produced an *over*-send and over-collecting produced an *under*-send —
1403 /// the opposite of the intuition in both cases — because the identical walk
1404 /// serves the `have` side, and an error there enters the answer through
1405 /// `union − had` with its sign flipped. That is precisely why the comparison
1406 /// is a symmetric difference and reports **both** counts: a guard that
1407 /// checked only for missing objects would have passed the first break, and a
1408 /// guard that checked only lengths would have named nothing.
1409 ///
1410 /// The premise is guarded too: with the arms wired to the same policy this
1411 /// test passes trivially, so the table size at each cap is asserted before
1412 /// any answer is compared.
1413 #[test]
1414 fn the_bounded_walk_answers_exactly_what_a_full_bitmap_table_answers() {
1415 use crate::reach::ReachPolicy;
1416
1417 let dir = tmpdir("reach-walk");
1418 let store = GitStore::open(&dir, "alice").unwrap();
1419 let (pack, _) = corpus_pack();
1420 store.put(&pack, &[]).unwrap();
1421 store.absorb_pending().unwrap();
1422
1423 let commits: Vec<String> = store.graph_snapshot().iter().map(|c| c.oid.clone()).collect();
1424 assert!(
1425 commits.len() > 2,
1426 "premise: the fixture needs more commits than the smallest cap under test, or every \
1427 arm bitmaps everything and the walk never runs — found {}",
1428 commits.len()
1429 );
1430
1431 let full = ReachPolicy {
1432 max_commits: usize::MAX,
1433 };
1434 let none = ReachPolicy { max_commits: 0 };
1435 let some = ReachPolicy { max_commits: 2 };
1436
1437 // The premise, measured off the tables themselves rather than assumed.
1438 // A cap that did not actually shrink the table would make every
1439 // comparison below a comparison of one code path with itself.
1440 let n_full = store.reach_bitmaps_with(full, false).unwrap().len();
1441 let n_none = store.reach_bitmaps_with(none, false).unwrap().len();
1442 let n_some = store.reach_bitmaps_with(some, false).unwrap().len();
1443 assert_eq!(
1444 n_full,
1445 commits.len(),
1446 "premise: the uncapped table must bitmap every commit"
1447 );
1448 assert_eq!(
1449 n_none, 0,
1450 "premise: the zero cap must produce NO bitmaps, or the walk arm is not a walk arm"
1451 );
1452 assert!(
1453 n_some > 0 && n_some < commits.len(),
1454 "premise: the middle cap must bitmap SOME commits and not all ({n_some} of {}), or \
1455 the stop-at-a-bitmap rule is never reached",
1456 commits.len()
1457 );
1458
1459 // A stride through the graph as `want`, each against its successor as
1460 // `have` — so the walk is asked about tips, roots and the middle.
1461 //
1462 // A stride and not every commit: `reachable_oids_with` rebuilds the
1463 // table on every call here (`cache: false`, and it must be — see that
1464 // method), so the exhaustive form is quadratic and ran for 334 s against
1465 // this fixture. Twelve probes spread across the history cost seconds and
1466 // cover the same three positions; the arms that matter are the three
1467 // caps, not the commit count.
1468 let stride = commits.len().div_ceil(12).max(1);
1469 let probes: Vec<usize> = (0..commits.len())
1470 .step_by(stride)
1471 .chain([commits.len() - 1])
1472 .collect();
1473 let mut walked_any = false;
1474 for i in probes {
1475 let w = &commits[i];
1476 let w_raw = hex::decode(w).unwrap();
1477 let h_raw = (i + 1 < commits.len()).then(|| hex::decode(&commits[i + 1]).unwrap());
1478 let haves: Vec<&[u8]> = h_raw.iter().map(|v| v.as_slice()).collect();
1479
1480 let reference: std::collections::BTreeSet<String> = store
1481 .reachable_oids_with(&[&w_raw], &haves, full, false)
1482 .unwrap()
1483 .into_iter()
1484 .collect();
1485
1486 for (label, policy) in [("cap 0", none), ("cap 2", some)] {
1487 let got: std::collections::BTreeSet<String> = store
1488 .reachable_oids_with(&[&w_raw], &haves, policy, false)
1489 .unwrap()
1490 .into_iter()
1491 .collect();
1492 let missing: Vec<&String> = reference.difference(&got).collect();
1493 let extra: Vec<&String> = got.difference(&reference).collect();
1494 assert!(
1495 missing.is_empty() && extra.is_empty(),
1496 "{label} vs the full table disagree for want {w}: MISSING {} object(s) the \
1497 full table selected, first {:?}; EXTRA {} object(s) the full table did not \
1498 select, first {:?}",
1499 missing.len(),
1500 missing.first(),
1501 extra.len(),
1502 extra.first()
1503 );
1504 }
1505 walked_any = true;
1506 }
1507 assert!(walked_any, "no commit was compared");
1508
1509 // And the answers are not vacuously equal because they are all empty.
1510 let tip = hex::decode(&commits[0]).unwrap();
1511 let n = store
1512 .reachable_oids_with(&[&tip], &[], none, false)
1513 .unwrap()
1514 .len();
1515 assert!(
1516 n > 1,
1517 "premise: a commit must reach more than itself for the comparison above to have any \
1518 content — got {n}"
1519 );
1520 }
1521
1522 /// **`want` minus `have` is an `andnot`, over objects and not just commits.**
1523 ///
1524 /// Built on a real repository's pack: the second commit's closure minus the
1525 /// first's must be exactly the objects the second commit introduced, and
1526 /// every returned oid must be one the store really holds.
1527 ///
1528 /// Seen RED by returning `union` instead of `union - had`: "removing the
1529 /// parent's closure removed nothing: 8 vs 8".
1530 ///
1531 /// MEASURED on the fixture: the child commit's closure is 8 objects, the
1532 /// parent's is 5, and the difference is 3 — the objects that commit
1533 /// introduced.
1534 #[test]
1535 fn reachable_is_want_minus_have_over_objects() {
1536 let dir = tmpdir("reach");
1537 let store = GitStore::open(&dir, "alice").unwrap();
1538 let (pack, _) = corpus_pack();
1539 store.put(&pack, &[]).unwrap();
1540 store.absorb_pending().unwrap();
1541 assert!(store.commit_count() > 1, "the fixture has no history");
1542
1543 // Two commits in parent→child order: the graph is folded that way.
1544 let (child, parent) = {
1545 let mut pair = None;
1546 for c in store.graph_snapshot() {
1547 if let Some(p) = c.parents.first() {
1548 pair = Some((c.oid.clone(), p.clone()));
1549 break;
1550 }
1551 }
1552 pair.expect("a commit with a parent")
1553 };
1554 let child_raw = hex::decode(&child).unwrap();
1555 let parent_raw = hex::decode(&parent).unwrap();
1556
1557 let all = store.reachable(&[&child_raw], &[]).unwrap();
1558 let delta = store.reachable(&[&child_raw], &[&parent_raw]).unwrap();
1559 let had = store.reachable(&[&parent_raw], &[]).unwrap();
1560
1561 assert!(
1562 !all.is_empty(),
1563 "a commit reaches at least itself and its tree"
1564 );
1565 assert!(
1566 delta.len() < all.len(),
1567 "removing the parent's closure removed nothing: {} vs {}",
1568 delta.len(),
1569 all.len()
1570 );
1571 assert_eq!(
1572 all.len(),
1573 delta.len() + had.iter().filter(|h| all.contains(h)).count(),
1574 "the three sets do not add up — want minus have is not an andnot"
1575 );
1576 for oid in delta.iter().chain(all.iter()) {
1577 assert!(
1578 store.has(oid).unwrap(),
1579 "reachable named {} which the store does not hold",
1580 hex::encode(oid)
1581 );
1582 }
1583 // The commit itself is in its own closure; its parent is not in the delta.
1584 assert!(all.contains(&child_raw));
1585 assert!(!delta.contains(&parent_raw));
1586 eprintln!(
1587 "closure {} objects, minus the parent's {} leaves {}",
1588 all.len(),
1589 had.len(),
1590 delta.len()
1591 );
1592 }
1593
1594 /// **The batch of one really is slower**, measured here rather than asserted
1595 /// from the plan — and the serial path is what `extents` takes for it.
1596 ///
1597 /// The ratio is machine- and load-dependent, so the guard on it is loose (it
1598 /// requires the batch path to be no *faster*, which is the direction the
1599 /// dispatch depends on) while the number is printed for the record. The
1600 /// dispatch itself is asserted exactly.
1601 ///
1602 /// MEASURED, release, oden, loadavg 4.94, 2687 objects: `lookup` 628 ns,
1603 /// `lookup_batch` of one 639 ns (**1.02x worse**, not the 1.7x the bare Arrow
1604 /// arms show), `lookup_batch` of 100 **161 ns/oid** — 3.9x better than the
1605 /// serial path. See [`lookup_path`] for why both numbers are recorded.
1606 ///
1607 /// Seen RED by making `lookup_path` return `Batch` for every n:
1608 /// "assertion `left == right` failed — left: Batch, right: Serial".
1609 ///
1610 /// The honest limit: a *timing* assertion cannot catch `extents` ignoring the
1611 /// dispatch, because the two paths differ by 2% here. That is exactly why the
1612 /// dispatch is a named function with an exact assertion on it rather than a
1613 /// comment above an `if`.
1614 #[test]
1615 fn the_batch_of_one_really_is_slower_than_the_serial_path() {
1616 assert_eq!(lookup_path(1), LookupPath::Serial);
1617 assert_eq!(lookup_path(2), LookupPath::Batch);
1618 assert_eq!(lookup_path(0), LookupPath::Batch);
1619 assert_eq!(BATCH_SATURATES_AT, 100);
1620
1621 let dir = tmpdir("batch");
1622 let store = GitStore::open(&dir, "alice").unwrap();
1623 let (pack, rows) = corpus_pack();
1624 store.put(&pack, &[]).unwrap();
1625 store.absorb_pending().unwrap();
1626 let oids: Vec<&[u8]> = rows.iter().map(|r| r.oid.as_slice()).collect();
1627
1628 // Both paths answer identically for one oid — the dispatch is an
1629 // optimisation, never a difference in the answer.
1630 for oid in oids.iter().take(64) {
1631 assert_eq!(
1632 store.extents(&[*oid]).unwrap(),
1633 store.index().extents_batch(&[*oid]),
1634 "the serial and batch paths disagree for {}",
1635 hex::encode(oid)
1636 );
1637 }
1638
1639 // 🔴 **Interleaved rounds, and the FASTEST of each is the reading.**
1640 //
1641 // This took one timed pass of each, one after the other, and compared
1642 // them. On a box running anything else that is a coin toss with extra
1643 // steps: the two passes are 20 000 lookups apart in time, so a
1644 // scheduler steal or a migration that lands on the first one alone
1645 // shifts the ratio by more than the 10 % the assertion allows. Seen RED
1646 // that way 2026-08-25 inside a full `cargo test` run, and green three
1647 // times in a row on the same binary in isolation — the failure was the
1648 // other 200 tests, not the dispatch.
1649 //
1650 // A stolen timeslice can only make a measurement SLOWER, never faster,
1651 // so the minimum over rounds is the least-perturbed sample of each and
1652 // the noise floor cancels out of the ratio. Interleaved rather than
1653 // batched per side, so a slow patch of wall clock hits both.
1654 let n = 20_000usize;
1655 const ROUNDS: usize = 5;
1656 let mut serial = f64::MAX;
1657 let mut batched = f64::MAX;
1658 for _ in 0..ROUNDS {
1659 let t = Instant::now();
1660 for i in 0..n {
1661 let _ = store.index().lookup(oids[i % oids.len()]);
1662 }
1663 serial = serial.min(t.elapsed().as_nanos() as f64 / n as f64);
1664
1665 let t = Instant::now();
1666 for i in 0..n {
1667 let _ = store.index().lookup_batch(&[oids[i % oids.len()]]);
1668 }
1669 batched = batched.min(t.elapsed().as_nanos() as f64 / n as f64);
1670 }
1671 // What the batch path is actually for: 100 oids in one call, which is
1672 // where it saturates.
1673 let hundred: Vec<&[u8]> = oids.iter().take(BATCH_SATURATES_AT).copied().collect();
1674 let per_oid_at_100 = {
1675 let rounds = n / BATCH_SATURATES_AT;
1676 let t = Instant::now();
1677 for _ in 0..rounds {
1678 let _ = store.index().lookup_batch(&hundred);
1679 }
1680 t.elapsed().as_nanos() as f64 / (rounds * hundred.len()) as f64
1681 };
1682 eprintln!(
1683 "load {}; {} objects: lookup {serial:.0} ns, lookup_batch[1] {batched:.0} ns \
1684 ({:.2}x, best of {ROUNDS} interleaved rounds), lookup_batch[{}] \
1685 {per_oid_at_100:.0} ns/oid",
1686 std::fs::read_to_string("/proc/loadavg")
1687 .unwrap_or_default()
1688 .trim(),
1689 store.index().len(),
1690 batched / serial,
1691 hundred.len(),
1692 );
1693 // 2026-08-10: the fused batch walk (no sort phase) collapsed the old
1694 // 1.02x batch-of-one penalty to a measured TIE — 604-620 ns both sides,
1695 // winner decided by noise, seen flipping run-to-run on an idle box. The
1696 // `Serial` dispatch for n=1 stays correct (equal time, two fewer Vec
1697 // allocations), so the guard keeps only the direction that would make
1698 // it WRONG: a batch of one materially faster than serial. 10% is
1699 // outside the tie's observed 0.3% jitter and inside any real win.
1700 //
1701 // RELEASE ONLY (2026-10-16). Every number above that this claim rests on
1702 // is a release build (~600 ns a lookup). Under `cargo test`'s debug
1703 // profile the same lookup is ~20 000 ns of unoptimised call layers, and
1704 // which side those layers weigh on is a property of the debug codegen,
1705 // not of the dispatch: measured on dwarf at load 27, debug, 0.90x — red
1706 // by 0.5 %, while the release build of the same code is the tie above.
1707 // A ratio of unoptimised code says nothing about the dispatch, so in a
1708 // debug build the reading is printed (above) and not judged; the exact
1709 // `lookup_path` asserts at the top of this test still run in both.
1710 if cfg!(debug_assertions) {
1711 eprintln!(
1712 "NOT JUDGED in a debug build: the batch-of-one ratio is a release claim \
1713 (run with --release to judge it)"
1714 );
1715 return;
1716 }
1717 assert!(
1718 batched >= serial * 0.9,
1719 "a batch of one measured MATERIALLY faster ({batched:.0} ns vs {serial:.0} ns) — \
1720 if that is repeatable the dispatch in `lookup_path` is wrong and must be \
1721 changed, not kept"
1722 );
1723 }
1724
1725 /// **`seal` folds the live logs into reserved Arrow sections**, and the
1726 /// generation numbers in the graph section are recomputed by that fold.
1727 ///
1728 /// Asserted by decoding the sections back out of their Arrow IPC bytes.
1729 ///
1730 /// Seen RED by dropping the `assign_generations` call from `refold`:
1731 /// "e58a5032ac3b36f15f08c759121efc8f08680e2b has generation 0".
1732 #[test]
1733 fn seal_emits_the_reserved_sections_and_recomputes_every_generation() {
1734 use znippy_common::{GUNNAR_GRAPH_MODULE, GUNNAR_REACH_MODULE, GUNNAR_REFS_MODULE};
1735
1736 let dir = tmpdir("seal");
1737 let store = GitStore::open(&dir, "alice").unwrap();
1738 let (pack, _) = corpus_pack();
1739 store.put(&pack, &[]).unwrap();
1740 store.absorb_pending().unwrap();
1741 let tip = store
1742 .graph_snapshot()
1743 .iter()
1744 .max_by_key(|c| c.generation)
1745 .expect("a graph")
1746 .clone();
1747 store
1748 .update_ref(
1749 "refs/heads/main",
1750 None,
1751 Some(&hex::decode(&tip.oid).unwrap()),
1752 )
1753 .unwrap();
1754
1755 let sections = store.seal().unwrap();
1756 let names: Vec<&str> = sections.iter().map(|s| s.module_name.as_str()).collect();
1757 for want in [GUNNAR_REFS_MODULE, GUNNAR_GRAPH_MODULE, GUNNAR_REACH_MODULE] {
1758 assert!(names.contains(&want), "{want} is not sealed: {names:?}");
1759 }
1760
1761 // Generations: a root is 1, a child is 1 + max(parents), and the graph
1762 // this store folded must say so.
1763 let graph = store.graph_snapshot();
1764 let by_oid: std::collections::HashMap<&str, &crate::graph::CommitNode> =
1765 graph.iter().map(|c| (c.oid.as_str(), c)).collect();
1766 let mut with_parents = 0usize;
1767 for c in &graph {
1768 assert!(c.generation >= 1, "{} has generation 0", c.oid);
1769 let known: Vec<&crate::graph::CommitNode> = c
1770 .parents
1771 .iter()
1772 .filter_map(|p| by_oid.get(p.as_str()).copied())
1773 .collect();
1774 if known.is_empty() {
1775 assert_eq!(c.generation, 1, "{} is a root here", c.oid);
1776 } else {
1777 with_parents += 1;
1778 let expect = 1 + known.iter().map(|p| p.generation).max().unwrap();
1779 assert_eq!(
1780 c.generation, expect,
1781 "{} must have generation {expect}",
1782 c.oid
1783 );
1784 assert!(
1785 c.generation > 1,
1786 "a commit with a parent must have generation > 1"
1787 );
1788 }
1789 }
1790 assert!(with_parents > 0, "no commit in the fixture has a parent");
1791 eprintln!(
1792 "{} commits sealed, {with_parents} with parents",
1793 graph.len()
1794 );
1795 }
1796
1797 /// **`seal` writes generation 0, and `gc` then has something to compact.**
1798 ///
1799 /// This is the hole the four working steps of `gc()` used to fall into:
1800 /// absorb, live set, retire, drop all ran, and then
1801 /// `NewGeneration::run(archive_path())` died on
1802 /// `stat …/repository.znippy: No such file or directory` because **nothing
1803 /// ever created generation 0**. Every other test in this file that reached
1804 /// step 5 hand-built the archive with `znippy_common::create_archive`, which
1805 /// is why the gap survived: the fixture was doing the store's job.
1806 ///
1807 /// Asserted on **applied output**, through znippy's own reader and nothing
1808 /// else:
1809 ///
1810 /// * the live pack comes back out of the archive **byte for byte**, via
1811 /// `extract_file_verified`, which reconstructs the entry and blake3-checks
1812 /// it against the index — so a wrong checksum, a wrong extent or a wrong
1813 /// `compressed` flag all fail here rather than being written and believed;
1814 /// * the tombstoned pack has **no** entry, because a row for it is exactly
1815 /// what would stop the compaction reclaiming its bytes;
1816 /// * the three reserved sections are addressable **out of the file**, not
1817 /// merely present in the returned vector;
1818 /// * `gc()` then runs end to end and `NewGeneration` produces
1819 /// `repository.g1.znippy`, verified, with the old generation gone.
1820 ///
1821 /// # Seen RED, and one of the reds was in this guard
1822 ///
1823 /// Every mutation below was applied to the real code, run, and reverted.
1824 ///
1825 /// 1. **The hole itself.** With `seal` returning the sections and writing
1826 /// nothing — the code as it stood — this fails at
1827 /// `seal did not create …/repository.znippy`, and `gc()` fails with the
1828 /// original `compacting …/repository.znippy after dropping 2683 dead
1829 /// rows: stat …/repository.znippy: No such file or directory`.
1830 /// 2. **A dropped row** — `.take(packs.len() - 1)` on the row loop:
1831 /// `objects.pack.2 is not in the sealed archive: ["objects.pack.0"]`.
1832 /// **This is the mutation that first found a hollow guard, and the guard
1833 /// was mine.** Against the two-pack fixture this test started with, the
1834 /// row dropped was the tombstoned one, which is skipped anyway — the
1835 /// mutation stayed GREEN. The fixture now pushes three packs and
1836 /// tombstones the middle one, so there are two live rows and dropping
1837 /// either is visible.
1838 /// 3. **No tombstone honoured** — the `retired.contains` skip disabled:
1839 /// `the tombstoned pack got an index row anyway … ["objects.pack.0",
1840 /// "objects.pack.2", "objects.pack.1"]`. A row for a retired pack is what
1841 /// would keep its payload alive through every future compaction.
1842 /// 4. **A corrupted extent** — `blob_offset: offset + 1`. It writes, lists
1843 /// and opens perfectly; it dies at the blake3 gate inside
1844 /// `extract_file_verified`: `checksum mismatch for objects.pack.0 at
1845 /// fdata_offset 0`.
1846 /// 5. **The checksum domain** — hashing the *path* instead of the bytes,
1847 /// which is precisely the mistake that produces an archive that writes
1848 /// cleanly and verifies wrong. Same gate, same message. Together 4 and 5
1849 /// are what establish that the checksum is blake3 over the bytes at
1850 /// `blob_offset`, and that this test can tell.
1851 /// 6. **The `compressed` flag** — `true` on bytes that were stored raw:
1852 /// `OpenZL getDecompressedSize: ZL_getDecompressedSize failed`. The flag
1853 /// is load-bearing, not decorative.
1854 /// 7. **A lying report** — `packs_retired: 0` in the `SealReport` literal:
1855 /// `the report miscounts the tombstoned packs`. The report is read back
1856 /// by a second seal, which also pins that a seal is a snapshot: taking it
1857 /// twice yields the same entries, not a second generation.
1858 /// 8. **No reserved builder attached** — the sections are still *returned*,
1859 /// so a guard that only inspected the return value would pass:
1860 /// `__gunnar_refs__ is not in the sealed archive's manifest`. That is why
1861 /// the sections are read back out of the file by module name.
1862 ///
1863 #[test]
1864 fn seal_writes_generation_zero_that_gc_can_compact() {
1865 use znippy_common::{
1866 read_reserved_section_bytes, ZnippyArchive, ZnippyReader, GUNNAR_GRAPH_MODULE,
1867 GUNNAR_REACH_MODULE, GUNNAR_REFS_MODULE,
1868 };
1869
1870 let dir = tmpdir("seal-g0");
1871 let store = GitStore::open(&dir, "alice").unwrap();
1872 // THREE packs, and the middle one is tombstoned before the seal. Two
1873 // live rows rather than one is not decoration: with a single live entry,
1874 // "dropped a row" and "wrote no rows at all" are the same failure, and a
1875 // `.take(packs.len() - 1)` mutation of the row loop stayed GREEN against
1876 // an earlier two-pack version of this fixture. Three packs with a gap in
1877 // the middle also pin the ordinal: it is the position among the
1878 // journal's `Pack` rows, so the survivors are 0 and **2**, not 0 and 1.
1879 let (pack, _) = corpus_pack();
1880 let (doomed, doomed_oid) = one_blob_pack(b"this pack is retired before the seal");
1881 let (kept, kept_oid) = one_blob_pack(b"this pack is unreferenced but not tombstoned");
1882 store.put(&pack, &[]).unwrap();
1883 let doomed_tx = store.put_pack(&doomed).unwrap();
1884 store.put_pack(&kept).unwrap();
1885 store.absorb_pending().unwrap();
1886 for (what, oid) in [("doomed", &doomed_oid), ("kept", &kept_oid)] {
1887 assert!(
1888 store.has(oid).unwrap(),
1889 "the {what} blob was never stored, so this fixture proves nothing"
1890 );
1891 }
1892
1893 // A ref, so the live set is not empty and `gc` is not refused.
1894 let root = store
1895 .graph_snapshot()
1896 .into_iter()
1897 .find(|c| c.generation == 1)
1898 .expect("a root commit");
1899 let root_raw = hex::decode(&root.oid).unwrap();
1900 store
1901 .update_ref("refs/heads/root", None, Some(&root_raw))
1902 .unwrap();
1903
1904 // Exactly one tombstone, written the way `gc` writes them. Named
1905 // explicitly rather than via `retire_dead_packs`, which would tombstone
1906 // BOTH one-blob packs and leave nothing to prove the ordinal with.
1907 crate::archive_write::retire_packs(
1908 &crate::archive_write::SafeWriter::journal_path(store.blobs_path()),
1909 &[doomed_tx.extent.unwrap().0],
1910 )
1911 .unwrap();
1912
1913 assert!(
1914 !store.archive_path().exists(),
1915 "something created the archive before the seal ran"
1916 );
1917 let sections = store.seal().unwrap();
1918
1919 // ── the archive exists and reads back through znippy's own reader ────
1920
1921 assert!(
1922 store.archive_path().exists(),
1923 "seal did not create {}",
1924 store.archive_path().display()
1925 );
1926 let ar = ZnippyArchive::open(store.archive_path()).unwrap();
1927 let listed = ar.list_files().unwrap();
1928 assert!(
1929 listed.contains(&"objects.pack.0".to_string()),
1930 "objects.pack.0 is not in the sealed archive: {listed:?}"
1931 );
1932 assert!(
1933 listed.contains(&"objects.pack.2".to_string()),
1934 "objects.pack.2 is not in the sealed archive: {listed:?}"
1935 );
1936 assert!(
1937 !listed.contains(&"objects.pack.1".to_string()),
1938 "the tombstoned pack got an index row anyway, so the compaction can never \
1939 reclaim its bytes: {listed:?}"
1940 );
1941 assert_eq!(
1942 listed.len(),
1943 2,
1944 "the seal wrote entries for packs the journal never acked as live: {listed:?}"
1945 );
1946
1947 // The bytes, reconstructed and blake3-checked by the reader itself.
1948 assert_eq!(
1949 ar.extract_file_verified("objects.pack.0").unwrap(),
1950 pack,
1951 "objects.pack.0 does not read back as the pack that was pushed"
1952 );
1953 assert_eq!(
1954 ar.extract_file_verified("objects.pack.2").unwrap(),
1955 kept,
1956 "objects.pack.2 does not read back as the pack that was pushed"
1957 );
1958
1959 // ── the reserved sections, addressed out of the file ─────────────────
1960 let names: Vec<&str> = sections.iter().map(|s| s.module_name.as_str()).collect();
1961 for want in [GUNNAR_REFS_MODULE, GUNNAR_GRAPH_MODULE, GUNNAR_REACH_MODULE] {
1962 assert!(names.contains(&want), "{want} was not returned: {names:?}");
1963 let bytes = read_reserved_section_bytes(store.archive_path(), want)
1964 .unwrap()
1965 .unwrap_or_else(|| panic!("{want} is not in the sealed archive's manifest"));
1966 assert!(!bytes.is_empty(), "{want} is sealed as zero bytes");
1967 }
1968
1969 // ── the report, and that a second seal is not a different archive ───
1970 //
1971 // A count nobody asserts on is a count that can be wrong for ever, so
1972 // the report is read here — and reading it means sealing again, which
1973 // pins the other half: a seal is a snapshot and re-taking it produces
1974 // the same entries, not a second generation.
1975 let again = store
1976 .seal_archive(store.reserved_sections().unwrap())
1977 .unwrap();
1978 assert_eq!(
1979 again.packs_sealed, 2,
1980 "the report miscounts the sealed packs"
1981 );
1982 assert_eq!(
1983 again.packs_retired, 1,
1984 "the report miscounts the tombstoned packs"
1985 );
1986 assert_eq!(
1987 again.packs_after_copy, 0,
1988 "nothing was pushed during this seal, so nothing can have raced it"
1989 );
1990 let mut once = listed.clone();
1991 once.sort();
1992 let mut twice = ZnippyArchive::open(store.archive_path())
1993 .unwrap()
1994 .list_files()
1995 .unwrap();
1996 twice.sort();
1997 // Sorted on both sides: `list_files` does not promise an order, and it
1998 // was observed returning the same two entries the other way round.
1999 assert_eq!(
2000 twice, once,
2001 "sealing twice produced a different set of entries"
2002 );
2003
2004 // ── and now the fifth step of `gc` has something to compact ──────────
2005 let report = store.gc().unwrap();
2006 assert_eq!(report.strategy, "NewGeneration");
2007 assert!(report.verified, "the new generation was not read back");
2008 assert_eq!(
2009 report.archive,
2010 store.archive_path().with_file_name("repository.g1.znippy"),
2011 "NewGeneration did not produce generation 1"
2012 );
2013 assert!(report.archive.exists(), "the new generation is not on disk");
2014 assert!(
2015 !store.archive_path().exists(),
2016 "the old generation was not retired"
2017 );
2018 assert!(
2019 report.bytes_after < report.bytes_before,
2020 "the compaction reclaimed nothing: {} → {} bytes — the retired pack's payload \
2021 should have gone",
2022 report.bytes_before,
2023 report.bytes_after
2024 );
2025
2026 // The live pack survives the compaction; the retired one's row never
2027 // existed, so its bytes are what got reclaimed.
2028 let g1 = ZnippyArchive::open(&report.archive).unwrap();
2029 assert_eq!(
2030 g1.extract_file_verified("objects.pack.0").unwrap(),
2031 pack,
2032 "generation 1 does not carry the live pack"
2033 );
2034 eprintln!(
2035 "seal: {} bytes, gc: {} → {} bytes, {}",
2036 std::fs::metadata(&report.archive)
2037 .map(|m| m.len())
2038 .unwrap_or(0),
2039 report.bytes_before,
2040 report.bytes_after,
2041 report.archive.display()
2042 );
2043 }
2044
2045 /// **GC in the order §13.20 fixes**: reachability, then the dead index rows,
2046 /// then base znippy's compaction.
2047 ///
2048 /// The archive is a real znippy archive built the way `gc`'s own fixture
2049 /// builds one, so step 3 really runs `compact_archive` and really produces a
2050 /// new generation. Steps 1 and 2 are asserted as applied output: the
2051 /// unreachable oid stops resolving and the reachable one still does.
2052 ///
2053 /// Seen RED by having `gc` drop rows *before* computing the live set — the
2054 /// order §13.20 fixes, inverted: "gc dropped the live object too". Seen RED a
2055 /// second time by making `drop_dead_rows` return `Ok(0)` without removing
2056 /// anything: "the dead object still resolves after gc".
2057 ///
2058 /// MEASURED on the corpus: a ref on the root commit leaves **130** live
2059 /// objects of 1352 — that commit, its tree and the 128 blobs the tree names
2060 /// — and the other 1222 rows go. (It read "5 of 2687" while the corpus was
2061 /// whatever pack this box's `read_dir` named first. Both numbers are
2062 /// properties of the fixture's root commit, which is exactly why one of
2063 /// them used to change when an unrelated repository was cloned.)
2064 #[test]
2065 fn gc_computes_reachability_drops_the_dead_rows_then_compacts() {
2066 let dir = tmpdir("gc");
2067 let store = GitStore::open(&dir, "alice").unwrap();
2068 let (pack, _) = corpus_pack();
2069 store.put(&pack, &[]).unwrap();
2070 store.absorb_pending().unwrap();
2071
2072 // A ref on one commit, and an object that commit cannot reach.
2073 let graph = store.graph_snapshot();
2074 let root = graph
2075 .iter()
2076 .find(|c| c.generation == 1)
2077 .expect("a root commit")
2078 .clone();
2079 let root_raw = hex::decode(&root.oid).unwrap();
2080 let live_from_root: std::collections::HashSet<Vec<u8>> = store
2081 .reachable(&[&root_raw], &[])
2082 .unwrap()
2083 .into_iter()
2084 .collect();
2085 let all: Vec<Vec<u8>> = store.index().oids_in_order().unwrap();
2086 let dead = all
2087 .iter()
2088 .find(|o| !live_from_root.contains(*o))
2089 .expect("the root does not reach everything in a real repository")
2090 .clone();
2091
2092 store
2093 .update_ref("refs/heads/root", None, Some(&root_raw))
2094 .unwrap();
2095
2096 // A real znippy archive for step 3 to compact.
2097 let files = vec![
2098 ("pack-0.pack".to_string(), pack.clone()),
2099 ("pack-1.pack".to_string(), pack.clone()),
2100 ];
2101 znippy_common::create_archive(store.archive_path(), &files, 3).unwrap();
2102
2103 let before = store.index().len();
2104 let report = store.gc().unwrap();
2105
2106 assert!(
2107 store.has(&root_raw).unwrap(),
2108 "gc dropped the live object too"
2109 );
2110 assert!(
2111 !store.has(&dead).unwrap(),
2112 "the dead object still resolves after gc"
2113 );
2114 assert!(
2115 store.index().len() < before,
2116 "gc dropped nothing: {} rows before and after",
2117 before
2118 );
2119 assert_eq!(
2120 store.index().len(),
2121 live_from_root.len(),
2122 "exactly the live set survives"
2123 );
2124 assert_eq!(report.strategy, "NewGeneration");
2125 assert!(report.verified, "the new generation was not read back");
2126 assert!(report.archive.exists(), "the new generation is not on disk");
2127 assert!(
2128 !store.archive_path().exists(),
2129 "the old generation was not retired"
2130 );
2131 eprintln!(
2132 "gc: {before} rows → {} live, {} → {} bytes, {}",
2133 store.index().len(),
2134 report.bytes_before,
2135 report.bytes_after,
2136 report.archive.display()
2137 );
2138
2139 // And a repository with no refs at all is refused rather than emptied.
2140 let dir2 = tmpdir("gc-norefs");
2141 let empty = GitStore::open(&dir2, "alice").unwrap();
2142 empty.put(&pack, &[]).unwrap();
2143 empty.absorb_pending().unwrap();
2144 let n = empty.index().len();
2145 assert!(empty.gc().is_err(), "a GC that would delete everything ran");
2146 assert_eq!(empty.index().len(), n, "the refused GC dropped rows anyway");
2147 }
2148
2149 // ── the GC's journal half: a pack that is *wholly* dead ──────────────────
2150
2151 /// What [`one_live_pack_and_one_doomed_pack`] hands back: an open store, and
2152 /// the three oids the guards ask about.
2153 struct DoomedFixture {
2154 dir: PathBuf,
2155 store: GitStore,
2156 /// The only object of the pack that dies whole.
2157 doomed: Vec<u8>,
2158 /// The commit a ref points at.
2159 root: Vec<u8>,
2160 /// That commit's tree — a live object that is not the ref itself.
2161 tree: Vec<u8>,
2162 }
2163
2164 /// One repository with two packs: a real one that a ref reaches into, and a
2165 /// one-blob pack nothing will ever point at.
2166 ///
2167 /// The second pack is what makes the difference visible: after a GC it has
2168 /// **no rows at all**, which is the state a pack that was in flight when the
2169 /// machine died also leaves behind.
2170 ///
2171 /// The store is returned **open**, so a guard runs its first GC in the same
2172 /// process lifetime that pushed. The reopen is the thing under test and it
2173 /// belongs in the guard, not in the fixture.
2174 fn one_live_pack_and_one_doomed_pack(tag: &str) -> DoomedFixture {
2175 let dir = tmpdir(tag);
2176 let (pack, _) = corpus_pack();
2177 let (doomed, doomed_oid) = one_blob_pack(b"no ref will ever reach this blob");
2178 let store = GitStore::open(&dir, "alice").unwrap();
2179 store.put(&pack, &[]).unwrap();
2180 store.put_pack(&doomed).unwrap();
2181 store.absorb_pending().unwrap();
2182 assert!(
2183 store.has(&doomed_oid).unwrap(),
2184 "the doomed blob was never stored, so this fixture proves nothing"
2185 );
2186
2187 let root = store
2188 .graph_snapshot()
2189 .into_iter()
2190 .find(|c| c.generation == 1)
2191 .expect("a root commit");
2192 let root_raw = hex::decode(&root.oid).unwrap();
2193 store
2194 .update_ref("refs/heads/root", None, Some(&root_raw))
2195 .unwrap();
2196 let tree = hex::decode(root.tree.as_ref().expect("the root commit names a tree")).unwrap();
2197 assert!(store.has(&tree).unwrap(), "the root's tree is not indexed");
2198
2199 // A real znippy archive, so the compaction step has something to compact.
2200 let files = vec![
2201 ("pack-0.pack".to_string(), pack.clone()),
2202 ("pack-1.pack".to_string(), pack.clone()),
2203 ];
2204 znippy_common::create_archive(store.archive_path(), &files, 3).unwrap();
2205 DoomedFixture {
2206 dir,
2207 store,
2208 doomed: doomed_oid,
2209 root: root_raw,
2210 tree,
2211 }
2212 }
2213 /// **A row that arrives after the GC took its snapshot survives the drop.**
2214 /// `live_set` walks from the refs as they are; a push landing during the GC
2215 /// appends rows no ref names yet, and to `live` they are unknown. The old
2216 /// `retain(|o| live.contains(o))` dropped them — a GC deleting a push in
2217 /// flight. Now the GC drops `seen − live`, and a row in neither set is kept.
2218 ///
2219 /// Staged without a concurrent push: one synthetic unreferenced row is
2220 /// appended BEFORE the snapshot (seen, not live → dead), one AFTER it (late
2221 /// → must survive), and the drop is asked to do its worst.
2222 ///
2223 /// # 🔴 Seen red
2224 ///
2225 /// With the snapshot taken after the late append — which is exactly what
2226 /// "dead = not in live, over the current index" amounted to — the late row
2227 /// lands in `dead`: "the late row is in neither set and is not dead" fires,
2228 /// and had it not, `store.has(&late)` would read false one line further
2229 /// down. Restored.
2230 #[test]
2231 fn a_row_appended_after_the_gc_scanned_survives_the_drop() {
2232 let dir = tmpdir("gc-late-row");
2233 let (pack, _rows) = corpus_pack();
2234 let store = GitStore::open(&dir, "alice").unwrap();
2235 store.put_pack(&pack).unwrap();
2236 store.wait_indexed();
2237 // A ref, or `live_set` refuses to compute a reachability that would
2238 // empty the repository — the right refusal, and not the one under test.
2239 let root = store
2240 .graph_snapshot()
2241 .into_iter()
2242 .find(|c| c.generation == 1)
2243 .expect("a root commit");
2244 let root_raw = hex::decode(&root.oid).unwrap();
2245 store
2246 .update_ref("refs/heads/root", None, Some(&root_raw))
2247 .unwrap();
2248 let fake = |b: u8| crate::index_layout::IndexEntry {
2249 oid: vec![b; 20],
2250 offset: 0,
2251 len: 0,
2252 obj_type: crate::index_layout::ObjType::Blob,
2253 uncompressed_size: 0,
2254 delta_base: 0,
2255 };
2256 let early = fake(0xe1);
2257 let late = fake(0x1a);
2258 store.index().append(std::slice::from_ref(&early)).unwrap();
2259
2260 // The GC's two snapshots, then reachability — in that order.
2261 let seen = store.index().oids_in_order().unwrap();
2262 let live = store.live_set().unwrap();
2263 // A push lands while the GC thinks.
2264 store.index().append(std::slice::from_ref(&late)).unwrap();
2265
2266 let dead = crate::git_ops::dead_oids(seen, &live);
2267 assert!(dead.contains(&early.oid), "the early unreferenced row is dead");
2268 assert!(!dead.contains(&late.oid), "the late row is in neither set and is not dead");
2269 let dropped = store.drop_dead_rows(&dead).unwrap();
2270 assert!(dropped >= 1, "the drop must have dropped the early row at least");
2271 assert!(!store.has(&early.oid).unwrap(), "the dead row must be gone");
2272 assert!(
2273 store.has(&late.oid).unwrap(),
2274 "a row appended after the GC's snapshot was dropped"
2275 );
2276 }
2277
2278
2279 /// The journal's rows, split into packs and tombstones, read off the disk.
2280 fn journal_state(dir: &Path) -> (Vec<(u64, u64)>, Vec<u64>) {
2281 use crate::archive_write::{acked_packs, read_journal, retired_offsets, SafeWriter};
2282 let rows = read_journal(&SafeWriter::journal_path(&dir.join("objects.pack"))).unwrap();
2283 let mut retired: Vec<u64> = retired_offsets(&rows).into_iter().collect();
2284 retired.sort_unstable();
2285 (acked_packs(&rows), retired)
2286 }
2287
2288 /// **THE BUG: a pack whose every object a GC found dead came back on the
2289 /// next open.**
2290 ///
2291 /// The `indexed` bit is derived as *extent in the journal, rows not in the
2292 /// index*, and a GC that drops every row of a pack produces exactly that
2293 /// state — so the reopen re-queued the pack and re-absorbed the objects the
2294 /// GC had just decided were dead. A partly dead pack keeps rows and was never
2295 /// affected, which is why this needs a pack that dies **whole**.
2296 ///
2297 /// Asserted on applied output on both sides of a process-lifetime boundary:
2298 /// the tombstone rows on disk, `has()` on the dead oid, and the object count
2299 /// after the reopen's drain has been waited for — a resurrection puts the row
2300 /// back, so counting is not a proxy for it, it *is* it.
2301 ///
2302 /// Seen RED by making `Absorber::adopt_journal` ignore the tombstones
2303 /// (`else if retired.contains(&extent.0)` → `else if false`), which is
2304 /// exactly the reader as it stood before this fix: "a dead object came back
2305 /// the instant the store was reopened". Restored.
2306 ///
2307 /// Seen RED a second time by not retiring anything in `GitOps::gc` (`let
2308 /// retired: Vec<u64> = Vec::new();` in place of the `retire_dead_packs`
2309 /// call — the writer as it stood before this fix): "gc retired 0 pack(s), not
2310 /// the one whose objects all died — left: 0, right: 1". Restored.
2311 ///
2312 /// Seen RED a third time by computing deadness the way a retirement written
2313 /// *after* the drop would have to (`occupied[i]` → `!occupied[i]` in
2314 /// `retire_dead_packs`): the same "gc retired 0 pack(s)" — after the drop an
2315 /// all-dead pack is indistinguishable from an unabsorbed one, which is the
2316 /// same confusion this whole change is about, one level up. Restored.
2317 ///
2318 /// And the derived half, seen RED by removing the `refold()` from
2319 /// `drop_dead_rows`: "the graph still holds 551 commits after gc dropped
2320 /// every dead one — left: 551, right: 1", with `reachable()` still selecting
2321 /// objects the index can no longer serve. Restored.
2322 #[test]
2323 fn a_wholly_dead_pack_does_not_come_back_when_the_store_is_reopened() {
2324 let DoomedFixture {
2325 dir,
2326 store,
2327 doomed: doomed_oid,
2328 root: root_raw,
2329 tree,
2330 } = one_live_pack_and_one_doomed_pack("gc-wholly-dead");
2331
2332 let (rows_after_gc, retired_by_gc) = {
2333 let report = store.gc().unwrap();
2334 assert_eq!(
2335 report.retired_packs, 1,
2336 "gc retired {} pack(s), not the one whose objects all died",
2337 report.retired_packs
2338 );
2339 assert!(
2340 !store.has(&doomed_oid).unwrap(),
2341 "the dead object still resolves in the store that dropped it"
2342 );
2343 assert!(store.has(&root_raw).unwrap(), "gc dropped the live commit");
2344
2345 // **Everything the indexer derived is dropped with the rows, in the
2346 // same process** — not merely rebuilt by the next open. The graph
2347 // holds one commit, and a selection over the bitmaps and the ordinal
2348 // space cannot name a dead object.
2349 assert_eq!(
2350 store.commit_count(),
2351 1,
2352 "the graph still holds {} commits after gc dropped every dead one",
2353 store.commit_count()
2354 );
2355 let selected = store.reachable(&[&root_raw], &[]).unwrap();
2356 assert!(
2357 !selected.contains(&doomed_oid),
2358 "a selection after gc still names a dead object — the bitmaps or the ordinal \
2359 space were not refolded"
2360 );
2361 assert_eq!(
2362 selected.len(),
2363 store.index().len(),
2364 "a selection after gc names {} objects and the index holds {}",
2365 selected.len(),
2366 store.index().len()
2367 );
2368 (store.index().len(), report.retired_packs)
2369 };
2370 drop(store);
2371
2372 // The tombstone is on disk, and it names the doomed pack's offset — not
2373 // the live one's.
2374 let (packs, retired) = journal_state(&dir);
2375 assert_eq!(packs.len(), 2, "both packs are still acked: {packs:?}");
2376 assert_eq!(
2377 retired,
2378 vec![packs[1].0],
2379 "the journal retired {retired:?}, and the doomed pack starts at {}",
2380 packs[1].0
2381 );
2382 assert_eq!(retired.len() as u64, retired_by_gc);
2383
2384 // ── the reopen: the whole point ──────────────────────────────────────
2385 let store = GitStore::open(&dir, "alice").unwrap();
2386 assert!(
2387 !store.has(&doomed_oid).unwrap(),
2388 "a dead object came back the instant the store was reopened"
2389 );
2390 store.wait_indexed();
2391 assert!(
2392 !store.has(&doomed_oid).unwrap(),
2393 "the reopen re-queued the wholly dead pack and its objects came back"
2394 );
2395 assert_eq!(
2396 store.index().len(),
2397 rows_after_gc,
2398 "the dead objects came back on the reopen: the store that dropped them holds \
2399 {rows_after_gc} rows, the reopened one holds {}",
2400 store.index().len()
2401 );
2402 assert_eq!(
2403 store.unindexed_packs(),
2404 0,
2405 "the retired pack is queued as work"
2406 );
2407 assert_eq!(
2408 store.absorb_pending().unwrap(),
2409 0,
2410 "a read falling back would re-absorb the retired pack"
2411 );
2412 // And the live side of the same repository is untouched by all of it.
2413 assert!(store.has(&root_raw).unwrap(), "the live commit is gone");
2414 assert!(store.has(&tree).unwrap(), "the live tree is gone");
2415 assert_eq!(
2416 store.commit_count(),
2417 1,
2418 "the graph after gc + reopen holds {} commits, not the one live one",
2419 store.commit_count()
2420 );
2421 }
2422
2423 /// **A partly dead pack is not retired, and comes through a GC and a reopen
2424 /// with its live objects.**
2425 ///
2426 /// This is the case that already worked and must keep working: the pack still
2427 /// has rows, so the crash-recovery diff calls it absorbed and nothing about
2428 /// it changes. Asserted on applied output: **no** tombstone appears in the
2429 /// journal, the live objects read back after the reopen, and the dead ones
2430 /// stay dead.
2431 ///
2432 /// Seen RED by retiring on `has_live[i]` instead of `!has_live[i]` in
2433 /// `retire_dead_packs` — i.e. tombstoning the packs that are *alive*: "gc
2434 /// retired the partly dead pack — its live objects are one reopen from a
2435 /// fallback that will never come — left: 1, right: 0", and the journal
2436 /// assertion below holds the same finding in on-disk bytes. The live objects
2437 /// still read back under that mutation, because a pack with rows is adopted
2438 /// as absorbed whatever the journal says about it — which is exactly why this
2439 /// guard asserts on the journal and not only on the reads. Restored.
2440 #[test]
2441 fn a_partly_dead_pack_is_never_retired_and_survives_a_reopen() {
2442 let dir = tmpdir("gc-partly-dead");
2443 let (pack, _) = corpus_pack();
2444 let store = GitStore::open(&dir, "alice").unwrap();
2445 store.put(&pack, &[]).unwrap();
2446 store.absorb_pending().unwrap();
2447
2448 let root = store
2449 .graph_snapshot()
2450 .into_iter()
2451 .find(|c| c.generation == 1)
2452 .expect("a root commit");
2453 let root_raw = hex::decode(&root.oid).unwrap();
2454 let live: std::collections::HashSet<Vec<u8>> = store
2455 .reachable(&[&root_raw], &[])
2456 .unwrap()
2457 .into_iter()
2458 .collect();
2459 let all: Vec<Vec<u8>> = store.index().oids_in_order().unwrap();
2460 let dead = all
2461 .iter()
2462 .find(|o| !live.contains(*o))
2463 .expect("a real pack holds more than one commit's closure")
2464 .clone();
2465 assert!(
2466 live.len() < all.len(),
2467 "the fixture pack is not partly dead: {} live of {}",
2468 live.len(),
2469 all.len()
2470 );
2471 store
2472 .update_ref("refs/heads/root", None, Some(&root_raw))
2473 .unwrap();
2474 let files = vec![("pack-0.pack".to_string(), pack.clone())];
2475 znippy_common::create_archive(store.archive_path(), &files, 3).unwrap();
2476
2477 let report = store.gc().unwrap();
2478 assert_eq!(
2479 report.retired_packs, 0,
2480 "gc retired the partly dead pack — its live objects are one reopen from a fallback \
2481 that will never come"
2482 );
2483 let rows_after_gc = store.index().len();
2484 drop(store);
2485
2486 let (packs, retired) = journal_state(&dir);
2487 assert_eq!(packs.len(), 1);
2488 assert!(
2489 retired.is_empty(),
2490 "gc tombstoned a partly dead pack: {retired:?}"
2491 );
2492
2493 let store = GitStore::open(&dir, "alice").unwrap();
2494 store.wait_indexed();
2495 assert_eq!(
2496 store.index().len(),
2497 rows_after_gc,
2498 "the reopen changed the row count of a partly dead pack's repository"
2499 );
2500 for oid in &live {
2501 assert!(
2502 store.has(oid).unwrap(),
2503 "the live object {} did not survive gc + reopen",
2504 hex::encode(oid)
2505 );
2506 }
2507 assert!(
2508 !store.has(&dead).unwrap(),
2509 "a dead object came back on the reopen of a partly dead pack"
2510 );
2511 // The pack was adopted as absorbed rather than re-queued: no work is
2512 // owed, and a read falling back would re-absorb it.
2513 assert_eq!(
2514 store.unindexed_packs(),
2515 0,
2516 "the surviving pack was re-queued"
2517 );
2518 assert_eq!(store.absorb_pending().unwrap(), 0);
2519 }
2520
2521 /// **An interruption at each step of the retirement leaves a readable
2522 /// archive and loses nothing.**
2523 ///
2524 /// The two steps are run as `GitOps::gc` runs them and the process boundary
2525 /// is a real one — the store is dropped and reopened between them, which is
2526 /// what a `kill -9` there would leave, because everything each step writes is
2527 /// fsynced before it returns.
2528 ///
2529 /// | killed after | on disk | what must be true |
2530 /// |---|---|---|
2531 /// | the tombstone | rows still there | **nothing is lost** — every object still reads, dead ones included |
2532 /// | the drop | rows gone, no compaction | the dead objects are gone and **stay** gone |
2533 /// | the compaction | a new generation | `gc.rs`'s own `an_interruption_at_every_step_leaves_a_readable_archive` |
2534 ///
2535 /// The first row is why the tombstone goes **first**: a GC killed there is a
2536 /// GC that did not happen, and the repository is exactly as it was. The
2537 /// reverse order has a window where the rows are gone and the journal still
2538 /// claims an unabsorbed pack, which is the resurrection this whole change is
2539 /// about.
2540 ///
2541 /// The archive itself is read back through the ordinary reader at every
2542 /// checkpoint, so "readable" is the whole file decoding, not a stat.
2543 ///
2544 /// Seen RED by computing deadness from `!occupied[i]` in `retire_dead_packs`
2545 /// — the shape a retirement written after the drop needs, which cannot see a
2546 /// pack that still has its rows: "the doomed pack was not retired: [] —
2547 /// left: 0, right: 1", at the first checkpoint, with every row still in the
2548 /// index. Restored.
2549 #[test]
2550 fn an_interruption_at_each_step_of_the_retirement_loses_nothing() {
2551 let DoomedFixture {
2552 dir,
2553 store,
2554 doomed: doomed_oid,
2555 root: root_raw,
2556 tree,
2557 } = one_live_pack_and_one_doomed_pack("gc-interrupt");
2558 let archive = dir.join("repository.znippy");
2559 let readable = |where_: &str| {
2560 crate::gc::read_back_every_entry(&archive)
2561 .unwrap_or_else(|e| panic!("the archive does not read back {where_}: {e:?}"));
2562 };
2563 readable("before anything ran");
2564
2565 // ── killed right after the tombstone, before any row was dropped ─────
2566 //
2567 // The live set is computed once and carried, which is what `gc()` does
2568 // with it too: it is an input to both steps, and computing it twice would
2569 // make this guard depend on a reachability fold rather than on the two
2570 // mutations it is here to interrupt.
2571 let (live, dead, rows_before) = {
2572 let seen = store.index().oids_in_order().unwrap();
2573 let live = store.live_set().unwrap();
2574 let dead = crate::git_ops::dead_oids(seen, &live);
2575 let retired = store.retire_dead_packs(&live, u64::MAX).unwrap();
2576 assert_eq!(
2577 retired.len(),
2578 1,
2579 "the doomed pack was not retired: {retired:?}"
2580 );
2581 let n = store.index().len();
2582 assert!(
2583 store.has(&doomed_oid).unwrap(),
2584 "retiring the journal row dropped an index row — the two steps are not separable"
2585 );
2586 (live, dead, n)
2587 };
2588 drop(store);
2589 let (packs, on_disk) = journal_state(&dir);
2590 assert_eq!(
2591 on_disk,
2592 vec![packs[1].0],
2593 "the tombstone is not on disk before a single row was dropped: {on_disk:?}"
2594 );
2595 readable("after the tombstone");
2596
2597 {
2598 // The reopen a crash there produces: everything is still here.
2599 let store = GitStore::open(&dir, "alice").unwrap();
2600 store.wait_indexed();
2601 assert_eq!(
2602 store.index().len(),
2603 rows_before,
2604 "a crash between the tombstone and the drop lost rows"
2605 );
2606 assert!(
2607 store.has(&doomed_oid).unwrap(),
2608 "a crash between the tombstone and the drop lost the objects of the pack it \
2609 retired — the GC had not decided anything yet"
2610 );
2611 assert!(store.has(&root_raw).unwrap() && store.has(&tree).unwrap());
2612 // And resuming is idempotent: the pack is already retired.
2613 assert!(
2614 store.retire_dead_packs(&live, u64::MAX).unwrap().is_empty(),
2615 "the resumed GC retired the same pack twice"
2616 );
2617 }
2618
2619 // ── killed after the drop, before the compaction ─────────────────────
2620 let rows_after_drop = {
2621 let store = GitStore::open(&dir, "alice").unwrap();
2622 let dropped = store.drop_dead_rows(&dead).unwrap();
2623 assert!(dropped > 0, "the drop dropped nothing");
2624 assert!(!store.has(&doomed_oid).unwrap());
2625 store.index().len()
2626 };
2627 readable("after the drop, before the compaction");
2628 let (_, still_retired) = journal_state(&dir);
2629 assert_eq!(still_retired.len(), 1, "the drop lost the tombstone");
2630
2631 {
2632 let store = GitStore::open(&dir, "alice").unwrap();
2633 store.wait_indexed();
2634 assert!(
2635 !store.has(&doomed_oid).unwrap(),
2636 "a crash between the drop and the compaction resurrected the dead objects"
2637 );
2638 assert_eq!(store.index().len(), rows_after_drop);
2639 assert!(store.has(&root_raw).unwrap() && store.has(&tree).unwrap());
2640
2641 // ── and the compaction still completes on the resumed store ──────
2642 let report = store.gc().unwrap();
2643 assert_eq!(
2644 report.retired_packs, 0,
2645 "the resumed gc retired an already-retired pack"
2646 );
2647 assert!(
2648 report.verified,
2649 "the resumed gc did not verify its generation"
2650 );
2651 assert!(report.archive.exists());
2652 crate::gc::read_back_every_entry(&report.archive)
2653 .expect("the new generation does not read back");
2654 assert!(!store.has(&doomed_oid).unwrap());
2655 assert!(store.has(&root_raw).unwrap());
2656 }
2657 }
2658
2659 /// `compact` leaves its source untouched and refuses to overwrite.
2660 ///
2661 /// Seen RED by removing the `dst.exists()` refusal: "the refusal must be the
2662 /// explicit one and must name the file: hard-linking …/repo.znippy to
2663 /// …/repo.g1.znippy — compaction runs against a second name for the same
2664 /// inode …". The first version of this guard only asserted *that* it failed,
2665 /// and that version stayed green under the same mutation, because `hard_link`
2666 /// refuses EEXIST as well. It now asserts which mechanism refused.
2667 #[test]
2668 fn compact_names_its_destination_and_never_overwrites() {
2669 let dir = tmpdir("compact");
2670 let src = dir.join("repo.znippy");
2671 let dst = dir.join("repo.g1.znippy");
2672 let files = vec![
2673 ("pack-0.pack".to_string(), vec![7u8; 400_000]),
2674 ("pack-1.pack".to_string(), vec![9u8; 400_000]),
2675 ];
2676 znippy_common::create_archive(&src, &files, 3).unwrap();
2677 let before = std::fs::read(&src).unwrap();
2678
2679 compact(&src, &dst).unwrap();
2680 assert!(dst.exists(), "compact produced no destination");
2681 assert_eq!(
2682 std::fs::read(&src).unwrap(),
2683 before,
2684 "compact modified its source"
2685 );
2686
2687 let err = compact(&src, &dst).expect_err("dst exists");
2688 // Asserted on WHICH mechanism refused, not merely that something did:
2689 // `hard_link` would fail with EEXIST too, so a guard that only checked
2690 // for an error could not tell the explicit refusal from the syscall's —
2691 // and the difference is whether a filesystem operation is attempted
2692 // against a file that must not be touched at all.
2693 assert!(
2694 err.to_string().contains("refusing to compact over it")
2695 && err.to_string().contains(&dst.display().to_string()),
2696 "the refusal must be the explicit one and must name the file: {err}"
2697 );
2698 assert_eq!(
2699 std::fs::read(&src).unwrap(),
2700 before,
2701 "the refused second compaction touched the source"
2702 );
2703 }
2704}
2705
2706#[cfg(test)]
2707mod emit_set_tests {
2708 use crate::index_layout::ObjectIndex as _;
2709 use crate::pack_walk::topological_order;
2710 use crate::store::tests::{corpus_pack, tmpdir};
2711 use crate::{GitOps, GitStore};
2712
2713 /// **A SUBSET emits a pack holding EXACTLY the subset, and stock git accepts
2714 /// it.**
2715 ///
2716 /// Emitting a whole pack proves the encoder; it cannot prove anything about
2717 /// a request whose delta bases fall outside it, because every base is
2718 /// present by construction. A subset is that case.
2719 ///
2720 /// # What this asserted until 2026-08-11, and why it inverted
2721 ///
2722 /// It asserted `entries.len() >= half.len()` under the heading *"closure
2723 /// pulled in nothing"* — the set was expected to come back **larger**,
2724 /// because `emit_set` added the delta bases. That is the defect: a base
2725 /// pulled into a narrowed request is an object the client did not ask for,
2726 /// and a tree pulled in that way owes children the pack does not carry. See
2727 /// [`GitStore::emit_set`] for the production failure and the counts.
2728 ///
2729 /// So the assertion is now an equality, and it is the strong direction: the
2730 /// emitted set is the requested set, entry for entry. The delta whose base
2731 /// was left out is emitted **whole**, which is what `pack-objects` does and
2732 /// what keeps the pack connected.
2733 #[test]
2734 fn a_subset_emits_exactly_the_subset_and_stock_git_accepts_it() {
2735 if std::process::Command::new("git")
2736 .arg("--version")
2737 .output()
2738 .is_err()
2739 {
2740 eprintln!("skipping: no git on PATH");
2741 return;
2742 }
2743 let dir = tmpdir("emit-subset");
2744 let store = GitStore::open(&dir, "alice").unwrap();
2745 let (pack, _) = corpus_pack();
2746 store.put_pack(&pack).unwrap();
2747 store.absorb_pending().unwrap();
2748
2749 let all = store.index().oids_in_order().unwrap();
2750 assert!(all.len() > 8, "corpus too small to take a subset of");
2751 // Every other object: enough deltas land with their base excluded that
2752 // the boundary rule has real work to do.
2753 let half: Vec<&[u8]> = all.iter().step_by(2).map(Vec::as_slice).collect();
2754 let rows = store.index().lookup_batch(&half);
2755 let inside: std::collections::HashSet<u64> =
2756 rows.iter().flatten().map(|r| r.offset).collect();
2757 let boundary = rows
2758 .iter()
2759 .flatten()
2760 .filter(|r| r.delta_base != 0 && !inside.contains(&r.delta_base))
2761 .count();
2762 assert!(
2763 boundary > 0,
2764 "this subset excludes no delta base, so it cannot tell an exact emission from a \
2765 closed one and the test proves nothing"
2766 );
2767
2768 let entries = store.emit_set(&half, true, None).unwrap();
2769 assert_eq!(
2770 entries.len(),
2771 half.len(),
2772 "the emitted set must be the requested set — {} asked, {} emitted",
2773 half.len(),
2774 entries.len()
2775 );
2776 assert_eq!(
2777 entries.iter().filter(|e| e.recompressed).count(),
2778 boundary,
2779 "exactly the entries whose base was excluded must be rebuilt whole"
2780 );
2781
2782 let (ordered, missing) = topological_order(entries);
2783 assert!(
2784 missing.is_empty(),
2785 "no entry may still name a base that is not here: {missing:?}"
2786 );
2787
2788 // Walk what we emitted, before handing it to git. If the header count
2789 // and the body disagree, that is the shape that crashes an indexer.
2790 {
2791 let mut b = Vec::new();
2792 store.emit_ordered(&ordered, &mut b).unwrap();
2793 let declared = u32::from_be_bytes([b[8], b[9], b[10], b[11]]);
2794 match crate::pack_walk::walk(&b, store.hash_kind().oid_len()) {
2795 Ok(w) => assert_eq!(
2796 w.entries.len(),
2797 declared as usize,
2798 "the pack header declares {declared} entries and the body walks to {}",
2799 w.entries.len()
2800 ),
2801 Err(e) => panic!("our own walk cannot read what we emitted: {e:#}"),
2802 }
2803 }
2804
2805 let mut bytes = Vec::new();
2806 let report = store.emit_ordered(&ordered, &mut bytes).unwrap();
2807 assert_eq!(
2808 report.copied + report.recompressed,
2809 report.written,
2810 "every entry is either copied or rebuilt, and the receipt must add up"
2811 );
2812 assert_eq!(
2813 report.recompressed as usize, boundary,
2814 "only the excluded-base entries may be rebuilt"
2815 );
2816
2817 // **No `--strict` here, and that is deliberate.** An arbitrary every-other
2818 // subset is not reachability-closed — a tree in it will name a blob that
2819 // is not — so `--strict`'s connectivity walk would be judging the
2820 // *caller's* selection, not this emitter. What is being asserted is
2821 // SELF-CONTAINMENT: that every delta in the pack resolves inside it,
2822 // which plain `index-pack` answers with `pack has N unresolved deltas`.
2823 // Connectivity has its own test, on a request that is closed:
2824 // `a_narrowed_clone_is_served_a_connected_pack_not_its_delta_bases`.
2825 let out_dir = tmpdir("emit-subset-idx");
2826 crate::git_oracle::assert_git_accepts(
2827 &out_dir,
2828 "dst.git",
2829 &bytes,
2830 crate::git_oracle::Strictness::SelfContained,
2831 );
2832 }
2833
2834 /// 🔴 **An oid this repository does not hold is REFUSED by name — it is not
2835 /// quietly left out of the pack.**
2836 ///
2837 /// `emit_set` used to `continue` past a `lookup_batch` miss. That is the
2838 /// silent-under-send shape: the request asks for N objects, the pack carries
2839 /// N-1, `PackStats` reports success and the *client* is the first to notice.
2840 /// It also made the two engines behind one contract disagree — gunnar's
2841 /// in-memory arm has always refused with *"emit_pack was asked for X, which
2842 /// this store does not hold"* — so a benchmark across the two was comparing
2843 /// two different contracts.
2844 ///
2845 /// # Why the assertions are shaped this way
2846 ///
2847 /// **RED before the change** on the first one: `emit_set` returned `Ok` with
2848 /// two entries for three asked oids, so `expect_err` panicked.
2849 ///
2850 /// The error must **name the oid**, because an operator reading
2851 /// `git.upload_pack.failed` needs the object, not the fact that something
2852 /// was missing; asserted on the rendered message.
2853 ///
2854 /// And the refusal must be *conditional*, or a function that errored
2855 /// unconditionally would pass the first two: the same store, the same two
2856 /// real oids and no absent one, must still emit.
2857 #[test]
2858 fn an_oid_the_store_does_not_hold_is_refused_by_name_not_skipped() {
2859 let dir = tmpdir("emit-missing");
2860 let store = GitStore::open(&dir, "alice").unwrap();
2861 let (pack, _) = corpus_pack();
2862 store.put_pack(&pack).unwrap();
2863 store.absorb_pending().unwrap();
2864
2865 let all = store.index().oids_in_order().unwrap();
2866 assert!(all.len() > 2, "corpus too small");
2867 // A well-formed oid of the right width that this store cannot hold: it
2868 // is not in the index, and asking for it is what a caller with a stale
2869 // set does.
2870 let absent = vec![0xABu8; store.hash_kind().oid_len()];
2871 assert!(
2872 store.index().lookup(&absent).is_none(),
2873 "the fixture oid must really be absent or this test asserts nothing"
2874 );
2875
2876 let asked: Vec<&[u8]> = vec![all[0].as_slice(), absent.as_slice(), all[1].as_slice()];
2877 let err = store
2878 .emit_set(&asked, true, None)
2879 .expect_err("a pack for an object the store lacks must be refused, not shortened");
2880 let msg = format!("{err:#}");
2881 assert!(
2882 msg.contains(&hex::encode(&absent)),
2883 "the refusal must name the oid it could not find; got: {msg}"
2884 );
2885
2886 // Conditional, not unconditional: drop the absent one and the same call
2887 // emits both objects.
2888 let present: Vec<&[u8]> = vec![all[0].as_slice(), all[1].as_slice()];
2889 let entries = store
2890 .emit_set(&present, true, None)
2891 .expect("two objects this store does hold must still emit");
2892 assert_eq!(entries.len(), 2);
2893 }
2894}
2895
2896/// **The zero-copy emit path: the same bytes, and none of the memory.**
2897///
2898/// [`crate::pack_walk::EntryBytes`] turned `EmitEntry.stored` from a `Vec<u8>`
2899/// that owned a `pread`ed copy of the entry into a 16-byte address resolved
2900/// against a mapping of the archive. That is a change to *where the bytes live*
2901/// and to **nothing else**, so the guards here are of two kinds: three that
2902/// require the emitted bytes to be unchanged, and one that requires the memory
2903/// to be gone. Either kind alone would pass on a broken change — an emitter that
2904/// held nothing and wrote garbage, or one that wrote perfectly and still held
2905/// the repository.
2906#[cfg(test)]
2907mod zero_copy_tests {
2908 use crate::index_layout::{ObjType, ObjectIndex as _};
2909 use crate::pack_walk::{EmitEntry, EntryBytes, topological_order};
2910 use crate::store::tests::{corpus_pack, tmpdir};
2911 use crate::{GitOps, GitStore};
2912
2913 /// A store holding the whole corpus pack, and every oid in it.
2914 fn corpus(name: &str) -> (std::path::PathBuf, GitStore, Vec<Vec<u8>>) {
2915 let dir = tmpdir(name);
2916 let store = GitStore::open(&dir, "alice").unwrap();
2917 let (pack, _) = corpus_pack();
2918 store.put_pack(&pack).unwrap();
2919 store.absorb_pending().unwrap();
2920 let all = store.index().oids_in_order().unwrap();
2921 assert!(all.len() > 100, "the corpus is too small to prove anything");
2922 (dir, store, all)
2923 }
2924
2925 /// The **old** shape of an emit set: every entry's bytes `pread` into an
2926 /// owned `Vec`, exactly as `emit_set` built them before 2026-08-14. The
2927 /// differential guards below emit this and the extent form and require the
2928 /// two packs to be identical.
2929 ///
2930 /// It reads through [`GitStore::read_extent`] — the `pread` path — so it
2931 /// shares no code with the mapping it is being compared against. A helper
2932 /// that resolved through `Mapped::get` would be comparing the change with
2933 /// itself.
2934 fn materialised(store: &GitStore, entries: &[EmitEntry]) -> Vec<EmitEntry> {
2935 entries
2936 .iter()
2937 .map(|e| {
2938 let mut e = e.clone();
2939 if let EntryBytes::Extent { offset, len } = e.stored {
2940 e.stored = EntryBytes::Owned(store.read_extent(offset, len).unwrap());
2941 }
2942 e
2943 })
2944 .collect()
2945 }
2946
2947 /// 🔴 **A full clone's pack is byte-identical before and after the change.**
2948 ///
2949 /// The strongest guard available, and the one the whole change stands on: a
2950 /// clone is built twice out of the same selection — once with every entry an
2951 /// `EntryBytes::Extent` resolved through the mapping, once with every entry
2952 /// `pread` into an owned `Vec` the way `emit_set` used to build it — and the
2953 /// two packs must be the same bytes. Not the same length, not the same
2954 /// object count: the same **bytes**, trailer included, which for a pack
2955 /// means the same headers, the same `OFS_DELTA` distances and the same sha1.
2956 ///
2957 /// A whole-repository selection, so `recompressed` is 0 and every one of the
2958 /// entries really is an extent — asserted, because a run in which they were
2959 /// all `Owned` would compare the old path with itself and pass vacuously.
2960 ///
2961 /// Seen RED by returning `snap.get(offset, len - 1)` from
2962 /// [`GitStore::extent`] — a mapping that hands back a *short* slice rather
2963 /// than `None`, which is the single most plausible way to get this wrong:
2964 /// "the mapped emit and the `pread` emit disagree at byte 1944: 0xe8 vs
2965 /// 0x3a (mapped 5651349 bytes, pread 5654037 bytes)".
2966 ///
2967 /// Seen RED a second way, by slicing `snap.get(offset + 1, len)` — an
2968 /// extent addressed one byte off: "resolving the emit set's stored bytes:
2969 /// resolving the stored bytes of 00065363b6d5f3edb20c8c17a5938503fcdcc941
2970 /// for emission: object type code 0 is not one git writes — refusing to
2971 /// guess it". That is phase 1's header parse refusing before phase 2 had
2972 /// written a byte of the pack, which is why the parse is there.
2973 #[test]
2974 fn a_full_clone_emits_the_identical_pack_from_extents_and_from_owned_bytes() {
2975 let (_dir, store, all) = corpus("zc-identical");
2976 let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
2977
2978 let entries = store.emit_set(&oids, true, None).unwrap();
2979 let extents = entries
2980 .iter()
2981 .filter(|e| matches!(e.stored, EntryBytes::Extent { .. }))
2982 .count();
2983 assert_eq!(
2984 extents,
2985 entries.len(),
2986 "a whole-repository selection must be ALL extents — {} of {} were owned, so this \
2987 comparison would be the old path against itself",
2988 entries.len() - extents,
2989 entries.len()
2990 );
2991
2992 let owned = materialised(&store, &entries);
2993 let (ordered_a, missing) = topological_order(entries);
2994 assert!(missing.is_empty(), "a full clone is closed: {missing:?}");
2995 let (ordered_b, _) = topological_order(owned);
2996
2997 let mut mapped_pack = Vec::new();
2998 let report = store.emit_ordered(&ordered_a, &mut mapped_pack).unwrap();
2999 let mut pread_pack = Vec::new();
3000 store.emit_ordered(&ordered_b, &mut pread_pack).unwrap();
3001
3002 assert_eq!(report.recompressed, 0, "a full clone re-deflates nothing");
3003 assert_eq!(report.copied, report.written);
3004 assert!(mapped_pack.len() > 1_000_000, "the corpus pack is tiny?");
3005 if mapped_pack != pread_pack {
3006 let at = mapped_pack
3007 .iter()
3008 .zip(pread_pack.iter())
3009 .position(|(a, b)| a != b);
3010 let i = at.unwrap_or(mapped_pack.len().min(pread_pack.len()));
3011 panic!(
3012 "the mapped emit and the `pread` emit disagree at byte {i}: {:#04x} vs {:#04x} \
3013 (mapped {} bytes, pread {} bytes)",
3014 mapped_pack.get(i).copied().unwrap_or(0),
3015 pread_pack.get(i).copied().unwrap_or(0),
3016 mapped_pack.len(),
3017 pread_pack.len()
3018 );
3019 }
3020 }
3021
3022 /// 🔴 **Every `OFS_DELTA`'s base distance still lands on its base.**
3023 ///
3024 /// The one thing phase 2 may not be parallelised around, asserted as applied
3025 /// output rather than as a receipt: the emitted pack is walked back, and for
3026 /// every `OfsDelta` entry the base its distance names must be an entry
3027 /// boundary in the same pack — which `PackWalk::closure` answers, and which
3028 /// no amount of correct-looking header encoding can satisfy by accident.
3029 /// `git index-pack` agrees separately in the tests beside this one; this is
3030 /// the guard that says *which* thing broke when it stops agreeing.
3031 ///
3032 /// The count of `OfsDelta` entries is asserted first. A pack with none would
3033 /// pass this vacuously, and a corpus that happened to hold none would make
3034 /// the whole of phase 2's serial argument untestable.
3035 ///
3036 /// Seen RED by encoding the distance as `here - base_at + 1` in
3037 /// `emit_pack`: "1345 of 1345 ofs-deltas name a base that is not an entry
3038 /// boundary — first at output offset 11". Seen RED a second, sharper way by
3039 /// emitting `emit_set`'s entries in their input order rather than
3040 /// `topological_order`'s: "entry at 3414738 deltas against archive offset
3041 /// 3414009, which is not in this pack — the set is not closed and was not
3042 /// ordered by `topological_order`".
3043 #[test]
3044 fn an_ofs_delta_still_names_its_base_by_the_right_distance() {
3045 let (_dir, store, all) = corpus("zc-ofs");
3046 let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
3047 let (ordered, _) = topological_order(store.emit_set(&oids, true, None).unwrap());
3048
3049 let mut pack = Vec::new();
3050 store.emit_ordered(&ordered, &mut pack).unwrap();
3051
3052 let walk = crate::pack_walk::walk(&pack, store.hash_kind().oid_len())
3053 .expect("our own walk must read what we emitted");
3054 let deltas = walk
3055 .entries
3056 .iter()
3057 .filter(|e| e.obj_type == ObjType::OfsDelta)
3058 .count();
3059 assert!(
3060 deltas > 0,
3061 "the corpus emitted no ofs-delta, so this proves nothing about distances"
3062 );
3063
3064 let closure = walk.closure();
3065 assert!(
3066 closure.broken_offsets.is_empty(),
3067 "{} of {deltas} ofs-deltas name a base that is not an entry boundary — first at \
3068 output offset {}",
3069 closure.broken_offsets.len(),
3070 closure.broken_offsets.first().copied().unwrap_or(0)
3071 );
3072 assert!(
3073 closure.external_refs.is_empty(),
3074 "a full clone must name no external base"
3075 );
3076 }
3077
3078 /// 🔴 **The `Owned` exception still carries a rebuilt delta's bytes.**
3079 ///
3080 /// `EntryBytes::Owned` is not vestigial: a delta whose base the request does
3081 /// not carry genuinely computes bytes that no extent addresses, and dropping
3082 /// the variant would have meant either shipping a delta the client cannot
3083 /// resolve or widening the request — the two failures
3084 /// `GitStore::emit_set`'s doc comment is written about.
3085 ///
3086 /// Asserted on applied output at both ends: the boundary entries are `Owned`
3087 /// and every other entry is an `Extent` (a mixed set, so the emitter is
3088 /// proved to handle both in one pack), and the resulting pack walks with
3089 /// its closure intact. The premise — that this subset really does cut delta
3090 /// chains — is asserted first.
3091 ///
3092 /// Seen RED by making the whole-rebuild arm of `emit_set` push
3093 /// `EntryBytes::Extent { offset: row.offset, len: row.len }` instead of the
3094 /// bytes it just built: "312 boundary entries are recompressed but 202 carry
3095 /// owned bytes — a rebuilt entry that points back at its stored extent ships
3096 /// the delta it was rebuilt to avoid". 202 and not 0, because the re-delta
3097 /// arm above it was untouched — which is what makes the count, and not a
3098 /// bare `is_some()`, the thing worth asserting.
3099 #[test]
3100 fn a_rebuilt_delta_still_carries_its_own_bytes_and_emits_beside_extents() {
3101 let (_dir, store, all) = corpus("zc-owned");
3102 let half: Vec<&[u8]> = all.iter().step_by(2).map(Vec::as_slice).collect();
3103
3104 let rows = store.index().lookup_batch(&half);
3105 let inside: std::collections::HashSet<u64> =
3106 rows.iter().flatten().map(|r| r.offset).collect();
3107 let boundary = rows
3108 .iter()
3109 .flatten()
3110 .filter(|r| r.delta_base != 0 && !inside.contains(&r.delta_base))
3111 .count();
3112 assert!(
3113 boundary > 0,
3114 "this subset cuts no delta chain, so the Owned path is never reached"
3115 );
3116
3117 let entries = store.emit_set(&half, true, None).unwrap();
3118 let rebuilt: Vec<&EmitEntry> = entries.iter().filter(|e| e.recompressed).collect();
3119 let with_bytes = rebuilt
3120 .iter()
3121 .filter(|e| e.stored.owned().is_some_and(|b| !b.is_empty()))
3122 .count();
3123 assert_eq!(
3124 with_bytes,
3125 rebuilt.len(),
3126 "{} boundary entries are recompressed but {with_bytes} carry owned bytes — a rebuilt \
3127 entry that points back at its stored extent ships the delta it was rebuilt to avoid",
3128 rebuilt.len()
3129 );
3130 assert_eq!(rebuilt.len(), boundary);
3131
3132 // The other half of the point: this is a MIXED pack, so the emitter is
3133 // resolving both variants in one pass.
3134 let as_extent = entries
3135 .iter()
3136 .filter(|e| matches!(e.stored, EntryBytes::Extent { .. }))
3137 .count();
3138 assert_eq!(
3139 as_extent,
3140 entries.len() - rebuilt.len(),
3141 "every entry that was NOT rebuilt must still be an extent"
3142 );
3143 assert!(
3144 as_extent > 0 && !rebuilt.is_empty(),
3145 "the set must be mixed"
3146 );
3147
3148 let (ordered, missing) = topological_order(entries);
3149 assert!(missing.is_empty(), "the emitted set is closed: {missing:?}");
3150 let mut pack = Vec::new();
3151 let report = store.emit_ordered(&ordered, &mut pack).unwrap();
3152 assert_eq!(report.recompressed as usize, boundary);
3153 let walk = crate::pack_walk::walk(&pack, store.hash_kind().oid_len())
3154 .expect("a mixed pack must still walk");
3155 assert!(
3156 walk.closure().broken_offsets.is_empty(),
3157 "a pack mixing extents and rebuilt bytes must still be self-contained"
3158 );
3159 }
3160
3161 /// 🔴 **An extent past the mapping falls back to `pread` and returns the
3162 /// SAME bytes.**
3163 ///
3164 /// `Mapped::get` answers `None` for an extent past the mapped end, which
3165 /// means *"go and pread it"* and never *"there are no bytes"*. The case is
3166 /// real and not hypothetical: a snapshot is taken once per emission and the
3167 /// blob file grows whenever a push lands, so any clone running across a push
3168 /// reads its tail this way.
3169 ///
3170 /// Constructed deliberately rather than raced: a snapshot is taken, a second
3171 /// pack is pushed, and the objects of that second pack are then emitted
3172 /// against the **stale** snapshot. Every one of their extents is past its
3173 /// end, so every one takes the fallback — asserted by `Mapped::get` refusing
3174 /// them, so a run where the snapshot happened to cover them cannot pass
3175 /// quietly.
3176 ///
3177 /// Seen RED by making [`GitStore::extent`]'s `None` arm
3178 /// `Ok(Cow::Owned(Vec::new()))` — the "absent means no bytes" misreading
3179 /// [`crate::archive_map::Mapped::get`]'s own doc comment forbids: "the
3180 /// fallback returned 0 bytes for the extent at (5653314, 43), which the
3181 /// mapping does not cover; `None` means pread, not empty".
3182 #[test]
3183 fn an_extent_past_the_mapping_falls_back_and_returns_the_same_bytes() {
3184 let (_dir, store, _all) = corpus("zc-fallback");
3185
3186 // The snapshot is taken NOW, before the second push.
3187 let stale = store.archive_snapshot().unwrap();
3188 let before = stale.len();
3189
3190 let (second, _) = crate::store::tests::one_blob_pack(b"a blob pushed after the snapshot\n");
3191 store.put_pack(&second).unwrap();
3192 store.absorb_pending().unwrap();
3193
3194 let after = store.archive_snapshot().unwrap().len();
3195 assert!(
3196 after > before,
3197 "the second push did not grow the blob file ({before} → {after}), so nothing is past \
3198 the stale mapping and this test proves nothing"
3199 );
3200
3201 // Every object of the second pack sits past the stale mapping's end.
3202 let fresh: Vec<Vec<u8>> = store
3203 .index()
3204 .oids_in_order()
3205 .unwrap()
3206 .into_iter()
3207 .filter(|oid| {
3208 store
3209 .index()
3210 .lookup(oid)
3211 .is_some_and(|r| r.offset >= before)
3212 })
3213 .collect();
3214 assert!(
3215 !fresh.is_empty(),
3216 "no object landed past the stale mapping's end"
3217 );
3218
3219 for oid in &fresh {
3220 let row = store.index().lookup(oid).unwrap();
3221 assert!(
3222 stale.get(row.offset, row.len).is_none(),
3223 "the stale mapping must NOT cover ({}, {}) or the fallback is never taken",
3224 row.offset,
3225 row.len
3226 );
3227 let fell_back = store.extent(&stale, row.offset, row.len).unwrap();
3228 let preaded = store.read_extent(row.offset, row.len).unwrap();
3229 assert_eq!(
3230 fell_back.len(),
3231 preaded.len(),
3232 "the fallback returned {} bytes for the extent at ({}, {}), which the mapping does \
3233 not cover; `None` means pread, not empty",
3234 fell_back.len(),
3235 row.offset,
3236 row.len
3237 );
3238 assert_eq!(
3239 fell_back.as_ref(),
3240 preaded.as_slice(),
3241 "the fallback bytes differ from the pread bytes at ({}, {})",
3242 row.offset,
3243 row.len
3244 );
3245 // A FRESH snapshot covers it, and agrees with both.
3246 let fresh_snap = store.archive_snapshot().unwrap();
3247 assert_eq!(
3248 fresh_snap.get(row.offset, row.len).unwrap(),
3249 preaded.as_slice(),
3250 "the remapped snapshot disagrees with the pread at ({}, {})",
3251 row.offset,
3252 row.len
3253 );
3254 }
3255 eprintln!(
3256 "{} object(s) resolved past a stale {before}-byte mapping and matched the pread",
3257 fresh.len()
3258 );
3259 }
3260
3261 /// 🔴 **The gatling fan-out resolves the identical bytes the serial pass
3262 /// does** — LAW 3's primitive, over data with no lock on it.
3263 ///
3264 /// Phase 1 is the one place the serving tier fans out, and the claim that
3265 /// makes it safe is a claim about the *data*: the blob file is append-only
3266 /// and `gc` truncates nothing, so N workers reading a snapshot of it share
3267 /// no mutable state and need no synchronisation. This asserts the observable
3268 /// consequence — the same set resolved on one thread and on four gives the
3269 /// same slices, in the same order, pointing at the same bytes.
3270 ///
3271 /// It calls
3272 /// [`resolve_with`](GitStore::resolve_with) directly rather than going
3273 /// through `resolve_emit_payloads`, because the production threshold is 4096
3274 /// entries and this corpus holds 1352: through the front door the gatling
3275 /// arm would never run and this test would be asserting the serial path
3276 /// against itself. The order matters as much as the content —
3277 /// `gatling_for_each` self-dispatches with no barrier, so a worker finishing
3278 /// unit 9 before unit 3 is normal, and an implementation that collected in
3279 /// **completion** order would produce a pack whose entries are shuffled and
3280 /// whose `OFS_DELTA` distances therefore point at the wrong objects.
3281 ///
3282 /// Seen RED by having the parallel arm return
3283 /// `gatling_for_each(n, workers, one).into_iter().rev().collect()` — the
3284 /// cheapest stand-in for "results not in index order": "the fan-out resolved
3285 /// entry 0 to different bytes than the serial pass did (2 workers): 46 bytes
3286 /// vs 1933, first differing at Some(0)".
3287 #[test]
3288 fn the_fan_out_resolves_the_identical_bytes_the_serial_pass_does() {
3289 let (_dir, store, all) = corpus("zc-gatling");
3290 let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
3291 let (ordered, _) = topological_order(store.emit_set(&oids, true, None).unwrap());
3292 assert!(
3293 ordered.len() > 1000,
3294 "too few entries to spread over workers"
3295 );
3296
3297 let snap = store.archive_snapshot().unwrap();
3298 let serial = store.resolve_with(&ordered, &snap, 1).unwrap();
3299 for workers in [2usize, 4, 8] {
3300 let parallel = store.resolve_with(&ordered, &snap, workers).unwrap();
3301 assert_eq!(
3302 parallel.len(),
3303 serial.len(),
3304 "the fan-out resolved {} entries against the serial pass's {} ({workers} workers)",
3305 parallel.len(),
3306 serial.len()
3307 );
3308 // Compared by hand rather than with `assert_eq!` on the slices: an
3309 // entry is kilobytes, and a failure that dumps two of them is a
3310 // failure nobody reads.
3311 for (i, (p, s)) in parallel.iter().zip(serial.iter()).enumerate() {
3312 if p.as_ref() != s.as_ref() {
3313 let at = p.iter().zip(s.iter()).position(|(a, b)| a != b);
3314 panic!(
3315 "the fan-out resolved entry {i} to different bytes than the serial pass \
3316 did ({workers} workers): {} bytes vs {}, first differing at {at:?}",
3317 p.len(),
3318 s.len()
3319 );
3320 }
3321 }
3322 }
3323
3324 // And the pack itself: emitted off the fan-out, walked back, closed.
3325 let mut pack = Vec::new();
3326 store.emit_ordered(&ordered, &mut pack).unwrap();
3327 let walk = crate::pack_walk::walk(&pack, store.hash_kind().oid_len()).unwrap();
3328 assert_eq!(walk.entries.len(), ordered.len());
3329 assert!(walk.closure().broken_offsets.is_empty());
3330 }
3331
3332 /// 🔴 **The pack is never materialised: the whole emit set's heap is a
3333 /// small fraction of the pack it will write.**
3334 ///
3335 /// # What this asserts, and how completely
3336 ///
3337 /// The emit set's heap is **accounted exactly**, not sampled: an
3338 /// [`EmitEntry`] holds a `Vec<EmitEntry>` slot, an oid, and — for the
3339 /// `Owned` exception only — payload bytes. There is nothing else it can
3340 /// hold, so `slots + oids + owned` is the complete cost of the set, and it
3341 /// is compared against the bytes that set will emit. Before 2026-08-14 the
3342 /// `owned` term alone *was* the pack: every entry's bytes `pread` into a
3343 /// `Vec`, all of them live at once, because the set is built in full before
3344 /// [`crate::pack_walk::emit_pack`] writes a byte.
3345 ///
3346 /// Seen RED by reverting `emit_set`'s base-inside arm to
3347 /// `EntryBytes::Owned(self.read_extent(row.offset, row.len)?)`: "the emit
3348 /// set owns 5653270 bytes of payload — 100.0 % of the 5653270 bytes it will
3349 /// emit — which is the whole pack held in memory before the first byte goes
3350 /// out".
3351 ///
3352 /// # What it does NOT prove, stated plainly
3353 ///
3354 /// **It is not an RSS test, and an RSS test here would be theatre.** This
3355 /// began as one, asserting that `/proc/self/statm` grows by less than the
3356 /// pack across `emit_set`. It was **blind**: run against the broken version
3357 /// above, which really does allocate and fill 5 653 270 bytes, resident set
3358 /// size did not move by a single page — 203 366 400 before and after, and
3359 /// 203 182 080 → 203 182 080 in the green run. The allocator was handing
3360 /// back arena pages this test process already had resident. A guard that
3361 /// cannot tell the defect from the fix is not a weak guard, it is no guard,
3362 /// and it is exactly what LAW 2 says to expect of one's own new guards. It
3363 /// is gone; the accounting above replaces it.
3364 ///
3365 /// **It says nothing about the peak of a real clone.** This corpus is 2.8 MB
3366 /// against `linux.git`'s ~6.4 GB, and the 2314 MB peak this change is aimed
3367 /// at was measured with `perf` and `/proc`, not with a unit test.
3368 ///
3369 /// **The residual it exposes is real and is printed rather than hidden.**
3370 /// The `slots + oids` term is `O(objects)` and survives this change: every
3371 /// entry still carries its own `Vec<u8>` oid, one heap allocation each. The
3372 /// printed projection to 13.8 M objects is the honest size of what is left
3373 /// to do, and it is not small.
3374 #[test]
3375 fn an_emit_set_holds_no_payload_bytes_of_the_pack_it_will_write() {
3376 let (_dir, store, all) = corpus("zc-memory");
3377 let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
3378
3379 let entries = store.emit_set(&oids, true, None).unwrap();
3380
3381 let owned: u64 = entries
3382 .iter()
3383 .map(|e| e.stored.owned().map_or(0, |b| b.len() as u64))
3384 .sum();
3385 let will_emit: u64 = entries.iter().map(|e| e.stored.len()).sum();
3386 assert!(
3387 will_emit > 1_000_000,
3388 "the corpus emits {will_emit} bytes, too few to distinguish held from addressed"
3389 );
3390 assert_eq!(
3391 owned,
3392 0,
3393 "the emit set owns {owned} bytes of payload — {:.1} % of the {will_emit} bytes it will \
3394 emit — which is the whole pack held in memory before the first byte goes out",
3395 100.0 * owned as f64 / will_emit as f64
3396 );
3397
3398 // The complete heap of the set, term by term.
3399 let slots = (entries.len() * std::mem::size_of::<EmitEntry>()) as u64;
3400 let oid_bytes: u64 = entries.iter().map(|e| e.oid.len() as u64).sum();
3401 let held = slots + oid_bytes + owned;
3402 assert!(
3403 held < will_emit / 8,
3404 "the emit set holds {held} bytes ({:.1} % of the {will_emit}-byte pack): {slots} of \
3405 slots, {oid_bytes} of oids, {owned} of payload",
3406 100.0 * held as f64 / will_emit as f64
3407 );
3408 eprintln!(
3409 "{} entries: {slots} B slots + {oid_bytes} B oids + {owned} B payload = {held} B held \
3410 for a {will_emit} B pack ({:.1} %). EmitEntry is {} B. Projected to linux.git's \
3411 ~13.8 M objects that residual is ~{:.2} GB — the payload copy is gone, the per-entry \
3412 slot and oid are NOT.",
3413 entries.len(),
3414 100.0 * held as f64 / will_emit as f64,
3415 std::mem::size_of::<EmitEntry>(),
3416 13.8e6 * (std::mem::size_of::<EmitEntry>() as f64 + 20.0) / 1e9,
3417 );
3418 }
3419}
3420
3421/// **A `REF_DELTA` whose base is inside the same pack**, on both sides of the
3422/// store: ingesting one, and serving one.
3423///
3424/// # Why this shape and no other
3425///
3426/// It is not exotic and it is not hand-rolled. `git index-pack --fix-thin`
3427/// completes a pushed **thin** pack by *appending the base object to the pack*
3428/// and leaving the delta naming that base by oid — so the base lands **after**
3429/// its dependant, in the same pack, named the one way a pack can name something
3430/// that is not a position. Every repository that has been pushed to more than
3431/// once and not `gc`'d holds packs of this shape, which is to say: the ordinary
3432/// case, not a corner.
3433///
3434/// The fixture is therefore built by **git itself** (`pack-objects --thin` fed
3435/// to `index-pack --fix-thin`) rather than assembled here, because a hand-built
3436/// pack could only ever prove that this crate agrees with this crate.
3437///
3438/// Two defects met on it, one per direction:
3439///
3440/// | direction | defect | fix |
3441/// |---|---|---|
3442/// | in | `external_bases_exist` demanded every ref base be in the **store**, so a pack carrying its own base was refused | ask the pack too, and only when about to refuse |
3443/// | out | `emit_set` copied the stored `REF_DELTA` through, and **gitoxide cannot read one whose base is in the pack** | re-head it as an `OFS_DELTA` naming the same base by distance |
3444#[cfg(test)]
3445mod in_pack_ref_delta_tests {
3446 use crate::index_layout::ObjType;
3447 use crate::pack_walk::{DeltaBase, topological_order};
3448 use crate::store::tests::tmpdir;
3449 use crate::{GitOps, GitStore};
3450 use std::path::Path;
3451 use std::process::Command;
3452
3453 fn git(dir: &Path, args: &[&str]) -> String {
3454 let out = Command::new("git")
3455 .current_dir(dir)
3456 .args(args)
3457 .env("GIT_AUTHOR_NAME", "t")
3458 .env("GIT_AUTHOR_EMAIL", "t@t")
3459 .env("GIT_COMMITTER_NAME", "t")
3460 .env("GIT_COMMITTER_EMAIL", "t@t")
3461 .env("GIT_AUTHOR_DATE", "2020-01-01T00:00:00Z")
3462 .env("GIT_COMMITTER_DATE", "2020-01-01T00:00:00Z")
3463 .output()
3464 .unwrap_or_else(|e| panic!("running git {args:?}: {e}"));
3465 assert!(
3466 out.status.success(),
3467 "git {args:?} failed: {}",
3468 String::from_utf8_lossy(&out.stderr)
3469 );
3470 String::from_utf8_lossy(&out.stdout).trim().to_string()
3471 }
3472
3473 /// A pack **git wrote** that contains a `REF_DELTA` whose base is an entry
3474 /// of the same pack.
3475 ///
3476 /// `None` when there is no `git` on `PATH`, so the suite still runs where
3477 /// the oracle is absent rather than failing for the wrong reason.
3478 fn fix_thin_pack(scratch: &Path) -> Option<Vec<u8>> {
3479 if Command::new("git").arg("--version").output().is_err() {
3480 eprintln!("skipping: no git on PATH");
3481 return None;
3482 }
3483 let repo = scratch.join("src");
3484 std::fs::create_dir_all(&repo).unwrap();
3485 git(&repo, &["init", "-q", "--initial-branch=main"]);
3486
3487 // Big enough, and changed little enough, that `pack-objects` really does
3488 // deltify the second version against the first. A two-line file would be
3489 // stored whole and this fixture would prove nothing — which the premise
3490 // assertion below refuses to let happen silently.
3491 let lines: Vec<String> = (0..400)
3492 .map(|i| format!("line {i} {}", "x".repeat(20)))
3493 .collect();
3494 std::fs::write(repo.join("f.txt"), lines.join("\n")).unwrap();
3495 git(&repo, &["add", "f.txt"]);
3496 git(&repo, &["commit", "-q", "-m", "c1"]);
3497
3498 let mut changed = lines.clone();
3499 changed[10] = "CHANGED".to_string();
3500 changed.push("appended line".to_string());
3501 std::fs::write(repo.join("f.txt"), changed.join("\n")).unwrap();
3502 git(&repo, &["commit", "-q", "-a", "-m", "c2"]);
3503
3504 // A THIN pack: c2's objects only, deltified against c1's, which it does
3505 // not carry.
3506 let head = git(&repo, &["rev-parse", "HEAD"]);
3507 let parent = git(&repo, &["rev-parse", "HEAD~1"]);
3508 let thin = {
3509 use std::io::Write as _;
3510 let mut child = Command::new("git")
3511 .current_dir(&repo)
3512 .args(["pack-objects", "--thin", "--revs", "--stdout"])
3513 .stdin(std::process::Stdio::piped())
3514 .stdout(std::process::Stdio::piped())
3515 .stderr(std::process::Stdio::piped())
3516 .spawn()
3517 .unwrap();
3518 write!(child.stdin.take().unwrap(), "{head}\n^{parent}\n").unwrap();
3519 let out = child.wait_with_output().unwrap();
3520 assert!(out.status.success(), "git pack-objects --thin failed");
3521 out.stdout
3522 };
3523
3524 // `--fix-thin` appends the bases INTO the pack. The ref-deltas keep
3525 // naming them by oid, and now those oids ARE in the pack.
3526 let idx = scratch.join("fixed.idx");
3527 {
3528 use std::io::Write as _;
3529 let mut child = Command::new("git")
3530 .current_dir(&repo)
3531 .args([
3532 "index-pack",
3533 "--fix-thin",
3534 "--stdin",
3535 "-o",
3536 idx.to_str().unwrap(),
3537 ])
3538 .stdin(std::process::Stdio::piped())
3539 .stdout(std::process::Stdio::piped())
3540 .stderr(std::process::Stdio::piped())
3541 .spawn()
3542 .unwrap();
3543 child.stdin.take().unwrap().write_all(&thin).unwrap();
3544 let out = child.wait_with_output().unwrap();
3545 assert!(
3546 out.status.success(),
3547 "git index-pack --fix-thin failed: {}",
3548 String::from_utf8_lossy(&out.stderr)
3549 );
3550 }
3551 let packdir = repo.join(".git/objects/pack");
3552 let pack = std::fs::read_dir(&packdir)
3553 .unwrap()
3554 .flatten()
3555 .map(|e| e.path())
3556 .find(|p| p.extension().is_some_and(|x| x == "pack"))
3557 .expect("git index-pack --fix-thin wrote no pack");
3558 Some(std::fs::read(pack).unwrap())
3559 }
3560
3561 /// The fixture is what it claims to be: at least one `REF_DELTA`, and every
3562 /// delta base resolvable **inside the pack alone**.
3563 ///
3564 /// Without this the two tests below could pass on a pack with no ref-delta
3565 /// in it at all, which is the vacuous green LAW 2 is about.
3566 fn assert_carries_its_own_ref_delta_base(pack: &[u8]) {
3567 let w = crate::pack_walk::walk(pack, 20).expect("git's own pack must walk");
3568 let refs = w
3569 .entries
3570 .iter()
3571 .filter(|e| e.obj_type == ObjType::RefDelta)
3572 .count();
3573 assert!(
3574 refs > 0,
3575 "the fixture has no ref-delta at all, so it cannot exercise either defect"
3576 );
3577 // Resolvable with NO external base source at all: every base a delta in
3578 // here names is carried in here.
3579 crate::resolve::resolve(
3580 pack,
3581 crate::GitHashKind::Sha1,
3582 0,
3583 &crate::resolve::NoBases,
3584 )
3585 .expect("the fixture must be self-contained, or `put_pack` is right to refuse it");
3586 }
3587
3588 /// 🔴 **znippy ingests a pack git wrote.**
3589 ///
3590 /// `pack_walk::closure()` reports every `REF_DELTA` base on `external_refs`
3591 /// — it cannot do otherwise, a walk knows no oids — and
3592 /// `external_bases_exist` demanded each one already be in the store. A pack
3593 /// carrying its own base was therefore refused, which is `git index-pack
3594 /// --fix-thin`'s ordinary output and `gunnar.multi_pack_serve`'s failure.
3595 ///
3596 /// Asserted on applied output, not on `Ok`: every object of the pack is
3597 /// afterwards *in the store's index*, so a `put_pack` that returned success
3598 /// having stored nothing would still be red.
3599 ///
3600 /// Seen RED by restoring the old check (refuse any ref base not already in
3601 /// the store):
3602 /// *"this pack deltas against 93c4207c152fa94c3978c75b136ff5a530ca16b6,
3603 /// which this repository does not have — the push is refused rather than
3604 /// stored with a dangling base"*.
3605 #[test]
3606 fn a_pack_carrying_its_own_ref_delta_base_is_ingested_not_refused() {
3607 let dir = tmpdir("inpack-put");
3608 let Some(pack) = fix_thin_pack(&dir) else {
3609 return;
3610 };
3611 assert_carries_its_own_ref_delta_base(&pack);
3612
3613 let store = GitStore::open(&dir.join("store"), "alice").unwrap();
3614 store
3615 .put_pack(&pack)
3616 .expect("a self-contained pack git itself wrote must be accepted");
3617 store.absorb_pending().unwrap();
3618
3619 let walked = crate::pack_walk::walk(&pack, 20).unwrap();
3620 let stored = store.index().oids_in_order().unwrap();
3621 assert_eq!(
3622 stored.len(),
3623 walked.entries.len(),
3624 "the pack has {} entries and the store indexed {}",
3625 walked.entries.len(),
3626 stored.len()
3627 );
3628 // And specifically the base that used to be the refusal.
3629 let base = walked
3630 .entries
3631 .iter()
3632 .find_map(|e| match &e.delta_base {
3633 DeltaBase::Ref(oid) => Some(oid.clone()),
3634 _ => None,
3635 })
3636 .expect("the premise asserted there is one");
3637 assert!(
3638 store.has(&base).unwrap(),
3639 "the in-pack ref-delta base {} is not in the store after the push",
3640 hex::encode(&base)
3641 );
3642 }
3643
3644 /// 🔴 **A pack this server emits is consumed by `gunnar_client`.**
3645 ///
3646 /// `gunnar_client` resolves a fetch with gitoxide, and
3647 /// `gix_pack::data::input::LookupRefDeltaObjectsIter` reads every
3648 /// `OBJ_REF_DELTA` as naming an object *the receiver already has*: it
3649 /// consults the local object database and the bases it has already spliced
3650 /// in, and never the pack it is reading. A clone's target holds nothing, so
3651 /// one stored ref-delta copied through is fatal for the **whole** pack.
3652 ///
3653 /// The receiver here is `gix_object::find::Never` — an object database that
3654 /// holds nothing, which is exactly `open_or_init_bare` at the moment a clone
3655 /// starts — so this is the client's own code answering, not a paraphrase of
3656 /// it.
3657 ///
3658 /// Three assertions, and all three are needed:
3659 ///
3660 /// 1. **gitoxide accepts it**, with an empty base source. This is the bug.
3661 /// 2. **stock git still accepts it.** git accepted the *broken* pack too,
3662 /// so a fix that satisfied gix and broke git would otherwise ship green.
3663 /// 3. **the object set is exactly what was asked for**, read back out of
3664 /// stock git by oid. A re-head that dropped, duplicated or corrupted an
3665 /// entry passes 1 and 2 — `index-pack` files an entry under the oid it
3666 /// computes from that entry's own bytes, so a wrong object is a
3667 /// *different* oid and only comparing the sets can see it.
3668 ///
3669 /// Seen RED, each separately — see the commit message for the exact texts.
3670 #[test]
3671 fn an_emitted_pack_names_no_base_inside_itself_by_oid() {
3672 let dir = tmpdir("inpack-emit");
3673 let Some(pack) = fix_thin_pack(&dir) else {
3674 return;
3675 };
3676 assert_carries_its_own_ref_delta_base(&pack);
3677
3678 let store = GitStore::open(&dir.join("store"), "alice").unwrap();
3679 store.put_pack(&pack).unwrap();
3680 store.absorb_pending().unwrap();
3681
3682 let all = store.index().oids_in_order().unwrap();
3683 let oids: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
3684 let (ordered, missing) = topological_order(store.emit_set(&oids, true, None).unwrap());
3685 assert!(
3686 missing.is_empty(),
3687 "an entry still names a base that is not here: {missing:?}"
3688 );
3689 let mut emitted = Vec::new();
3690 store.emit_ordered(&ordered, &mut emitted).unwrap();
3691
3692 // ── the premise: the emitted pack really does carry a delta, or nothing
3693 // below distinguishes a fix from a pack of whole objects ────────────
3694 let w = crate::pack_walk::walk(&emitted, 20).expect("our own walk reads what we emitted");
3695 let deltas = w
3696 .entries
3697 .iter()
3698 .filter(|e| !matches!(e.delta_base, DeltaBase::None))
3699 .count();
3700 assert!(
3701 deltas > 0,
3702 "the emitted pack has no delta at all, so it cannot show how a base is named"
3703 );
3704
3705 // ── 1. THE BUG: no base may be named by oid ───────────────────────────
3706 let named_by_oid = w.closure().external_refs;
3707 assert!(
3708 named_by_oid.is_empty(),
3709 "a clone's pack names {} base(s) by oid — gitoxide resolves those against the \
3710 receiver's store and a clone's receiver is empty: {:?}",
3711 named_by_oid.len(),
3712 named_by_oid.iter().map(hex::encode).collect::<Vec<_>>()
3713 );
3714
3715 // …and the client's own code says so, not only our reading of it.
3716 let out = dir.join("gix-out");
3717 std::fs::create_dir_all(&out).unwrap();
3718 let stop = std::sync::atomic::AtomicBool::new(false);
3719 let mut cur = std::io::Cursor::new(emitted.as_slice());
3720 let outcome = gix_pack::Bundle::write_to_directory(
3721 &mut cur,
3722 Some(&out),
3723 &mut gix_features::progress::Discard,
3724 &stop,
3725 // An object database that holds nothing — a fresh clone target.
3726 Some(gix_object::find::Never),
3727 gix_hash::Kind::Sha1,
3728 gix_pack::bundle::write::Options::default(),
3729 );
3730 let outcome = match outcome {
3731 Ok(o) => o,
3732 // gix 0.88 reports an `Exn`, not a `std::error::Error`.
3733 Err(e) => panic!(
3734 "gitoxide refused a pack this store emitted: {}",
3735 chain(&gix_error::Error::from(e))
3736 ),
3737 };
3738 assert_eq!(
3739 outcome.index.num_objects as usize,
3740 oids.len(),
3741 "gitoxide indexed {} objects for a request of {}",
3742 outcome.index.num_objects,
3743 oids.len()
3744 );
3745
3746 // ── 2. stock git must still accept it ─────────────────────────────────
3747 crate::git_oracle::assert_git_accepts(
3748 &dir,
3749 "stock.git",
3750 &emitted,
3751 crate::git_oracle::Strictness::SelfContained,
3752 );
3753
3754 // ── 3. exactly the object set that was asked for, by oid, read back out
3755 // of stock git ───────────────────────────────────────────────────
3756 let read_back = crate::git_oracle::git_reads_back(&dir, "readback.git", &emitted).unwrap();
3757 assert_eq!(
3758 read_back.len(),
3759 oids.len(),
3760 "the pack carries {} objects, the request asked for {}",
3761 read_back.len(),
3762 oids.len()
3763 );
3764 let got: std::collections::BTreeSet<String> =
3765 read_back.iter().map(|(oid, _)| oid.clone()).collect();
3766 let want: std::collections::BTreeSet<String> = oids.iter().map(hex::encode).collect();
3767 assert_eq!(
3768 got, want,
3769 "the emitted pack is not the requested set — missing {:?}, unexpected {:?}",
3770 want.difference(&got).collect::<Vec<_>>(),
3771 got.difference(&want).collect::<Vec<_>>()
3772 );
3773 }
3774
3775 /// **RED / GREEN perf probe** for the per-push fold cost (lane ZNIPPY-WRITE).
3776 ///
3777 /// Not part of the correctness suite (`#[ignore]`) — a manual measurement:
3778 /// ```text
3779 /// cargo test -p znippy-plugin-git --release the_incremental_push_fold_is_flat -- --ignored --nocapture
3780 /// ```
3781 /// Warms a store to `n` commits, then pushes a fixed `d`-commit delta and
3782 /// times `put_pack + absorb_pending` (the delta resolve is ~`3d` objects,
3783 /// so the number is dominated by the fold). Separately times a *forced*
3784 /// whole-store `refold` (graph cache removed) to isolate the fold from the
3785 /// resolve. The RED reading is both columns growing with `n`; the GREEN
3786 /// reading (after the incremental fold) is `delta_absorb_ms` flat in `n`.
3787 #[test]
3788 #[cfg(feature = "bench-kernels")]
3789 #[ignore = "manual perf probe; run with --features bench-kernels --ignored --nocapture"]
3790 fn the_incremental_push_fold_is_flat() {
3791 use crate::object::GitObjectKind;
3792 fn split(obj: &[u8]) -> (GitObjectKind, Vec<u8>) {
3793 let nul = obj.iter().position(|&b| b == 0).expect("canonical header");
3794 let kind = match std::str::from_utf8(&obj[..nul])
3795 .unwrap()
3796 .split(' ')
3797 .next()
3798 .unwrap()
3799 {
3800 "commit" => GitObjectKind::Commit,
3801 "tree" => GitObjectKind::Tree,
3802 "blob" => GitObjectKind::Blob,
3803 "tag" => GitObjectKind::Tag,
3804 other => panic!("unknown kind {other}"),
3805 };
3806 (kind, obj[nul + 1..].to_vec())
3807 }
3808 use std::time::Instant;
3809 let d = 8usize;
3810 println!("n\tcommits\tforced_refold_ms\tdelta_absorb_ms\tdelta_objs");
3811 for &n in &[256usize, 1024, 4096, 8192] {
3812 let repo = crate::bench_kernels::synth_repo(
3813 n + d,
3814 4,
3815 0,
3816 crate::object::GitHashKind::Sha256,
3817 );
3818 // synth_repo appends blob,tree,commit per commit, in order.
3819 let split_at = 3 * n;
3820 let base: Vec<(GitObjectKind, Vec<u8>)> =
3821 repo.objects[..split_at].iter().map(|(_, b)| split(b)).collect();
3822 let delta: Vec<(GitObjectKind, Vec<u8>)> =
3823 repo.objects[split_at..].iter().map(|(_, b)| split(b)).collect();
3824 let delta_objs = delta.len();
3825 let pack_base = crate::pack_fixture::whole_object_pack(&base);
3826 let pack_delta = crate::pack_fixture::whole_object_pack(&delta);
3827
3828 let dir = tmpdir(&format!("fold-scale-{n}"));
3829 let store = GitStore::open(&dir, "alice").unwrap();
3830 store.put_pack(&pack_base).unwrap();
3831 store.absorb_pending().unwrap();
3832
3833 // Isolated fold: drop the persisted graph so refold must recompute,
3834 // exactly what a push's token change forces.
3835 let _ = std::fs::remove_file(store.root().join("objects.graph"));
3836 let t = Instant::now();
3837 store.refold().unwrap();
3838 let forced_refold_ms = t.elapsed().as_secs_f64() * 1000.0;
3839
3840 // Real incremental push.
3841 store.put_pack(&pack_delta).unwrap();
3842 let t2 = Instant::now();
3843 store.absorb_pending().unwrap();
3844 let delta_absorb_ms = t2.elapsed().as_secs_f64() * 1000.0;
3845
3846 let commits = store.commit_count();
3847 assert_eq!(commits, n + d, "graph must hold every commit");
3848 println!(
3849 "{n}\t{commits}\t{forced_refold_ms:.2}\t{delta_absorb_ms:.2}\t{delta_objs}"
3850 );
3851 }
3852 }
3853
3854 /// **RSS floor attribution probe** (lane ZNIPPY-WRITE). Opens `k` warm
3855 /// stores over the corpus pack, once with the default redb page-cache
3856 /// ceiling and once with a small one, and reports resident-set growth per
3857 /// store — the discriminator for whether the per-store floor is the redb
3858 /// cache. `#[ignore]`, manual:
3859 /// ```text
3860 /// cargo test -p znippy-plugin-git --release the_store_floor_rss -- --ignored --nocapture
3861 /// ```
3862 #[test]
3863 #[ignore = "manual RSS probe; run with --ignored --nocapture"]
3864 fn the_store_floor_rss() {
3865 use crate::arms::StoreConfig;
3866 fn vmrss_kb() -> u64 {
3867 let s = std::fs::read_to_string("/proc/self/status").unwrap_or_default();
3868 for l in s.lines() {
3869 if let Some(v) = l.strip_prefix("VmRSS:") {
3870 return v.trim().trim_end_matches(" kB").trim().parse().unwrap_or(0);
3871 }
3872 }
3873 0
3874 }
3875 fn threads() -> usize {
3876 std::fs::read_dir("/proc/self/task").map(|d| d.count()).unwrap_or(0)
3877 }
3878 let pack = crate::pack_fixture::corpus().pack.clone();
3879 let k = 16usize;
3880 for (label, cache) in [("default(64MiB)", 64 << 20), ("small(1MiB)", 1 << 20)] {
3881 let base = vmrss_kb();
3882 let t0 = threads();
3883 let mut stores = Vec::with_capacity(k);
3884 for i in 0..k {
3885 let dir = tmpdir(&format!("floor-{label}-{i}"));
3886 let store = GitStore::<crate::index_layout::OneTableFourColumns>::open_with_arms(
3887 &dir,
3888 "alice",
3889 crate::object::GitHashKind::Sha1,
3890 StoreConfig::DEFAULT.with_redb_cache_bytes(cache),
3891 )
3892 .unwrap();
3893 store.put_pack(&pack).unwrap();
3894 store.absorb_pending().unwrap();
3895 // Warm the cache the way a served repo is.
3896 for r in store.refs().unwrap() {
3897 if let Some(oid) = r.oid {
3898 let _ = store.has(&oid);
3899 }
3900 }
3901 stores.push(store);
3902 }
3903 let live = vmrss_kb();
3904 let per = (live.saturating_sub(base)) as f64 / k as f64 / 1024.0;
3905 println!(
3906 "FLOOR cache={label} k={k} rss_growth={:.1}MB per_store={:.2}MB threads_delta={}",
3907 (live.saturating_sub(base)) as f64 / 1024.0,
3908 per,
3909 threads().saturating_sub(t0)
3910 );
3911 drop(stores);
3912 }
3913 }
3914
3915 /// gix errors nest their real cause; the top line alone names the operation
3916 /// and nothing about why. The same walk `gunnar-client::error::chain` does,
3917 /// and here for the same reason: without it the red above reads *"Failed to
3918 /// write pack"* and diagnoses nothing.
3919 fn chain(err: &(dyn std::error::Error + 'static)) -> String {
3920 let mut out = err.to_string();
3921 let mut cursor = err.source();
3922 for _ in 0..16 {
3923 let Some(next) = cursor else { break };
3924 let text = next.to_string();
3925 if !out.ends_with(&text) {
3926 out.push_str(": ");
3927 out.push_str(&text);
3928 }
3929 cursor = next.source();
3930 }
3931 out
3932 }
3933}