znippy_plugin_git/git_ops.rs
1//! **`GitOps` — ours, typed, and base znippy never sees it.**
2//!
3//! §13.2: `ArchiveTypePlugin` is base znippy's contract and
4//! [`crate::NativeGitPlugin`] implements it — base stores, the plugin annotates,
5//! exactly as maven, python, rust-toolchain and media already work. `GitOps` is
6//! the *other* trait: the one gunnar calls, with git's own vocabulary in it.
7//!
8//! §13.3: `ArchiveTypePlugin::run_command(cmd, args)` is **CLI-only string
9//! dispatch** and delegates here. A server's hot path does not go through
10//! stringly-typed dispatch, so nothing below takes a `&str` command or an
11//! `args: &[String]`.
12//!
13//! ```text
14//! gunnar ──typed──► GitOps (this file, twelve methods)
15//! │
16//! ┌────────────────┼──────────────────┬─────────────┐
17//! ▼ ▼ ▼ ▼
18//! ArchiveWrite ObjectIndex RefLog Gc
19//! SafeWriter ObjectReadStack push log NewGeneration
20//! (blob fsync, (stree → Arrow (Arrow IPC (link, compact,
21//! journal row) → redb tail) frame = tx) verify, rename)
22//! ```
23//!
24//! Every arm under that diagram already existed and was already measured. This
25//! file adds no storage, no index and no durability mechanism of its own — if a
26//! method here looks like it is inventing one, that is a bug.
27//!
28//! # The one thing that is NOT here
29//!
30//! `run_command` still lives on the plugin in [`crate::lib`], and it now
31//! delegates. Nothing git-shaped went into base znippy for any of this.
32
33use std::collections::{BTreeMap, HashMap};
34use std::fs::File;
35use std::path::{Path, PathBuf};
36use std::sync::{Arc, Mutex, RwLock};
37
38use anyhow::{anyhow, bail, Context, Result};
39use znippy_common::ReservedSection;
40
41use crate::arms::StoreConfig;
42use crate::exploded::ExplodedStats;
43use crate::exploded_cache::ExplodedCache;
44use crate::gc::{Gc, GcReport};
45use crate::graph::{assign_generations, CommitNode};
46use crate::index_layout::{ObjectIndex, OneTableFourColumns};
47use crate::indexer::{IndexJob, ObjectAbsorb, PushPath};
48use crate::object::{GitHashKind, GitObjectKind};
49use crate::pack_walk::PackWalk;
50use crate::read_stack::{ObjectReadStack, RebuildTriggers};
51use crate::refs::{RefLog, RefUpdate};
52use crate::resolve::BaseSource;
53
54// ── the contract, now owned by `git-storage-trait` ────────────────────────────
55//
56// `GitOps` (the neutral ELEVEN) and its value types live in the
57// `git-storage-trait` crate at this repo's root, so a gix backend
58// (edda/storage-git-gix) can implement the same contract without linking Arrow
59// or OpenZL. The twelfth method, `seal()`, returns `znippy_common::
60// ReservedSection` (an Arrow type a gix backend has no analog for), so it is an
61// **inherent** method on [`GitStore`] / [`SelectedStore`], not part of the
62// trait. The re-exports below keep every existing path
63// (`znippy_plugin_git::git_ops::{GitOps, Oid, …}`) alive.
64pub use git_storage_trait::{Extent, GitOps, Oid, RefRow, Stored, TxId};
65
66
67// ── the batch shape, which is measured and must not be papered over ───────────
68
69/// Which path a lookup of `n` oids takes.
70///
71/// The number this dispatch was written for is `index_layout_bench`'s: on the bare
72/// Arrow arms, `lookup_batch` with a **single** element costs 818 ns against
73/// `lookup`'s 482 ns — 1.7x worse — because the batch path allocates a result
74/// vector, a key vector and a second pass over them to serve one oid.
75///
76/// **RE-MEASURED against the shipping read stack, and the magnitude does not
77/// carry over.** oden 2026-08-07, release, 1-min loadavg 4.94 (another tenant's
78/// work — see LAW: never measure on a busy box, and read these as a ratio rather
79/// than as absolutes), `ObjectReadStack<OneTableFourColumns>` over a real
80/// repository's 2687-object pack, 20 000 iterations
81/// ([`crate::store::tests::the_batch_of_one_really_is_slower_than_the_serial_path`]):
82///
83/// | call | per oid |
84/// |---|---:|
85/// | `lookup` | **628 ns** |
86/// | `lookup_batch` with 1 oid | **639 ns** — 1.02x worse |
87/// | `lookup_batch` with 100 oids | **161 ns** — 3.9x better |
88///
89/// So: the *direction* holds and the dispatch is right — a batch of one is never
90/// faster — but on this stack at this size it costs 2%, not 70%. The real finding
91/// is the third row: the batch path is worth **3.9x** at
92/// [`BATCH_SATURATES_AT`], which is a much stronger reason to keep the two paths
93/// apart than the batch-of-one penalty ever was. Both numbers are stated because
94/// quoting only the 1.7x would be quoting a figure this stack does not reproduce.
95///
96/// The dispatch is a named function with its own guard precisely so it cannot
97/// quietly become `lookup_batch` for every n.
98#[derive(Debug, Clone, Copy, PartialEq, Eq)]
99pub enum LookupPath {
100 /// One oid: `ObjectIndex::lookup`.
101 Serial,
102 /// Two or more: `ObjectIndex::lookup_batch`, one pass.
103 Batch,
104}
105
106/// Oids at which the batch path stops getting faster. Reported, not enforced:
107/// splitting a bigger batch would cost a pass and buy nothing.
108pub const BATCH_SATURATES_AT: usize = 100;
109
110/// The dispatch, in one place.
111pub fn lookup_path(n: usize) -> LookupPath {
112 if n == 1 {
113 LookupPath::Serial
114 } else {
115 LookupPath::Batch
116 }
117}
118
119// ── the store ─────────────────────────────────────────────────────────────────
120
121/// A pack whose bytes are durable but whose objects the index has not absorbed.
122///
123/// §13.12's `indexed` bit, in the form this store needs it: membership in
124/// [`PackState::pending`] *is* the bit being clear. A read that arrives while it
125/// is non-empty falls back to absorbing it — slower, never wrong.
126#[derive(Debug, Clone, Copy, PartialEq, Eq)]
127struct PendingPack {
128 pack_id: u64,
129 offset: u64,
130 len: u64,
131 /// The journal queued it on open (a restart), not a push in this process.
132 /// What makes the progress row say `resumed 0%` after a reboot.
133 resumed: bool,
134}
135
136impl PendingPack {
137 fn from_job(j: IndexJob) -> Self {
138 Self {
139 pack_id: j.pack_id,
140 offset: j.offset,
141 len: j.len,
142 resumed: false,
143 }
144 }
145}
146
147/// **§13.12's `indexed` bit — one bit per pack, indexed by pack ordinal.**
148///
149/// A plain bitset, and the two rejected alternatives say why:
150///
151/// * **not a hash set.** Pack ordinals are dense and small: they are assigned
152/// `0..N` within **one repository** (§13.14 scopes a negotiation, and therefore
153/// this store, to one repository — account-wide they would be sparse and the
154/// array would grow with the account's total packs). Dense means the bit is one
155/// shift and one AND with no hashing, and 10 000 packs is **1250 bytes** of
156/// `words`, which lives in L1.
157/// * **not a bloom filter.** A bloom's false positive would be *actively wrong*
158/// here: it would report a pack absorbed when it is not, the read path would
159/// stop falling back, and negotiation would be told `absent` about an object
160/// whose bytes are durable — which makes a client withhold objects and lose
161/// data. The bitset has no such mode. `1` means absorbed, `0` means fall back,
162/// and both are exact.
163///
164/// **The bit is never stored.** It is derived on open by diffing two facts that
165/// are already durable — the journal's extents against the index's rows — in
166/// [`Absorber::adopt_journal`]. There is nothing for a derived bit to drift from.
167///
168/// **Why the extents sit beside it.** A `0` bit is only actionable with the
169/// extent to absorb, so `extent[i]` carries it; `None` means "this store has not
170/// been told about ordinal `i`", which is not the same as "not absorbed" and must
171/// not be counted as work.
172///
173/// **Why `absorbed` is a separate fact from `extent`.** The ack path does two
174/// things in this order: record the pack pending, then hand its extent to the
175/// account's channel. The indexer's worker is a different thread and can reach
176/// `absorb` *before* the first of those returns. With only a pending list, that
177/// race ends in a bit that is set after the rows are already in and is never
178/// cleared, so every later read pays a pointless fallback for the life of the
179/// process. The bit is what the two writers agree on: [`PackState::note`] will
180/// not queue a pack that is already done, and `absorb` will not do the work
181/// twice.
182///
183/// Touched only under short critical sections — never across an absorb, which is
184/// what the separate [`Absorber::gate`] is for. **The ack path must never block
185/// behind index work.**
186#[derive(Default)]
187struct PackState {
188 /// One bit per pack ordinal, 64 to a word. `1` = its rows are in the index.
189 words: Vec<u64>,
190 /// The extent of each known ordinal, so a `0` bit can be acted on.
191 extent: Vec<Option<Extent>>,
192 /// Known ordinals whose bit is `0` — what [`GitStore::unindexed_packs`]
193 /// reports, kept as a count so the read path's check is a field load rather
194 /// than a scan.
195 unabsorbed: usize,
196}
197
198impl PackState {
199 /// The bit.
200 fn is_absorbed(&self, pack_id: u64) -> bool {
201 let i = pack_id as usize;
202 self.words
203 .get(i / 64)
204 .is_some_and(|w| w >> (i % 64) & 1 == 1)
205 }
206
207 /// Record a durable pack whose rows are not in yet. The bit **going clear**.
208 ///
209 /// **Idempotent, and the extent slot is the single thing that makes it so.**
210 /// A second `note` for an ordinal that already has one — the ack path
211 /// arriving after the drain has already absorbed the pack, or a re-queue on
212 /// open racing the same pack's push — finds the slot filled and counts no new
213 /// work. [`PackState::mark_absorbed`] fills that same slot, so a pack that is
214 /// already done cannot be queued behind its own completion, and there is no
215 /// second condition here that could disagree with the first.
216 fn note(&mut self, pack_id: u64, extent: Extent) {
217 let i = pack_id as usize;
218 self.grow_to(i);
219 if self.extent[i].is_none() {
220 self.extent[i] = Some(extent);
221 self.unabsorbed += 1;
222 }
223 }
224
225 /// Room for ordinal `i` in both arrays. Dense ordinals are what makes this a
226 /// `resize` and not an insert.
227 fn grow_to(&mut self, i: usize) {
228 if self.words.len() <= i / 64 {
229 self.words.resize(i / 64 + 1, 0);
230 }
231 if self.extent.len() <= i {
232 self.extent.resize(i + 1, None);
233 }
234 }
235
236 /// The bit **going up**: this pack's rows are in the index.
237 fn mark_absorbed(&mut self, pack_id: u64, extent: Extent) {
238 let i = pack_id as usize;
239 self.grow_to(i);
240 if self.words[i / 64] >> (i % 64) & 1 == 0 {
241 self.words[i / 64] |= 1 << (i % 64);
242 if self.extent[i].is_some() {
243 self.unabsorbed -= 1;
244 }
245 }
246 self.extent[i] = Some(extent);
247 }
248
249 /// Every known pack whose bit is clear, in ordinal order.
250 fn pending(&self) -> Vec<PendingPack> {
251 self.extent
252 .iter()
253 .enumerate()
254 .filter(|(i, e)| e.is_some() && !self.is_absorbed(*i as u64))
255 .map(|(i, e)| {
256 let (offset, len) = e.expect("filtered on Some");
257 PendingPack {
258 pack_id: i as u64,
259 offset,
260 len,
261 resumed: true,
262 }
263 })
264 .collect()
265 }
266}
267
268/// The derived tables: the commit graph, the tree payloads the reachability
269/// bitmaps are built from, and this store's own ordinal space.
270///
271/// **The ordinal space is this struct's, not the projection's.** An
272/// [`crate::index_layout::IndexRow::ordinal`] is a row address inside one
273/// projection generation and is re-derived by every rebuild, so a bitmap built
274/// over projection ordinals would silently address different objects after any
275/// rebuild. The bitmaps here are over `ordinal`, which is rebuilt in the same
276/// fold that rebuilds them — the two cannot drift because one call produces both.
277/// **Nothing in here is a source of truth.** Every field is folded from the
278/// `objects` table and §14's exploded table by [`Absorber::refold`], and both of
279/// those are on disk — which is what makes a clean reopen come back with the
280/// same graph it shut down with. Before the exploded table existed, `graph` and
281/// `trees` were *accumulated* here as packs were absorbed and stored nowhere, so
282/// a clean shutdown lost them silently: MEASURED on oden 2026-08-08, a reopened
283/// store over a fully absorbed 2687-object pack had all 2687 rows and **0 of 551
284/// commits**, with `reachable()` returning a commit alone instead of its closure
285/// and `gc` seeing an empty live set.
286#[derive(Default)]
287struct Derived {
288 /// Parents-before-children, generations assigned. Folded from the exploded
289 /// table's commit payloads at open, and **extended** after a push: the
290 /// previous rows plus the pushed commits, re-numbered by
291 /// [`crate::graph::assign_generations`]. The hex spelling is kept for the
292 /// sealed graph section and `graph_snapshot`; nothing that serves a request
293 /// reads it.
294 graph: Vec<CommitNode>,
295 /// [`Derived::graph`] in integers — rows, ordinals, parent rows — rebuilt
296 /// beside it at every fold with one hex decode per node, so no serving path
297 /// touches a hex string. Behind an `Arc` so a negotiation borrows it with a
298 /// refcount bump and a fold that lands mid-answer replaces the pointer, not
299 /// the rows under the reader.
300 ord_graph: Arc<crate::reach::OrdGraph>,
301 /// **The ordinal space, and it is append-only.**
302 ///
303 /// A sorted base laid down by the first fold, then every object a later
304 /// push brings in appended behind it ([`crate::reach::OrdinalTable`]). The
305 /// sorted space this replaced shifted every ordinal above the new oids on
306 /// every push, so every stored bitmap had to be rewritten bit by bit
307 /// (`carry_reach`: 512 entries × every object of the repository, after a
308 /// one-commit push — that is the 4 s of CPU GUN-PERF's Z-2 measured behind
309 /// a 3-object fetch on git.git). Here a push appends and the table stands.
310 ///
311 /// The space is re-sorted only when the appended part outgrows
312 /// [`ORDINAL_COMPACT_FRACTION`] of the base or the store seals, and then
313 /// through ONE remap table ([`crate::reach::OrdinalTable::remap_to`]) —
314 /// one lookup per object, one array index per bit.
315 ordinals: crate::reach::OrdinalTable,
316 /// The live reachability table over `ordinals`, or empty if nothing has
317 /// needed it yet. Behind an `Arc` (2026-08-14, a measurement, not a
318 /// tidy-up: the negotiation used to deep-copy every bitmap twice per
319 /// request). Survives every fold; see [`Absorber::refold`].
320 reach: Arc<crate::reach::ReachTable>,
321 /// row → entry index into [`Derived::reach`], rebuilt with `ord_graph` at
322 /// every fold so the pair can never be out of step.
323 reach_by_row: Arc<Vec<u32>>,
324 /// blake3 over the sorted oid list the last fold saw — the token the
325 /// persisted `objects.graph` is validated against.
326 token: [u8; 32],
327 /// Verbatim packs, so any absorbed object's content can be re-derived from
328 /// the truth on demand. `(archive offset, len)`.
329 packs: Vec<(u64, u64)>,
330}
331
332/// **The compaction rule**: a fold re-sorts the ordinal space when the appended
333/// part is larger than this fraction of the base (and than
334/// [`ORDINAL_COMPACT_MIN`] objects), renumbering the reachability table once
335/// through one remap table. Until then every push appends and touches no
336/// bitmap. A quarter keeps the appended index small (it is searched separately,
337/// through a sorted index) and the renumbering rare.
338pub(crate) const ORDINAL_COMPACT_FRACTION: usize = 4;
339/// The floor under the rule: a tiny repository is never compacted for a
340/// handful of objects.
341pub(crate) const ORDINAL_COMPACT_MIN: usize = 4096;
342/// **How often an extended table is written back.** The persisted table stays
343/// valid across pushes it does not know about — a load `follow`s the store's
344/// current oid list and extends for the commits it lacks — so writing it after
345/// every push would buy nothing but the write. A full build and a compaction
346/// always persist; an extension persists every this-many folds, and the drop
347/// of the store persists whatever is left.
348pub(crate) const REACH_PERSIST_EVERY: u64 = 16;
349/// The delta-chain memo a full reachability build reads its trees through
350/// ([`crate::resolve::ChainCache`]): bounded, FIFO by bytes, dropped with the
351/// build. 128 MiB holds the last few thousand materialised trees of a packed
352/// repository, which is where the next chain's base is.
353pub(crate) const REACH_BUILD_CHAIN_CACHE_BYTES: usize = 128 << 20;
354/// …and no more than this per object the store holds (floor 8 MiB), so a
355/// small repository's build does not reserve a large repository's memo.
356pub(crate) const REACH_BUILD_CHAIN_CACHE_PER_OBJECT: usize = 512;
357
358/// **The object-level index, and the one thing that writes into it.**
359///
360/// Its own allocation rather than a field group inside [`GitStore`], and §13.9-12
361/// is the only reason: the drain runs on the account indexer's worker thread, so
362/// whatever it absorbs into cannot be reachable solely through a `&GitStore`.
363/// Splitting it out is what lets the background worker and a falling-back read
364/// call **the same** absorb instead of two copies of it (LAW 5) — the alternative
365/// was a second implementation on the worker side, which is precisely the
366/// twinning that law forbids.
367///
368/// It is a **leaf**: nothing in here points back at the store, so the worker's
369/// `Arc` closes no cycle and the store still drops (and joins its worker)
370/// normally.
371/// Entries below which [`GitStore::resolve_emit_payloads`] does **not** fan out.
372///
373/// A `std::thread::scope` spawn is ~10–20 µs a worker and the per-entry work it
374/// would divide is a bounds check plus a varint; a 100-object fetch would pay
375/// four spawns to save four microseconds of parsing. The threshold is what keeps
376/// `crate::serve`'s "one core's worth per transfer" true for every request that
377/// is not a bulk clone — and a bulk clone is the only request the copy this
378/// replaces was ever measured on.
379const FAN_OUT_AT: usize = 4096;
380
381/// The folded commit graph, cached beside the verbatim pack so a reopen *reads*
382/// it instead of re-deriving it from every commit payload (S-065).
383const GRAPH_CACHE_FILE: &str = "objects.graph";
384/// `objects.graph` framing: 4-byte magic + 1-byte version + 32-byte token, then
385/// the Arrow IPC commit graph.
386const GRAPH_CACHE_MAGIC: &[u8; 4] = b"GRPH";
387const GRAPH_CACHE_HEADER: usize = 4 + 1 + 32;
388
389/// The live reachability table and its ordinal space, cached beside the
390/// verbatim pack so a reopen *reads* them instead of walking every tree again
391/// (GUN-PERF Z-1: 85 to 150 s on git.git after every start). See
392/// [`Absorber::load_persisted_reach`] for the frame.
393const REACH_CACHE_FILE: &str = "objects.reach";
394const REACH_CACHE_MAGIC: &[u8; 4] = b"RECH";
395/// magic (4) + version (1) + oid_len (u32) + base bytes (u64) + extra bytes (u64).
396const REACH_CACHE_HEADER: usize = 4 + 1 + 4 + 8 + 8;
397
398/// **`objects.ordinal` is no longer written, and an existing one is removed at
399/// open.** It was S-082's memo of the ordinal space — every oid in sorted order,
400/// with a `(epoch, count)` header validated against the redb tail — so that a
401/// cold open could *read* the space instead of scanning the whole redb tail to
402/// rebuild it. Since 2026-09-04 the read stack persists its projection
403/// (`objects.index`) and the sorted oid list IS that index's oid column, mapped;
404/// [`ObjectReadStack::oids_raw`] hands it back with a merge of the few redb rows
405/// since the last fold. The scan the memo spared no longer exists, and a second
406/// file carrying the same bytes under a bespoke header is exactly the kind of
407/// file this store should not have. On the kernel mirror it was 278 964 902 B —
408/// 20.0 B/object, the oid column to the byte. The name is kept only so the
409/// stale file can be found and deleted.
410const ORDINAL_CACHE_FILE: &str = "objects.ordinal";
411
412/// The validity token for a persisted commit graph: blake3 over the sorted raw
413/// oids of **every** object in the store.
414///
415/// It is both necessary and sufficient. The graph is a pure, deterministic
416/// function of the commit payloads, which are content-addressed by their oids,
417/// so an identical object set yields a byte-identical graph — and this hashes
418/// the whole object set, in the one order `refold` already sorts it into, with
419/// no read the fold did not already make. A mismatch (any push, any gc) forces a
420/// recompute; a 256-bit blake3 makes an accidental match of two different object
421/// sets not a case worth engineering against.
422/// **`seen − live`: what a GC may drop.** `seen` is every oid the index held
423/// when the GC started (`oids_in_order`), `live` the reachable set computed
424/// after it. The difference is the dead set, and it is a POSITIVE set on
425/// purpose: `retain(|o| live.contains(o))` would also drop every row that
426/// arrived between the snapshot and the drop — a push in flight, whose refs
427/// are not updated yet — because to `live` those are simply unknown. A row in
428/// neither set is kept. `live` is hex, `seen` raw; the decode happens once
429/// here rather than once per row in the retain predicate.
430pub(crate) fn dead_oids(
431 seen: Vec<Vec<u8>>,
432 live: &std::collections::HashSet<String>,
433) -> std::collections::HashSet<Vec<u8>> {
434 let live_raw: std::collections::HashSet<Vec<u8>> =
435 live.iter().filter_map(|h| hex::decode(h).ok()).collect();
436 seen.into_iter().filter(|o| !live_raw.contains(o)).collect()
437}
438
439fn graph_token(oids_raw: &[u8]) -> [u8; 32] {
440 let mut h = znippy_common::blake3::Hasher::new();
441 h.update(oids_raw);
442 *h.finalize().as_bytes()
443}
444
445/// Commits that get a reachability bitmap on the **live** selection path.
446///
447/// # It was `usize::MAX`, and that was the single most expensive line in a fetch
448///
449/// The comment that stood here said a sampled table *"is for the sealed archive
450/// where the cost is disk"* and that every commit must be bitmapped on the live
451/// path. That was not a preference, it was forced:
452/// [`GitStore::reachable_oids`] had no walk, so a `want` with no bitmap could
453/// only contribute itself — a silent under-send — and `crate::serve`'s `select`
454/// therefore had to refuse unless *"in the graph"* and *"has a bitmap"* were the
455/// same fact.
456///
457/// MEASURED on oden 2026-08-14, one 4-object negotiated fetch out of the
458/// znippy-served `nornir` mirror (1 581 commits, 12 455 objects): the
459/// `select.bitmap` stage was **75 % of the request's CPU**, and **530 218 of its
460/// 575 349 allocations were this build** — which `refold` throws away again on
461/// the next push. The gix arm answered the identical fetch in 6.8 ms because it
462/// *reads* a `.bitmap` instead of building one.
463///
464/// [`crate::reach::accumulate`] is the walk that makes the cap legal, and 512 is
465/// the number [`crate::reach::ReachPolicy::default`] already uses for the sealed
466/// archive — the same figure git's own bitmap selection is of the order of, and
467/// deliberately not a second, different constant to keep in step.
468const LIVE_REACH_COMMITS: usize = 512;
469
470/// [`LIVE_REACH_COMMITS`], or `ZNIPPY_GIT_REACH_COMMITS`.
471///
472/// **An operator knob, not an arm.** It selects no implementation and changes no
473/// byte of any answer: the walk covers whatever the table does not, so the cap
474/// trades build cost against walk cost and nothing else. That property is the
475/// whole subject of
476/// `the_bounded_walk_answers_exactly_what_a_full_bitmap_table_answers`, which
477/// drives this very knob to its two extremes over one store.
478///
479/// Read **once per process** through [`crate::arms::read_env`], so it is
480/// counted with every other environment read this crate makes, and so a
481/// reachability request does not cost a `getenv` — until 2026-08-21 it did,
482/// once per `reach_bitmaps`/`reachable_oids` call, uncounted.
483pub(crate) fn live_reach_policy() -> crate::reach::ReachPolicy {
484 static MAX: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
485 let max_commits = *MAX.get_or_init(|| {
486 crate::arms::read_env(crate::arms::ENV_REACH_COMMITS)
487 .and_then(|v| v.parse::<usize>().ok())
488 .unwrap_or(LIVE_REACH_COMMITS)
489 });
490 crate::reach::ReachPolicy { max_commits }
491}
492
493/// Ceiling on phase-1 workers **per request**.
494///
495/// The serving tier admits ~32 concurrent transfers, and the fan-out primitive
496/// has no reentrancy detection, so an uncapped `gatling_for_each` here is
497/// 32 × ncores threads on a 128-core box. Four is a bound that leaves the
498/// admission gate in charge of the machine.
499const MAX_RESOLVE_WORKERS: usize = 4;
500
501/// Workers for phase 1: [`MAX_RESOLVE_WORKERS`], or `ZNIPPY_GIT_EMIT_WORKERS`.
502///
503/// **An operator knob, not an arm.** It selects no implementation and changes no
504/// byte of the emitted pack — a pack emitted at one worker and at sixteen is
505/// byte-identical, because phase 1 only decides *where each entry's bytes are*
506/// and phase 2 writes them in one serial order either way. `0` is read as "as
507/// many as this box has cores", which is `gatling_for_each`'s own convention.
508///
509/// Read **once per process** through [`crate::arms::read_env`] — see
510/// [`live_reach_policy`] for why; this one was a `getenv` per emitted request
511/// until 2026-08-21.
512fn emit_workers() -> usize {
513 static WORKERS: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
514 *WORKERS.get_or_init(|| {
515 crate::arms::read_env(crate::arms::ENV_EMIT_WORKERS)
516 .and_then(|v| v.parse::<usize>().ok())
517 .unwrap_or(MAX_RESOLVE_WORKERS)
518 })
519}
520
521pub(crate) struct Absorber<S: ObjectIndex = OneTableFourColumns> {
522 blobs: PathBuf,
523 /// Read handle on the verbatim packs, for `pread` of an extent.
524 ///
525 /// The **fallback**, since 2026-08-14, not the ordinary path: `map` below
526 /// answers every extent the mapping covers, and this reads the ones past its
527 /// end (a push that landed after the snapshot was taken).
528 ///
529 /// **It is the writer's own open file description** (2026-08-30, lane AH),
530 /// not a second `File::open`: this reads with `pread` at an explicit offset
531 /// and shares no cursor with `append`, so a private descriptor bought a
532 /// kernel object and a page-cache alias per open repository and no
533 /// isolation. See [`crate::archive_write::open_blobs`] and
534 /// `tests/fd_per_store.rs`.
535 reader: Arc<File>,
536 /// The same file, mapped read-only. See [`crate::archive_map`] for why an
537 /// append-only file `gc` never rewrites is safe to map, and for the ~20 % of
538 /// serve cycles the `pread`-per-object shape was spending on copies.
539 map: crate::archive_map::ArchiveMap,
540 hash: GitHashKind,
541 /// The `objects` table: stree → Arrow → redb tail.
542 ///
543 /// `S` is the payload layout the projection is built in
544 /// ([`crate::arms::IndexArm`]) and it is a **type parameter, not a field**:
545 /// this is the hot path, every lookup goes through it, and the whole reason
546 /// the arms exist is to measure the differences between them. A `dyn` here
547 /// would put a virtual call on every single oid.
548 objects: ObjectReadStack<S>,
549 /// **§14's exploded objects table** — oid → resolved type and content, built
550 /// eagerly by this absorb and by nothing else.
551 ///
552 /// Beside the `objects` table rather than inside it because the two have
553 /// different lifetimes: `objects` is the index and losing it loses the
554 /// repository's addressing, while this one is derived, verifiable against the
555 /// verbatim bytes and deletable at any time. They are separate files so that
556 /// "droppable" is a `rm`, not a schema migration.
557 ///
558 /// SWAPPED 2026-08-13 from the redb `ExplodedTable` to **one Arrow IPC
559 /// table**, payload in the row. redb held 16.8 GB of kernel payload in a
560 /// 204 GB file — 12× — and serialised the gatling fan-out behind its one
561 /// writer at 96.8% of a single core. See [`crate::exploded_arrow`].
562 ///
563 /// **BUDGETED AND EVICTABLE since lane Q (2026-08-30)**: the two-pool
564 /// [`ExplodedCache`] — the indexer's eager pool (commits and tags) beside a
565 /// lazy pool the FIRST reader of a blob or tree fills and a byte budget
566 /// empties. The unbounded eager tier it replaces MEASURED **141.7 GiB** for
567 /// the 8.12 GiB `torvalds/linux` pack on the rig, 17× the pack, which is
568 /// more than one appliance holds for all its repositories together.
569 exploded: Arc<ExplodedCache>,
570 derived: RwLock<Derived>,
571 /// How many folds since open read the persisted commit graph off disk vs
572 /// recomputed it from the exploded table. Counters, not clocks — the only
573 /// thing that can say which path a fold took (LAW 2). See
574 /// [`Absorber::graph_source_stats`].
575 graph_read: std::sync::atomic::AtomicU64,
576 graph_recomputed: std::sync::atomic::AtomicU64,
577 /// **Is `objects.graph` read and written at all** ([`crate::arms::ENV_GRAPH_CACHE`],
578 /// read once at open). `false` folds from the exploded table every time and
579 /// leaves no file: a store that serves no reads holds no derived memo.
580 graph_cache: std::sync::atomic::AtomicBool,
581 /// How many folds CARRIED the reachability table across (renumbered, no
582 /// tree opened) vs how many reads had to build it from scratch. The only
583 /// witness that the carry ran — the bytes a carried and a rebuilt table
584 /// answer with are identical by construction (LAW 2). See
585 /// [`Absorber::reach_source_stats`].
586 reach_carried: std::sync::atomic::AtomicU64,
587 reach_rebuilt: std::sync::atomic::AtomicU64,
588 /// How many folds READ the persisted table (`objects.reach`), how many
589 /// EXTENDED it for pushed commits, and how many extensions are not written
590 /// back yet. See [`Absorber::reach_stats`].
591 reach_loaded: std::sync::atomic::AtomicU64,
592 reach_extended: std::sync::atomic::AtomicU64,
593 reach_unpersisted: std::sync::atomic::AtomicU64,
594 /// Where the last fold's **ordinal space** came from: how many oids straight
595 /// off the persisted index's column, how many off a scan of the redb tail.
596 /// The same shape as [`Self::graph_read`]/[`Self::graph_recomputed`], and for
597 /// the same reason: which source produced the space is not visible in the
598 /// (identical) bytes, only in a counter (LAW 2). See
599 /// [`Absorber::ordinal_source_stats`].
600 ordinal_folded: std::sync::atomic::AtomicU64,
601 ordinal_tail: std::sync::atomic::AtomicU64,
602 packs: Mutex<PackState>,
603 /// Serialises **absorption**, which is long. Held across a whole pack.
604 ///
605 /// Two absorbs of one pack are duplicated *work* and nothing worse: every
606 /// writer under it is keyed by oid — [`ObjectReadStack::append`] swallows an
607 /// identical second row, the exploded table overwrites its own key, and the
608 /// commit graph is folded from that table rather than pushed at, so it cannot
609 /// double. (It could before §14's table existed, when the graph was a `Vec`
610 /// accumulated per absorb; that is why this paragraph used to say the gate
611 /// was load-bearing for correctness and now does not.) The gate is still what
612 /// makes the ordering below hold: **the rows go in, and only then is the bit
613 /// cleared**, so a read either finds the bit still clear (and waits here) or
614 /// finds the rows. Neither order can produce "absent".
615 gate: Mutex<()>,
616 /// **The progress counter** ([`crate::progress`]) a dashboard or a stats
617 /// call reads. One per store, registered process-wide.
618 progress: Arc<crate::progress::IndexProgress>,
619 /// Set by [`GitStore::cancel_index`]: the repository is being deleted. The
620 /// resolver checks it every few thousand objects and aborts the pack; the
621 /// drain skips every pack still queued.
622 cancel: std::sync::atomic::AtomicBool,
623 /// Names the store in log lines: its root directory.
624 label: String,
625 /// The memory watchdog this store's worker pauses on. The process-wide one
626 /// unless a test hands in a fake ([`GitStore::set_mem_watchdog`]).
627 watchdog: RwLock<Arc<crate::memguard::MemWatchdog>>,
628 /// **Pending packs a reader has already scanned**, keyed by pack offset:
629 /// the walk and an oid-sorted row table, built by ONE read-only pass and
630 /// kept until the pack is absorbed. Every later lookup into that pack is
631 /// a binary search; a content read inflates one chain. Without this,
632 /// every `has` during the index window re-resolved the whole pack —
633 /// MEASURED on the rig 2026-08-30: a `push --mirror` of linux stalled
634 /// pre-ack, ~800 ref updates × one 8 GiB resolve each.
635 scanned: Mutex<HashMap<u64, Arc<ScannedPack>>>,
636 /// One scan build at a time: a second reader waits for the first reader's
637 /// pass rather than starting its own. That is waiting on a *reader*, not
638 /// on the indexer.
639 scan_build: Mutex<()>,
640 /// **Packs this store has stopped trying to index**, by ordinal, with how
641 /// many attempts it took to stop and the reason the last one gave.
642 ///
643 /// 🔴 The state this replaces was *no state*. A pack whose absorb failed
644 /// stayed pending, every absorb loop hit it FIRST (they run in ordinal
645 /// order) and aborted on it with `?`, so the packs behind it were never
646 /// reached — and the next push, and the one after that, re-did the same
647 /// doomed resolve and was refused for it. MEASURED on oden 2026-08-30
648 /// (lane AI): one pack that could not be absorbed refused **every** push to
649 /// that repository from then on, 21 of 140, and lane AD's standby re-asked
650 /// for it every 300 s for 21 minutes. Work that cannot succeed has to stop
651 /// and say why; see [`MAX_ABSORB_ATTEMPTS`] and [`Absorber::absorb_each`].
652 unindexable: Mutex<HashMap<u64, Unindexable>>,
653}
654
655/// A pack this store will not try to index again, and why.
656#[derive(Debug, Clone)]
657pub struct Unindexable {
658 /// The pack's ordinal.
659 pub pack_id: u64,
660 /// Its extent in the verbatim archive. The bytes are still there.
661 pub extent: Extent,
662 /// How many absorbs were spent before it stopped.
663 pub attempts: u32,
664 /// The last attempt's root cause — the backend's own sentence, which is
665 /// what a client and an operator both get told.
666 pub reason: String,
667}
668
669/// How many times one pack's absorb is attempted before the store stops.
670///
671/// Three, and the reason it is not one: an absorb can fail for a reason outside
672/// the pack — the memory watchdog gave up, the repository was being deleted, a
673/// transient read error — and a single bad moment must not condemn durable
674/// bytes. It is not thirty either, because the resolve is deterministic in the
675/// store's own state: a pack that failed three times with nothing else changing
676/// fails the fourth time for the same reason, and every one of those attempts
677/// is paid for by a client waiting on a push.
678pub const MAX_ABSORB_ATTEMPTS: u32 = 3;
679
680// **The pack this thread is inside an absorb of**, if any.
681//
682// `absorb_gated` holds `Absorber::gate` for the whole pack, and the resolver it
683// runs asks this very store for delta bases through `BaseSource::content`. That
684// re-entry must not take the gate again — a `std::sync::Mutex` is not
685// reentrant, so it would not be a wait, it would be a deadlock — and it must
686// not treat the pack it is halfway through as something it can absorb. One
687// `Cell` answers both questions, and it is the same guard shape
688// `scan_pending`'s `SCANNING` uses for the same reason.
689thread_local! {
690 static IN_ABSORB: std::cell::Cell<Option<u64>> = const { std::cell::Cell::new(None) };
691}
692
693/// Sets [`IN_ABSORB`] for the duration of one absorb and restores whatever was
694/// there before — absorbs nest (a pack whose base is in another pending pack).
695struct InAbsorb(Option<u64>);
696
697impl InAbsorb {
698 fn enter(pack_id: u64) -> Self {
699 InAbsorb(IN_ABSORB.with(|c| c.replace(Some(pack_id))))
700 }
701}
702
703impl Drop for InAbsorb {
704 fn drop(&mut self) {
705 IN_ABSORB.with(|c| c.set(self.0));
706 }
707}
708
709/// One pending pack, scanned read-only: what a lookup needs and nothing that
710/// the index will hold. O(objects) memory — for linux ~56 B of walk + 32 B of
711/// row per object, ~1.2 GB — and gone when the pack's bit goes up.
712///
713/// **Nothing on the production read path builds one of these any more.** A
714/// reader that needs an un-indexed pack indexes it instead
715/// ([`Absorber::index_pending_now`]); this remains for the one caller that
716/// cannot — a resolver already inside `absorb_gated`, which may consult a scan
717/// somebody else already built but must never start one.
718struct ScannedPack {
719 walk: crate::pack_walk::PackWalk,
720 /// `(oid, entry index, resolved size)`, sorted by oid.
721 rows: Vec<(Vec<u8>, u32, u64)>,
722 archive_offset: u64,
723}
724
725impl ScannedPack {
726 fn find(&self, oid: &[u8]) -> Option<(usize, u64)> {
727 self.rows
728 .binary_search_by(|(o, _, _)| o.as_slice().cmp(oid))
729 .ok()
730 .map(|i| (self.rows[i].1 as usize, self.rows[i].2))
731 }
732
733 fn row(&self, oid: &[u8]) -> Option<crate::index_layout::IndexRow> {
734 let (entry, size) = self.find(oid)?;
735 let e = &self.walk.entries[entry];
736 Some(crate::index_layout::IndexRow {
737 // Not ranked: the row is not in the index yet.
738 ordinal: u32::MAX,
739 offset: e.offset + self.archive_offset,
740 len: e.len,
741 obj_type: e.obj_type,
742 uncompressed_size: size,
743 delta_base: match &e.delta_base {
744 crate::pack_walk::DeltaBase::Offset(o) => o + self.archive_offset,
745 _ => 0,
746 },
747 })
748 }
749}
750
751/// One repository's git store.
752///
753/// | on disk | what it is | who owns the format |
754/// |---|---|---|
755/// | `<root>/objects.pack` | the verbatim pack bytes, appended | [`SafeWriter`] |
756/// | `<root>/objects.pack.journal` | the durable extent rows | [`SafeWriter`] |
757/// | `<root>/objects.tail` | the un-sealed index tail | [`ObjectReadStack`] (redb) |
758/// | `<root>/objects.exploded` | §14's resolved content, **droppable** | [`ExplodedArchive`] (one Arrow IPC table) |
759/// | `<root>/refs.log` | the ref transaction log | [`RefLog`] (Arrow IPC frames) |
760/// | `<root>/<name>.znippy` | the sealed archive GC compacts | base znippy |
761///
762/// Nothing in that table is new. The store is the composition, not a fifth
763/// format.
764/// # Which implementation of each trait, and how a caller picks
765///
766/// Three of the four boxes in the diagram at the top of this file have more than
767/// one implementation. [`StoreConfig`] is the choice, [`crate::arms`] documents
768/// what each arm promises, and the dispatch is split deliberately:
769///
770/// | trait | selected by | dispatch | cost |
771/// |---|---|---|---|
772/// | [`ObjectIndex`] | the type parameter `S` | **static** | none — the hot path is monomorphised |
773/// | [`ArchiveWrite`](crate::archive_write::ArchiveWrite) | [`StoreConfig::writer`] | `dyn`, inside [`PushPath`] as it already was | one virtual call **per pack append**, not per object |
774/// | [`Gc`] | [`StoreConfig::gc`] | `dyn` | one virtual call per `gc()` run |
775///
776/// `S` defaults to [`OneTableFourColumns`], so `GitStore` still names exactly
777/// what it named before and every existing caller compiles unchanged.
778pub struct GitStore<S: ObjectIndex = OneTableFourColumns> {
779 root: PathBuf,
780 /// The verbatim pack bytes. The **truth** (§14).
781 blobs: PathBuf,
782 /// The sealed znippy archive `gc` compacts. Only GC and `seal` touch it.
783 archive: PathBuf,
784 /// The selected [`ArchiveWrite`](crate::archive_write::ArchiveWrite) arm
785 /// plus the per-account index channel. The ack path.
786 push: PushPath,
787 account: String,
788 hash: GitHashKind,
789 /// The object-level index and the absorb behind it — **shared with the
790 /// account indexer's worker**, which is how a push ends in object rows
791 /// without a read having to ask for them.
792 absorber: Arc<Absorber<S>>,
793 refs: RefLog,
794 /// Serialises the ref log's read-compare-append. A CAS that read the
795 /// namespace, then appended, without holding this could be overtaken between
796 /// the two and would overwrite the update it did not see.
797 ref_gate: Mutex<()>,
798 /// The selected [`Gc`] arm. Built once, here, and run by
799 /// [`GitOps::gc`] — a `Box<dyn>` because a GC is a whole-archive
800 /// compaction that happens on a maintenance timer, so one virtual call
801 /// against it is unmeasurable and a third type parameter on every signature
802 /// in the crate is not free to read.
803 gc: Box<dyn Gc + Send + Sync>,
804 /// What this store was built with, for a bench row or a config dump. A
805 /// description of the arms already constructed — **nothing consults it on a
806 /// serving path**.
807 arms: StoreConfig,
808}
809
810impl GitStore<OneTableFourColumns> {
811 /// Open (creating) the store for one repository under `root`.
812 ///
813 /// **The default arms, and it reads no environment to get them.** An
814 /// operator who exported `ZNIPPY_GIT_WRITER=fast` does not change what this
815 /// builds — a caller that never asked for a selector must not have its
816 /// durability contract moved out from under it. Use
817 /// [`open_from_env`] when the selection is wanted.
818 ///
819 /// ⚠ One variable *is* read here, and it is not an arm:
820 /// `ZNIPPY_GIT_REDB_CACHE_BYTES` (see
821 /// [`crate::arms::redb_cache_bytes`]). It selects no implementation,
822 /// changes no durability contract and leaves the bytes on disk identical —
823 /// it caps redb's page cache, whose own default is 1 GiB *per database*, of
824 /// which this store opens two. The sentence above is about arms, and a
825 /// memory ceiling is not one.
826 pub fn open(root: &Path, account: &str) -> Result<Self> {
827 Self::open_with(root, account, GitHashKind::Sha1)
828 }
829
830 /// Same, with an explicit oid width. sha256 repositories exist and the store
831 /// does not get to assume otherwise; every oid it stores is
832 /// [`GitHashKind::oid_len`] bytes wide and the tail refuses a mixed batch.
833 ///
834 /// # What a reopen recovers, and why nothing had to be stored for it
835 ///
836 /// A store opened over an `objects.pack` some earlier process was still
837 /// indexing has to end up in the same state that process would have reached.
838 /// §13.12's `indexed` bit is what says so, and it is **derived here rather
839 /// than persisted**: a pack is unabsorbed *iff* its extent is in the journal
840 /// and its rows are not in the index, and both of those are already durable
841 /// (see [`Absorber::adopt_journal`]). The packs that come back clear are put
842 /// straight onto the account's channel, so the drain finishes the interrupted
843 /// work with no read having to ask — and until it does, every read falls back
844 /// exactly as it would have before the crash. **Slower, never wrong**, across
845 /// a restart as well as within one.
846 ///
847 /// The bits are set **before** the channel is fed, which is the same ordering
848 /// the ack path uses for the same reason: a read that arrives in between finds
849 /// the bit clear and waits, never an index that does not mention the pack.
850 ///
851 /// # What a reopen recovers of the DERIVED tables, and how that was fixed
852 ///
853 /// [`Derived::graph`] and [`Derived::trees`] come back **in full**, for every
854 /// pack, whether or not anything is re-queued. They are folded from §14's
855 /// exploded table ([`Absorber::refold`]), which is on disk, so a clean
856 /// shutdown carries them across.
857 ///
858 /// This was a live correctness hole until 2026-08-08 and it is recorded
859 /// rather than quietly repaired. The graph was *accumulated in RAM* as packs
860 /// were absorbed and stored nowhere; a store killed mid-absorb got it back
861 /// because the interrupted pack was re-queued and re-resolved, but a store
862 /// that shut down **cleanly** re-queued nothing and came back with an empty
863 /// graph. MEASURED on oden 2026-08-08: 2687 rows and **0 of 551 commits**,
864 /// `reachable` on a commit returning that commit alone instead of its
865 /// closure — a *quiet* wrong answer — and `gc` seeing an empty live set. The
866 /// fix is the table, because the payloads the graph is folded from had no
867 /// home; [`tests::a_clean_shutdown_and_reopen_keeps_the_graph_and_reachable`]
868 /// is what proves it, over a real process that exits normally.
869 ///
870 /// The exploded table is still **droppable**: if its row count is short of
871 /// the `objects` table's, [`Absorber::adopt_journal`] declares no pack
872 /// absorbed and every one of them is re-queued and re-exploded. Absent means
873 /// rebuild, never wrong (§13.12, applied unchanged to a derived table).
874 ///
875 /// **Also the default arms, and also no environment read** — see
876 /// [`open`](GitStore::open).
877 pub fn open_with(root: &Path, account: &str, hash: GitHashKind) -> Result<Self> {
878 // The default ARMS, and the environment's CEILING: the one variable
879 // this constructor reads, because an operator capping a server's
880 // footprint must not have to pick a different constructor to do it.
881 // See [`crate::arms::redb_cache_bytes`].
882 Self::open_with_arms(
883 root,
884 account,
885 hash,
886 StoreConfig::DEFAULT.with_redb_cache_bytes(crate::arms::redb_cache_bytes()?),
887 )
888 }
889}
890
891/// Where a stored entry's delta base sits, relative to the request being served.
892///
893/// The `Outside` arm carries the base in **both** namings because the two delta
894/// forms know only one each — an `OFS_DELTA` its archive offset, a `REF_DELTA`
895/// its oid — and [`GitStore::emit_set`]'s thin arm has to ask "does the client
896/// hold this" in whichever one the entry speaks.
897enum BaseOf {
898 /// Not a delta at all. There is nothing to place it after and nothing to
899 /// re-head; the stored entry is copied as it stands.
900 Whole,
901 /// A delta whose base **is** in the request, at this archive offset.
902 ///
903 /// The offset is carried rather than recomputed because a `REF_DELTA` does
904 /// not know it — it names its base by oid, and its own `delta_base` column
905 /// is `0` by construction — and the emitter needs it twice: as the
906 /// `EmitEntry::delta_base` that [`crate::pack_walk::topological_order`]
907 /// orders on, and as the base an in-pack ref-delta is re-headed to point at.
908 Inside { at: u64 },
909 /// A delta whose base the request does not carry.
910 Outside {
911 /// The base's archive offset, `0` when the entry named an oid instead.
912 offset: u64,
913 /// The base's oid, `None` when the entry named an offset instead — or
914 /// when the entry is too short to read one out of.
915 oid: Option<Vec<u8>>,
916 },
917}
918
919/// The bases a **thin** pack may name without carrying them: what the receiver
920/// already held before this transfer, keyed both ways.
921///
922/// Built by [`GitStore::client_bases`], which is where the argument for trusting
923/// it is written down.
924struct ExternalBases {
925 /// The held object's oid, keyed by its archive offset — the coordinate an
926 /// `OFS_DELTA` names its base in, and the only reason this map exists.
927 by_offset: HashMap<u64, OidKey>,
928 /// The same objects keyed the way a `REF_DELTA` names them.
929 held: std::collections::HashSet<OidKey>,
930}
931
932/// **A raw oid as a fixed-size, `Copy` hash key — 20 bytes for SHA-1, 32 for
933/// SHA-256, and no heap allocation either way.**
934///
935/// The two maps in [`ExternalBases`] are repository-sized: the client's `have`
936/// closure on an incremental fetch is very nearly every object in the store.
937/// Keying them by `Vec<u8>` is one heap allocation per entry per request —
938/// 12 112 of them for a 69-object fetch, measured on oden 2026-08-15 — for a
939/// question (*does the client hold this base?*) that never looks at more than
940/// `oid_len` bytes.
941///
942/// Padded to the widest hash rather than parameterised by width: a store holds
943/// exactly one hash kind, so every key in one set has the same real length and
944/// two different oids can never collide through the padding. The length is
945/// carried anyway, because a key that silently compared equal across widths is
946/// the kind of wrong that only shows up in a mixed-hash mirror.
947#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
948struct OidKey {
949 bytes: [u8; 32],
950 len: u8,
951}
952
953impl OidKey {
954 /// The key for `oid`, or a refusal for one wider than any git hash.
955 fn new(oid: &[u8]) -> Result<Self> {
956 let len = oid.len();
957 if len == 0 || len > 32 {
958 bail!("{len} is not the width of any git object id (1..=32 bytes)");
959 }
960 let mut bytes = [0u8; 32];
961 bytes[..len].copy_from_slice(oid);
962 Ok(OidKey {
963 bytes,
964 len: len as u8,
965 })
966 }
967
968 /// The oid back, at its real width.
969 fn as_slice(&self) -> &[u8] {
970 &self.bytes[..self.len as usize]
971 }
972}
973
974impl<S: ObjectIndex + 'static> GitStore<S> {
975 /// **The one constructor**, with every arm named.
976 ///
977 /// `S` picks the [`ObjectIndex`] layout at compile time; `arms` picks the
978 /// [`ArchiveWrite`](crate::archive_write::ArchiveWrite) and [`Gc`]
979 /// implementations at run time. [`open`](GitStore::open) and
980 /// [`open_with`](GitStore::open_with) are this function with
981 /// [`StoreConfig::DEFAULT`], which is why nothing about them moved.
982 ///
983 /// `arms.index` is **not** consulted here — `S` is the index arm, and a
984 /// value that disagreed with the type would be a second source of truth.
985 /// [`open_from_env`] is the one place the two are tied together, and it does
986 /// it by choosing `S` from the value.
987 ///
988 /// ```no_run
989 /// use znippy_plugin_git::arms::{GcArm, StoreConfig, WriterArm};
990 /// use znippy_plugin_git::index_layout::PackedPayload;
991 /// use znippy_plugin_git::{GitHashKind, GitStore};
992 ///
993 /// // A rebuildable import mirror: the ceiling writer, the packed payload
994 /// // layout, and an in-place compaction.
995 /// let arms = StoreConfig::DEFAULT
996 /// .with_writer(WriterArm::Fast)
997 /// .with_gc(GcArm::CompactInPlace);
998 /// let store = GitStore::<PackedPayload>::open_with_arms(
999 /// std::path::Path::new("/srv/repo"),
1000 /// "alice",
1001 /// GitHashKind::Sha1,
1002 /// arms,
1003 /// )?;
1004 /// # Ok::<(), anyhow::Error>(())
1005 /// ```
1006 pub fn open_with_arms(
1007 root: &Path,
1008 account: &str,
1009 hash: GitHashKind,
1010 arms: StoreConfig,
1011 ) -> Result<Self> {
1012 std::fs::create_dir_all(root)
1013 .with_context(|| format!("creating the store root {}", root.display()))?;
1014 let blobs = root.join("objects.pack");
1015 let writer = arms.writer.create(&blobs)?;
1016 // `None` for an arm that keeps no durable ack log — see
1017 // [`crate::arms::WriterArm::journal`]. Everything below that touches the
1018 // journal is written against that `Option` rather than assuming one.
1019 let journal = arms.writer.journal(&blobs);
1020 // **The redb ceiling**, carried in on the config rather
1021 // than read here: `StoreConfig::from_env` reads it with the arms, and
1022 // `open`/`open_with` put the environment's value on `DEFAULT` before
1023 // getting here, so every constructor still honours the variable and
1024 // `arms.to_string()` prints what was actually applied. Not an arm — see
1025 // [`crate::arms::redb_cache_bytes`] — redb's own default is 1 GiB *per
1026 // database*, and this store opens exactly ONE of them, the
1027 // `ObjectReadStack` below. The `let _ = cache_bytes` two lines down is
1028 // what is left of the second.
1029 let cache_bytes = arms.redb_cache_bytes;
1030 let objects = ObjectReadStack::<S>::open(
1031 &root.join("objects.tail"),
1032 RebuildTriggers::default(),
1033 cache_bytes,
1034 )?;
1035 let _ = cache_bytes; // no page cache to size: the blob is pread, the index is a slice
1036 // A store written before 2026-09-04 carries `objects.ordinal`, the memo
1037 // of the sorted oid list the read stack's persisted index now IS. Its
1038 // bytes are never read again, so it goes on the first open by this
1039 // build — 278 964 902 B on the kernel mirror. Absent is the normal case
1040 // and not an event; anything else that stops the delete is logged, not
1041 // fatal, because the store does not depend on the file being gone.
1042 match std::fs::remove_file(root.join(ORDINAL_CACHE_FILE)) {
1043 Ok(()) => tracing::info!(
1044 target: "znippy.git",
1045 path = %root.join(ORDINAL_CACHE_FILE).display(),
1046 "removed a stale objects.ordinal — the persisted index's oid column is the \
1047 ordinal space now"
1048 ),
1049 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
1050 Err(e) => tracing::warn!(
1051 target: "znippy.git",
1052 path = %root.join(ORDINAL_CACHE_FILE).display(),
1053 error = %e,
1054 "could not remove a stale objects.ordinal; it is dead weight, not a fault"
1055 ),
1056 }
1057 let exploded = Arc::new(ExplodedCache::open(
1058 &root.join("objects.exploded"),
1059 crate::exploded_arrow::ExplodePolicy::from_env()?,
1060 crate::exploded_cache::budget_from_env(root)?,
1061 )?)
1062 .registered();
1063 // **One descriptor on the blob file, not three.** The writer opened it
1064 // read/write and `flock`ed it; the absorber's `pread` fallback and the
1065 // index workers' `read_exact_at` both address it positionally, so they
1066 // take the same open file description rather than two more of their
1067 // own. 5 descriptors per open store → 3, which is the divisor of
1068 // gunnar's `open_repo_budget` (`tests/fd_per_store.rs`).
1069 let absorber = Arc::new(Absorber::<S>::open(
1070 &blobs,
1071 writer.blobs_shared(),
1072 hash,
1073 objects,
1074 exploded,
1075 )?);
1076
1077 // **The `indexed` bit, derived.** Every extent this archive has ever
1078 // acked, diffed against the rows the index holds. `SafeWriter::create`
1079 // above appends to that journal and never truncates it, which is what
1080 // makes the older extents still be here to diff against.
1081 //
1082 // An arm with no journal has nothing to derive the bit from, and that is
1083 // the honest consequence of choosing it rather than a gap: a
1084 // `FastWriter` store has no durable record that a pack was acked, so a
1085 // reopen re-queues nothing and there is nothing for it to re-queue.
1086 let acked = match journal.as_deref() {
1087 Some(p) if p.exists() => crate::archive_write::read_journal(p).with_context(|| {
1088 format!(
1089 "reading {} — it is the durable half of the indexed bit and a store \
1090 cannot be opened without knowing which packs it owes index work for",
1091 p.display()
1092 )
1093 })?,
1094 _ => Vec::new(),
1095 };
1096 let requeue = absorber.adopt_journal(&acked)?;
1097
1098 // **The wiring.** The absorber goes into the push path, which hands it to
1099 // every account indexer it starts, so the drain behind `push_pack` ends
1100 // in object rows instead of one pack row. Nothing about the ack changes:
1101 // `append` still returns after the two fsyncs and this is all downstream
1102 // of that (§13.9).
1103 let push = PushPath::with_absorber(
1104 writer,
1105 &blobs,
1106 journal,
1107 absorber.clone() as Arc<dyn ObjectAbsorb>,
1108 )?;
1109 let refs = RefLog::new(root.join("refs.log"));
1110 let archive = root.join("repository.znippy");
1111
1112 let store = Self {
1113 root: root.to_path_buf(),
1114 blobs,
1115 archive,
1116 push,
1117 account: account.to_string(),
1118 hash,
1119 absorber,
1120 refs,
1121 ref_gate: Mutex::new(()),
1122 gc: arms.gc.create(),
1123 arms,
1124 };
1125 // The interrupted work, back on the channel it fell off. Nothing reads
1126 // here and nothing blocks: this is the same 24-byte handoff a push makes,
1127 // and the drain picks it up on its own thread (§13.9-11).
1128 let indexer = store.push.indexer(&store.account);
1129 for p in &requeue {
1130 indexer.submit(IndexJob {
1131 pack_id: p.pack_id,
1132 offset: p.offset,
1133 len: p.len,
1134 })?;
1135 }
1136 // A store reopened over an existing tail already knows its objects; the
1137 // derived tables are rebuilt from them rather than from a side-car,
1138 // because a side-car is a second truth.
1139 store.refold()?;
1140 Ok(store)
1141 }
1142
1143 /// The arms this store was built with. A description of what was
1144 /// constructed, not a switch — nothing on a serving path reads it.
1145 pub fn arms(&self) -> StoreConfig {
1146 self.arms
1147 }
1148
1149 /// The selected writer's name, as it goes on a bench row.
1150 pub fn writer_name(&self) -> &'static str {
1151 self.push.name()
1152 }
1153
1154 /// What the selected writer's `append` actually promises. Printed next to a
1155 /// throughput, because a throughput without it is not a comparison.
1156 pub fn writer_durability(&self) -> &'static str {
1157 self.push.durability()
1158 }
1159
1160 /// The selected [`Gc`] arm.
1161 pub(crate) fn gc_arm(&self) -> &dyn Gc {
1162 self.gc.as_ref()
1163 }
1164
1165 /// Where the verbatim packs are.
1166 pub fn blobs_path(&self) -> &Path {
1167 &self.blobs
1168 }
1169
1170 /// The sealed archive GC operates on.
1171 pub fn archive_path(&self) -> &Path {
1172 &self.archive
1173 }
1174
1175 pub fn root(&self) -> &Path {
1176 &self.root
1177 }
1178
1179 pub fn hash_kind(&self) -> GitHashKind {
1180 self.hash
1181 }
1182
1183 /// The `objects` table, for a caller that wants the index directly (the
1184 /// bench does).
1185 pub fn index(&self) -> &ObjectReadStack<S> {
1186 &self.absorber.objects
1187 }
1188
1189 /// Objects in the `objects` table.
1190 pub fn object_count(&self) -> usize {
1191 self.absorber.objects.len()
1192 }
1193
1194 /// Commits in the graph.
1195 pub fn commit_count(&self) -> usize {
1196 self.absorber
1197 .derived
1198 .read()
1199 .expect("derived lock")
1200 .graph
1201 .len()
1202 }
1203
1204 /// `(graph_read_from_disk, graph_recomputed)` since this store was opened —
1205 /// how many folds read the persisted `objects.graph` vs re-derived it.
1206 pub fn graph_source_stats(&self) -> (u64, u64) {
1207 self.absorber.graph_source_stats()
1208 }
1209
1210 /// `(reach_carried, reach_rebuilt)` since this store was opened — how many
1211 /// folds carried the reachability table across (renumbered, no tree
1212 /// opened) vs how many reads built it from scratch. The witness for the
1213 /// clone-after-push lever; see `REACH_CARRY_MAX_FRACTION`.
1214 pub fn reach_source_stats(&self) -> (u64, u64) {
1215 self.absorber.reach_source_stats()
1216 }
1217
1218 /// `(folded_rows, tail_rows)` of the ordinal space the last fold built — how
1219 /// many oids came off the persisted index's oid column and how many off a
1220 /// scan of the redb tail. Replaces S-082's `(read, recomputed)`: there is no
1221 /// memo to read any more, the index IS the space.
1222 pub fn ordinal_source_stats(&self) -> (u64, u64) {
1223 self.absorber.ordinal_source_stats()
1224 }
1225
1226 /// The folded ordinal space — every oid in ordinal order, hex. For the tests
1227 /// that assert a memo read reproduces the scanned space exactly (same oids,
1228 /// same order, same ordinals — the ordinal is the index into this vector).
1229 ///
1230 /// Formatted on demand from `oids_raw`: there is no resident hex `Vec` any
1231 /// more (S-082), and a test that wants hex encodes the flat buffer here
1232 /// rather than the store carrying a second copy for it.
1233 #[cfg(test)]
1234 pub(crate) fn ordinal_oids(&self) -> Vec<String> {
1235 let oid_len = self.hash.oid_len();
1236 self.absorber
1237 .derived
1238 .read()
1239 .expect("derived lock")
1240 .ordinals
1241 .view()
1242 .concat()
1243 .chunks_exact(oid_len)
1244 .map(hex::encode)
1245 .collect()
1246 }
1247
1248 /// **One object's resolved content** — §14's exploded table, in one lookup.
1249 ///
1250 /// Not one of the twelve and deliberately not on [`GitOps`]: the twelve are
1251 /// the wire vocabulary and they hand back **stored** bytes ([`GitOps::get`]),
1252 /// which for a delta entry is a delta. This is the other question — *what
1253 /// does this object contain* — and it is the one §14's table exists to make
1254 /// a point lookup instead of a chain walk.
1255 ///
1256 /// It absorbs any pending pack first, for the same reason
1257 /// [`GitOps::has`] does: a store with durable bytes it has not indexed cannot
1258 /// answer "absent" without lying.
1259 pub fn content(&self, oid: Oid<'_>) -> Result<Option<(GitObjectKind, Vec<u8>)>> {
1260 // No absorb here any more: a pack the worker has not reached is read
1261 // for this one oid by `Absorber::resolved`'s third arm — on this thread,
1262 // from the mmapped pack, without waiting for the worker.
1263 self.absorber.resolved(oid)
1264 }
1265
1266 /// **One object, with the bill for producing it.**
1267 ///
1268 /// [`Self::content`] plus what the read cost: `None` for a cost when the
1269 /// exploded pools answered (nothing was rebuilt), `Some(cost)` when the
1270 /// object was inflated out of the pack. What a measurement needs and a
1271 /// server does not — the cache admission decision is made inside, not by
1272 /// the caller.
1273 pub fn content_costed(
1274 &self,
1275 oid: Oid<'_>,
1276 ) -> Result<Option<(GitObjectKind, Vec<u8>, Option<crate::resolve::ReadCost>)>> {
1277 self.absorber.resolved_costed(oid)
1278 }
1279
1280 /// §14's table, counted: rows, rows written, reads it served, reads that fell
1281 /// through to re-deriving from the verbatim truth.
1282 ///
1283 /// Applied output. Both content paths return identical bytes, so `served`
1284 /// against `rederived` is the only thing that can say which one ran.
1285 pub fn exploded_stats(&self) -> ExplodedStats {
1286 self.absorber.exploded.engine_stats()
1287 }
1288
1289 /// **What the exploded tier is holding and what it is allowed to hold** —
1290 /// the row a dashboard prints (`explode 3.1G/8G`) and `Admin.StorageInfo`
1291 /// carries, per repository.
1292 pub fn explode_cache_use(&self) -> crate::exploded_cache::CacheUse {
1293 self.absorber.exploded.use_now()
1294 }
1295
1296 /// Rows in the EAGER pool only — the graph the push wrote, with nothing a
1297 /// reader has since cached. What `adopt_journal` compares against the index
1298 /// and what a test asking "did the push explode the graph" must ask,
1299 /// because [`Self::exploded_stats`]'s `rows` counts both pools and a read
1300 /// moves it.
1301 pub fn exploded_eager_rows(&self) -> Result<u64> {
1302 self.absorber.exploded.eager_rows()
1303 }
1304
1305 /// **Set this repository's explode budget** — what the control plane calls
1306 /// when an account's quota changes. Takes effect at the next rotation; it
1307 /// never refuses a read, never refuses a write and never deletes on the
1308 /// caller's thread. A bigger budget buys a bigger cache and therefore faster
1309 /// first reads; a smaller one costs re-inflation, never an error.
1310 pub fn set_explode_budget(&self, bytes: u64) {
1311 self.absorber.exploded.set_budget(bytes);
1312 }
1313
1314 /// Flush the lazy pool, rotate it and evict it back under budget — on
1315 /// **this** thread. What a test and a `gc` call; a reader never does, it
1316 /// starts a write-back thread and walks away.
1317 pub fn explode_cache_maintain(&self) -> Result<u64> {
1318 self.absorber.exploded.maintain()
1319 }
1320
1321 /// **`repo gc` reclaims the cache**: every lazy segment unlinked, the graph
1322 /// pool kept. Returns the bytes that left the filesystem.
1323 pub fn drop_explode_cache(&self) -> Result<u64> {
1324 self.absorber.exploded.drop_cache()
1325 }
1326
1327 /// Where §14's table lives. Deleting this file while no store holds it is the
1328 /// sanctioned way to drop it; the next open re-explodes every pack.
1329 pub fn exploded_path(&self) -> &Path {
1330 self.absorber.exploded.path()
1331 }
1332
1333 /// Every live row of one kind, with its payload — what the graph fold reads,
1334 /// exposed so a measurement can ask **the table** how many bytes it holds
1335 /// rather than dividing a file size by what the pack claimed.
1336 pub fn exploded_of_kind(&self, kind: GitObjectKind) -> Result<Vec<(Vec<u8>, Vec<u8>)>> {
1337 self.absorber.exploded.of_kind(kind)
1338 }
1339
1340 /// The commit graph as it stands, generations included. A snapshot: the
1341 /// caller holds no lock and the next fold replaces it wholesale.
1342 pub fn graph_snapshot(&self) -> Vec<CommitNode> {
1343 self.absorber
1344 .derived
1345 .read()
1346 .expect("derived lock")
1347 .graph
1348 .clone()
1349 }
1350
1351 /// Packs whose objects are not in the index yet — §13.12's bit, counted.
1352 pub fn unindexed_packs(&self) -> usize {
1353 self.absorber.unindexed_packs()
1354 }
1355
1356 /// This account's indexer: the thread that drains the channel a push queues
1357 /// on. `is_indexed(pack_id)` on it is §13.12's bit at the pack level, and it
1358 /// is set only once that pack's **objects** are in the index.
1359 pub fn indexer(&self) -> std::sync::Arc<crate::indexer::AccountIndexer> {
1360 self.push.indexer(&self.account)
1361 }
1362
1363 /// Block until the background drain has absorbed everything this store has
1364 /// pushed. What a maintenance tick, a seal or a test uses instead of racing
1365 /// it.
1366 pub fn wait_indexed(&self) {
1367 self.push.pool().wait_caught_up();
1368 }
1369
1370 /// **Stop this store's index work — the repository is being deleted.**
1371 ///
1372 /// Sets the cancel flag the resolver polls every few thousand objects and
1373 /// the drain checks before every pack. The pack in flight is **aborted at
1374 /// its next checkpoint** (milliseconds), not finished: its rows are keyed
1375 /// by oid and the exploded tier is droppable, and the whole directory is
1376 /// about to be removed anyway. Every pack still queued is skipped. After
1377 /// this the store can be dropped without the drop joining a worker that is
1378 /// hours into a kernel pack — which is what `Drop for AccountIndexer` would
1379 /// otherwise do.
1380 pub fn cancel_index(&self) {
1381 self.absorber.cancel.store(true, std::sync::atomic::Ordering::SeqCst);
1382 self.push.pool().cancel_all();
1383 }
1384
1385 /// Put a different memory watchdog under this store's worker — a test's
1386 /// fake `/proc/meminfo`. Takes effect from the next pack.
1387 pub fn set_mem_watchdog(&self, dog: Arc<crate::memguard::MemWatchdog>) {
1388 *self.absorber.watchdog.write().unwrap_or_else(|e| e.into_inner()) = dog;
1389 }
1390
1391 /// Has [`cancel_index`](Self::cancel_index) been called?
1392 pub fn index_cancelled(&self) -> bool {
1393 self.absorber.cancel.load(std::sync::atomic::Ordering::SeqCst)
1394 }
1395
1396 /// This store's index progress counter, for a dashboard or a stats call.
1397 pub fn index_progress(&self) -> crate::progress::Snapshot {
1398 self.absorber.progress.snapshot()
1399 }
1400
1401 /// **The indexer's half of the split**, and not one of the twelve.
1402 ///
1403 /// Runs on the account indexer's worker (§13.9), and on a read that arrives
1404 /// before that worker got there — the same function either way, which is why
1405 /// there is no such thing as a background index that behaves differently from
1406 /// the fallback one.
1407 pub fn absorb_pending(&self) -> Result<usize> {
1408 self.absorber.absorb_pending()
1409 }
1410
1411 /// **Make every durable object addressable, on this thread, now** —
1412 /// [`git_storage_trait::GitServe::index_now`]'s implementation and the ack
1413 /// path's one call. See [`Absorber::index_pending_now`] for why the
1414 /// connectivity check is the only caller allowed to wait here.
1415 pub fn index_now(&self) -> Result<usize> {
1416 self.absorber.index_pending_now()
1417 }
1418
1419 /// **The packs this store has stopped trying to index**, with the reason
1420 /// each one stopped for.
1421 ///
1422 /// Empty is the ordinary answer and the only good one. A non-empty list is
1423 /// a repository that is missing objects whose bytes are durable, and it is
1424 /// the fact an operator needs before wondering why a push was refused or a
1425 /// standby is behind — see [`MAX_ABSORB_ATTEMPTS`].
1426 pub fn unindexable_packs(&self) -> Vec<Unindexable> {
1427 self.absorber.unindexable_packs()
1428 }
1429
1430 /// **The object ids a stored pack introduced**, and not one of the twelve.
1431 ///
1432 /// [`GitOps::put_pack`] cannot answer this and never will: the walk it does
1433 /// reads pack *entries*, and a pack entry does not carry its oid — that is
1434 /// why [`crate::pack_walk::Closure`] can only name the `REF_DELTA` bases and
1435 /// says so. The oids exist once the indexer has absorbed the pack, so this
1436 /// absorbs first rather than racing the drain, and is therefore the one
1437 /// caller that deliberately pays §13.9's deferral back.
1438 ///
1439 /// **Only call it when something needs the set.** A push whose policy has no
1440 /// gate over objects must not pay this, which is the whole reason it is a
1441 /// separate call and not a field on `TxId`.
1442 ///
1443 /// The extent *is* the identity: §14 stores a pack verbatim as one
1444 /// contiguous append, so "introduced by this pack" and "indexed at an offset
1445 /// inside this pack's extent" are the same set, with no pack column to keep
1446 /// in step. Bases the push deltas against live in *other* extents and are
1447 /// excluded by construction — which is what a thin push needs, and what
1448 /// reading a written `.idx` back could never do.
1449 ///
1450 /// Cost is one pass over the index, so it is O(objects in the repository),
1451 /// not O(objects pushed). Recorded rather than hidden: a per-pack ordinal
1452 /// range would make it O(pushed), and needs the indexer to keep one.
1453 pub fn oids_in_extent(&self, extent: Extent) -> Result<Vec<Vec<u8>>> {
1454 let (start, len) = extent;
1455 // **Never waits on the indexer.** If the gate is free this thread
1456 // absorbs the pending packs (the same work, done now, by the caller
1457 // that needs it); if the worker holds it, the extent is resolved
1458 // read-only on this thread and its oids are the answer — the same
1459 // resolver, no sink, no rows written.
1460 if self.try_absorb_pending()?.is_none() {
1461 let snap = self.archive_snapshot()?;
1462 let bytes = self.extent(&snap, start, len)?;
1463 let walked = crate::pack_walk::walk(&bytes, self.hash.oid_len())?;
1464 let mut oids: Vec<Vec<u8>> = Vec::with_capacity(walked.entries.len());
1465 let limits = crate::resolve::ResolveLimits {
1466 base_budget: (crate::resolve::base_budget_from_env() / 4).max(1),
1467 ..crate::resolve::ResolveLimits::none()
1468 };
1469 crate::resolve::resolve_streaming(
1470 &bytes,
1471 &walked,
1472 self.hash,
1473 start,
1474 &*self.absorber,
1475 &crate::exploded::NoSink,
1476 limits,
1477 &mut |rows| {
1478 oids.extend(rows.drain(..).map(|r| r.oid));
1479 Ok(())
1480 },
1481 )
1482 .context("resolving the pushed pack read-only to enumerate its objects")?;
1483 return Ok(oids);
1484 }
1485 let end = start.saturating_add(len);
1486 let all = self.index().oids_in_order()?;
1487 let refs: Vec<&[u8]> = all.iter().map(Vec::as_slice).collect();
1488 let extents = self.index().extents_batch(&refs);
1489 Ok(all
1490 .into_iter()
1491 .zip(extents)
1492 .filter_map(|(oid, at)| match at {
1493 Some((offset, _)) if offset >= start && offset < end => Some(oid),
1494 _ => None,
1495 })
1496 .collect())
1497 }
1498
1499 /// **Everything needed to emit a pack for EXACTLY `oids`** — nothing added,
1500 /// nothing dropped.
1501 ///
1502 /// Not one of the twelve. It sits beside [`absorb_pending`](Self::absorb_pending)
1503 /// for the same reason those do: it is about how storage is *shaped*, not
1504 /// about what a caller stores or reads.
1505 ///
1506 /// # 🔴 It used to add the delta bases, and that shipped broken clones
1507 ///
1508 /// Until 2026-08-11 this closed the request over its delta bases: an
1509 /// `OFS_DELTA` names its base by position, so a base outside the set cannot
1510 /// be encoded, and *adding the base* looked like the answer that is always
1511 /// safe. It is not, and the failure is silent on the server:
1512 ///
1513 /// ```text
1514 /// git clone --bare --single-branch --branch base <url>
1515 /// fatal: did not receive expected object 8601ec33920b7d701c9887fa04916501136d6e90
1516 /// fatal: fetch-pack: invalid index-pack output
1517 /// event="git.upload_pack.served" objects=33773 ← the server said SUCCESS
1518 /// ```
1519 ///
1520 /// Measured on the `h2h-linear-sha1-2048c-1024f-16k` fixture: `base` reaches
1521 /// **31 805** objects, the delta-base closure added **1 968** more — 33 773,
1522 /// exactly the count the server logged — and 507 of those additions were
1523 /// **trees**. A tree that is not reachable from the wants drags its own
1524 /// children in as a *requirement*, because `git index-pack
1525 /// --check-self-contained-and-connected` (which is what a clone runs, and
1526 /// which sets `strict`) walks every received object's links and then demands
1527 /// each one exist. Those 507 trees named **204** objects the pack did not
1528 /// contain. `8601ec33…` is one of them.
1529 ///
1530 /// So a delta base pulled into a narrowed request is not free: it satisfies
1531 /// the *pack format* and violates *connectivity*. Closing over the added
1532 /// bases' children in turn does terminate — the same fixture reaches a fixed
1533 /// point at 34 534 objects — but it is an over-send by construction, and on
1534 /// a `--filter=blob:none` fetch it adds back exactly the blobs the filter
1535 /// excluded, which is the defect
1536 /// `emit_pack_emits_exactly_the_set_it_is_given_and_never_its_closure`
1537 /// exists to forbid.
1538 ///
1539 /// # What it does instead: the base decides how the entry is encoded
1540 ///
1541 /// The set is never widened. For each requested object:
1542 ///
1543 /// * **base inside the request** → the stored entry is copied byte for byte,
1544 /// which is the whole point of this engine and is what a full clone does
1545 /// for every single object;
1546 /// * **base outside the request** → the entry is **rebuilt** from §14's
1547 /// resolved content and marked
1548 /// [`EmitEntry::recompressed`](crate::pack_walk::EmitEntry::recompressed) —
1549 /// as a computed delta against a base the request *does* carry where one
1550 /// can be found (see [`Self::in_request_chain_ancestor`] and
1551 /// [`crate::delta`]), and whole where it cannot.
1552 ///
1553 /// That is what stock `pack-objects` does — it reuses a stored delta only
1554 /// when the base is also being packed — and it keeps the receipt a fact:
1555 /// `copied` and `recompressed` add up to the objects written, and
1556 /// `recompressed` is **0 for a whole-repository clone** because such a
1557 /// selection contains every base. The cost lands on the boundary a narrowed
1558 /// request cuts through and nowhere else: 1 968 of 33 773 entries, 5.9 %, on
1559 /// the fixture above.
1560 ///
1561 /// `ofs_delta_ok` is the client's `ofs-delta` capability. An old client that
1562 /// cannot parse one is not refused any more: the entry is re-headed as a
1563 /// `REF_DELTA` naming its base by oid, **carrying the same compressed delta
1564 /// payload**, so that arm stays a copy too. Only a base outside the request
1565 /// costs a rebuild.
1566 ///
1567 /// # `thin_haves` — the boundary entry that does NOT have to be rebuilt
1568 ///
1569 /// `Some(tips)` says the client consented to a thin pack *and* the
1570 /// negotiation found common ground; [`crate::serve::GitStore::emit_oids`] is
1571 /// where the two halves are required together, because either alone is a
1572 /// failed fetch rather than a smaller one. Given it, a base outside the
1573 /// request is checked against [`the client's own objects`](GitStore::client_bases),
1574 /// and one the client already holds is named by oid in a `REF_DELTA` that
1575 /// carries the **stored delta stream unchanged**. That entry counts as
1576 /// `copied`, because it is: nothing was inflated and nothing was deflated.
1577 ///
1578 /// `None` — a clone, or a client that did not ask for a thin pack — behaves
1579 /// exactly as before, and a **full clone is byte-identical either way**,
1580 /// because a whole-repository request has no base outside itself for any of
1581 /// this to apply to.
1582 ///
1583 /// It does nothing for the *clone* half of the boundary cost, and cannot: a
1584 /// clone's receiver holds nothing, so there is no external base to name.
1585 /// That half is the arm below it — [`crate::delta`] computes a delta against
1586 /// something the pack **does** carry, which is the only move left when there
1587 /// is no receiver to lean on. The two do not overlap: `thin_haves` reaches a
1588 /// boundary entry first and costs nothing when it fires, so a fetch stays a
1589 /// pure copy and only a clone pays for a computed delta.
1590 ///
1591 /// # An oid this repository does not hold is REFUSED
1592 ///
1593 /// It used to be skipped, which is the same silent-under-send shape as the
1594 /// closure defect above wearing a different hat: N asked, N-1 emitted,
1595 /// `PackStats` reporting success, and the client the first to know. gunnar's
1596 /// in-memory arm has always refused it, so the skip also meant the two
1597 /// engines behind one contract disagreed about what a missing object means.
1598 /// A partial-clone filter does not depend on the skip and cannot: a filter
1599 /// *removes* oids from the selection, it never adds one the store lacks.
1600 ///
1601 /// # Cost
1602 ///
1603 /// O(objects requested). The previous shape read `oids_in_order()` and
1604 /// `lookup_batch` over the **whole repository** on every call, to build an
1605 /// offset→oid map it needed only for the closure; with no closure the
1606 /// selection's own offsets answer "is this base in the request", so that
1607 /// scan is gone and no reverse index has to exist.
1608 pub fn emit_set(
1609 &self,
1610 oids: &[Oid<'_>],
1611 ofs_delta_ok: bool,
1612 thin_haves: Option<&[Oid<'_>]>,
1613 ) -> Result<Vec<crate::pack_walk::EmitEntry>> {
1614 use crate::index_layout::ObjType;
1615 use std::collections::hash_map::Entry;
1616
1617 // If the gate is free, absorb now; if the worker holds it, the rows
1618 // come from the index plus a read-only pass over the pending packs
1619 // (`lookup_batch_never_wait` below). Never a wait.
1620 let _ = self
1621 .try_absorb_pending()
1622 .context("absorbing pending index work before emitting a pack")?;
1623
1624 // Deduplicate, preserving the caller's order. A repeated oid in the
1625 // request must not become a repeated entry in the pack.
1626 //
1627 // **A map and not a set**, keyed oid → position in `asked` (which is the
1628 // position in `rows`). The membership question is the one it always was;
1629 // what the position buys is the base's *archive offset* for a
1630 // `REF_DELTA`, which names its base by oid and can learn where that base
1631 // sits no other way. It is a `usize` per requested object on a vector
1632 // this function already builds — not a second pass and not a second
1633 // index lookup.
1634 let mut asked: Vec<&[u8]> = Vec::with_capacity(oids.len());
1635 let mut seen: HashMap<&[u8], usize> = HashMap::with_capacity(oids.len());
1636 for oid in oids {
1637 if let Entry::Vacant(slot) = seen.entry(*oid) {
1638 slot.insert(asked.len());
1639 asked.push(*oid);
1640 }
1641 }
1642
1643 let rows = self.lookup_batch_never_wait(&asked)?;
1644
1645 // **One mapping for the whole emit set.** Only the header probes below
1646 // read through it — the entries themselves are recorded as extents and
1647 // resolved by [`Self::resolve_emit_payloads`], which takes its own
1648 // snapshot of the same [`ArchiveMap`](crate::archive_map::ArchiveMap) and
1649 // therefore the same `Arc` unless a push landed in between.
1650 let snap = self.archive_snapshot()?;
1651
1652 // The two questions "is this base in the request" are asked in the two
1653 // coordinate systems the two delta forms use, and both are answered off
1654 // the request itself rather than off the index.
1655 //
1656 // **Offset → position in the request**, not a bare set of offsets. The
1657 // membership question is the same one it always was; what the position
1658 // buys is the base's *oid*, which the re-delta arm below needs in order
1659 // to read the base's content. It is the same single pass over `rows`
1660 // that built the set, so it costs a `usize` per requested object and no
1661 // extra work — and it is emphatically not the whole-repository
1662 // offset→oid scan the old delta-base closure used to do.
1663 let in_set_offsets: HashMap<u64, usize> = rows
1664 .iter()
1665 .enumerate()
1666 .filter_map(|(i, r)| r.as_ref().map(|r| (r.offset, i)))
1667 .collect();
1668 let oid_len = self.hash_kind().oid_len();
1669
1670 // The base of the last boundary entry that was re-deltified, kept so a
1671 // run of entries hanging off one ancestor inflates it once. One slot and
1672 // not an LRU: measured on `h2h-linear-sha1-2048c-1024f-16k`, 961
1673 // boundary entries name 855 distinct stored bases, so there is no reuse
1674 // worth a cache that could hold a repository's worth of objects — and
1675 // this crate is under a live RSS budget.
1676 let mut base_content: Option<(u64, Vec<u8>)> = None;
1677
1678 // What the client already holds, built **lazily and at most once**. A
1679 // clone never asks for it (`thin_haves` is `None`), and a fetch every one
1680 // of whose bases is inside the request never reaches the line that builds
1681 // it either — only a request that actually cuts a delta chain pays, which
1682 // is the only request it can help.
1683 let mut client_bases: Option<ExternalBases> = None;
1684
1685 let mut out = Vec::with_capacity(asked.len());
1686 for (oid, row) in asked.iter().zip(rows.iter()) {
1687 // **An oid this repository does not hold is REFUSED, not dropped.**
1688 //
1689 // This used to `continue`, and that is the same silent-under-send
1690 // shape as the closure defect above: the request asks for N objects,
1691 // the pack carries N-1, the receipt says success and the client is
1692 // the first to find out. It also made the two engines disagree about
1693 // what a missing object *means* — gunnar's in-memory arm has always
1694 // said *"emit_pack was asked for X, which this store does not hold;
1695 // a pack emitter that skipped it would send a pack no client can
1696 // close"* — so a benchmark across the two was comparing two
1697 // contracts.
1698 //
1699 // No legitimate caller relies on the skip. Every set that reaches
1700 // here is built out of this store's own index: `select` answers from
1701 // the reachability bitmaps, and gunnar's own walk resolves each id
1702 // against the store before it selects it. A partial-clone filter is
1703 // no exception and is the case worth naming, because it is the one
1704 // that *looks* like it might be: `--filter=blob:none` **removes**
1705 // oids from the selection, it never adds one the store lacks, so a
1706 // filtered request is a smaller set of objects that are all present
1707 // and this refusal never fires on it.
1708 let Some(row) = row.as_ref() else {
1709 bail!(
1710 "this pack was asked for {}, which this repository does not hold; emitting \
1711 the rest would be a short pack reported as a success, so this refuses \
1712 instead",
1713 hex::encode(oid)
1714 );
1715 };
1716
1717 // Which base this entry names, if any, and whether the request holds
1718 // it. A `REF_DELTA`'s `delta_base` column is 0 by construction —
1719 // `DeltaBase::as_offset` reports 0 for a ref base and 0 is also the
1720 // no-base sentinel — so its base is read out of the entry's own
1721 // bytes, directly after the type/size varint.
1722 //
1723 // A base that is *outside* the request is carried in **both**
1724 // coordinate systems rather than as a bare `false`, because that is
1725 // what the thin arm below needs: an `OFS_DELTA` knows only where its
1726 // base sits, a `REF_DELTA` knows only what it is called, and
1727 // "does the client hold it" has to be asked in whichever one the
1728 // entry speaks.
1729 let base = match row.obj_type {
1730 ObjType::OfsDelta => match in_set_offsets.get(&row.delta_base) {
1731 Some(_) => BaseOf::Inside {
1732 at: row.delta_base,
1733 },
1734 None => BaseOf::Outside {
1735 offset: row.delta_base,
1736 oid: None,
1737 },
1738 },
1739 ObjType::RefDelta => {
1740 let head = self.extent(&snap, row.offset, row.len.min(64))?;
1741 let (_, _, n) = crate::pack_walk::type_and_size_of(&head)?;
1742 match head.get(n..n + oid_len) {
1743 // In the request — **and where**. `rows[i]` is `None`
1744 // only for an oid this store does not hold, which the
1745 // refusal above turns into an error the moment the loop
1746 // reaches it; until then the base is treated as outside,
1747 // which is the answer that cannot be wrong.
1748 Some(b) => match seen
1749 .get(b)
1750 .and_then(|i| rows[*i].as_ref())
1751 .map(|r| r.offset)
1752 {
1753 Some(at) => BaseOf::Inside { at },
1754 None => BaseOf::Outside {
1755 offset: 0,
1756 oid: Some(b.to_vec()),
1757 },
1758 },
1759 // A truncated entry names no base this can read. It is
1760 // not "inside", and it is not offered to the client
1761 // either — it falls through to the whole rebuild, which
1762 // is the answer that cannot be wrong.
1763 None => BaseOf::Outside {
1764 offset: 0,
1765 oid: None,
1766 },
1767 }
1768 }
1769 _ => BaseOf::Whole,
1770 };
1771
1772 if let BaseOf::Whole | BaseOf::Inside { .. } = base {
1773 let inside_at = match base {
1774 BaseOf::Inside { at } => Some(at),
1775 _ => None,
1776 };
1777 // **The whole-clone arm, and it reads not one byte here.** The
1778 // entry is recorded as its address in the archive; the bytes are
1779 // resolved against the mapping at emit time and go straight from
1780 // the page cache to the wire. This line used to be a `pread`
1781 // into a fresh `Vec` — 13.8 M of each on a `linux.git` clone, all
1782 // of them held at once, which is the 2314 MB peak RSS
1783 // [`crate::pack_walk::EntryBytes`] documents.
1784 let extent = crate::pack_walk::EntryBytes::Extent {
1785 offset: row.offset,
1786 len: row.len,
1787 };
1788 let (stored, obj_type, delta_base) = match (row.obj_type, inside_at) {
1789 // An `OFS_DELTA` for a client that cannot read one: re-head it
1790 // as a `REF_DELTA` naming the same base by oid. The compressed
1791 // payload is the delta stream either way, so this is still a
1792 // copy in the `recompressed` sense — nothing is inflated — but
1793 // the concatenation of a new header and an old payload exists
1794 // nowhere on disk, so it cannot be an extent and is `Owned`.
1795 (ObjType::OfsDelta, Some(_)) if !ofs_delta_ok => {
1796 let base_oid = self.oid_at_offset(row.delta_base, &asked, &rows)?;
1797 let bytes = self.extent(&snap, row.offset, row.len)?;
1798 (
1799 crate::pack_walk::EntryBytes::Owned(Self::as_ref_delta(
1800 &bytes, &base_oid,
1801 )?),
1802 ObjType::RefDelta,
1803 0,
1804 )
1805 }
1806 // 🚨 **A stored `REF_DELTA` whose base is IN THIS PACK is
1807 // re-headed as an `OFS_DELTA`, and that is not an
1808 // optimisation.**
1809 //
1810 // gitoxide — which is `gunnar_client`, `gunnar`'s own
1811 // `receive-pack`, and every other gix-based receiver — reads
1812 // every `OBJ_REF_DELTA` in an incoming pack as naming an
1813 // object *the receiver already has*.
1814 // `gix_pack::data::input::LookupRefDeltaObjectsIter` consults
1815 // the local object database and the bases it has already
1816 // spliced in, and **nowhere else**: never inside the pack it
1817 // is reading. A base that is in the pack and not in the
1818 // receiver's store is `input::Error::NotFound`, and the fetch
1819 // dies with `The object <base> could not be decoded or wasn't
1820 // found` — with the whole pack rejected, on a clone whose
1821 // target is empty by definition, so *every* such entry is
1822 // fatal. That is `S-003` / gitoxide#2882, and `gunnar-wire`'s
1823 // `Selection::client_has` carries the same rule for gunnar's
1824 // own emitter.
1825 //
1826 // Stock git resolves it happily, which is why this survived
1827 // five green `git clone` runs against the store that could
1828 // not serve one `gunnar_client` fetch (`gunnar.rewrite`
1829 // round 0 and `gunnar.year` week 0, all four znippy columns,
1830 // 2026-08-14 — the server logged the same request `served`).
1831 //
1832 // The entry keeps its compressed delta stream byte for byte;
1833 // only the header changes, from "base named by oid" to "base
1834 // named by distance", which is the naming every receiver in
1835 // existence resolves inside the pack. This is where such an
1836 // entry comes from in the first place: `git index-pack
1837 // --fix-thin` appends the base into the pack and leaves the
1838 // ref-delta naming it by oid, so any repository pushed to
1839 // more than once holds them.
1840 //
1841 // # What it costs, stated rather than glossed
1842 //
1843 // `Owned`, so this is the one arm that gives up the zero-copy
1844 // property for an entry it does **not** rebuild: the
1845 // compressed delta stream is memcpy'd once, because a new
1846 // header concatenated with an old payload exists nowhere on
1847 // disk and no extent can address it. Bounded by the
1848 // compressed size of the ref-deltas whose base is in the
1849 // request — not by the repository, and not by the pack.
1850 //
1851 // It could be avoided: `emit_pack` re-encodes every header
1852 // from `EmitEntry::obj_type` while `header_len` finds the
1853 // payload from the STORED bytes' own type, so an `Extent`
1854 // over the untouched ref-delta declared as an `OfsDelta`
1855 // would emit the identical pack with no copy at all. That is
1856 // deliberately not done: it makes an `EmitEntry` disagree
1857 // with the bytes it points at, and the entry model being
1858 // trustworthy is worth more than the memcpy. If the RSS
1859 // budget ever says otherwise, the honest form is a third
1860 // `EntryBytes` variant that carries the new header beside
1861 // the extent, not a silent disagreement.
1862 (ObjType::RefDelta, Some(at)) if ofs_delta_ok => {
1863 let bytes = self.extent(&snap, row.offset, row.len)?;
1864 (
1865 crate::pack_walk::EntryBytes::Owned(Self::as_ofs_delta(
1866 &bytes, oid_len, row.offset, at,
1867 )?),
1868 ObjType::OfsDelta,
1869 at,
1870 )
1871 }
1872 // Everything else copies. A `REF_DELTA` left as one here is
1873 // the `!ofs_delta_ok` client, which cannot be handed a
1874 // distance at all — and which is stock git older than 1.4.4,
1875 // never a gix receiver, since gix has always advertised
1876 // `ofs-delta`.
1877 _ => (extent, row.obj_type, row.delta_base),
1878 };
1879 out.push(crate::pack_walk::EmitEntry {
1880 oid: oid.to_vec(),
1881 stored,
1882 obj_type,
1883 uncompressed_size: row.uncompressed_size,
1884 delta_base,
1885 offset: row.offset,
1886 recompressed: false,
1887 deltified: false,
1888 });
1889 continue;
1890 }
1891
1892 // ── the base is outside the request ──────────────────────────────
1893 //
1894 // **Thin first.** If the client consented to a thin pack and the
1895 // negotiation vouches that it already holds this base, the entry
1896 // goes out as a `REF_DELTA` naming that base by oid, carrying the
1897 // **same compressed delta stream**. Nothing is inflated, nothing is
1898 // deflated, and the pack is smaller than the whole object by exactly
1899 // the margin the delta was worth. That is what `caps.thin` has always
1900 // been the allowance for.
1901 //
1902 // An `OFS_DELTA` cannot survive this: it names its base by a
1903 // backwards distance *within the pack*, and the base is not in the
1904 // pack. Re-heading it as a `REF_DELTA` is the same header swap the
1905 // no-`ofs-delta` client path does, over an unchanged payload.
1906 if let Some(haves) = thin_haves {
1907 if client_bases.is_none() {
1908 client_bases = Some(self.client_bases(haves)?);
1909 }
1910 let ext = client_bases
1911 .as_ref()
1912 .expect("the client's bases were just built");
1913 let base_oid: Option<Vec<u8>> = match &base {
1914 // Unreachable: the arm above `continue`s on both. Matched
1915 // rather than `unreachable!()` so that a future arm falling
1916 // through here declines the thin path instead of panicking
1917 // inside a serving thread.
1918 BaseOf::Whole | BaseOf::Inside { .. } => None,
1919 BaseOf::Outside {
1920 oid: Some(named), ..
1921 } => ext
1922 .held
1923 .contains(&OidKey::new(named)?)
1924 .then(|| named.clone()),
1925 BaseOf::Outside {
1926 offset,
1927 oid: None,
1928 } => ext.by_offset.get(offset).map(|k| k.as_slice().to_vec()),
1929 };
1930 if let Some(base_oid) = base_oid {
1931 // A stored `REF_DELTA` already names its base by oid, so its
1932 // bytes go out untouched and stay an extent. An `OFS_DELTA`
1933 // has to be re-headed, and the result is new bytes.
1934 let stored = if row.obj_type == ObjType::OfsDelta {
1935 let bytes = self.extent(&snap, row.offset, row.len)?;
1936 crate::pack_walk::EntryBytes::Owned(Self::as_ref_delta(&bytes, &base_oid)?)
1937 } else {
1938 crate::pack_walk::EntryBytes::Extent {
1939 offset: row.offset,
1940 len: row.len,
1941 }
1942 };
1943 out.push(crate::pack_walk::EmitEntry {
1944 oid: oid.to_vec(),
1945 stored,
1946 obj_type: ObjType::RefDelta,
1947 uncompressed_size: row.uncompressed_size,
1948 // The base is NOT in this pack, so nothing may be ordered
1949 // after it: `topological_order` reads 0 as "names no base
1950 // in here", which for an external base is the truth.
1951 delta_base: 0,
1952 offset: row.offset,
1953 // Copied. The payload never left the archive's bytes.
1954 recompressed: false,
1955 deltified: false,
1956 });
1957 continue;
1958 }
1959 }
1960
1961 // The base is not being sent and the client cannot supply it, so the
1962 // stored bytes cannot go out as they are. Two answers remain, and
1963 // the first is worth ~4× the second on the wire.
1964 let (kind, body) = self.content(oid)?.ok_or_else(|| {
1965 anyhow!(
1966 "{} is a delta whose base is outside this request, so it must be sent whole, \
1967 and this repository cannot produce its content — neither §14's exploded \
1968 table nor a re-derivation from the verbatim packs answered",
1969 hex::encode(oid)
1970 )
1971 })?;
1972
1973 // ── re-delta against a base the request DOES carry ────────────────
1974 //
1975 // The candidate comes from the entry's own stored delta chain: the
1976 // nearest ancestor that is inside the request. That is the packer's
1977 // original similarity judgement, already made and already written
1978 // down — no window, no sort, no scan of the repository.
1979 let candidate = if crate::delta::enabled() {
1980 self.in_request_chain_ancestor(row.delta_base, row.offset, &in_set_offsets)?
1981 } else {
1982 None
1983 };
1984 if let Some(base_at) = candidate {
1985 let at = *in_set_offsets
1986 .get(&base_at)
1987 .expect("the ancestor was found by lookup in this very map");
1988 // Reuse the last base if this entry hangs off the same one.
1989 if base_content.as_ref().is_none_or(|(o, _)| *o != base_at) {
1990 let base_oid = asked[at];
1991 let (_, base_body) = self.content(base_oid)?.ok_or_else(|| {
1992 anyhow!(
1993 "{} is in this request and is the delta base chosen for {}, and this \
1994 repository cannot produce its content",
1995 hex::encode(base_oid),
1996 hex::encode(oid)
1997 )
1998 })?;
1999 base_content = Some((base_at, base_body));
2000 }
2001 let (_, base_body) = base_content.as_ref().expect("just filled");
2002
2003 // `None` is the encoder declining — an unrelated base, or a
2004 // delta that came out no smaller than the object. Falling
2005 // through to the whole rebuild is then strictly right.
2006 if let Some(d) = crate::delta::delta(base_body, &body)? {
2007 let stored = Self::ofs_delta_entry(&d, row.offset, base_at)?;
2008 // A client with no `ofs-delta` is served the same computed
2009 // delta under a `REF_DELTA` head naming the base by oid —
2010 // the identical header swap the base-inside arm above does,
2011 // over a payload that is already built.
2012 // **The `Owned` exception, in its primary shape.** These
2013 // bytes were computed here and exist on no disk, so no
2014 // extent addresses them; carrying them on the entry is
2015 // correct and is bounded by the boundary a narrowed request
2016 // cuts through (961 of 33 773 entries on the fixture below),
2017 // never by the repository.
2018 let (stored, obj_type, delta_base) = if ofs_delta_ok {
2019 (
2020 crate::pack_walk::EntryBytes::Owned(stored),
2021 ObjType::OfsDelta,
2022 base_at,
2023 )
2024 } else {
2025 (
2026 crate::pack_walk::EntryBytes::Owned(Self::as_ref_delta(
2027 &stored, asked[at],
2028 )?),
2029 ObjType::RefDelta,
2030 0,
2031 )
2032 };
2033 out.push(crate::pack_walk::EmitEntry {
2034 oid: oid.to_vec(),
2035 stored,
2036 obj_type,
2037 uncompressed_size: row.uncompressed_size,
2038 delta_base,
2039 offset: row.offset,
2040 // Inflated and re-deflated, so NOT a copy — and the
2041 // receipt says which of the two rebuilds ran.
2042 recompressed: true,
2043 deltified: true,
2044 });
2045 continue;
2046 }
2047 }
2048
2049 // Ship the object whole rather than dragging an unreachable base —
2050 // and its children — into a clone.
2051 out.push(crate::pack_walk::EmitEntry {
2052 oid: oid.to_vec(),
2053 // Inflated and re-deflated from §14's resolved content: the
2054 // `Owned` exception again, and again bounded by the boundary.
2055 stored: crate::pack_walk::EntryBytes::Owned(Self::whole_entry(kind, &body)?),
2056 obj_type: crate::serve::resolved_type(kind),
2057 uncompressed_size: body.len() as u64,
2058 // Whole: it names no base, and nothing may order it after one.
2059 delta_base: 0,
2060 offset: row.offset,
2061 recompressed: true,
2062 deltified: false,
2063 });
2064 }
2065 Ok(out)
2066 }
2067
2068 /// **PHASE 1 of emitting: resolve every entry's stored bytes, in parallel.**
2069 ///
2070 /// Emission splits in two, and the split is not a preference — it is where
2071 /// the ordering dependency actually is:
2072 ///
2073 /// | phase | what it does | why |
2074 /// |---|---|---|
2075 /// | 1, **here**, parallel | address → bytes for every entry | the archive is immutable, so N workers over disjoint (or shared) extents need **no lock at all** |
2076 /// | 2, [`crate::pack_walk::emit_pack`], serial | assign output offsets, encode `OFS_DELTA` distances, write, hash | an `OFS_DELTA` names its base by **distance back in the output pack**, so entry *n* cannot be encoded until every earlier entry's byte length is known |
2077 ///
2078 /// Phase 2 is not parallelisable and no attempt is made: the distance chain
2079 /// is a genuine serial dependency, and a running sha1 is another.
2080 ///
2081 /// # No lock, and why that is a fact rather than an intention
2082 ///
2083 /// [`Self::adopt_journal`] states the storage property this rests on — *"the
2084 /// blob file is append-only and `gc` truncates nothing, so a retired pack's
2085 /// bytes are still exactly where the journal says they are"*. Every byte a
2086 /// snapshot covers is therefore immutable for the life of the file, so the
2087 /// workers below share one `&Mapped` and read it concurrently with no
2088 /// synchronisation of any kind. There is no mutex here; if one were needed
2089 /// the data model would be wrong.
2090 ///
2091 /// # 🚨 This is a fan-out inside the serving tier, and `crate::serve`'s
2092 /// header says that tier is serial
2093 ///
2094 /// It did, and the reason it gave is still true and still respected: the
2095 /// constellation's fan-out primitive has no reentrancy detection, so an
2096 /// unbounded fan-out per request inside an admission gate that already runs
2097 /// 32 transfers spawns W² threads. That is why this is **bounded twice**:
2098 ///
2099 /// * it does not fan out at all below [`FAN_OUT_AT`] entries, so every fetch
2100 /// small enough for thread-spawn to dominate runs on the calling thread
2101 /// exactly as before, and
2102 /// * it never exceeds [`MAX_RESOLVE_WORKERS`] (override with
2103 /// `ZNIPPY_GIT_EMIT_WORKERS`), so the worst case is 32 × 4 and not
2104 /// 32 × ncores.
2105 ///
2106 /// # What the fan-out is worth — stated honestly
2107 ///
2108 /// Resolving a mapped extent is a bounds check and a pointer, so the work
2109 /// this parallelises is **not** the resolve itself. It is the two things
2110 /// beside it: the first-touch **page fault** on each entry's header (the
2111 /// grammar is parsed here, which is what forces that fault, and it validates
2112 /// the entry before phase 2 commits a byte to the wire), and the `pread`
2113 /// fallback for any extent the mapping does not cover. On a warm page cache
2114 /// with the whole archive mapped, phase 1 is nearly free and the fan-out
2115 /// buys correspondingly little — the 20 % of cycles this change is aimed at
2116 /// is recovered by *not copying*, not by threads. I could not measure the
2117 /// fan-out's own contribution separately: oden was at loadavg 25.9 while
2118 /// this was written (2026-08-14), and a shared box cannot answer a question
2119 /// that fine.
2120 pub(crate) fn resolve_emit_payloads<'a>(
2121 &self,
2122 entries: &'a [crate::pack_walk::EmitEntry],
2123 snap: &'a crate::archive_map::Mapped,
2124 ) -> Result<Vec<std::borrow::Cow<'a, [u8]>>>
2125 where
2126 Self: Sync,
2127 {
2128 let workers = if entries.len() < FAN_OUT_AT {
2129 1
2130 } else {
2131 emit_workers()
2132 };
2133 self.resolve_with(entries, snap, workers)
2134 }
2135
2136 /// [`Self::resolve_emit_payloads`] with the worker count named rather than
2137 /// decided — `1` runs inline on the calling thread.
2138 ///
2139 /// Split out so the fan-out is **testable at a size a test fixture can
2140 /// reach**. The production threshold is 4096 entries and the corpus these
2141 /// guards run on holds 2687, so a test that went through the front door
2142 /// would exercise the serial arm every time and the gatling arm never;
2143 /// `the_fan_out_resolves_the_identical_bytes_the_serial_pass_does` calls
2144 /// this with 1 and with 4 over the same set and requires the two to agree
2145 /// byte for byte.
2146 pub(crate) fn resolve_with<'a>(
2147 &self,
2148 entries: &'a [crate::pack_walk::EmitEntry],
2149 snap: &'a crate::archive_map::Mapped,
2150 workers: usize,
2151 ) -> Result<Vec<std::borrow::Cow<'a, [u8]>>>
2152 where
2153 Self: Sync,
2154 {
2155 use znippy_zoomies::gatling_forkjoin::gatling_for_each;
2156
2157 let one = |i: usize| -> Result<std::borrow::Cow<'a, [u8]>> {
2158 let e = &entries[i];
2159 let bytes = match &e.stored {
2160 // The bytes live on the entry — the rebuilt-delta exception.
2161 // Borrowed, never cloned: `EntryBytes::Owned` is already the
2162 // only allocation this path makes and copying it would double it.
2163 crate::pack_walk::EntryBytes::Owned(v) => std::borrow::Cow::Borrowed(v.as_slice()),
2164 crate::pack_walk::EntryBytes::Extent { offset, len } => {
2165 self.extent(snap, *offset, *len)?
2166 }
2167 };
2168 // Parse the entry's own header now. Two things fall out: a malformed
2169 // entry is refused **before** phase 2 has written any of the pack,
2170 // and the first cache line of every entry is touched here — on
2171 // whichever worker got it — rather than one at a time on the serial
2172 // writer.
2173 crate::pack_walk::type_and_size_of(&bytes).with_context(|| {
2174 format!(
2175 "resolving the stored bytes of {} for emission",
2176 hex::encode(&e.oid)
2177 )
2178 })?;
2179 Ok(bytes)
2180 };
2181
2182 if workers <= 1 {
2183 return (0..entries.len()).map(one).collect();
2184 }
2185 gatling_for_each(entries.len(), workers, one)
2186 .into_iter()
2187 .collect()
2188 }
2189
2190 /// **Both phases, one call** — the only way a caller in this crate turns an
2191 /// ordered emit set into pack bytes.
2192 ///
2193 /// It is three lines, and it is a function rather than three lines because
2194 /// LAW 5's *fix by construction* applies exactly here: the snapshot has to
2195 /// outlive the slices taken from it, and a caller that took its own snapshot
2196 /// per entry, or dropped it between the phases, would be writing a pack out
2197 /// of a mapping that no longer exists. Routing `emit_oids` and every test
2198 /// through one writer makes that impossible to get wrong twice.
2199 pub(crate) fn emit_ordered(
2200 &self,
2201 ordered: &[crate::pack_walk::EmitEntry],
2202 out: &mut dyn std::io::Write,
2203 ) -> Result<crate::pack_walk::EmitReport>
2204 where
2205 Self: Sync,
2206 {
2207 let snap = self
2208 .archive_snapshot()
2209 .context("mapping the archive to emit")?;
2210 let payloads = self
2211 .resolve_emit_payloads(ordered, &snap)
2212 .context("resolving the emit set's stored bytes")?;
2213 crate::pack_walk::emit_pack(
2214 ordered,
2215 self.hash_kind(),
2216 out,
2217 &|i| Ok(payloads[i].as_ref()),
2218 )
2219 }
2220
2221 /// **The nearest ancestor of `from`'s stored delta chain that is inside the
2222 /// request**, or `None` when the chain leaves the request and never comes
2223 /// back.
2224 ///
2225 /// `from` is the archive offset of a base already known to be *outside* the
2226 /// request; `entry_at` is the offset of the entry that named it.
2227 ///
2228 /// # Why this cannot make a cycle, and why that is not a comment but an
2229 /// invariant
2230 ///
2231 /// The answer is required to satisfy `answer < entry_at`. In a well-formed
2232 /// pack a base always precedes the delta that names it, so every edge in the
2233 /// stored graph already points from a higher offset to a lower one; a new
2234 /// edge with the same property keeps the emitted graph a DAG **by
2235 /// construction**, whatever else the request contains. That matters because
2236 /// the obvious better candidate — the entry's own delta *children*, one
2237 /// revision away instead of two to nine — is exactly the set that would
2238 /// close a cycle: a child's stored base is this entry, so pointing this
2239 /// entry at the child makes `topological_order` unable to order either, and
2240 /// `emit_pack` refuses the set. Measured cost of the restriction on
2241 /// `h2h-linear-sha1-2048c-1024f-16k`: ~1 470 bytes per boundary entry
2242 /// against ~314 for the cyclic candidate, and 5 800 for shipping it whole.
2243 ///
2244 /// An `OFS_DELTA` decreases the offset every hop, so the bound is free
2245 /// there; a `REF_DELTA` names its base by oid and could point anywhere, so
2246 /// the comparison is made rather than assumed.
2247 ///
2248 /// # Cost
2249 ///
2250 /// One 64-byte `pread` per hop, no inflate, no allocation sized by an
2251 /// object — the same walk and the same ceiling as
2252 /// [`crate::serve::GitStore::resolved_type_at`], for the same reason: a
2253 /// pushed pack's back-references are not under this server's control, so a
2254 /// cycle must terminate rather than spin. Measured hop counts on the fixture
2255 /// above: 2 for 392 of the 859 that resolve, 3 for 260, and 9 at the worst.
2256 fn in_request_chain_ancestor(
2257 &self,
2258 from: u64,
2259 entry_at: u64,
2260 in_request: &HashMap<u64, usize>,
2261 ) -> Result<Option<u64>> {
2262 use crate::index_layout::{ObjType, ObjectIndex as _};
2263
2264 /// git's own `pack.depth` ceiling, doubled — the same bound
2265 /// `resolved_type_at` carries and for the same reason.
2266 const MAX_LINKS: usize = 100;
2267 /// Enough of an entry to read its type/size varint plus either an
2268 /// `OFS_DELTA` distance or a `REF_DELTA` oid.
2269 const HEADER_PROBE: u64 = 64;
2270
2271 let oid_len = self.hash_kind().oid_len();
2272 let snap = self.archive_snapshot()?;
2273 let mut at = from;
2274 for _ in 0..MAX_LINKS {
2275 if at != 0 && at < entry_at && in_request.contains_key(&at) {
2276 return Ok(Some(at));
2277 }
2278 if at == 0 {
2279 return Ok(None);
2280 }
2281 // Clamped, not `read_extent`: a short entry at the archive's end
2282 // has fewer than HEADER_PROBE bytes after it (FORGE-SHALLOW).
2283 let head = self.header_probe(&snap, at, HEADER_PROBE)?;
2284 let (t, _, n) = crate::pack_walk::type_and_size_of(&head)?;
2285 at = match t {
2286 // A whole object with no base: the chain ends here, and it was
2287 // not in the request.
2288 ObjType::Commit | ObjType::Tree | ObjType::Blob | ObjType::Tag => {
2289 return Ok(None)
2290 }
2291 ObjType::OfsDelta => {
2292 let (distance, _) = crate::pack_walk::ofs_distance_of(&head[n..])?;
2293 match at.checked_sub(distance) {
2294 Some(next) if next != 0 => next,
2295 // A base before the start of the archive is a corrupt
2296 // back-reference. The whole rebuild is always a correct
2297 // answer, so this declines rather than failing a clone
2298 // over an entry it was only ever going to optimise.
2299 _ => return Ok(None),
2300 }
2301 }
2302 ObjType::RefDelta => {
2303 let Some(base_oid) = head.get(n..n + oid_len) else {
2304 return Ok(None);
2305 };
2306 match self.index().lookup(base_oid) {
2307 Some(row) => row.offset,
2308 None => return Ok(None),
2309 }
2310 }
2311 };
2312 }
2313 Ok(None)
2314 }
2315
2316 /// An `OFS_DELTA` pack entry carrying `d`, deflated.
2317 ///
2318 /// The distance written here is the **archive** distance, which is the same
2319 /// coordinate every stored entry's distance is in;
2320 /// [`crate::pack_walk::emit_pack`] rewrites it into output coordinates for
2321 /// every entry it emits, copied or not, so this only has to be well-formed
2322 /// and self-consistent — [`crate::pack_walk::header_len`] parses it back to
2323 /// find where the payload starts.
2324 ///
2325 /// `Compression::fast` for the same reason [`Self::whole_entry`] uses it,
2326 /// and the trade is far better here: the buffer being deflated is the delta,
2327 /// which on the measured fixture is ~1.4 KB against the object's 16 KiB, so
2328 /// this path deflates roughly a **tenth** of the bytes the whole rebuild it
2329 /// replaces does.
2330 fn ofs_delta_entry(d: &[u8], entry_at: u64, base_at: u64) -> Result<Vec<u8>> {
2331 use crate::index_layout::ObjType;
2332 use std::io::Write as _;
2333
2334 let distance = entry_at.checked_sub(base_at).ok_or_else(|| {
2335 anyhow!(
2336 "a computed delta at archive offset {entry_at} names a base at {base_at}, which \
2337 is after it — an ofs-delta distance is backwards and this would not encode"
2338 )
2339 })?;
2340 let mut out = Vec::with_capacity(d.len() / 2 + 32);
2341 crate::pack_walk::encode_type_and_size(&mut out, ObjType::OfsDelta, d.len() as u64);
2342 crate::pack_walk::encode_ofs_distance(&mut out, distance);
2343 let mut z = flate2::write::ZlibEncoder::new(out, flate2::Compression::fast());
2344 z.write_all(d)?;
2345 Ok(z.finish()?)
2346 }
2347
2348 /// **The objects the receiver already held before this transfer**, in both
2349 /// coordinate systems a delta can name a base in.
2350 ///
2351 /// `haves` is the negotiated common **tips**, exactly as
2352 /// [`git_storage_trait::GitServe::emit_pack`] defines them, and the closure
2353 /// over them is this store's own — the contract deliberately passes tips
2354 /// rather than the walk's exclusion set, because the closure is
2355 /// repository-sized and the engine can rebuild it from its own bitmaps.
2356 ///
2357 /// # This is a VOUCHER, and the caller has to be able to stand behind it
2358 ///
2359 /// Every object named here may be pointed at by a delta the pack does not
2360 /// carry. If the receiver turns out **not** to hold one, its
2361 /// `index-pack --fix-thin` dies with `pack has N unresolved deltas`, so a
2362 /// caller that cannot vouch must not pass `caps.thin`. The case that matters
2363 /// is a **partial clone**: a `blob:none` client's `have` tips reach blobs it
2364 /// was deliberately never sent, so the closure over-states what it holds and
2365 /// a filtered fetch must not be marked thin.
2366 ///
2367 /// # Cost
2368 ///
2369 /// One bitmap union plus one point lookup per held object, and it happens at
2370 /// most once per request — and only on a request that has a base outside
2371 /// itself to place. The `by_offset` map exists because an `OFS_DELTA` knows
2372 /// its base only as an archive offset; building it over what the *client*
2373 /// holds rather than over the repository is what keeps this off the
2374 /// whole-repository scan the closure used to do.
2375 ///
2376 /// **And it is spent in raw bytes now, end to end.** This used to ask
2377 /// [`GitOps::reachable`] — hex out of the ordinal space, `hex::decode`d
2378 /// straight back to bytes, then a third `to_vec` per object to key the
2379 /// offset map, three heap allocations per held object for a set the size of
2380 /// the client's history. It asks [`GitStore::reachable_raw`] instead: one
2381 /// buffer for the closure, one hash table of [`OidKey`]s over it, and
2382 /// nothing per object.
2383 fn client_bases(&self, haves: &[Oid<'_>]) -> Result<ExternalBases> {
2384 use crate::index_layout::ObjectIndex as _;
2385
2386 // From the last COMMITTED fold if the indexer holds the gate: a closure
2387 // is a fact about immutable commits, and a `have` the fold does not
2388 // know yet contributes nothing — a smaller voucher, the safe direction
2389 // (fewer thin bases, never a base the client lacks). A thin fetch that
2390 // met an index build used to be refused here with "index in progress",
2391 // and gunnar re-emitted without the voucher (GF-4) to get past it.
2392 let held = self.reachable_raw_committed(haves).context(
2393 "closing over the negotiated common tips, to learn which bases the client can supply \
2394 for itself",
2395 )?;
2396 let refs: Vec<&[u8]> = held.iter().collect();
2397 let rows = self.index().lookup_batch(&refs);
2398 let mut by_offset = HashMap::with_capacity(refs.len());
2399 let mut keys = std::collections::HashSet::with_capacity(refs.len());
2400 for (oid, row) in refs.iter().zip(rows.iter()) {
2401 let key = OidKey::new(oid)?;
2402 if let Some(row) = row {
2403 by_offset.insert(row.offset, key);
2404 }
2405 keys.insert(key);
2406 }
2407 Ok(ExternalBases {
2408 by_offset,
2409 held: keys,
2410 })
2411 }
2412
2413 /// The oid of the entry at `offset`, looked up **in the request** — which is
2414 /// the only place it can be, because this is called for a base the request
2415 /// was just shown to contain.
2416 ///
2417 /// Linear over the request rather than a map, because it runs only on the
2418 /// `ofs-delta`-less client path: a capability no git since 1.4.4 omits, and
2419 /// one that has to be *advertised absent* to reach here at all.
2420 fn oid_at_offset(
2421 &self,
2422 offset: u64,
2423 asked: &[&[u8]],
2424 rows: &[Option<crate::index_layout::IndexRow>],
2425 ) -> Result<Vec<u8>> {
2426 asked
2427 .iter()
2428 .zip(rows)
2429 .find(|(_, r)| r.as_ref().is_some_and(|r| r.offset == offset))
2430 .map(|(oid, _)| oid.to_vec())
2431 .ok_or_else(|| {
2432 anyhow!(
2433 "the entry at archive offset {offset} was checked to be in this request and \
2434 then could not be found in it"
2435 )
2436 })
2437 }
2438
2439 /// Re-head a stored `OFS_DELTA` as a `REF_DELTA` naming `base_oid`.
2440 ///
2441 /// The compressed delta stream is copied unchanged; only the header differs,
2442 /// because the two forms differ in **how the base is named** and in nothing
2443 /// else. This is why a client without `ofs-delta` costs bytes and not CPU.
2444 fn as_ref_delta(stored: &[u8], base_oid: &[u8]) -> Result<Vec<u8>> {
2445 use crate::index_layout::ObjType;
2446 let (t, stated_size, n) = crate::pack_walk::type_and_size_of(stored)?;
2447 if t != ObjType::OfsDelta {
2448 bail!("only an ofs-delta can be re-headed as a ref-delta, this entry is {t:?}");
2449 }
2450 let (_, d) = crate::pack_walk::ofs_distance_of(stored.get(n..).unwrap_or(&[]))?;
2451 let mut out = Vec::with_capacity(stored.len() + base_oid.len());
2452 crate::pack_walk::encode_type_and_size(&mut out, ObjType::RefDelta, stated_size);
2453 out.extend_from_slice(base_oid);
2454 out.extend_from_slice(stored.get(n + d..).unwrap_or(&[]));
2455 Ok(out)
2456 }
2457
2458 /// Re-head a stored `REF_DELTA` as an `OFS_DELTA` naming the base that sits
2459 /// at `base_at` — **the inverse of [`Self::as_ref_delta`]**, and the fix for
2460 /// the one pack shape gitoxide cannot read.
2461 ///
2462 /// The compressed delta stream is copied unchanged; only the header differs,
2463 /// because the two forms differ in how the base is *named* and in nothing
2464 /// else. The reason this has to exist is in the caller: a `REF_DELTA` whose
2465 /// base is inside the same pack is resolved by stock git and refused by
2466 /// every gix-based receiver.
2467 ///
2468 /// # The distance written here is a placeholder, and that is by design
2469 ///
2470 /// Exactly as [`Self::ofs_delta_entry`] documents: the distance an
2471 /// `OFS_DELTA` carries is relative to a position in the pack it is *in*, so
2472 /// [`crate::pack_walk::emit_pack`] re-encodes it for every entry from the
2473 /// output offsets it is assigning, and [`crate::pack_walk::header_len`]
2474 /// parses whatever is written here only to find where the payload starts. It
2475 /// therefore has to be well-formed and self-consistent, and nothing more.
2476 ///
2477 /// It cannot simply be `entry_at - base_at`: a `REF_DELTA` is allowed to
2478 /// name a base that comes **after** it in the pack, and `git index-pack
2479 /// --fix-thin` produces exactly that — it appends the base at the end. So a
2480 /// forward reference encodes as `1`, and `emit_pack` writes the real
2481 /// backwards distance once `topological_order` has put the base first.
2482 fn as_ofs_delta(
2483 stored: &[u8],
2484 oid_len: usize,
2485 entry_at: u64,
2486 base_at: u64,
2487 ) -> Result<Vec<u8>> {
2488 use crate::index_layout::ObjType;
2489 let (t, stated_size, n) = crate::pack_walk::type_and_size_of(stored)?;
2490 if t != ObjType::RefDelta {
2491 bail!("only a ref-delta can be re-headed as an ofs-delta, this entry is {t:?}");
2492 }
2493 let payload = stored.get(n + oid_len..).ok_or_else(|| {
2494 anyhow!(
2495 "a ref-delta entry at archive offset {entry_at} is {} bytes, which is not even its \
2496 header plus a {oid_len}-byte base oid",
2497 stored.len()
2498 )
2499 })?;
2500 let mut out = Vec::with_capacity(payload.len() + 32);
2501 crate::pack_walk::encode_type_and_size(&mut out, ObjType::OfsDelta, stated_size);
2502 crate::pack_walk::encode_ofs_distance(
2503 &mut out,
2504 entry_at.checked_sub(base_at).filter(|d| *d > 0).unwrap_or(1),
2505 );
2506 out.extend_from_slice(payload);
2507 Ok(out)
2508 }
2509
2510 /// A whole (non-delta) pack entry for `body`: type/size header, then the
2511 /// body deflated.
2512 ///
2513 /// The **only** place in this crate that deflates on a serving path, and the
2514 /// only reason it exists is a delta whose base the request does not contain.
2515 /// `Compression::fast` deliberately: these bytes are re-compressed because
2516 /// they *cannot* be copied, so the trade is wire size against a clone's
2517 /// latency, and the entries this runs on are a small boundary fraction of any
2518 /// real request.
2519 fn whole_entry(kind: crate::object::GitObjectKind, body: &[u8]) -> Result<Vec<u8>> {
2520 use std::io::Write as _;
2521 let mut out = Vec::with_capacity(body.len() / 2 + 32);
2522 crate::pack_walk::encode_type_and_size(
2523 &mut out,
2524 crate::serve::resolved_type(kind),
2525 body.len() as u64,
2526 );
2527 let mut z = flate2::write::ZlibEncoder::new(out, flate2::Compression::fast());
2528 z.write_all(body)?;
2529 Ok(z.finish()?)
2530 }
2531
2532 /// Take the absorb gate and hold it.
2533 ///
2534 /// The only way to observe "a read that arrived before the drain" as a
2535 /// **state** rather than as a race: with this held, the worker is parked at
2536 /// the top of its absorb, so the pack is provably durable, provably queued
2537 /// and provably not indexed. Its guard is in this module's tests, and nothing
2538 /// on a serving path takes it.
2539 #[cfg(test)]
2540 pub(crate) fn hold_absorb_gate(&self) -> std::sync::MutexGuard<'_, ()> {
2541 self.absorber.gate.lock().expect("absorb gate poisoned")
2542 }
2543
2544 /// `pread` an extent out of the verbatim archive, into a fresh `Vec`.
2545 ///
2546 /// **The fallback and the owning path.** A caller that can hold a borrow
2547 /// wants [`Self::archive_snapshot`] plus [`Self::extent`] instead: this one
2548 /// costs a syscall, an allocation and a kernel→user copy, which is the shape
2549 /// [`crate::archive_map`] exists to get off the emit path.
2550 pub(crate) fn read_extent(&self, offset: u64, len: u64) -> Result<Vec<u8>> {
2551 self.absorber.read_extent(offset, len)
2552 }
2553
2554 /// One mapping of the verbatim archive, to be taken **once per operation**
2555 /// and sliced N times. See [`crate::archive_map::ArchiveMap::snapshot`].
2556 pub(crate) fn archive_snapshot(&self) -> Result<Arc<crate::archive_map::Mapped>> {
2557 self.absorber.map.snapshot()
2558 }
2559
2560 /// **The one resolver**: an extent's bytes, borrowed from `snap` when it
2561 /// covers them and `pread` into a fresh buffer when it does not.
2562 ///
2563 /// `Cow::Borrowed` is the ordinary answer and the whole point — no syscall,
2564 /// no allocation, no copy, just a bounds-checked slice of the page cache.
2565 /// `Cow::Owned` happens for an extent past the mapped end, which means the
2566 /// blob file grew after the snapshot was taken (a push landed mid-clone).
2567 /// `Mapped::get` returning `None` is *"go and pread it"* and never *"there
2568 /// are no bytes"*, so the two arms return the **same bytes** and differ only
2569 /// in what they cost — which is the property
2570 /// `an_extent_past_the_mapping_falls_back_and_returns_the_same_bytes`
2571 /// asserts.
2572 pub(crate) fn extent<'m>(
2573 &self,
2574 snap: &'m crate::archive_map::Mapped,
2575 offset: u64,
2576 len: u64,
2577 ) -> Result<std::borrow::Cow<'m, [u8]>> {
2578 match snap.get(offset, len) {
2579 Some(b) => Ok(std::borrow::Cow::Borrowed(b)),
2580 None => Ok(std::borrow::Cow::Owned(self.read_extent(offset, len)?)),
2581 }
2582 }
2583
2584 /// **A header probe: AT MOST `len` bytes at `offset`, fewer at the archive's end.**
2585 ///
2586 /// [`Self::extent`] is for an extent the index vouches for and refuses a
2587 /// short read, which is right there: a short extent is a truncated archive.
2588 /// A header probe is not such an extent. It asks for a fixed ceiling
2589 /// ([`crate::serve`]'s `HEADER_PROBE`) to cover the longest header, and an
2590 /// entry shorter than that at the END of the archive has fewer bytes after
2591 /// it. That is not truncation and refusing it cost a live repository every
2592 /// shallow clone (FORGE-SHALLOW, git.gunnar.rs 2026-10-28: edda's 41-byte
2593 /// ref-delta, `short read at (16843881, 64)`). The parsers read a grammar,
2594 /// not a fixed width, so they take what is there. The same clamp
2595 /// [`ArchiveWindow`] gives [`crate::resolve::materialize_at`].
2596 pub(crate) fn header_probe<'m>(
2597 &self,
2598 snap: &'m crate::archive_map::Mapped,
2599 offset: u64,
2600 len: u64,
2601 ) -> Result<std::borrow::Cow<'m, [u8]>> {
2602 clamped_window(&self.absorber, snap, offset, len)
2603 }
2604
2605 /// Record a durable pack as pending index work.
2606 ///
2607 /// The **bit going clear**, and the ack path's last act before it returns.
2608 pub(crate) fn queue(&self, pack_id: u64, extent: Extent) -> Result<()> {
2609 self.absorber.queue(pack_id, extent)
2610 }
2611
2612 fn reach_bitmaps(&self) -> Result<Arc<crate::reach::ReachTable>> {
2613 self.absorber.reach_bitmaps()
2614 }
2615
2616 /// [`Absorber::reach_bitmaps_with`], for the differential guard.
2617 #[cfg(test)]
2618 pub(crate) fn reach_bitmaps_with(
2619 &self,
2620 policy: crate::reach::ReachPolicy,
2621 cache: bool,
2622 ) -> Result<Arc<crate::reach::ReachTable>> {
2623 self.absorber.reach_bitmaps_with(policy, cache)
2624 }
2625
2626 /// `(loaded, extended, unpersisted)` — how many folds read the persisted
2627 /// reachability table, how many extended it for pushed commits, and how
2628 /// many extensions have not been written back yet. The witnesses for the
2629 /// persisted and incremental paths (LAW 2): the bytes a loaded, an extended
2630 /// and a rebuilt table answer with are identical by construction.
2631 pub fn reach_stats(&self) -> (u64, u64, u64) {
2632 self.absorber.reach_stats()
2633 }
2634 pub(crate) fn refold(&self) -> Result<()> {
2635 self.absorber.refold()
2636 }
2637}
2638
2639impl<S: ObjectIndex> Absorber<S> {
2640 /// `shared` is the writer's descriptor on the same file
2641 /// ([`crate::archive_write::ArchiveWrite::blobs_shared`]); `None` opens one,
2642 /// which is what a caller with no writer does.
2643 fn open(
2644 blobs: &Path,
2645 shared: Option<Arc<File>>,
2646 hash: GitHashKind,
2647 objects: ObjectReadStack<S>,
2648 exploded: Arc<ExplodedCache>,
2649 ) -> Result<Self> {
2650 let reader = match shared {
2651 Some(f) => f,
2652 None => Arc::new(
2653 File::open(blobs)
2654 .with_context(|| format!("opening {} for reads", blobs.display()))?,
2655 ),
2656 };
2657 let label = blobs
2658 .parent()
2659 .map(|p| p.display().to_string())
2660 .unwrap_or_else(|| blobs.display().to_string());
2661 let graph_cache = crate::arms::graph_cache_from_env()?;
2662 if !graph_cache {
2663 // A memo left by an earlier open with the cache on: this store keeps none.
2664 let stale = blobs.with_file_name(GRAPH_CACHE_FILE);
2665 match std::fs::remove_file(&stale) {
2666 Ok(()) => {}
2667 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
2668 Err(e) => tracing::warn!(
2669 target: "znippy.git",
2670 path = %stale.display(),
2671 error = %e,
2672 "the graph cache is off but a stale objects.graph could not be removed"
2673 ),
2674 }
2675 }
2676 Ok(Self {
2677 blobs: blobs.to_path_buf(),
2678 reader,
2679 map: crate::archive_map::ArchiveMap::new(blobs),
2680 hash,
2681 objects,
2682 exploded,
2683 derived: RwLock::new(Derived::default()),
2684 graph_read: std::sync::atomic::AtomicU64::new(0),
2685 graph_recomputed: std::sync::atomic::AtomicU64::new(0),
2686 graph_cache: std::sync::atomic::AtomicBool::new(graph_cache),
2687 reach_carried: std::sync::atomic::AtomicU64::new(0),
2688 reach_rebuilt: std::sync::atomic::AtomicU64::new(0),
2689 reach_loaded: std::sync::atomic::AtomicU64::new(0),
2690 reach_extended: std::sync::atomic::AtomicU64::new(0),
2691 reach_unpersisted: std::sync::atomic::AtomicU64::new(0),
2692 ordinal_folded: std::sync::atomic::AtomicU64::new(0),
2693 ordinal_tail: std::sync::atomic::AtomicU64::new(0),
2694 packs: Mutex::new(PackState::default()),
2695 gate: Mutex::new(()),
2696 progress: crate::progress::IndexProgress::new(&label),
2697 cancel: std::sync::atomic::AtomicBool::new(false),
2698 label,
2699 watchdog: RwLock::new(crate::memguard::global()),
2700 scanned: Mutex::new(HashMap::new()),
2701 scan_build: Mutex::new(()),
2702 unindexable: Mutex::new(HashMap::new()),
2703 })
2704 }
2705
2706 /// **Tree payloads, on demand** — [`crate::reach::TreeSource`] over this
2707 /// store.
2708 ///
2709 /// One lookup per tree the walk actually visits, instead of a resident map
2710 /// of every tree in the repository. Where the bytes come from is
2711 /// [`Absorber::resolved`]'s business and nobody else's: the exploded cache
2712 /// if the tree is in it, its own delta chain out of the mmapped pack if it
2713 /// is not — and the read then writes it back, so a second walk finds it
2714 /// warm.
2715 ///
2716 /// It must not take the `derived` lock, and it does not: `resolved` reads
2717 /// the exploded pools, the `objects` index and the pending packs, none of
2718 /// which is `derived`. Both callers build one of these **inside** their
2719 /// `derived.write()` section, and that is only sound because of this
2720 /// sentence.
2721 fn tree_source(&self) -> TreesFromStore<'_, S> {
2722 TreesFromStore { abs: self, chains: None }
2723 }
2724
2725 /// [`Self::tree_source`] with a [`crate::resolve::ChainCache`] of
2726 /// [`REACH_BUILD_CHAIN_CACHE_BYTES`] behind it — for the one reader that
2727 /// walks every tree: the full reachability build. Dropped with the build.
2728 fn tree_source_for_build(&self, objects: usize) -> TreesFromStore<'_, S> {
2729 // Sized to the repository: a small one must not pay the full memo
2730 // (MEASURED: a 28 251-object repository peaked at 274 MB with the full
2731 // 128 MiB memo against 121 MB without — the memo was most of it).
2732 let cap = objects
2733 .saturating_mul(REACH_BUILD_CHAIN_CACHE_PER_OBJECT)
2734 .clamp(8 << 20, REACH_BUILD_CHAIN_CACHE_BYTES);
2735 TreesFromStore {
2736 abs: self,
2737 chains: Some(Mutex::new(crate::resolve::ChainCache::new(cap))),
2738 }
2739 }
2740
2741 /// Packs whose objects are not in the index yet — §13.12's bit, counted.
2742 fn unindexed_packs(&self) -> usize {
2743 self.packs.lock().expect("pack state lock").unabsorbed
2744 }
2745
2746 /// **Index everything durable, here, now — the ack path's half of §13.9.**
2747 ///
2748 /// # The two hours, in one function
2749 ///
2750 /// MEASURED by lane P2 on the oden rig 2026-08-30. `git.receive_pack.checking`
2751 /// fired at 07:43:49; its very first `peel` of a pushed tip missed the
2752 /// exploded table and missed the `objects` index (the worker was minutes
2753 /// into a 13 948 244-object pack), and fell into [`Self::scan_pending`],
2754 /// which ran **a second, read-only, throw-away resolve of the whole 8.12 GiB
2755 /// pack on the connection thread**:
2756 ///
2757 /// ```text
2758 /// 09:20:21 index.scanned "a reader scanned a pending pack for itself:
2759 /// 13948244 objects in 5792442 ms"
2760 /// ```
2761 ///
2762 /// 1h36m32s on the 16 GB box and 1h44m14s on the 8 GB one, for rows it then
2763 /// threw away with the comment *"keeping them would be the index, and the
2764 /// worker is building that"*. That sentence is the defect: two resolvers
2765 /// over the same bytes, each holding its own O(pack) `PackWalk::entries` and
2766 /// oid table, on four vCPUs — which is also **why the watchdog paused**. The
2767 /// reader's second copy is what ate the `MemAvailable` the indexer tripped
2768 /// over, and neither pass ever finished the check.
2769 ///
2770 /// # Who may call this
2771 ///
2772 /// **The ack path, and nothing else.** It takes [`Absorber::gate`], so it
2773 /// waits for whoever is absorbing — and *"nobody waits on the indexer"* is a
2774 /// law with its own red test
2775 /// (`a_read_before_the_drain_is_served_from_the_pack_and_never_waits`): a
2776 /// clone, an `ls-remote`, a `control status` must be answered while the
2777 /// drain is parked, and they still are, through [`Self::scan_pending`].
2778 ///
2779 /// The connectivity check is the one caller for which that is the wrong
2780 /// trade. It has to prove that every object reachable from 2874 new tips is
2781 /// present, which means reading them; an entry's oid exists only once its
2782 /// delta chain has been applied; so the pack must be resolved before the
2783 /// check can answer, and the only question is whether it is resolved **once
2784 /// or twice**. This makes it once. `git receive-pack` makes the same choice
2785 /// and calls it `index-pack`.
2786 ///
2787 /// If the account worker already holds the gate, this blocks on it and then
2788 /// finds `is_absorbed` set and returns: whoever got there first did the work.
2789 ///
2790 /// Returns how many packs this call absorbed; `0` means it could not (it is
2791 /// nested inside an absorb of the only pending pack, or there was nothing
2792 /// to do).
2793 fn index_pending_now(&self) -> Result<usize> {
2794 if self.unindexed_packs() == 0 {
2795 return Ok(0);
2796 }
2797 // **Nested**: this thread is already inside `absorb_gated` and holds the
2798 // gate, so taking it again is a deadlock, not a wait. The other pending
2799 // packs are still ours to absorb inline — that is lane U's case, a thin
2800 // push whose base sits in a pack pushed a moment earlier — and the one
2801 // we are in the middle of is not.
2802 if let Some(current) = IN_ABSORB.with(|c| c.get()) {
2803 let jobs: Vec<PendingPack> = {
2804 let s = self.packs.lock().map_err(|_| anyhow!("pack state poisoned"))?;
2805 s.pending()
2806 };
2807 return self.absorb_each(&jobs, Some(current));
2808 }
2809 let _gate = self.gate.lock().map_err(|_| anyhow!("absorb gate poisoned"))?;
2810 let jobs: Vec<PendingPack> = {
2811 let s = self.packs.lock().map_err(|_| anyhow!("pack state poisoned"))?;
2812 s.pending()
2813 };
2814 // `absorb_gated` is a no-op for a pack somebody absorbed while we were
2815 // queued on the gate, so the loser of the race pays nothing.
2816 self.absorb_each(&jobs, None)
2817 }
2818
2819 /// **Absorb the pending packs only if nobody else is absorbing.**
2820 ///
2821 /// `Some(n)` = the gate was free and this caller did the work; `None` = the
2822 /// worker (or another reader) holds the gate — possibly paused by the
2823 /// memory watchdog for as long as the box is short — and this caller did
2824 /// **not** wait for it. The owner's rule, 2026-08-30: *nobody waits on the
2825 /// indexer.* A read that needs the index now answers from the pack for
2826 /// itself ([`Self::scan_pending`]) and a read that needs the whole graph
2827 /// says so by name rather than blocking behind a kernel pack.
2828 fn try_absorb_pending(&self) -> Result<Option<usize>> {
2829 if self.unindexed_packs() == 0 {
2830 return Ok(Some(0));
2831 }
2832 let _gate = match self.gate.try_lock() {
2833 Ok(g) => g,
2834 Err(std::sync::TryLockError::WouldBlock) => return Ok(None),
2835 Err(std::sync::TryLockError::Poisoned(_)) => bail!("absorb gate poisoned"),
2836 };
2837 let jobs: Vec<PendingPack> = {
2838 let s = self
2839 .packs
2840 .lock()
2841 .map_err(|_| anyhow!("pack state poisoned"))?;
2842 s.pending()
2843 };
2844 self.absorb_each(&jobs, None).map(Some)
2845 }
2846
2847 /// **A read served from the pack on the reader's own request, never by
2848 /// waiting for the worker.**
2849 ///
2850 /// Every oid in `wants` is looked up in the `objects` table first — the
2851 /// rows the worker has already landed, batch by batch, are visible there
2852 /// with no lock in common with it. The ones still missing, if any pack is
2853 /// still un-indexed, are found by **resolving the pending packs read-only
2854 /// on this thread**: the same resolver the worker runs
2855 /// ([`crate::resolve::resolve_streaming`]), over the same mmapped bytes,
2856 /// with a bounded base set, but with **no sink, no gate and no rows
2857 /// written** — it produces rows for the caller and nothing else. The pass
2858 /// stops the moment every wanted oid has been seen.
2859 ///
2860 /// `capture` names one oid whose **content** the caller wants as well (a
2861 /// [`GitStore::content`] read); it rides through a
2862 /// [`crate::exploded::CaptureOne`] sink and comes back in the second slot.
2863 ///
2864 /// # Cost, stated
2865 ///
2866 /// An oid that is in a pending pack costs one read-only pass up to where it
2867 /// sits; an oid that is **absent** costs a full pass over every pending
2868 /// pack — for a kernel pack, minutes of one core. That is the price of
2869 /// never answering "absent" for an object whose bytes are durable, paid by
2870 /// the reader that asked and by nobody else. The rows it computes are not
2871 /// kept: keeping them would be the index, and the worker is building that.
2872 fn scan_pending(
2873 &self,
2874 wants: &[&[u8]],
2875 capture: Option<&[u8]>,
2876 ) -> Result<(Vec<Option<crate::index_layout::IndexRow>>, Option<(GitObjectKind, Vec<u8>)>)> {
2877 use crate::index_layout::IndexRow;
2878 let mut out: Vec<Option<IndexRow>> = vec![None; wants.len()];
2879 let mut missing: HashMap<&[u8], Vec<usize>> = HashMap::with_capacity(wants.len());
2880 for (i, w) in wants.iter().enumerate() {
2881 missing.entry(*w).or_default().push(i);
2882 }
2883 if missing.is_empty() && capture.is_none() {
2884 return Ok((out, None));
2885 }
2886 // **No nested BUILD.** The resolver asks this absorber (as its
2887 // `BaseSource`) for a thin pack's external base, and that base may
2888 // itself be un-indexed — a second *scan build* started from inside the
2889 // first, and a third from inside that, without end.
2890 //
2891 // The guard used to answer `None` outright when re-entered, and that
2892 // was wider than the recursion it was written for: a base sitting in
2893 // **another pack that is already scanned** is one binary search away,
2894 // and refusing it made a thin push on top of a just-acked one fail
2895 // `[remote rejected] … missing necessary objects` — lane U's rig
2896 // finding, 2026-08-30. So a nested call now consults the scans that
2897 // exist and builds none: a lookup terminates, a build is what does not.
2898 thread_local! { static SCANNING: std::cell::Cell<bool> = const { std::cell::Cell::new(false) }; }
2899 let nested = SCANNING.with(|s| s.get());
2900 struct Unset(bool);
2901 impl Drop for Unset {
2902 fn drop(&mut self) {
2903 if !self.0 {
2904 SCANNING.with(|s| s.set(false));
2905 }
2906 }
2907 }
2908 SCANNING.with(|s| s.set(true));
2909 let _unset = Unset(nested);
2910 let pending: Vec<PendingPack> = {
2911 let s = self
2912 .packs
2913 .lock()
2914 .map_err(|_| anyhow!("pack state poisoned"))?;
2915 s.pending()
2916 };
2917 if pending.is_empty() {
2918 return Ok((out, None));
2919 }
2920 let snap = self.map.snapshot()?;
2921 let mut content: Option<(GitObjectKind, Vec<u8>)> = None;
2922 for job in &pending {
2923 if missing.is_empty() && (capture.is_none() || content.is_some()) {
2924 break;
2925 }
2926 let scanned = match self.cached_scan(job.offset) {
2927 Some(s) => s,
2928 // Inside another scan: no build, so the recursion the guard
2929 // exists for cannot happen, and a pack nobody has scanned yet
2930 // is simply not consulted for this base.
2931 None if nested => continue,
2932 None => self.scanned_pack(&snap, job)?,
2933 };
2934 missing.retain(|oid, slots| match scanned.row(oid) {
2935 Some(r) => {
2936 for &i in slots.iter() {
2937 out[i] = Some(r.clone());
2938 }
2939 false
2940 }
2941 None => true,
2942 });
2943 if let Some(want) = capture
2944 && content.is_none()
2945 && let Some((entry, _)) = scanned.find(want)
2946 {
2947 let bytes: std::borrow::Cow<'_, [u8]> = match snap.get(job.offset, job.len) {
2948 Some(b) => std::borrow::Cow::Borrowed(b),
2949 None => std::borrow::Cow::Owned(self.read_extent(job.offset, job.len)?),
2950 };
2951 let sp = scanned.clone();
2952 let offset_of = move |oid: &[u8]| sp.find(oid).map(|(e, _)| sp.walk.entries[e].offset);
2953 content = Some(crate::resolve::materialize_entry(
2954 &bytes,
2955 &scanned.walk,
2956 entry,
2957 self.hash.oid_len(),
2958 &offset_of,
2959 self,
2960 )?);
2961 }
2962 }
2963 Ok((out, content))
2964 }
2965
2966 /// The scan of one pending pack **if it is already built** — a lookup, and
2967 /// nothing that could recurse. What a nested `scan_pending` is allowed.
2968 fn cached_scan(&self, offset: u64) -> Option<Arc<ScannedPack>> {
2969 self.scanned
2970 .lock()
2971 .unwrap_or_else(|e| e.into_inner())
2972 .get(&offset)
2973 .cloned()
2974 }
2975
2976 /// The scan of one pending pack: cached, or built now by one read-only
2977 /// pass (the same resolver, bounded bases, no sink, no rows written).
2978 fn scanned_pack(&self, snap: &crate::archive_map::Mapped, job: &PendingPack) -> Result<Arc<ScannedPack>> {
2979 if let Some(s) = self.scanned.lock().unwrap_or_else(|e| e.into_inner()).get(&job.offset) {
2980 return Ok(s.clone());
2981 }
2982 let _one_at_a_time = self.scan_build.lock().unwrap_or_else(|e| e.into_inner());
2983 if let Some(s) = self.scanned.lock().unwrap_or_else(|e| e.into_inner()).get(&job.offset) {
2984 return Ok(s.clone());
2985 }
2986 let t0 = std::time::Instant::now();
2987 let bytes: std::borrow::Cow<'_, [u8]> = match snap.get(job.offset, job.len) {
2988 Some(b) => std::borrow::Cow::Borrowed(b),
2989 None => std::borrow::Cow::Owned(self.read_extent(job.offset, job.len)?),
2990 };
2991 let walk = crate::pack_walk::walk(&bytes, self.hash.oid_len())?;
2992 let mut rows: Vec<(Vec<u8>, u32, u64)> = Vec::with_capacity(walk.entries.len());
2993 // A reader's own budget: a quarter of the worker's, no watchdog —
2994 // the pause is the worker's and only the worker's.
2995 let limits = crate::resolve::ResolveLimits {
2996 base_budget: (crate::resolve::base_budget_from_env() / 4).max(1),
2997 cancel: Some(&self.cancel),
2998 label: &self.label,
2999 ..crate::resolve::ResolveLimits::none()
3000 };
3001 // Entry index = position in offset order; the rows come out in
3002 // resolution order, so map offset → index by binary search.
3003 let report = crate::resolve::resolve_streaming(
3004 &bytes,
3005 &walk,
3006 self.hash,
3007 job.offset,
3008 self,
3009 &crate::exploded::NoSink,
3010 limits,
3011 &mut |batch| {
3012 for r in batch.drain(..) {
3013 let pack_off = r.offset - job.offset;
3014 let idx = walk
3015 .entries
3016 .binary_search_by_key(&pack_off, |e| e.offset)
3017 .map_err(|_| anyhow!("resolved row at {pack_off} has no walk entry"))?;
3018 rows.push((r.oid, idx as u32, r.uncompressed_size));
3019 }
3020 Ok(())
3021 },
3022 )
3023 .context("resolving a pending pack read-only for lookups")?;
3024 rows.sort_unstable_by(|a, b| a.0.cmp(&b.0));
3025 let scanned = Arc::new(ScannedPack {
3026 walk,
3027 rows,
3028 archive_offset: job.offset,
3029 });
3030 tracing::info!(
3031 target: "gunnar::index",
3032 event = "index.scanned",
3033 repo = %self.label,
3034 pack_id = job.pack_id,
3035 objects = scanned.rows.len(),
3036 reinflations = report.reinflations,
3037 elapsed_ms = t0.elapsed().as_millis() as u64,
3038 "a reader scanned a pending pack for itself: {} objects in {} ms",
3039 scanned.rows.len(),
3040 t0.elapsed().as_millis()
3041 );
3042 self.scanned
3043 .lock()
3044 .unwrap_or_else(|e| e.into_inner())
3045 .insert(job.offset, scanned.clone());
3046 Ok(scanned)
3047 }
3048
3049 /// **Drop everything this store holds that can be rebuilt** — what the
3050 /// watchdog's flush reaches: the readers' scanned packs (`scanned`, ~1.2
3051 /// GB on linux, rebuilt by the next lookup's pass) and the exploded
3052 /// table's oid tree and batch cache. The rows on disk are untouched.
3053 ///
3054 /// **Returns the rows it actually dropped**, and that number is the point.
3055 /// Lane P2's rig logged this event 190 and 627 times with
3056 /// `scanned_rows_dropped=0` on every single one and kept waiting anyway,
3057 /// because the caller could not tell a spill that freed a gigabyte from one
3058 /// that freed nothing. Three zeros in a row now end the pause
3059 /// ([`crate::memguard::PauseEnd::NothingToGive`]).
3060 ///
3061 /// The exploded table's own caches are an mmap index and a small batch
3062 /// cache; they are dropped here too and deliberately not counted, because
3063 /// they are not where a kernel-sized pack's memory is.
3064 fn spill(&self) -> u64 {
3065 let dropped = {
3066 let mut s = self.scanned.lock().unwrap_or_else(|e| e.into_inner());
3067 let n: usize = s.values().map(|p| p.rows.len()).sum();
3068 s.clear();
3069 n
3070 };
3071 self.exploded.drop_caches();
3072 tracing::info!(
3073 target: "gunnar::index",
3074 event = "index.spilled",
3075 repo = %self.label,
3076 scanned_rows_dropped = dropped,
3077 "the store dropped its rebuildable caches for the memory watchdog"
3078 );
3079 dropped as u64
3080 }
3081
3082 /// This store's watchdog (see the field).
3083 fn watchdog(&self) -> Arc<crate::memguard::MemWatchdog> {
3084 self.watchdog.read().unwrap_or_else(|e| e.into_inner()).clone()
3085 }
3086
3087 /// One oid, never waiting: the index, then the pending packs.
3088 fn lookup_never_wait(&self, oid: &[u8]) -> Result<Option<crate::index_layout::IndexRow>> {
3089 if let Some(r) = self.objects.lookup(oid) {
3090 return Ok(Some(r));
3091 }
3092 // **Check-then-act, and the answer is to look again, not to wait.**
3093 // The index miss above and the bit read below are two reads with the
3094 // drain running between them: the pack's rows can land AND its bit
3095 // clear after the miss, and "no pending pack, so absent" was then a
3096 // lie about a durable object — the one answer §13.12 forbids
3097 // (`no_read_is_told_absent_while_the_drain_is_running` caught it,
3098 // 1 absent answer in 1352, 2 of 10 full-suite runs under load,
3099 // 2026-09-28). The bit clears only AFTER the rows go in, so once it
3100 // reads clear a second index read is authoritative.
3101 if self.unindexed_packs() == 0 {
3102 return Ok(self.objects.lookup(oid));
3103 }
3104 let (mut rows, _) = self.scan_pending(&[oid], None)?;
3105 if let Some(r) = rows.pop().flatten() {
3106 return Ok(Some(r));
3107 }
3108 // The same window one step later: the pack was absorbed and left the
3109 // pending list between the bit read and the scan.
3110 Ok(self.objects.lookup(oid))
3111 }
3112
3113 /// Many oids, never waiting: one index pass, then one read-only pass over
3114 /// the pending packs for whatever the index did not have.
3115 fn lookup_batch_never_wait(&self, oids: &[&[u8]]) -> Result<Vec<Option<crate::index_layout::IndexRow>>> {
3116 let mut rows = self.objects.lookup_batch(oids);
3117 if rows.iter().all(Option::is_some) {
3118 return Ok(rows);
3119 }
3120 // See `lookup_never_wait`: a miss followed by a clear bit is re-read,
3121 // because the drain may have landed the rows between the two.
3122 if self.unindexed_packs() == 0 {
3123 return Ok(self.objects.lookup_batch(oids));
3124 }
3125 let missing: Vec<&[u8]> = oids
3126 .iter()
3127 .zip(&rows)
3128 .filter(|(_, r)| r.is_none())
3129 .map(|(o, _)| *o)
3130 .collect();
3131 let (found, _) = self.scan_pending(&missing, None)?;
3132 let mut by_oid: HashMap<&[u8], crate::index_layout::IndexRow> = HashMap::with_capacity(found.len());
3133 for (o, r) in missing.iter().zip(found) {
3134 if let Some(r) = r {
3135 by_oid.insert(*o, r);
3136 }
3137 }
3138 for (o, slot) in oids.iter().zip(rows.iter_mut()) {
3139 if slot.is_none()
3140 && let Some(r) = by_oid.get(*o)
3141 {
3142 *slot = Some(r.clone());
3143 }
3144 }
3145 // Absorbed between the bit read and the scan: whatever is still
3146 // missing gets one more index read.
3147 if rows.iter().any(Option::is_none) {
3148 for (o, slot) in oids.iter().zip(rows.iter_mut()) {
3149 if slot.is_none() {
3150 *slot = self.objects.lookup(o);
3151 }
3152 }
3153 }
3154 Ok(rows)
3155 }
3156
3157 /// The bit going clear: this pack is durable and its objects are not in.
3158 ///
3159 /// Skips a pack the drain already absorbed — the ack path queues *after* it
3160 /// has handed the extent to the channel, so the worker can legitimately be
3161 /// finished before this runs, and clearing a bit for a pack that is already
3162 /// indexed would leave it clear for ever.
3163 fn queue(&self, pack_id: u64, extent: Extent) -> Result<()> {
3164 let mut s = self
3165 .packs
3166 .lock()
3167 .map_err(|_| anyhow!("pack state poisoned"))?;
3168 s.note(pack_id, extent);
3169 Ok(())
3170 }
3171
3172 /// **The bit, derived from the two durable facts, on open.**
3173 ///
3174 /// `journal` is every row this archive's journal holds, in append order.
3175 /// [`acked_packs`](crate::archive_write::acked_packs) is what turns that into
3176 /// the packs: its index *is* the pack ordinal — dense by construction, and
3177 /// re-derived the same way by every later open. For each one, the question is
3178 /// only *does this pack have any rows in the index*, which
3179 /// [`ObjectReadStack::extents_with_rows`] answers in one early-terminating
3180 /// scan rather than per pack.
3181 ///
3182 /// Returns the packs whose bit came up **clear**: durable bytes, no rows. The
3183 /// caller re-queues them on the account's channel, which is what makes a
3184 /// pack that was mid-absorb when the process died get absorbed by the next
3185 /// one instead of being durable-but-invisible for ever.
3186 ///
3187 /// The verbatim extents are also handed to [`Derived::packs`] — *all* of
3188 /// them, not only the unabsorbed ones. That is what lets [`BaseSource`]
3189 /// re-derive a delta base's content from the truth after a restart; without
3190 /// it a reopened store knows the rows of a pack it can no longer find the
3191 /// bytes of.
3192 ///
3193 /// # The one case where "no rows" does not mean "never absorbed" — and the
3194 /// third durable fact that now tells them apart
3195 ///
3196 /// [`GitOps::gc`] is the only thing that removes rows. A pack **all** of
3197 /// whose objects it finds dead therefore ends with a journal extent and no
3198 /// rows, which is byte for byte the state a pack that was in flight when the
3199 /// machine died leaves behind. This diff cannot tell them apart, and it must
3200 /// not guess: re-queueing the first resurrects every dead object, and *not*
3201 /// re-queueing the second loses a pack whose bytes were acked.
3202 ///
3203 /// So `gc` says which it is, durably, before it drops a row:
3204 /// [`retire_packs`](crate::archive_write::retire_packs) appends a tombstone
3205 /// naming the retired pack's offset, and a pack named by one is never
3206 /// re-queued here. The bit is still derived from durable facts only — there
3207 /// are now three of them (the journal's pack rows, the journal's tombstones,
3208 /// the index's rows) and none of them is a cache of another.
3209 ///
3210 /// A **partly** dead pack is not tombstoned and still has rows, so it takes
3211 /// the `absorbed` branch exactly as it always did.
3212 ///
3213 /// A tombstoned pack that *does* still have rows is a `gc` that was killed
3214 /// between the tombstone and the drop. It is treated as absorbed, which is
3215 /// what its rows say: nothing is lost, and the next `gc` finishes the job.
3216 ///
3217 /// Its extent still goes to [`Derived::packs`] either way. The blob file is
3218 /// append-only and `gc` truncates nothing, so a retired pack's bytes are
3219 /// still exactly where the journal says they are — and a live object may
3220 /// legitimately be a delta against a base that is itself unreachable, so
3221 /// dropping the extent from that list would break the resolve rather than
3222 /// tidy it.
3223 ///
3224 /// # The second durable fact: §14's exploded table
3225 ///
3226 /// A pack counts as absorbed only if the exploded table is **whole**, which
3227 /// is `exploded.rows() >= objects.len()` — one row per object, because §14's
3228 /// table is eager and skips nothing. If it is short, the table has been
3229 /// dropped (or a process died between the two commits) and **no** pack counts
3230 /// as absorbed: every one is re-queued and re-exploded from the verbatim
3231 /// truth. That is the whole of "droppable": deleting the file is a supported
3232 /// operation whose only cost is one re-ingest, and §13.12's rule holds
3233 /// unchanged over it — absent means fall back, never means wrong.
3234 ///
3235 /// `>=` and not `==` on purpose. A resolve that failed part way leaves rows
3236 /// for the objects it got to, and a `gc` retires both tables together, so the
3237 /// table can legitimately hold *more* than the index names; only being
3238 /// **short** is evidence of a gap.
3239 fn adopt_journal(&self, journal: &[Extent]) -> Result<Vec<PendingPack>> {
3240 if journal.is_empty() {
3241 return Ok(Vec::new());
3242 }
3243 // The packs, in ordinal order, and the offsets a `gc` retired. A
3244 // tombstone is a row and not a pack, so it holds no ordinal.
3245 let packs = crate::archive_write::acked_packs(journal);
3246 let retired = crate::archive_write::retired_offsets(journal);
3247 if packs.is_empty() {
3248 return Ok(Vec::new());
3249 }
3250 // Under `Graph` the table is short **by design** — commits and trees
3251 // only — so the row-count comparison would call every store dropped and
3252 // re-queue every pack on every open. It is weakened to "not empty",
3253 // which still catches the case this rule exists for (the file deleted),
3254 // and is stated rather than hidden: making it exact needs a durable
3255 // commit-and-tree count, which the index does not publish today.
3256 // Only `Full` explodes one row per object, so only `Full` can compare
3257 // counts. Under `Commits` (the shipped default) and `Graph` the eager
3258 // pool is short **by design**, and the comparison would call every store
3259 // dropped and re-queue every pack on every open — a Linux-sized re-index
3260 // per restart. The weakened test still catches what the rule exists for
3261 // (the file deleted) and is stated rather than hidden: making it exact
3262 // needs a durable per-kind count, which the index does not publish.
3263 //
3264 // The LAZY pool is deliberately not consulted. Its size is a function of
3265 // what has been READ, so it says nothing about whether a pack was
3266 // absorbed, and an eviction must never look like a lost index.
3267 let exploded_is_whole = match self.exploded.policy() {
3268 crate::exploded_arrow::ExplodePolicy::Full => {
3269 self.exploded.eager_rows()? >= self.objects.len() as u64
3270 }
3271 crate::exploded_arrow::ExplodePolicy::Off => true,
3272 _ => self.objects.len() == 0 || self.exploded.eager_rows()? > 0,
3273 };
3274 let has_rows = if exploded_is_whole {
3275 self.objects.extents_with_rows(&packs)?
3276 } else {
3277 vec![false; packs.len()]
3278 };
3279 {
3280 let mut d = self
3281 .derived
3282 .write()
3283 .map_err(|_| anyhow!("derived poisoned"))?;
3284 d.packs = packs.clone();
3285 }
3286 let mut s = self
3287 .packs
3288 .lock()
3289 .map_err(|_| anyhow!("pack state poisoned"))?;
3290 let mut requeue = Vec::new();
3291 for (i, (&extent, &absorbed)) in packs.iter().zip(&has_rows).enumerate() {
3292 let pack_id = i as u64;
3293 if absorbed {
3294 s.mark_absorbed(pack_id, extent);
3295 } else if retired.contains(&extent.0) {
3296 // A `gc` dropped every row this pack had and said so durably
3297 // *before* dropping them. There is no index work owed for it,
3298 // ever: re-queueing it is precisely how its dead objects would
3299 // come back. The bit goes **up** rather than being left clear,
3300 // because a clear bit is what makes a read fall back to
3301 // absorbing — which is the same resurrection by another door.
3302 s.mark_absorbed(pack_id, extent);
3303 } else {
3304 s.note(pack_id, extent);
3305 requeue.push(PendingPack {
3306 pack_id,
3307 offset: extent.0,
3308 len: extent.1,
3309 resumed: true,
3310 });
3311 }
3312 }
3313 Ok(requeue)
3314 }
3315
3316 /// **The indexer's half of the split**, and not one of the twelve.
3317 ///
3318 /// Reads each pending pack's verbatim bytes back out of the archive,
3319 /// resolves them to oids ([`crate::resolve`]), appends the rows to the
3320 /// `objects` table and re-folds the derived tables. This is what §13.9-12
3321 /// puts behind the channel; it runs after the ack, never before one.
3322 ///
3323 /// Two callers, one function: the account indexer's worker (through
3324 /// [`ObjectAbsorb`]) and a read that arrived before that worker did. The
3325 /// second is §13.12's fallback in its literal form — *slower, never wrong*.
3326 fn absorb_pending(&self) -> Result<usize> {
3327 // Nothing to do, and — this is the point — **no gate taken**: the common
3328 // case is a store whose drain has kept up, and it must not queue behind
3329 // one that is running.
3330 if self.unindexed_packs() == 0 {
3331 return Ok(0);
3332 }
3333 let _gate = self
3334 .gate
3335 .lock()
3336 .map_err(|_| anyhow!("absorb gate poisoned"))?;
3337 let jobs: Vec<PendingPack> = {
3338 let s = self
3339 .packs
3340 .lock()
3341 .map_err(|_| anyhow!("pack state poisoned"))?;
3342 s.pending()
3343 };
3344 self.absorb_each(&jobs, None)
3345 }
3346
3347 /// **Every pack it can, skipping the ones it has already given up on.**
3348 ///
3349 /// The one loop all four absorb entry points run (LAW 5): they were four
3350 /// copies of `for job in &jobs { self.absorb_gated(job)?; }`, and that `?`
3351 /// is the bug this function exists to remove. Two rules, and both are
3352 /// consequences of the same sentence — *work that can never succeed must
3353 /// stop and say why*:
3354 ///
3355 /// 1. **A failure does not abandon the packs behind it.** The loop runs to
3356 /// the end and reports the FIRST failure afterwards, so a pack that
3357 /// cannot be indexed no longer stops the push that arrived after it from
3358 /// being indexed and acked.
3359 /// 2. **A pack that has been given up on is not attempted again.** After
3360 /// [`MAX_ABSORB_ATTEMPTS`] it is recorded in
3361 /// [`Absorber::unindexable`] with the reason, logged once as
3362 /// `index.unindexable`, and skipped from then on. Its objects are then
3363 /// genuinely absent from the index, which every read path already knows
3364 /// how to say — a push that needs one is refused by name at
3365 /// `external_bases_exist`, and a clone that needs one fails the
3366 /// connectivity check. What no longer happens is the store re-resolving
3367 /// it on every request for the rest of the process's life.
3368 ///
3369 /// `skip` is the pack this thread is standing inside an absorb of
3370 /// ([`IN_ABSORB`]).
3371 fn absorb_each(&self, jobs: &[PendingPack], skip: Option<u64>) -> Result<usize> {
3372 let mut absorbed = 0usize;
3373 let mut first: Option<anyhow::Error> = None;
3374 for job in jobs {
3375 if skip == Some(job.pack_id) || self.given_up_on(job.pack_id) {
3376 continue;
3377 }
3378 match self.absorb_gated(job) {
3379 Ok(()) => absorbed += 1,
3380 Err(e) => {
3381 // Cancellation is the repository being deleted: there is no
3382 // queue left worth draining and nothing to record.
3383 if self.cancel.load(std::sync::atomic::Ordering::Relaxed) {
3384 return Err(e);
3385 }
3386 self.note_absorb_failure(job, &e);
3387 if first.is_none() {
3388 first = Some(e);
3389 }
3390 }
3391 }
3392 }
3393 match first {
3394 None => Ok(absorbed),
3395 Some(e) => Err(e),
3396 }
3397 }
3398
3399 /// Every pack this store has given up on, ordinal order.
3400 fn unindexable_packs(&self) -> Vec<Unindexable> {
3401 let mut v: Vec<Unindexable> = self
3402 .unindexable
3403 .lock()
3404 .unwrap_or_else(|e| e.into_inner())
3405 .values()
3406 .cloned()
3407 .collect();
3408 v.sort_by_key(|u| u.pack_id);
3409 v
3410 }
3411
3412 /// Has this store stopped trying to index that pack?
3413 fn given_up_on(&self, pack_id: u64) -> bool {
3414 self.unindexable
3415 .lock()
3416 .unwrap_or_else(|e| e.into_inner())
3417 .contains_key(&pack_id)
3418 }
3419
3420 /// Count one failed absorb, and stop at [`MAX_ABSORB_ATTEMPTS`].
3421 ///
3422 /// The reason recorded is `root_cause`, not the outermost wrapper: that is
3423 /// the sentence the backend actually has to offer, and lane Z's rule is that
3424 /// the client gets it verbatim rather than a layer's paraphrase of it.
3425 fn note_absorb_failure(&self, job: &PendingPack, err: &anyhow::Error) {
3426 let giving_up;
3427 {
3428 let mut map = self.unindexable.lock().unwrap_or_else(|e| e.into_inner());
3429 // A pack already given up on is not attempted, so an entry here is
3430 // always a fresh count.
3431 let e = map.entry(job.pack_id).or_insert_with(|| Unindexable {
3432 pack_id: job.pack_id,
3433 extent: (job.offset, job.len),
3434 attempts: 0,
3435 reason: String::new(),
3436 });
3437 e.attempts += 1;
3438 e.reason = format!("{}", err.root_cause());
3439 if e.attempts < MAX_ABSORB_ATTEMPTS {
3440 let attempts = e.attempts;
3441 map.remove(&job.pack_id);
3442 tracing::warn!(
3443 target: "znippy.git",
3444 store = %self.label,
3445 pack = job.pack_id,
3446 attempts,
3447 max = MAX_ABSORB_ATTEMPTS,
3448 error = %err.root_cause(),
3449 "a pack could not be indexed; it stays pending and will be tried again"
3450 );
3451 return;
3452 }
3453 giving_up = Some(e.clone());
3454 }
3455 if let Some(u) = giving_up {
3456 tracing::error!(
3457 target: "znippy.git",
3458 store = %self.label,
3459 event = "index.unindexable",
3460 pack = u.pack_id,
3461 offset = u.extent.0,
3462 len = u.extent.1,
3463 attempts = u.attempts,
3464 error = %u.reason,
3465 "this store has STOPPED trying to index this pack after {} attempts. Its bytes \
3466 are durable and untouched, and every object it introduced is absent from the \
3467 index — a push that deltas against one is refused by name and a clone that \
3468 needs one fails its connectivity check, both immediately. Nothing will retry \
3469 it: re-push the pack, or delete and re-create the repository.",
3470 u.attempts
3471 );
3472 }
3473 }
3474
3475 /// One pack, **with [`Absorber::gate`] already held**.
3476 ///
3477 /// The order at the end is the correctness argument: the rows are in the
3478 /// index *before* the bit is cleared. A read that finds the bit still clear
3479 /// blocks on the gate and then finds the rows; a read that finds it set finds
3480 /// the rows too. There is no interleaving in which a durable object answers
3481 /// "absent" — which during negotiation would make a client withhold objects
3482 /// and lose data.
3483 fn absorb_gated(&self, job: &PendingPack) -> Result<()> {
3484 if self.cancel.load(std::sync::atomic::Ordering::Relaxed) {
3485 // The repository is being deleted: no pack still queued is worth a
3486 // byte of work, and the drain must get through its queue fast so
3487 // the store can be dropped and the directory removed.
3488 return Err(crate::resolve::ResolveError::cancelled());
3489 }
3490 {
3491 let s = self
3492 .packs
3493 .lock()
3494 .map_err(|_| anyhow!("pack state poisoned"))?;
3495 if s.is_absorbed(job.pack_id) {
3496 // Already in, by whoever reached it first. Re-resolving is now
3497 // *correct* — the exploded table is keyed by oid and the graph is
3498 // folded from it rather than pushed at, so a second absorb of one
3499 // pack changes nothing (that idempotence is what makes dropping
3500 // the table and re-queueing every pack a safe operation). It is
3501 // still a whole pack re-read, re-inflated and re-hashed for
3502 // nothing, so it is skipped.
3503 return Ok(());
3504 }
3505 }
3506 // From here to the end of `absorb_one` this thread holds the gate, and
3507 // the resolver is about to ask this same store for delta bases. See
3508 // [`IN_ABSORB`]: the re-entry must not take the gate again and must not
3509 // try to absorb the pack it is standing in.
3510 let _in = InAbsorb::enter(job.pack_id);
3511 self.absorb_one(job).map_err(|e| {
3512 // The entry stays pending by construction — it was never removed —
3513 // so the bytes are durable, the pack is still un-indexed, and reads
3514 // keep falling back rather than answering absent.
3515 e.context(format!(
3516 "absorbing pack {} at ({}, {}) — its bytes are durable and it stays \
3517 un-indexed; reads will keep falling back rather than answer absent",
3518 job.pack_id, job.offset, job.len
3519 ))
3520 })?;
3521 let mut s = self
3522 .packs
3523 .lock()
3524 .map_err(|_| anyhow!("pack state poisoned"))?;
3525 s.mark_absorbed(job.pack_id, (job.offset, job.len));
3526 drop(s);
3527 // The index answers for it now; the reader's scan is dead weight.
3528 self.scanned.lock().unwrap_or_else(|e| e.into_inner()).remove(&job.offset);
3529 Ok(())
3530 }
3531
3532 /// One pack: verbatim bytes in, `objects` rows **and** §14's exploded rows
3533 /// out, from **one** walk and **one** resolve.
3534 ///
3535 /// The exploded rows are not a second pass and not a second resolver (LAW 5):
3536 /// [`crate::resolve::resolve_walked`] hands every object it produces to the
3537 /// sink as it produces it, including the blob payloads it is about to drop,
3538 /// so the side table falls out of the pass that was already being made.
3539 ///
3540 /// Ordering is the correctness argument, and it is the same one the drain
3541 /// makes one level up: **the exploded rows are committed before the fold that
3542 /// reads them**, and the fold runs before this returns, so there is no
3543 /// interleaving in which the graph is folded over a table that does not yet
3544 /// hold this pack's commits.
3545 fn absorb_one(&self, job: &PendingPack) -> Result<()> {
3546 let t0 = std::time::Instant::now();
3547 self.progress.begin(job.pack_id, job.len, job.resumed);
3548 // **A live row every 10 s for as long as this pack takes**, carrying
3549 // `peak_rss` and the re-inflation count. Those two rode only on
3550 // `index.finished`, so lane P2's two rig boxes — which ran for 2h14m
3551 // and 2h12m and never finished a pack — produced neither, in exactly
3552 // the failure they were built for. The ticker is a thread and not a
3553 // hook inside the resolve loop on purpose: it prints while the resolver
3554 // is PAUSED too, which is when an operator most needs it.
3555 let _tick = crate::progress::ProgressTicker::start(self.progress.clone());
3556 let result = self.absorb_one_inner(job, t0);
3557 self.progress.sample_memory();
3558 self.progress.finish();
3559 result
3560 }
3561
3562 /// The body of [`absorb_one`](Self::absorb_one), so `finish` runs on every
3563 /// exit and the progress row never sticks at 99 %.
3564 fn absorb_one_inner(&self, job: &PendingPack, t0: std::time::Instant) -> Result<()> {
3565 use crate::progress::Phase;
3566 let progress = &self.progress;
3567 // **The pack is read where it already is.** `read_extent` was a
3568 // `pread` of the whole extent into a fresh `Vec` — 8.12 GiB of
3569 // anonymous heap for torvalds/linux, before one byte was inflated, on a
3570 // 16 GB box. The mapping costs nothing the kernel cannot reclaim
3571 // ([`crate::archive_map`]), and `MemAvailable` — what the watchdog
3572 // reads — knows that.
3573 let snap = self.map.snapshot()?;
3574 let bytes: std::borrow::Cow<'_, [u8]> = match snap.get(job.offset, job.len) {
3575 Some(b) => std::borrow::Cow::Borrowed(b),
3576 // Past the mapping: the file grew after the snapshot. Same bytes,
3577 // one `pread` — the exception, not the shape.
3578 None => std::borrow::Cow::Owned(self.read_extent(job.offset, job.len)?),
3579 };
3580 progress.set_phase(Phase::Walk);
3581 let walked = crate::pack_walk::walk(&bytes, self.hash.oid_len())?;
3582 let t_walk = t0.elapsed();
3583 let dog = self.watchdog();
3584 let spill = || self.spill();
3585 let limits = crate::resolve::ResolveLimits::for_indexer(
3586 &self.label,
3587 &self.cancel,
3588 progress,
3589 dog,
3590 &spill,
3591 );
3592 let mut batches = 0u64;
3593 let mut appended = 0u64;
3594 let mut entries: Vec<crate::index_layout::IndexEntry> = Vec::with_capacity(limits.row_batch);
3595 let report = crate::resolve::resolve_streaming(
3596 &bytes,
3597 &walked,
3598 self.hash,
3599 job.offset,
3600 self,
3601 &*self.exploded,
3602 limits,
3603 &mut |rows| {
3604 // **One batch of rows lands, then the next.** The exploded
3605 // tier already flushed itself every 64 MiB; the index rows
3606 // used to wait for the whole pack. Now the batch is the unit:
3607 // rows go to the `objects` table as they are produced, and the
3608 // pack's own "indexed" bit still goes up only after the LAST
3609 // batch — `mark_absorbed` is the caller's, after this returns.
3610 entries.clear();
3611 entries.extend(rows.iter().map(|r| r.index_entry()));
3612 rows.clear();
3613 self.objects.append(&entries)?;
3614 appended += entries.len() as u64;
3615 batches += 1;
3616 Ok(())
3617 },
3618 )?;
3619 drop(entries);
3620 let t_resolve = t0.elapsed() - t_walk;
3621 progress.set_phase(Phase::Flush);
3622 // Down before anything folds over it.
3623 self.exploded.flush()?;
3624 // A push is what fills the disk, so the push is what asks the cache to
3625 // give space back (the disk line, `exploded_cache::DISK_LINE_PERCENT`).
3626 // Until 2026-09-28 only a reader's write-back started the maintenance
3627 // thread, so a store nobody read never looked at the disk. `kick`
3628 // spawns and returns: the indexer does not wait for an unlink.
3629 self.exploded.kick();
3630
3631 {
3632 let mut d = self
3633 .derived
3634 .write()
3635 .map_err(|_| anyhow!("derived poisoned"))?;
3636 // A reopen already listed every acked extent (`adopt_journal`), so
3637 // this is the first absorb of a pack pushed in *this* process — or a
3638 // re-queued one that is already listed. `find` is what reads this
3639 // list, so a duplicate would not be wrong, only unbounded.
3640 if !d.packs.contains(&(job.offset, job.len)) {
3641 d.packs.push((job.offset, job.len));
3642 }
3643 }
3644 progress.set_phase(Phase::Fold);
3645 let t_fold0 = std::time::Instant::now();
3646 // The graph and the tree payloads are folded from the exploded table, not
3647 // pushed at from here. That is what makes a re-absorb idempotent — which
3648 // it has to be, because dropping the table re-queues every pack.
3649 self.refold()?;
3650 let _ = self.objects.maybe_rebuild()?;
3651 let t_fold = t_fold0.elapsed();
3652 // **The audit line**: every number is applied output of this absorb.
3653 // `peak_rss` is the process's VmHWM off /proc/self/status — the high
3654 // water mark of the whole server, which on the appliance is the box.
3655 tracing::info!(
3656 target: "gunnar::index",
3657 event = "index.finished",
3658 repo = %self.label,
3659 pack_id = job.pack_id,
3660 pack_bytes = job.len,
3661 peak_rss = crate::memguard::peak_rss_bytes(),
3662 mem_available = self.watchdog().available().unwrap_or(0),
3663 objects = report.rows,
3664 batches = batches,
3665 rows_appended = appended,
3666 reinflations = report.reinflations,
3667 evictions = report.evictions,
3668 watchdog_drops = report.watchdog_drops,
3669 peak_base_bytes = report.peak_base_bytes,
3670 pauses = report.pauses,
3671 paused_ms = report.paused_ms,
3672 walk_ms = t_walk.as_millis() as u64,
3673 resolve_ms = t_resolve.as_millis() as u64,
3674 fold_ms = t_fold.as_millis() as u64,
3675 total_ms = t0.elapsed().as_millis() as u64,
3676 resumed = job.resumed,
3677 "indexed {} objects in {} batches: walk {} ms, resolve {} ms, fold {} ms; peak bases {}, {} evictions, {} re-inflations, {} pauses ({} ms); peak rss {}",
3678 report.rows,
3679 batches,
3680 t_walk.as_millis(),
3681 t_resolve.as_millis(),
3682 t_fold.as_millis(),
3683 crate::memguard::fmt_bytes(report.peak_base_bytes),
3684 report.evictions,
3685 report.reinflations,
3686 report.pauses,
3687 report.paused_ms,
3688 crate::memguard::fmt_bytes(crate::memguard::peak_rss_bytes())
3689 );
3690 Ok(())
3691 }
3692
3693 /// Recompute everything derived from the object set: the commit graph, the
3694 /// tree payloads, the ordinal space, the generation numbers, and (lazily) the
3695 /// bitmaps.
3696 ///
3697 /// **The graph is folded from §14's exploded table, not accumulated.** That
3698 /// is the whole fix for the clean-reopen hole: commit and tree payloads are
3699 /// on disk, so this call produces the same graph after a restart as before
3700 /// one, and it is called at the end of [`GitStore::open_with_arms`] for
3701 /// exactly that reason. It also makes a re-absorb idempotent — the old
3702 /// accumulate-into-a-`Vec` shape doubled every commit if a pack was absorbed
3703 /// twice, which is why dropping the table and re-queueing everything was not
3704 /// a safe operation before this.
3705 ///
3706 /// **Generations are stored and must be recomputed in any fold** (§13,
3707 /// decided). A generation number is `1 + max(parents)`, so it is invalidated
3708 /// the moment history's shape changes — which is exactly what absorbing a
3709 /// pack does. [`assign_generations`] is the one writer of that number and it
3710 /// is called here, over the whole graph, rather than incrementally: an
3711 /// incremental update would have to know which descendants moved, and being
3712 /// wrong about that produces an ancestry cutoff that skips real ancestors.
3713 ///
3714 /// # Cost
3715 ///
3716 /// One range scan of the exploded table's commits and one of its trees, per
3717 /// absorbed pack. It is the same order as the `tail_oids` scan below, which
3718 /// this function already made on every absorb, and it reads **no blob
3719 /// payload** — that is what the kind index in
3720 /// [`crate::exploded`] is for.
3721 ///
3722 /// ## And that order is the whole repository, on every push
3723 ///
3724 /// Said plainly, because "the same order as the scan below" reads like a
3725 /// reassurance and is not one: **nothing here is proportional to the push.**
3726 /// Every call rebuilds every tree payload, every commit node, every
3727 /// generation number, a `HashMap<String, u32>` over every oid in the
3728 /// repository and a sorted `Vec<String>` of the same — for a push of one
3729 /// commit exactly as for a push of ten thousand.
3730 ///
3731 /// MEASURED on oden 2026-08-11, release, quiet box (`some avg10` 0.00). The
3732 /// **same** 100-commit increment, pushed onto the same history seeded to
3733 /// four different depths, server CPU for the push:
3734 ///
3735 /// | objects already in the repository | CPU |
3736 /// |---:|---:|
3737 /// | 9 614 | 0.12 s |
3738 /// | 35 350 | 0.23 s |
3739 /// | 103 988 | 0.45 s |
3740 /// | 206 925 | 0.77 s |
3741 ///
3742 /// That is a straight line: **≈88 ms fixed, plus ≈3.3 µs per object already
3743 /// stored**, and the fit predicts the four points to within 20 ms. It is
3744 /// also why a push is slower than the ack path suggests — §13.9 puts the
3745 /// index behind a channel, but `put_refs` checks every ref target with
3746 /// `has`, `has` calls `lookup_one`, and `lookup_one` absorbs pending packs
3747 /// first. The drain is off the *ack* and squarely on the *push*.
3748 ///
3749 /// This is not fixed here and the numbers are recorded so the next attempt
3750 /// starts from them rather than from a guess. The obvious lever — fold
3751 /// incrementally — is the one the paragraph above rules out for
3752 /// generations, so it needs its own argument and its own gate.
3753 fn refold(&self) -> Result<()> {
3754 // Everything derived is computed **before** the write lock, which is then
3755 // held only for the swap: the fold is the long part and nothing that
3756 // reads `derived` should queue behind it.
3757 //
3758 // Ordinals first — **read, not re-scanned, when the tail has not moved.**
3759 // S-082's fix, mirroring the commit-graph memo one paragraph down:
3760 // `objects.ordinal` beside the pack holds `oids_raw` (every oid in the
3761 // store, in ordinal = sorted-raw order), and a cold open reads its couple
3762 // of megabytes back sequentially in milliseconds instead of walking the
3763 // whole redb B-tree — REFOLD measured that scan at ~65% (0.45 s) of a
3764 // 136k-object reopen, and proved a flat buffer measured the same, so the
3765 // cost is the SCAN, not the allocation. It is a memo, not a second truth:
3766 // the redb tail stays authoritative and the file is discarded unless its
3767 // (epoch, count) both match the live tail, so a push, a gc or a torn
3768 // write all fall back to the scan and re-persist.
3769 //
3770 // The raw concatenation is BOTH the ordinal space and the validity token
3771 // for the commit graph below, so the graph is read whenever the object
3772 // set is unchanged; the hex form and the ordinal map fold out of the same
3773 // bytes, so ordinal `o` names the same object in every representation by
3774 // construction rather than by agreeing computations (LAW 5). The tail
3775 // hands oids in oid order, which is byte order over fixed-width oids and
3776 // therefore already the sorted concatenation the memo stores.
3777 //
3778 // **2026-09-04: the memo is gone, because the scan it spared is gone.**
3779 // The read stack persists its projection now (`objects.index`), and the
3780 // sorted oid list IS that index's oid column — one contiguous copy out
3781 // of a mapped buffer, plus a merge of the few rows redb still holds
3782 // since the last fold. No file of its own, no header, no epoch to
3783 // validate: the projection is current by construction. What is kept is
3784 // the provenance — how many oids came off the index and how many off
3785 // the tail — because that is the only thing that can prove the space
3786 // was not a scan (both paths yield identical bytes).
3787 let space = self.objects.oids_raw()?;
3788 self.ordinal_folded
3789 .store(space.folded_rows, std::sync::atomic::Ordering::Relaxed);
3790 self.ordinal_tail
3791 .store(space.tail_rows, std::sync::atomic::Ordering::Relaxed);
3792 let oids_raw = space.raw;
3793 let token = graph_token(&oids_raw);
3794
3795 // The commit graph — **read, not recomputed, when the object set has not
3796 // moved.** S-065's fix: `objects.graph` beside the pack holds the folded
3797 // graph with generations, written by this same call at derive time and
3798 // read here. It is not a second truth — it is a memoized projection
3799 // validated on every read against the authoritative object set (the
3800 // token is a blake3 over every oid in the store), and an absent file, a
3801 // stale token or a decode failure all fall back to the fold and
3802 // re-persist. Same shape as lane REFCAS persisting a tag's peeled target.
3803 //
3804 // On a fold, commit payloads are read out of §14's exploded table (no
3805 // tree payload — that was lane Q's 28.24 GB and most of the 50.9 GiB RSS
3806 // on `torvalds/linux`), parsed, and generation-numbered. That is the
3807 // whole of the O(commits) open cost this persistence removes.
3808 let oid_len = self.hash.oid_len();
3809 let cached = if self.graph_cache.load(std::sync::atomic::Ordering::Relaxed) {
3810 self.load_persisted_graph(&token)?
3811 } else {
3812 None
3813 };
3814
3815 // The whole fold runs under the write lock. It is the short part now:
3816 // after a push the graph is extended by the pushed commits, the ordinal
3817 // space appends, and the reachability table gets entries for the new
3818 // tips — all O(pushed), whatever the repository's size. The one long
3819 // fold is the first one of a store with no memo, and nothing reads a
3820 // store before its open returns.
3821 let mut d = self
3822 .derived
3823 .write()
3824 .map_err(|_| anyhow!("derived poisoned"))?;
3825
3826 // ── the ordinal space: follow the store, never renumber ───────────
3827 //
3828 // `appended` is the range of ordinals this fold added. `None` means the
3829 // space was (re)built from the sorted list — the first fold, or an
3830 // object removed under the table (gc, retain) — and every entry of the
3831 // reachability table was dropped with it.
3832 let mut space_loaded = false;
3833 if d.ordinals.is_empty() && d.reach.is_empty() {
3834 // A cold open: the persisted table, if it is still exact.
3835 if let Some((ordinals, table)) = self.load_persisted_reach(oid_len)? {
3836 d.ordinals = ordinals;
3837 d.reach = Arc::new(table);
3838 space_loaded = true;
3839 }
3840 }
3841 let appended: Option<std::ops::Range<u32>> = if d.ordinals.is_empty() {
3842 None
3843 } else {
3844 d.ordinals.follow(&oids_raw)
3845 };
3846 if appended.is_none() {
3847 d.ordinals = crate::reach::OrdinalTable::from_sorted(oids_raw.clone(), oid_len);
3848 d.reach = Arc::default();
3849 space_loaded = false;
3850 }
3851 if space_loaded {
3852 self.reach_loaded
3853 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
3854 }
3855
3856 // ── the commit graph ──────────────────────────────────────────────
3857 let graph: Vec<CommitNode> = match cached {
3858 Some(g) => {
3859 self.graph_read.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
3860 g
3861 }
3862 None => {
3863 self.graph_recomputed
3864 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
3865 let g = match &appended {
3866 // **Extend, do not re-derive.** The previous fold's rows are
3867 // exact (a commit is immutable); only the objects this fold
3868 // appended can be new commits, and the exploded table says
3869 // which of them are, without resolving a single delta.
3870 Some(r) if !d.graph.is_empty() => {
3871 let mut nodes = std::mem::take(&mut d.graph);
3872 let space = d.ordinals.view();
3873 for o in r.clone() {
3874 let Some(oid) = space.oid(o) else { continue };
3875 if let Some((GitObjectKind::Commit, payload)) = self.exploded.content(oid)? {
3876 let h = crate::object::parse_commit(&payload);
3877 nodes.push(CommitNode {
3878 oid: hex::encode(oid),
3879 parents: h.parents,
3880 tree: h.tree,
3881 committer_time: h.committer_time,
3882 generation: 0,
3883 });
3884 }
3885 }
3886 assign_generations(nodes)
3887 }
3888 _ => {
3889 let commits = self.exploded.of_kind(GitObjectKind::Commit)?;
3890 let g: Vec<CommitNode> = commits
3891 .iter()
3892 .map(|(oid, payload)| {
3893 let h = crate::object::parse_commit(payload);
3894 CommitNode {
3895 oid: hex::encode(oid),
3896 parents: h.parents,
3897 tree: h.tree,
3898 committer_time: h.committer_time,
3899 generation: 0,
3900 }
3901 })
3902 .collect();
3903 assign_generations(g)
3904 }
3905 };
3906 // Persist for the next open. A write failure is not fatal to the
3907 // fold — the graph is correct in memory and the next open simply
3908 // recomputes — so it is logged, not propagated.
3909 if !self.graph_cache.load(std::sync::atomic::Ordering::Relaxed) {
3910 // Switched off: no memo on this disk.
3911 } else if let Err(e) = self.persist_graph(&token, &g) {
3912 tracing::warn!(store = %self.label, "could not persist objects.graph: {e:#}");
3913 }
3914 g
3915 }
3916 };
3917 let mut ord_graph = crate::reach::OrdGraph::from_nodes(&graph, &d.ordinals.view());
3918
3919 // ── the reachability table: extend for the pushed commits ─────────
3920 //
3921 // Entries describe immutable commits over a space that just appended,
3922 // so every one of them stands. What is new is the pushed commits; the
3923 // new tips (and a sample of a long pushed branch) get entries by the
3924 // bounded walk, inside this fold, so the first fetch after the push
3925 // pays nothing. See `crate::reach::extend_reach_table`.
3926 let mut reach_changed = false;
3927 if !d.reach.is_empty() {
3928 if let Some(r) = &appended {
3929 let new_rows: Vec<u32> = (0..ord_graph.len() as u32)
3930 .filter(|&row| {
3931 let o = ord_graph.ords[row as usize];
3932 o != crate::reach::NONE && o >= r.start
3933 })
3934 .collect();
3935 if !new_rows.is_empty() {
3936 let trees = self.tree_source();
3937 let facts = crate::reach::ObjectFacts {
3938 ordinal: d.ordinals.view(),
3939 trees: &trees,
3940 oid_len,
3941 };
3942 let started = std::time::Instant::now();
3943 let extended = crate::reach::extend_reach_table(
3944 &d.reach,
3945 &ord_graph,
3946 &facts,
3947 &new_rows,
3948 live_reach_policy(),
3949 );
3950 tracing::info!(
3951 store = %self.label,
3952 new_commits = new_rows.len(),
3953 entries = extended.len(),
3954 elapsed_ms = started.elapsed().as_millis() as u64,
3955 "fold: extended the reachability table for the pushed commits"
3956 );
3957 d.reach = Arc::new(extended);
3958 self.reach_extended
3959 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
3960 reach_changed = true;
3961 }
3962 }
3963 self.reach_carried
3964 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
3965 }
3966
3967 // ── compaction: re-sort the space once it has grown enough ────────
3968 let mut compacted = false;
3969 if d.ordinals.extra_len() > d.ordinals.base_len() / ORDINAL_COMPACT_FRACTION
3970 && d.ordinals.extra_len() > ORDINAL_COMPACT_MIN
3971 {
3972 let fresh = crate::reach::OrdinalTable::from_sorted(oids_raw, oid_len);
3973 let remap = d.ordinals.remap_to(&fresh.view());
3974 let kept = d.reach.remapped(&remap).unwrap_or_default();
3975 tracing::info!(
3976 store = %self.label,
3977 appended = d.ordinals.extra_len(),
3978 base = d.ordinals.base_len(),
3979 entries = kept.len(),
3980 "fold: compacted the ordinal space (re-sorted, table renumbered once)"
3981 );
3982 d.ordinals = fresh;
3983 d.reach = Arc::new(kept);
3984 ord_graph = crate::reach::OrdGraph::from_nodes(&graph, &d.ordinals.view());
3985 compacted = true;
3986 reach_changed = true;
3987 }
3988
3989 d.graph = graph;
3990 d.reach_by_row = Arc::new(d.reach.by_row(&ord_graph));
3991 d.ord_graph = Arc::new(ord_graph);
3992 d.token = token;
3993 tracing::info!(
3994 store = %self.label,
3995 graph_len = d.graph.len(),
3996 space_len = d.ordinals.len(),
3997 appended = appended.as_ref().map(|r| r.len()).unwrap_or(0),
3998 space_rebuilt = appended.is_none(),
3999 reach_entries = d.reach.len(),
4000 reach_loaded = space_loaded,
4001 "fold: derived tables"
4002 );
4003 if reach_changed && !d.reach.is_empty() {
4004 let due = self
4005 .reach_unpersisted
4006 .fetch_add(1, std::sync::atomic::Ordering::Relaxed)
4007 + 1;
4008 if compacted || due >= REACH_PERSIST_EVERY {
4009 self.persist_reach_locked(&d);
4010 }
4011 }
4012 Ok(())
4013 }
4014
4015 /// Write the live reachability table and its space, if the memo is on.
4016 /// A failure is logged, never propagated: the table is right in memory and
4017 /// the next open rebuilds.
4018 fn persist_reach_locked(&self, d: &Derived) {
4019 if !self.graph_cache.load(std::sync::atomic::Ordering::Relaxed) || d.reach.is_empty() {
4020 return;
4021 }
4022 match self.persist_reach(&d.ordinals, &d.reach) {
4023 Ok(()) => {
4024 self.reach_unpersisted
4025 .store(0, std::sync::atomic::Ordering::Relaxed);
4026 }
4027 Err(e) => tracing::warn!(store = %self.label, "could not persist objects.reach: {e:#}"),
4028 }
4029 }
4030 /// `(reach_carried, reach_rebuilt)` — how many folds since open carried the
4031 /// reachability table across (renumbered) vs how many reads built it from
4032 /// scratch. See [`tests::a_push_carries_the_reach_table_across_the_fold`].
4033 fn reach_source_stats(&self) -> (u64, u64) {
4034 (
4035 self.reach_carried.load(std::sync::atomic::Ordering::Relaxed),
4036 self.reach_rebuilt.load(std::sync::atomic::Ordering::Relaxed),
4037 )
4038 }
4039
4040 /// `(loaded, extended, unpersisted)`; see [`GitStore::reach_stats`].
4041 fn reach_stats(&self) -> (u64, u64, u64) {
4042 (
4043 self.reach_loaded.load(std::sync::atomic::Ordering::Relaxed),
4044 self.reach_extended.load(std::sync::atomic::Ordering::Relaxed),
4045 self.reach_unpersisted.load(std::sync::atomic::Ordering::Relaxed),
4046 )
4047 }
4048
4049 /// Where the folded commit graph is cached, beside the verbatim pack:
4050 /// `<root>/objects.graph`.
4051 fn graph_cache_path(&self) -> PathBuf {
4052 self.blobs.with_file_name(GRAPH_CACHE_FILE)
4053 }
4054
4055 /// The persisted graph and the token it was written under, or `None` when
4056 /// the cache is absent, truncated, an unknown version, or fails to decode —
4057 /// every one of which is a signal to recompute and re-persist rather than an
4058 /// error, because the cache is a memo and the exploded table is the truth.
4059 fn load_persisted_graph(&self, token: &[u8; 32]) -> Result<Option<Vec<CommitNode>>> {
4060 let bytes = match std::fs::read(self.graph_cache_path()) {
4061 Ok(b) => b,
4062 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
4063 Err(e) => {
4064 return Err(e).with_context(|| {
4065 format!("reading {}", self.graph_cache_path().display())
4066 });
4067 }
4068 };
4069 // Frame: b"GRPH" (4) + version (1) + token (32) + Arrow IPC graph.
4070 //
4071 // **The token is checked BEFORE the decode.** It used to be after: every
4072 // fold read and fully decoded the old graph — a `String` per commit and
4073 // per parent — and then threw it away, because after any push the token
4074 // cannot match. The miss is now one 32-byte compare.
4075 if bytes.len() < GRAPH_CACHE_HEADER || &bytes[..4] != GRAPH_CACHE_MAGIC || bytes[4] != 1 {
4076 return Ok(None);
4077 }
4078 if &bytes[5..GRAPH_CACHE_HEADER] != token {
4079 return Ok(None);
4080 }
4081 match crate::graph::decode_graph(&bytes[GRAPH_CACHE_HEADER..]) {
4082 Ok(g) => Ok(Some(g)),
4083 Err(_) => Ok(None),
4084 }
4085 }
4086
4087 fn reach_cache_path(&self) -> PathBuf {
4088 self.blobs.with_file_name(REACH_CACHE_FILE)
4089 }
4090
4091 /// **The persisted reachability table**: its ordinal space (base and the
4092 /// appended part, raw) and the entries as Arrow IPC. `Ok(None)` when absent,
4093 /// stale or malformed — all three mean "build it again", never a panic:
4094 /// the IPC part goes through znippy-common's guard and the raw parts are
4095 /// length-checked before a single slice is taken.
4096 ///
4097 /// Frame: b"RECH" (4) + version (1) + oid_len (u32) + base bytes (u64) +
4098 /// extra bytes (u64) + base + extra + Arrow IPC `(commit_ordinal, bitmap)`.
4099 fn load_persisted_reach(&self, oid_len: usize) -> Result<Option<(crate::reach::OrdinalTable, crate::reach::ReachTable)>> {
4100 if !self.graph_cache.load(std::sync::atomic::Ordering::Relaxed) {
4101 return Ok(None);
4102 }
4103 let bytes = match std::fs::read(self.reach_cache_path()) {
4104 Ok(b) => b,
4105 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
4106 Err(e) => {
4107 return Err(e).with_context(|| format!("reading {}", self.reach_cache_path().display()));
4108 }
4109 };
4110 if bytes.len() < REACH_CACHE_HEADER || &bytes[..4] != REACH_CACHE_MAGIC || bytes[4] != 1 {
4111 return Ok(None);
4112 }
4113 let w = u32::from_le_bytes(bytes[5..9].try_into().expect("4 bytes")) as usize;
4114 let base_len = u64::from_le_bytes(bytes[9..17].try_into().expect("8 bytes")) as usize;
4115 let extra_len = u64::from_le_bytes(bytes[17..25].try_into().expect("8 bytes")) as usize;
4116 if w != oid_len {
4117 return Ok(None);
4118 }
4119 let Some(ipc_at) = REACH_CACHE_HEADER.checked_add(base_len).and_then(|a| a.checked_add(extra_len)) else {
4120 return Ok(None);
4121 };
4122 if ipc_at > bytes.len() {
4123 return Ok(None);
4124 }
4125 let base = &bytes[REACH_CACHE_HEADER..REACH_CACHE_HEADER + base_len];
4126 let extra = &bytes[REACH_CACHE_HEADER + base_len..ipc_at];
4127 let Some(ordinals) = crate::reach::OrdinalTable::from_base_and_extra(base.to_vec(), extra.to_vec(), oid_len) else {
4128 return Ok(None);
4129 };
4130 match crate::reach::decode_live_table(&bytes[ipc_at..], ordinals.len() as u32) {
4131 Ok(table) if !table.is_empty() => Ok(Some((ordinals, table))),
4132 Ok(_) => Ok(None),
4133 Err(e) => {
4134 tracing::warn!(store = %self.label, "objects.reach is unreadable, rebuilding: {e:#}");
4135 Ok(None)
4136 }
4137 }
4138 }
4139
4140 /// Write the live table and its space durably: encode, write a temp file,
4141 /// fsync it, then rename over the cache — the rename is the atomic publish.
4142 fn persist_reach(&self, ordinals: &crate::reach::OrdinalTable, table: &crate::reach::ReachTable) -> Result<()> {
4143 use std::io::Write as _;
4144 let ipc = crate::reach::encode_live_table(table)?;
4145 let (base, extra) = (ordinals.base_raw(), ordinals.extra_raw());
4146 let mut out = Vec::with_capacity(REACH_CACHE_HEADER + base.len() + extra.len() + ipc.len());
4147 out.extend_from_slice(REACH_CACHE_MAGIC);
4148 out.push(1);
4149 out.extend_from_slice(&(ordinals.oid_len() as u32).to_le_bytes());
4150 out.extend_from_slice(&(base.len() as u64).to_le_bytes());
4151 out.extend_from_slice(&(extra.len() as u64).to_le_bytes());
4152 out.extend_from_slice(base);
4153 out.extend_from_slice(extra);
4154 out.extend_from_slice(&ipc);
4155 let path = self.reach_cache_path();
4156 let tmp = path.with_extension("reach.tmp");
4157 {
4158 let mut f = File::create(&tmp).with_context(|| format!("creating {}", tmp.display()))?;
4159 f.write_all(&out).with_context(|| format!("writing {}", tmp.display()))?;
4160 f.sync_all().with_context(|| format!("fsync {}", tmp.display()))?;
4161 }
4162 std::fs::rename(&tmp, &path).with_context(|| format!("publishing {}", path.display()))?;
4163 Ok(())
4164 }
4165 /// Write the folded graph and its token durably: encode, write a temp file,
4166 /// fsync it, then rename over the cache — the rename is the atomic publish,
4167 /// so a reader never sees a half-written graph.
4168 fn persist_graph(&self, token: &[u8; 32], nodes: &[CommitNode]) -> Result<()> {
4169 use std::io::Write as _;
4170 let ipc = crate::graph::encode_graph(nodes)?;
4171 let mut out = Vec::with_capacity(GRAPH_CACHE_HEADER + ipc.len());
4172 out.extend_from_slice(GRAPH_CACHE_MAGIC);
4173 out.push(1);
4174 out.extend_from_slice(token);
4175 out.extend_from_slice(&ipc);
4176
4177 let path = self.graph_cache_path();
4178 let tmp = path.with_extension("graph.tmp");
4179 {
4180 let mut f = File::create(&tmp)
4181 .with_context(|| format!("creating {}", tmp.display()))?;
4182 f.write_all(&out)
4183 .with_context(|| format!("writing {}", tmp.display()))?;
4184 f.sync_all()
4185 .with_context(|| format!("fsync {}", tmp.display()))?;
4186 }
4187 std::fs::rename(&tmp, &path)
4188 .with_context(|| format!("publishing {}", path.display()))?;
4189 Ok(())
4190 }
4191
4192 /// `(graph_read_from_disk, graph_recomputed)` — how many folds since open
4193 /// read the persisted graph vs recomputed it. The only thing that can prove
4194 /// the persisted path ran; see
4195 /// [`tests::a_reopen_reads_the_persisted_graph_it_does_not_recompute`].
4196 fn graph_source_stats(&self) -> (u64, u64) {
4197 (
4198 self.graph_read.load(std::sync::atomic::Ordering::Relaxed),
4199 self.graph_recomputed
4200 .load(std::sync::atomic::Ordering::Relaxed),
4201 )
4202 }
4203
4204 /// `(folded_rows, tail_rows)` of the ordinal space the last fold built — how
4205 /// many oids came straight off the persisted index's oid column and how many
4206 /// off a scan of the redb tail. The only thing that can prove the space was
4207 /// not a scan (LAW 2), because both sources yield identical bytes; see
4208 /// [`tests::the_ordinal_space_comes_off_the_index_not_a_scan`].
4209 fn ordinal_source_stats(&self) -> (u64, u64) {
4210 (
4211 self.ordinal_folded.load(std::sync::atomic::Ordering::Relaxed),
4212 self.ordinal_tail.load(std::sync::atomic::Ordering::Relaxed),
4213 )
4214 }
4215
4216 /// The table, built on first use after a fold that could not carry one
4217 /// (the first fold, a gc) and kept across every fold after that.
4218 fn reach_bitmaps(&self) -> Result<Arc<crate::reach::ReachTable>> {
4219 self.reach_bitmaps_with(live_reach_policy(), true)
4220 }
4221
4222 /// [`Absorber::reach_bitmaps`] with the policy **named** and the cache
4223 /// optional. The table alone; [`Absorber::reach_table_with`] is the one
4224 /// the negotiation calls, because it needs the row index too.
4225 fn reach_bitmaps_with(
4226 &self,
4227 policy: crate::reach::ReachPolicy,
4228 cache: bool,
4229 ) -> Result<Arc<crate::reach::ReachTable>> {
4230 Ok(self.reach_table_with(policy, cache)?.0)
4231 }
4232
4233 /// **The table and its row index, as one pair that is always in step.**
4234 ///
4235 /// `cache` is false for the differential guard, which builds the table at
4236 /// two very different caps over one store and requires the two to answer
4237 /// identically; letting either poison `d.reach` would make the second arm
4238 /// read the first one's table and the comparison would be of a thing
4239 /// against itself.
4240 fn reach_table_with(
4241 &self,
4242 policy: crate::reach::ReachPolicy,
4243 cache: bool,
4244 ) -> Result<(Arc<crate::reach::ReachTable>, Arc<Vec<u32>>)> {
4245 {
4246 let d = self
4247 .derived
4248 .read()
4249 .map_err(|_| anyhow!("derived poisoned"))?;
4250 if cache && !d.reach.is_empty() {
4251 // Two refcount bumps, nothing copied.
4252 return Ok((Arc::clone(&d.reach), Arc::clone(&d.reach_by_row)));
4253 }
4254 if d.graph.is_empty() {
4255 return Ok((Arc::default(), Arc::default()));
4256 }
4257 }
4258 let mut d = self
4259 .derived
4260 .write()
4261 .map_err(|_| anyhow!("derived poisoned"))?;
4262 // Built before the facts so it outlives them, and deliberately NOT
4263 // holding the `derived` guard: `Absorber::resolved` does not take that
4264 // lock, which is what makes a live tree lookup legal from inside this
4265 // write section at all.
4266 let trees = self.tree_source_for_build(d.ordinals.len());
4267 let oid_len = self.hash.oid_len();
4268 let facts = crate::reach::ObjectFacts {
4269 ordinal: d.ordinals.view(),
4270 trees: &trees,
4271 oid_len,
4272 };
4273 // The build reads (nearly) every tree in the repository once, and the
4274 // lazy exploded cache writes each one back for the next reader.
4275 // MEASURED on git.git (dwarf, 330 535 tree reads): suppressing the
4276 // write-back changed nothing (27.1 s of reads against 25.2 s with it,
4277 // noise on a shared box); the cost is the delta chain each tree is
4278 // inflated out of, not the cache. So the cache is kept warm — the
4279 // bounded walks a later fetch makes read trees through it.
4280 let started = std::time::Instant::now();
4281 let built = crate::reach::build_reach_table(&d.ord_graph, &facts, policy);
4282 let by_row = built.by_row(&d.ord_graph);
4283 self.reach_rebuilt
4284 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
4285 let chains = trees.chain_stats();
4286 tracing::info!(
4287 store = %self.label,
4288 entries = built.len(),
4289 commits = d.graph.len(),
4290 objects = d.ordinals.len(),
4291 elapsed_ms = started.elapsed().as_millis() as u64,
4292 chain_hits = chains.map(|c| c.0).unwrap_or(0),
4293 chain_misses = chains.map(|c| c.1).unwrap_or(0),
4294 chain_bytes = chains.map(|c| c.2).unwrap_or(0),
4295 cache,
4296 "built the reachability table from scratch"
4297 );
4298 if std::env::var_os(crate::arms::ENV_REACH_TIMING).is_some() {
4299 eprintln!("reach build chain memo: {chains:?} (hits, misses, bytes)");
4300 }
4301 let built = Arc::new(built);
4302 let by_row = Arc::new(by_row);
4303 if cache {
4304 d.reach = Arc::clone(&built);
4305 d.reach_by_row = Arc::clone(&by_row);
4306 // A full build is always worth writing down: it is the one cost a
4307 // restart would otherwise pay again.
4308 self.persist_reach_locked(&d);
4309 }
4310 Ok((built, by_row))
4311 }
4312 // `ordinal_space()` lived here: `d.oids.clone()`, a fresh `Vec<String>` of
4313 // every oid in the store, handed to `reachable_oids` twice per request. It
4314 // is gone, and so is `d.oids` itself (S-082): the serve path borrows
4315 // `d.oids_raw` as an `OrdinalSpace` under the read guard it was already
4316 // going to take, and reads oid ⇄ ordinal straight off it.
4317
4318 /// `pread` an extent out of the verbatim archive, **into uninitialised
4319 /// capacity** — no zero-fill before the read.
4320 ///
4321 /// This used to be `vec![0u8; len]` + `read_exact_at`, and under emit
4322 /// concurrency the zeroing was not a detail: profiled on oden 2026-08-12
4323 /// (gunnar serving 32 concurrent clones, perf on the server pid), ~44 % of
4324 /// serve CPU was memory zeroing/copying — `memset` 18.1 % plus kernel
4325 /// page-zeroing 7.8 % — against ~12 % for one clone, because 32 threads
4326 /// churning pack-scale buffers recycle mimalloc freelist blocks that must
4327 /// each be memset before `pread` immediately overwrites every byte.
4328 ///
4329 /// Soundness: no reference to uninitialised memory is ever formed. The
4330 /// bytes are written through a raw pointer into the `Vec`'s spare
4331 /// capacity, `EINTR` retries, a short read refuses, and `set_len` runs
4332 /// only after every one of `len` bytes has been written.
4333 fn read_extent(&self, offset: u64, len: u64) -> Result<Vec<u8>> {
4334 use std::os::fd::AsRawFd;
4335 let want = usize::try_from(len).context("extent length overflows usize")?;
4336 let mut buf: Vec<u8> = Vec::with_capacity(want);
4337 let fd = self.reader.as_raw_fd();
4338 let mut filled = 0usize;
4339 while filled < want {
4340 let n = unsafe {
4341 libc::pread(
4342 fd,
4343 buf.as_mut_ptr().add(filled).cast(),
4344 want - filled,
4345 i64::try_from(offset + filled as u64).context("extent offset overflows off_t")?,
4346 )
4347 };
4348 if n < 0 {
4349 let err = std::io::Error::last_os_error();
4350 if err.kind() == std::io::ErrorKind::Interrupted {
4351 continue;
4352 }
4353 return Err(err).with_context(|| {
4354 format!("reading ({offset}, {len}) out of {}", self.blobs.display())
4355 });
4356 }
4357 if n == 0 {
4358 bail!(
4359 "short read at ({offset}, {len}) out of {}: {filled} of {want} bytes before \
4360 EOF — the archive is truncated relative to its index",
4361 self.blobs.display()
4362 );
4363 }
4364 filled += n as usize;
4365 }
4366 // Every byte of `want` is now initialised.
4367 unsafe { buf.set_len(want) };
4368 Ok(buf)
4369 }
4370}
4371
4372/// **The drain's object-level ingress**, and the reason a push ends in object
4373/// rows without anybody reading first.
4374///
4375/// The account indexer holds this as an `Arc<dyn ObjectAbsorb>` and calls it once
4376/// per drained job, on its own thread, after the ack. It is the *same* absorb a
4377/// falling-back read takes, gate and all (LAW 5): there is no background copy of
4378/// this logic to drift.
4379impl<S: ObjectIndex> ObjectAbsorb for Absorber<S> {
4380 fn absorb(&self, job: IndexJob) -> Result<()> {
4381 // **The memory watchdog, BEFORE the pack and BEFORE the gate**
4382 // ([`crate::memguard`]). Inside the pack the resolver checks again
4383 // every few thousand objects; this is the check that keeps a burst of
4384 // packs from starting one more absorb on a box that is already short.
4385 // Taken before the gate on purpose: a paused worker holds nothing —
4386 // not the gate, not a row, not a base — so a reader that wants to
4387 // absorb for itself can, and every other reader scans the pack.
4388 if self.cancel.load(std::sync::atomic::Ordering::Relaxed) {
4389 return Err(crate::resolve::ResolveError::cancelled());
4390 }
4391 let dog = self.watchdog();
4392 if dog.breached().is_some() {
4393 self.progress.begin(job.pack_id, job.len, false);
4394 self.progress.set_phase(crate::progress::Phase::Paused);
4395 self.progress.add_pause();
4396 let r = dog.pause_until_ok(&self.label, Some(&self.cancel), &mut || Ok(self.spill()));
4397 self.progress.finish();
4398 let paused = r?;
4399 // The pause before the pack has the same rule as the one inside it:
4400 // a give-up is not a resume. Starting an 8 GiB pack on a box that
4401 // just told us it has nothing left is how the parked hours began.
4402 if paused.end.gave_up() {
4403 return Err(crate::resolve::ResolveError::out_of_memory(
4404 paused.end.as_str(),
4405 paused.available,
4406 paused.budget,
4407 0,
4408 0,
4409 ));
4410 }
4411 }
4412 let _gate = self
4413 .gate
4414 .lock()
4415 .map_err(|_| anyhow!("absorb gate poisoned"))?;
4416 self.absorb_gated(&PendingPack::from_job(job))
4417 }
4418}
4419
4420impl<S: ObjectIndex + 'static> GitStore<S> {
4421 /// The push path, for the two `put` halves.
4422 pub(crate) fn push_path(&self) -> &PushPath {
4423 &self.push
4424 }
4425
4426 pub(crate) fn account(&self) -> &str {
4427 &self.account
4428 }
4429
4430 pub(crate) fn ref_log(&self) -> &RefLog {
4431 &self.refs
4432 }
4433
4434 pub(crate) fn ref_gate(&self) -> &Mutex<()> {
4435 &self.ref_gate
4436 }
4437
4438 /// The closure check's index half: a `REF_DELTA`'s base has to exist —
4439 /// **in the store, or in this very pack**.
4440 ///
4441 /// This is the one part of receive-pack that reads `objects.oid` (§13's
4442 /// table), and on the path a real push takes it reads nothing else.
4443 ///
4444 /// # The second half of that sentence used to be missing, and it refused
4445 /// packs git itself writes
4446 ///
4447 /// [`crate::pack_walk::PackWalk::closure`] cannot do better on its own:
4448 /// a walk reports what the bytes say, and the bytes of a `REF_DELTA` name a
4449 /// base by **oid**, which is a fact about resolved content that no walk can
4450 /// know. So every ref base came back on `external_refs` and every one of
4451 /// them was required to be in the store already.
4452 ///
4453 /// A `REF_DELTA` naming a base *inside the same pack* is ordinary and git
4454 /// writes it constantly: `git index-pack --fix-thin` completes a pushed thin
4455 /// pack by **appending the base object to the pack** and leaving the delta
4456 /// naming it by oid. Every such pack was refused with *"this pack deltas
4457 /// against X, which this repository does not have"*, on a store that was
4458 /// being handed a perfectly ordinary, self-contained packfile — the base was
4459 /// entry 1 of 25 in the pack's own `.idx` (`gunnar.multi_pack_serve`,
4460 /// 2026-08-14). The same missing check is `S-003` / gitoxide#2882 in
4461 /// `LookupRefDeltaObjectsIter`; it is the same defect wearing the receiving
4462 /// hat instead of the sending one.
4463 ///
4464 /// # What it costs, and where
4465 ///
4466 /// **Nothing on any push that was going to succeed before.** Every base
4467 /// found in the store short-circuits exactly as it did, and a pack with no
4468 /// ref-delta at all never gets past the first loop.
4469 ///
4470 /// The pack is resolved only when the alternative is *refusing it*, and then
4471 /// the resolve is the cheapest honest answer available: an entry's oid is
4472 /// knowable only by applying its delta chain, which is precisely what
4473 /// [`crate::resolve::resolve_walked`] does — the same resolver the absorb is
4474 /// about to run over the same pack anyway, reused rather than twinned
4475 /// (LAW 5). A push that is genuinely thin against a base nobody has still
4476 /// fails, one resolve later, and still names the oid.
4477 pub(crate) fn external_bases_exist(&self, bytes: &[u8], w: &PackWalk) -> Result<()> {
4478 let c = w.closure();
4479 if !c.broken_offsets.is_empty() {
4480 bail!(
4481 "this pack is corrupt: {} delta base offset(s) do not land on an entry — first {}",
4482 c.broken_offsets.len(),
4483 c.broken_offsets[0]
4484 );
4485 }
4486 // The fast path, unchanged: a base the store already holds is settled
4487 // without reading one byte of the pack.
4488 let mut unheld: Vec<&[u8]> = Vec::new();
4489 for oid in &c.external_refs {
4490 if self.lookup_one(oid)?.is_none() {
4491 unheld.push(oid.as_slice());
4492 }
4493 }
4494 if unheld.is_empty() {
4495 return Ok(());
4496 }
4497
4498 // Only now, and only because the alternative is refusing the push: does
4499 // the pack supply these itself? `resolve_walked` answers with the
4500 // store behind it, so a base that really is external is still found the
4501 // cheap way and only a genuinely absent one fails.
4502 let resolved = crate::resolve::resolve_walked(
4503 bytes,
4504 w,
4505 self.hash_kind(),
4506 0,
4507 &*self.absorber,
4508 &crate::exploded::NoSink,
4509 )
4510 .with_context(|| {
4511 format!(
4512 "this pack deltas against {} object(s) this repository does not have, so it was \
4513 resolved to find out whether the pack carries them itself — first {}",
4514 unheld.len(),
4515 hex::encode(unheld[0])
4516 )
4517 })?;
4518 let in_pack: std::collections::HashSet<&[u8]> =
4519 resolved.iter().map(|r| r.oid.as_slice()).collect();
4520 for oid in unheld {
4521 if !in_pack.contains(oid) {
4522 bail!(
4523 "this pack deltas against {}, which this repository does not have and which \
4524 the pack does not carry either — the push is refused rather than stored with \
4525 a dangling base",
4526 hex::encode(oid)
4527 );
4528 }
4529 }
4530 Ok(())
4531 }
4532
4533 /// One oid, on the **serial** path. See [`lookup_path`].
4534 pub(crate) fn lookup_one(&self, oid: Oid<'_>) -> Result<Option<crate::index_layout::IndexRow>> {
4535 // **Never waits on the indexer.** The rows the worker has landed so far
4536 // are in the index; an oid it has not reached is resolved from the
4537 // pending pack on this thread (`Absorber::scan_pending`).
4538 self.absorber.lookup_never_wait(oid)
4539 }
4540
4541 /// Many oids, on the **batch** path, never waiting on the indexer.
4542 pub(crate) fn lookup_batch_never_wait(
4543 &self,
4544 oids: &[Oid<'_>],
4545 ) -> Result<Vec<Option<crate::index_layout::IndexRow>>> {
4546 self.absorber.lookup_batch_never_wait(oids)
4547 }
4548
4549 /// Absorb the pending packs on this thread **if the gate is free**;
4550 /// `None` = the worker holds it and this caller did not wait. See
4551 /// `Absorber::try_absorb_pending`.
4552 pub fn try_absorb_pending(&self) -> Result<Option<usize>> {
4553 self.absorber.try_absorb_pending()
4554 }
4555
4556 /// The named refusal for a request that needs the whole derived graph
4557 /// while a pack is still being indexed: `reachable` cannot be answered
4558 /// from a graph that does not hold the pack's commits yet, and waiting for
4559 /// the worker is forbidden. The message carries the progress row's
4560 /// numbers so the client knows when to come back.
4561 fn index_in_progress(&self) -> anyhow::Error {
4562 let p = self.index_progress();
4563 anyhow!(
4564 "index in progress for this repository: pack {} {}% ({}/{} objects, {}) — reachability \
4565 needs the whole graph; retry when the index row reaches 100 %",
4566 p.pack_id,
4567 p.percent(),
4568 p.objects_done,
4569 p.objects_total,
4570 p.phase.as_str()
4571 )
4572 }
4573
4574 /// The current ref namespace, as the log folds to it. The whole-snapshot
4575 /// accessor; [`ref_get`](Self::ref_get) is the keyed one.
4576 pub(crate) fn ref_state(&self) -> Result<BTreeMap<String, crate::refs::RefState>> {
4577 self.refs.current()
4578 }
4579
4580 /// One ref by name, O(log r), without cloning the namespace — for `head`
4581 /// and a single-ref CAS. See [`crate::refs::RefLog::get`].
4582 pub(crate) fn ref_get(&self, name: &str) -> Result<Option<crate::refs::RefState>> {
4583 self.refs.get(name)
4584 }
4585
4586 /// Reachability over the graph, as roaring bitmaps, in **this store's**
4587 /// ordinal space.
4588 ///
4589 /// Returns `(want ∪ closure) − (have ∪ closure)` as oid hex. Every object —
4590 /// commit, tree and blob — is in the bitmaps, because
4591 /// [`crate::reach::build_reach`] walks the trees; that is why this can answer
4592 /// with objects rather than only with commits.
4593 ///
4594 /// **Hex is for the maintenance path, not the serving one.** The answer is
4595 /// computed in raw bytes by [`GitStore::reachable_raw_with`] and encoded
4596 /// here, at one `String` per object; every caller that serves a request asks
4597 /// for the raw form instead. See [`Derived::oids_raw`] for what that cost.
4598 pub(crate) fn reachable_oids(&self, want: &[Oid<'_>], have: &[Oid<'_>]) -> Result<Vec<String>> {
4599 Ok(self
4600 .reachable_raw_with(want, have, live_reach_policy(), true)?
4601 .iter()
4602 .map(hex::encode)
4603 .collect())
4604 }
4605
4606 /// [`GitStore::reachable_oids`] with the bitmap policy **named** and the
4607 /// cache optional.
4608 ///
4609 /// One body, not two (LAW 5): both are [`GitStore::reachable_raw_with`], so
4610 /// the guarded path and the shipped path are the same code.
4611 #[cfg(test)]
4612 pub(crate) fn reachable_oids_with(
4613 &self,
4614 want: &[Oid<'_>],
4615 have: &[Oid<'_>],
4616 policy: crate::reach::ReachPolicy,
4617 cache: bool,
4618 ) -> Result<Vec<String>> {
4619 Ok(self
4620 .reachable_raw_with(want, have, policy, cache)?
4621 .iter()
4622 .map(hex::encode)
4623 .collect())
4624 }
4625
4626 /// `(want ∪ closure) − (have ∪ closure)`, in **raw oid bytes**, in one
4627 /// allocation.
4628 ///
4629 /// The answer every serving caller wants and the one the bitmaps naturally
4630 /// produce: an ordinal indexes straight into [`Derived::oids_raw`], so
4631 /// nothing is encoded, decoded or allocated per object on the way out.
4632 pub(crate) fn reachable_raw(
4633 &self,
4634 want: &[Oid<'_>],
4635 have: &[Oid<'_>],
4636 ) -> Result<git_storage_trait::OidList> {
4637 self.reachable_raw_with(want, have, live_reach_policy(), true)
4638 }
4639
4640 /// **The closure of `tips` out of the last committed fold, never waiting
4641 /// on the indexer and never refusing for it.** For the thin-fetch voucher
4642 /// ([`Self::client_bases`]): a tip the fold does not hold contributes
4643 /// nothing, which is the safe direction for a voucher. A `want` must not
4644 /// be answered this way — a stale graph would under-send — so this takes
4645 /// the tips as `have` and asks for nothing else.
4646 pub(crate) fn reachable_raw_committed(&self, tips: &[Oid<'_>]) -> Result<git_storage_trait::OidList> {
4647 Ok(self
4648 .select_with_gate(&[], tips, live_reach_policy(), true, false, GateBusy::UseCommitted)?
4649 .map(|s| s.client_has)
4650 .unwrap_or_default())
4651 }
4652
4653 /// [`GitStore::reachable_raw`] with the bitmap policy **named** and the
4654 /// cache optional.
4655 ///
4656 /// It exists because the answer is defined to be **independent of the
4657 /// policy** — [`crate::reach::accumulate`] covers whatever the table does
4658 /// not — and a property like that can only be tested by driving the policy.
4659 /// A test that could not reach it would be asserting that one code path
4660 /// equals itself. `cache: false` keeps one arm's table from being read by
4661 /// the next, which is the same trap in a different hat.
4662 pub(crate) fn reachable_raw_with(
4663 &self,
4664 want: &[Oid<'_>],
4665 have: &[Oid<'_>],
4666 policy: crate::reach::ReachPolicy,
4667 cache: bool,
4668 ) -> Result<git_storage_trait::OidList> {
4669 Ok(self
4670 .select_with(want, have, policy, cache, false)?
4671 .map(|s| s.objects)
4672 .unwrap_or_default())
4673 }
4674
4675 /// **`select`, in integers, under one read guard** — the whole answer
4676 /// [`crate::serve::GitServe::select`] hands back, computed once: `want −
4677 /// have` (raw, one allocation), the commits among them (a bitmap AND, not a
4678 /// hash probe per object) and the closure of `have` (which this already
4679 /// computed to subtract it — it used to be computed a second time to hand
4680 /// back as `client_has`).
4681 ///
4682 /// `require_commit_tips` is `select`'s refusal: a `want` that is not a
4683 /// commit row declines the whole request (`Ok(None)`) rather than
4684 /// under-sending. Without it (the `reachable` contract) a non-commit want
4685 /// contributes itself and what it peels to, and a want the store does not
4686 /// hold at all is an error.
4687 pub(crate) fn select_with(
4688 &self,
4689 want: &[Oid<'_>],
4690 have: &[Oid<'_>],
4691 policy: crate::reach::ReachPolicy,
4692 cache: bool,
4693 require_commit_tips: bool,
4694 ) -> Result<Option<Selected>> {
4695 self.select_with_gate(want, have, policy, cache, require_commit_tips, GateBusy::Refuse)
4696 }
4697
4698 /// [`Self::select_with`] with the busy-gate answer **named**.
4699 fn select_with_gate(
4700 &self,
4701 want: &[Oid<'_>],
4702 have: &[Oid<'_>],
4703 policy: crate::reach::ReachPolicy,
4704 cache: bool,
4705 require_commit_tips: bool,
4706 busy: GateBusy,
4707 ) -> Result<Option<Selected>> {
4708 // The graph is folded by the absorb, so a pending pack has to be
4709 // absorbed before reachability can be right. If the gate is free this
4710 // thread does it; if the worker holds it — hours into a kernel pack,
4711 // or paused by the watchdog — a whole-graph question is refused by
4712 // name, not waited on, and a voucher is answered from the last
4713 // committed fold (see [`GateBusy`]).
4714 if self.try_absorb_pending()?.is_none() {
4715 match busy {
4716 GateBusy::Refuse => return Err(self.index_in_progress()),
4717 GateBusy::UseCommitted => {}
4718 }
4719 }
4720 // `reach_table_with` may take the derived WRITE lock to build, so it
4721 // runs to completion before the read guard below is taken. Getting that
4722 // order wrong is a self-deadlock, not a slow path.
4723 let (table, by_row) = self.absorber.reach_table_with(policy, cache)?;
4724 // **One read guard, held across the whole answer, and nothing cloned
4725 // out of it.** Borrowing is not a lock held longer than the work: the
4726 // guard is a reader, folds take the writer, and a fold that lands
4727 // mid-answer would invalidate the rows this is translating.
4728 let d = self
4729 .absorber
4730 .derived
4731 .read()
4732 .map_err(|_| anyhow!("derived poisoned"))?;
4733 let space = d.ordinals.view();
4734 let oid_len = self.hash.oid_len();
4735 let graph: &crate::reach::OrdGraph = &d.ord_graph;
4736 let view = crate::reach::ReachView {
4737 table: &table,
4738 by_row: &by_row,
4739 graph,
4740 };
4741 let trees = self.absorber.tree_source();
4742 let facts = crate::reach::ObjectFacts {
4743 ordinal: space,
4744 trees: &trees,
4745 oid_len,
4746 };
4747
4748 if require_commit_tips {
4749 if graph.is_empty() {
4750 return Ok(None);
4751 }
4752 for oid in want {
4753 let Some(o) = space.ordinal_of_raw(oid) else { return Ok(None) };
4754 if !graph.is_commit(o) {
4755 return Ok(None);
4756 }
4757 }
4758 }
4759
4760 let mut union = roaring::RoaringBitmap::new();
4761 // **The bitmapped wants first, in one pass, then the rest.** The union
4762 // is closed after every step, so a want whose commit is already in it
4763 // is answered by `accumulate`'s first check and walks nothing — and on
4764 // a clone of git.git, 1 006 wants, that is nearly every tag once the
4765 // branch tips are in. Union is commutative, so the order is free.
4766 let mut later: Vec<&[u8]> = Vec::new();
4767 for oid in want {
4768 if let Some(o) = space.ordinal_of_raw(oid) {
4769 if let Some(bm) = graph.row_of(o).and_then(|row| view.bitmap_of_row(row)) {
4770 union |= bm;
4771 continue;
4772 }
4773 }
4774 later.push(oid);
4775 }
4776 for oid in later {
4777 let Some(o) = space.ordinal_of_raw(oid) else {
4778 bail!(
4779 "want {} is not in this repository — a negotiation must not be answered from a \
4780 partial set",
4781 hex::encode(oid)
4782 );
4783 };
4784 match graph.row_of(o) {
4785 // A commit: its whole closure, in one OR — or the bounded walk
4786 // down to the first commit that has one. A commit the graph
4787 // holds is always answerable; only a want the store does not
4788 // hold at all is refused, above.
4789 Some(row) => match view.bitmap_of_row(row) {
4790 Some(bm) => union |= bm,
4791 None => crate::reach::accumulate(row, &view, &facts, &mut union),
4792 },
4793 // Not a commit: a TAG or a blob asked for directly. It
4794 // contributes itself and — since 2026-08-31 — what it points
4795 // at: the bitmaps are built over the commit graph and no tree
4796 // points at a tag, so a tag is in nobody's closure and this is
4797 // the only place its target can enter. A tag object is only
4798 // ever reachable as a REF TIP, so peeling here is complete.
4799 None => {
4800 union.insert(o);
4801 peel_tag_into(self.absorber.as_ref(), o, &view, &facts, &mut union);
4802 }
4803 }
4804 }
4805 let mut had = roaring::RoaringBitmap::new();
4806 let mut later: Vec<&[u8]> = Vec::new();
4807 for oid in have {
4808 if let Some(o) = space.ordinal_of_raw(oid) {
4809 if let Some(bm) = graph.row_of(o).and_then(|row| view.bitmap_of_row(row)) {
4810 had |= bm;
4811 continue;
4812 }
4813 }
4814 later.push(oid);
4815 }
4816 for oid in later {
4817 // An unknown `have` is normal: the client may have objects we do
4818 // not. It contributes nothing, which is the safe direction — we
4819 // send more, never less.
4820 let Some(o) = space.ordinal_of_raw(oid) else { continue };
4821 match graph.row_of(o) {
4822 // The same walk on the exclude side, and it must be the same
4823 // walk: an under-counted `have` over-sends (harmless, wasteful)
4824 // but an under-counted `want` under-sends, and answering the
4825 // two sides by different rules is how a `want − have` stops
4826 // being a subtraction of like for like.
4827 Some(row) => match view.bitmap_of_row(row) {
4828 Some(bm) => had |= bm,
4829 None => crate::reach::accumulate(row, &view, &facts, &mut had),
4830 },
4831 None => {
4832 had.insert(o);
4833 }
4834 }
4835 }
4836
4837 let delta = &union - &had;
4838 let commits = &delta & graph.commits();
4839 // One allocation per list, sized before the loop. The ordinal space is
4840 // a flat buffer, so each object is a `copy_from_slice` of `oid_len`
4841 // bytes into it and nothing else.
4842 let to_list = |bm: &roaring::RoaringBitmap| -> Result<git_storage_trait::OidList> {
4843 let mut out = git_storage_trait::OidList::with_capacity(bm.len() as usize, oid_len);
4844 for o in bm {
4845 let raw = space
4846 .oid(o)
4847 .ok_or_else(|| anyhow!("ordinal {o} is outside this store's ordinal space"))?;
4848 out.push(raw)?;
4849 }
4850 Ok(out)
4851 };
4852 let objects = to_list(&delta)?;
4853 // The trait's `ReachSet::commits` is a `Vec<Vec<u8>>`; one allocation
4854 // per selected commit, never per object.
4855 let commits: Vec<Vec<u8>> = commits
4856 .iter()
4857 .map(|o| space.oid(o).map(<[u8]>::to_vec).ok_or_else(|| anyhow!("ordinal {o} is outside this store's ordinal space")))
4858 .collect::<Result<_>>()?;
4859 // What the client held BEFORE this transfer: the closure of `have`, and
4860 // only it. Empty for a clone, which is the request with no excludes at
4861 // all and therefore the one that pays nothing for this.
4862 let client_has = if have.is_empty() { git_storage_trait::OidList::new() } else { to_list(&had)? };
4863 Ok(Some(Selected {
4864 objects,
4865 commits,
4866 client_has,
4867 }))
4868 }
4869
4870}
4871
4872/// What a reachability question does when the indexer holds the absorb gate.
4873#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4874pub(crate) enum GateBusy {
4875 /// Refuse by name ("index in progress"): a `want` the pending pack
4876 /// brought cannot be served from a graph that does not hold it, and a
4877 /// stale answer would under-send.
4878 Refuse,
4879 /// Answer from the last committed fold: for the closure of what the
4880 /// client already HOLDS, a commit the fold does not know contributes
4881 /// nothing, and a smaller voucher is the safe direction.
4882 UseCommitted,
4883}
4884
4885/// What [`GitStore::select_with`] answers: the three lists of
4886/// [`git_storage_trait::ReachSet`], with `objects` still in one allocation.
4887pub(crate) struct Selected {
4888 pub objects: git_storage_trait::OidList,
4889 pub commits: Vec<Vec<u8>>,
4890 pub client_has: git_storage_trait::OidList,
4891}
4892
4893impl<S: ObjectIndex + 'static> GitStore<S> {
4894 /// Every object reachable from every ref: what a GC must keep.
4895 pub(crate) fn live_set(&self) -> Result<std::collections::HashSet<String>> {
4896 let refs = self.ref_state()?;
4897 let tips: Vec<Vec<u8>> = refs
4898 .values()
4899 .filter_map(|s| s.target.as_deref())
4900 .filter_map(|t| hex::decode(t).ok())
4901 .collect();
4902 if tips.is_empty() {
4903 bail!(
4904 "this repository has no ref pointing at an object, so every object in it would be \
4905 dead. A GC that would delete everything is refused: name a ref first"
4906 );
4907 }
4908 let borrowed: Vec<Oid<'_>> = tips.iter().map(|t| t.as_slice()).collect();
4909 let mut live: std::collections::HashSet<String> =
4910 self.reachable_oids(&borrowed, &[])?.into_iter().collect();
4911 // The peeled targets of annotated tags are reachable too, and a tag
4912 // object is not a commit so it has no bitmap of its own.
4913 for s in refs.values() {
4914 if let Some(p) = &s.peeled {
4915 live.insert(p.clone());
4916 }
4917 if let Some(t) = &s.target {
4918 live.insert(t.clone());
4919 }
4920 }
4921 Ok(live)
4922 }
4923
4924 /// Drop every row the live set does not name. The index is append-only for
4925 /// everything except this.
4926 ///
4927 /// **Both tables, in one call.** §14's exploded table is retired with the
4928 /// `objects` table because its row count against that one's is what says
4929 /// whether it is whole ([`Absorber::adopt_journal`]): dropping from one and
4930 /// not the other would leave every later open re-exploding the entire
4931 /// repository. The count returned is still the index's — that is what a
4932 /// [`GcReport`] means by a dropped row.
4933 /// Drop exactly the rows in `dead` — a set the GC computed as "seen at its
4934 /// start and not reachable", never "not in `live`". The distinction is a
4935 /// row that arrived after the GC took its snapshot: it is in neither set,
4936 /// and the predicate below keeps it. See [`dead_oids`].
4937 pub(crate) fn drop_dead_rows(&self, dead: &std::collections::HashSet<Vec<u8>>) -> Result<u64> {
4938 let dropped = self
4939 .absorber
4940 .objects
4941 .retain(&|oid: &[u8]| !dead.contains(oid))?;
4942 self.absorber
4943 .exploded
4944 .retain(&|oid: &[u8]| !dead.contains(oid))?;
4945 // **Everything folded from those two tables, refolded.** The commit
4946 // graph, the tree payloads, the ordinal space and the bitmaps are all
4947 // derivations of the rows that just went, and a derivation that still
4948 // names a dropped oid is *wrong* rather than merely stale — a `reachable`
4949 // over a graph holding dead commits selects objects the index can no
4950 // longer serve. This is the same [`Absorber::refold`] every absorb and
4951 // every open already calls, over the table this call just changed;
4952 // nothing about how the graph is rebuilt is touched here.
4953 //
4954 // The bitmaps go with it: they address **ordinals**, and dropping rows
4955 // renumbers the ordinal space, so `refold` clears them and the next
4956 // `reach_bitmaps()` builds them over the new numbering. Carrying them
4957 // across would be carrying an index into a different array.
4958 self.absorber.refold()?;
4959 Ok(dropped)
4960 }
4961
4962 /// **The journal's half of a GC: retire every pack whose objects are now all
4963 /// dead, before a single row is dropped.**
4964 ///
4965 /// Returns the offsets retired.
4966 ///
4967 /// # Why this exists
4968 ///
4969 /// §13.12's `indexed` bit is derived on open as *extent in the journal, rows
4970 /// not in the index* ([`Absorber::adopt_journal`]). Dropping every row of a
4971 /// pack produces that state exactly, so without this call the next open
4972 /// re-queues the pack and re-absorbs the objects a GC just decided were dead.
4973 /// A **partly** dead pack keeps rows and was never affected; this is only
4974 /// about the all-dead case.
4975 ///
4976 /// # What it does not do
4977 ///
4978 /// It does not truncate `objects.pack`. The blob file is append-only and a
4979 /// pack in the middle of it cannot be cut out without moving every extent
4980 /// after it — which would invalidate every index row in the archive. The dead
4981 /// pack's bytes stay; what changes is that nothing will ever index them
4982 /// again. Reclaiming those bytes is a rewrite of the blob file, and it is not
4983 /// this function.
4984 ///
4985 /// An arm with no journal ([`FastWriter`](crate::archive_write::FastWriter))
4986 /// has nothing to retire and nothing that re-queues, so it returns empty.
4987 ///
4988 /// # Cost
4989 ///
4990 /// One [`ObjectReadStack::extents_with_rows`] scan plus one batch lookup of
4991 /// the live set. Both are already the shape a GC pays elsewhere, and both run
4992 /// once per `gc()`, not per pack.
4993 /// Where the journal's acked extents end right now — the GC's snapshot of
4994 /// "packs that existed when I started". A pack acked past this point during
4995 /// the GC belongs to a push in flight and is not the GC's to judge.
4996 pub(crate) fn acked_end(&self) -> Result<u64> {
4997 let Some(journal) = self.arms.writer.journal(&self.blobs) else {
4998 return Ok(u64::MAX);
4999 };
5000 if !journal.exists() {
5001 return Ok(0);
5002 }
5003 let rows = crate::archive_write::read_journal(&journal)?;
5004 Ok(crate::archive_write::acked_packs(&rows)
5005 .iter()
5006 .map(|(o, l)| o + l)
5007 .max()
5008 .unwrap_or(0))
5009 }
5010
5011 /// Tombstone every acked pack that holds no live object — among the packs
5012 /// whose extent ends at or before `acked_end`, the GC's snapshot from
5013 /// [`acked_end`](Self::acked_end). A pack pushed after that snapshot has
5014 /// objects `live` never saw, and without the bound it would be retired for
5015 /// being unknown, which is how a GC would delete a push in flight.
5016 pub(crate) fn retire_dead_packs(
5017 &self,
5018 live: &std::collections::HashSet<String>,
5019 acked_end: u64,
5020 ) -> Result<Vec<u64>> {
5021 let Some(journal) = self.arms.writer.journal(&self.blobs) else {
5022 return Ok(Vec::new());
5023 };
5024 if !journal.exists() {
5025 return Ok(Vec::new());
5026 }
5027 let rows = crate::archive_write::read_journal(&journal)?;
5028 let packs: Vec<(u64, u64)> = crate::archive_write::acked_packs(&rows)
5029 .into_iter()
5030 .filter(|(o, l)| o + l <= acked_end)
5031 .collect();
5032 let already = crate::archive_write::retired_offsets(&rows);
5033 if packs.is_empty() {
5034 return Ok(Vec::new());
5035 }
5036
5037 // Which packs hold any row at all right now. A pack with none is either
5038 // already retired or was never absorbed — neither is this call's
5039 // business, and calling the second one dead would lose it.
5040 let occupied = self.absorber.objects.extents_with_rows(&packs)?;
5041
5042 // Which packs hold a row that survives. Placed by binary search over the
5043 // pack starts, the same way the scan above places a row.
5044 let mut order: Vec<usize> = (0..packs.len()).filter(|&i| packs[i].1 > 0).collect();
5045 order.sort_unstable_by_key(|&i| packs[i].0);
5046 let starts: Vec<u64> = order.iter().map(|&i| packs[i].0).collect();
5047 let live_oids: Vec<Vec<u8>> = live.iter().filter_map(|h| hex::decode(h).ok()).collect();
5048 let borrowed: Vec<&[u8]> = live_oids.iter().map(|o| o.as_slice()).collect();
5049 let mut has_live = vec![false; packs.len()];
5050 for (offset, _) in self
5051 .absorber
5052 .objects
5053 .extents_batch(&borrowed)
5054 .into_iter()
5055 .flatten()
5056 {
5057 let p = starts.partition_point(|&s| s <= offset);
5058 if p == 0 {
5059 continue;
5060 }
5061 let i = order[p - 1];
5062 if offset < packs[i].0 + packs[i].1 {
5063 has_live[i] = true;
5064 }
5065 }
5066
5067 let dead: Vec<u64> = (0..packs.len())
5068 .filter(|&i| occupied[i] && !has_live[i] && !already.contains(&packs[i].0))
5069 .map(|i| packs[i].0)
5070 .collect();
5071 // Durable before the rows go. The one ordering this function has.
5072 crate::archive_write::retire_packs(&journal, &dead)?;
5073 Ok(dead)
5074 }
5075
5076 /// **Write generation 0** at [`archive_path`](Self::archive_path), carrying
5077 /// the verbatim packs this store has acked plus `reserved`.
5078 ///
5079 /// The whole of the decision-making lives in
5080 /// [`seal_generation_zero`](crate::archive_write::seal_generation_zero); this
5081 /// is the three paths it needs, taken off the same fields the push path and
5082 /// the GC read, so a seal cannot address a different blob file or a different
5083 /// journal from the one that acked the packs.
5084 pub(crate) fn seal_archive(
5085 &self,
5086 reserved: Vec<ReservedSection>,
5087 ) -> Result<crate::archive_write::SealReport> {
5088 crate::archive_write::seal_generation_zero(
5089 &self.blobs,
5090 self.arms.writer.journal(&self.blobs).as_deref(),
5091 &self.archive,
5092 reserved,
5093 )
5094 }
5095
5096 /// The sections a sealed archive carries: the ref log, the commit graph and
5097 /// the bitmaps.
5098 ///
5099 /// The sealed archive numbers its objects in **sorted** order, the live
5100 /// table in the store's append-only order, so the table is renumbered once
5101 /// here through one remap table (one lookup per object, one index per bit)
5102 /// — the one place the two spaces meet.
5103 pub(crate) fn reserved_sections(&self) -> Result<Vec<ReservedSection>> {
5104 use znippy_common::{GUNNAR_GRAPH_MODULE, GUNNAR_REACH_MODULE};
5105 let mut out = vec![self.refs.seal_section()?];
5106 // May take the derived WRITE lock to build: before the read guard.
5107 let _ = self.reach_bitmaps()?;
5108 let d = self
5109 .absorber
5110 .derived
5111 .read()
5112 .map_err(|_| anyhow!("derived poisoned"))?;
5113 if !d.graph.is_empty() {
5114 out.push(ReservedSection::arrow(
5115 GUNNAR_GRAPH_MODULE,
5116 crate::graph::graph_schema(),
5117 vec![crate::graph::build_graph_batch(&d.graph)?],
5118 ));
5119 }
5120 if !d.reach.is_empty() {
5121 let sorted = self.absorber.objects.oids_raw()?.raw;
5122 let oid_len = self.hash.oid_len();
5123 let target = crate::reach::OrdinalSpace::new(&sorted, oid_len);
5124 let remap = d.ordinals.remap_to(&target);
5125 if let Some(entries) = d.reach.to_entries(&target, Some(&remap)) {
5126 out.push(ReservedSection::arrow(
5127 GUNNAR_REACH_MODULE,
5128 crate::reach::reach_schema(),
5129 vec![crate::reach::build_reach_batch(&entries)?],
5130 ));
5131 } else {
5132 tracing::warn!(
5133 store = %self.absorber.label,
5134 "seal: the live reachability table names an object the sorted space lacks; \
5135 no __gunnar_reach__ section is written"
5136 );
5137 }
5138 }
5139 Ok(out)
5140 }
5141}
5142
5143// ── the runtime selector ──────────────────────────────────────────────────────
5144
5145/// **A [`GitStore`] whose index arm was chosen at run time.**
5146///
5147/// The [`ObjectIndex`] arm is a *type*, so a value can only choose it by naming
5148/// every monomorphisation. This enum is that list, and it is an enum rather than
5149/// a `Box<dyn GitOps>` for two reasons:
5150///
5151/// * a boxed trait object can only offer the twelve, and the arms differ in
5152/// things the twelve deliberately do not expose — how many Arrow bytes the
5153/// projection actually materialised, which writer is on the ack path. A bench
5154/// or an operator dump needs those, and a guard needs them to assert that the
5155/// selector selected **on applied output** rather than on a label;
5156/// * there is no allocation and no vtable: the `match` is one predictable branch
5157/// at the top of a call, and everything under it — the `stree` probe, the
5158/// Arrow gather, the redb tail — is the same fully static code
5159/// [`GitStore::<S>::open_with_arms`](GitStore::open_with_arms) produces,
5160/// because `S` is known inside each arm.
5161///
5162/// # What it costs, stated rather than hidden
5163///
5164/// One branch per [`GitOps`] call — **not per object**. `extents(&[1000 oids])`
5165/// is one branch and a thousand monomorphised lookups behind it. The binary
5166/// carries three copies of the store, which is the price of picking a type at
5167/// run time and there is no version of this that does not pay it.
5168///
5169/// A caller that knows its arm at compile time should **not** come through here:
5170/// [`GitStore::open`] and [`GitStore::open_with_arms`] hand back a concrete
5171/// store with no branch at all, and that includes every caller that wants the
5172/// default.
5173pub enum SelectedStore {
5174 OneTableFourColumns(GitStore<OneTableFourColumns>),
5175 FourTables(GitStore<crate::index_layout::FourTables>),
5176 PackedPayload(GitStore<crate::index_layout::PackedPayload>),
5177}
5178
5179/// One expression, evaluated against whichever concrete store this is. The
5180/// delegation below is generated from it so there is no second copy of any
5181/// method body (LAW 5).
5182macro_rules! on_arm {
5183 ($self:ident, $s:ident => $body:expr) => {
5184 match $self {
5185 SelectedStore::OneTableFourColumns($s) => $body,
5186 SelectedStore::FourTables($s) => $body,
5187 SelectedStore::PackedPayload($s) => $body,
5188 }
5189 };
5190}
5191
5192impl SelectedStore {
5193 /// The arms this store was built with.
5194 pub fn arms(&self) -> StoreConfig {
5195 on_arm!(self, s => s.arms())
5196 }
5197
5198 /// **The explode band** — bytes held in each pool, the budget, and what has
5199 /// been evicted since this store was opened. `Admin.StorageInfo` and the
5200 /// dashboard's `explode 3.1G/8G` cell.
5201 pub fn explode_cache_use(&self) -> crate::exploded_cache::CacheUse {
5202 on_arm!(self, s => s.explode_cache_use())
5203 }
5204
5205 /// **This repository's explode quota, set live.** What the control plane
5206 /// calls when an account's plan changes. It never refuses a read and never
5207 /// deletes on the caller's thread: over budget evicts at the next rotation,
5208 /// and an account at its limit still serves every object — from the pack,
5209 /// which is slower and never wrong.
5210 pub fn set_explode_budget(&self, bytes: u64) {
5211 on_arm!(self, s => s.set_explode_budget(bytes))
5212 }
5213
5214 /// **`repo gc` reclaims the cache**: every lazy segment unlinked, the graph
5215 /// pool kept. Returns the bytes that left the filesystem.
5216 pub fn drop_explode_cache(&self) -> anyhow::Result<u64> {
5217 on_arm!(self, s => s.drop_explode_cache())
5218 }
5219
5220 /// The selected writer's name, as it goes on a bench row.
5221 pub fn writer_name(&self) -> &'static str {
5222 on_arm!(self, s => s.writer_name())
5223 }
5224
5225 /// What the selected writer's `append` promises.
5226 pub fn writer_durability(&self) -> &'static str {
5227 on_arm!(self, s => s.writer_durability())
5228 }
5229
5230 /// **Arrow IPC bytes the selected index arm actually materialised.**
5231 ///
5232 /// The applied output that distinguishes one layout from another: a packed
5233 /// 25-byte column, four columns in one section and four independent
5234 /// sections are three different numbers for the same objects. This is what
5235 /// a guard reads to prove the selector selected, because a *name* would
5236 /// prove only that a name was copied.
5237 pub fn index_ipc_bytes(&self) -> usize {
5238 on_arm!(self, s => s.index().ipc_bytes())
5239 }
5240
5241 /// **The name the built projection reports about itself** —
5242 /// `OneTableFourColumns`, `FourTables` or `PackedPayload`.
5243 ///
5244 /// The other half of the applied-output pair [`index_ipc_bytes`] starts:
5245 /// the byte count separates the arms once objects are in the store, and
5246 /// this separates them from the instant it is built, including on an empty
5247 /// store where all three materialise nothing. It is the value
5248 /// [`crate::arms::IndexArm::projection_name`] exists to be compared with —
5249 /// what the layout calls itself, never what the selector asked for — so a
5250 /// server logging it is stating what it built rather than repeating its own
5251 /// environment back.
5252 ///
5253 /// [`index_ipc_bytes`]: SelectedStore::index_ipc_bytes
5254 pub fn index_name(&self) -> &'static str {
5255 on_arm!(self, s => s.index().projection_name())
5256 }
5257
5258 /// The oids a pushed extent introduced — [`GitStore::oids_in_extent`],
5259 /// delegated.
5260 ///
5261 /// Not one of the twelve (a pack's extent is a znippy concept), so it does
5262 /// not arrive with the [`GitOps`] impl below and has to be forwarded by
5263 /// hand. A server that reports the objects a push introduced needs it, and
5264 /// a server that picked its index arm at run time still needs it.
5265 pub fn oids_in_extent(&self, extent: Extent) -> Result<Vec<Vec<u8>>> {
5266 on_arm!(self, s => s.oids_in_extent(extent))
5267 }
5268
5269 /// Objects in the `objects` table.
5270 pub fn object_count(&self) -> usize {
5271 on_arm!(self, s => s.object_count())
5272 }
5273
5274 /// Block until the background drain has absorbed everything pushed.
5275 pub fn wait_indexed(&self) {
5276 on_arm!(self, s => s.wait_indexed())
5277 }
5278
5279 /// Stop this store's index work; see [`GitStore::cancel_index`].
5280 pub fn cancel_index(&self) {
5281 on_arm!(self, s => s.cancel_index())
5282 }
5283
5284 /// This store's index progress; see [`GitStore::index_progress`].
5285 pub fn index_progress(&self) -> crate::progress::Snapshot {
5286 on_arm!(self, s => s.index_progress())
5287 }
5288
5289 /// Rebuild the projection, so the layout under test is what answers rather
5290 /// than the redb tail.
5291 pub fn rebuild_projection(&self) -> Result<()> {
5292 on_arm!(self, s => s.index().rebuild())
5293 }
5294
5295 /// The twelfth method — inherent, not on [`GitOps`], because it returns
5296 /// Arrow `ReservedSection`s a gix backend has no analog for.
5297 pub fn seal(&self) -> Result<Vec<ReservedSection>> {
5298 on_arm!(self, s => s.seal())
5299 }
5300}
5301
5302impl GitOps for SelectedStore {
5303 fn put(&self, pack: &[u8], refs: &[RefUpdate]) -> Result<TxId> {
5304 on_arm!(self, s => s.put(pack, refs))
5305 }
5306 fn put_pack(&self, bytes: &[u8]) -> Result<TxId> {
5307 on_arm!(self, s => s.put_pack(bytes))
5308 }
5309 // Delegated like every other method, and that is the point: without these
5310 // two lines a `SelectedStore` would silently inherit the trait's *default*
5311 // `put_pack_observed` — the one that publishes nothing — and every store
5312 // gunnar actually opens goes through this enum. The count would have been
5313 // implemented in `store.rs` and unreachable in production.
5314 fn put_pack_observed(
5315 &self,
5316 bytes: &[u8],
5317 progress: &dyn git_storage_trait::PackProgress,
5318 ) -> Result<TxId> {
5319 on_arm!(self, s => s.put_pack_observed(bytes, progress))
5320 }
5321 fn publishes_pack_progress(&self) -> bool {
5322 on_arm!(self, s => s.publishes_pack_progress())
5323 }
5324 fn put_refs(&self, updates: &[RefUpdate]) -> Result<TxId> {
5325 on_arm!(self, s => s.put_refs(updates))
5326 }
5327 fn get(&self, oid: Oid<'_>) -> Result<Option<Stored>> {
5328 on_arm!(self, s => s.get(oid))
5329 }
5330 fn has(&self, oid: Oid<'_>) -> Result<bool> {
5331 on_arm!(self, s => s.has(oid))
5332 }
5333 fn size(&self, oid: Oid<'_>) -> Result<Option<u64>> {
5334 on_arm!(self, s => s.size(oid))
5335 }
5336 fn extents(&self, oids: &[Oid<'_>]) -> Result<Vec<Option<Extent>>> {
5337 on_arm!(self, s => s.extents(oids))
5338 }
5339 fn refs(&self) -> Result<Vec<RefRow>> {
5340 on_arm!(self, s => s.refs())
5341 }
5342 fn update_ref(&self, name: &str, old: Option<Oid<'_>>, new: Option<Oid<'_>>) -> Result<TxId> {
5343 on_arm!(self, s => s.update_ref(name, old, new))
5344 }
5345 fn put_refs_cas(&self, edits: &[git_storage_trait::RefCas<'_>]) -> Result<TxId> {
5346 on_arm!(self, s => s.put_refs_cas(edits))
5347 }
5348 fn reachable(&self, want: &[Oid<'_>], have: &[Oid<'_>]) -> Result<Vec<Vec<u8>>> {
5349 on_arm!(self, s => s.reachable(want, have))
5350 }
5351 fn gc(&self) -> Result<GcReport> {
5352 on_arm!(self, s => s.gc())
5353 }
5354}
5355
5356/// The reading contract, delegated the same way [`GitOps`] is.
5357///
5358/// Every method is one predictable branch over a monomorphised body — the same
5359/// arrangement, and the same cost argument, [`SelectedStore`] makes for the
5360/// eleven. It is here rather than in [`crate::serve`] only because the macro
5361/// that generates it lives in this module.
5362impl crate::serve::GitServe for SelectedStore {
5363 fn read(&self, oid: Oid<'_>) -> Result<Option<(crate::index_layout::ObjType, Vec<u8>)>> {
5364 on_arm!(self, s => crate::serve::GitServe::read(s, oid))
5365 }
5366 fn header(&self, oid: Oid<'_>) -> Result<Option<(crate::index_layout::ObjType, u64)>> {
5367 on_arm!(self, s => crate::serve::GitServe::header(s, oid))
5368 }
5369 fn sizes(&self, oids: &[Oid<'_>]) -> Result<Vec<Option<u64>>> {
5370 on_arm!(self, s => crate::serve::GitServe::sizes(s, oids))
5371 }
5372 fn index_now(&self) -> Result<usize> {
5373 on_arm!(self, s => crate::serve::GitServe::index_now(s))
5374 }
5375 fn head(&self) -> Result<Option<RefRow>> {
5376 on_arm!(self, s => crate::serve::GitServe::head(s))
5377 }
5378 fn set_head(&self, target: &str) -> Result<TxId> {
5379 on_arm!(self, s => crate::serve::GitServe::set_head(s, target))
5380 }
5381 /// `objects` is the **exact set to emit**, not tips to close over — the
5382 /// contract's word, and this forwarder uses it so that the name cannot drift
5383 /// back into `want` one arm at a time. See [`crate::serve::GitServe::emit_pack`].
5384 fn emit_pack(
5385 &self,
5386 objects: &[Oid<'_>],
5387 have: &[Oid<'_>],
5388 caps: &git_storage_trait::Caps,
5389 out: &mut dyn std::io::Write,
5390 ) -> Result<git_storage_trait::PackStats> {
5391 on_arm!(self, s => crate::serve::GitServe::emit_pack(s, objects, have, caps, out))
5392 }
5393 fn select(
5394 &self,
5395 want: &[Oid<'_>],
5396 have: &[Oid<'_>],
5397 ) -> Result<Option<git_storage_trait::ReachSet>> {
5398 on_arm!(self, s => crate::serve::GitServe::select(s, want, have))
5399 }
5400}
5401
5402/// **Open a store with all three arms chosen at run time.**
5403///
5404/// The one place [`crate::arms::IndexArm`] — a value — is turned into `S` — a
5405/// type. See [`SelectedStore`] for what the resulting handle costs.
5406pub fn open_selected(
5407 root: &Path,
5408 account: &str,
5409 hash: GitHashKind,
5410 arms: StoreConfig,
5411) -> Result<SelectedStore> {
5412 use crate::arms::IndexArm;
5413 use crate::index_layout::{FourTables, PackedPayload};
5414 Ok(match arms.index {
5415 IndexArm::OneTableFourColumns => {
5416 SelectedStore::OneTableFourColumns(GitStore::<OneTableFourColumns>::open_with_arms(
5417 root, account, hash, arms,
5418 )?)
5419 }
5420 IndexArm::FourTables => SelectedStore::FourTables(GitStore::<FourTables>::open_with_arms(
5421 root, account, hash, arms,
5422 )?),
5423 IndexArm::PackedPayload => SelectedStore::PackedPayload(
5424 GitStore::<PackedPayload>::open_with_arms(root, account, hash, arms)?,
5425 ),
5426 })
5427}
5428
5429/// **The env-driven front door**: read the three variables **once**, here, and
5430/// build the store they name.
5431///
5432/// This is the whole runtime selector a server or a bench needs — one call, one
5433/// triple of `getenv`s, and from then on the store holds built objects and reads
5434/// nothing. There is deliberately no per-operation lookup of any kind: a
5435/// `std::env::var` inside a serving loop would be one syscall per object served,
5436/// which is a real defect that was found and fixed in gunnar the day before this
5437/// was written. [`crate::arms::env_reads`] counts every read this crate makes so
5438/// that "once, at construction" is an assertion rather than a claim.
5439///
5440/// An unset variable takes the shipping default, so a process with a clean
5441/// environment gets exactly [`GitStore::open`]'s combination.
5442pub fn open_from_env(root: &Path, account: &str, hash: GitHashKind) -> Result<SelectedStore> {
5443 let arms = StoreConfig::from_env()?;
5444 open_selected(root, account, hash, arms)
5445}
5446
5447impl<S: ObjectIndex> Absorber<S> {
5448 /// **One object's resolved content, and §14's table is what answers.**
5449 ///
5450 /// Two paths, and they are not interchangeable even though they return
5451 /// identical bytes:
5452 ///
5453 /// 1. **the exploded table** — one point lookup, no inflate, no delta chain;
5454 /// 2. **re-derived from the verbatim truth** — find which pack holds the
5455 /// object, read the whole pack back, resolve it. Correct and slow, and it
5456 /// exists because the table is droppable: a store whose table has been
5457 /// deleted, or whose re-explosion has not caught up, must still answer.
5458 ///
5459 /// The counters are the only way to tell them apart. Identical bytes come
5460 /// back either way, so a byte comparison proves nothing about which path ran
5461 /// — that is the identity-value trap in its exact form, and it is why
5462 /// [`crate::exploded::ExplodedStats::rederived`] is bumped here rather than
5463 /// inferred.
5464 ///
5465 /// On the [`Absorber`] rather than on [`GitStore`] because that is where the
5466 /// absorb runs: the background worker has no store handle, and a base lookup
5467 /// that needed one would put the whole store behind the drain.
5468 fn resolved(&self, oid: &[u8]) -> Result<Option<(GitObjectKind, Vec<u8>)>> {
5469 Ok(self
5470 .resolved_costed(oid)?
5471 .map(|(kind, payload, _)| (kind, payload)))
5472 }
5473
5474 /// [`Self::resolved`] through a [`crate::resolve::ChainCache`]: the
5475 /// reachability builder's read, which materialises neighbouring chains by
5476 /// the hundred thousand.
5477 fn resolved_cached(
5478 &self,
5479 oid: &[u8],
5480 cache: &mut crate::resolve::ChainCache,
5481 ) -> Result<Option<(GitObjectKind, Vec<u8>)>> {
5482 Ok(self
5483 .resolved_costed_with(oid, Some(cache))?
5484 .map(|(kind, payload, _)| (kind, payload)))
5485 }
5486
5487 /// [`Self::resolved`] and the bill. `None` for the cost means a pool
5488 /// answered and nothing was rebuilt.
5489 #[allow(clippy::type_complexity)]
5490 fn resolved_costed(
5491 &self,
5492 oid: &[u8],
5493 ) -> Result<Option<(GitObjectKind, Vec<u8>, Option<crate::resolve::ReadCost>)>> {
5494 self.resolved_costed_with(oid, None)
5495 }
5496
5497 #[allow(clippy::type_complexity)]
5498 fn resolved_costed_with(
5499 &self,
5500 oid: &[u8],
5501 cache: Option<&mut crate::resolve::ChainCache>,
5502 ) -> Result<Option<(GitObjectKind, Vec<u8>, Option<crate::resolve::ReadCost>)>> {
5503 // (1) §14's table.
5504 if let Some((kind, payload)) = self.exploded.content(oid)? {
5505 return Ok(Some((kind, payload, None)));
5506 }
5507 // (2) the verbatim truth, re-derived.
5508 let Some(row) = self.objects.lookup(oid) else {
5509 // (3) not indexed yet: the pending packs, read-only, on this
5510 // reader's own thread — never by waiting for the worker.
5511 //
5512 // **This arm is the ack path's trap and it is not the ack path's
5513 // fix.** `a_read_before_the_drain_is_served_from_the_pack_and_never_waits`
5514 // is the law here: a clone must be answered while the drain is
5515 // parked, so a read may not take the absorb gate. What must not
5516 // happen is the ACK arriving here, and it no longer does —
5517 // `GitServe::index_now` runs before the connectivity check, so by
5518 // the time the check reads there is nothing pending to scan. See
5519 // [`Absorber::index_pending_now`].
5520 if self.unindexed_packs() == 0 {
5521 return Ok(None);
5522 }
5523 self.exploded.note_rederived();
5524 let (_, content) = self.scan_pending(&[], Some(oid))?;
5525 // The pending scan resolves through a `PackWalk` rather than a
5526 // chain walk, so it does not produce a bill. `None` is honest here
5527 // and is not "it was free".
5528 return Ok(content.map(|(k, p)| (k, p, None)));
5529 };
5530 // ── ONE CHAIN, not one pack ─────────────────────────────────────
5531 //
5532 // The row's `offset` addresses the entry in the archive directly, so the
5533 // object is materialised by following its own delta chain out of the
5534 // mapping — `d` inflates and `d − 1` delta applications, nothing else
5535 // touched. This used to `read_extent` the WHOLE containing pack and
5536 // re-resolve it: for `torvalds/linux` an 8.12 GiB read and a 13.9 M
5537 // object resolve **per object served**, which is the cost that forced
5538 // the exploded tier to be built eagerly at push time — and that tier
5539 // MEASURED 141.7 GiB for an 8.12 GiB pack (17×). Lazy explode is only
5540 // affordable because of this function
5541 // (`ideas/lazy-explode-index-only-at-push.md`).
5542 //
5543 // It also does not care whether the containing pack has been ABSORBED:
5544 // the bytes were fsynced before the ack, so an object whose row is in
5545 // the index is readable the instant the row is. The pending-pack scan
5546 // below is for the window before the row exists at all.
5547 self.exploded.note_rederived();
5548 let snap = self.map.snapshot()?;
5549 let src = ArchiveWindow { abs: self, snap: &snap };
5550 let offset_of = |base: &[u8]| self.objects.lookup(base).map(|r| r.offset);
5551 match crate::resolve::materialize_at_with(
5552 &src,
5553 row.offset,
5554 self.hash.oid_len(),
5555 &offset_of,
5556 self,
5557 cache,
5558 ) {
5559 Ok((kind, payload, cost)) => {
5560 self.keep_for_the_next_reader(oid, kind, &payload, cost);
5561 Ok(Some((kind, payload, Some(cost))))
5562 }
5563 Err(e) => {
5564 // A chain that cannot be followed out of the mapping is not a
5565 // wrong answer to hand back: fall through to the pending scan,
5566 // which re-derives from the pack the row came in on.
5567 //
5568 // **And if that finds nothing either, the ORIGINAL error is
5569 // what comes back — never `Ok(None)`.** The row says this store
5570 // holds the object; a failed read of an object we hold is a
5571 // store defect, and answering "absent" for it turned that defect
5572 // into a refusal of somebody else's push (lane AI, 2026-08-30 —
5573 // see `crate::resolve::MAX_DELTA_LINKS`). Absence is a fact the
5574 // index states, not a shape an error decays into.
5575 if self.unindexed_packs() == 0 {
5576 return Err(e);
5577 }
5578 match self.scan_pending(&[], Some(oid))? {
5579 (_, Some((k, p))) => Ok(Some((k, p, None))),
5580 (_, None) => Err(e),
5581 }
5582 }
5583 }
5584 }
5585}
5586
5587impl<S: ObjectIndex> Absorber<S> {
5588 /// **The write-back, and it is off the reader's path.**
5589 ///
5590 /// The reader has its bytes already — this buffers a copy into the lazy
5591 /// pool's newest segment (a `memcpy` under one lock, no I/O, no fsync, no
5592 /// eviction) and, once 4 MiB have accumulated, starts a thread to do the
5593 /// writing. That is the owner's law from the design note in one function:
5594 /// *nobody waits*. A failure here is not the read's failure — the bytes
5595 /// were correct before the cache was consulted and are correct after it
5596 /// refuses — so it is counted and swallowed rather than propagated.
5597 ///
5598 /// **It does not always write.** A shallow, small object is cheaper to
5599 /// inflate again than to keep — a whole object is one `inflate` of one zlib
5600 /// stream at one offset — so the cache declines it and counts the read as
5601 /// `direct_reads` (`exploded_cache::worth_caching`). That is the owner's
5602 /// refinement, 2026-08-30: *do not explode uniformly*. The reader that just
5603 /// paid the bill is the only thing that knows what it was, which is why the
5604 /// cost comes back out of [`crate::resolve::materialize_at`] rather than
5605 /// being predicted from an index column.
5606 ///
5607 /// Two readers racing on one oid is benign: identical bytes, and the table
5608 /// keeps the first row it filed under that key.
5609 fn keep_for_the_next_reader(
5610 &self,
5611 oid: &[u8],
5612 kind: GitObjectKind,
5613 payload: &[u8],
5614 cost: crate::resolve::ReadCost,
5615 ) {
5616 match self.exploded.write_back(oid, kind, payload, cost) {
5617 Ok(true) => self.exploded.kick(),
5618 Ok(false) => {}
5619 Err(e) => {
5620 tracing::debug!(
5621 target: "znippy.git",
5622 store = %self.label,
5623 error = %e,
5624 "the exploded cache would not take a write-back; the read is unaffected"
5625 );
5626 }
5627 }
5628 }
5629}
5630
5631/// [`crate::reach::TreeSource`] over a live store — see
5632/// [`Absorber::tree_source`].
5633/// The oid a tag object points at, out of its header, **raw**.
5634///
5635/// A tag object's payload begins `object <hex>\n` — that is the format, not a
5636/// heuristic, and it is the whole parse. Deliberately NOT reached for with a
5637/// git object library: this crate is a shipped archiver and `gix-object` is a
5638/// dev-dependency here, so pulling it into the read path to read one line would
5639/// put the pack pipeline back in an archive-only build (S-071). Exactly the
5640/// hash width this store uses, and hex: a shorter or longer field is a
5641/// malformed tag, and guessing at one would put a wrong oid into a
5642/// reachability answer — the one place a wrong answer is silent.
5643fn tag_target_raw(payload: &[u8], oid_len: usize) -> Option<[u8; 32]> {
5644 let line = payload.split(|b| *b == b'\n').next()?;
5645 let rest = line.strip_prefix(b"object ")?;
5646 if rest.len() != oid_len * 2 || oid_len > 32 {
5647 return None;
5648 }
5649 let mut out = [0u8; 32];
5650 hex::decode_to_slice(rest, &mut out[..oid_len]).ok()?;
5651 Some(out)
5652}
5653
5654/// Add what a tag points at — and what THAT reaches — to a reachability union.
5655///
5656/// A tag may point at a commit (its closure, by bitmap or bounded walk), at
5657/// another tag (followed, bounded), at a tree (every object under it — a tree
5658/// alone would be the same bug one level down) or at a blob (itself). An
5659/// unreadable object makes the closure SHORT, and a short closure is refused
5660/// downstream by name (`missing necessary objects`) rather than served as
5661/// though it were complete — the same contract `TreesFromStore::tree`
5662/// documents.
5663fn peel_tag_into<S: ObjectIndex>(
5664 abs: &Absorber<S>,
5665 tag: u32,
5666 view: &crate::reach::ReachView<'_>,
5667 facts: &crate::reach::ObjectFacts<'_>,
5668 union: &mut roaring::RoaringBitmap,
5669) {
5670 const MAX_TAG_HOPS: usize = 8;
5671 let oid_len = facts.oid_len;
5672 let mut cursor = tag;
5673 for _ in 0..MAX_TAG_HOPS {
5674 let Some(raw) = facts.ordinal.oid(cursor) else { return };
5675 let Ok(Some((GitObjectKind::Tag, payload))) = abs.resolved(raw) else { return };
5676 let Some(target) = tag_target_raw(&payload, oid_len) else { return };
5677 let target = &target[..oid_len];
5678 let Some(t) = facts.ordinal.ordinal_of_raw(target) else { return };
5679 if let Some(row) = view.graph.row_of(t) {
5680 match view.bitmap_of_row(row) {
5681 Some(bm) => *union |= bm,
5682 None => crate::reach::accumulate(row, view, facts, union),
5683 }
5684 return;
5685 }
5686 match abs.resolved(target) {
5687 // `fill_tree` sets the tree's own bit and everything under it; it
5688 // must see the bit CLEAR, so the tree is not inserted first.
5689 Ok(Some((GitObjectKind::Tree, _))) => {
5690 crate::reach::fill_tree(t, facts, union);
5691 return;
5692 }
5693 Ok(Some((GitObjectKind::Tag, _))) => {
5694 union.insert(t);
5695 cursor = t;
5696 }
5697 // A blob, or unreadable: it is in, and it has no closure.
5698 _ => {
5699 union.insert(t);
5700 return;
5701 }
5702 }
5703 }
5704}
5705
5706struct TreesFromStore<'a, S: ObjectIndex> {
5707 abs: &'a Absorber<S>,
5708 /// The delta-chain memo of a full build; `None` for the bounded walks.
5709 chains: Option<Mutex<crate::resolve::ChainCache>>,
5710}
5711
5712impl<S: ObjectIndex> TreesFromStore<'_, S> {
5713 /// `(hits, misses, bytes)` of the chain memo, if there is one.
5714 fn chain_stats(&self) -> Option<(u64, u64, usize)> {
5715 let c = self.chains.as_ref()?.lock().ok()?;
5716 let (h, m) = c.stats();
5717 Some((h, m, c.bytes()))
5718 }
5719}
5720
5721impl<S: ObjectIndex> crate::reach::TreeSource for TreesFromStore<'_, S> {
5722 fn tree(&self, oid: &[u8]) -> Option<std::borrow::Cow<'_, [u8]>> {
5723 // A read that fails is `None`, which is what an object this archive
5724 // does not hold has always answered here — the walk stops at it. It is
5725 // NOT silently dropping an error that matters: an unreadable tree makes
5726 // the closure short, and a short closure is refused downstream by name
5727 // (`missing necessary objects`) rather than served as if it were
5728 // complete.
5729 let read = match &self.chains {
5730 Some(m) => match m.lock() {
5731 Ok(mut c) => self.abs.resolved_cached(oid, &mut c),
5732 Err(_) => self.abs.resolved(oid),
5733 },
5734 None => self.abs.resolved(oid),
5735 };
5736 match read {
5737 Ok(Some((GitObjectKind::Tree, payload))) => Some(std::borrow::Cow::Owned(payload)),
5738 // A blob, a tag, a gitlink, or absent: all "not a tree here".
5739 Ok(_) => None,
5740 Err(e) => {
5741 tracing::debug!(
5742 target: "znippy.git",
5743 store = %self.abs.label,
5744 oid = %hex::encode(oid),
5745 error = %e,
5746 "a reachability walk could not read a tree; the walk stops there"
5747 );
5748 None
5749 }
5750 }
5751 }
5752}
5753/// The verbatim archive as [`crate::resolve::ArchiveBytes`]: **the mapping
5754/// where it reaches, a `pread` past its end.**
5755///
5756/// One snapshot is taken per read and sliced as many times as the chain is
5757/// deep, which is [`crate::archive_map::ArchiveMap::snapshot`]'s contract. The
5758/// `pread` arm is the one a push that landed after the snapshot takes; both
5759/// arms return the same bytes and differ only in what they cost.
5760struct ArchiveWindow<'a, S: ObjectIndex> {
5761 abs: &'a Absorber<S>,
5762 snap: &'a crate::archive_map::Mapped,
5763}
5764
5765impl<S: ObjectIndex> crate::resolve::ArchiveBytes for ArchiveWindow<'_, S> {
5766 fn at(&self, at: u64, len: u64) -> Result<std::borrow::Cow<'_, [u8]>> {
5767 clamped_window(self.abs, self.snap, at, len)
5768 }
5769}
5770
5771/// At most `len` bytes of the archive at `at`: the mapping where it reaches, a
5772/// `pread` past its end, and **fewer bytes at the end of the file** instead of
5773/// a refusal. A probe near the end of the archive legitimately gets fewer bytes
5774/// than it asked for, so the window is clamped rather than refused — the
5775/// parsers read a grammar, not a fixed width.
5776fn clamped_window<'m, S: ObjectIndex>(
5777 abs: &Absorber<S>,
5778 snap: &'m crate::archive_map::Mapped,
5779 at: u64,
5780 len: u64,
5781) -> Result<std::borrow::Cow<'m, [u8]>> {
5782 let mapped = snap.len();
5783 if at < mapped {
5784 let have = (mapped - at).min(len);
5785 if let Some(b) = snap.get(at, have) {
5786 return Ok(std::borrow::Cow::Borrowed(b));
5787 }
5788 }
5789 let end = abs.reader.metadata().map(|m| m.len()).unwrap_or(mapped);
5790 if at >= end {
5791 return Ok(std::borrow::Cow::Owned(Vec::new()));
5792 }
5793 let have = (end - at).min(len);
5794 Ok(std::borrow::Cow::Owned(abs.read_extent(at, have)?))
5795}
5796
5797/// A thin pack's external delta base, answered from §14's exploded table.
5798///
5799/// Before that table existed this had to read and re-resolve a whole pack per
5800/// base, and it deliberately cached nothing — because a cache here *would* have
5801/// been the undecided table. It is decided now (eager, 2026-08-08), so the cache
5802/// is the table and this is one point lookup.
5803///
5804/// **`None` means absent and `Err` means unreadable, and they are not the same
5805/// answer.** A base this store does not hold still comes back `Ok(None)` and
5806/// [`crate::resolve::ResolveError::missing_base`] refuses the pack by name.
5807///
5808/// # 🔴 This was `.ok().flatten()`
5809///
5810/// A store that HELD the base and could not reconstruct it answered exactly
5811/// like one that never had it, so the resolver blamed the pushed pack: *"this
5812/// repository genuinely does not have that object"*, about an object one index
5813/// lookup away. That sentence went to the client on the `ng` line and to the
5814/// operator in the log, and it named the wrong party — lane AI, 2026-08-30, 21
5815/// of 140 ordinary pushes. The error travels now.
5816impl<S: ObjectIndex> BaseSource for Absorber<S> {
5817 fn content(&self, oid: &[u8]) -> Result<Option<(GitObjectKind, Vec<u8>)>> {
5818 self.resolved(oid).with_context(|| {
5819 format!(
5820 "reading the delta base {} out of this store — it is not absent, it could not \
5821 be reconstructed",
5822 hex::encode(oid)
5823 )
5824 })
5825 }
5826}
5827
5828/// **A table that was extended and not yet written back is written on drop.**
5829/// The extension persists every [`REACH_PERSIST_EVERY`] folds, so a clean
5830/// shutdown would otherwise leave the last few pushes' entries to be walked
5831/// again at the next open. Bounded and best-effort: a failure is logged.
5832impl<S: ObjectIndex> Drop for Absorber<S> {
5833 fn drop(&mut self) {
5834 if self.reach_unpersisted.load(std::sync::atomic::Ordering::Relaxed) == 0 {
5835 return;
5836 }
5837 if let Ok(d) = self.derived.read() {
5838 self.persist_reach_locked(&d);
5839 }
5840 }
5841}
5842
5843#[cfg(test)]
5844mod tests {
5845 use super::*;
5846 use crate::archive_write::{read_journal, SafeWriter};
5847 use crate::object::GitObjectKind;
5848 use crate::store::tests::{one_blob_pack, corpus_pack, tmpdir};
5849 use std::collections::HashSet;
5850 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering as AtomicOrdering};
5851 use std::time::{Duration, Instant};
5852 use znippy_zoomies::background::Job;
5853 use znippy_zoomies::gatling_forkjoin::gatling_for_each;
5854
5855 fn loadavg() -> String {
5856 std::fs::read_to_string("/proc/loadavg")
5857 .unwrap_or_default()
5858 .split_whitespace()
5859 .take(3)
5860 .collect::<Vec<_>>()
5861 .join(" ")
5862 }
5863
5864 /// **The one representation answers exactly what the two it replaced did.**
5865 ///
5866 /// Before S-082 the store carried the ordinal space three times: `oids_raw`
5867 /// (flat bytes), `oids` (hex `Vec<String>`) and `ordinal`
5868 /// (`HashMap<String, u32>`). The last two are gone; oid ⇄ ordinal is now a
5869 /// slice and a binary search over `oids_raw`, through
5870 /// [`crate::reach::OrdinalSpace`]. This rebuilds those two OLD tables inline
5871 /// from the SAME `oids_raw` the store folded, and asserts the new lookups
5872 /// return the identical answer **for every oid in a real corpus** — the
5873 /// property that lets the field be removed without a server ever naming the
5874 /// wrong object in a pack, which no exit code could see.
5875 ///
5876 /// Seen RED by an off-by-one binary search (`Equal => Some(mid as u32 + 1)`
5877 /// in [`crate::reach::OrdinalSpace::ordinal_of_raw`]): every
5878 /// `ordinal_of_raw` / `ordinal_of_hex` below disagreed with the old
5879 /// `HashMap`, and the closure comparison at the end disagreed as well.
5880 #[test]
5881 fn the_one_ordinal_space_answers_what_the_two_it_replaced_did() {
5882 let dir = tmpdir("ordinal-space-one-representation");
5883 let (pack, rows) = corpus_pack();
5884 let store = GitStore::open(&dir, "alice").unwrap();
5885 store.put_pack(&pack).unwrap();
5886 store.wait_indexed();
5887
5888 let oid_len = store.hash_kind().oid_len();
5889 let raw = {
5890 let d = store.absorber.derived.read().unwrap();
5891 d.ordinals.view().concat()
5892 };
5893 // The two tables `refold` used to build, rebuilt here from the same
5894 // bytes — this IS the "old" answer the new lookups must equal.
5895 let old_oids: Vec<String> = raw.chunks_exact(oid_len).map(hex::encode).collect();
5896 let old_ordinal: std::collections::HashMap<String, u32> = old_oids
5897 .iter()
5898 .enumerate()
5899 .map(|(i, o)| (o.clone(), i as u32))
5900 .collect();
5901
5902 assert!(
5903 !old_oids.is_empty(),
5904 "the fixture pack ({} entries) folded to an EMPTY ordinal space, so every \
5905 comparison below would hold vacuously",
5906 rows.len()
5907 );
5908 assert_eq!(
5909 raw.len(),
5910 old_oids.len() * oid_len,
5911 "the raw ordinal space is {} bytes for {} oids of {oid_len} bytes",
5912 raw.len(),
5913 old_oids.len()
5914 );
5915
5916 let space = crate::reach::OrdinalSpace::new(&raw, oid_len);
5917 assert_eq!(space.len(), old_oids.len(), "the space holds a different count");
5918 let mut checked = 0usize;
5919 for (o, h) in old_oids.iter().enumerate() {
5920 let want_ord = old_ordinal[h];
5921 // oid → ordinal, hex and raw, both == the old HashMap answer.
5922 assert_eq!(
5923 space.ordinal_of_hex(h),
5924 Some(want_ord),
5925 "ordinal_of_hex({h}) != the old HashMap answer {want_ord}"
5926 );
5927 let raw_oid = hex::decode(h).unwrap();
5928 assert_eq!(
5929 space.ordinal_of_raw(&raw_oid),
5930 Some(want_ord),
5931 "ordinal_of_raw disagrees with the old HashMap at {h}"
5932 );
5933 // ordinal → oid == the old hex Vec's slice, in both spellings.
5934 assert_eq!(
5935 space.oid(o as u32),
5936 Some(raw_oid.as_slice()),
5937 "oid({o}) != the old oids_raw slice"
5938 );
5939 assert_eq!(
5940 space.oid(o as u32).map(hex::encode).as_deref(),
5941 Some(h.as_str()),
5942 "oid({o}) hex != the old hex Vec entry {h}"
5943 );
5944 checked += 1;
5945 }
5946 assert_eq!(checked, old_oids.len());
5947
5948 // A non-member answers None both ways, exactly as `HashMap::get` did for
5949 // an absent key — the binary search must not land on a neighbour.
5950 let absent = vec![0xffu8; oid_len];
5951 assert_eq!(space.ordinal_of_raw(&absent), None, "an absent oid resolved to an ordinal");
5952 assert_eq!(
5953 space.ordinal_of_hex(&hex::encode(&absent)),
5954 None,
5955 "an absent hex oid resolved to an ordinal"
5956 );
5957 // A wrong-width key is absent, not a panic.
5958 assert_eq!(space.ordinal_of_raw(&absent[..oid_len - 1]), None);
5959 assert_eq!(space.oid(old_oids.len() as u32), None, "an out-of-range ordinal named an oid");
5960
5961 // And the answer built out of it: the raw closure IS the hex closure,
5962 // for a real tip rather than for an empty request.
5963 let tip = store
5964 .graph_snapshot()
5965 .into_iter()
5966 .max_by_key(|c| c.generation)
5967 .expect("a graph");
5968 let tip_raw = hex::decode(&tip.oid).unwrap();
5969 let flat = store.reachable_raw(&[&tip_raw], &[]).unwrap();
5970 let hexed = store.reachable_oids(&[&tip_raw], &[]).unwrap();
5971 assert!(
5972 flat.len() > 1,
5973 "the tip's closure is {} object(s) — too small to tell an ordering bug from a \
5974 coincidence",
5975 flat.len()
5976 );
5977 assert_eq!(
5978 flat.iter().map(hex::encode).collect::<Vec<_>>(),
5979 hexed,
5980 "the serving answer and the maintenance answer name different objects"
5981 );
5982 assert!(flat.contains(&tip_raw), "a closure without its own tip");
5983 eprintln!(
5984 "load {}; {} ordinals agree in both spellings, tip closure {} objects",
5985 loadavg(),
5986 checked,
5987 flat.len()
5988 );
5989 }
5990
5991 /// Distinct commits in a resolved pack — what the graph must hold exactly
5992 /// once.
5993 fn commits_in(rows: &[crate::resolve::Resolved]) -> usize {
5994 rows.iter()
5995 .filter(|r| r.kind == GitObjectKind::Commit)
5996 .map(|r| r.oid.clone())
5997 .collect::<HashSet<Vec<u8>>>()
5998 .len()
5999 }
6000
6001 /// **A push ends in object rows, and no read asked for them.**
6002 ///
6003 /// This is the seam this file wires: before it, `put` produced a *pack* row
6004 /// on the account indexer's channel and the object rows appeared only when
6005 /// somebody read (or called `absorb_pending` by hand). Here nothing is read
6006 /// between `put_pack` and `wait_indexed` — the only thing that could have
6007 /// built the rows is the drain.
6008 ///
6009 /// Asserted on applied output: one row per pack entry, every oid resolving
6010 /// out of the index, and the pack's indexed bit up. `absorb_failures` is
6011 /// asserted too, because a drain that absorbed nothing and recorded the
6012 /// failure would otherwise look like a drain that was never called.
6013 ///
6014 /// Seen RED by making the drain skip the absorber
6015 /// (`match absorber.as_deref()` → `match None::<&dyn ObjectAbsorb>` in
6016 /// `indexer::index_worker`): "the drain built no object rows: 0 of 2687".
6017 /// Restored.
6018 ///
6019 /// MEASURED on oden, 2026-08-08, release, three runs at **1-minute loadavg
6020 /// 5.77** (another tenant's work — oden is shared, so this is not an idle
6021 /// box): 2687 objects out of a 5 653 302-byte real pack absorbed in
6022 /// 161–164 ms, **16 421–16 673 object rows/s**, behind an ack of 29.1–29.4 ms.
6023 /// An earlier set at loadavg 16.94 gave 15 528 rows/s, so the figure moves
6024 /// about 7% with the box.
6025 ///
6026 /// Read it as a *floor* rather than as the engine's speed: the drain here is
6027 /// one pack, so it never uses more than one absorb, and the 162 ms covers a
6028 /// full inflate-and-delta-apply of every entry, a SHA-1 per object, the redb
6029 /// append and a whole-tail re-fold. Nothing about it has been optimised and
6030 /// no arm of it has been isolated. The rate is printed with the loadavg of
6031 /// its own run so a later number is never compared against a different
6032 /// machine state by accident.
6033 #[test]
6034 fn a_push_ends_in_object_rows_and_no_read_asked_for_them() {
6035 let dir = tmpdir("drain-rows");
6036 let store = GitStore::open(&dir, "alice").unwrap();
6037 let (pack, rows) = corpus_pack();
6038
6039 let t = Instant::now();
6040 let tx = store.put_pack(&pack).unwrap();
6041 let ack = t.elapsed();
6042 let pack_id = tx.pack_id.expect("a pack push assigns an id");
6043
6044 // Nothing is read here. The drain is the only thing that can move.
6045 store.wait_indexed();
6046 let drained = t.elapsed();
6047
6048 assert_eq!(
6049 store.object_count(),
6050 rows.len(),
6051 "the drain built no object rows: {} of {}",
6052 store.object_count(),
6053 rows.len()
6054 );
6055 assert_eq!(store.unindexed_packs(), 0, "the bit is still clear");
6056 assert!(
6057 store.indexer().is_indexed(pack_id),
6058 "the pack's indexed bit is not up after the drain"
6059 );
6060 assert_eq!(
6061 store.indexer().absorb_failures(),
6062 0,
6063 "the drain recorded an absorb failure: {:?}",
6064 store.indexer().last_absorb_error()
6065 );
6066
6067 // Every oid answers **out of the rows**, not out of a fallback: the
6068 // index is asked directly, so nothing can absorb behind this loop.
6069 for (i, r) in rows.iter().enumerate() {
6070 let row = store
6071 .index()
6072 .lookup(&r.oid)
6073 .unwrap_or_else(|| panic!("object {i} {} has no row", hex::encode(&r.oid)));
6074 assert_eq!(
6075 (row.offset, row.len),
6076 (tx.extent.unwrap().0 + r.offset, r.len)
6077 );
6078 assert_eq!(row.uncompressed_size, r.uncompressed_size);
6079 }
6080 assert_eq!(
6081 store.absorb_pending().unwrap(),
6082 0,
6083 "the drain left work for a read to do"
6084 );
6085 assert_eq!(
6086 store.commit_count(),
6087 commits_in(&rows),
6088 "the commit graph does not match the pack"
6089 );
6090
6091 let secs = (drained - ack).as_secs_f64();
6092 eprintln!(
6093 "load {}; {} objects in {} bytes: ack {:.0} µs, drain {:.1} ms, {:.0} object rows/s",
6094 loadavg(),
6095 rows.len(),
6096 pack.len(),
6097 ack.as_secs_f64() * 1e6,
6098 (drained - ack).as_secs_f64() * 1e3,
6099 rows.len() as f64 / secs.max(1e-9),
6100 );
6101 }
6102
6103 // ── the bit across a restart ─────────────────────────────────────────────
6104
6105 /// Where the child of the kill guard finds its fixture and leaves its marker.
6106 const KILL_DIR: &str = "GUNNAR_KILL_MID_ABSORB_DIR";
6107
6108 /// **The other half of [`a_store_reopened_after_a_kill_mid_absorb_requeues_the_pack`]:
6109 /// the process that dies.**
6110 ///
6111 /// It runs as a child of that test and it **aborts on purpose**, which is why
6112 /// it is `#[ignore]`d: `SIGABRT` from inside a test binary is a failing test
6113 /// run, and the point is for the *parent* to see the signal.
6114 ///
6115 /// The kill lands where it has to land — after `put_pack` has returned, so
6116 /// the bytes and the journal row are on the platter, and before any row is in
6117 /// the index, which the held absorb gate makes a **state** rather than a
6118 /// hope. Nothing is simulated: no injected error, no fault flag, no early
6119 /// return. The process is gone.
6120 #[test]
6121 #[ignore = "spawned by a_store_reopened_after_a_kill_mid_absorb_requeues_the_pack; it aborts on purpose"]
6122 fn the_child_that_dies_between_durable_bytes_and_indexed_rows() {
6123 let Ok(dir) = std::env::var(KILL_DIR) else {
6124 // Run by hand with `--ignored` and no fixture: there is nothing to
6125 // kill over, so do nothing rather than abort somebody's test run.
6126 return;
6127 };
6128 let dir = PathBuf::from(dir);
6129 let pack = std::fs::read(dir.join("fixture.pack")).expect("the parent's fixture");
6130
6131 let store = GitStore::open(&dir, "alice").expect("open");
6132 // The drain is parked in front of its absorb for as long as this lives,
6133 // so no row can land before the abort below.
6134 let _gate = store.hold_absorb_gate();
6135 let tx = store.put_pack(&pack).expect("the ack path");
6136 assert_eq!(tx.pack_id, Some(0), "a fresh archive starts at ordinal 0");
6137 assert_eq!(store.unindexed_packs(), 1, "the pack is not queued");
6138 assert_eq!(store.index().len(), 0, "a row landed before the kill");
6139 std::fs::write(dir.join("ready"), b"durable, not indexed").expect("marker");
6140
6141 // Not `panic!`, not `exit`: the process stops here with a live gate, a
6142 // live worker and a live redb handle, which is what a machine that dies
6143 // mid-absorb leaves behind.
6144 std::process::abort();
6145 }
6146
6147 /// **A store reopened over a pack that was still being absorbed re-queues
6148 /// it, answers correctly the whole way through, and gets its derived tables
6149 /// back.**
6150 ///
6151 /// The interruption is a **real** one: a child process is spawned, pushes the
6152 /// pack, and is killed by `SIGABRT` between "bytes durable" and "rows
6153 /// indexed". The parent then asserts what the kill left on disk — one journal
6154 /// row, the pack verbatim, **zero** index rows — before opening a store over
6155 /// it, so the recovery is being tested against the state it claims to
6156 /// recover from and not against a lucky one.
6157 ///
6158 /// Three things are then asserted, all on applied output:
6159 ///
6160 /// 1. **the pack is re-queued** — the account indexer of the *new* process
6161 /// publishes ordinal 0, which it can only do for a job somebody submitted;
6162 /// 2. **a read is correct throughout** — `has` is asked the instant the store
6163 /// is open, before any drain has been waited for, and must answer `true`;
6164 /// 3. **`commit_count()` and `reachable()` come back** — the derived tables
6165 /// are rebuilt by the same absorb, so the graph holds exactly the pack's
6166 /// commits and a tip's closure is non-empty.
6167 ///
6168 /// Seen RED by making `open_with` re-queue nothing (`absorber
6169 /// .adopt_journal(&acked)?` → an empty `Vec`, which is what this store did
6170 /// before this change): "a durable object read `absent` from a store reopened
6171 /// over the pack that holds it — the crash-recovery bit did not re-queue
6172 /// anything". Restored.
6173 ///
6174 /// Seen RED a second time by inverting the diff in
6175 /// [`Absorber::adopt_journal`] (`if absorbed` → `if !absorbed`), so a pack
6176 /// with **no** rows has its bit set and a pack with rows is re-queued: the
6177 /// same first assertion fires, because a set bit over an empty index is
6178 /// exactly the wrong answer a bloom filter's false positive would have
6179 /// produced — the read stops falling back and says `absent` about an object
6180 /// whose bytes are durable. Restored.
6181 ///
6182 /// **Why the assertions are not immediately after the reopen.** The trap the
6183 /// drain guards already record applies here too, mirrored: right after
6184 /// `GitStore::open` the drain may legitimately have finished the re-queued
6185 /// pack already, so `unindexed_packs() == 1` is not assertable and is not
6186 /// asserted. What *is* assertable at that instant is the answer to a read,
6187 /// which must be `true` whichever side of the drain it lands on — and that is
6188 /// exactly the property §13.12 promises.
6189 ///
6190 /// **And it still holds now that a `gc` can retire a pack's journal row.**
6191 /// The tombstone tells "indexed then garbage-collected" from "never
6192 /// indexed", and this pack is the second: nothing retired it, so it must
6193 /// still be re-queued. Seen RED by making `adopt_journal` treat *every* pack
6194 /// as retired (`retired.contains(&extent.0)` → `true`): "a durable object
6195 /// read `absent` from a store reopened over the pack that holds it — the
6196 /// crash-recovery bit did not re-queue anything". Restored.
6197 #[test]
6198 fn a_store_reopened_after_a_kill_mid_absorb_requeues_the_pack() {
6199 use std::os::unix::process::ExitStatusExt;
6200
6201 let dir = tmpdir("kill-mid-absorb");
6202 let (pack, rows) = corpus_pack();
6203 std::fs::write(dir.join("fixture.pack"), &pack).unwrap();
6204
6205 let status = std::process::Command::new(std::env::current_exe().unwrap())
6206 .args([
6207 "--exact",
6208 "--ignored",
6209 "--nocapture",
6210 "git_ops::tests::the_child_that_dies_between_durable_bytes_and_indexed_rows",
6211 ])
6212 .env(KILL_DIR, &dir)
6213 .status()
6214 .expect("spawn the child that dies");
6215 assert_eq!(
6216 status.signal(),
6217 Some(6),
6218 "the child did not die by SIGABRT — it {status:?}, so nothing was interrupted"
6219 );
6220 assert!(
6221 dir.join("ready").exists(),
6222 "the child never reached the kill point"
6223 );
6224
6225 // ── what the kill left on disk ───────────────────────────────────────
6226 let blobs = dir.join("objects.pack");
6227 let acked = read_journal(&SafeWriter::journal_path(&blobs)).unwrap();
6228 assert_eq!(
6229 acked.len(),
6230 1,
6231 "the pack's extent is not durable: {acked:?}"
6232 );
6233 assert_eq!(acked[0].1, pack.len() as u64);
6234 let on_disk = std::fs::read(&blobs).unwrap();
6235 assert_eq!(
6236 &on_disk[acked[0].0 as usize..(acked[0].0 + acked[0].1) as usize],
6237 &pack[..],
6238 "the verbatim bytes did not survive the kill"
6239 );
6240 {
6241 // Opened and dropped before the store takes redb's file lock.
6242 let tail = ObjectReadStack::<OneTableFourColumns>::open(
6243 &dir.join("objects.tail"),
6244 RebuildTriggers::default(),
6245 crate::arms::DEFAULT_REDB_CACHE_BYTES,
6246 )
6247 .unwrap();
6248 assert_eq!(
6249 tail.len(),
6250 0,
6251 "the kill landed after the rows were indexed, not between the bytes and the rows \
6252 — this guard would then be testing nothing"
6253 );
6254 }
6255
6256 // ── the reopen ───────────────────────────────────────────────────────
6257 let store = GitStore::open(&dir, "alice").unwrap();
6258 assert!(
6259 store.has(&rows[0].oid).unwrap(),
6260 "a durable object read `absent` from a store reopened over the pack that holds it — \
6261 the crash-recovery bit did not re-queue anything"
6262 );
6263
6264 store.wait_indexed();
6265 assert_eq!(
6266 store.object_count(),
6267 rows.len(),
6268 "the re-queued pack's objects never reached the index — left: {}, right: {}",
6269 store.object_count(),
6270 rows.len()
6271 );
6272 assert!(
6273 store.indexer().is_indexed(0),
6274 "the re-queued pack's indexed bit never went up in the new process"
6275 );
6276 assert_eq!(store.unindexed_packs(), 0);
6277 assert_eq!(store.absorb_pending().unwrap(), 0);
6278 for r in rows.iter().take(512) {
6279 let row = store
6280 .index()
6281 .lookup(&r.oid)
6282 .unwrap_or_else(|| panic!("{} has no row after recovery", hex::encode(&r.oid)));
6283 assert_eq!((row.offset, row.len), (acked[0].0 + r.offset, r.len));
6284 }
6285
6286 // ── the derived tables, which were the second symptom ────────────────
6287 assert_eq!(
6288 store.commit_count(),
6289 commits_in(&rows),
6290 "the commit graph did not come back: {} of {}",
6291 store.commit_count(),
6292 commits_in(&rows)
6293 );
6294 let tip = store
6295 .graph_snapshot()
6296 .into_iter()
6297 .max_by_key(|c| c.generation)
6298 .expect("a graph");
6299 let tip_raw = hex::decode(&tip.oid).unwrap();
6300 let closure = store.reachable(&[&tip_raw], &[]).unwrap();
6301 assert!(
6302 !closure.is_empty(),
6303 "reachable() came back empty after the reopen"
6304 );
6305 assert!(closure.contains(&tip_raw));
6306 eprintln!(
6307 "load {}; killed mid-absorb: {} journal extent(s) recovered {} object rows, {} \
6308 commits, closure of the tip {} objects",
6309 loadavg(),
6310 acked.len(),
6311 store.object_count(),
6312 store.commit_count(),
6313 closure.len(),
6314 );
6315 }
6316
6317 /// **A clean reopen re-queues nothing, and the next push gets a fresh
6318 /// ordinal.**
6319 ///
6320 /// The other side of the diff. A store whose pack was fully absorbed before
6321 /// it closed must come back with the bit **set**: re-absorbing would be a
6322 /// whole pack re-resolved for nothing, and — because the commit graph is a
6323 /// `Vec` — it is the same doubling the drain's own guard exists to prevent.
6324 ///
6325 /// "Re-queued nothing" is asserted deterministically rather than by a timing
6326 /// window: the account indexer in this process publishes a pack only for a
6327 /// job somebody submitted, so `is_indexed(0)` being **false** after
6328 /// `wait_indexed` is proof that open submitted none.
6329 ///
6330 /// The second half is the ordinal. Pack ids are the journal's row numbers, so
6331 /// a reopened store must resume after them. Asserted on applied output rather
6332 /// than on the id alone: the newly pushed pack's object has to be findable,
6333 /// and it is exactly what goes missing when a fresh pack is handed an
6334 /// absorbed pack's ordinal — `queue` skips it, the drain skips it, and its
6335 /// durable bytes are never indexed by anybody.
6336 ///
6337 /// Seen RED by making `Absorber::adopt_journal` treat every journal extent as
6338 /// unabsorbed (`let has_rows = vec![false; journal.len()]`): "a fully
6339 /// absorbed pack was re-queued on open — the diff is not exact".
6340 ///
6341 /// Seen RED a second time by `AtomicU64::new(packs_already_acked(...)?)` →
6342 /// `AtomicU64::new(0)` in `PushPath::with_absorber`: "a reopened store handed
6343 /// a fresh pack the ordinal of an absorbed one — left: Some(0), right:
6344 /// Some(1)". With that same edit **and the id assertion deleted**, the
6345 /// applied-output one goes too — "the second push's object never reached the
6346 /// index — its ordinal collided with an absorbed pack's, so every writer
6347 /// skipped it" — which is what says this guard is about the objects and not
6348 /// about a counter. Restored.
6349 #[test]
6350 fn a_clean_reopen_requeues_nothing_and_the_next_push_gets_a_fresh_ordinal() {
6351 let dir = tmpdir("clean-reopen");
6352 let (pack, rows) = corpus_pack();
6353 let (second, blob_oid) = one_blob_pack(b"a push that arrives after the restart");
6354
6355 {
6356 let store = GitStore::open(&dir, "alice").unwrap();
6357 store.put_pack(&pack).unwrap();
6358 store.wait_indexed();
6359 assert_eq!(
6360 store.object_count(),
6361 rows.len(),
6362 "the first run did not index"
6363 );
6364 }
6365
6366 let store = GitStore::open(&dir, "alice").unwrap();
6367 store.wait_indexed();
6368 assert!(
6369 !store.indexer().is_indexed(0),
6370 "a fully absorbed pack was re-queued on open — the diff is not exact"
6371 );
6372 assert_eq!(store.unindexed_packs(), 0);
6373 assert_eq!(
6374 store.absorb_pending().unwrap(),
6375 0,
6376 "the reopen left index work"
6377 );
6378 assert_eq!(
6379 store.object_count(),
6380 rows.len(),
6381 "the reopened store lost rows"
6382 );
6383 assert!(store.has(&rows[0].oid).unwrap());
6384
6385 let tx = store.put_pack(&second).unwrap();
6386 assert_eq!(
6387 tx.pack_id,
6388 Some(1),
6389 "a reopened store handed a fresh pack the ordinal of an absorbed one"
6390 );
6391 store.wait_indexed();
6392 assert!(
6393 store.has(&blob_oid).unwrap(),
6394 "the second push's object never reached the index — its ordinal collided with an \
6395 absorbed pack's, so every writer skipped it"
6396 );
6397 assert_eq!(store.object_count(), rows.len() + 1);
6398 assert!(store.indexer().is_indexed(1));
6399 // And the journal is a log: both packs' extents are in it, in order.
6400 let acked = read_journal(&SafeWriter::journal_path(store.blobs_path())).unwrap();
6401 assert_eq!(
6402 acked.len(),
6403 2,
6404 "the reopen truncated the journal: {acked:?}"
6405 );
6406 assert_eq!(acked[1], tx.extent.unwrap());
6407 }
6408
6409 /// **What the derive-on-open diff costs at a realistic pack count.**
6410 ///
6411 /// Not a guard — a measurement, with two assertions on it that keep it from
6412 /// measuring the wrong thing: every absorbed pack must come back `true`, and
6413 /// the one pack that has no rows must come back `false`.
6414 ///
6415 /// Both cases are timed because they are different algorithms in practice.
6416 /// The **clean reopen** — every pack absorbed — terminates as soon as each
6417 /// extent has been hit once, which happens after a few rows per pack because
6418 /// the tail is in oid order and an oid says nothing about which pack its
6419 /// object came from. The **crash** case has one pack with no rows at all, and
6420 /// proving an absence means reading the tail to the end.
6421 ///
6422 /// MEASURED on oden 2026-08-08, release, 1000 real pushed packs / 3686 object
6423 /// rows, 1-minute loadavg 2.77: **0.38 ms** clean and 0.39 ms with one pack
6424 /// un-absorbed (0.40 / 0.39 ms at loadavg 13.06, so the figure barely moves
6425 /// with the box). At this row count the two cases are the same, because a
6426 /// 3686-row tail is scanned to the end faster than the early exit saves
6427 /// anything; they diverge with the object count, and the table on
6428 /// [`ObjectReadStack::extents_with_rows`] carries that at a million rows
6429 /// (0.74 ms against 109 ms).
6430 ///
6431 /// The printed line carries the loadavg of its own run, the pack count and
6432 /// the row count, because none of the three is comparable across machines or
6433 /// across a busy box.
6434 #[test]
6435 fn the_derive_on_open_diff_costs_milliseconds_at_a_realistic_pack_count() {
6436 const PACKS: usize = 1000;
6437 let dir = tmpdir("diff-cost");
6438 let (real, rows) = corpus_pack();
6439
6440 {
6441 let store = GitStore::open(&dir, "alice").unwrap();
6442 store.put_pack(&real).unwrap();
6443 for i in 0..PACKS - 1 {
6444 let (p, _) = one_blob_pack(format!("pack number {i}").as_bytes());
6445 store.put_pack(&p).unwrap();
6446 }
6447 store.wait_indexed();
6448 assert_eq!(
6449 store.object_count(),
6450 rows.len() + PACKS - 1,
6451 "the fixture did not absorb"
6452 );
6453 }
6454
6455 let acked = read_journal(&SafeWriter::journal_path(&dir.join("objects.pack"))).unwrap();
6456 assert_eq!(acked.len(), PACKS, "one journal row per pack");
6457 // The tail on its own, so what is timed is the diff rather than redb's
6458 // open and the projection build that a store's open also pays for.
6459 let tail = ObjectReadStack::<OneTableFourColumns>::open(
6460 &dir.join("objects.tail"),
6461 RebuildTriggers::default(),
6462 crate::arms::DEFAULT_REDB_CACHE_BYTES,
6463 )
6464 .unwrap();
6465
6466 let t = Instant::now();
6467 let hit = tail.extents_with_rows(&acked).unwrap();
6468 let clean = t.elapsed();
6469 assert!(
6470 hit.iter().all(|&h| h),
6471 "an absorbed pack was reported unabsorbed"
6472 );
6473
6474 // One pack in flight when the machine died: durable bytes, no rows.
6475 let mut in_flight = acked.clone();
6476 let end = acked[PACKS - 1].0 + acked[PACKS - 1].1;
6477 in_flight.push((end, 4096));
6478 let t = Instant::now();
6479 let hit = tail.extents_with_rows(&in_flight).unwrap();
6480 let crashed = t.elapsed();
6481 assert_eq!(
6482 hit.iter().filter(|h| !**h).count(),
6483 1,
6484 "the pack with no rows is not the only one reported unabsorbed"
6485 );
6486
6487 eprintln!(
6488 "load {}; {PACKS} packs / {} object rows: derive-on-open diff {:.2} ms clean \
6489 (early exit), {:.2} ms with one pack un-absorbed (full tail scan)",
6490 loadavg(),
6491 tail.len(),
6492 clean.as_secs_f64() * 1e3,
6493 crashed.as_secs_f64() * 1e3,
6494 );
6495 }
6496
6497 /// **The bit is a bitset over dense ordinals, and one bit is one pack.**
6498 ///
6499 /// Ordinals that straddle word boundaries (63, 64, 65) are the whole point:
6500 /// a shift that ignores the word split, or a word index that does not, sets
6501 /// a neighbour's bit — and a neighbour's bit set means a durable pack is
6502 /// declared absorbed with no rows behind it.
6503 ///
6504 /// Seen RED by `>> (i % 64) & 1` → `>> (i % 63) & 1` in
6505 /// `PackState::is_absorbed`: "the bit for ordinal 64 did not go up". Restored.
6506 ///
6507 /// Seen RED a second time by `if self.extent[i].is_none()` → `if
6508 /// self.extent[i].is_none() || true` in `PackState::note`, i.e. letting a
6509 /// re-queue of a pack that is already recorded count as new work — which is
6510 /// the race the ack path really produces, since the drain can absorb before
6511 /// `queue` runs: "an absorbed pack was queued again — left: 6, right: 5".
6512 /// Restored.
6513 #[test]
6514 fn the_indexed_bit_is_a_bitset_over_dense_pack_ordinals() {
6515 let ids = [0u64, 1, 63, 64, 65, 4095];
6516 let mut s = PackState::default();
6517 for id in ids {
6518 s.note(id, (id * 4096 + 12, 4096));
6519 }
6520 assert_eq!(s.unabsorbed, ids.len());
6521 assert!(
6522 ids.iter().all(|&i| !s.is_absorbed(i)),
6523 "a bit is up already"
6524 );
6525
6526 s.mark_absorbed(64, (64 * 4096 + 12, 4096));
6527 assert!(s.is_absorbed(64), "the bit for ordinal 64 did not go up");
6528 for other in [0u64, 1, 63, 65, 4095] {
6529 assert!(
6530 !s.is_absorbed(other),
6531 "setting ordinal 64 also set ordinal {other} — one bit spilled onto a neighbour"
6532 );
6533 }
6534 assert_eq!(s.unabsorbed, ids.len() - 1);
6535 assert_eq!(
6536 s.pending().iter().map(|p| p.pack_id).collect::<Vec<_>>(),
6537 vec![0, 1, 63, 65, 4095],
6538 "the pending list is not the clear bits"
6539 );
6540 assert_eq!(
6541 s.pending()[0],
6542 PendingPack {
6543 pack_id: 0,
6544 offset: 12,
6545 len: 4096,
6546 resumed: true,
6547 },
6548 "a pending pack lost the extent it must be absorbed from"
6549 );
6550
6551 // Both writers are idempotent, because both of them race.
6552 s.mark_absorbed(64, (64 * 4096 + 12, 4096));
6553 s.note(64, (64 * 4096 + 12, 4096));
6554 assert!(s.is_absorbed(64));
6555 assert_eq!(
6556 s.unabsorbed,
6557 ids.len() - 1,
6558 "an absorbed pack was queued again"
6559 );
6560
6561 // One bit per pack, and that is the whole cost: 4096 ordinals is 64
6562 // words. A hash set of the same ordinals is an order of magnitude more,
6563 // and a bloom would be smaller and wrong.
6564 assert_eq!(s.words.len(), 64);
6565 assert_eq!(s.words.len() * 8, 512, "4096 packs is 512 bytes of bits");
6566 }
6567
6568 /// **A read that arrives before the drain is served from the pack, on its
6569 /// own thread, and never waits for the worker.**
6570 ///
6571 /// The owner's rule, 2026-08-30: *nobody waits on the indexer.* Until then
6572 /// this test asserted the opposite — that a read parked behind the absorb
6573 /// gate stayed unanswered until the drain released it, which on a kernel
6574 /// pack is hours, and under the memory watchdog is *as long as the box is
6575 /// short*. Now the gate is the absorbers' and only the absorbers': the read
6576 /// finds no row, sees a pending pack, and resolves that pack read-only
6577 /// from the mmapped bytes for the one oid it wants
6578 /// (`Absorber::scan_pending`). The drain is still parked, the index is
6579 /// still empty, and the answer is **present** — from the verbatim truth.
6580 ///
6581 /// Seen RED by making `lookup_never_wait` return `Ok(None)` when the index
6582 /// misses: "a read with the drain parked answered absent for a durable
6583 /// object". And by putting `self.gate.lock()` back in front of it: the
6584 /// 2-second bound below trips with the reader still parked.
6585 #[test]
6586 fn a_read_before_the_drain_is_served_from_the_pack_and_never_waits() {
6587 let dir = tmpdir("drain-before");
6588 let store = Arc::new(GitStore::open(&dir, "alice").unwrap());
6589 let (pack, rows) = corpus_pack();
6590 let (second, blob_oid) = one_blob_pack(b"a second push while the drain is parked");
6591
6592 // The drain is parked here, before it can absorb anything.
6593 let gate = store.hold_absorb_gate();
6594 let tx = store.put_pack(&pack).unwrap();
6595 let pack_id = tx.pack_id.unwrap();
6596 assert_eq!(store.unindexed_packs(), 1, "the pack is not queued");
6597 assert_eq!(store.index().len(), 0, "the index already holds rows");
6598 assert!(!store.indexer().is_indexed(pack_id), "the bit is already up");
6599
6600 // (1) a second push does not wait either: the ack path never meets the gate.
6601 let t = Instant::now();
6602 let tx2 = store.put_pack(&second).unwrap();
6603 assert_eq!(tx2.pack_id, Some(pack_id + 1));
6604 assert!(t.elapsed() < Duration::from_secs(2), "the push waited on the parked drain");
6605
6606 // (2) reads into exactly that state: present, from the pack, bounded.
6607 let oid = rows[0].oid.clone();
6608 let last = rows[rows.len() - 1].oid.clone();
6609 let t = Instant::now();
6610 assert!(store.has(&oid).unwrap(), "a read with the drain parked answered absent for a durable object");
6611 assert!(store.has(&last).unwrap(), "the last object of the pack answered absent");
6612 assert!(store.has(&blob_oid).unwrap(), "the second (still pending) pack's object answered absent");
6613 let (kind, body) = store.content(&blob_oid).unwrap().expect("content from the pending pack");
6614 assert_eq!(kind, GitObjectKind::Blob);
6615 assert_eq!(body, b"a second push while the drain is parked");
6616 let stored = store.get(&oid).unwrap().expect("get from the pending pack");
6617 assert_eq!(stored.extent.0, rows[0].offset + tx.extent.unwrap().0);
6618 assert_eq!(
6619 store.size(&last).unwrap(),
6620 Some(rows[rows.len() - 1].uncompressed_size),
6621 "size from the pending pack"
6622 );
6623 let absent = vec![0xEEu8; 20];
6624 assert!(!store.has(&absent).unwrap(), "an oid in no pack must still be absent");
6625 let elapsed = t.elapsed();
6626 assert!(elapsed < Duration::from_secs(20), "reads took {elapsed:?} with the drain parked");
6627 // The drain is STILL parked and the index STILL empty: nothing above
6628 // absorbed anything, and nothing waited.
6629 assert_eq!(store.index().len(), 0, "a read absorbed the pack behind the parked gate");
6630 assert!(!store.indexer().is_indexed(pack_id));
6631 assert_eq!(store.unindexed_packs(), 2);
6632 eprintln!("{} reads served from two pending packs with the drain parked, in {elapsed:?}", 6);
6633
6634 // (3) a whole-graph request is refused by name, not waited on.
6635 let err = store.reachable(&[oid.as_slice()], &[]).unwrap_err();
6636 assert!(err.to_string().contains("index in progress"), "{err:#}");
6637
6638 drop(gate);
6639 // And afterwards it is served from the rows.
6640 store.wait_indexed();
6641 assert_eq!(store.unindexed_packs(), 0);
6642 assert!(store.indexer().is_indexed(pack_id));
6643 assert_eq!(store.index().len(), rows.len() + 1);
6644 assert_eq!(store.absorb_pending().unwrap(), 0);
6645 for r in rows.iter().take(256) {
6646 assert!(
6647 store.index().lookup(&r.oid).is_some(),
6648 "{} is not in the rows after the drain",
6649 hex::encode(&r.oid)
6650 );
6651 }
6652 assert!(store.reachable(&[oid.as_slice()], &[]).is_ok(), "reachable after the drain");
6653 }
6654
6655 /// **The memory watchdog pauses the indexer worker and nobody else.**
6656 ///
6657 /// A fake `/proc/meminfo` under the floor is put under the process-wide
6658 /// watchdog; the worker parks in front of its batch (`index_worker`'s
6659 /// pre-batch check) and stays parked. Meanwhile a push is acked, every
6660 /// kind of read answers from the pack, and the progress snapshot returns —
6661 /// all bounded, none of them behind the pause. Then the fake is raised,
6662 /// the worker resumes, and the pack is indexed normally.
6663 ///
6664 /// Seen RED with the pre-batch check taking the absorb gate before pausing
6665 /// (the reads' scan does not need the gate, but `absorb_pending`-based
6666 /// reads did): every read parked for the length of the pause.
6667 #[test]
6668 fn a_paused_indexer_blocks_no_push_and_no_read() {
6669 let dir = tmpdir("paused-indexer");
6670 let meminfo = dir.join("meminfo");
6671 let write = |avail_kib: u64| {
6672 std::fs::write(&meminfo, format!("MemTotal: 16000000 kB\nMemAvailable: {avail_kib} kB\n"))
6673 .unwrap()
6674 };
6675 write(1_000_000);
6676 let dog = Arc::new(crate::memguard::MemWatchdog::with(
6677 &meminfo,
6678 3_200_000 * 1024,
6679 Duration::from_millis(10),
6680 ));
6681 let store = Arc::new(GitStore::open(&dir, "alice").unwrap());
6682 // This store's worker only: every other store in the process keeps
6683 // the real watchdog.
6684 store.set_mem_watchdog(dog.clone());
6685 let (pack, rows) = corpus_pack();
6686 let (second, blob_oid) = one_blob_pack(b"pushed while the indexer is paused");
6687
6688 let t_push = Instant::now();
6689 let tx = store.put_pack(&pack).unwrap();
6690 let pack_id = tx.pack_id.unwrap();
6691 // Give the worker time to reach its pre-batch check and park.
6692 let mut parked = false;
6693 for _ in 0..200 {
6694 if dog.is_paused() {
6695 parked = true;
6696 break;
6697 }
6698 std::thread::sleep(Duration::from_millis(10));
6699 }
6700 assert!(parked, "the worker never paused under a meminfo 1 GB < 3.2 GB");
6701 assert!(!store.indexer().is_indexed(pack_id), "indexed while paused");
6702 assert_eq!(store.index().len(), 0);
6703
6704 // The ack path: a push while paused.
6705 let tx2 = store.put_pack(&second).unwrap();
6706 assert_eq!(tx2.pack_id, Some(pack_id + 1));
6707 assert!(t_push.elapsed() < Duration::from_secs(5), "a push waited on the paused worker");
6708
6709 // Reads while paused: present, from the pack.
6710 let t = Instant::now();
6711 assert!(store.has(&rows[0].oid).unwrap());
6712 assert!(store.has(&blob_oid).unwrap());
6713 let (kind, body) = store.content(&blob_oid).unwrap().unwrap();
6714 assert_eq!((kind, body.as_slice()), (GitObjectKind::Blob, &b"pushed while the indexer is paused"[..]));
6715 assert_eq!(
6716 store.extents(&[rows[0].oid.as_slice(), rows[1].oid.as_slice(), blob_oid.as_slice()])
6717 .unwrap()
6718 .iter()
6719 .filter(|e| e.is_some())
6720 .count(),
6721 3,
6722 "the batch path answered absent while paused"
6723 );
6724 assert!(t.elapsed() < Duration::from_secs(20), "reads waited on the paused worker");
6725 // The progress row: information, never a gate.
6726 let snap = store.index_progress();
6727 assert_eq!(snap.root, store.root().display().to_string());
6728 assert!(dog.is_paused(), "the worker resumed on its own — the fake still says 1 GB");
6729 assert!(!store.indexer().is_indexed(pack_id), "indexed while still paused");
6730
6731 // A whole-graph request while the worker is paused: the gate is free
6732 // (a paused worker holds nothing), so the READER absorbs the pending
6733 // packs on its own thread and answers — the worker is still parked.
6734 let reach = store.reachable(&[rows[0].oid.as_slice()], &[]).unwrap();
6735 assert!(!reach.is_empty());
6736 assert_eq!(store.unindexed_packs(), 0, "the reader did not absorb for itself");
6737 assert!(dog.is_paused(), "the worker is still parked; the reader did the work");
6738 assert_eq!(store.index().len(), rows.len() + 1);
6739
6740 // Memory comes back: the worker resumes, finds nothing to do, and the
6741 // bits go up through the ordinary drain.
6742 write(9_000_000);
6743 store.wait_indexed();
6744 assert!(store.indexer().is_indexed(pack_id));
6745 assert!(store.indexer().is_indexed(pack_id + 1));
6746 assert_eq!(dog.pauses(), 1, "one pause, one resume");
6747 assert!(!dog.is_paused());
6748 }
6749
6750 /// **The bit is cleared after the rows go in, never before.**
6751 ///
6752 /// The ordering inside `absorb_gated` is the whole reason a read cannot be
6753 /// told "absent" while the drain is running, and the only way to see it is to
6754 /// read *during* one. Four gatling workers (LAW 3 — never rayon, and never a
6755 /// raw thread pool) hammer `has` over the pack's oids from the moment
6756 /// `put_pack` returns until the pack's bit is up.
6757 ///
6758 /// Asserted on applied output: not one of those reads may answer absent for
6759 /// an object whose bytes are already durable.
6760 ///
6761 /// Seen RED by clearing the bit *before* absorbing (a
6762 /// `s.pending.retain(|p| p.pack_id != job.pack_id)` added in `absorb_gated`
6763 /// above the `self.absorb_one(job)` call): "4 reader(s) were told a durable
6764 /// object was absent — **32 865 593** absent answers over 2687 objects".
6765 /// Restored. The count is that large because the readers spin until the bit
6766 /// goes up and, with the bit cleared first, every pass over the sample missed
6767 /// for the whole length of the absorb.
6768 ///
6769 /// The honest limit: this is a race, so a green run proves the window was not
6770 /// hit, not that no window exists. That is why the printed line carries how
6771 /// many of the reads actually landed while the pack was still un-indexed —
6772 /// **a run where that number is 0 proved nothing**, and it is reported rather
6773 /// than asserted because forcing it would mean pausing the drain, which is
6774 /// the one thing this test must not do. MEASURED 2026-08-08 on oden at
6775 /// loadavg 16.94: 336 of 336 reads landed early.
6776 #[test]
6777 fn no_read_is_told_absent_while_the_drain_is_running() {
6778 let dir = tmpdir("drain-during");
6779 let store = GitStore::open(&dir, "alice").unwrap();
6780 let (pack, rows) = corpus_pack();
6781 let oids: Vec<&[u8]> = rows.iter().map(|r| r.oid.as_slice()).collect();
6782
6783 let tx = store.put_pack(&pack).unwrap();
6784 let pack_id = tx.pack_id.unwrap();
6785
6786 let early = AtomicU64::new(0);
6787 let absent = AtomicU64::new(0);
6788 let reads = AtomicU64::new(0);
6789 let workers = 4usize;
6790 gatling_for_each(workers, workers, |w| loop {
6791 let still_pending = store.unindexed_packs() > 0;
6792 for oid in oids.iter().skip(w).step_by(workers * 8) {
6793 if still_pending {
6794 early.fetch_add(1, AtomicOrdering::Relaxed);
6795 }
6796 reads.fetch_add(1, AtomicOrdering::Relaxed);
6797 if !store.has(oid).unwrap() {
6798 absent.fetch_add(1, AtomicOrdering::Relaxed);
6799 }
6800 }
6801 if store.indexer().is_indexed(pack_id) {
6802 break;
6803 }
6804 });
6805
6806 let absent = absent.load(AtomicOrdering::Acquire);
6807 assert_eq!(
6808 absent,
6809 0,
6810 "{workers} reader(s) were told a durable object was absent — {absent} absent answers \
6811 over {} objects",
6812 rows.len()
6813 );
6814 assert_eq!(store.object_count(), rows.len());
6815 eprintln!(
6816 "load {}; {} reads across {workers} gatling workers, {} of them while the pack was \
6817 still un-indexed",
6818 loadavg(),
6819 reads.load(AtomicOrdering::Acquire),
6820 early.load(AtomicOrdering::Acquire),
6821 );
6822 }
6823
6824 /// **One pack is absorbed once, whichever of the two callers gets there
6825 /// first.**
6826 ///
6827 /// Both callers are made to attempt it, deterministically: with the gate
6828 /// held, the drain is parked inside `ObjectAbsorb::absorb` and a reader is
6829 /// parked inside `absorb_pending`, both past their own early exits. Releasing
6830 /// the gate lets exactly one do the work; the other must take the skip.
6831 ///
6832 /// The observable is the **commit graph**, not the object table:
6833 /// `ObjectReadStack::append` is keyed by oid and would swallow a second
6834 /// identical row, but the graph is a `Vec` and a second absorb pushes every
6835 /// commit into it again — which silently doubles the fold every reachability
6836 /// bitmap is built from.
6837 ///
6838 /// Seen RED by disabling the `s.absorbed.contains(&job.pack_id)` skip in
6839 /// `absorb_gated`: "the commit graph carries **1102** commits, the pack has
6840 /// **551**" — exactly double, both callers having done the whole fold.
6841 /// Restored.
6842 #[test]
6843 fn a_pack_is_absorbed_once_even_when_both_callers_race_for_it() {
6844 let dir = tmpdir("drain-once");
6845 let store = Arc::new(GitStore::open(&dir, "alice").unwrap());
6846 let (pack, rows) = corpus_pack();
6847
6848 let gate = store.hold_absorb_gate();
6849 store.put_pack(&pack).unwrap();
6850
6851 let s = store.clone();
6852 let reader = Job::spawn(move || s.absorb_pending().unwrap());
6853 // Both are now blocked on the gate: the drain inside `absorb`, the reader
6854 // inside `absorb_pending`, each having already decided there is work.
6855 std::thread::sleep(Duration::from_millis(100));
6856 drop(gate);
6857
6858 let by_read = reader.join().unwrap();
6859 store.wait_indexed();
6860
6861 assert_eq!(
6862 store.commit_count(),
6863 commits_in(&rows),
6864 "the commit graph carries {} commits, the pack has {}",
6865 store.commit_count(),
6866 commits_in(&rows)
6867 );
6868 assert_eq!(store.object_count(), rows.len());
6869 assert_eq!(store.unindexed_packs(), 0);
6870 eprintln!(
6871 "load {}; the read absorbed {by_read} pack(s), the drain absorbed the rest",
6872 loadavg()
6873 );
6874 }
6875
6876 // ── §14's exploded table ─────────────────────────────────────────────────
6877
6878 /// **A push produces an exploded row for every object in the pack, and each
6879 /// one hashes back to its own oid.**
6880 ///
6881 /// Eager, decided 2026-08-08: no threshold, no size cap, no type skipped —
6882 /// so the row count is the *object* count and not some subset of it. Nothing
6883 /// is read between `put_pack` and `wait_indexed`, so the only thing that
6884 /// could have built the rows is the background drain, which is the half of
6885 /// §14 that says "built by the same background indexer".
6886 ///
6887 /// The assertion is applied output in its strongest available form: every
6888 /// object's content is read back out of the table, re-serialised canonically
6889 /// and **re-hashed**, and the hash has to be the oid it was filed under. A
6890 /// row count alone could be satisfied by 2687 rows of the wrong bytes; a
6891 /// re-hash cannot.
6892 ///
6893 /// Seen RED by deleting the `sink.explode(&oid, kind, &payload)?` call in
6894 /// `resolve::resolve_walked`: "the drain built no exploded rows: 0 of 2687".
6895 /// Restored.
6896 ///
6897 /// Seen RED a second time by making the sink skip blobs in
6898 /// `ExplodedTable::explode` (`if kind == GitObjectKind::Blob { return Ok(()) }`
6899 /// at the top — which is what a *lazy* or size-capped table would look like
6900 /// from here): "the drain built no exploded rows: 1420 of 2687". Restored,
6901 /// because §14's open question was closed EAGER and disk is explicitly not a
6902 /// consideration — and because the 1267 rows that edit removes are exactly
6903 /// the blobs a content read wants most.
6904 #[test]
6905 fn a_push_produces_an_exploded_row_for_every_object_in_the_pack() {
6906 let dir = tmpdir("exploded-every-object");
6907 let store = GitStore::open(&dir, "alice").unwrap();
6908 let (pack, rows) = corpus_pack();
6909
6910 let t = Instant::now();
6911 store.put_pack(&pack).unwrap();
6912 let ack = t.elapsed();
6913 store.wait_indexed();
6914 // **Drain, printed beside ack**, because the two answer different
6915 // questions and only one of them was ever in doubt. Ack is unchanged by
6916 // construction — nothing on it touches this table. Drain is where the
6917 // eager resolve and the sink both live, and it is the column redb's
6918 // 2026-08-08 measurement recorded as 316.2–329.8 ms against 166.7–168.6
6919 // ms with the sink stubbed out. Without it here, "did the medium buy
6920 // anything but disk" has no answer in this repository.
6921 let drain = t.elapsed();
6922
6923 let distinct: HashSet<Vec<u8>> = rows.iter().map(|r| r.oid.clone()).collect();
6924 let stats = store.exploded_stats();
6925 // **The push explodes the GRAPH, not the content.** Under the shipped
6926 // policy the drain writes commits and trees and leaves blobs in the
6927 // pack; the assertion is therefore two-sided, because "some rows" would
6928 // pass on a table that wrote one row and on one that wrote all of them,
6929 // and it is exactly the second that costs 17× the pack.
6930 let graph_rows = store.exploded_of_kind(GitObjectKind::Commit).unwrap().len();
6931 assert_eq!(
6932 store.exploded_eager_rows().unwrap(),
6933 graph_rows as u64,
6934 "the drain wrote {} eager rows and there are {} commits — the eager pool \
6935 holds something other than the graph",
6936 store.exploded_eager_rows().unwrap(),
6937 graph_rows
6938 );
6939 assert!(
6940 stats.rows > 0 && stats.rows < distinct.len() as u64,
6941 "the eager pool holds {} of {} objects; lazy explode means neither 0 \
6942 (the graph would be blind) nor all of them (that is the 22.4x tier)",
6943 stats.rows,
6944 distinct.len()
6945 );
6946
6947 // Every one of them, re-hashed. Bytes that came back wrong cannot pass.
6948 let hash = store.hash_kind();
6949 let mut checked = 0usize;
6950 for oid in &distinct {
6951 let (kind, payload) = store
6952 .content(oid)
6953 .unwrap()
6954 .unwrap_or_else(|| panic!("{} has no exploded row", hex::encode(oid)));
6955 let rehashed = hash.oid_of(&crate::object::canonical(kind, &payload));
6956 assert_eq!(
6957 &rehashed,
6958 oid,
6959 "the exploded row filed under {} contains an object that hashes to {}",
6960 hex::encode(oid),
6961 hex::encode(&rehashed)
6962 );
6963 checked += 1;
6964 }
6965 assert_eq!(checked, distinct.len());
6966 // **Every object came back byte-correct**, and the ones the eager pool
6967 // does not hold came back by inflating their own delta chain out of the
6968 // mapped pack ([`crate::resolve::materialize_at`]). So the re-derivation
6969 // count is the blob count and not zero — asserting zero here is what a
6970 // test written for the eager tier does, and it would now be asserting
6971 // that lazy explode is off.
6972 let after_reads = store.exploded_stats();
6973 assert!(
6974 after_reads.rederived > 0,
6975 "no read re-derived anything, so either every object is eager or the \
6976 lazy path never ran"
6977 );
6978 assert_eq!(
6979 after_reads.rederived + after_reads.served,
6980 checked as u64,
6981 "reads {} = served {} + rederived {} must account for every object",
6982 checked,
6983 after_reads.served,
6984 after_reads.rederived
6985 );
6986 // **What it costs on disk, measured rather than assumed.** §14 says the
6987 // price is "both copies are held" and the decision was taken with "we
6988 // don't care if disk is tripled" — so the real multiple belongs in the
6989 // output where anybody can read it, not in a sentence. It is not 3x on a
6990 // real pack: the verbatim bytes are deflated *and* delta-encoded, the
6991 // table holds raw inflated content, and redb pages it.
6992 let inflated: u64 = rows.iter().map(|r| r.uncompressed_size).sum();
6993 let table = std::fs::metadata(store.exploded_path())
6994 .map(|m| m.len())
6995 .unwrap_or(0);
6996 eprintln!(
6997 "load {}; {} pack entries → {} exploded rows, all {} re-hashed to their own oid; \
6998 ack {:.1} ms, drain {:.1} ms, {:.0} rows/s; disk: pack {:.1} MiB verbatim, objects \
6999 inflate to {:.1} MiB, table file {:.1} MiB ({:.0}x the pack, {:.3}x the payload)",
7000 loadavg(),
7001 rows.len(),
7002 stats.rows,
7003 checked,
7004 ack.as_secs_f64() * 1e3,
7005 drain.as_secs_f64() * 1e3,
7006 stats.rows as f64 / drain.as_secs_f64(),
7007 pack.len() as f64 / (1 << 20) as f64,
7008 inflated as f64 / (1 << 20) as f64,
7009 table as f64 / (1 << 20) as f64,
7010 table as f64 / pack.len() as f64,
7011 table as f64 / inflated.max(1) as f64,
7012 );
7013 }
7014
7015 /// **The amplification measurement, at whatever scale you point it at.**
7016 ///
7017 /// [`a_push_produces_an_exploded_row_for_every_object_in_the_pack`] prints
7018 /// the same ratio on a 5.4 MiB fixture and is a *guard*, so it has to stay
7019 /// fast. This is the same arithmetic with no assertion about size and no
7020 /// fixture of its own: give it a pack and it reports what the table cost.
7021 ///
7022 /// ```text
7023 /// ZNIPPY_EXPLODE_BENCH_PACK=/path/to/pack-….pack \
7024 /// cargo test -p znippy-plugin-git --lib the_table_costs_what_it_holds \
7025 /// -- --ignored --nocapture
7026 /// ```
7027 ///
7028 /// It exists because the 12× figure that killed redb — 16.8 GB of resolved
7029 /// `linux.git` content in a 204 GB file — was taken by hand off a bench run
7030 /// that no longer exists, and a number nobody can re-take is a number that
7031 /// rots. `#[ignore]`d and pack-less by default: with no pack named it
7032 /// returns, so it never fails a suite for the absence of a corpus nobody
7033 /// promised to keep.
7034 #[test]
7035 #[ignore = "needs a pack: ZNIPPY_EXPLODE_BENCH_PACK=<path> cargo test … -- --ignored --nocapture"]
7036 fn the_table_costs_what_it_holds() {
7037 let Ok(path) = std::env::var("ZNIPPY_EXPLODE_BENCH_PACK") else {
7038 eprintln!("no ZNIPPY_EXPLODE_BENCH_PACK named; nothing measured");
7039 return;
7040 };
7041 let pack = std::fs::read(&path).unwrap_or_else(|e| panic!("reading {path}: {e}"));
7042 let dir = tmpdir("exploded-cost");
7043 let store = GitStore::open(&dir, "alice").unwrap();
7044
7045 let t = Instant::now();
7046 store.put_pack(&pack).unwrap();
7047 let ack = t.elapsed();
7048 store.wait_indexed();
7049 let drain = t.elapsed();
7050
7051 let stats = store.exploded_stats();
7052 let table = std::fs::metadata(store.exploded_path())
7053 .map(|m| m.len())
7054 .unwrap_or(0);
7055 // The payload the table actually holds, asked of the table rather than
7056 // of the pack — a pack entry's `uncompressed_size` is what git recorded,
7057 // and this is what we stored. Only the second one can be divided into
7058 // the file size and mean anything.
7059 let mut held = 0u64;
7060 for kind in [
7061 GitObjectKind::Commit,
7062 GitObjectKind::Tree,
7063 GitObjectKind::Blob,
7064 GitObjectKind::Tag,
7065 ] {
7066 held += store
7067 .exploded_of_kind(kind)
7068 .unwrap()
7069 .iter()
7070 .map(|(_, p)| p.len() as u64)
7071 .sum::<u64>();
7072 }
7073 let mib = |n: u64| n as f64 / (1 << 20) as f64;
7074 eprintln!(
7075 "load {}; pack {} — {:.1} MiB verbatim → {} rows holding {:.1} MiB, table file \
7076 {:.1} MiB = {:.3}x the payload and {:.1}x the pack; ack {:.0} ms, drain {:.0} ms",
7077 loadavg(),
7078 path,
7079 mib(pack.len() as u64),
7080 stats.rows,
7081 mib(held),
7082 mib(table),
7083 table as f64 / held.max(1) as f64,
7084 table as f64 / pack.len() as f64,
7085 ack.as_secs_f64() * 1e3,
7086 drain.as_secs_f64() * 1e3,
7087 );
7088 }
7089
7090 /// Where the child of the clean-shutdown guard finds its fixture and leaves
7091 /// its markers.
7092 const CLEAN_DIR: &str = "GUNNAR_CLEAN_SHUTDOWN_DIR";
7093
7094 /// **The other half of
7095 /// [`a_clean_shutdown_and_reopen_keeps_the_graph_and_reachable`]: the process
7096 /// that exits normally.**
7097 ///
7098 /// A real process, a real push, a real drain, and then a real `drop` — every
7099 /// worker joined, every redb handle closed, no signal, exit status 0. That is
7100 /// what makes the parent's reopen a *clean* reopen rather than a recovery,
7101 /// and it is the case the crash guard cannot reach: after a clean shutdown
7102 /// every pack's bit is legitimately set and **nothing is re-queued**, so the
7103 /// derived tables have to have been on disk or they are gone.
7104 ///
7105 /// `#[ignore]`d because it is a fixture, not a guard — run on its own it has
7106 /// no directory to work in and returns.
7107 #[test]
7108 #[ignore = "spawned by a_clean_shutdown_and_reopen_keeps_the_graph_and_reachable"]
7109 fn the_child_that_pushes_and_shuts_down_cleanly() {
7110 let Ok(dir) = std::env::var(CLEAN_DIR) else {
7111 return;
7112 };
7113 let dir = PathBuf::from(dir);
7114 let pack = std::fs::read(dir.join("fixture.pack")).expect("the parent's fixture");
7115
7116 let store = GitStore::open(&dir, "alice").expect("open");
7117 store.put_pack(&pack).expect("the ack path");
7118 store.wait_indexed();
7119 assert_eq!(store.unindexed_packs(), 0, "the child shut down mid-index");
7120
7121 let tip = store
7122 .graph_snapshot()
7123 .into_iter()
7124 .max_by_key(|c| c.generation)
7125 .expect("the child built no graph at all");
7126 let tip_raw = hex::decode(&tip.oid).unwrap();
7127 let closure = store.reachable(&[&tip_raw], &[]).unwrap();
7128 assert!(!closure.is_empty(), "the child's own reachable() was empty");
7129
7130 std::fs::write(dir.join("tip"), &tip.oid).expect("marker");
7131 std::fs::write(dir.join("closure"), closure.len().to_string()).expect("marker");
7132 std::fs::write(dir.join("commits"), store.commit_count().to_string()).expect("marker");
7133 std::fs::write(dir.join("objects"), store.object_count().to_string()).expect("marker");
7134
7135 // **The clean shutdown.** Not a signal and not an abort: the store drops,
7136 // which closes the channel, joins the account indexer's worker and closes
7137 // redb. The process then returns normally with status 0.
7138 drop(store);
7139 std::fs::write(dir.join("clean"), b"closed").expect("marker");
7140 }
7141
7142 /// **`commit_count()` and `reachable()` are correct after a CLEAN shutdown
7143 /// and reopen.** This is the guard the exploded table exists to make pass.
7144 ///
7145 /// It was the live bug: the commit graph was folded from commit and tree
7146 /// *payloads* which were stored nowhere, accumulated in RAM as packs were
7147 /// absorbed. A store killed mid-absorb got them back only because the
7148 /// interrupted pack was re-queued and re-resolved
7149 /// ([`a_store_reopened_after_a_kill_mid_absorb_requeues_the_pack`]); a store
7150 /// that shut down **cleanly** re-queued nothing, and MEASURED on oden
7151 /// 2026-08-08 came back with 2687 rows and **0 of 551 commits** —
7152 /// `reachable()` returning a commit alone instead of its closure, and `gc`
7153 /// seeing an empty live set. Both are quiet wrong answers.
7154 ///
7155 /// The shape is the crash guard's: a **real** child process, which here exits
7156 /// normally, and the parent asserts against what is on disk. Two assertions
7157 /// carry it and they have to be in this order:
7158 ///
7159 /// 1. **`commit_count()` immediately after `open`**, before anything is
7160 /// waited on — so the graph can only have come from the fold over the
7161 /// exploded table that `open` performs, not from a re-absorb;
7162 /// 2. **`is_indexed(0)` is false after `wait_indexed()`** — the account
7163 /// indexer in *this* process publishes a pack only for a job somebody
7164 /// submitted, so a clear bit proves open re-queued nothing and the graph
7165 /// above is not a recovery artefact.
7166 ///
7167 /// `reachable()` is then asserted against the closure the child computed on
7168 /// the same tip, so it is the same answer and not merely a non-empty one.
7169 ///
7170 /// Seen RED by deleting `store.refold()?` from the end of
7171 /// `GitStore::open_with_arms`, so nothing folds the table on open: "the
7172 /// commit graph did not survive a clean shutdown: 0 of 551 commits — the
7173 /// exploded table is a warm cache, not a durable table", left 0 right 551.
7174 /// That is the recorded bug's own number, reproduced. Restored.
7175 ///
7176 /// Seen RED a second time by deleting `self.exploded.flush()?` from
7177 /// `Absorber::absorb_one`, which is the **durability** edit: the rows then
7178 /// live only in the un-flushed buffer, which answers perfectly inside the
7179 /// child and is gone when the process ends. It fires assertion (2) rather
7180 /// than (1) — "the reopen re-queued the pack, so the graph above is a
7181 /// recovery artefact rather than the table's" — and that is worth writing
7182 /// down, because assertion (1) *passed*: the 64 MiB flush threshold had
7183 /// tripped once mid-pack, leaving 1798 of 2687 rows on disk which happened
7184 /// to include all 551 commits. A guard with only assertion (1) would have
7185 /// called that edit green. Restored.
7186 /// **An annotated tag reaches the commit it points at.**
7187 ///
7188 /// # The bug this is the guard for
7189 ///
7190 /// Reachability bitmaps are built over the commit graph, and no tree points
7191 /// at a tag, so a tag is in nobody's closure. `reachable`'s `want` loop had
7192 /// one arm for "not a commit": insert the object itself, and stop. So a
7193 /// `want` that was an annotated tag answered with **one object** — the tag —
7194 /// and the commit it names, plus that commit's whole history, was left out.
7195 ///
7196 /// It stayed invisible because a tag is nearly always pushed alongside the
7197 /// branch that carries the same commit, and then the branch's closure
7198 /// covers it. It surfaces the moment a tag is the ONLY thing naming a
7199 /// commit — which is what gunnar's replication does, one ref at a time. Six
7200 /// repositories on the live fleet could not replicate on 2026-08-31, one of
7201 /// them with 25 tags hanging off a single commit no branch carried, all
7202 /// refused by the far end as `missing necessary objects … (peeling a tag)`.
7203 ///
7204 /// So the fixture asserts the premise it needs: the tag is the only `want`.
7205 ///
7206 /// # 🔴 Seen red
7207 ///
7208 /// Against the arm as it was (`union.insert(o)` and nothing more):
7209 /// `the tag's closure is 1 object, so it carries the tag and nothing it
7210 /// points at`. With the peel: the commit, its trees and its blobs.
7211 #[test]
7212 fn an_annotated_tag_reaches_the_commit_it_points_at() {
7213 let dir = tmpdir("tag-peel");
7214 let store = GitStore::open(&dir, "alice").unwrap();
7215 let (pack, rows) = corpus_pack();
7216 store.put(&pack, &[]).unwrap();
7217 store.wait_indexed();
7218
7219 // Any commit will do: the tag is the only `want`, so nothing else can
7220 // pull this commit's closure in behind our backs.
7221 let commit = rows
7222 .iter()
7223 .find(|r| r.kind == GitObjectKind::Commit)
7224 .expect("the corpus has commits");
7225 let commit_hex = hex::encode(&commit.oid);
7226 let commit_closure = store.reachable(&[&commit.oid], &[]).unwrap();
7227 assert!(
7228 commit_closure.len() > 1,
7229 "the fixture's commit closes over nothing, so this test could not tell a peel from a no-op"
7230 );
7231
7232 let (tag_pack, tag_oid) = crate::store::tests::one_tag_pack(&commit_hex, "v1.0");
7233 store.put(&tag_pack, &[]).unwrap();
7234 store.wait_indexed();
7235
7236 let closure = store.reachable(&[&tag_oid], &[]).unwrap();
7237 assert!(
7238 closure.contains(&tag_oid),
7239 "the tag itself is not in its own closure"
7240 );
7241 assert!(
7242 closure.contains(&commit.oid),
7243 "the tag's closure is {} object(s), so it carries the tag and nothing it points at — the commit {} it names is missing and a mirror built from this answer cannot be read",
7244 closure.len(),
7245 &commit_hex[..12]
7246 );
7247 // Not merely the commit: everything the commit reaches, or a clone from
7248 // this answer is short by a tree.
7249 for oid in &commit_closure {
7250 assert!(
7251 closure.contains(oid),
7252 "the tag's closure is missing {} which the commit it points at reaches",
7253 hex::encode(oid)
7254 );
7255 }
7256 }
7257
7258 #[test]
7259 fn a_clean_shutdown_and_reopen_keeps_the_graph_and_reachable() {
7260 let dir = tmpdir("clean-shutdown-graph");
7261 let (pack, rows) = corpus_pack();
7262 std::fs::write(dir.join("fixture.pack"), &pack).unwrap();
7263
7264 let status = std::process::Command::new(std::env::current_exe().unwrap())
7265 .args([
7266 "--exact",
7267 "--ignored",
7268 "--nocapture",
7269 "git_ops::tests::the_child_that_pushes_and_shuts_down_cleanly",
7270 ])
7271 .env(CLEAN_DIR, &dir)
7272 .status()
7273 .expect("spawn the child that shuts down cleanly");
7274 assert!(
7275 status.success(),
7276 "the child did not exit cleanly — it {status:?}, so this is a crash guard and not a \
7277 clean-shutdown one"
7278 );
7279 assert!(
7280 dir.join("clean").exists(),
7281 "the child never reached its clean shutdown"
7282 );
7283
7284 let tip_hex = std::fs::read_to_string(dir.join("tip")).unwrap();
7285 let closure_before: usize = std::fs::read_to_string(dir.join("closure"))
7286 .unwrap()
7287 .parse()
7288 .unwrap();
7289 let commits_before: usize = std::fs::read_to_string(dir.join("commits"))
7290 .unwrap()
7291 .parse()
7292 .unwrap();
7293 assert_eq!(
7294 commits_before,
7295 commits_in(&rows),
7296 "the child's own graph was already wrong, so the reopen proves nothing"
7297 );
7298
7299 // ── the reopen, and (1): the graph before anything is waited on ──────
7300 let store = GitStore::open(&dir, "alice").unwrap();
7301 assert_eq!(
7302 store.commit_count(),
7303 commits_in(&rows),
7304 "the commit graph did not survive a clean shutdown: {} of {} commits — the exploded \
7305 table is a warm cache, not a durable table",
7306 store.commit_count(),
7307 commits_in(&rows)
7308 );
7309 let tip_raw = hex::decode(&tip_hex).unwrap();
7310 let closure = store.reachable(&[&tip_raw], &[]).unwrap();
7311 assert_eq!(
7312 closure.len(),
7313 closure_before,
7314 "reachable() answers differently after a clean reopen: {} objects against the {} the \
7315 same tip closed over before the shutdown",
7316 closure.len(),
7317 closure_before
7318 );
7319 assert!(closure.contains(&tip_raw));
7320
7321 // ── (2): and none of it came from a re-absorb ────────────────────────
7322 store.wait_indexed();
7323 assert!(
7324 !store.indexer().is_indexed(0),
7325 "the reopen re-queued the pack, so the graph above is a recovery artefact rather \
7326 than the table's"
7327 );
7328 assert_eq!(store.unindexed_packs(), 0);
7329 assert_eq!(
7330 store.absorb_pending().unwrap(),
7331 0,
7332 "the reopen left index work"
7333 );
7334 assert_eq!(store.object_count(), rows.len());
7335 assert_eq!(
7336 store.exploded_stats().written,
7337 0,
7338 "this process wrote exploded rows, so the table was rebuilt rather than read"
7339 );
7340
7341 eprintln!(
7342 "load {}; clean shutdown → reopen: {} rows, {} commits, tip closure {} objects \
7343 (rebuilt nothing)",
7344 loadavg(),
7345 store.object_count(),
7346 store.commit_count(),
7347 closure.len(),
7348 );
7349 }
7350
7351 /// **A reopen READS `objects.graph`; it does not recompute it — and the
7352 /// read graph equals the recomputed one.** This is S-065's fix, and the only
7353 /// thing that can say which path a fold took is a counter, so the assertion
7354 /// is on [`GitStore::graph_source_stats`], not on the (identical) graph
7355 /// bytes — the identity-value trap, since both paths yield the same graph.
7356 ///
7357 /// One test exercises **both** arms so the counter cannot be blindly stuck:
7358 ///
7359 /// 1. A first store builds the graph, persists it, and shuts down cleanly.
7360 /// 2. `objects.graph` is deleted, so the reopen has no cache: it **recomputes**
7361 /// (`(read, recomputed) == (0, 1)`) and re-persists. Its graph and its
7362 /// `reachable()` are captured — this is the recomputed truth.
7363 /// 3. A second reopen finds the cache the step above wrote: it **reads**
7364 /// (`(1, 0)`), and its graph and `reachable()` must equal step 2's exactly
7365 /// — read == recomputed.
7366 ///
7367 /// # 🔴 Seen red
7368 ///
7369 /// With the persistence removed (open always folds), step 3 is `(0, 1)`:
7370 /// "the reopen recomputed the graph instead of reading objects.graph", left
7371 /// `(0, 1)` right `(1, 0)`. With the token check inverted so a stale cache is
7372 /// accepted, step 3's `reachable()` diverges from step 2's and the read==
7373 /// recomputed assertion fires. Restored.
7374 #[test]
7375 fn a_reopen_reads_the_persisted_graph_it_does_not_recompute() {
7376 let dir = tmpdir("persisted-graph");
7377 let (pack, rows) = corpus_pack();
7378 let tip_hex;
7379 {
7380 let store = GitStore::open(&dir, "alice").unwrap();
7381 store.put_pack(&pack).unwrap();
7382 store.wait_indexed();
7383 tip_hex = store
7384 .graph_snapshot()
7385 .into_iter()
7386 .max_by_key(|c| c.generation)
7387 .expect("a graph")
7388 .oid;
7389 assert!(
7390 dir.join("objects.graph").exists(),
7391 "an absorb never persisted the graph, so a reopen has nothing to read"
7392 );
7393 }
7394 let tip_raw = hex::decode(&tip_hex).unwrap();
7395
7396 // ── (2) no cache → recompute, and capture the recomputed truth ───────
7397 std::fs::remove_file(dir.join("objects.graph")).unwrap();
7398 let (recomputed_graph, recomputed_closure) = {
7399 let store = GitStore::open(&dir, "alice").unwrap();
7400 assert_eq!(
7401 store.graph_source_stats(),
7402 (0, 1),
7403 "with no objects.graph the reopen should have recomputed the graph"
7404 );
7405 assert_eq!(store.commit_count(), commits_in(&rows));
7406 let g = store.graph_snapshot();
7407 let c = store.reachable(&[&tip_raw], &[]).unwrap();
7408 assert!(!c.is_empty());
7409 (g, c)
7410 };
7411
7412 // ── (3) cache present → read, and it must equal the recomputed truth ──
7413 let store = GitStore::open(&dir, "alice").unwrap();
7414 assert_eq!(
7415 store.graph_source_stats(),
7416 (1, 0),
7417 "the reopen recomputed the graph instead of reading objects.graph"
7418 );
7419 assert_eq!(
7420 store.graph_snapshot(),
7421 recomputed_graph,
7422 "the graph READ from objects.graph differs from the one refold RECOMPUTES"
7423 );
7424 assert_eq!(
7425 store.reachable(&[&tip_raw], &[]).unwrap(),
7426 recomputed_closure,
7427 "reachable() over the persisted graph differs from reachable() over the recomputed one"
7428 );
7429 assert_eq!(store.commit_count(), commits_in(&rows));
7430 }
7431
7432 /// **The graph cache switched off leaves no `objects.graph`** — and the store
7433 /// still answers the same graph and closure, folded from the exploded table.
7434 /// The switch is flipped on the absorber directly: `ZNIPPY_GIT_GRAPH_CACHE`
7435 /// is read at open, and setting it process-wide would race the test above.
7436 ///
7437 /// Seen red with the persist guard removed: "the graph cache is off and
7438 /// objects.graph was written anyway".
7439 #[test]
7440 fn with_the_graph_cache_off_no_objects_graph_is_written_and_the_graph_is_the_same() {
7441 let on_dir = tmpdir("graph-cache-on");
7442 let off_dir = tmpdir("graph-cache-off");
7443 let (pack, rows) = corpus_pack();
7444 let on = GitStore::open(&on_dir, "alice").unwrap();
7445 on.put_pack(&pack).unwrap();
7446 on.wait_indexed();
7447 let off = GitStore::open(&off_dir, "alice").unwrap();
7448 off.absorber.graph_cache.store(false, std::sync::atomic::Ordering::Relaxed);
7449 std::fs::remove_file(off_dir.join("objects.graph")).ok();
7450 off.put_pack(&pack).unwrap();
7451 off.wait_indexed();
7452 assert!(on_dir.join("objects.graph").exists(), "the default still persists the graph");
7453 assert!(
7454 !off_dir.join("objects.graph").exists(),
7455 "the graph cache is off and objects.graph was written anyway"
7456 );
7457 assert_eq!(off.graph_source_stats().0, 0, "a store with the cache off never reads one");
7458 assert_eq!(on.graph_snapshot(), off.graph_snapshot());
7459 assert_eq!(off.commit_count(), commits_in(&rows));
7460 }
7461
7462 /// **The token gates the read: a cache whose token does not match the
7463 /// object set is rejected and the graph is recomputed.** This is what makes
7464 /// `objects.graph` a memo rather than a second truth — and a push, a gc, or
7465 /// a torn write all reach the store through exactly this door.
7466 ///
7467 /// The object set is left unchanged and the token in the cache is corrupted
7468 /// directly, so the ONE thing under test is the token comparison and not
7469 /// some incidental object-count difference.
7470 ///
7471 /// # 🔴 Seen red
7472 ///
7473 /// With the token comparison dropped (`Some((_, g)) => g`), the corrupted
7474 /// cache is read and the reopen is `(1, 0)`: "a cache with the wrong token
7475 /// was trusted". Restored.
7476 #[test]
7477 fn a_graph_whose_token_does_not_match_is_rejected() {
7478 let dir = tmpdir("wrong-token-graph");
7479 let (pack, _rows) = corpus_pack();
7480 {
7481 let store = GitStore::open(&dir, "alice").unwrap();
7482 store.put_pack(&pack).unwrap();
7483 store.wait_indexed();
7484 }
7485 // Flip a byte inside the 32-byte token (offset 5..37) without touching
7486 // the graph body, so only the token is wrong.
7487 let path = dir.join("objects.graph");
7488 let mut bytes = std::fs::read(&path).unwrap();
7489 assert!(bytes.len() > 37 && &bytes[..4] == b"GRPH");
7490 bytes[6] ^= 0xff;
7491 std::fs::write(&path, &bytes).unwrap();
7492
7493 let store = GitStore::open(&dir, "alice").unwrap();
7494 assert_eq!(
7495 store.graph_source_stats(),
7496 (0, 1),
7497 "a cache with the wrong token was trusted instead of being recomputed"
7498 );
7499 }
7500
7501 /// **The ordinal space comes off the persisted index, not off a scan — and
7502 /// no `objects.ordinal` is ever written.** This replaces S-082's memo test:
7503 /// the memo spared a cold open the redb scan that rebuilt the sorted oid
7504 /// list, and since 2026-09-04 that list IS the persisted index's oid column,
7505 /// so there is nothing to spare and no file to validate. What is asserted:
7506 ///
7507 /// 1. A first store absorbs the corpus; its ordinal space equals the tail in
7508 /// oid order and every oid is accounted to one of the two sources. Then
7509 /// it folds explicitly, so redb is empty and the index holds everything.
7510 /// 2. The reopen is warm: its space comes **entirely off the index** —
7511 /// `(folded, tail) == (n, 0)` — and equals step 1's space, closure and
7512 /// counts exactly. `objects.ordinal` does not exist after either session.
7513 ///
7514 /// # 🔴 Seen red
7515 ///
7516 /// With `ObjectReadStack::oids_raw` returning only the redb tail on a
7517 /// persisting stack, step 2's space is empty (redb holds nothing after the
7518 /// fold) and `reachable()` over it is empty: "the ordinal space after the
7519 /// warm reopen is not the tail in oid order". Restored.
7520 #[test]
7521 fn the_ordinal_space_comes_off_the_index_not_a_scan() {
7522 let dir = tmpdir("ordinal-off-the-index");
7523 let (pack, rows) = corpus_pack();
7524 let tip_hex;
7525 let (first_oids, first_closure, first_objects, first_commits);
7526 {
7527 let store = GitStore::open(&dir, "alice").unwrap();
7528 store.put_pack(&pack).unwrap();
7529 store.wait_indexed();
7530 tip_hex = store
7531 .graph_snapshot()
7532 .into_iter()
7533 .max_by_key(|c| c.generation)
7534 .expect("a graph")
7535 .oid;
7536 let tip_raw = hex::decode(&tip_hex).unwrap();
7537 let n = store.object_count() as u64;
7538 let (folded, tail) = store.ordinal_source_stats();
7539 assert_eq!(folded + tail, n, "every oid is accounted to the index or the tail");
7540 let oids = store.ordinal_oids();
7541 let scanned: Vec<String> =
7542 store.index().oids_in_order().unwrap().iter().map(hex::encode).collect();
7543 assert_eq!(oids, scanned, "the ordinal space is not the repository in oid order");
7544 assert!(
7545 !dir.join("objects.ordinal").exists(),
7546 "no memo may be written: the index's oid column is the ordinal space"
7547 );
7548 // Fold, so the reopen has everything on the index and nothing in redb.
7549 store.index().rebuild().unwrap();
7550 first_closure = store.reachable(&[&tip_raw], &[]).unwrap();
7551 assert!(!first_closure.is_empty());
7552 first_oids = oids;
7553 first_objects = store.object_count();
7554 first_commits = store.commit_count();
7555 }
7556 let tip_raw = hex::decode(&tip_hex).unwrap();
7557
7558 let store = GitStore::open(&dir, "alice").unwrap();
7559 let n = store.object_count() as u64;
7560 assert_eq!(
7561 store.ordinal_source_stats(),
7562 (n, 0),
7563 "after a fold the warm reopen's ordinal space must come entirely off the index"
7564 );
7565 assert_eq!(
7566 store.ordinal_oids(),
7567 first_oids,
7568 "the ordinal space after the warm reopen is not the tail in oid order"
7569 );
7570 assert_eq!(
7571 store.reachable(&[&tip_raw], &[]).unwrap(),
7572 first_closure,
7573 "reachable() over the index's ordinal space differs from the first session's"
7574 );
7575 assert_eq!(store.object_count(), first_objects);
7576 assert_eq!(store.commit_count(), first_commits);
7577 assert_eq!(store.commit_count(), commits_in(&rows));
7578 assert!(!dir.join("objects.ordinal").exists(), "still no memo after the reopen");
7579 }
7580
7581 /// **A stale `objects.ordinal` left by an older binary is removed at open.**
7582 /// The file is dead weight now (279 MB on the kernel mirror) and its
7583 /// contents are never read, so a store that carries one loses it the first
7584 /// time this build opens it — and keeps working.
7585 ///
7586 /// # 🔴 Seen red
7587 ///
7588 /// With the `remove_file` in `open_with_arms` dropped, the file survives the
7589 /// open: "a stale objects.ordinal must be removed at open". Restored.
7590 #[test]
7591 fn a_stale_objects_ordinal_from_an_older_binary_is_removed_at_open() {
7592 let dir = tmpdir("stale-ordinal");
7593 let (pack, _rows) = corpus_pack();
7594 {
7595 let store = GitStore::open(&dir, "alice").unwrap();
7596 store.put_pack(&pack).unwrap();
7597 store.wait_indexed();
7598 }
7599 // The old memo's shape: magic, version, oid_len, epoch, count, oids.
7600 let path = dir.join("objects.ordinal");
7601 let mut bytes = b"ORDL".to_vec();
7602 bytes.push(1);
7603 bytes.push(20);
7604 bytes.extend_from_slice(&7u64.to_le_bytes());
7605 bytes.extend_from_slice(&1u64.to_le_bytes());
7606 bytes.extend_from_slice(&[0xabu8; 20]);
7607 std::fs::write(&path, &bytes).unwrap();
7608 assert!(path.exists(), "precondition");
7609
7610 let store = GitStore::open(&dir, "alice").unwrap();
7611 assert!(!path.exists(), "a stale objects.ordinal must be removed at open");
7612 assert!(store.object_count() > 0, "and the store still answers");
7613 }
7614
7615 /// **A content read is served from the table, not re-derived** — and the only
7616 /// thing that can say so is a counter.
7617 ///
7618 /// Both paths return the **identical** bytes: the table hands back what the
7619 /// resolver produced, and the fallback re-resolves the same pack and produces
7620 /// it again. A byte comparison therefore proves nothing at all about which
7621 /// one ran — this is the identity-value trap in its exact form, and it fired
7622 /// on this codebase once already. So the assertion is on
7623 /// [`ExplodedStats::served`] against [`ExplodedStats::rederived`], both of
7624 /// which are bumped where the work happens.
7625 ///
7626 /// The fallback is not asserted away: it is exercised in the same test, on a
7627 /// store whose table has been dropped, and the counters move the other way.
7628 /// A guard that only saw the fast path could not tell a working counter from
7629 /// one that is never incremented.
7630 ///
7631 /// Seen RED by deleting the `if let Some(hit) = self.exploded.content(oid)?`
7632 /// early return in `Absorber::resolved`, so every read re-derives: "512 content
7633 /// reads re-resolved a whole pack instead of hitting the table — served 0,
7634 /// rederived 512". The bytes were still correct on every one of them, which
7635 /// is the point. Restored.
7636 ///
7637 /// Seen RED a second time by bumping `served` unconditionally at the top of
7638 /// `ExplodedTable::content`, before the `pending`/redb probe — the counter
7639 /// then reports a hit whether or not the table had the object. The first half
7640 /// fires on the double count, "512 content reads re-resolved a whole pack
7641 /// instead of hitting the table — served 1024, rederived 0", and
7642 /// `exploded::tests::a_row_round_trips_and_the_kind_index_finds_it` fires with
7643 /// it ("a content read was not counted", left 5 right 2). Restored — a
7644 /// counter that cannot be wrong about a miss is the only kind worth
7645 /// asserting on.
7646 #[test]
7647 fn a_content_read_is_served_from_the_table_and_not_re_derived() {
7648 const READS: usize = 512;
7649 let dir = tmpdir("exploded-served");
7650 let (pack, rows) = corpus_pack();
7651 let oids: Vec<Vec<u8>> = rows
7652 .iter()
7653 .map(|r| r.oid.clone())
7654 .collect::<HashSet<Vec<u8>>>()
7655 .into_iter()
7656 .take(READS)
7657 .collect();
7658
7659 {
7660 let store = GitStore::open(&dir, "alice").unwrap();
7661 store.put_pack(&pack).unwrap();
7662 store.wait_indexed();
7663
7664 // ── FIRST read: the lazy path, timed ────────────────────────
7665 let before = store.exploded_stats();
7666 let t = Instant::now();
7667 let mut first: Vec<(GitObjectKind, Vec<u8>)> = Vec::with_capacity(oids.len());
7668 for oid in &oids {
7669 first.push(
7670 store
7671 .content(oid)
7672 .unwrap()
7673 .expect("the store lost an object"),
7674 );
7675 }
7676 let cold = t.elapsed();
7677 let after = store.exploded_stats();
7678 let rederived = after.rederived - before.rederived;
7679 assert_eq!(
7680 rederived + (after.served - before.served),
7681 oids.len() as u64,
7682 "{} reads = served {} + rederived {} must account for every one",
7683 oids.len(),
7684 after.served - before.served,
7685 rederived
7686 );
7687 assert!(
7688 rederived > 0,
7689 "not one read re-derived: either every object is eager or the lazy \
7690 path never ran, and this test then proves nothing about either"
7691 );
7692
7693 // The write-back is off the read path, so it has to be waited for
7694 // HERE and only here — a reader never does.
7695 store.explode_cache_maintain().unwrap();
7696 // **How many of those reads the cache DECLINED.** A shallow, small
7697 // object is cheaper to inflate again than to keep, so it is read
7698 // directly and nothing is written
7699 // (`exploded_cache::worth_caching`). Those objects are re-derived
7700 // on every read for ever, on purpose, and the second pass below
7701 // has to expect exactly that many — not zero, which is what this
7702 // test asserted before the admission policy existed and would now
7703 // be asserting that the policy is off.
7704 let declined = store.explode_cache_use().direct_reads;
7705 assert!(
7706 declined > 0,
7707 "not one of {} reads was declined as cheap; either every object in \
7708 this corpus is expensive or the admission policy is not running",
7709 oids.len()
7710 );
7711
7712 // ── SECOND read: the design note's "= today" column ──────────
7713 let mid = store.exploded_stats();
7714 let t = Instant::now();
7715 for (i, oid) in oids.iter().enumerate() {
7716 let again = store.content(oid).unwrap().expect("the store lost an object");
7717 assert_eq!(again, first[i], "the second read of an object disagreed with the first");
7718 }
7719 let warm = t.elapsed();
7720 let end = store.exploded_stats();
7721 assert_eq!(
7722 end.rederived - mid.rederived,
7723 declined,
7724 "the second pass re-derived {} objects and the cache declined {declined} — \
7725 every EXPENSIVE object must come out of the table the second time, and \
7726 every cheap one must be read directly again",
7727 end.rederived - mid.rederived
7728 );
7729 assert_eq!(
7730 end.served - mid.served,
7731 oids.len() as u64 - declined,
7732 "the expensive objects did not come out of the table on the second pass"
7733 );
7734 eprintln!(
7735 "load {}; {} content reads: FIRST {:.1} ms ({:.1} µs/read, {} re-derived from \
7736 the pack, {declined} of them too cheap to cache) → SECOND {:.1} ms \
7737 ({:.1} µs/read) = {:.1}x",
7738 loadavg(),
7739 oids.len(),
7740 cold.as_secs_f64() * 1e3,
7741 cold.as_secs_f64() * 1e6 / oids.len() as f64,
7742 rederived,
7743 warm.as_secs_f64() * 1e3,
7744 warm.as_secs_f64() * 1e6 / oids.len() as f64,
7745 cold.as_secs_f64() / warm.as_secs_f64().max(f64::MIN_POSITIVE),
7746 );
7747 }
7748
7749 // ── and the fallback, so the counter is not a constant ───────────────
7750 //
7751 // The table is dropped and the *drain* is what would rebuild it, so the
7752 // reads have to happen before it gets there. `absorb_pending` is called
7753 // by hand first so the fallback under test is the content path's and not
7754 // the index's.
7755 std::fs::remove_file(dir.join("objects.exploded")).unwrap();
7756 // **And the lazy pool with it.** Dropping only the eager table leaves
7757 // the write-backs of the pass above sitting in `objects.exploded.cN`,
7758 // and those would answer — the fallback would then be measured on a
7759 // store that still has a cache, which is the exact shape of a test that
7760 // cannot fail. One `drop_explode_cache` is the sanctioned door, and it
7761 // is what `repo gc` calls.
7762 for e in std::fs::read_dir(&dir).unwrap().flatten() {
7763 if e.file_name().to_string_lossy().starts_with("objects.exploded.c") {
7764 std::fs::remove_file(e.path()).unwrap();
7765 }
7766 }
7767 let store = GitStore::open(&dir, "alice").unwrap();
7768 let gate = store.hold_absorb_gate();
7769 let before = store.exploded_stats();
7770 let mut rederived = 0u64;
7771 for oid in oids.iter().take(8) {
7772 // Straight at the absorber: the store's `content` would absorb the
7773 // re-queued pack first and rebuild the very table this is testing
7774 // the absence of.
7775 assert!(
7776 store.absorber.resolved(oid).unwrap().is_some(),
7777 "the verbatim truth could not re-derive {}",
7778 hex::encode(oid)
7779 );
7780 rederived += 1;
7781 }
7782 let after = store.exploded_stats();
7783 // The drain starts the instant the store opens; on a loaded box it can
7784 // rebuild the small corpus's table before the gate above is taken, and
7785 // then the reads are legitimately served from it. Seen 2026-08-30 under
7786 // a parallel suite. That run proves nothing about the fallback and says
7787 // so, rather than failing on a race the test does not control.
7788 let drain_won = store.unindexed_packs() == 0;
7789 drop(gate);
7790 if drain_won {
7791 eprintln!("the drain rebuilt the table before the gate was held; the fallback was not observed this run");
7792 return;
7793 }
7794 assert_eq!(
7795 after.served - before.served,
7796 0,
7797 "a dropped table still reported {} reads served",
7798 after.served - before.served
7799 );
7800 assert_eq!(
7801 after.rederived - before.rederived,
7802 rederived,
7803 "the fallback ran but was not counted"
7804 );
7805 }
7806
7807 /// ★ **THE READ-AFTER-ACK GAP (lane U → lane Q): every read path answers
7808 /// with the indexer PAUSED, and none of them waits for it.**
7809 ///
7810 /// MEASURED on the rig 2026-08-30 (lane U, both ISOs, timing-dependent):
7811 /// right after an ACKED push a `clone` came back
7812 /// `remote error: object … is not in this repository (peeling a tag)` and a
7813 /// `push --mirror` on top got `[remote rejected] … missing necessary
7814 /// objects`. Both of those are gunnar's `Graph::load` — `GitServe::read` —
7815 /// answering `None` for an object whose bytes were fsynced before the client
7816 /// was told the push had landed. §13.12 says an index miss falls back to the
7817 /// journal; the tag-peel and connectivity-check paths were not observed to.
7818 ///
7819 /// So this holds the absorb gate — the indexer paused, exactly the state
7820 /// lane P's watchdog puts it in — and requires **`has`, `header` and `read`
7821 /// to answer for every object of a just-acked pack**, byte-correct, and to
7822 /// do it inside a deadline. The deadline is not decoration: "nobody waits"
7823 /// (the design note's rule) is violated by a read that blocks on the gate
7824 /// just as badly as by one that answers absent, and a blocking read looks
7825 /// exactly like a correct one to an assertion that has no clock.
7826 ///
7827 /// Seen RED by returning `Ok(None)` from `Absorber::resolved`'s pending arm:
7828 /// 1352 of 1352 objects report "the store said absent for an object it had
7829 /// already acked".
7830 #[test]
7831 fn every_read_path_answers_a_just_acked_push_with_the_indexer_paused() {
7832 use crate::serve::GitServe as _;
7833 let dir = tmpdir("read-after-ack");
7834 let store = Arc::new(GitStore::open(&dir, "alice").unwrap());
7835 let (pack, rows) = corpus_pack();
7836
7837 // The ack: `put_pack` returning is what the client is told.
7838 store.put_pack(&pack).unwrap();
7839
7840 // The indexer, paused. A watchdog pause (lane P) pauses the indexer and
7841 // nothing else, so this is the shape the appliance is really in.
7842 let gate = store.hold_absorb_gate();
7843 assert!(
7844 store.unindexed_packs() > 0,
7845 "the drain absorbed the pack before the gate was taken, so this run \
7846 read an INDEXED store and proves nothing about the window"
7847 );
7848
7849 let distinct: Vec<Vec<u8>> = rows
7850 .iter()
7851 .map(|r| r.oid.clone())
7852 .collect::<HashSet<Vec<u8>>>()
7853 .into_iter()
7854 .collect();
7855 let hash = store.hash_kind();
7856 let n = distinct.len();
7857
7858 // On another thread, so the deadline measures the reads and not this
7859 // test's own patience.
7860 let (tx, rx) = std::sync::mpsc::channel::<Result<(usize, usize, usize)>>();
7861 let reader = {
7862 let store = store.clone();
7863 let distinct = distinct.clone();
7864 std::thread::spawn(move || {
7865 let mut had = 0usize;
7866 let mut headed = 0usize;
7867 let mut readback = 0usize;
7868 let r = (|| -> Result<()> {
7869 for oid in &distinct {
7870 if store.has(oid)? {
7871 had += 1;
7872 }
7873 if crate::serve::GitServe::header(&*store, oid)?.is_some() {
7874 headed += 1;
7875 }
7876 // The tag-peel / connectivity-check path, byte for byte.
7877 let Some((kind, payload)) = crate::serve::GitServe::read(&*store, oid)? else {
7878 continue;
7879 };
7880 let k = match kind {
7881 crate::index_layout::ObjType::Commit => GitObjectKind::Commit,
7882 crate::index_layout::ObjType::Tree => GitObjectKind::Tree,
7883 crate::index_layout::ObjType::Blob => GitObjectKind::Blob,
7884 crate::index_layout::ObjType::Tag => GitObjectKind::Tag,
7885 // A read resolves the chain, so a delta type coming
7886 // back out of it is the store contradicting itself.
7887 other => anyhow::bail!("`read` answered with a {other:?}"),
7888 };
7889 let rehashed = hash.oid_of(&crate::object::canonical(k, &payload));
7890 if &rehashed == oid {
7891 readback += 1;
7892 }
7893 }
7894 Ok(())
7895 })();
7896 let _ = tx.send(r.map(|_| (had, headed, readback)));
7897 })
7898 };
7899
7900 // The deadline. Generous — a cold pending scan of the corpus is
7901 // milliseconds — and it is against a WALL clock while the gate is held,
7902 // so a read that parks on the absorber trips it.
7903 let out = rx
7904 .recv_timeout(Duration::from_secs(20))
7905 .expect(
7906 "a read path blocked on the paused indexer for 20 s: 'nobody waits' is the \
7907 design note's rule and a read that waits for the drain is a bug by definition",
7908 );
7909 let (had, headed, readback) = out.expect("a read path errored inside the index window");
7910 drop(gate);
7911 let _ = reader.join();
7912
7913 assert_eq!(
7914 (had, headed, readback),
7915 (n, n, n),
7916 "the store answered for {had}/{n} `has`, {headed}/{n} `header` and {readback}/{n} \
7917 byte-correct `read`s of a pack it had already ACKED — an object the client was \
7918 told had landed must never read as absent"
7919 );
7920 }
7921
7922 /// ★ **An account at its explode budget serves EVERY object, correctly —
7923 /// only slower.**
7924 ///
7925 /// The quota is a cache size, not a storage limit: over budget the server
7926 /// EVICTS, it does not refuse. So a repository squeezed to a budget far
7927 /// under what it holds must still answer every read byte-for-byte, and the
7928 /// only thing that may change is where the bytes came from — the table, or
7929 /// the pack via [`crate::resolve::materialize_at`].
7930 ///
7931 /// This is requirement (c) of the quota: *"an account at its budget must
7932 /// still serve every read correctly, only slower"*. The `rederived`
7933 /// counter is what proves the eviction really happened, because identical
7934 /// bytes come back either way and a byte comparison cannot tell a working
7935 /// cache from a dead one — the identity-value trap in its exact form.
7936 ///
7937 /// Seen RED by having `write_back` return the payload's bytes from a
7938 /// half-written segment (`out.truncate(1)`): "the store returned the wrong
7939 /// bytes for an object under a squeezed budget".
7940 #[test]
7941 fn an_account_at_its_budget_serves_every_object_only_slower() {
7942 let dir = tmpdir("explode-budget");
7943 let store = GitStore::open(&dir, "alice").unwrap();
7944 let (pack, rows) = corpus_pack();
7945 store.put_pack(&pack).unwrap();
7946 store.wait_indexed();
7947
7948 let oids: Vec<Vec<u8>> = rows
7949 .iter()
7950 .map(|r| r.oid.clone())
7951 .collect::<HashSet<Vec<u8>>>()
7952 .into_iter()
7953 .collect();
7954
7955 // Every object once, unsqueezed: the truth to compare against.
7956 let mut truth: Vec<(GitObjectKind, Vec<u8>)> = Vec::with_capacity(oids.len());
7957 for oid in &oids {
7958 truth.push(store.content(oid).unwrap().expect("the store lost an object"));
7959 }
7960 store.explode_cache_maintain().unwrap();
7961
7962 // Now the plan changes: a budget far under what this repository holds.
7963 store.set_explode_budget(64 << 10);
7964 let before = store.exploded_stats();
7965 for _ in 0..3 {
7966 for (i, oid) in oids.iter().enumerate() {
7967 let got = store
7968 .content(oid)
7969 .unwrap()
7970 .expect("an object went absent when the budget was lowered");
7971 assert_eq!(
7972 got, truth[i],
7973 "the store returned the wrong bytes for an object under a squeezed budget"
7974 );
7975 }
7976 store.explode_cache_maintain().unwrap();
7977 }
7978 let after = store.exploded_stats();
7979 let band = store.explode_cache_use();
7980
7981 assert!(
7982 band.evicted_bytes > 0 || band.lazy_bytes <= 64 << 10,
7983 "the budget was lowered to 64 KiB and the lazy pool holds {} B with nothing \\
7984 evicted — the quota is not enforced, so this run proves nothing about reading \\
7985 under one: {band:?}",
7986 band.lazy_bytes
7987 );
7988 assert!(
7989 after.rederived > before.rederived,
7990 "not one read re-derived from the pack under a 64 KiB budget, so the cache was \\
7991 never squeezed and 'only slower' was never tested"
7992 );
7993 eprintln!(
7994 "load {}; {} objects read 3x under a 64 KiB budget: {} re-derived, {} served, \
7995{} evicted bytes over {} segments; every byte identical",
7996 loadavg(),
7997 oids.len(),
7998 after.rederived - before.rederived,
7999 after.served - before.served,
8000 band.evicted_bytes,
8001 band.evicted_segments,
8002 );
8003 }
8004
8005 /// ★ **A THIN push on top of a just-acked one is accepted with the indexer
8006 /// PAUSED** — lane U's second rig symptom.
8007 ///
8008 /// MEASURED on the rig 2026-08-30 (lane U): right after an acked push,
8009 /// `git push --mirror` of the next pack came back
8010 /// `[remote rejected] … missing necessary objects`, on a varying set of
8011 /// repositories run to run — a race, not a repository. A thin pack names
8012 /// its delta bases by oid and expects the server to have them; the server
8013 /// **did** have them, in a pack it had acked and not yet indexed, and
8014 /// `scan_pending`'s re-entrancy guard answered `None` for every base looked
8015 /// up from inside a scan rather than only for the ones that would have
8016 /// started a second scan.
8017 ///
8018 /// Seen RED by restoring that guard (`if nested { return Ok((out, None)) }`):
8019 /// *"reading the thin pack's object errored inside the index window:
8020 /// resolving a pending pack read-only for lookups — this pack is thin: it
8021 /// deltas against 42088154… which is not in it, and no base source could
8022 /// produce that object's content"*. The **push** is accepted either way —
8023 /// `external_bases_exist` finds the base with a top-level `lookup_one`
8024 /// before it ever resolves — so the assertion that discriminates is the
8025 /// READ with the gate still held, and an earlier draft of this test that
8026 /// dropped the gate first passed under both arms.
8027 #[test]
8028 fn a_thin_push_on_top_of_an_unindexed_one_is_accepted() {
8029 let dir = tmpdir("thin-on-unindexed");
8030 let store = GitStore::open(&dir, "alice").unwrap();
8031 let (pack, rows) = corpus_pack();
8032 store.put_pack(&pack).unwrap();
8033
8034 // The indexer, paused — so the base can only be found in the pack that
8035 // was acked a moment ago.
8036 let gate = store.hold_absorb_gate();
8037 assert!(
8038 store.unindexed_packs() > 0,
8039 "the drain absorbed the first pack before the gate was taken; this run \
8040 pushed onto an INDEXED store and proves nothing"
8041 );
8042
8043 // A base with a payload, and its real size: a `REF_DELTA` declares the
8044 // source size and the resolver checks it, so a made-up number would fail
8045 // for a reason that has nothing to do with the window under test.
8046 let base = rows
8047 .iter()
8048 .find(|r| r.kind == GitObjectKind::Blob && r.uncompressed_size > 0)
8049 .expect("the corpus has a blob");
8050 let thin = crate::pack_fixture::thin_ref_delta_pack(
8051 &base.oid,
8052 base.uncompressed_size,
8053 b"lane Q was here\n",
8054 );
8055
8056 let tx = store.put_pack(&thin).unwrap_or_else(|e| {
8057 panic!("the thin push was refused: {e:#}");
8058 });
8059 assert!(tx.pack_id.is_some(), "the thin pack was not stored");
8060
8061 // **And its object reads back with the gate STILL HELD.** This is the
8062 // nested case and the one the guard was too wide for: the read scans
8063 // pack B, B's entry is a ref-delta, its base is in pack A, and
8064 // resolving it is a second `scan_pending` from inside the first. A
8065 // lookup into A's already-built scan terminates; only a *build* would
8066 // not.
8067 let hash = store.hash_kind();
8068 let oid = hash.oid_of(&crate::object::canonical(
8069 GitObjectKind::Blob,
8070 b"lane Q was here\n",
8071 ));
8072 let got = store.content(&oid);
8073 drop(gate);
8074 let (kind, payload) = got
8075 .expect("reading the thin pack's object errored inside the index window")
8076 .expect("the thin pack's own object is absent after its push was acked");
8077 assert_eq!(kind, GitObjectKind::Blob);
8078 assert_eq!(payload, b"lane Q was here\n");
8079 }
8080
8081 /// **Dropping the table and reopening still answers correctly — fall back,
8082 /// then rebuild.**
8083 ///
8084 /// §14 calls the resolved table *droppable*: derived, verifiable against the
8085 /// verbatim truth, and deletable at any time without consulting a client.
8086 /// This is that sentence as an operation — `rm objects.exploded` between two
8087 /// opens — and §13.12's rule applied to it unchanged: **absent means fall
8088 /// back, never means wrong.**
8089 ///
8090 /// Three things are asserted, in this order:
8091 ///
8092 /// 1. the file really is gone and the reopened store really does start with
8093 /// an empty table — otherwise the rest is testing a warm one;
8094 /// 2. **`reachable()` is already correct**, asked before any drain has been
8095 /// waited on: the fall-back absorb runs inline and rebuilds what the
8096 /// answer needs;
8097 /// 3. the table is **rebuilt** — one row per object again, the graph back,
8098 /// and the rows written by *this* process, which is what says a rebuild
8099 /// happened rather than a survival.
8100 ///
8101 /// Seen RED by `let exploded_is_whole = self.exploded.rows()? >= self.objects.len()
8102 /// as u64;` → `= true;` in `Absorber::adopt_journal`, so a dropped table is
8103 /// never noticed and nothing is re-queued. Assertion (2) fires first:
8104 /// **"reachable() after dropping the table: 1 objects, against 2683
8105 /// before"** — one object, the tip alone, which is the recorded bug's exact
8106 /// symptom ("`reachable` on a commit returns that commit alone rather than
8107 /// its closure") arrived at from the other direction. Restored.
8108 ///
8109 /// Seen RED a second time by `>=` → `>` in the same expression, which makes a
8110 /// *whole* table look short. This guard stays green — the rebuild is
8111 /// correct — and two others fire instead:
8112 /// [`a_clean_shutdown_and_reopen_keeps_the_graph_and_reachable`]'s "the
8113 /// reopen re-queued the pack, so the graph above is a recovery artefact
8114 /// rather than the table's", and the pre-existing
8115 /// [`a_clean_reopen_requeues_nothing_and_the_next_push_gets_a_fresh_ordinal`]'s
8116 /// "a fully absorbed pack was re-queued on open — the diff is not exact".
8117 /// That pair is the point: one guard catches the table not being rebuilt when
8118 /// it should be, the others catch it being rebuilt when it should not.
8119 /// Restored.
8120 #[test]
8121 fn dropping_the_exploded_table_and_reopening_still_answers() {
8122 let dir = tmpdir("exploded-dropped");
8123 let (pack, rows) = corpus_pack();
8124 let tip_hex;
8125 let closure_before;
8126 {
8127 let store = GitStore::open(&dir, "alice").unwrap();
8128 store.put_pack(&pack).unwrap();
8129 store.wait_indexed();
8130 let tip = store
8131 .graph_snapshot()
8132 .into_iter()
8133 .max_by_key(|c| c.generation)
8134 .expect("a graph");
8135 let tip_raw = hex::decode(&tip.oid).unwrap();
8136 closure_before = store.reachable(&[&tip_raw], &[]).unwrap().len();
8137 tip_hex = tip.oid;
8138 }
8139
8140 // (1) — the drop.
8141 let table = dir.join("objects.exploded");
8142 assert!(table.exists(), "the store never built an exploded table");
8143 std::fs::remove_file(&table).unwrap();
8144 {
8145 let fresh = crate::exploded_arrow::ExplodedArchive::open(&table).unwrap();
8146 assert_eq!(fresh.rows().unwrap(), 0, "the drop did not drop anything");
8147 }
8148 // Opening an absent table does not create one — so this only has
8149 // anything to remove if the assertion above was inspecting a live file.
8150 let _ = std::fs::remove_file(&table);
8151
8152 // (2) — a correct answer with the table gone.
8153 let store = GitStore::open(&dir, "alice").unwrap();
8154 let tip_raw = hex::decode(&tip_hex).unwrap();
8155 let closure = store.reachable(&[&tip_raw], &[]).unwrap();
8156 assert_eq!(
8157 closure.len(),
8158 closure_before,
8159 "reachable() after dropping the table: {} objects, against {} before",
8160 closure.len(),
8161 closure_before
8162 );
8163 assert!(store.has(&rows[0].oid).unwrap());
8164
8165 // (3) — and it is back. The GRAPH is what comes back at push time; a
8166 // blob comes back when somebody reads it, which is the whole of lazy
8167 // explode, so the count to compare against is the graph's and not the
8168 // pack's.
8169 store.wait_indexed();
8170 let stats = store.exploded_stats();
8171 let distinct: HashSet<Vec<u8>> = rows.iter().map(|r| r.oid.clone()).collect();
8172 // Commits, and NOT `of_kind(Tree)`: under the shipped policy a tree is
8173 // lazy, so the trees this test's own reads cached are in the lazy pool
8174 // and `of_kind` unions both.
8175 let graph_rows = store.exploded_of_kind(GitObjectKind::Commit).unwrap().len();
8176 // The EAGER pool, not `stats.rows`: step (2) above read objects through
8177 // the store, and every one of those reads wrote its object back into the
8178 // lazy pool, which `stats.rows` also counts. Comparing the sum against
8179 // the graph would be comparing "what the push wrote" against "what the
8180 // push wrote plus what this test happened to read".
8181 let eager = store.exploded_eager_rows().unwrap();
8182 assert_eq!(
8183 eager, graph_rows as u64,
8184 "the dropped table came back holding {eager} rows against a {}-object graph",
8185 graph_rows
8186 );
8187 assert!(
8188 eager > 0 && eager < distinct.len() as u64,
8189 "the rebuilt pool holds {eager} of {} objects — neither 0 nor all of them is \
8190 the lane Q shape",
8191 distinct.len()
8192 );
8193 assert!(
8194 stats.written > 0,
8195 "the table came back without this process writing a row — it was never dropped"
8196 );
8197 assert_eq!(
8198 store.commit_count(),
8199 commits_in(&rows),
8200 "the graph did not come back with the table: {} of {}",
8201 store.commit_count(),
8202 commits_in(&rows)
8203 );
8204 assert_eq!(
8205 store.object_count(),
8206 rows.len(),
8207 "the rebuild lost index rows"
8208 );
8209 eprintln!(
8210 "load {}; dropped and rebuilt: {} rows written, {} commits, tip closure {} objects",
8211 loadavg(),
8212 stats.written,
8213 store.commit_count(),
8214 closure.len(),
8215 );
8216 }
8217
8218 // ─────────────────────────────────────────────────────────────────────
8219 // lane ENGINE-OPT scaffold: a graded linear-history generator, the O(C)
8220 // negotiation timing test, and the cross-push memo-invalidation guard.
8221 // ─────────────────────────────────────────────────────────────────────
8222
8223 /// A linear commit history of `n` commits as a clone pack, optionally
8224 /// chained onto `start_parent` (an oid already in the store). Each commit
8225 /// carries its own one-entry tree and one unique blob, so the object count
8226 /// is `3n` and the graph is a single parent chain `n` deep — the shape that
8227 /// makes a negotiation's cost O(commits), which is what this lane memoizes.
8228 fn linear_history_pack_from(
8229 n: usize,
8230 start_parent: Option<&[u8]>,
8231 ) -> (Vec<u8>, Vec<Vec<u8>>) {
8232 use crate::object::canonical;
8233 use crate::pack_fixture::{build_pack, Item};
8234 let hash = GitHashKind::Sha1;
8235 let mut items: Vec<Item> = Vec::with_capacity(n * 3);
8236 let mut commits: Vec<Vec<u8>> = Vec::with_capacity(n);
8237 let mut prev: Option<Vec<u8>> = start_parent.map(|p| p.to_vec());
8238 for c in 0..n {
8239 let body = format!("blob for commit {c} of {n}, parent {prev:?}\n").into_bytes();
8240 let blob_oid = hash.oid_of(&canonical(GitObjectKind::Blob, &body));
8241 items.push(Item::Whole(GitObjectKind::Blob, body));
8242
8243 let mut tpayload = Vec::new();
8244 tpayload.extend_from_slice(b"100644 f.bin\0");
8245 tpayload.extend_from_slice(&blob_oid);
8246 let tree_oid = hash.oid_of(&canonical(GitObjectKind::Tree, &tpayload));
8247 items.push(Item::Whole(GitObjectKind::Tree, tpayload));
8248
8249 let mut cbody = format!("tree {}\n", hex::encode(&tree_oid));
8250 if let Some(parent) = &prev {
8251 cbody.push_str(&format!("parent {}\n", hex::encode(parent)));
8252 }
8253 cbody.push_str(
8254 "author A U Thor <a@example.com> 1700000000 +0000\n\
8255 committer A U Thor <a@example.com> 1700000000 +0000\n\n\
8256 commit\n",
8257 );
8258 let cbody = cbody.into_bytes();
8259 let commit_oid = hash.oid_of(&canonical(GitObjectKind::Commit, &cbody));
8260 items.push(Item::Whole(GitObjectKind::Commit, cbody));
8261 prev = Some(commit_oid.clone());
8262 commits.push(commit_oid);
8263 }
8264 (build_pack(&items), commits)
8265 }
8266
8267 /// The O(commits) negotiation cost, graded. `reachable` rebuilt the
8268 /// `by_commit` and graph maps on every call — O(C) regardless of how few
8269 /// objects the request touched — so the `root` query (closure ~1) isolates
8270 /// that map-build cost from the walk, and the `tip` query (full closure)
8271 /// pays both. Median of nine, min–max, with the loadavg of its own run.
8272 #[test]
8273 #[ignore = "perf measurement; run with --release --ignored --nocapture"]
8274 fn bench_reachable_negotiation() {
8275 for &n in &[48usize, 2_000, 20_000] {
8276 let dir = tmpdir(&format!("bench-reach-{n}"));
8277 let (pack, commits) = linear_history_pack_from(n, None);
8278 let store = GitStore::open(&dir, "alice").unwrap();
8279 store.put_pack(&pack).unwrap();
8280 store.wait_indexed();
8281 let root = commits.first().unwrap().clone();
8282 let tip = commits.last().unwrap().clone();
8283 // warm the reach bitmaps once so the loop times the negotiation, not
8284 // the one-time bitmap build.
8285 let _ = store.reachable(&[&tip], &[]).unwrap();
8286
8287 let reps = 9usize;
8288 let (mut root_t, mut tip_t) = (Vec::new(), Vec::new());
8289 for _ in 0..reps {
8290 let t = Instant::now();
8291 let _ = store.reachable(&[&root], &[]).unwrap();
8292 root_t.push(t.elapsed());
8293 let t = Instant::now();
8294 let _ = store.reachable(&[&tip], &[]).unwrap();
8295 tip_t.push(t.elapsed());
8296 }
8297 root_t.sort();
8298 tip_t.sort();
8299 let tip_closure = store.reachable(&[&tip], &[]).unwrap().len();
8300 eprintln!(
8301 "load {}; n={n}: reachable(root, closure~1) med {:?} [{:?}..{:?}] | \
8302 reachable(tip, closure {}) med {:?} [{:?}..{:?}]",
8303 loadavg(),
8304 root_t[reps / 2],
8305 root_t[0],
8306 root_t[reps - 1],
8307 tip_closure,
8308 tip_t[reps / 2],
8309 tip_t[0],
8310 tip_t[reps - 1],
8311 );
8312 }
8313 }
8314
8315 /// **A negotiation answered from the memoized reach view is the answer a
8316 /// cold fold gives — including across a push that changes the fold.**
8317 ///
8318 /// Lane ENGINE-OPT memoizes the `by_commit` and graph lookup maps that
8319 /// `reachable` used to rebuild on every call, in `Derived`, folded by the
8320 /// same call that folds `commit_raw`. The bug this discipline exists to
8321 /// catch is a memo that SURVIVES a fold it should not: after a push adds
8322 /// commits, the warm store must answer with the new closure, byte-identical
8323 /// (order included — both spaces sort the same oids) to a fresh store
8324 /// opened cold on the same bytes.
8325 ///
8326 /// Seen RED by skipping the memo rebuild in `refold` (leaving `graph_index`
8327 /// from before the push): the post-push tip is then not a known commit,
8328 /// `reachable` contributes it alone, and the warm answer is a handful of
8329 /// objects where the cold fold gives the whole chain — the assertion names
8330 /// both lengths.
8331 #[test]
8332 fn reachable_memo_matches_a_cold_fold_across_a_push() {
8333 let dir = tmpdir("reach-memo-across-push");
8334 let (pack_a, commits_a) = linear_history_pack_from(40, None);
8335 let store = GitStore::open(&dir, "alice").unwrap();
8336 store.put_pack(&pack_a).unwrap();
8337 store.wait_indexed();
8338 let tip_a = commits_a.last().unwrap().clone();
8339 let warm_a = store.reachable(&[&tip_a], &[]).unwrap();
8340 assert!(
8341 warm_a.len() > 10,
8342 "A's closure is too small to tell a stale memo from a coincidence: {}",
8343 warm_a.len()
8344 );
8345
8346 // Push B: a second linear history chained ONTO A's tip, so B's tip
8347 // closes over A too. This is a fold that genuinely grows the graph.
8348 let (pack_b, commits_b) = linear_history_pack_from(40, Some(&tip_a));
8349 store.put_pack(&pack_b).unwrap();
8350 store.wait_indexed();
8351 let tip_b = commits_b.last().unwrap().clone();
8352 let warm_b_full = store.reachable(&[&tip_b], &[]).unwrap();
8353 let warm_b_minus_a = store.reachable(&[&tip_b], &[&tip_a]).unwrap();
8354 assert!(
8355 warm_b_full.len() > warm_a.len(),
8356 "the push did not grow the closure ({} !> {}), so the memo could be \
8357 stale and this test would not know",
8358 warm_b_full.len(),
8359 warm_a.len()
8360 );
8361
8362 // The cold oracle: a fresh handle on the same bytes, folded from disk.
8363 drop(store);
8364 let cold = GitStore::open(&dir, "alice").unwrap();
8365 cold.wait_indexed();
8366 let cold_b_full = cold.reachable(&[&tip_b], &[]).unwrap();
8367 let cold_b_minus_a = cold.reachable(&[&tip_b], &[&tip_a]).unwrap();
8368
8369 assert_eq!(
8370 warm_b_full, cold_b_full,
8371 "warm memo != cold fold for want=tip_b (a memo that survived the push)"
8372 );
8373 assert_eq!(
8374 warm_b_minus_a, cold_b_minus_a,
8375 "warm memo != cold fold for want=tip_b have=tip_a"
8376 );
8377 // tip_a is still answered identically after the push (its closure did
8378 // not change; only the store grew around it).
8379 let cold_a = cold.reachable(&[&tip_a], &[]).unwrap();
8380 assert_eq!(
8381 warm_a, cold_a,
8382 "tip_a's closure changed across the push"
8383 );
8384 eprintln!(
8385 "load {}; A closure {}, B closure {}, B−A {}",
8386 loadavg(),
8387 warm_a.len(),
8388 warm_b_full.len(),
8389 warm_b_minus_a.len(),
8390 );
8391 }
8392
8393 /// **The clone-after-push lever, witnessed.** A push folds; the fold used
8394 /// to drop the reachability table and the next reader rebuilt it by
8395 /// walking every commit's trees — 0.2 ms per object, 10 s on a 51k-object
8396 /// repository, measured 2026-09-05. Now the fold CARRIES the table
8397 /// (renumbered) when the graph grew by no more than the carry rule allows.
8398 /// RED before the carry: `reach_rebuilt` went 1 → 2 across the push.
8399 #[test]
8400 fn a_push_carries_the_reach_table_across_the_fold() {
8401 let dir = tmpdir("reach-carry-across-push");
8402 let (pack_a, commits_a) = linear_history_pack_from(40, None);
8403 let store = GitStore::open(&dir, "alice").unwrap();
8404 store.put_pack(&pack_a).unwrap();
8405 store.wait_indexed();
8406 let tip_a = commits_a.last().unwrap().clone();
8407 let warm_a = store.reachable(&[&tip_a], &[]).unwrap();
8408 assert_eq!(store.reach_source_stats(), (0, 1), "the first read builds the table once");
8409
8410 // A push of 30 commits onto A's tip: within the carry rule (64 or a
8411 // tenth of 40, whichever is larger), so the fold carries.
8412 let (pack_b, commits_b) = linear_history_pack_from(30, Some(&tip_a));
8413 store.put_pack(&pack_b).unwrap();
8414 store.wait_indexed();
8415 let tip_b = commits_b.last().unwrap().clone();
8416 let warm_b = store.reachable(&[&tip_b], &[]).unwrap();
8417 let warm_b_minus_a = store.reachable(&[&tip_b], &[&tip_a]).unwrap();
8418 assert_eq!(
8419 store.reach_source_stats(),
8420 (1, 1),
8421 "the fold carried the table; no read rebuilt it"
8422 );
8423 assert!(warm_b.len() > warm_a.len(), "the push grew the closure");
8424 assert_eq!(warm_b_minus_a.len(), warm_b.len() - warm_a.len(), "tip_b − tip_a is exactly the pushed objects");
8425 assert_eq!(store.reachable(&[&tip_a], &[]).unwrap(), warm_a, "tip_a's closure survived the renumbering");
8426
8427 // The cold oracle: the same bytes folded from disk with a fresh table.
8428 drop(store);
8429 let cold = GitStore::open(&dir, "alice").unwrap();
8430 cold.wait_indexed();
8431 assert_eq!(cold.reachable(&[&tip_b], &[]).unwrap(), warm_b, "carried != rebuilt for want=tip_b");
8432 assert_eq!(
8433 cold.reachable(&[&tip_b], &[&tip_a]).unwrap(),
8434 warm_b_minus_a,
8435 "carried != rebuilt for want=tip_b have=tip_a"
8436 );
8437 // The cold handle READ the table the warm one's drop wrote back
8438 // (extended for the push), followed the space (nothing to append) and
8439 // had nothing to extend — nothing was built from scratch.
8440 assert_eq!(cold.reach_source_stats(), (1, 0), "the cold handle kept the persisted table and rebuilt nothing");
8441 let (loaded, extended, _) = cold.reach_stats();
8442 assert_eq!((loaded, extended), (1, 0), "loaded once, nothing left to extend");
8443 }
8444
8445 /// **A push of any size extends the table; nothing is rebuilt.** Under
8446 /// the sorted ordinal space a push past the carry allowance (a tenth of
8447 /// the history) dropped the table and the next read walked every tree in
8448 /// the repository again; under the append-only space the fold gives the
8449 /// pushed tip an entry by the bounded walk and the table stands. RED on the
8450 /// old code: `reach_source_stats` went (0, 1) → (0, 2) across the push.
8451 #[test]
8452 fn a_push_of_any_size_extends_the_reach_table_and_rebuilds_nothing() {
8453 let dir = tmpdir("reach-extend-not-rebuild");
8454 let (pack_a, commits_a) = linear_history_pack_from(20, None);
8455 let store = GitStore::open(&dir, "alice").unwrap();
8456 store.put_pack(&pack_a).unwrap();
8457 store.wait_indexed();
8458 let tip_a = commits_a.last().unwrap().clone();
8459 let warm_a = store.reachable(&[&tip_a], &[]).unwrap();
8460 assert_eq!(store.reach_source_stats(), (0, 1));
8461 // 100 commits onto 20: five times the history, far past the old allowance.
8462 let (pack_b, commits_b) = linear_history_pack_from(100, Some(&tip_a));
8463 store.put_pack(&pack_b).unwrap();
8464 store.wait_indexed();
8465 let tip_b = commits_b.last().unwrap().clone();
8466 let warm_b = store.reachable(&[&tip_b], &[]).unwrap();
8467 assert_eq!(store.reach_source_stats(), (1, 1), "the fold kept the table; no read rebuilt it");
8468 let (_, extended, unpersisted) = store.reach_stats();
8469 assert_eq!(extended, 1, "the fold extended the table for the pushed commits");
8470 assert_eq!(unpersisted, 1, "one extension since the last write-back");
8471 assert_eq!(warm_b.len(), warm_a.len() + 300, "100 commits × (commit, tree, blob)");
8472 assert_eq!(store.reachable(&[&tip_a], &[]).unwrap(), warm_a, "tip_a's closure is untouched");
8473 drop(store);
8474 let cold = GitStore::open(&dir, "alice").unwrap();
8475 cold.wait_indexed();
8476 assert_eq!(cold.reachable(&[&tip_b], &[]).unwrap(), warm_b, "extended warm != cold");
8477 // The drop wrote the extended table back; the cold open read it whole
8478 // and had nothing to extend.
8479 let (loaded, extended, _) = cold.reach_stats();
8480 assert_eq!((loaded, extended), (1, 0), "read back the table the drop persisted; nothing to extend");
8481 assert_eq!(cold.reach_source_stats(), (1, 0));
8482 }
8483
8484 /// **A reopen reads the table it built; it does not walk every tree
8485 /// again.** GUN-PERF Z-1: the first read after every start of a git.git
8486 /// store rebuilt the bitmaps, 85 to 150 s. The table (and the ordinal space
8487 /// it is over) is persisted beside the graph and read back. RED: the cold
8488 /// handle reported one rebuild and nothing kept.
8489 #[test]
8490 fn a_reopen_reads_the_persisted_reach_table_and_rebuilds_nothing() {
8491 let dir = tmpdir("reach-persisted-reopen");
8492 let (pack, commits) = linear_history_pack_from(12, None);
8493 let store = GitStore::open(&dir, "alice").unwrap();
8494 store.put_pack(&pack).unwrap();
8495 store.wait_indexed();
8496 let tip = commits.last().unwrap().clone();
8497 let root = commits.first().unwrap().clone();
8498 let warm = store.reachable(&[&tip], &[]).unwrap();
8499 let warm_delta = store.reachable(&[&tip], &[&root]).unwrap();
8500 assert_eq!(store.reach_source_stats(), (0, 1));
8501 drop(store);
8502 assert!(dir.join("objects.reach").is_file(), "the build persisted objects.reach");
8503 let cold = GitStore::open(&dir, "alice").unwrap();
8504 cold.wait_indexed();
8505 assert_eq!(cold.reachable(&[&tip], &[]).unwrap(), warm);
8506 assert_eq!(cold.reachable(&[&tip], &[&root]).unwrap(), warm_delta);
8507 assert_eq!(cold.reach_source_stats(), (1, 0), "read back, nothing rebuilt");
8508 }
8509
8510 /// **The thin-fetch voucher answers while the indexer holds the gate.**
8511 /// `client_bases` is the closure of what the client already holds, the
8512 /// set a thin delta may name a base from. It was refused with "index in
8513 /// progress" whenever a pack was being absorbed — so a thin fetch that met
8514 /// an index build failed, and gunnar (GF-4, 71023a43) re-emits without the
8515 /// voucher to get past it. The voucher is answerable from the LAST
8516 /// COMMITTED fold: a closure is a fact about immutable commits, and a
8517 /// `have` the fold does not know yet contributes nothing — a smaller
8518 /// voucher, which is the safe direction (fewer thin bases, never a base
8519 /// the client lacks). Nothing waits on the worker and nothing races: the
8520 /// derived tables are swapped under their write lock.
8521 /// RED: `client_bases` with the gate parked is `Err("index in progress")`.
8522 #[test]
8523 fn client_bases_answers_from_the_last_fold_while_the_gate_is_held() {
8524 let dir = tmpdir("client-bases-during-fold");
8525 let (pack_a, commits_a) = linear_history_pack_from(10, None);
8526 let store = GitStore::open(&dir, "alice").unwrap();
8527 store.put_pack(&pack_a).unwrap();
8528 store.wait_indexed();
8529 let tip_a = commits_a.last().unwrap().clone();
8530 let settled = store.client_bases(&[&tip_a]).unwrap();
8531 assert_eq!(settled.held.len(), 30, "10 commits × (commit, tree, blob)");
8532
8533 // Park the drain, then push: the second pack stays pending, as it is
8534 // while the worker is minutes into a big one.
8535 let gate = store.hold_absorb_gate();
8536 let (pack_b, commits_b) = linear_history_pack_from(5, Some(&tip_a));
8537 store.put_pack(&pack_b).unwrap();
8538 let tip_b = commits_b.last().unwrap().clone();
8539 assert_eq!(store.unindexed_packs(), 1, "premise: the push is pending behind the parked gate");
8540
8541 let t = Instant::now();
8542 let during = store
8543 .client_bases(&[&tip_a])
8544 .expect("the voucher must come from the last committed fold, not be refused");
8545 assert!(t.elapsed() < Duration::from_secs(2), "the voucher waited on the parked drain");
8546 assert_eq!(during.held, settled.held, "the last fold's closure of tip_a, exactly");
8547 // A `have` the fold does not know yet is not a refusal either: it
8548 // vouches for nothing, which over-sends and never under-vouches.
8549 let unknown = store.client_bases(&[&tip_b]).expect("an unknown have is an empty voucher, not an error");
8550 assert!(unknown.held.is_empty(), "a commit the fold has not seen vouches for nothing");
8551 // The whole-graph question is still refused by name: a `want` the
8552 // fold does not hold cannot be served from a stale graph.
8553 let err = store.reachable(&[tip_b.as_slice()], &[]).unwrap_err();
8554 assert!(err.to_string().contains("index in progress"), "{err:#}");
8555
8556 drop(gate);
8557 store.wait_indexed();
8558 let after = store.client_bases(&[&tip_b]).unwrap();
8559 assert_eq!(after.held.len(), 45, "once absorbed, tip_b's closure is whole");
8560 }
8561
8562 /// **The persisted table refuses hostile bytes and falls back to a build.**
8563 /// `objects.reach` is bytes off a disk: a torn write, a truncation or a
8564 /// foreign file must come back as "build it again", never a panic.
8565 #[test]
8566 fn a_hostile_persisted_reach_table_is_rebuilt_not_trusted() {
8567 let dir = tmpdir("reach-hostile-persisted");
8568 let (pack, commits) = linear_history_pack_from(12, None);
8569 let store = GitStore::open(&dir, "alice").unwrap();
8570 store.put_pack(&pack).unwrap();
8571 store.wait_indexed();
8572 let tip = commits.last().unwrap().clone();
8573 let warm = store.reachable(&[&tip], &[]).unwrap();
8574 drop(store);
8575 let path = dir.join("objects.reach");
8576 let bytes = std::fs::read(&path).expect("the build persisted objects.reach");
8577 for (name, hostile) in [
8578 ("truncated", bytes[..bytes.len() / 2].to_vec()),
8579 ("garbage", b"RECH\x01not a table at all, not even a header".to_vec()),
8580 ("bitflipped", {
8581 let mut b = bytes.clone();
8582 let at = b.len() - 7;
8583 b[at] ^= 0xff;
8584 b
8585 }),
8586 ] {
8587 std::fs::write(&path, &hostile).unwrap();
8588 let cold = GitStore::open(&dir, "alice").unwrap();
8589 cold.wait_indexed();
8590 assert_eq!(cold.reachable(&[&tip], &[]).unwrap(), warm, "{name}: the answer must not change");
8591 let (loaded, _, _) = cold.reach_stats();
8592 let (_, rebuilt) = cold.reach_source_stats();
8593 assert!(
8594 loaded == 0 || rebuilt == 0,
8595 "{name}: either refused and rebuilt (loaded 0) or decoded exactly (rebuilt 0); got loaded {loaded} rebuilt {rebuilt}"
8596 );
8597 drop(cold);
8598 }
8599 }
8600
8601 /// ★ **A push alone makes the cache give space back at the disk line.**
8602 /// Owner 2026-09-28: the indexing thread should also de-cache.
8603 ///
8604 /// The lazy pool is filled into several segments under a 10 % disk, then
8605 /// the disk reads 85 % and a pack is pushed. No reader touches the store
8606 /// after that, so only the indexer can have started the cleanup.
8607 ///
8608 /// Seen RED by removing `self.exploded.kick()` from `absorb_one_inner`:
8609 /// the pool kept every segment after the push.
8610 #[test]
8611 fn a_push_past_the_disk_line_evicts_the_oldest_cache_without_a_read() {
8612 let dir = tmpdir("push-disk-line");
8613 let store = GitStore::open(&dir, "rickard").unwrap();
8614 let c = store.absorber.exploded.clone();
8615 c.set_global_line(0);
8616 c.set_segment_bytes_for_test(256 << 10);
8617 c.set_disk_reading_for_test(10);
8618 // Incompressible, deep enough to be admitted: the cache must hold it.
8619 let mut x: u32 = 0x9E37_79B9;
8620 let big: Vec<u8> = (0..64 << 10)
8621 .map(|_| {
8622 x = x.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
8623 (x >> 24) as u8
8624 })
8625 .collect();
8626 let cost = crate::resolve::ReadCost {
8627 depth: crate::exploded_cache::CHEAP_DEPTH + 8,
8628 inflated: big.len() as u64,
8629 external_base: false,
8630 };
8631 for i in 0..24u32 {
8632 let mut oid = vec![0u8; 20];
8633 oid[..4].copy_from_slice(&i.to_be_bytes());
8634 c.write_back(&oid, GitObjectKind::Blob, &big, cost).unwrap();
8635 c.maintain().unwrap();
8636 }
8637 let before = c.use_now();
8638 assert!(before.segments >= 4, "precondition: rotated into segments, has {}", before.segments);
8639 assert_eq!(before.evicted_segments, 0, "precondition: nothing evicted under a 10 % disk");
8640
8641 c.set_disk_reading_for_test(85);
8642 let (pack, _) = one_blob_pack(b"pushed after the disk crossed its line\n");
8643 store.put(&pack, &[]).unwrap();
8644 store.wait_indexed();
8645 let deadline = Instant::now() + Duration::from_secs(10);
8646 while c.use_now().evicted_segments == 0 && Instant::now() < deadline {
8647 std::thread::sleep(Duration::from_millis(20));
8648 }
8649 let after = c.use_now();
8650 assert!(after.evicted_segments > 0, "the push did not make the cache give anything back");
8651 assert_eq!(after.segments, 1, "past the line the pool keeps only its live segment");
8652 }
8653}