gunnar_server/storage.rs
1//! **Which storage engine this binary carries, and which one it is running.**
2//!
3//! There is one storage contract in the constellation and it is
4//! [`git_storage_trait::GitOps`] — *the eleven* — which lives in znippy beside
5//! its main implementation. This module is the *only* place in gunnar that names
6//! an implementation of it.
7//!
8//! ```text
9//! git-storage-trait :: GitOps
10//! ┌───────────┴───────────┐
11//! znippy-plugin-git storage-git-gix
12//! (Arrow-IPC archive) (gix-odb / gix-ref)
13//! `engine-znippy` `engine-gix`
14//! └───────────┬───────────┘
15//! Engine::open_repo
16//! ──▶ Arc<OpenRepo>
17//! ```
18//!
19//! # What used to be here, and why it is gone
20//!
21//! A second, older seam: `trait StorageEngine` with `install()`, `installed()`
22//! and a `static ENGINE: OnceLock<&'static dyn StorageEngine>`. It existed
23//! because the engine was chosen by *linking a different crate* — `gunnar-znippy`
24//! — which had to run `install()` before `run()`, and therefore needed a `main`
25//! of its own. That crate declared `[[bin]] gunnar` and `[[bin]] gunnar-server`,
26//! two names the build declaration had already assigned to `gunnar-cli` and to
27//! this package, while being absent from that declaration itself: an undeclared
28//! second writer of two declared artifact names. It is why every matrix-built
29//! binary answered *"this binary carries no storage engine"*.
30//!
31//! The engine is a **feature** now. There is nothing left to install, so there is
32//! no `OnceLock`, no `install`, and no engine-free build — see the
33//! [`compile_error!`] below. `gunnar-znippy` is deleted.
34//!
35//! # Both engines may be compiled in at once
36//!
37//! Cargo features are additive and must be, so this is the correct idiom rather
38//! than a compromise:
39//!
40//! | build | engines linked | selection |
41//! |---|---|---|
42//! | `--features engine-znippy,engine-gix` | both | **runtime**, `GUNNAR_ENGINE`, defaulting to znippy |
43//! | default | znippy only | nothing to select |
44//! | `--no-default-features --features server,engine-gix` | gix only | nothing to select |
45//!
46//! **The rule.** When both are compiled the default is **znippy**, and asking for
47//! an engine that was not compiled in fails loudly, at startup, before anything
48//! is served. It never silently falls back to the other: a server quietly
49//! serving from the wrong engine is the worst possible outcome of this seam, and
50//! it is the one failure a measurement cannot see.
51//!
52//! That rule is kept **by construction** rather than by a check. [`Engine`]'s
53//! variants are themselves `#[cfg]`-gated, so in a gix-only build the value
54//! `Engine::Znippy` does not exist and cannot be constructed by any path,
55//! mistaken or otherwise. [`Engine::from_name`] still *knows* the name, which is
56//! what lets it refuse with "this binary was built without it" rather than with
57//! "unknown engine".
58//!
59//! # znippy's build-time C toolchain, and why this module is where the cost lands
60//!
61//! Cargo reads a path dependency's manifest at **resolve** time whether or not
62//! the feature selecting it is on. Naming `znippy-plugin-git` here therefore
63//! makes a sibling znippy checkout a hard requirement of *every* build of this
64//! workspace. MEASURED 2026-08-10 — see this crate's `Cargo.toml`, which records
65//! the command, the output and the 30 packages it adds to `Cargo.lock`.
66
67use std::io::BufRead;
68use std::path::{Path, PathBuf};
69use std::sync::Arc;
70
71use anyhow::{anyhow, bail, Context, Result};
72use gunnar_store::git_storage_trait::GitServe;
73use gunnar_store::RepoStore;
74
75#[cfg(not(any(feature = "engine-znippy", feature = "engine-gix")))]
76compile_error!(
77 "gunnar-server must be built with at least one storage engine: `engine-znippy` (the \
78 default) or `engine-gix`. A server binary with no engine is not a smaller server — it \
79 is the defect this seam replaced: it built, it shipped, it answered every push with \
80 `this binary carries no storage engine`, and the build matrix could not tell it from \
81 the real one."
82);
83
84#[cfg(feature = "engine-gix")]
85mod gix_engine;
86mod pack_end;
87mod store_cache;
88#[cfg(feature = "engine-znippy")]
89pub mod writer;
90/// **The engine seam.** `znippy_plugin_git::` may be named here and nowhere
91/// else — `xtask`'s `Rule::EngineByName` says so, because this file's own
92/// header claims to be the only place in gunnar that names an implementation.
93///
94/// Both sides of the 2026-08-31 merge had built this module and given it a
95/// different name and a different scope. Master's `archive` moved the SIX names
96/// that broke the rule (`archive_in`/`out`/`out_to`, `parse_explode_budget`,
97/// `human_bytes`); this one moves the whole adapter — `store_root`, `shutdown`,
98/// `cancel_index`, `index_progress`, `explode_use`, `set_explode_budget`,
99/// `drop_explode_cache`, `open`, `ingest`, `maintain` — and is the answer that
100/// makes the next engine a file beside `gix_engine.rs` rather than a `#[cfg]`
101/// in `admin.rs`. So the wider one wins and `archive.rs` retires into it, with
102/// master's type discipline carried across: the archive receipts are the
103/// plugin's, re-exported, never re-declared.
104///
105/// `pub` because `admin.rs` and `control_plane.rs` have to reach it: a seam
106/// nothing outside can reach is a rule nobody can keep.
107#[cfg(feature = "engine-znippy")]
108pub mod znippy_engine;
109
110/// **An encrypted repository's push, checked before a byte of it is kept**
111/// (crypt-level2 item 4; `crate::encrypted_quarantine` re-exports it). Here,
112/// not beside `transport.rs`, because it walks the pushed pack with the znippy
113/// engine's own reader — and the engine is named in this seam and nowhere else.
114#[cfg(feature = "engine-znippy")]
115pub mod encrypted_quarantine;
116
117/// The writer arm the znippy engine built its stores on (`uring` / `safe`),
118/// for the dashboard's disk cell; `None` before the first store, or on an
119/// engine that has no arms.
120pub fn writer_arm() -> Option<&'static str> {
121 #[cfg(feature = "engine-znippy")]
122 {
123 writer::selected().map(|s| s.arm.as_str())
124 }
125 #[cfg(not(feature = "engine-znippy"))]
126 {
127 None
128 }
129}
130
131/// **Release every store this process cached, on the way out.**
132///
133/// Engine-neutral, and deliberately not a method on [`Engine`]: it is not "shut
134/// down the engine this run selected", it is "let go of everything anything
135/// opened", and a build carrying both engines has two caches whichever one the
136/// request used.
137///
138/// The znippy arm is where this earns its keep — see
139/// [`znippy_engine::shutdown`] for the measurement and for why the cache itself
140/// stays. It is idempotent, and safe to call in a process that opened no
141/// repository at all: the `OnceLock` behind the cache is initialised on demand
142/// and an empty map clears to an empty map.
143///
144/// The gix arm now has something to release too. It has no database to be told
145/// it was closed, but since 2026-08-11 it caches a `GixStore` per store root for
146/// correctness — one handle so that a push refreshes the view the reader serves
147/// from ([`store_cache`]) — and those hold mmaps over every `.pack` and `.idx`
148/// they have touched. The line the old comment said would be needed "later" is
149/// the second one below.
150pub fn shutdown() {
151 #[cfg(feature = "engine-znippy")]
152 znippy_engine::shutdown();
153 #[cfg(feature = "engine-gix")]
154 gix_engine::shutdown();
155}
156
157/// The environment variable that picks an engine in a build carrying both.
158///
159/// An environment variable and not a flag, for now, because the flag would live
160/// on [`gunnar_common::cli`]'s `ServeArgs` — a crate this change does not own —
161/// and every verb that opens a repository needs the choice, not just `serve`.
162/// The variable is read exactly once, in [`Engine::selected`], on the startup
163/// path.
164pub const ENGINE_ENV: &str = "GUNNAR_ENGINE";
165
166/// **A storage engine this binary actually carries.**
167///
168/// The variants are `#[cfg]`-gated on purpose: a value of this type is proof
169/// that the engine it names is linked. There is no "selected but absent" state
170/// to guard against, because it cannot be represented.
171#[derive(Debug, Clone, Copy, PartialEq, Eq)]
172pub enum Engine {
173 /// `znippy-plugin-git` — Arrow-IPC in a znippy archive.
174 #[cfg(feature = "engine-znippy")]
175 Znippy,
176 /// `storage-git-gix` — `gix-odb` / `gix-ref` on a plain filesystem.
177 #[cfg(feature = "engine-gix")]
178 Gix,
179}
180
181/// The engine a build carrying znippy runs unless told otherwise.
182#[cfg(feature = "engine-znippy")]
183pub const DEFAULT_ENGINE: Engine = Engine::Znippy;
184
185/// With no znippy there is exactly one engine left, and it is the default by
186/// arithmetic rather than by preference. The `compile_error!` above is what
187/// makes this exhaustive.
188#[cfg(all(not(feature = "engine-znippy"), feature = "engine-gix"))]
189pub const DEFAULT_ENGINE: Engine = Engine::Gix;
190
191impl std::fmt::Display for Engine {
192 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
193 f.write_str(self.name())
194 }
195}
196
197/// **Two resolver workers on a twin.** MEASURED 2026-09-15
198/// (`tests/twin_snapshot_memory.rs`): one worker per core, each inflating a whole blob, grew
199/// RssAnon +133 MiB for a 512 MiB snapshot install even with explode and graph off. Handed to
200/// `znippy_plugin_git::set_resolve_workers_default` by [`Engine::hold_twin_memory_bound`].
201pub const TWIN_RESOLVE_WORKERS: usize = 2;
202/// **16 MiB of resolver lookahead on a twin** (same measurement) — the bytes the workers may
203/// run ahead of the ordered output by. znippy's own default is 32 MiB.
204pub const TWIN_RESOLVE_LOOKAHEAD_BYTES: u64 = 16 << 20;
205/// **16 MiB explode budget on a twin** (same measurement). With the policy `off` nothing is
206/// stored at all; the cap is what keeps a later policy change bounded.
207pub const TWIN_EXPLODE_BUDGET_BYTES: u64 = 16 << 20;
208
209/// **Where the nordisk repositories live** — the ONE place an engine's source
210/// URL is spelled: the nordisk organisation on git.gunnar.rs, the repositories'
211/// home. A link only (`Admin.StorageInfo`'s `repo_url`); nothing clones it.
212pub const NORDISK_REPOS: &str = "https://git.gunnar.rs/nordisk";
213
214impl Engine {
215 /// Every engine compiled into this binary, in preference order.
216 pub const COMPILED: &'static [Engine] = &[
217 #[cfg(feature = "engine-znippy")]
218 Engine::Znippy,
219 #[cfg(feature = "engine-gix")]
220 Engine::Gix,
221 ];
222
223 /// The name this engine is selected by and reported under.
224 pub fn name(self) -> &'static str {
225 match self {
226 #[cfg(feature = "engine-znippy")]
227 Engine::Znippy => "znippy",
228 #[cfg(feature = "engine-gix")]
229 Engine::Gix => "gix",
230 }
231 }
232
233 /// **Can replication read a repository this engine stores?**
234 ///
235 /// It reads one by opening it as a *filesystem git repository* — `gix::open`
236 /// in [`crate::mirror`], then `gunnar-client`'s send-pack, which is proven
237 /// against the real `git receive-pack` and must not be twinned. An engine
238 /// that writes no `.git` layout cannot be read that way, and the failure
239 /// lands inside the retry loop where nobody sees it: the server accepts
240 /// every push and the standby stays empty.
241 ///
242 /// So it is asked at start-up instead, and a server that cannot do what it
243 /// was configured to do refuses to come up. Closing the gap means a
244 /// server-side send-pack over `GitServe` — the store already emits packs
245 /// for `upload-pack` — at which point this predicate goes away rather than
246 /// gaining an arm.
247 pub fn replication_can_read_repositories(self) -> bool {
248 match self {
249 #[cfg(feature = "engine-znippy")]
250 Engine::Znippy => false,
251 #[cfg(feature = "engine-gix")]
252 Engine::Gix => true,
253 }
254 }
255
256 /// **The engine's crate**, as it is named on the Storage page and linked.
257 pub fn plugin(self) -> &'static str {
258 match self {
259 #[cfg(feature = "engine-znippy")]
260 Engine::Znippy => "znippy-plugin-git",
261 #[cfg(feature = "engine-gix")]
262 Engine::Gix => "storage-git-gix",
263 }
264 }
265
266 /// **What it stores and how**, in one line — the Storage page's tooltip.
267 pub fn summary(self) -> &'static str {
268 match self {
269 #[cfg(feature = "engine-znippy")]
270 Engine::Znippy => "Arrow-IPC archive; objects + refs",
271 #[cfg(feature = "engine-gix")]
272 Engine::Gix => "gix-odb packs + gix-ref (gitoxide); objects + refs",
273 }
274 }
275
276 /// **Where the engine's source lives**, off [`NORDISK_REPOS`]: znippy's
277 /// own repository; `storage-git-gix` is a crate of edda's (built on
278 /// gitoxide's `gix-odb` / `gix-ref`).
279 pub fn repo_url(self) -> String {
280 match self {
281 #[cfg(feature = "engine-znippy")]
282 Engine::Znippy => format!("{NORDISK_REPOS}/znippy"),
283 #[cfg(feature = "engine-gix")]
284 Engine::Gix => format!("{NORDISK_REPOS}/edda"),
285 }
286 }
287
288 /// **The engine, in parts** — name, one-line description, source
289 /// repository — for `Admin.StorageInfo`'s `engine` (2026-10-02).
290 pub fn report(self) -> gunnar_proto::StorageEngine {
291 gunnar_proto::StorageEngine {
292 name: self.plugin().to_owned(),
293 description: self.summary().to_owned(),
294 repo_url: self.repo_url(),
295 }
296 }
297
298 /// What answers, in one line, for an operator reading `storage info`:
299 /// `znippy-plugin-git (Arrow-IPC archive; objects + refs)`, made of the
300 /// same parts [`Self::report`] sends apart.
301 pub fn describe(self) -> String {
302 format!("{} ({})", self.plugin(), self.summary())
303 }
304
305 /// The engines this binary carries, as a human-readable list.
306 pub fn compiled_list() -> String {
307 Self::COMPILED
308 .iter()
309 .map(|e| e.name())
310 .collect::<Vec<_>>()
311 .join(", ")
312 }
313
314 /// **Resolve a name to an engine, or refuse.**
315 ///
316 /// Three outcomes, and the middle one is the whole point:
317 ///
318 /// * a name this build carries → that engine;
319 /// * a name this build *knows* but did not compile in → refused, saying so;
320 /// * anything else → refused as unknown.
321 ///
322 /// Never a fallback. See the module docs.
323 pub fn from_name(name: &str) -> Result<Engine> {
324 match name.trim().to_ascii_lowercase().as_str() {
325 #[cfg(feature = "engine-znippy")]
326 "znippy" => Ok(Engine::Znippy),
327 #[cfg(not(feature = "engine-znippy"))]
328 "znippy" => Err(not_compiled_in("znippy")),
329
330 #[cfg(feature = "engine-gix")]
331 "gix" => Ok(Engine::Gix),
332 #[cfg(not(feature = "engine-gix"))]
333 "gix" => Err(not_compiled_in("gix")),
334
335 other => Err(anyhow!(
336 "unknown storage engine {other:?}; the engine names are `znippy` and `gix`, and \
337 this binary carries {}",
338 Self::compiled_list()
339 )),
340 }
341 }
342
343 /// **The engine this process runs**, decided once, on the startup path.
344 ///
345 /// [`ENGINE_ENV`] if it is set and non-empty, [`DEFAULT_ENGINE`] otherwise.
346 /// An unset variable is *not* "no engine": it is the default, which is
347 /// znippy wherever znippy is compiled in.
348 pub fn selected() -> Result<Engine> {
349 match std::env::var(ENGINE_ENV) {
350 Ok(raw) if !raw.trim().is_empty() => Self::from_name(&raw).with_context(|| {
351 format!("{ENGINE_ENV}={raw:?} selects the storage engine for this process")
352 }),
353 _ => Ok(DEFAULT_ENGINE),
354 }
355 }
356
357 /// Open — creating if absent — the repository at `git_dir`.
358 ///
359 /// Unconditional. "It holds nothing yet" is the normal state of a
360 /// repository's first push, not a reason to answer with something else:
361 /// gating on what a repository already holds is what refuses every first
362 /// push, which is a mistake this seam has already made once.
363 pub fn open_repo(self, git_dir: &Path, hash_kind: gix_hash::Kind) -> Result<Arc<OpenRepo>> {
364 match self {
365 #[cfg(feature = "engine-znippy")]
366 Engine::Znippy => znippy_engine::open(git_dir, hash_kind),
367 #[cfg(feature = "engine-gix")]
368 Engine::Gix => gix_engine::open(git_dir, hash_kind),
369 }
370 }
371
372 /// Stop any index work on the repository at `git_dir` — the engine half
373 /// of `repo delete`. A no-op for an engine without a background indexer.
374 pub fn cancel_index(self, git_dir: &Path) {
375 match self {
376 #[cfg(feature = "engine-znippy")]
377 Engine::Znippy => znippy_engine::cancel_index(git_dir),
378 #[cfg(feature = "engine-gix")]
379 Engine::Gix => {
380 let _ = git_dir;
381 }
382 }
383 }
384
385 /// Index work in flight across every open repository, as
386 /// `(git_dir, snapshot)`. Empty for an engine without a background indexer.
387 pub fn index_progress(self) -> Vec<(PathBuf, IndexSnapshot)> {
388 match self {
389 #[cfg(feature = "engine-znippy")]
390 Engine::Znippy => znippy_engine::index_progress()
391 .into_iter()
392 .map(|(dir, s)| {
393 (
394 dir,
395 IndexSnapshot {
396 active: s.active,
397 resumed: s.resumed,
398 pack_id: s.pack_id,
399 objects_done: s.objects_done,
400 objects_total: s.objects_total,
401 bytes_done: s.bytes_done,
402 bytes_total: s.bytes_total,
403 phase: s.phase.as_str(),
404 started_at_ms: s.started_at_ms,
405 reinflations: s.reinflations,
406 pauses: s.pauses,
407 },
408 )
409 })
410 .collect(),
411 #[cfg(feature = "engine-gix")]
412 Engine::Gix => Vec::new(),
413 }
414 }
415
416 /// **Every open repository's explode band**, as `(git_dir, band)`.
417 ///
418 /// The exploded tier is a read cache with a byte budget, not storage: a
419 /// blob is written into it by the first reader that inflates it and
420 /// unlinked from it when the budget says so. Both numbers are here because
421 /// an operator asked to see a repository at its limit — an account at its
422 /// budget still serves every object, only slower, and nothing else on the
423 /// dashboard would say which repositories those are.
424 ///
425 /// A `metadata` call per pool, no store lock. Empty for an engine with no
426 /// such tier.
427 pub fn explode_use(self) -> Vec<(PathBuf, ExplodeBand)> {
428 match self {
429 #[cfg(feature = "engine-znippy")]
430 Engine::Znippy => znippy_engine::explode_use(),
431 #[cfg(feature = "engine-gix")]
432 Engine::Gix => Vec::new(),
433 }
434 }
435
436 /// **Indexed objects of every open store**, as `(git_dir, count)` — the
437 /// `objects` meter of `Usage.Read`, i.e. what this process is paying RAM
438 /// for (~382 B an object, measured). A closed store is absent, not zero.
439 ///
440 /// Empty on the gix engine: it keeps no in-RAM object index per open
441 /// store (its odb is mmapped pack indexes), so there is no per-object RAM
442 /// price to report and reporting one would be a made-up bill.
443 pub fn open_object_counts(self) -> Vec<(PathBuf, u64)> {
444 match self {
445 #[cfg(feature = "engine-znippy")]
446 Engine::Znippy => znippy_engine::open_object_counts(),
447 #[cfg(feature = "engine-gix")]
448 Engine::Gix => Vec::new(),
449 }
450 }
451
452 /// **Set the explode budget of every open repository under `git_dir`.**
453 ///
454 /// A prefix, so one call moves a whole account. Returns how many stores
455 /// were changed — applied output, so a control plane that reached nothing
456 /// (every repository closed) can say so instead of reporting success.
457 pub fn set_explode_budget(self, git_dir: &Path, bytes: u64) -> usize {
458 match self {
459 #[cfg(feature = "engine-znippy")]
460 Engine::Znippy => znippy_engine::set_explode_budget(git_dir, bytes),
461 #[cfg(feature = "engine-gix")]
462 Engine::Gix => {
463 let _ = (git_dir, bytes);
464 0
465 }
466 }
467 }
468
469 /// **A twin keeps no derived memo** (`.nornir/twin-transaction-log.md` C2): the znippy
470 /// engine's persisted commit graph (`objects.graph`) is off for every store this process
471 /// opens from now on. Process-wide, because the box's role is — called when stores open as
472 /// a twin and when a fresh box becomes one. The gix engine keeps no such file.
473 pub fn hold_no_graph_memo(self) {
474 match self {
475 #[cfg(feature = "engine-znippy")]
476 Engine::Znippy => znippy_engine::set_graph_cache_default(false),
477 #[cfg(feature = "engine-gix")]
478 Engine::Gix => {}
479 }
480 }
481
482 /// **A twin holds no read cache and resolves narrow** (`twin-memory-bound`, and lane T3's
483 /// one-medium run): the znippy engine's explode policy is `off`, its explode budget is
484 /// [`TWIN_EXPLODE_BUDGET_BYTES`], and the indexer's resolver runs [`TWIN_RESOLVE_WORKERS`]
485 /// wide with [`TWIN_RESOLVE_LOOKAHEAD_BYTES`] of lookahead, for every store this
486 /// process opens from now on. MEASURED: +36 MiB of RssAnon for a 512 MiB snapshot install
487 /// against a 64 MiB ceiling, against +149 MiB unbounded — and, with the policy left alone,
488 /// five `objects.exploded` files on a twin's volume that a twin may not hold.
489 ///
490 /// Process-wide, because the box's role is. It is a SIBLING of
491 /// [`Engine::hold_no_graph_memo`] and not folded into it, because the two answer different
492 /// files (`objects.graph` against `objects.exploded` and the resolver's heap) and a reader
493 /// who assumed the graph door covered the cache is exactly how the twin came to hold five
494 /// of them. The gix engine has neither.
495 pub fn hold_twin_memory_bound(self) {
496 match self {
497 #[cfg(feature = "engine-znippy")]
498 Engine::Znippy => znippy_engine::hold_twin_memory_bound(),
499 #[cfg(feature = "engine-gix")]
500 Engine::Gix => {}
501 }
502 }
503
504 /// **And the stores already open when that happened** — their exploded pools unlinked and
505 /// capped, since a store keeps the policy it opened with. Returns the bytes unlinked.
506 pub fn bound_open_stores_to_twin(self) -> u64 {
507 match self {
508 #[cfg(feature = "engine-znippy")]
509 Engine::Znippy => znippy_engine::bound_open_stores_to_twin(),
510 #[cfg(feature = "engine-gix")]
511 Engine::Gix => 0,
512 }
513 }
514
515 /// **Hold this store's indexer to a memory reading other than `/proc/meminfo`**
516 /// (IMPORT-2): `Some((file, floor))` makes the indexer's guard read `file` (the
517 /// cgroup's memory in meminfo shape, [`crate::import_room::cgroup_meminfo`]) and pause
518 /// under `floor` bytes; `None` gives it back the process-wide guard. `false` when the
519 /// store is not open or the engine has no such guard.
520 pub fn hold_index_memory(self, git_dir: &Path, meminfo: Option<(&Path, u64)>) -> bool {
521 match self {
522 #[cfg(feature = "engine-znippy")]
523 Engine::Znippy => znippy_engine::hold_index_memory(git_dir, meminfo),
524 #[cfg(feature = "engine-gix")]
525 Engine::Gix => {
526 let _ = (git_dir, meminfo);
527 false
528 }
529 }
530 }
531
532 /// **`repo gc` reclaims the read cache**: every lazily-exploded object
533 /// unlinked, the commit-and-tree pool kept because that is the graph.
534 /// Returns the bytes that left the filesystem.
535 pub fn drop_explode_cache(self, git_dir: &Path) -> u64 {
536 match self {
537 #[cfg(feature = "engine-znippy")]
538 Engine::Znippy => znippy_engine::drop_explode_cache(git_dir),
539 #[cfg(feature = "engine-gix")]
540 Engine::Gix => {
541 let _ = git_dir;
542 0
543 }
544 }
545 }
546
547 /// **Take one pushed packfile straight into storage, off the wire.**
548 ///
549 /// The bytes go from the transport into the engine and are durable when this
550 /// returns. Nothing is staged through a packfile on the way: a seam spelled
551 /// in paths cannot express a server without them, which is why this one is
552 /// spelled in bytes.
553 ///
554 /// Everything but `want_objects` is engine-neutral and lives in
555 /// [`ingest_common`] — the pack header, the trailer and
556 /// [`GitOps::put_pack`](git_storage_trait::GitOps::put_pack) are the same
557 /// three facts whichever engine stored them. Enumerating the ids a push
558 /// introduced is **not** neutral: a pack entry carries no oid, so an engine
559 /// answers it out of its own index or not at all.
560 ///
561 /// # `progress`, and why it is threaded rather than fetched
562 ///
563 /// `Some(sink)` asks the engine to say how many objects of this pack it has
564 /// taken in **while it is still taking them in**
565 /// ([`GitOps::put_pack_observed`](git_storage_trait::GitOps::put_pack_observed)).
566 /// The count belongs to the code that splits the pack and to nothing else —
567 /// deriving it anywhere on the way here means inflating every entry a second
568 /// time, which is a second pack decoder (one implementation) — so the only thing this
569 /// layer can do is carry the sink down and let the walk publish.
570 ///
571 /// # `before_store`, and why the pushed pack is framed HERE
572 ///
573 /// Called with exactly the bytes the engine is about to store, before it stores them — the
574 /// transaction log's `Pack` frame (`gunnar_core::txlog`, the twin). This path opens the
575 /// engine by `git_dir` and never passes the `OpenRepo::logged` tee, so without it a pushed
576 /// pack was never a frame while its ref update was: MEASURED 2026-09-15 under KVM, the twin
577 /// and a restored primary both refused `refs/heads/main would point at …, which this
578 /// repository does not have`. An error from the hook refuses the push before a byte lands.
579 ///
580 /// `None` is not a degraded mode: a push over SSH or smart HTTP has no job
581 /// record and nobody polling one, so it asks for nothing and pays for
582 /// nothing. The mirror queue is the caller that has a console watching.
583 pub fn ingest_push(
584 self,
585 git_dir: &Path,
586 hash_kind: gix_hash::Kind,
587 pack: &mut dyn BufRead,
588 objects: ObjectSet,
589 progress: Option<&dyn git_storage_trait::PackProgress>,
590 before_store: Option<BeforeStore<'_>>,
591 ) -> Result<PushReceipt> {
592 match self {
593 #[cfg(feature = "engine-znippy")]
594 Engine::Znippy => {
595 znippy_engine::ingest(git_dir, hash_kind, pack, objects, progress, before_store)
596 }
597 #[cfg(feature = "engine-gix")]
598 Engine::Gix => {
599 gix_engine::ingest(git_dir, hash_kind, pack, objects, progress, before_store)
600 }
601 }
602 }
603
604 /// `gunnar gc` for one repository: seal, then reclaim.
605 ///
606 /// The order is the correctness argument rather than a preference. Sealing
607 /// folds pending index work into the derived tables; reclaiming computes its
608 /// live set out of those tables. Reclaiming first would judge liveness
609 /// against a graph missing everything pushed since the last drain, and
610 /// tombstone objects reachable only from those commits.
611 ///
612 /// Only the znippy arm has a seal. The gix arm's `gc` reads the packs and
613 /// the ref store directly, so there is no pending index work for a seal to
614 /// fold in, and it reports `sections_sealed: 0` — which is the truth and not
615 /// a stub.
616 pub fn maintain(self, git_dir: &Path, hash_kind: gix_hash::Kind) -> Result<MaintenanceReport> {
617 match self {
618 #[cfg(feature = "engine-znippy")]
619 Engine::Znippy => znippy_engine::maintain(git_dir, hash_kind),
620 #[cfg(feature = "engine-gix")]
621 Engine::Gix => gix_engine::maintain(git_dir, hash_kind),
622 }
623 }
624}
625
626/// The refusal a build that does not carry `name` gives when asked for it.
627///
628/// Written out in full rather than as "unsupported", because the operator's next
629/// action is a rebuild and this is where they find out which flag.
630#[allow(dead_code)]
631fn not_compiled_in(name: &str) -> anyhow::Error {
632 anyhow!(
633 "this binary was built without the `{name}` storage engine; it carries {}. Rebuild with \
634 `--features engine-{name}`, or select an engine that is here. Refusing rather than \
635 serving from a different engine: a server that quietly answers out of the wrong store \
636 is worse than one that will not start.",
637 Engine::compiled_list()
638 )
639}
640
641/// **One repository, opened on whichever engine this process carries.**
642///
643/// The value [`Engine::open_repo`] hands back, and the *only* implementor of
644/// [`RepoStore`] in gunnar. It is a **value**, not a seam: nothing in it can be
645/// implemented by a backend and no engine can behave differently through it —
646/// the engine-specific part is the one line in each adapter that constructs it.
647/// It replaced two near-identical `RepoStore` impls (`ZnippyRepo`, `GixRepo`)
648/// that differed only in which concrete store they held.
649///
650/// # Why it exists at all
651///
652/// `gunnar-server` is a *serving* process, and everything it does with a
653/// repository needs the **reading** contract:
654/// [`UploadPack::new`](gunnar_wire::UploadPack::new) and
655/// [`ReceivePack::new`](gunnar_wire::ReceivePack::new) take `&dyn GitServe`,
656/// `gunnar-policy`'s ancestry walk and signature gate take `&dyn GitServe`, and
657/// `HEAD` is reachable only through [`GitServe::head`] / [`GitServe::set_head`]
658/// because the `GitServe` HEAD ruling settled that it is not a row in `GitOps::refs`.
659///
660/// # ONE handle, since [`RepoStore::git`] widened — 2026-08-10
661///
662/// This struct carried **two** fat pointers to the same store, `ops` and
663/// `serve`, for one day: `RepoStore::git()` returned `Arc<dyn GitOps>`, the
664/// hand-off from the reading half to the storage half is one-way, and `dyn` has
665/// no downcast — so a server that had only the narrow handle could not serve a
666/// clone from it. The fix was one line in `gunnar-store`:
667/// `git()` hands out `Arc<dyn GitServe>`, which is still one method and still
668/// One repository's index work, engine-neutral. See
669/// `znippy_plugin_git::progress::Snapshot` for the source of every field.
670#[derive(Debug, Clone, PartialEq, Eq)]
671pub struct IndexSnapshot {
672 pub active: bool,
673 pub resumed: bool,
674 pub pack_id: u64,
675 pub objects_done: u64,
676 pub objects_total: u64,
677 pub bytes_done: u64,
678 pub bytes_total: u64,
679 pub phase: &'static str,
680 pub started_at_ms: u64,
681 pub reinflations: u64,
682 pub pauses: u64,
683}
684
685/// One repository's exploded read cache: what it holds, what it may hold, and
686/// what has already been evicted from it. Engine-neutral by construction —
687/// `znippy_plugin_git::exploded_cache::CacheUse` copied field for field, so a
688/// dashboard cell cannot tell which engine answered.
689#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
690pub struct ExplodeBand {
691 /// The pool the indexer writes at push time (commits and trees). Never
692 /// evicted: it is what `graph`, `reach` and a tag peel read.
693 pub eager_bytes: u64,
694 /// The pool first reads fill and the budget empties.
695 pub lazy_bytes: u64,
696 /// The cap. `0` is unlimited.
697 pub budget: u64,
698 pub segments: u64,
699 pub evicted_segments: u64,
700 /// Bytes that actually left the filesystem, measured before the unlink.
701 pub evicted_bytes: u64,
702 pub written_back: u64,
703}
704
705impl ExplodeBand {
706 pub fn used(&self) -> u64 {
707 self.eager_bytes + self.lazy_bytes
708 }
709 /// True while the repository is at or over its quota — the row an operator
710 /// wants to find. Not an error state: it reads correctly and more slowly.
711 pub fn at_limit(&self) -> bool {
712 self.budget != 0 && self.used() >= self.budget
713 }
714}
715
716impl IndexSnapshot {
717 pub fn percent(&self) -> u64 {
718 (self.objects_done * 100)
719 .checked_div(self.objects_total)
720 .map_or(0, |percent| percent.min(100))
721 }
722}
723
724/// one authority because `GitServe` **is** `GitOps`.
725///
726/// So the second field is gone, and with it the paragraph that explained why
727/// two fields were not two handles and why they could not be one upcast (trait
728/// upcasting stabilised in Rust 1.86 and this workspace declares `1.85`). There
729/// is one `Arc`, one refcount cell, and one view of one transaction.
730pub struct OpenRepo {
731 git_dir: PathBuf,
732 hash_kind: gix_hash::Kind,
733 serve: Arc<dyn GitServe>,
734 /// The engine's store UNDER the transaction-log tee, when `serve` is the
735 /// tee. The twin applies the primary's frames through this one, so its
736 /// own log holds the primary's numbers and never a second copy.
737 raw: Option<Arc<dyn GitServe>>,
738}
739
740impl OpenRepo {
741 /// Wrap one engine's concrete store.
742 ///
743 /// `S: GitServe` is the requirement that keeps an engine honest: a backend
744 /// that implements only the storage half cannot be opened by this server,
745 /// and the refusal is a compile error in the engine's adapter rather than a
746 /// repository that opens and then cannot answer a clone.
747 pub(crate) fn new<S: GitServe + 'static>(
748 git_dir: &Path,
749 hash_kind: gix_hash::Kind,
750 store: Arc<S>,
751 ) -> Arc<Self> {
752 Arc::new(OpenRepo {
753 git_dir: git_dir.to_path_buf(),
754 hash_kind,
755 serve: store as Arc<dyn GitServe>,
756 raw: None,
757 })
758 }
759
760 /// **The reading contract**: inflated objects, headers, `HEAD`, `emit_pack`,
761 /// `select` — and, by the `GitServe: GitOps` supertrait, the eleven as well.
762 ///
763 /// This is what the serving path holds. It blocks: every caller below is
764 /// already inside `spawn_blocking` or on a control-plane worker, and
765 /// The two-tier rule keeps the serving tier serial — request concurrency, no
766 /// intra-request fan-out.
767 pub fn serve(&self) -> &dyn GitServe {
768 self.serve.as_ref()
769 }
770
771 /// **The same store, behind the transaction log.** Every write through
772 /// the returned handle is a frame in `log` before the engine sees it
773 /// (`gunnar_core::txlog::tee::LoggedGit`); every read is the engine's.
774 /// `repo` is the catalog key and `hash` its hash kind, as the frame
775 /// records them for the twin.
776 pub(crate) fn logged(
777 self: &Arc<Self>,
778 log: Arc<gunnar_core::txlog::Log>,
779 repo: &str,
780 hash: &str,
781 ) -> Arc<Self> {
782 let tee = gunnar_core::txlog::tee::LoggedGit::new(Arc::clone(&self.serve), log, repo, hash);
783 Arc::new(OpenRepo {
784 git_dir: self.git_dir.clone(),
785 hash_kind: self.hash_kind,
786 serve: Arc::new(tee) as Arc<dyn GitServe>,
787 raw: Some(Arc::clone(&self.serve)),
788 })
789 }
790
791 /// **The store with no log in front of it.** The twin's apply path and
792 /// nothing else: a write here is invisible to this box's transaction log.
793 pub fn unlogged(&self) -> &dyn GitServe {
794 self.raw.as_deref().unwrap_or(self.serve.as_ref())
795 }
796
797 /// Where this repository lives. The inherent twin of
798 /// [`RepoStore::git_dir`], so callers that only want a path do not have to
799 /// have the trait in scope.
800 pub fn git_dir(&self) -> &Path {
801 &self.git_dir
802 }
803
804 /// This repository's hash algorithm — asked, never defaulted. See
805 /// [`RepoStore::hash_kind`].
806 pub fn hash_kind(&self) -> gix_hash::Kind {
807 self.hash_kind
808 }
809}
810
811impl RepoStore for OpenRepo {
812 fn git_dir(&self) -> &Path {
813 &self.git_dir
814 }
815
816 fn hash_kind(&self) -> gix_hash::Kind {
817 self.hash_kind
818 }
819
820 fn git(&self) -> Arc<dyn GitServe> {
821 Arc::clone(&self.serve)
822 }
823}
824
825/// **How badly the caller needs the object set a push introduced.**
826///
827/// A `bool` said only *"enumerate it"*, and that conflated two callers with
828/// very different stakes:
829///
830/// * a **policy gate** over the pushed objects cannot decide without the set,
831/// so an engine that cannot produce one must refuse the push — passing it on
832/// an answer the engine never gave is the silent-yes this whole seam exists
833/// to prevent;
834/// * the **push attestation** wants it and can record its absence
835/// (`new_object_oids_unknown`), so an engine that cannot produce one must still
836/// serve the push.
837///
838/// With one flag, teaching the attestation to ask made every push refuse on the
839/// `gix` arm, which cannot enumerate at all. Two answers, one type.
840#[derive(Debug, Clone, Copy, PartialEq, Eq)]
841pub enum ObjectSet {
842 /// A gate needs it. An engine that cannot answer must return an error.
843 Required,
844 /// Wanted for the record. An engine that cannot answer returns `None` and
845 /// the push is served.
846 Wanted,
847 /// Nobody asked. Do not pay for it.
848 NotNeeded,
849}
850
851/// The hook [`Engine::ingest_push`] runs on the pack before a byte of it is
852/// stored; an error refuses the push.
853pub type BeforeStore<'a> = &'a dyn Fn(&[u8]) -> Result<()>;
854
855/// What one push stored, read back off what was written.
856///
857/// Nothing here is restated from the request: the object count and version come
858/// off the stored pack's own 12-byte header, and `pack_id` is its trailing
859/// checksum.
860#[derive(Debug, Clone, PartialEq, Eq, Default)]
861pub struct PushReceipt {
862 /// From the pack's own header.
863 pub num_objects: u32,
864 /// The pack format version, likewise.
865 pub version: u32,
866 /// The pack's trailing checksum, which is its identity.
867 pub pack_id: Vec<u8>,
868 /// Bytes stored, which for a verbatim store is bytes received.
869 pub pack_bytes: u64,
870 /// The object ids this push introduced, when the caller asked for them.
871 ///
872 /// `None` is **not asked for**, never "none were introduced" — a gate handed
873 /// an empty set would pass, and one handed `None` refuses.
874 pub new_oids: Option<Vec<Vec<u8>>>,
875 /// Wall time of the ingest.
876 pub elapsed_ms: u64,
877}
878
879/// What one maintenance run did. Applied outputs, read off the result.
880#[derive(Debug, Clone, PartialEq, Eq, Default)]
881pub struct MaintenanceReport {
882 /// Which strategy ran, by its own name.
883 pub strategy: String,
884 /// Size before and after — the pair, because a "reclaimed" figure alone
885 /// cannot tell a run that compacted from one that did nothing.
886 pub bytes_before: u64,
887 pub bytes_after: u64,
888 /// Live index rows carried across. Maintenance never changes this, so a run
889 /// that did is a bug and this is how it becomes visible.
890 pub rows: u64,
891 /// Packs whose every object was dead, and whose rows were tombstoned.
892 pub retired_packs: u64,
893 /// Whether the result was read back and checked. `false` is not failure —
894 /// an in-place compaction verifies after it has already committed.
895 pub verified: bool,
896 /// Reserved sections the seal produced. Always `0` on an engine with no
897 /// seal; see [`Engine::maintain`].
898 pub sections_sealed: usize,
899}
900
901// ── the engine-neutral half of a push ────────────────────────────────────────
902
903/// One pushed pack, read off the wire, with the three facts its own bytes carry.
904///
905/// [`Debug`] is written out rather than derived: `bytes` is a whole packfile,
906/// and a derive would put tens of megabytes of it into any `expect_err` message
907/// or `tracing` line that ever touched one.
908pub(crate) struct PushedPack {
909 pub bytes: PackBytes,
910 /// From the pack's own 12-byte header, so nothing derived can disagree.
911 pub num_objects: u32,
912 pub version: u32,
913}
914
915/// Packs at or above this many bytes are spooled to disk while they arrive.
916pub(crate) const SPOOL_ABOVE: usize = 64 << 20;
917
918/// **How many of the first `consumed` bytes of a pushed pack are in its spool
919/// file, on the disk** — the rule [`read_pushed_pack_spooling_at`] follows,
920/// stated once for the space ledger (`crate::push_space::Gated`), which gives a
921/// spooled byte's promise back the moment it is written: below the threshold
922/// the pack is on the heap and nothing is on the disk; the moment it reaches
923/// the threshold everything consumed so far is written out, and from then on
924/// each chunk is written BEFORE it is consumed.
925/// `the_spool_holds_what_spooled_on_disk_says_at_every_consume` holds the two
926/// to each other.
927pub(crate) fn spooled_on_disk(consumed: u64, spool_above: u64) -> u64 {
928 if consumed >= spool_above {
929 consumed
930 } else {
931 0
932 }
933}
934
935/// The pushed bytes: on the heap for an ordinary push, **on disk and mapped**
936/// for a big one.
937///
938/// MEASURED on the KVM rig 2026-08-30: the linux mirror push (8.12
939/// GiB) killed an 8 GB and a 12 GB appliance *while the client was still
940/// sending* — `Kernel panic: System is deadlocked on memory` with no
941/// `store.znippy.ingested` line — because the whole pack was one `Vec<u8>`
942/// grown by `extend_from_slice` (a 16 GiB capacity step past 8 GiB). Above
943/// [`SPOOL_ABOVE`] the bytes now go to a temp file beside the store as they
944/// arrive and come back as a read-only mapping: page cache the kernel can
945/// reclaim, which `MemAvailable` (and the indexer's watchdog) counts as free.
946/// The walk, the closure check and the verbatim append read the mapping the
947/// same way they read a `Vec`.
948pub(crate) enum PackBytes {
949 Heap(Vec<u8>),
950 Spooled {
951 /// Removed when this drops — after `put_pack` has copied the bytes
952 /// into the archive.
953 _file: tempfile::NamedTempFile,
954 map: memmap2::Mmap,
955 },
956}
957
958impl std::ops::Deref for PackBytes {
959 type Target = [u8];
960 fn deref(&self) -> &[u8] {
961 match self {
962 PackBytes::Heap(v) => v,
963 PackBytes::Spooled { map, .. } => map,
964 }
965 }
966}
967
968impl PartialEq<Vec<u8>> for PackBytes {
969 fn eq(&self, other: &Vec<u8>) -> bool {
970 &**self == other.as_slice()
971 }
972}
973
974impl std::fmt::Debug for PackBytes {
975 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
976 match self {
977 PackBytes::Heap(v) => write!(f, "{} byte(s) on the heap", v.len()),
978 PackBytes::Spooled { map, .. } => write!(f, "{} byte(s) spooled", map.len()),
979 }
980 }
981}
982
983impl std::fmt::Debug for PushedPack {
984 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
985 f.debug_struct("PushedPack")
986 .field("bytes", &format_args!("{} byte(s)", self.bytes.len()))
987 .field("num_objects", &self.num_objects)
988 .field("version", &self.version)
989 .finish()
990 }
991}
992
993/// **Read exactly one packfile off the wire and read its header back out of
994/// it.**
995///
996/// One copy, called from both engine arms (one implementation): the header is a property of
997/// the file, not of the engine that stored it.
998///
999/// # The shape that does not line up, and the deadlock it caused
1000///
1001/// The seam hands a `&mut dyn BufRead` positioned on the packfile's first byte
1002/// and gives **no length**, because the pkt-line framing does not carry one.
1003/// `GitOps::put_pack` needs the bytes *as a slice*, because storing them
1004/// verbatim is a single append of a caller-owned buffer. Until 2026-08-11 the
1005/// two were joined by `std::io::copy`, i.e. by reading to end of stream, under
1006/// a comment that said *"nothing follows the pack on that channel"*.
1007///
1008/// Nothing follows the pack, and that is the bug. A `receive-pack` client does
1009/// not hang up after its pack: it holds the channel open and waits for the
1010/// status report, which is the next thing this server owes it. So the server
1011/// waited for an EOF the client would only send once it had been answered, and
1012/// the client waited for an answer the server would only send once it had seen
1013/// EOF. `gunnar.real_repo_clone` sat in that state for 452 seconds with **0 s
1014/// of CPU on either side**, holding all 12 750 objects of the pack it was
1015/// waiting for more of; see [`pack_end`] for the full measurement.
1016///
1017/// It survived because every client the suite drives happens to close its write
1018/// end — stock `git push` does, smart HTTP does — and gunnar's own client, which
1019/// does not have to, is the one that found it.
1020///
1021/// So the pack is **delimited**, the way `git index-pack --stdin` delimits it:
1022/// [`pack_end::PackScanner`] walks the entries and stops on the last byte of the
1023/// trailer, and this loop consumes from the reader only what the scanner claims.
1024/// Everything after it — the report the client is waiting for, or nothing at all
1025/// — is left in the stream, untouched.
1026/// Both engine arms pass a directory to spool a big pack into; `None` keeps
1027/// every pack on the heap, which is what the tests below want and what
1028/// [`read_pushed_pack`] is.
1029pub(crate) fn read_pushed_pack_spooling(
1030 pack: &mut dyn BufRead,
1031 hash_kind: gix_hash::Kind,
1032 spool_dir: Option<&Path>,
1033) -> Result<PushedPack> {
1034 read_pushed_pack_spooling_at(pack, hash_kind, spool_dir, SPOOL_ABOVE)
1035}
1036
1037/// [`read_pushed_pack_spooling`] with the threshold as a parameter — a test's
1038/// door to the spool path without a 64 MiB fixture.
1039pub(crate) fn read_pushed_pack_spooling_at(
1040 pack: &mut dyn BufRead,
1041 hash_kind: gix_hash::Kind,
1042 spool_dir: Option<&Path>,
1043 spool_above: usize,
1044) -> Result<PushedPack> {
1045 use std::io::Write as _;
1046 let mut scanner = pack_end::PackScanner::new(hash_kind.len_in_bytes());
1047 let mut bytes = Vec::new();
1048 let mut spool: Option<(tempfile::NamedTempFile, u64)> = None;
1049 while !scanner.done() {
1050 // `fill_buf`/`consume` and not `read`: the scanner is shown a *peek* and
1051 // answers with a prefix length, so the byte after the trailer is never
1052 // taken out of the reader. A `read` would already have removed it.
1053 let used = {
1054 let available = pack
1055 .fill_buf()
1056 .context("reading the pushed packfile off the wire")?;
1057 if available.is_empty() {
1058 if bytes.is_empty() {
1059 bail!("an empty push is not a pack");
1060 }
1061 bail!(
1062 "the pushed pack ended after {} byte(s), in the middle of {}. The client \
1063 hung up mid-push; nothing was stored",
1064 bytes.len(),
1065 scanner.waiting_for()
1066 );
1067 }
1068 let used = scanner.feed(available)?;
1069 match &mut spool {
1070 Some((f, n)) => {
1071 f.write_all(&available[..used])
1072 .context("spooling the pushed pack")?;
1073 *n += used as u64;
1074 }
1075 None => {
1076 bytes.extend_from_slice(&available[..used]);
1077 if let (Some(dir), true) = (spool_dir, bytes.len() >= spool_above) {
1078 // Past the threshold: everything so far goes to disk
1079 // and the heap copy is released. One row, once.
1080 let mut f = tempfile::Builder::new()
1081 .prefix("incoming-")
1082 .suffix(".pack")
1083 .tempfile_in(dir)
1084 .with_context(|| {
1085 format!("spooling the pushed pack under {}", dir.display())
1086 })?;
1087 f.write_all(&bytes).context("spooling the pushed pack")?;
1088 let n = bytes.len() as u64;
1089 bytes = Vec::new();
1090 tracing::info!(
1091 target: "gunnar::store",
1092 event = "push.spooled",
1093 dir = %dir.display(),
1094 threshold = spool_above,
1095 "a pushed pack over {} MiB is spooled to disk while it arrives, not held in RAM",
1096 spool_above >> 20
1097 );
1098 spool = Some((f, n));
1099 }
1100 }
1101 }
1102 used
1103 };
1104 pack.consume(used);
1105 }
1106 let (version, num_objects) = scanner.header();
1107 let bytes = match spool {
1108 None => PackBytes::Heap(bytes),
1109 Some((f, n)) => {
1110 f.as_file().sync_data().ok();
1111 // SAFETY: the file is ours alone — created with a unique name,
1112 // written only above, never written again while mapped, and
1113 // removed when the mapping's owner drops.
1114 let map =
1115 unsafe { memmap2::Mmap::map(f.as_file()) }.context("mapping the spooled pack")?;
1116 tracing::info!(
1117 target: "gunnar::store",
1118 event = "push.spooled.mapped",
1119 bytes = n,
1120 "the spooled pack is complete and mapped for the walk and the append"
1121 );
1122 PackBytes::Spooled { _file: f, map }
1123 }
1124 };
1125 Ok(PushedPack {
1126 bytes,
1127 num_objects,
1128 version,
1129 })
1130}
1131
1132/// The pack's trailing checksum, which is its identity.
1133pub(crate) fn pack_trailer(bytes: &[u8], hash_kind: gix_hash::Kind) -> Result<Vec<u8>> {
1134 let len = hash_kind.len_in_bytes();
1135 bytes
1136 .len()
1137 .checked_sub(len)
1138 .and_then(|at| bytes.get(at..))
1139 .map(<[u8]>::to_vec)
1140 .ok_or_else(|| anyhow!("the pushed pack has no trailing {hash_kind} checksum"))
1141}
1142
1143// `open_secret_log` used to live here — an Arrow secret log at
1144// `<git_dir>/gunnar-secrets`, created by both arms at `open_repo` time to
1145// satisfy `RepoStore::secrets()`.
1146//
1147// It is **deleted** (2026-08-10), and the removal is worth more than the eight
1148// lines it took back. It ran `create_dir_all` on the way past, so opening *any*
1149// repository minted a directory, an `O_EXCL` lock file and a `secrets.arrow`
1150// beside it — per repository, for a store that was never read, on an engine
1151// whose whole design goal is one file per repository. The only secret store the
1152// server ever read was the *control* repository's, and that one is now a table
1153// in `control.redb` (`gunnar_core::secrets`). Neither arm has a secret of its
1154// own to keep, so neither carries one.
1155
1156#[cfg(test)]
1157mod tests {
1158 use super::pack_end::fixture::fixture_pack;
1159 use super::*;
1160
1161 /// ★ **The engine is reported in parts: its name, what it is, and where
1162 /// its source lives** (owner, 2026-10-02: the Storage page read "Engine
1163 /// Backend znippy-plugin-git (Arrow-IPC archive; objects + refs)"; it
1164 /// shows the name, the description as a tooltip, and links the name). The
1165 /// repository URL comes from ONE constant per forge.
1166 #[test]
1167 fn the_engine_is_reported_as_name_description_and_source_repository() {
1168 for engine in Engine::COMPILED {
1169 let r = engine.report();
1170 assert!(
1171 !r.name.is_empty() && !r.name.contains(' '),
1172 "{engine:?}: the name is one word: {r:?}"
1173 );
1174 assert!(
1175 !r.description.is_empty() && !r.description.contains(&r.name),
1176 "{engine:?}: {r:?}"
1177 );
1178 assert!(r.repo_url.starts_with("https://"), "{engine:?}: {r:?}");
1179 assert_eq!(
1180 engine.describe(),
1181 format!("{} ({})", r.name, r.description),
1182 "the CLI's line is the same parts"
1183 );
1184 }
1185 #[cfg(feature = "engine-znippy")]
1186 {
1187 let z = Engine::Znippy.report();
1188 assert_eq!(
1189 (z.name.as_str(), z.description.as_str()),
1190 ("znippy-plugin-git", "Arrow-IPC archive; objects + refs")
1191 );
1192 assert_eq!(z.repo_url, format!("{NORDISK_REPOS}/znippy"));
1193 assert_eq!(
1194 z.repo_url, "https://git.gunnar.rs/nordisk/znippy",
1195 "znippy's home is nordisk on git.gunnar.rs"
1196 );
1197 }
1198 #[cfg(feature = "engine-gix")]
1199 {
1200 let g = Engine::Gix.report();
1201 assert_eq!(g.name, "storage-git-gix");
1202 assert_eq!(
1203 g.repo_url,
1204 format!("{NORDISK_REPOS}/edda"),
1205 "storage-git-gix's source is edda"
1206 );
1207 }
1208 }
1209
1210 /// [`read_pushed_pack_spooling`] with no spool directory: every pack stays
1211 /// on the heap. **Test-only.** Both engine arms pass a real directory, so
1212 /// a `pub(crate)` version of this shim was dead code in every shipped
1213 /// binary while looking like a production entry point — it is the shape
1214 /// `dead_code` caught on 2026-09-03.
1215 fn read_pushed_pack(pack: &mut dyn BufRead, hash_kind: gix_hash::Kind) -> Result<PushedPack> {
1216 read_pushed_pack_spooling(pack, hash_kind, None)
1217 }
1218
1219 /// **A `receive-pack` client, as it behaves after it has sent its pack: it
1220 /// waits, holding the channel open, for the status report.**
1221 ///
1222 /// Every byte of the pack is handed over. Then the reader does the only
1223 /// honest thing a socket can do here — it refuses to produce a byte,
1224 /// because there is not going to be one until the server speaks — and it
1225 /// counts how many times it was asked.
1226 ///
1227 /// **The refusal is an error rather than a block on purpose.** A reader
1228 /// that blocked would turn this into a test with a stopwatch in it, which
1229 /// passes on a fast machine and fails on a slow one and proves nothing
1230 /// about either. `reads_past_the_pack` is the assertion that cannot lie:
1231 /// the correct number is zero, and it is zero for reasons of protocol, not
1232 /// of timing.
1233 struct ClientWaitingForItsReport {
1234 stream: Vec<u8>,
1235 at: usize,
1236 pack_len: usize,
1237 reads_past_the_pack: usize,
1238 }
1239
1240 impl ClientWaitingForItsReport {
1241 fn holding(pack: &[u8]) -> Self {
1242 ClientWaitingForItsReport {
1243 stream: pack.to_vec(),
1244 at: 0,
1245 pack_len: pack.len(),
1246 reads_past_the_pack: 0,
1247 }
1248 }
1249
1250 /// The same client, with a report already sitting behind the pack. Used
1251 /// to prove the reader is left *positioned* correctly and not merely
1252 /// unblocked: a delimiter that stopped one byte late would pass the
1253 /// test above and desynchronise the session here.
1254 fn with_a_report_behind_the_pack(pack: &[u8], report: &[u8]) -> Self {
1255 let mut stream = pack.to_vec();
1256 stream.extend_from_slice(report);
1257 ClientWaitingForItsReport {
1258 stream,
1259 at: 0,
1260 pack_len: pack.len(),
1261 reads_past_the_pack: 0,
1262 }
1263 }
1264
1265 fn what_is_left(&self) -> &[u8] {
1266 &self.stream[self.at..]
1267 }
1268 }
1269
1270 impl std::io::Read for ClientWaitingForItsReport {
1271 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1272 let available = self.fill_buf()?.len().min(buf.len());
1273 buf[..available].copy_from_slice(&self.stream[self.at..self.at + available]);
1274 self.consume(available);
1275 Ok(available)
1276 }
1277 }
1278
1279 impl BufRead for ClientWaitingForItsReport {
1280 fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
1281 if self.at >= self.pack_len && self.at >= self.stream.len() {
1282 self.reads_past_the_pack += 1;
1283 return Err(std::io::Error::new(
1284 std::io::ErrorKind::WouldBlock,
1285 "the client sent its whole pack and is waiting for the status report; this \
1286 read would block for ever",
1287 ));
1288 }
1289 // Handed over in small pieces, because a socket does that and a
1290 // delimiter that only works on one whole buffer is a delimiter that
1291 // works in tests.
1292 let end = self.stream.len().min(self.at + 997);
1293 Ok(&self.stream[self.at..end])
1294 }
1295
1296 fn consume(&mut self, amount: usize) {
1297 self.at += amount;
1298 }
1299 }
1300
1301 /// **RED-WHEN-BROKEN: the push that deadlocked `gunnar.real_repo_clone`.**
1302 ///
1303 /// Watched go red against the `std::io::copy` this replaced, 2026-08-11:
1304 ///
1305 /// ```text
1306 /// ---- storage::tests::the_pack_read_stops_at_the_trailer_and_never_waits_for_eof stdout ----
1307 /// reading the pushed packfile off the wire
1308 /// Caused by: the client sent its whole pack and is waiting for the status
1309 /// report; this read would block for ever
1310 /// ```
1311 ///
1312 /// On the wire that error is not an error. It is 452 seconds of two
1313 /// processes at 0% CPU with an empty socket between them.
1314 /// **A big pack is spooled, not held**: with the threshold at
1315 /// zero the same bytes come back through a mapping of a temp file beside
1316 /// the store, the file exists while the pack is held and is gone when it
1317 /// drops. Seen RED by returning `PackBytes::Heap` from the spool arm: the
1318 /// temp dir stays empty.
1319 #[test]
1320 fn a_pack_over_the_threshold_is_spooled_to_disk_and_mapped() {
1321 let pack = fixture_pack();
1322 let dir = tempfile::tempdir().expect("tempdir");
1323 let mut client = std::io::Cursor::new(pack.clone());
1324 let pushed =
1325 read_pushed_pack_spooling_at(&mut client, gix_hash::Kind::Sha1, Some(dir.path()), 0)
1326 .expect("a well-formed pack");
1327 assert!(
1328 matches!(pushed.bytes, PackBytes::Spooled { .. }),
1329 "{:?}",
1330 pushed.bytes
1331 );
1332 assert_eq!(pushed.bytes, pack, "the mapped pack is not the pack sent");
1333 assert_eq!(pushed.bytes.len(), pack.len());
1334 let spooled: Vec<_> = std::fs::read_dir(dir.path()).unwrap().flatten().collect();
1335 assert_eq!(
1336 spooled.len(),
1337 1,
1338 "one incoming-*.pack while the pack is held"
1339 );
1340 assert!(spooled[0]
1341 .file_name()
1342 .to_string_lossy()
1343 .starts_with("incoming-"));
1344 assert_eq!(
1345 pack_trailer(&pushed.bytes, gix_hash::Kind::Sha1)
1346 .unwrap()
1347 .len(),
1348 20
1349 );
1350 drop(pushed);
1351 assert_eq!(
1352 std::fs::read_dir(dir.path()).unwrap().count(),
1353 0,
1354 "the spool file outlived the pack"
1355 );
1356 // Under the threshold: the heap, and nothing on disk.
1357 let mut client = std::io::Cursor::new(pack.clone());
1358 let pushed = read_pushed_pack_spooling_at(
1359 &mut client,
1360 gix_hash::Kind::Sha1,
1361 Some(dir.path()),
1362 usize::MAX,
1363 )
1364 .unwrap();
1365 assert!(matches!(pushed.bytes, PackBytes::Heap(_)));
1366 assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0);
1367 }
1368
1369 /// ★ The space ledger's rule for the spool ([`spooled_on_disk`]) is what
1370 /// the spool file actually holds, at every consume of the read — the ledger
1371 /// gives a byte's spool promise back at `consume`, so a rule that ran ahead
1372 /// of the write would be a byte nobody counts.
1373 #[test]
1374 fn the_spool_holds_what_spooled_on_disk_says_at_every_consume() {
1375 struct Probe<'a> {
1376 inner: std::io::Cursor<Vec<u8>>,
1377 dir: &'a Path,
1378 consumed: u64,
1379 threshold: u64,
1380 checked: u32,
1381 }
1382 impl std::io::Read for Probe<'_> {
1383 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
1384 std::io::Read::read(&mut self.inner, buf)
1385 }
1386 }
1387 impl BufRead for Probe<'_> {
1388 fn fill_buf(&mut self) -> std::io::Result<&[u8]> {
1389 let b = self.inner.fill_buf()?;
1390 Ok(&b[..b.len().min(37)])
1391 }
1392 fn consume(&mut self, n: usize) {
1393 self.inner.consume(n);
1394 self.consumed += n as u64;
1395 let on_disk: u64 = std::fs::read_dir(self.dir)
1396 .unwrap()
1397 .flatten()
1398 .map(|e| e.metadata().unwrap().len())
1399 .sum();
1400 assert_eq!(
1401 on_disk,
1402 spooled_on_disk(self.consumed, self.threshold),
1403 "after {} byte(s) consumed",
1404 self.consumed
1405 );
1406 self.checked += 1;
1407 }
1408 }
1409 let pack = fixture_pack();
1410 let threshold = (pack.len() / 3) as u64;
1411 let dir = tempfile::tempdir().unwrap();
1412 let mut probe = Probe {
1413 inner: std::io::Cursor::new(pack.clone()),
1414 dir: dir.path(),
1415 consumed: 0,
1416 threshold,
1417 checked: 0,
1418 };
1419 let pushed = read_pushed_pack_spooling_at(
1420 &mut probe,
1421 gix_hash::Kind::Sha1,
1422 Some(dir.path()),
1423 threshold as usize,
1424 )
1425 .unwrap();
1426 assert_eq!(pushed.bytes, pack);
1427 assert!(probe.checked > 3, "{} consume(s)", probe.checked);
1428 }
1429
1430 #[test]
1431 fn the_pack_read_stops_at_the_trailer_and_never_waits_for_eof() {
1432 let pack = fixture_pack();
1433 let mut client = ClientWaitingForItsReport::holding(&pack);
1434
1435 let pushed = read_pushed_pack(&mut client, gix_hash::Kind::Sha1)
1436 .expect("the pack is complete; nothing more is owed before the report");
1437
1438 assert_eq!(
1439 client.reads_past_the_pack, 0,
1440 "the server made {} read(s) past the last byte of the pack. There is nobody to \
1441 answer them: the client is waiting for the status report those reads come before, \
1442 and on a socket this is not an error, it is a deadlock",
1443 client.reads_past_the_pack
1444 );
1445 assert_eq!(pushed.bytes, pack, "the stored pack is not the pack sent");
1446 assert_eq!(pushed.num_objects, 4);
1447 assert_eq!(pushed.version, 2);
1448 }
1449
1450 /// The reader is left **positioned** on the byte after the trailer, not
1451 /// merely unblocked. Smart HTTP already worked with an over-reading server
1452 /// because its body ends; this is what a session that carries on needs.
1453 #[test]
1454 fn what_follows_the_pack_is_left_in_the_stream() {
1455 let pack = fixture_pack();
1456 let report = b"0009done\n0000";
1457 let mut client = ClientWaitingForItsReport::with_a_report_behind_the_pack(&pack, report);
1458
1459 let pushed =
1460 read_pushed_pack(&mut client, gix_hash::Kind::Sha1).expect("a well-formed pack");
1461
1462 assert_eq!(pushed.bytes.len(), pack.len());
1463 assert_eq!(
1464 client.what_is_left(),
1465 report,
1466 "the bytes after the pack were swallowed, so whatever spoke next would be read \
1467 out of the middle of the pack"
1468 );
1469 assert_eq!(client.reads_past_the_pack, 0);
1470 }
1471
1472 /// A client that hangs up mid-pack is a **refusal**, not a stored half
1473 /// push. The old code could not tell this from a completed one: both ended
1474 /// at EOF.
1475 #[test]
1476 fn a_client_that_hangs_up_mid_pack_is_refused_and_says_where() {
1477 let pack = fixture_pack();
1478 let truncated = &pack[..pack.len() - 4];
1479 let mut client = std::io::Cursor::new(truncated.to_vec());
1480
1481 let err = read_pushed_pack(&mut client, gix_hash::Kind::Sha1)
1482 .expect_err("half a pack must not be accepted as a push");
1483 let message = format!("{err:#}");
1484 assert!(
1485 message.contains("trailing checksum") && message.contains("nothing was stored"),
1486 "the refusal must name what never arrived: {message}"
1487 );
1488 }
1489
1490 /// A push with no bytes at all keeps the message it always had.
1491 #[test]
1492 fn an_empty_push_is_still_not_a_pack() {
1493 let mut nothing = std::io::Cursor::new(Vec::new());
1494 let err = read_pushed_pack(&mut nothing, gix_hash::Kind::Sha1).expect_err("no bytes");
1495 assert!(format!("{err:#}").contains("an empty push is not a pack"));
1496 }
1497
1498 /// The default is znippy wherever znippy is compiled in. Asserted rather
1499 /// than commented, because "defaulting to the other engine" is a change no
1500 /// output of a working server would show.
1501 #[test]
1502 #[cfg(feature = "engine-znippy")]
1503 fn the_default_is_znippy() {
1504 assert_eq!(DEFAULT_ENGINE, Engine::Znippy);
1505 assert_eq!(Engine::COMPILED[0], Engine::Znippy);
1506 }
1507
1508 /// Every compiled engine answers to its own name. A rename that broke this
1509 /// would make `GUNNAR_ENGINE` silently unusable.
1510 #[test]
1511 fn every_compiled_engine_answers_to_its_name() {
1512 for engine in Engine::COMPILED {
1513 assert_eq!(
1514 Engine::from_name(engine.name()).expect("a compiled engine resolves"),
1515 *engine
1516 );
1517 }
1518 }
1519
1520 /// An unknown name is refused, and the message says what is here.
1521 #[test]
1522 fn an_unknown_engine_is_refused() {
1523 let err = Engine::from_name("postgres").expect_err("not an engine");
1524 let msg = format!("{err:#}");
1525 assert!(msg.contains("unknown storage engine"), "{msg}");
1526 assert!(msg.contains(&Engine::compiled_list()), "{msg}");
1527 }
1528
1529 /// **The rule, asserted in the build that can violate it.**
1530 ///
1531 /// In a znippy-only build, asking for gix must refuse and say the engine was
1532 /// not compiled in — never fall back to znippy. This test only exists in the
1533 /// build where the fallback is possible, which is the build where it matters.
1534 #[test]
1535 #[cfg(all(feature = "engine-znippy", not(feature = "engine-gix")))]
1536 fn an_engine_that_was_not_compiled_in_refuses_rather_than_falling_back() {
1537 let err = Engine::from_name("gix").expect_err("gix is not in this build");
1538 let msg = format!("{err:#}");
1539 assert!(
1540 msg.contains("built without the `gix` storage engine"),
1541 "{msg}"
1542 );
1543 assert!(msg.contains("--features engine-gix"), "{msg}");
1544 }
1545
1546 /// The same, the other way round, in the gix-only build.
1547 #[test]
1548 #[cfg(all(feature = "engine-gix", not(feature = "engine-znippy")))]
1549 fn znippy_is_refused_when_it_was_not_compiled_in() {
1550 let err = Engine::from_name("znippy").expect_err("znippy is not in this build");
1551 assert!(
1552 format!("{err:#}").contains("built without the `znippy` storage engine"),
1553 "{err:#}"
1554 );
1555 }
1556}