znippy_plugin_git/

lib.rs

//! znippy handler for **git object stores** — the `git` package format.
//!
//! One archive holds one git repository (D12: one `.znippy` per repo). The
//! consumer is `gunnar`, a pure-Rust git server whose `repack()` writes the cold
//! tier.
//!
//! ## The two tiers a repository can be stored as
//!
//! | tier | data entries | what it preserves |
//! |---|---|---|
//! | **objects** | one per object, named by its **oid hex**, holding its canonical bytes `"<type> <size>\0<content>"` | the oid is checkable by re-hashing the entry |
//! | **packs** | `pack-<id>.pack` + `pack-<id>.idx` | the client's own **deflate** and every **delta chain** |
//!
//! The object tier is the one this format was designed around and is what the
//! `__gunnar_*` sub-indexes describe. **The pack tier is what gunnar actually
//! seals**, and the reason is measured: a git object in a pack is already
//! deflated and often a delta, so storing it inflated per oid costs an OpenZL
//! decode plus a fresh zlib deflate on every fetch, for ever, and forecloses
//! pack-copy permanently. gunnar measured an archive of 60000 small git objects
//! at 3.4× *larger* than its input. See [`sections::GitIndexBuilder::pack_tier`].
//!
//! ## What this format adds on top of a plain archive
//!
//! | column / module | carries |
//! |---|---|
//! | `object_type` | `blob` / `tree` / `commit` / `tag` for the object tier; `packfile` / `pack-index` for the pack tier — typed listing with no reads |
//! | `object_size` | content length, so quota gates and size analytics are index-only |
//! | `__gunnar_oid__` | an `stree` over the first 8 bytes of every oid → its lookup row |
//! | `__gunnar_graph__` | the commit graph as Arrow: `oid, parents[], tree, committer_time, generation` |
//! | `__gunnar_reach__` | per-commit reachability bitmaps (roaring) over object ordinals |
//! | `__gunnar_refs__` / `__gunnar_secrets__` | the server's push logs, on **either** tier |
//!
//! The last three object modules are emitted for the object tier only — a pack
//! carries its own `.idx`, which is a better oid index than `__gunnar_oid__`
//! because it addresses pack offsets.
//!
//! All of these modules are **reserved** (`znippy_common::is_reserved_module`),
//! so `znippy list`, `decompress`, the iceberg sink and the manifest readers skip
//! them exactly as they skip the lookup and the trie.
//!
//! ## Laws it honours
//!
//! **P-1** one archive = one ecosystem (`--format git`); **P-2** it writes only
//! the columns it declared; **P-3** it is discovered through [`meta`]; **P-4** a
//! malformed or hostile entry never panics — `object_type` degrades to
//! `unknown` and the archive still writes.
//!
//! ## Who writes the reserved sections
//!
//! [`GitIndexBuilder`] does, handed to `ArrowIpcSink::with_reserved_builder` by
//! the writer that knows the object set — gunnar's `repack()`. It is deliberately
//! **not** wired into `znippy compress --format git`: the CLI's small-file batch
//! pass never sees objects that take the big-file path, so a CLI-built index
//! would be silently incomplete, and a silently incomplete oid index is worse
//! than none. `znippy compress --format git` therefore writes the two columns and
//! no reserved sections; `znippy run git lookup|graph` reads archives that carry
//! them.
//!
//! Native builtin, registered in `znippy-cli/src/handlers.rs::builtin_handlers`.
//! Deliberately **not** a WASM plugin: the oid index's hot path is `stree`, which
//! needs AVX2, an mmap and prefetch, none of which wasm offers.

use std::collections::HashMap;

use znippy_common::arrow::datatypes::{DataType, Field};
use znippy_common::plugin::{
    ArchiveTypePlugin, ExtensionRow, ExtensionValue, HandlerCommand, HandlerMeta,
};

/// The measured workloads behind the `znippy.git_*` benches (feature
/// `bench-kernels`). Not linked into a shipped archiver.
#[cfg(feature = "bench-kernels")]
pub mod bench_kernels;
/// The push path's archive writer: one trait, three durability contracts
/// ([`FastWriter`](archive_write::FastWriter),
/// [`SafeWriter`](archive_write::SafeWriter),
/// [`UringWriter`](uring_write::UringWriter)).
pub mod archive_write;
pub(crate) mod archive_map;
/// **Which implementation of each trait a store is built on.** One selector,
/// read once at construction, never per operation — [`arms::StoreConfig`].
pub mod arms;
pub mod graph;
pub mod index_layout;
/// The shared, off-the-ack-path index builder — one indexer per account,
/// gatling fan-out, index tables built last.
pub mod indexer;
pub mod memguard;
pub mod progress;
pub mod object;
pub mod oid_index;
/// Computing a git delta, for the one entry shape this engine cannot copy: a
/// stored delta whose base falls outside the request and whose receiver holds
/// nothing. The only place in this crate that computes one.
pub mod delta;
/// One pass over a pushed packfile's entries — the split the closure check
/// falls out of. Ours, gated against gix's grammar entry for entry.
pub mod pack_walk;
pub mod pushlog;
/// The read path as one stack: Ragnar `stree` in front, the Arrow index tables
/// it points into, and the redb un-sealed tail it falls through to.
pub mod read_stack;
/// Pack entries to oids: the indexer's half of the split, off the ack path.
pub mod resolve;
/// §14's exploded objects table: resolved content, derived from the verbatim
/// truth, built eagerly by the same indexer, droppable at any time.
pub mod exploded;
pub mod exploded_arrow;
pub mod exploded_cache;
pub use exploded_cache::{suppress_write_back, write_back_suppressed, SuppressWriteBack};
pub mod forest;
pub mod reach;
pub mod refs;
/// Store-side replication to another gunnar. **Deliberately empty** — signature
/// and contract only, no transport.
pub mod replicate;
/// One repository's stored bytes in ONE FILE: [`replicate`]'s decisions with a
/// body, and a file instead of a transport. `archive_out` / `archive_in`.
pub mod archive_file;
pub mod secrets;
pub mod store;
/// Garbage collection: one trait, two implementations
/// ([`gc::CompactInPlace`] in place, [`gc::NewGeneration`] into a new
/// generation — the default).
pub mod gc;
/// `GitOps` — the typed API gunnar calls, and the store that implements it.
/// Base znippy never sees this trait.
pub mod git_ops;
/// `GitServe` — the **reading** contract: decoded reads, the `HEAD` accessor
/// pair, negotiation and pack emission. Layered above [`git_ops`], answered out
/// of Arrow IPC, and **containing no gix** — see the module header for why that
/// is a rule rather than a coincidence.
pub mod serve;
/// **Stock `git` as the arbiter, inside a repository.** Test support only —
/// nothing on a serving, storing or indexing path may call it, because it forks
/// `git`. It is `pub` because `tests/concurrent_push.rs` compiles against the
/// library, and it exists because `git index-pack --strict` *segfaults* outside
/// a repository, which is how an oracle came to be incapable of failing.
pub mod git_oracle;
/// io_uring arm of [`archive_write`]. Linux only — the module is empty
/// elsewhere rather than substituting a `pwrite` path under an io_uring name.
pub mod uring_write;
pub mod sections;
#[cfg(test)]
mod thin_storm;
/// **The corpus the tests run on, generated rather than found.** Test support
/// only, and compiled only under `cfg(test)`: it exists because the fixture
/// used to be whichever repository `read_dir` handed back first on one
/// developer's box. See the module header for the eight guards that measured.
#[cfg(test)]
pub(crate) mod pack_fixture;

pub use archive_write::{ArchiveWrite, Faults, FastWriter, SafeWriter};
pub use arms::{
    ALL_ENV, DEFAULT_REDB_CACHE_BYTES, ENV_BOUNDARY_DELTA, ENV_EMIT_WORKERS, ENV_EXPLODE, ENV_GC,
    ENV_BASE_BUDGET_BYTES, ENV_INDEX, ENV_INDEX_BATCH_ROWS, ENV_MEM_BUDGET_BYTES, ENV_MEM_BUDGET_PCT,
    ENV_MEMINFO, ENV_REACH_COMMITS, ENV_REDB_CACHE, ENV_WRITER, GcArm, IndexArm, StoreConfig,
    WriterArm, env_reads, env_reads_here, redb_cache_bytes, set_graph_cache_default, ENV_GRAPH_CACHE,
};
pub use indexer::{AccountIndexer, IndexJob, IndexRow, IndexerPool, Lookup, PushPath};
#[cfg(target_os = "linux")]
pub use uring_write::UringWriter;

pub use graph::{CommitNode, decode_graph, graph_schema, read_graph};
pub use object::{
    GitHashKind, GitObject, GitObjectKind, PACKFILE_TYPE, PACK_INDEX_TYPE, PackFileKind, canonical,
    is_oid_path, pack_path_kind, parse_canonical, parse_commit, tree_entries,
};
pub use oid_index::{GitOidIndex, OidEntry, OidHit, OidLayout, key_for_oid};
pub use pushlog::{
    CompactionPolicy, CompactionReport, Finish, PushLog, PushLogScan, scan_frames,
};
pub use reach::{ReachEntry, ReachPolicy, decode_reach, read_reach, reach_schema};
pub use refs::{RefLog, RefState, RefUpdate, read_refs, refs_schema};
pub use secrets::{SecretState, SecretUpdate, SecretsLog, read_secrets, secrets_schema};
pub use sections::GitIndexBuilder;
pub use git_ops::{
    GitOps, GitStore, LookupPath, Oid, RefRow, SelectedStore, Stored, TxId, lookup_path,
    open_from_env, open_selected,
};
pub use serve::{Caps, GitServe, PackStats, ReachSet, HEAD};
/// The typed ref rejection, re-exported so a consumer of this crate reaches it
/// without also naming the contract crate. `RefRejection::of(&err)` is the only
/// sanctioned way to ask whether a ref write lost a race — never a message.
pub use git_storage_trait::{Observed, RefCas, RefRejection, RefTarget};

/// DenseUnion / `pkg_type` discriminant. Clear of the built-ins (1–25), media
/// (25), skidbladnir (40) and rust-toolchain (41).
pub const GIT_TYPE_ID: i8 = 42;

/// `object_type` written when the entry is not parseable as a canonical git
/// object. A distinct, queryable state — never a silent `blob`.
pub const UNKNOWN_OBJECT_TYPE: &str = "unknown";

/// Native git-object handler.
pub struct NativeGitPlugin;

impl NativeGitPlugin {
    pub fn new() -> Self {
        NativeGitPlugin
    }
}

impl Default for NativeGitPlugin {
    fn default() -> Self {
        Self::new()
    }
}

impl ArchiveTypePlugin for NativeGitPlugin {
    fn name(&self) -> &str {
        "git"
    }

    fn type_id(&self) -> i8 {
        GIT_TYPE_ID
    }

    fn meta(&self) -> HandlerMeta {
        HandlerMeta {
            name: "git".into(),
            aliases: vec!["gunnar".into(), "git-objects".into()],
            type_id: GIT_TYPE_ID,
            ecosystem: "Git object store (gunnar cold tier — one archive per repository)".into(),
            // Left empty on purpose, and it is not an oversight. A loose git
            // object carries no extension at all — its name IS its oid — and
            // the pack entries this handler also claims are matched on their
            // whole `pack-<oid>.<ext>` shape, not on a suffix. An extension
            // list here would claim every `.pack` and `.idx` in any archive,
            // which is wider than the truth. `matches_path` is the authority.
            extensions: Vec::new(),
            description: "Stores a git repository: either canonical objects keyed by oid hex, \
                          with the reserved __gunnar_oid__ (stree) / __gunnar_graph__ (commit \
                          graph) / __gunnar_reach__ (reachability bitmaps) sub-indexes, or the \
                          pack tier (pack-<id>.pack / .idx) that preserves the client's deflate \
                          and its delta chains"
                .into(),
            commands: vec![
                HandlerCommand::new(
                    "inspect",
                    "Print type/size/sha1/sha256 for a file of canonical git object bytes",
                ),
                HandlerCommand::new(
                    "lookup",
                    "Resolve an oid against an archive's __gunnar_oid__ index: `git lookup <archive> <oid>`",
                ),
                HandlerCommand::new(
                    "graph",
                    "Print an archive's commit graph (oid, generation, time, parents)",
                ),
                HandlerCommand::new(
                    "refs",
                    "Print a live store's ref namespace: `git refs <store-root>`",
                ),
            ],
        }
    }

    /// **CLI-only string dispatch** (§13.3). A server never comes through here:
    /// it holds a [`GitStore`] and calls [`GitOps`] directly, typed.
    ///
    /// `refs` is what that delegation looks like — open the store, call the trait
    /// method, print what it returned, and carry no logic of its own. The three
    /// commands beside it are older and are **archive inspection**, not store
    /// operations: they read a *sealed* archive's sections, which is a different
    /// thing from a live store and is deliberately not modelled as a `GitOps`
    /// call. They are left as they are rather than bent through a trait that does
    /// not describe them.
    fn run_command(&self, cmd: &str, args: &[String]) -> anyhow::Result<()> {
        match cmd {
            "refs" => {
                let root = args
                    .first()
                    .ok_or_else(|| anyhow::anyhow!("usage: git refs <store-root>"))?;
                // Everything below this line is printing. The answer comes from
                // the trait.
                let store = GitStore::open(std::path::Path::new(root), "cli")?;
                for r in store.refs()? {
                    println!(
                        "{}\t{}{}",
                        r.name,
                        r.oid.as_deref().map(hex::encode).unwrap_or_else(|| "-".into()),
                        match (&r.peeled, &r.symref_target) {
                            (Some(p), _) => format!("\t^{}", hex::encode(p)),
                            (None, Some(t)) => format!("\t-> {t}"),
                            (None, None) => String::new(),
                        }
                    );
                }
                Ok(())
            }
            "inspect" => {
                let path = args
                    .first()
                    .ok_or_else(|| anyhow::anyhow!("usage: git inspect <object-file>"))?;
                let data = std::fs::read(path)?;
                match parse_canonical(&data) {
                    Some(obj) => {
                        println!("type:   {}", obj.kind.as_str());
                        println!("size:   {}", obj.payload.len());
                        println!("sha1:   {}", GitHashKind::Sha1.oid_hex_of(&data));
                        println!("sha256: {}", GitHashKind::Sha256.oid_hex_of(&data));
                        Ok(())
                    }
                    None => anyhow::bail!(
                        "'{path}' is not canonical git object bytes (`<type> <size>\\0<content>`)"
                    ),
                }
            }
            "lookup" => {
                let (archive, oid) = match (args.first(), args.get(1)) {
                    (Some(a), Some(o)) => (a, o),
                    _ => anyhow::bail!("usage: git lookup <archive> <oid-hex>"),
                };
                let index = GitOidIndex::open(std::path::Path::new(archive))?.ok_or_else(|| {
                    anyhow::anyhow!("'{archive}' carries no __gunnar_oid__ index")
                })?;
                match index.lookup_hex(oid) {
                    Some(hit) => {
                        println!("oid:        {oid}");
                        println!("lookup_row: {}", hit.lookup_row);
                        println!("ordinal:    {}", hit.ordinal);
                        Ok(())
                    }
                    None => anyhow::bail!("oid '{oid}' is not in '{archive}'"),
                }
            }
            "graph" => {
                let archive = args
                    .first()
                    .ok_or_else(|| anyhow::anyhow!("usage: git graph <archive>"))?;
                let nodes = read_graph(std::path::Path::new(archive))?.ok_or_else(|| {
                    anyhow::anyhow!("'{archive}' carries no __gunnar_graph__ section")
                })?;
                println!("commits: {}", nodes.len());
                for n in &nodes {
                    println!(
                        "  {} gen={} time={} parents=[{}]",
                        n.oid,
                        n.generation,
                        n.committer_time.map(|t| t.to_string()).unwrap_or_else(|| "-".into()),
                        n.parents.join(" ")
                    );
                }
                Ok(())
            }
            other => anyhow::bail!("git: unknown subcommand '{other}'"),
        }
    }

    /// A git archive's entries are either **oids** — the whole `relative_path`
    /// is 40 or 64 lowercase hex characters — or the **packs** of a repository's
    /// cold tier, `pack-<id>.pack` / `pack-<id>.idx`.
    ///
    /// Both are the same ecosystem and so belong to one handler (P-1). Which of
    /// the two an archive holds is not a variant of the format: it is what the
    /// writer had. gunnar's `repack()` produces a consolidated pack and seals
    /// that, because a pack preserves the pushing client's own deflate and its
    /// delta chains, and an inflated per-oid tier destroys both permanently.
    fn matches_path(&self, path: &str) -> bool {
        is_oid_path(path) || pack_path_kind(path).is_some()
    }

    fn schema_fields(&self) -> Vec<Field> {
        vec![
            Field::new("object_type", DataType::Utf8, true),
            // UInt32: the index's extension-column writer materialises UInt32 and
            // Utf8 only. A git object larger than 4 GiB saturates here; its exact
            // byte length is always available from the base `uncompressed_size`
            // column, which is u64 and is the authority.
            Field::new("object_size", DataType::UInt32, true),
        ]
    }

    fn extract_metadata(&self, path: &str, data: &[u8]) -> Option<ExtensionRow> {
        if !self.matches_path(path) {
            return None;
        }
        let mut f: HashMap<String, ExtensionValue> = HashMap::new();

        // A pack is typed by its magic, never by its name. A name is a claim
        // the writer made; the magic is what the bytes are. `pack-<hex>.pack`
        // over something that is not a packfile degrades to `unknown` exactly
        // as malformed object bytes do (P-4) — a wrong type in the index is
        // worse than an honest absence, because it is queried and believed.
        if let Some(kind) = pack_path_kind(path) {
            let typed = if data.starts_with(kind.magic()) {
                kind.as_str()
            } else {
                UNKNOWN_OBJECT_TYPE
            };
            f.insert("object_type".into(), ExtensionValue::Str(typed.into()));
            f.insert(
                "object_size".into(),
                ExtensionValue::U32(data.len().min(u32::MAX as usize) as u32),
            );
            return Some(ExtensionRow { fields: f });
        }

        match parse_canonical(data) {
            Some(obj) => {
                f.insert("object_type".into(), ExtensionValue::Str(obj.kind.as_str().into()));
                f.insert(
                    "object_size".into(),
                    ExtensionValue::U32(obj.payload.len().min(u32::MAX as usize) as u32),
                );
            }
            None => {
                // P-4: a decompress bomb, a truncated object or plain garbage
                // still writes a row — typed `unknown`, sized by what is there.
                f.insert(
                    "object_type".into(),
                    ExtensionValue::Str(UNKNOWN_OBJECT_TYPE.into()),
                );
                f.insert(
                    "object_size".into(),
                    ExtensionValue::U32(data.len().min(u32::MAX as usize) as u32),
                );
            }
        }
        Some(ExtensionRow { fields: f })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::object::canonical;

    fn get<'a>(row: &'a ExtensionRow, k: &str) -> Option<&'a ExtensionValue> {
        row.fields.get(k)
    }

    #[test]
    fn claims_oid_paths_and_nothing_else() {
        let p = NativeGitPlugin::new();
        assert!(p.matches_path(&"a1b2c3d4".repeat(5)));       // 40 hex
        assert!(p.matches_path(&"0f".repeat(32)));            // 64 hex
        assert!(!p.matches_path("refs/heads/main"));
        assert!(!p.matches_path("objects/pack/pack-abc.pack"));
        assert!(!p.matches_path(&"A1B2C3D4".repeat(5)));
    }

    /// The pack tier's two file names are claimed, and only in the shape git
    /// itself writes. A bare `.pack` suffix is not enough: an archive of
    /// arbitrary files must not have its entries retyped as git packs.
    #[test]
    fn claims_the_pack_tier_by_its_whole_name_not_by_a_suffix() {
        let p = NativeGitPlugin::new();
        let id40 = "a1b2c3d4".repeat(5);
        let id64 = "0f".repeat(32);
        assert_eq!(pack_path_kind(&format!("pack-{id40}.pack")), Some(PackFileKind::Data));
        assert_eq!(pack_path_kind(&format!("pack-{id64}.idx")), Some(PackFileKind::Index));
        assert!(p.matches_path(&format!("pack-{id64}.pack")));
        assert!(p.matches_path(&format!("objects/pack/pack-{id40}.idx")));

        for not_ours in [
            "pack-nothex.pack",
            "somefile.pack",
            "pack-.pack",
            &format!("pack-{id40}.bitmap"),
            &format!("pack-{}.pack", "A1B2C3D4".repeat(5)),
            &format!("{id40}.pack"),
        ] {
            assert_eq!(pack_path_kind(not_ours), None, "wrongly claimed {not_ours}");
        }
        assert!(!p.matches_path("somefile.pack"));
    }

    /// A pack is typed by its **magic**, never by the name the writer chose.
    /// A file called `pack-….pack` that does not start with `PACK` is
    /// `unknown` — the same honest state malformed object bytes get (P-4).
    #[test]
    fn a_pack_is_typed_by_its_magic_and_a_liar_is_unknown() {
        let p = NativeGitPlugin::new();
        let id = "0f".repeat(32);

        let mut pack = b"PACK".to_vec();
        pack.extend_from_slice(&2u32.to_be_bytes());
        pack.extend_from_slice(&7u32.to_be_bytes());
        let row = p.extract_metadata(&format!("pack-{id}.pack"), &pack).unwrap();
        assert_eq!(get(&row, "object_type"), Some(&ExtensionValue::Str(PACKFILE_TYPE.into())));
        assert_eq!(get(&row, "object_size"), Some(&ExtensionValue::U32(pack.len() as u32)));

        let idx = b"\xfftOc\x00\x00\x00\x02".to_vec();
        let row = p.extract_metadata(&format!("pack-{id}.idx"), &idx).unwrap();
        assert_eq!(get(&row, "object_type"), Some(&ExtensionValue::Str(PACK_INDEX_TYPE.into())));

        // Now the liars. Each is named as a pack and is not one.
        for (name, bytes) in [
            (format!("pack-{id}.pack"), &b"NOTAPACK"[..]),
            (format!("pack-{id}.idx"), &b"PACK\0\0\0\x02"[..]),
            (format!("pack-{id}.pack"), &b""[..]),
        ] {
            let row = p.extract_metadata(&name, bytes).expect("a claimed path always yields a row");
            assert_eq!(
                get(&row, "object_type"),
                Some(&ExtensionValue::Str(UNKNOWN_OBJECT_TYPE.into())),
                "{name} carries {bytes:?}, which is not that kind of file — it must not be typed \
                 from its name"
            );
        }
    }

    #[test]
    fn types_and_sizes_come_from_the_stored_bytes() {
        let p = NativeGitPlugin::new();
        let body = b"hello world";
        let bytes = canonical(GitObjectKind::Blob, body);
        let oid = GitHashKind::Sha256.oid_hex_of(&bytes);
        let row = p.extract_metadata(&oid, &bytes).unwrap();
        assert_eq!(get(&row, "object_type"), Some(&ExtensionValue::Str("blob".into())));
        assert_eq!(get(&row, "object_size"), Some(&ExtensionValue::U32(body.len() as u32)));

        let commit = canonical(GitObjectKind::Commit, b"tree x\n\nmsg\n");
        let coid = GitHashKind::Sha1.oid_hex_of(&commit);
        let crow = p.extract_metadata(&coid, &commit).unwrap();
        assert_eq!(get(&crow, "object_type"), Some(&ExtensionValue::Str("commit".into())));
    }

    #[test]
    fn garbage_degrades_to_unknown_and_never_panics() {
        let p = NativeGitPlugin::new();
        let oid = "d".repeat(64);
        for bad in [&b""[..], &b"\0\0\0"[..], &[0xffu8; 4096][..], &b"blob 99\0short"[..]] {
            let row = p.extract_metadata(&oid, bad).expect("a claimed path always yields a row");
            assert_eq!(
                get(&row, "object_type"),
                Some(&ExtensionValue::Str(UNKNOWN_OBJECT_TYPE.into())),
                "unparseable bytes must be typed `unknown`, never guessed"
            );
            assert_eq!(get(&row, "object_size"), Some(&ExtensionValue::U32(bad.len() as u32)));
        }
    }

    #[test]
    fn a_path_that_is_not_an_oid_yields_no_row() {
        let p = NativeGitPlugin::new();
        assert!(p.extract_metadata("HEAD", b"ref: refs/heads/main\n").is_none());
    }

    #[test]
    fn meta_is_the_discovery_record() {
        let m = NativeGitPlugin::new().meta();
        assert_eq!(m.name, "git");
        assert_eq!(m.type_id, GIT_TYPE_ID);
        assert!(m.aliases.contains(&"gunnar".to_string()));
        assert_eq!(m.commands.len(), 4);
        assert!(
            m.commands.iter().any(|c| c.name == "refs"),
            "the GitOps-delegating command is not advertised"
        );
    }
}