znippy_plugin_git/

sections.rs

//! `GitIndexBuilder` — turns a set of git objects into the three reserved
//! sections a `git`-format archive carries.
//!
//! The caller (gunnar's `repack()`, or a test) hands over the canonical bytes of
//! every object it is sealing. The builder derives each object's oid **by
//! hashing those bytes**, so the oid it indexes and the `relative_path` the
//! archive stores cannot drift apart — there is no second place where an oid is
//! spelled.
//!
//! Payloads are retained only for commits and trees. Blobs are leaves in the
//! reachability graph and contribute nothing but their oid, so their bytes are
//! dropped as soon as they are hashed. On a real repository that is the
//! difference between holding the metadata and holding the repository.
//!
//! The oid index needs lookup **row numbers**, which only exist once the sink has
//! sorted the lookup — hence [`GitIndexBuilder::into_reserved_builder`], which
//! defers exactly that step to `ArrowIpcSink::finish`.
//!
//! ## Two tiers, one seal path
//!
//! A `git` archive holds a repository one of two ways, and the builder has a
//! constructor for each:
//!
//! | constructor | data entries | derived sections |
//! |---|---|---|
//! | [`GitIndexBuilder::new`] | one per object, named by its oid | `__gunnar_oid__`, `__gunnar_graph__`, `__gunnar_reach__` |
//! | [`GitIndexBuilder::pack_tier`] | `pack-<id>.pack` / `.idx` | none — the pack's own `.idx` is the index |
//!
//! [`with_section`](GitIndexBuilder::with_section) — the refs and secrets push
//! logs — works identically on both, because it is the *server's* log and has
//! nothing to do with how the objects are laid out. That is the reason both
//! tiers route through this one type instead of growing a second writer.

use std::collections::HashMap;

use anyhow::{Result, anyhow, bail};
use znippy_common::{
    GUNNAR_GRAPH_MODULE, GUNNAR_OID_MODULE, GUNNAR_REACH_MODULE, ReservedSection,
    ReservedSectionBuilder,
};

use crate::graph::{CommitNode, assign_generations, build_graph_batch, graph_schema};
use crate::object::{GitHashKind, GitObjectKind, parse_canonical};
use crate::oid_index::{OidEntry, build_section};
use crate::reach::{ObjectFacts, OrdinalSpace, ReachPolicy, build_reach, build_reach_batch, reach_schema};

/// Accumulates objects and emits the reserved sections.
pub struct GitIndexBuilder {
    hash: GitHashKind,
    /// oid hex → object kind, in insertion order via `order`.
    kinds: HashMap<String, GitObjectKind>,
    order: Vec<String>,
    /// Tree payloads, kept for the reachability closure.
    trees: HashMap<String, Vec<u8>>,
    /// Commit rows, before generation numbers are assigned.
    commits: Vec<CommitNode>,
    reach: ReachPolicy,
    /// When false, `__gunnar_reach__` is not emitted at all — an archive that
    /// declares no bitmaps is a different thing from one that declares empty
    /// bitmaps, and a consumer must be able to tell them apart.
    emit_reach: bool,
    /// This archive's objects live in **packfiles**, not as one entry per oid.
    ///
    /// See [`GitIndexBuilder::pack_tier`]. The three derived sections are then
    /// not emitted at all, because the pack's own `.idx` already is the oid
    /// index and emitting an empty `__gunnar_oid__` beside it would state, in a
    /// section consumers trust, that the archive holds no objects.
    objects_in_packs: bool,
    /// Already-built sections handed in by the caller — the sealed
    /// `__gunnar_refs__` and `__gunnar_secrets__` push logs.
    ///
    /// They come in ready-made rather than being built here because their
    /// contents are a *log*, owned and appended to by the running server, not
    /// something derivable from the object set. Routing them through this one
    /// builder is what keeps a single seal path (LAW 5): gunnar's `repack()`
    /// hands over refs and secrets, and every reserved section the archive ends
    /// up carrying is emitted by the same code.
    extra: Vec<ReservedSection>,
}

impl GitIndexBuilder {
    pub fn new(hash: GitHashKind) -> Self {
        Self {
            hash,
            kinds: HashMap::new(),
            order: Vec::new(),
            trees: HashMap::new(),
            commits: Vec::new(),
            reach: ReachPolicy::default(),
            emit_reach: true,
            objects_in_packs: false,
            extra: Vec::new(),
        }
    }

    /// A builder for a **pack-tier** archive: one whose data entries are
    /// `pack-<id>.pack` / `pack-<id>.idx` rather than one entry per object.
    ///
    /// This is what gunnar's `repack()` seals. The reasoning is not znippy's to
    /// re-argue but is worth stating, because it decides what this builder must
    /// *not* emit: a git object inside a pack is already deflated and often a
    /// delta, so a tier that stores inflated content per oid pays a full
    /// inflate-plus-deflate round trip on every fetch, for ever, and can never
    /// be pack-copied. Measured on gunnar's own fixture, a znippy archive of
    /// 60000 small git objects came out **3.4× larger than its input** — the
    /// tier inflated rather than compressed.
    ///
    /// So the three derived sections are not emitted:
    ///
    /// * `__gunnar_oid__` — the pack's `.idx` **is** the oid index, and a far
    ///   better one: it addresses pack offsets, which is what a reader needs.
    /// * `__gunnar_graph__` / `__gunnar_reach__` — derived from object payloads
    ///   this builder never sees.
    ///
    /// They are omitted rather than emitted empty. An empty `__gunnar_oid__`
    /// beside a pack holding a million objects is not a smaller truth, it is a
    /// false one, and [`GitOidIndex::open`](crate::GitOidIndex::open) returning
    /// `None` is the state a consumer can act on.
    ///
    /// [`with_section`](Self::with_section) still works and is the point: refs
    /// and secrets are sealed the same way, through this one composition point,
    /// whichever tier the archive holds (LAW 5).
    pub fn pack_tier(hash: GitHashKind) -> Self {
        Self {
            objects_in_packs: true,
            ..Self::new(hash)
        }
    }

    /// True when this builder seals a pack tier rather than loose objects.
    pub fn is_pack_tier(&self) -> bool {
        self.objects_in_packs
    }

    /// Seal an already-built reserved section — the `__gunnar_refs__` or
    /// `__gunnar_secrets__` push log — alongside the object sections.
    ///
    /// Refused unless the module is reserved: a section the manifest readers do
    /// not classify as reserved is merged into the *data* index, which silently
    /// corrupts `list`, `decompress` and the iceberg sink. Catching it here
    /// names the offending module, instead of surfacing later as a wrong file
    /// count.
    pub fn with_section(mut self, section: ReservedSection) -> Result<Self> {
        if !znippy_common::is_reserved_module(&section.module_name) {
            bail!(
                "'{}' is not a reserved module — sealing it would merge it into the data index",
                section.module_name
            );
        }
        self.extra.push(section);
        Ok(self)
    }

    pub fn with_reach_policy(mut self, policy: ReachPolicy) -> Self {
        self.reach = policy;
        self
    }

    /// Do not emit `__gunnar_reach__`. Use when the bitmaps' build cost is not
    /// wanted for this repack.
    pub fn without_reach(mut self) -> Self {
        self.emit_reach = false;
        self
    }

    pub fn hash_kind(&self) -> GitHashKind {
        self.hash
    }

    pub fn len(&self) -> usize {
        self.order.len()
    }

    pub fn is_empty(&self) -> bool {
        self.order.is_empty()
    }

    /// Add one object from its canonical bytes. Returns the oid hex — which is
    /// also the `relative_path` the archive must store the object under.
    pub fn push_canonical(&mut self, canonical_bytes: &[u8]) -> Result<String> {
        // Refused rather than accepted-and-ignored. A pack-tier builder emits no
        // oid index, so an object handed to it would be silently absent from
        // every section — and the archive would still seal, still verify, and
        // still be wrong. The two tiers are alternatives, not a mixture.
        if self.objects_in_packs {
            bail!(
                "this is a pack-tier archive: its objects live in `pack-<id>.pack`, and the \
                 pack's own `.idx` is the oid index. Use `GitIndexBuilder::new` to seal loose \
                 objects instead."
            );
        }
        let obj = parse_canonical(canonical_bytes)
            .ok_or_else(|| anyhow!("not a canonical git object (`<type> <size>\\0<content>`)"))?;
        let oid = self.hash.oid_hex_of(canonical_bytes);
        if self.kinds.insert(oid.clone(), obj.kind).is_none() {
            self.order.push(oid.clone());
        }
        match obj.kind {
            GitObjectKind::Tree => {
                self.trees.insert(oid.clone(), obj.payload.to_vec());
            }
            GitObjectKind::Commit => {
                let h = crate::object::parse_commit(obj.payload);
                self.commits.push(CommitNode {
                    oid: oid.clone(),
                    parents: h.parents,
                    tree: h.tree,
                    committer_time: h.committer_time,
                    generation: 0,
                });
            }
            _ => {}
        }
        Ok(oid)
    }

    /// Add many objects at once.
    pub fn extend_canonical<'a, I: IntoIterator<Item = &'a [u8]>>(
        &mut self,
        objects: I,
    ) -> Result<Vec<String>> {
        objects.into_iter().map(|b| self.push_canonical(b)).collect()
    }

    /// The distinct objects in oid-lexicographic order — the ordinal space,
    /// whose index IS each oid's ordinal. This is the order `__gunnar_reach__`
    /// bitmaps address and the `ordinal` column of `__gunnar_oid__` records. The
    /// hex→ordinal map that used to accompany it is gone: oid ⇄ ordinal is
    /// served off the raw buffer built below by [`crate::reach::OrdinalSpace`].
    fn sorted_oids(&self) -> Vec<&str> {
        let mut sorted: Vec<&str> = self.order.iter().map(|s| s.as_str()).collect();
        sorted.sort_unstable();
        sorted
    }

    /// Build the sections against an explicit path → first-lookup-row mapping.
    ///
    /// Separated from [`into_reserved_builder`](Self::into_reserved_builder) so
    /// it is testable without sealing an archive, and so the sink path and the
    /// direct path are the same code (LAW 5).
    pub fn build_sections(&self, first_row: &HashMap<&str, u64>) -> Result<Vec<ReservedSection>> {
        // A pack tier derives nothing from object payloads it never saw. Not an
        // early return over an empty vector by accident: see
        // [`GitIndexBuilder::pack_tier`] for why absent beats empty here.
        if self.objects_in_packs {
            return Ok(Vec::new());
        }
        let sorted = self.sorted_oids();

        let mut entries = Vec::with_capacity(sorted.len());
        // The flat, sorted, fixed-width ordinal space — built once here, in the
        // same loop that decodes each oid, and borrowed as an `OrdinalSpace`
        // below. Sorted order IS ordinal order, so this needs no sort of its own.
        let mut oids_raw = Vec::with_capacity(sorted.len() * self.hash.oid_len());
        for (i, oid_hex) in sorted.iter().enumerate() {
            let raw = hex::decode(oid_hex)
                .map_err(|e| anyhow!("oid {oid_hex} is not hex: {e}"))?;
            if raw.len() != self.hash.oid_len() {
                bail!(
                    "oid {oid_hex} is {} bytes, expected {} for {:?}",
                    raw.len(),
                    self.hash.oid_len(),
                    self.hash
                );
            }
            let row = *first_row.get(*oid_hex).ok_or_else(|| {
                anyhow!(
                    "object {oid_hex} was indexed but no archive entry has it as its \
                     relative_path — the archive and the git index disagree"
                )
            })?;
            oids_raw.extend_from_slice(&raw);
            entries.push(OidEntry { oid: raw, lookup_row: row, ordinal: i as u32 });
        }

        let mut sections = vec![ReservedSection::raw(
            GUNNAR_OID_MODULE,
            build_section(&entries, self.hash)?,
        )];

        let commits = assign_generations(self.commits.clone());
        sections.push(ReservedSection::arrow(
            GUNNAR_GRAPH_MODULE,
            graph_schema(),
            vec![build_graph_batch(&commits)?],
        ));

        if self.emit_reach {
            let facts = ObjectFacts {
                ordinal: OrdinalSpace::new(&oids_raw, self.hash.oid_len()),
                trees: &self.trees,
                oid_len: self.hash.oid_len(),
            };
            let reach = build_reach(&commits, &facts, self.reach);
            sections.push(ReservedSection::arrow(
                GUNNAR_REACH_MODULE,
                reach_schema(),
                vec![build_reach_batch(&reach)?],
            ));
        }
        Ok(sections)
    }

    /// Every reserved section this archive will carry: the object sections from
    /// [`build_sections`](Self::build_sections), then the caller's push logs.
    ///
    /// The one composition point. Both the direct path and the sink path go
    /// through it, so a section added here cannot reach one and miss the other.
    pub fn finish_sections(mut self, first_row: &HashMap<&str, u64>) -> Result<Vec<ReservedSection>> {
        let mut sections = self.build_sections(first_row)?;
        sections.append(&mut self.extra);
        Ok(sections)
    }

    /// Hand the builder to `ArrowIpcSink::with_reserved_builder`.
    ///
    /// The closure runs at seal time with the sink's final sorted lookup, so
    /// `__gunnar_oid__` points at the rows the archive actually has. It fails
    /// loudly — rather than emitting a silently wrong index — if an object it
    /// was given is not present in the archive as a `relative_path`.
    pub fn into_reserved_builder(self) -> ReservedSectionBuilder {
        Box::new(move |view| {
            let rows = view.first_rows();
            let first_row: HashMap<&str, u64> = rows.into_iter().collect();
            self.finish_sections(&first_row)
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::object::{GitObjectKind, canonical};
    use crate::oid_index::GitOidIndex;
    use znippy_common::ReservedPayload;

    #[test]
    fn a_missing_archive_entry_is_a_loud_error_not_a_wrong_index() {
        let mut b = GitIndexBuilder::new(GitHashKind::Sha256);
        let oid = b.push_canonical(&canonical(GitObjectKind::Blob, b"hi")).unwrap();
        let empty: HashMap<&str, u64> = HashMap::new();
        let err = b.build_sections(&empty).unwrap_err().to_string();
        assert!(err.contains(&oid), "error should name the missing oid: {err}");
    }

    /// A pack-tier archive emits the push logs and **nothing derived**. The
    /// distinction that matters is absent-versus-empty: an empty
    /// `__gunnar_oid__` beside a pack of a million objects would read, to every
    /// consumer, as "this archive holds nothing".
    #[test]
    fn a_pack_tier_emits_the_push_logs_and_no_derived_section() {
        let dir = tempfile::tempdir().unwrap();
        let refs = crate::refs::RefLog::new(dir.path().join("refs.log"));
        refs.push(&[crate::refs::RefUpdate::set("refs/heads/main", &"a".repeat(64))])
            .unwrap();

        let b = GitIndexBuilder::pack_tier(GitHashKind::Sha256)
            .with_section(refs.seal_section().unwrap())
            .unwrap();
        assert!(b.is_pack_tier());

        let empty: HashMap<&str, u64> = HashMap::new();
        let derived = b.build_sections(&empty).unwrap();
        assert!(
            derived.is_empty(),
            "a pack tier must derive no section from objects it never saw, got {:?}",
            derived.iter().map(|s| s.module_name.clone()).collect::<Vec<_>>()
        );

        let all = b.finish_sections(&empty).unwrap();
        let names: Vec<&str> = all.iter().map(|s| s.module_name.as_str()).collect();
        assert_eq!(
            names,
            vec![znippy_common::GUNNAR_REFS_MODULE],
            "the push log is the only section a pack tier carries"
        );
    }

    /// The mirror, and the reason the test above is not vacuous: the SAME calls
    /// on a loose-object builder do emit the derived sections. Without this a
    /// builder that had stopped emitting them entirely would still pass.
    #[test]
    fn a_loose_object_builder_still_emits_all_three_derived_sections() {
        let mut b = GitIndexBuilder::new(GitHashKind::Sha256);
        let oid = b.push_canonical(&canonical(GitObjectKind::Blob, b"hi")).unwrap();
        assert!(!b.is_pack_tier());
        let rows: HashMap<&str, u64> = [(oid.as_str(), 0u64)].into_iter().collect();
        let names: Vec<String> = b
            .build_sections(&rows)
            .unwrap()
            .iter()
            .map(|s| s.module_name.clone())
            .collect();
        assert_eq!(
            names,
            vec![
                znippy_common::GUNNAR_OID_MODULE,
                znippy_common::GUNNAR_GRAPH_MODULE,
                znippy_common::GUNNAR_REACH_MODULE
            ]
        );
    }

    /// Handing an object to a pack-tier builder is refused, not ignored. If it
    /// were ignored the archive would seal, verify and be silently wrong: the
    /// object would be in no section at all.
    #[test]
    fn a_pack_tier_refuses_a_loose_object_rather_than_dropping_it() {
        let mut b = GitIndexBuilder::pack_tier(GitHashKind::Sha256);
        let err = b
            .push_canonical(&canonical(GitObjectKind::Blob, b"hi"))
            .unwrap_err()
            .to_string();
        assert!(
            err.contains("pack-tier") || err.contains("pack tier") || err.contains("pack-<id>"),
            "the refusal must say which tier this is: {err}"
        );
        assert_eq!(b.len(), 0, "the refused object must not have been recorded");
    }

    #[test]
    fn ordinals_are_oid_lexicographic_and_match_the_oid_index() {
        let mut b = GitIndexBuilder::new(GitHashKind::Sha1);
        let mut oids = Vec::new();
        for i in 0..40u32 {
            oids.push(b.push_canonical(&canonical(GitObjectKind::Blob, format!("b{i}").as_bytes())).unwrap());
        }
        let rows: HashMap<&str, u64> =
            oids.iter().enumerate().map(|(i, o)| (o.as_str(), i as u64 * 2)).collect();
        let sections = b.build_sections(&rows).unwrap();

        let ReservedPayload::Raw(oid_bytes) = &sections[0].payload else {
            panic!("first section must be the raw oid index")
        };
        let index = GitOidIndex::parse(oid_bytes.clone()).unwrap();

        let mut sorted = oids.clone();
        sorted.sort();
        for (i, o) in sorted.iter().enumerate() {
            let hit = index.lookup_hex(o).unwrap();
            assert_eq!(hit.ordinal, i as u32, "ordinal must be the oid-lexicographic rank");
            assert_eq!(hit.lookup_row, rows[o.as_str()]);
        }
    }
}