grit_lib/
pack_write.rs

1//! Write pack index files and install pack streams into an object directory.
2//!
3//! Used by embedders and tooling that build pack bytes in memory (for example
4//! [`crate::transfer::build_pack`]) and need a Git-compatible `.pack`/`.idx` pair
5//! on disk without shelling out to `git index-pack`.
6//!
7//! ```rust
8//! use grit_lib::pack_write::InstallPackOptions;
9//! # fn main() {
10//! let _ = InstallPackOptions::default();
11//! # }
12//! ```
13
14use std::fs;
15use std::io::Read;
16use std::path::{Path, PathBuf};
17
18use flate2::read::ZlibDecoder;
19
20use crate::error::{Error, Result};
21use crate::objects::{HashAlgo, ObjectId, ObjectKind};
22use crate::pack::{
23    clear_pack_cache, hash_object_bytes, parse_ofs_delta_base, parse_pack_object_header,
24    skip_one_pack_object, PackedType,
25};
26
27/// One `(object id, pack offset)` entry used when writing a v2 index.
28#[derive(Debug, Clone, PartialEq, Eq)]
29pub struct PackIndexWriteEntry {
30    /// Canonical object id for the stored object.
31    pub oid: ObjectId,
32    /// Byte offset of the object header within the `.pack` file.
33    pub offset: u64,
34}
35
36/// Options controlling [`install_pack_stream`].
37#[derive(Debug, Clone)]
38pub struct InstallPackOptions {
39    /// When set, use this stem for `pack/<stem>.pack` instead of deriving one from the pack checksum.
40    pub stem: Option<String>,
41    /// Hash width for this repository (`20` for SHA-1, `32` for SHA-256).
42    pub hash_algo: HashAlgo,
43}
44
45impl Default for InstallPackOptions {
46    fn default() -> Self {
47        Self {
48            stem: None,
49            hash_algo: HashAlgo::Sha1,
50        }
51    }
52}
53
54/// Validate a PACK header and return `(body_end, object_count)` for `hash_algo`.
55fn validated_pack_body(pack: &[u8], hash_algo: HashAlgo) -> Result<(usize, usize)> {
56    let hash_bytes = hash_algo.len();
57    if pack.len() < 12 + hash_bytes {
58        return Err(Error::CorruptObject("pack file too small".into()));
59    }
60    if &pack[0..4] != b"PACK" {
61        return Err(Error::CorruptObject("invalid PACK signature".into()));
62    }
63    let version = u32::from_be_bytes(pack[4..8].try_into().unwrap_or([0; 4]));
64    if version != 2 && version != 3 {
65        return Err(Error::CorruptObject(format!(
66            "unsupported pack version {version}"
67        )));
68    }
69    let count = u32::from_be_bytes(pack[8..12].try_into().unwrap_or([0; 4])) as usize;
70    let body_end = pack.len() - hash_bytes;
71    Ok((body_end, count))
72}
73
74/// Scan a pack containing only whole (non-delta) objects and return sorted index entries.
75///
76/// # Errors
77///
78/// Returns [`Error::CorruptObject`] when the stream is malformed or contains delta objects.
79pub fn enumerate_whole_pack_entries(
80    pack: &[u8],
81    hash_algo: HashAlgo,
82) -> Result<Vec<PackIndexWriteEntry>> {
83    let hash_bytes = hash_algo.len();
84    let (body_end, count) = validated_pack_body(pack, hash_algo)?;
85    let mut pos = 12usize;
86    let mut out = Vec::with_capacity(count);
87    while pos < body_end {
88        let offset =
89            u64::try_from(pos).map_err(|_| Error::CorruptObject("pack offset overflow".into()))?;
90        let (packed_type, size) = parse_pack_object_header(pack, &mut pos)?;
91        match packed_type {
92            PackedType::Commit | PackedType::Tree | PackedType::Blob | PackedType::Tag => {
93                let mut dec = ZlibDecoder::new(&pack[pos..]);
94                let mut data = Vec::with_capacity(size as usize);
95                dec.read_to_end(&mut data)
96                    .map_err(|e| Error::Zlib(e.to_string()))?;
97                pos += dec.total_in() as usize;
98                let kind = packed_type_to_kind(packed_type)?;
99                let oid_bytes = hash_object_bytes(kind, &data, hash_bytes)?;
100                let oid = ObjectId::from_bytes(&oid_bytes)?;
101                out.push(PackIndexWriteEntry { oid, offset });
102            }
103            PackedType::RefDelta | PackedType::OfsDelta => {
104                return Err(Error::CorruptObject(
105                    "enumerate_whole_pack_entries requires a non-delta pack".into(),
106                ));
107            }
108        }
109    }
110    if out.len() != count {
111        return Err(Error::CorruptObject(format!(
112            "pack header object count {count} != scanned {}",
113            out.len()
114        )));
115    }
116    out.sort_by(|a, b| a.oid.as_bytes().cmp(b.oid.as_bytes()));
117    Ok(out)
118}
119
120/// Write a version-2 pack index for `pack_path` using precomputed `entries`.
121///
122/// `entries` must be sorted by OID (as produced by [`enumerate_whole_pack_entries`]).
123///
124/// # Errors
125///
126/// Returns [`Error::Io`] or [`Error::CorruptObject`] on failure.
127fn write_pack_index_v2(
128    idx_path: &Path,
129    pack_path: &Path,
130    entries: &[PackIndexWriteEntry],
131    hash_bytes: usize,
132) -> Result<()> {
133    let pack_bytes = fs::read(pack_path).map_err(Error::Io)?;
134    if pack_bytes.len() < hash_bytes {
135        return Err(Error::CorruptObject(
136            "pack too small for idx trailer".into(),
137        ));
138    }
139    let mut fanout = [0u32; 256];
140    for byte in 0u32..256 {
141        let count = entries
142            .iter()
143            .filter(|e| u32::from(e.oid.as_bytes()[0]) <= byte)
144            .count();
145        fanout[byte as usize] = u32::try_from(count).unwrap_or(u32::MAX);
146    }
147    let mut buf = Vec::new();
148    buf.extend_from_slice(b"\xfftOc");
149    buf.extend_from_slice(&2u32.to_be_bytes());
150    for f in fanout {
151        buf.extend_from_slice(&f.to_be_bytes());
152    }
153    for e in entries {
154        buf.extend_from_slice(e.oid.as_bytes());
155    }
156    for e in entries {
157        let start = e.offset as usize;
158        let mut end = start;
159        skip_one_pack_object(&pack_bytes, &mut end, e.offset, hash_bytes)
160            .map_err(|err| Error::CorruptObject(format!("idx crc walk: {err}")))?;
161        let crc = crc32fast::hash(&pack_bytes[start..end]);
162        buf.extend_from_slice(&crc.to_be_bytes());
163    }
164    let (offsets32, large_offsets) = pack_index_v2_offset_tables(entries);
165    for v in offsets32 {
166        buf.extend_from_slice(&v.to_be_bytes());
167    }
168    for lo in large_offsets {
169        buf.extend_from_slice(&lo.to_be_bytes());
170    }
171    buf.extend_from_slice(&pack_bytes[pack_bytes.len() - hash_bytes..]);
172    let idx_hash = match hash_bytes {
173        20 => HashAlgo::Sha1.checksum_digest(&buf),
174        32 => HashAlgo::Sha256.checksum_digest(&buf),
175        _ => {
176            return Err(Error::CorruptObject(format!(
177                "unsupported hash width {hash_bytes} for pack index"
178            )));
179        }
180    };
181    buf.extend_from_slice(&idx_hash);
182    fs::write(idx_path, buf).map_err(Error::Io)?;
183    Ok(())
184}
185
186/// Install `pack` under `objects_dir/pack/` and write the matching v2 `.idx`.
187///
188/// Returns paths to the `.pack` and `.idx` files. Clears the in-process pack cache so
189/// subsequent reads observe the new pack.
190///
191/// # Errors
192///
193/// Returns [`Error::CorruptObject`] when `pack` cannot be indexed, or [`Error::Io`] on failure.
194pub fn install_pack_stream(
195    objects_dir: &Path,
196    pack: &[u8],
197    opts: InstallPackOptions,
198) -> Result<(PathBuf, PathBuf)> {
199    let hash_bytes = opts.hash_algo.len();
200    let entries = enumerate_whole_pack_entries(pack, opts.hash_algo)?;
201    let pack_dir = objects_dir.join("pack");
202    fs::create_dir_all(&pack_dir).map_err(Error::Io)?;
203    let stem = opts.stem.unwrap_or_else(|| {
204        let trailer = &pack[pack.len() - hash_bytes..];
205        hex::encode(&trailer[..hash_bytes.min(12)])
206    });
207    let pack_path = pack_dir.join(format!("pack-{stem}.pack"));
208    let idx_path = pack_dir.join(format!("pack-{stem}.idx"));
209    fs::write(&pack_path, pack).map_err(Error::Io)?;
210    write_pack_index_v2(&idx_path, &pack_path, &entries, hash_bytes)?;
211    clear_pack_cache();
212    Ok((pack_path, idx_path))
213}
214
215/// Uncompressed payload size of all whole objects in `pack`.
216///
217/// # Errors
218///
219/// Returns [`Error::CorruptObject`] when `pack` is malformed.
220#[must_use = "callers use this for size reporting"]
221pub fn whole_pack_uncompressed_size(pack: &[u8], hash_algo: HashAlgo) -> Result<u64> {
222    let hash_bytes = hash_algo.len();
223    let (body_end, header_count) = validated_pack_body(pack, hash_algo)?;
224    let mut pos = 12usize;
225    let mut total = 0u64;
226    let mut seen = 0usize;
227    while pos < body_end {
228        let (packed_type, size) = parse_pack_object_header(pack, &mut pos)?;
229        match packed_type {
230            PackedType::Commit | PackedType::Tree | PackedType::Blob | PackedType::Tag => {
231                total = total.saturating_add(size);
232                let mut dec = ZlibDecoder::new(&pack[pos..]);
233                let mut tmp = Vec::with_capacity(size as usize);
234                dec.read_to_end(&mut tmp)
235                    .map_err(|e| Error::Zlib(e.to_string()))?;
236                pos += dec.total_in() as usize;
237                seen += 1;
238            }
239            PackedType::RefDelta => {
240                pos += hash_bytes;
241                let mut dec = ZlibDecoder::new(&pack[pos..]);
242                let mut tmp = Vec::with_capacity(size as usize);
243                dec.read_to_end(&mut tmp)
244                    .map_err(|e| Error::Zlib(e.to_string()))?;
245                pos += dec.total_in() as usize;
246                seen += 1;
247            }
248            PackedType::OfsDelta => {
249                let obj_start = pos;
250                let _base = parse_ofs_delta_base(pack, &mut pos, obj_start as u64)?;
251                let mut dec = ZlibDecoder::new(&pack[pos..]);
252                let mut tmp = Vec::with_capacity(size as usize);
253                dec.read_to_end(&mut tmp)
254                    .map_err(|e| Error::Zlib(e.to_string()))?;
255                pos += dec.total_in() as usize;
256                seen += 1;
257            }
258        }
259    }
260    if seen != header_count {
261        return Err(Error::CorruptObject(format!(
262            "pack header object count {header_count} != scanned {seen}"
263        )));
264    }
265    Ok(total)
266}
267
268/// Build the v2 index 32-bit offset column and optional 64-bit large-offset table.
269///
270/// Offsets below `0x8000_0000` are stored literally. Larger offsets (or any offset
271/// that would set the high bit) are stored as `0x8000_0000 | index` into `large`,
272/// followed by the real offset in the 64-bit table (Git pack index v2).
273#[must_use]
274pub fn pack_index_v2_offset_tables(entries: &[PackIndexWriteEntry]) -> (Vec<u32>, Vec<u64>) {
275    let mut offsets32 = Vec::with_capacity(entries.len());
276    let mut large = Vec::new();
277    for e in entries {
278        if e.offset < 0x8000_0000 {
279            offsets32.push(e.offset as u32);
280        } else {
281            let idx = u32::try_from(large.len()).unwrap_or(u32::MAX);
282            offsets32.push(0x8000_0000 | idx);
283            large.push(e.offset);
284        }
285    }
286    (offsets32, large)
287}
288
289fn packed_type_to_kind(pt: PackedType) -> Result<ObjectKind> {
290    match pt {
291        PackedType::Commit => Ok(ObjectKind::Commit),
292        PackedType::Tree => Ok(ObjectKind::Tree),
293        PackedType::Blob => Ok(ObjectKind::Blob),
294        PackedType::Tag => Ok(ObjectKind::Tag),
295        PackedType::RefDelta | PackedType::OfsDelta => Err(Error::CorruptObject(
296            "delta object in whole-object enumeration".into(),
297        )),
298    }
299}
300
301#[cfg(test)]
302mod tests {
303    use super::*;
304    use crate::objects::ObjectKind;
305    use crate::odb::Odb;
306    use crate::pack::read_pack_index;
307    use crate::transfer::{build_pack, PackBuildOptions};
308
309    #[cfg_attr(
310        not(feature = "full-tests"),
311        ignore = "starts system git or is heavy; run with --features full-tests"
312    )]
313    #[test]
314    fn install_pack_passes_git_verify_pack() {
315        let tmp = tempfile::tempdir().expect("tempdir");
316        let odb = Odb::new(tmp.path());
317        let blob = odb
318            .write(ObjectKind::Blob, b"synth pack-write test\n")
319            .expect("write");
320        let pack = build_pack(
321            &odb,
322            &[blob],
323            &[],
324            &PackBuildOptions {
325                delta: false,
326                ..PackBuildOptions::default()
327            },
328        )
329        .expect("build");
330        let (pack_path, idx_path) = install_pack_stream(
331            tmp.path(),
332            &pack,
333            InstallPackOptions {
334                hash_algo: HashAlgo::Sha1,
335                ..InstallPackOptions::default()
336            },
337        )
338        .expect("install");
339        let idx = read_pack_index(&idx_path).expect("read idx");
340        assert_eq!(idx.len(), 1);
341        let offset = idx
342            .find_offset(&blob)
343            .expect("blob in idx")
344            .expect("offset");
345        assert!(offset > 0, "pack index offset is positive");
346        let pack_str = pack_path.to_str().expect("pack path utf-8");
347        let out = grit_test_support::git_cmd(&["verify-pack", "-s", pack_str])
348            .in_dir(tmp.path())
349            .exec();
350        assert!(out.ok(), "{}", out.dump("git verify-pack"));
351        assert_eq!(
352            idx.find_offset(&blob).expect("lookup"),
353            Some(idx.offset_at(0).expect("off0")),
354            "idx offset matches blob"
355        );
356        assert_eq!(
357            idx.find_offset(&blob).expect("lookup"),
358            Some(offset),
359            "idx offset stable"
360        );
361    }
362
363    #[test]
364    fn pack_index_v2_large_offset_table_round_trips() {
365        let entries = vec![
366            PackIndexWriteEntry {
367                oid: ObjectId::from_hex("0000000000000000000000000000000000000001").unwrap(),
368                offset: 100,
369            },
370            PackIndexWriteEntry {
371                oid: ObjectId::from_hex("0000000000000000000000000000000000000002").unwrap(),
372                offset: 0x8000_0000 + 512,
373            },
374            PackIndexWriteEntry {
375                oid: ObjectId::from_hex("0000000000000000000000000000000000000003").unwrap(),
376                offset: (1u64 << 33) + 99,
377            },
378        ];
379        let (offsets32, large) = pack_index_v2_offset_tables(&entries);
380        assert_eq!(offsets32[0], 100);
381        assert_eq!(offsets32[1], 0x8000_0000);
382        assert_eq!(offsets32[2], 0x8000_0001);
383        assert_eq!(large, vec![0x8000_0000 + 512, (1u64 << 33) + 99]);
384        let mut decoded_large = 0usize;
385        for (i, raw) in offsets32.iter().enumerate() {
386            let off = if (*raw & 0x8000_0000) == 0 {
387                u64::from(*raw)
388            } else {
389                let slot = decoded_large;
390                decoded_large += 1;
391                large[slot]
392            };
393            assert_eq!(off, entries[i].offset, "entry {i}");
394        }
395    }
396
397    #[test]
398    fn whole_pack_uncompressed_size_rejects_malformed_input() {
399        use crate::error::Error;
400        let err = whole_pack_uncompressed_size(&[], HashAlgo::Sha1).unwrap_err();
401        assert!(matches!(err, Error::CorruptObject(_)));
402        let truncated = b"PACK";
403        let err = whole_pack_uncompressed_size(truncated, HashAlgo::Sha1).unwrap_err();
404        assert!(matches!(err, Error::CorruptObject(_)));
405        let bad_sig = b"NOPE\x00\x00\x00\x02\x00\x00\x00\x00";
406        let err = whole_pack_uncompressed_size(bad_sig, HashAlgo::Sha1).unwrap_err();
407        assert!(matches!(err, Error::CorruptObject(_)));
408    }
409
410    #[test]
411    fn whole_pack_uncompressed_size_empty_pack_is_zero() {
412        let mut pack = Vec::from(b"PACK" as &[u8]);
413        pack.extend_from_slice(&2u32.to_be_bytes());
414        pack.extend_from_slice(&0u32.to_be_bytes());
415        pack.extend_from_slice(&[0u8; 20]);
416        assert_eq!(
417            whole_pack_uncompressed_size(&pack, HashAlgo::Sha1).unwrap(),
418            0
419        );
420    }
421
422    #[test]
423    fn enumerate_whole_pack_matches_build_pack() {
424        let odb = Odb::new(tempfile::tempdir().expect("tempdir").path());
425        let a = odb.write(ObjectKind::Blob, b"aaa").unwrap();
426        let b = odb.write(ObjectKind::Blob, b"bbb").unwrap();
427        let pack = build_pack(
428            &odb,
429            &[a, b],
430            &[],
431            &PackBuildOptions {
432                delta: false,
433                ..Default::default()
434            },
435        )
436        .unwrap();
437        let entries = enumerate_whole_pack_entries(&pack, HashAlgo::Sha1).unwrap();
438        assert_eq!(entries.len(), 2);
439        assert!(entries
440            .windows(2)
441            .all(|w| w[0].oid.as_bytes() <= w[1].oid.as_bytes()));
442    }
443}
grit_lib/pack_write.rs

grit_lib/
pack_write.rs
