1#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))]
4
5use flate2::{Compress, Compression, FlushCompress, Status};
6use sley_core::{CancelFlag, GitError, ObjectFormat, ObjectId, Result, StreamingDigest};
7use sley_formats::Bundle;
8use sley_object::{EncodedObject, ObjectType};
9use std::borrow::Borrow;
10use std::cell::RefCell;
11use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
12use std::fmt;
13use std::fs::File;
14use std::io::{Read, Seek, SeekFrom, Write};
15use std::ops::Range;
16use std::path::Path;
17use std::sync::Arc;
18
19mod bounded_read;
26mod delta;
27mod fix_thin;
28mod index;
29pub mod inflate;
30mod limits;
31mod read;
32mod stream;
33mod write;
34
35pub use bounded_read::*;
36pub(crate) use delta::*;
37pub use fix_thin::*;
38pub use index::*;
39pub use limits::{MAX_READ_DELTA_CHAIN_DEPTH, PACK_OBJECT_COUNT_PREALLOC_CAP};
40pub(crate) use limits::{checked_pack_object_count, pack_entry_prealloc};
41pub use read::*;
42pub use stream::*;
43pub use write::*;
44
45#[derive(Debug, Clone, PartialEq, Eq)]
46pub struct PackEntry {
47 pub oid: ObjectId,
48 pub compressed_size: u64,
49 pub uncompressed_size: u64,
50 pub offset: u64,
51}
52#[derive(Debug, Clone, PartialEq, Eq)]
53pub struct RepackPolicy {
54 pub write_bitmaps: bool,
55 pub cruft_packs: bool,
56 pub geometric_factor: Option<u8>,
57}
58
59#[derive(Debug, Clone, PartialEq, Eq)]
60pub struct PackFile {
61 pub version: u32,
62 pub entries: Vec<PackObject>,
63 pub checksum: ObjectId,
64}
65
66#[derive(Debug, Clone, PartialEq, Eq)]
67pub struct PackObject {
68 pub entry: PackEntry,
69 pub object: EncodedObject,
70}
71
72#[derive(Debug, Clone, PartialEq, Eq)]
75pub struct PackVerifyStat {
76 pub oid: ObjectId,
78 pub object_type: ObjectType,
80 pub size: u64,
82 pub size_in_pack: u64,
85 pub offset: u64,
87 pub delta_depth: u32,
89 pub base_oid: Option<ObjectId>,
92}
93
94#[derive(Debug, Clone, PartialEq, Eq)]
97pub struct PackVerifyStats {
98 pub objects: Vec<PackVerifyStat>,
99 pub checksum: ObjectId,
100}
101
102#[derive(Debug, Clone, PartialEq, Eq)]
103pub struct PackWrite {
104 pub pack: Vec<u8>,
105 pub index: Vec<u8>,
106 pub checksum: ObjectId,
107 pub entries: Vec<PackIndexEntry>,
108 pub delta_count: u32,
109}
110
111#[derive(Debug, Clone, PartialEq, Eq)]
112pub struct PackWriteSummary {
113 pub index: Vec<u8>,
114 pub checksum: ObjectId,
115 pub entries: Vec<PackIndexEntry>,
116 pub delta_count: u32,
117 pub pack_size: u64,
118}
119
120#[derive(Debug, Clone, Copy, PartialEq, Eq)]
121pub struct PackInput<'a> {
122 pub oid: &'a ObjectId,
123 pub object: &'a EncodedObject,
124}
125#[derive(Debug, Clone, Copy, PartialEq, Eq)]
126enum PackObjectKind {
127 Commit,
128 Tree,
129 Blob,
130 Tag,
131 OfsDelta,
132 RefDelta,
133}
134
135#[derive(Debug, Clone, PartialEq, Eq)]
136enum ParsedPackEntry {
137 Resolved(PackObject),
138 Delta {
139 base: DeltaBase,
140 compressed_size: u64,
141 delta_size: u64,
142 offset: u64,
143 delta: Vec<u8>,
144 },
145}
146
147#[derive(Debug, Clone, PartialEq, Eq)]
148enum DeltaBase {
149 Offset(u64),
150 Ref(ObjectId),
151}
152
153struct OnDiskEntry {
157 offset: u64,
158 base: Option<DeltaBase>,
159 stream_size: u64,
160}
161#[derive(Debug, Clone, Copy, PartialEq, Eq)]
162struct EntryHeader {
163 kind: PackObjectKind,
164 size: u64,
165}
166fn next_byte(bytes: &[u8], offset: &mut usize) -> Result<u8> {
167 let Some(byte) = bytes.get(*offset).copied() else {
168 return Err(GitError::InvalidFormat(
169 "truncated pack entry header".into(),
170 ));
171 };
172 *offset += 1;
173 Ok(byte)
174}
175
176fn u16_be(bytes: &[u8]) -> u16 {
177 u16::from_be_bytes([bytes[0], bytes[1]])
178}
179
180fn u32_be(bytes: &[u8]) -> u32 {
181 u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])
182}
183
184fn u64_be(bytes: &[u8]) -> u64 {
185 u64::from_be_bytes([
186 bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7],
187 ])
188}
189fn checked_range(
190 start: usize,
191 count: usize,
192 width: usize,
193 total: usize,
194) -> Result<std::ops::Range<usize>> {
195 let len = count
196 .checked_mul(width)
197 .ok_or_else(|| GitError::InvalidFormat("pack index table overflow".into()))?;
198 let end = start
199 .checked_add(len)
200 .ok_or_else(|| GitError::InvalidFormat("pack index table overflow".into()))?;
201 if end > total {
202 return Err(GitError::InvalidFormat("truncated pack index table".into()));
203 }
204 Ok(start..end)
205}
206
207fn validate_position_permutation(positions: &[u32]) -> Result<()> {
208 let mut seen = vec![false; positions.len()];
209 for position in positions {
210 let idx = *position as usize;
211 if idx >= positions.len() {
212 return Err(GitError::InvalidFormat(format!(
213 "invalid rev-index position {position}"
214 )));
215 }
216 if seen[idx] {
217 return Err(GitError::InvalidFormat(format!(
218 "invalid rev-index position {position}"
219 )));
220 }
221 seen[idx] = true;
222 }
223 Ok(())
224}
225
226thread_local! {
232 pub(crate) static INFLATE: RefCell<flate2::Decompress> = RefCell::new(flate2::Decompress::new(true));
233}
234
235#[cfg(test)]
236mod tests {
237 use super::*;
238 use flate2::Compression;
239 use flate2::read::ZlibDecoder;
240 use flate2::write::ZlibEncoder;
241 use sley_core::AtomicCancel;
242 use std::fs;
243 use std::io::{Cursor, Read, Write};
244 use std::path::{Path, PathBuf};
245 use std::process::Command;
246 use std::time::{SystemTime, UNIX_EPOCH};
247
248 fn multi_blob_pack(count: usize) -> Vec<u8> {
249 let objects = (0..count)
250 .map(|idx| {
251 EncodedObject::new(
252 ObjectType::Blob,
253 format!("stream cancel pack object {idx}\n").into_bytes(),
254 )
255 })
256 .collect::<Vec<_>>();
257 PackFile::write_undeltified(&objects, ObjectFormat::Sha1)
258 .expect("test operation should succeed")
259 .pack
260 }
261
262 #[test]
263 fn index_pack_stream_already_cancelled_returns_cancelled() {
264 let pack = multi_blob_pack(4);
265 let source = AtomicCancel::new();
266 source.cancel();
267 let mut reader = Cursor::new(pack.as_slice());
268 let err = PackIndex::write_v2_for_pack_reader_to_trailer_with_progress_and_cancel(
269 &mut reader,
270 ObjectFormat::Sha1,
271 CancelFlag::new(&source),
272 |_| {},
273 )
274 .expect_err("pre-cancelled index should fail");
275 assert_eq!(err, GitError::Cancelled);
276 }
277
278 #[test]
279 fn index_pack_stream_cancel_mid_stream_returns_cancelled() {
280 let pack = multi_blob_pack(8);
281 let source = AtomicCancel::new();
282 let mut saw_object = false;
283 let mut reader = Cursor::new(pack.as_slice());
284 let err = PackIndex::write_v2_for_pack_reader_to_trailer_with_progress_and_cancel(
285 &mut reader,
286 ObjectFormat::Sha1,
287 CancelFlag::new(&source),
288 |progress| {
289 if progress.received_objects >= 1 {
290 saw_object = true;
291 source.cancel();
292 }
293 },
294 )
295 .expect_err("mid-stream cancel should fail");
296 assert!(
297 saw_object,
298 "progress should have reported at least one object before cancel"
299 );
300 assert_eq!(err, GitError::Cancelled);
301 }
302
303 #[test]
304 fn index_pack_stream_with_progress_still_works_without_cancel() {
305 let pack = multi_blob_pack(3);
306 let mut samples = 0u32;
307 let mut reader = Cursor::new(pack.as_slice());
308 let build = PackIndex::write_v2_for_pack_reader_to_trailer_with_progress(
309 &mut reader,
310 ObjectFormat::Sha1,
311 |_| {
312 samples += 1;
313 },
314 )
315 .expect("uncancelled index should succeed");
316 assert_eq!(build.entries.len(), 3);
317 assert!(samples >= 2, "header + completion progress expected");
318 }
319
320 #[test]
321 fn write_packed_from_source_respects_cancel_between_windows() {
322 let format = ObjectFormat::Sha1;
323 let count = PACK_STREAM_COMPRESSION_WINDOW_OBJECTS + 4;
325 let objects = (0..count)
326 .map(|idx| {
327 EncodedObject::new(
328 ObjectType::Blob,
329 format!("write-cancel object {idx}\n").into_bytes(),
330 )
331 })
332 .collect::<Vec<_>>();
333 let object_ids = objects
334 .iter()
335 .map(|object| object.object_id(format).expect("oid"))
336 .collect::<Vec<_>>();
337 let object_map = object_ids
338 .iter()
339 .copied()
340 .zip(objects.into_iter().map(Arc::new))
341 .collect::<HashMap<_, _>>();
342
343 let source = AtomicCancel::new();
344 source.cancel();
345 let mut written = Vec::new();
346 let err = PackFile::write_packed_from_source_to_writer_with_cancel(
347 &object_ids,
348 format,
349 &PackWriteOptions::new().with_reorder(false),
350 |oid| {
351 object_map
352 .get(oid)
353 .cloned()
354 .ok_or_else(|| GitError::not_found(format!("missing test object {oid}")))
355 },
356 &mut written,
357 CancelFlag::new(&source),
358 )
359 .expect_err("pre-cancelled pack write should fail");
360 assert_eq!(err, GitError::Cancelled);
361 assert!(
362 written.is_empty() || written.len() < 32,
363 "should not have finished a full pack after cancel"
364 );
365 }
366
367 fn delta_pack_options(prefer_ofs_delta: bool) -> PackWriteOptions {
368 PackWriteOptions::new()
369 .with_prefer_ofs_delta(prefer_ofs_delta)
370 .with_reorder(false)
371 }
372
373 #[test]
374 fn parses_single_blob_pack() {
375 let pack = single_object_pack(ObjectFormat::Sha1, ObjectType::Blob, b"hello\n");
376 let parsed = PackFile::parse_sha1(&pack).expect("test operation should succeed");
377 assert_eq!(parsed.version, 2);
378 assert_eq!(parsed.entries.len(), 1);
379 let object = &parsed.entries[0].object;
380 assert_eq!(object.object_type, ObjectType::Blob);
381 assert_eq!(object.body, b"hello\n");
382 assert_eq!(
383 parsed.entries[0].entry.oid.to_hex(),
384 "ce013625030ba8dba906f756967f9e9ca394464a"
385 );
386 }
387
388 #[test]
389 fn parses_single_blob_pack_sha256() {
390 let pack = single_object_pack(ObjectFormat::Sha256, ObjectType::Blob, b"hello\n");
391 let parsed =
392 PackFile::parse(&pack, ObjectFormat::Sha256).expect("test operation should succeed");
393 assert_eq!(parsed.version, 2);
394 assert_eq!(parsed.entries.len(), 1);
395 let object = &parsed.entries[0].object;
396 assert_eq!(object.object_type, ObjectType::Blob);
397 assert_eq!(object.body, b"hello\n");
398 assert_eq!(
399 parsed.entries[0].entry.oid,
400 object
401 .object_id(ObjectFormat::Sha256)
402 .expect("test operation should succeed")
403 );
404 }
405
406 #[test]
407 fn parses_bundle_pack_payload_with_bundle_format() {
408 let pack = single_object_pack(ObjectFormat::Sha1, ObjectType::Blob, b"bundle\n");
409 let oid = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"bundle\n")
410 .expect("test operation should succeed");
411 let bundle_bytes = format!("# v2 git bundle\n{oid} refs/heads/main\n\n")
412 .into_bytes()
413 .into_iter()
414 .chain(pack)
415 .collect::<Vec<_>>();
416 let bundle = Bundle::parse(&bundle_bytes, ObjectFormat::Sha1)
417 .expect("test operation should succeed");
418
419 let parsed = PackFile::parse_bundle(&bundle).expect("test operation should succeed");
420 assert_eq!(parsed.entries.len(), 1);
421 assert_eq!(parsed.entries[0].object.object_type, ObjectType::Blob);
422 assert_eq!(parsed.entries[0].object.body, b"bundle\n");
423 }
424
425 fn lying_size_blob_pack(format: ObjectFormat, declared_size: u64, real_body: &[u8]) -> Vec<u8> {
431 let mut pack = Vec::new();
432 pack.extend_from_slice(b"PACK");
433 pack.extend_from_slice(&2u32.to_be_bytes());
434 pack.extend_from_slice(&1u32.to_be_bytes());
435 write_pack_entry_header_kind(&mut pack, 3, declared_size);
437 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
438 encoder
439 .write_all(real_body)
440 .expect("test operation should succeed");
441 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
442 let checksum =
443 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
444 pack.extend_from_slice(checksum.as_bytes());
445 pack
446 }
447
448 #[test]
461 fn rejects_decompression_bomb_header_without_oom() {
462 for &declared in &[u64::MAX, 100 * 1024 * 1024 * 1024, u64::from(u32::MAX) * 4] {
463 let pack = lying_size_blob_pack(ObjectFormat::Sha1, declared, b"tiny\n");
464 let handle = std::thread::spawn(move || PackFile::parse_sha1(&pack));
465 let result = handle.join();
466 assert!(
468 result.is_ok(),
469 "parsing a bomb header (declared={declared}) panicked instead of erroring cleanly"
470 );
471 let parse_result = result.expect("parse thread should not panic on a bomb header");
473 assert!(
474 parse_result.is_err(),
475 "bomb header (declared={declared}) should be rejected as invalid"
476 );
477 }
478 }
479
480 fn lying_result_size_delta_pack(
487 format: ObjectFormat,
488 declared_result_size: u64,
489 delta_kind: DeltaKind,
490 ) -> Vec<u8> {
491 let base = b"hello";
492 let result = b"hello world"; let mut delta = Vec::new();
496 write_delta_varint(&mut delta, base.len() as u64);
497 write_delta_varint(&mut delta, declared_result_size);
498 let suffix = &result[base.len()..];
500 delta.push(0x90); delta.push(base.len() as u8);
502 delta.push(suffix.len() as u8);
503 delta.extend_from_slice(suffix);
504
505 let mut pack = Vec::new();
506 pack.extend_from_slice(b"PACK");
507 pack.extend_from_slice(&2u32.to_be_bytes());
508 pack.extend_from_slice(&2u32.to_be_bytes());
509
510 let base_offset = pack.len();
511 write_entry_header(&mut pack, ObjectType::Blob, base.len() as u64);
512 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
513 encoder
514 .write_all(base)
515 .expect("test operation should succeed");
516 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
517
518 let delta_offset = pack.len();
519 write_pack_entry_header_kind(
520 &mut pack,
521 match delta_kind {
522 DeltaKind::Offset => 6,
523 DeltaKind::Ref => 7,
524 },
525 delta.len() as u64,
526 );
527 match delta_kind {
528 DeltaKind::Offset => write_ofs_delta_offset(&mut pack, delta_offset - base_offset),
529 DeltaKind::Ref => {
530 let base_oid = sley_core::object_id_for_bytes(format, "blob", base)
531 .expect("test operation should succeed");
532 pack.extend_from_slice(base_oid.as_bytes());
533 }
534 }
535 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
536 encoder
537 .write_all(&delta)
538 .expect("test operation should succeed");
539 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
540
541 let checksum =
542 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
543 pack.extend_from_slice(checksum.as_bytes());
544 pack
545 }
546
547 #[test]
557 fn rejects_delta_result_size_bomb_without_oom() {
558 let bombs: &[u64] = &[u64::MAX, 1024 * 1024 * 1024 * 1024];
559 for &declared in bombs {
560 for delta_kind in [DeltaKind::Ref, DeltaKind::Offset] {
561 let pack = lying_result_size_delta_pack(ObjectFormat::Sha1, declared, delta_kind);
562 let handle = std::thread::spawn(move || PackFile::parse_sha1(&pack));
563 let join_result = handle.join();
564 assert!(
565 join_result.is_ok(),
566 "delta bomb (declared={declared}, kind={delta_kind:?}) panicked/aborted \
567 instead of erroring cleanly"
568 );
569 let parse_result =
570 join_result.expect("parse thread should not panic on a delta bomb");
571 assert!(
572 parse_result.is_err(),
573 "delta bomb (declared={declared}, kind={delta_kind:?}) should be rejected \
574 as invalid (result.len() != declared)"
575 );
576 }
577 }
578 }
579
580 #[test]
584 fn applies_legitimate_delta_after_result_size_bound() {
585 for delta_kind in [DeltaKind::Ref, DeltaKind::Offset] {
586 let base = b"hello";
587 let result = b"hello world";
588 let pack = two_object_delta_pack(ObjectFormat::Sha1, base, result, delta_kind);
589 let parsed = PackFile::parse_sha1(&pack).expect("legitimate delta should resolve");
590 assert_eq!(parsed.entries.len(), 2);
591 assert_eq!(parsed.entries[0].object.body, base);
592 assert_eq!(parsed.entries[1].object.body, result);
593 }
594 }
595
596 #[test]
597 fn bounded_delta_application_polls_cancel_between_commands() {
598 let base = vec![b'x'; 4096];
599 let mut result = base.clone();
600 result.extend_from_slice(b"changed");
601 let delta = DeltaIndex::new(&base).delta(&result).expect("delta");
602 let plan = plan_pack_delta(&base, &delta).expect("plan");
603 let mut out = Vec::with_capacity(result.len());
604 let source = AtomicCancel::new();
605 source.cancel();
606
607 assert_eq!(
608 apply_pack_delta_exact(&base, &delta, plan, &mut out, CancelFlag::new(&source)),
609 Err(GitError::Cancelled)
610 );
611 assert!(out.is_empty());
612 }
613
614 #[test]
615 fn legacy_delta_preserves_final_size_mismatch_and_copy_heavy_growth() {
616 let base = b"x";
617 let overproducing = vec![1, 1, 2, b'x', b'y'];
618 assert_eq!(
619 apply_pack_delta(base, &overproducing),
620 Err(GitError::InvalidObject(
621 "delta result size mismatch: expected 1, got 2".into()
622 ))
623 );
624
625 let copies = 4096usize;
626 let mut copy_heavy = Vec::new();
627 write_delta_varint(&mut copy_heavy, base.len() as u64);
628 write_delta_varint(&mut copy_heavy, copies as u64);
629 for _ in 0..copies {
630 copy_heavy.extend_from_slice(&[0x91, 0, 1]);
632 }
633 assert_eq!(
634 apply_pack_delta(base, ©_heavy).expect("copy-heavy legacy delta"),
635 vec![b'x'; copies]
636 );
637 }
638
639 #[test]
640 fn rejects_bundle_pack_payload_with_wrong_object_format() {
641 let pack = single_object_pack(ObjectFormat::Sha1, ObjectType::Blob, b"bundle\n");
642 let oid = sley_core::object_id_for_bytes(ObjectFormat::Sha256, "blob", b"bundle\n")
643 .expect("test operation should succeed");
644 let bundle_bytes =
645 format!("# v3 git bundle\n@object-format=sha256\n{oid} refs/heads/main\n\n")
646 .into_bytes()
647 .into_iter()
648 .chain(pack)
649 .collect::<Vec<_>>();
650 let bundle = Bundle::parse(&bundle_bytes, ObjectFormat::Sha1)
651 .expect("test operation should succeed");
652
653 assert!(PackFile::parse_bundle(&bundle).is_err());
654 }
655
656 fn assert_pack_index_view_matches_owned(index: &[u8], format: ObjectFormat) {
657 let owned = PackIndex::parse(index, format).expect("test operation should succeed");
658 let view = PackIndexView::parse(index, format).expect("test operation should succeed");
659 let owned_view =
660 PackIndexViewData::parse(Arc::from(index.to_vec().into_boxed_slice()), format)
661 .expect("test operation should succeed");
662
663 assert_eq!(view.version, owned.version);
664 assert_eq!(view.count, owned.entries.len());
665 assert_eq!(view.count(), owned.entries.len());
666 assert_eq!(view.fanout(), &owned.fanout);
667 assert_eq!(view.pack_checksum, owned.pack_checksum);
668 assert_eq!(view.index_checksum, owned.index_checksum);
669 assert_eq!(owned_view.version, owned.version);
670 assert_eq!(owned_view.count(), owned.entries.len());
671 assert_eq!(owned_view.fanout(), &owned.fanout);
672 assert_eq!(owned_view.pack_checksum, owned.pack_checksum);
673 assert_eq!(owned_view.index_checksum, owned.index_checksum);
674 for entry in &owned.entries {
675 let owned_found = owned
676 .find(&entry.oid)
677 .expect("test operation should succeed");
678 let expected = Some(PackIndexLookup {
679 crc32: owned_found.crc32,
680 offset: owned_found.offset,
681 });
682 assert_eq!(view.find(&entry.oid), expected);
683 assert_eq!(owned_view.find(&entry.oid), expected);
684 }
685 }
686
687 #[test]
688 fn writes_pack_and_index_that_round_trip() {
689 let object = EncodedObject::new(ObjectType::Blob, b"hello\n".to_vec());
690 let written = PackFile::write_undeltified_sha1(std::slice::from_ref(&object))
691 .expect("test operation should succeed");
692 let pack = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
693 let index =
694 PackIndex::parse_v2_sha1(&written.index).expect("test operation should succeed");
695 let oid = object
696 .object_id(ObjectFormat::Sha1)
697 .expect("test operation should succeed");
698 assert_eq!(pack.entries[0].object, object);
699 assert_eq!(index.pack_checksum, pack.checksum);
700 assert_eq!(
701 index
702 .find(&oid)
703 .expect("test operation should succeed")
704 .offset,
705 12
706 );
707 }
708
709 #[test]
710 fn pack_index_view_matches_owned_index_for_generated_sha1_pack() {
711 let objects = (0..8)
712 .map(|idx| {
713 EncodedObject::new(
714 ObjectType::Blob,
715 format!("borrowed pack index view sha1 object {idx}\n").into_bytes(),
716 )
717 })
718 .collect::<Vec<_>>();
719 let written = PackFile::write_packed(&objects, ObjectFormat::Sha1)
720 .expect("test operation should succeed");
721
722 assert_pack_index_view_matches_owned(&written.index, ObjectFormat::Sha1);
723
724 let view =
725 PackIndexView::parse_v2_sha1(&written.index).expect("test operation should succeed");
726 let missing = sley_core::object_id_for_bytes(
727 ObjectFormat::Sha1,
728 "blob",
729 b"not present in borrowed index\n",
730 )
731 .expect("test operation should succeed");
732 assert_eq!(view.find(&missing), None);
733 }
734
735 #[test]
736 fn writes_sha256_pack_and_index_that_round_trip() {
737 let object = EncodedObject::new(ObjectType::Blob, b"hello sha256\n".to_vec());
738 let written =
739 PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha256)
740 .expect("test operation should succeed");
741 let pack = PackFile::parse(&written.pack, ObjectFormat::Sha256)
742 .expect("test operation should succeed");
743 let index = PackIndex::parse(&written.index, ObjectFormat::Sha256)
744 .expect("test operation should succeed");
745 let oid = object
746 .object_id(ObjectFormat::Sha256)
747 .expect("test operation should succeed");
748 assert_eq!(pack.entries[0].object, object);
749 assert_eq!(index.pack_checksum, pack.checksum);
750 assert_eq!(index.pack_checksum.format(), ObjectFormat::Sha256);
751 assert_eq!(index.index_checksum.format(), ObjectFormat::Sha256);
752 assert_eq!(
753 index
754 .find(&oid)
755 .expect("test operation should succeed")
756 .offset,
757 12
758 );
759 }
760
761 #[test]
762 fn pack_index_view_matches_owned_index_for_generated_sha256_pack() {
763 let objects = (0..4)
764 .map(|idx| {
765 EncodedObject::new(
766 ObjectType::Blob,
767 format!("borrowed pack index view sha256 object {idx}\n").into_bytes(),
768 )
769 })
770 .collect::<Vec<_>>();
771 let written = PackFile::write_undeltified(&objects, ObjectFormat::Sha256)
772 .expect("test operation should succeed");
773
774 assert_pack_index_view_matches_owned(&written.index, ObjectFormat::Sha256);
775 }
776
777 #[test]
778 fn indexes_existing_sha256_pack_bytes() {
779 let object = EncodedObject::new(ObjectType::Blob, b"index raw sha256 pack\n".to_vec());
780 let written =
781 PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha256)
782 .expect("test operation should succeed");
783
784 let indexed = PackIndex::write_v2_for_pack(&written.pack, ObjectFormat::Sha256)
785 .expect("test operation should succeed");
786 let index = PackIndex::parse(&indexed.index, ObjectFormat::Sha256)
787 .expect("test operation should succeed");
788
789 assert_eq!(indexed.pack_checksum, written.checksum);
790 assert_eq!(indexed.entries, written.entries);
791 assert_eq!(index.pack_checksum, written.checksum);
792 assert_eq!(index.entries, written.entries);
793 }
794
795 #[test]
796 fn indexes_existing_delta_pack_bytes() {
797 let (base, changed) = similar_blob_objects();
798 let options = delta_pack_options(true);
799 let written = PackFile::write_packed_with_options(
800 &[base, changed.clone()],
801 ObjectFormat::Sha1,
802 &options,
803 )
804 .expect("test operation should succeed");
805
806 let indexed = PackIndex::write_v2_for_pack_sha1(&written.pack)
807 .expect("test operation should succeed");
808 let index =
809 PackIndex::parse_v2_sha1(&indexed.index).expect("test operation should succeed");
810 let changed_oid = changed
811 .object_id(ObjectFormat::Sha1)
812 .expect("test operation should succeed");
813
814 assert_eq!(indexed.pack_checksum, written.checksum);
815 assert_eq!(indexed.entries, written.entries);
816 assert_eq!(
817 index
818 .find(&changed_oid)
819 .expect("test operation should succeed")
820 .offset,
821 written.entries[1].offset
822 );
823 assert_eq!(
824 index
825 .find(&changed_oid)
826 .expect("test operation should succeed")
827 .crc32,
828 written.entries[1].crc32
829 );
830 }
831
832 #[test]
833 fn writes_ref_delta_pack_and_index_that_round_trip() {
834 let (base, changed) = similar_blob_objects();
835 let options = delta_pack_options(false);
836 let written = PackFile::write_packed_with_options(
837 &[base.clone(), changed.clone()],
838 ObjectFormat::Sha1,
839 &options,
840 )
841 .expect("test operation should succeed");
842 let mut second_offset = written.entries[1].offset as usize;
843 let header = parse_entry_header(&written.pack, &mut second_offset)
844 .expect("test operation should succeed");
845 assert_eq!(header.kind, PackObjectKind::RefDelta);
846
847 let pack = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
848 let index =
849 PackIndex::parse_v2_sha1(&written.index).expect("test operation should succeed");
850 let oid = changed
851 .object_id(ObjectFormat::Sha1)
852 .expect("test operation should succeed");
853 assert_eq!(pack.entries[0].object, base);
854 assert_eq!(pack.entries[1].object, changed);
855 assert_eq!(index.pack_checksum, pack.checksum);
856 assert_eq!(
857 index
858 .find(&oid)
859 .expect("test operation should succeed")
860 .offset,
861 written.entries[1].offset
862 );
863 }
864
865 #[test]
866 fn read_object_at_matches_full_parse_for_ofs_delta_pack() {
867 let (base, changed) = similar_blob_objects();
868 let options = delta_pack_options(true);
869 let written = PackFile::write_packed_with_options(
870 &[base, changed.clone()],
871 ObjectFormat::Sha1,
872 &options,
873 )
874 .expect("test operation should succeed");
875 let mut second = written.entries[1].offset as usize;
877 assert_eq!(
878 parse_entry_header(&written.pack, &mut second)
879 .expect("test operation should succeed")
880 .kind,
881 PackObjectKind::OfsDelta
882 );
883 let parsed = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
885 for po in &parsed.entries {
886 let got =
887 read_object_at_arc(&written.pack, po.entry.offset, ObjectFormat::Sha1, |_| {
888 Ok(None)
889 })
890 .expect("test operation should succeed");
891 assert_eq!(*got, po.object, "offset {}", po.entry.offset);
892 }
893 }
894
895 #[derive(Default)]
898 struct MapHeaderTypeCache(HashMap<u64, (ObjectType, u64)>);
899
900 impl HeaderTypeCache for MapHeaderTypeCache {
901 fn get(&self, pack_offset: u64) -> Option<(ObjectType, u64)> {
902 self.0.get(&pack_offset).copied()
903 }
904 fn put(&mut self, pack_offset: u64, header: (ObjectType, u64)) {
905 self.0.insert(pack_offset, header);
906 }
907 }
908
909 #[test]
910 fn read_object_header_at_cached_matches_uncached_cold_and_warm_for_ofs_delta() {
911 let (base, changed) = similar_blob_objects();
912 let options = delta_pack_options(true);
913 let written =
914 PackFile::write_packed_with_options(&[base, changed], ObjectFormat::Sha1, &options)
915 .expect("test operation should succeed");
916 let mut second = written.entries[1].offset as usize;
918 assert_eq!(
919 parse_entry_header(&written.pack, &mut second)
920 .expect("test operation should succeed")
921 .kind,
922 PackObjectKind::OfsDelta
923 );
924
925 let parsed = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
926 let mut cache = MapHeaderTypeCache::default();
927 for po in &parsed.entries {
928 let uncached =
929 read_object_header_at(&written.pack, po.entry.offset, ObjectFormat::Sha1, |_| {
930 Ok(None)
931 })
932 .expect("test operation should succeed");
933 assert_eq!(
935 uncached,
936 (po.object.object_type, po.object.body.len() as u64),
937 "uncached header at offset {}",
938 po.entry.offset
939 );
940 let cold = read_object_header_at_with_cache(
942 &written.pack,
943 po.entry.offset,
944 ObjectFormat::Sha1,
945 |_| Ok(None),
946 &mut cache,
947 )
948 .expect("test operation should succeed");
949 assert_eq!(cold, uncached, "cold cache at offset {}", po.entry.offset);
950 }
951 for po in &parsed.entries {
954 let warm = read_object_header_at_with_cache(
955 &written.pack,
956 po.entry.offset,
957 ObjectFormat::Sha1,
958 |_| panic!("warm cache must not re-walk the chain"),
959 &mut cache,
960 )
961 .expect("test operation should succeed");
962 assert_eq!(
963 warm,
964 (po.object.object_type, po.object.body.len() as u64),
965 "warm cache at offset {}",
966 po.entry.offset
967 );
968 }
969 }
970
971 #[test]
972 fn read_object_at_matches_full_parse_for_ref_delta_pack() {
973 let (base, changed) = similar_blob_objects();
974 let options = delta_pack_options(false);
975 let written = PackFile::write_packed_with_options(
976 &[base, changed.clone()],
977 ObjectFormat::Sha1,
978 &options,
979 )
980 .expect("test operation should succeed");
981 let parsed = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
982 let by_oid: HashMap<ObjectId, Arc<EncodedObject>> = parsed
983 .entries
984 .iter()
985 .map(|po| (po.entry.oid, Arc::new(po.object.clone())))
986 .collect();
987 for po in &parsed.entries {
988 let got =
989 read_object_at_arc(&written.pack, po.entry.offset, ObjectFormat::Sha1, |oid| {
990 Ok(by_oid.get(oid).cloned())
991 })
992 .expect("test operation should succeed");
993 assert_eq!(*got, po.object);
994 }
995 }
996
997 #[derive(Default)]
1001 struct CountingDeltaCache {
1002 map: std::cell::RefCell<HashMap<u64, Arc<EncodedObject>>>,
1003 hits: std::cell::Cell<usize>,
1004 inserts: std::cell::Cell<usize>,
1005 }
1006
1007 impl PackDeltaCache for CountingDeltaCache {
1008 fn get(&self, offset: u64) -> Option<Arc<EncodedObject>> {
1009 let hit = self.map.borrow().get(&offset).cloned();
1010 if hit.is_some() {
1011 self.hits.set(self.hits.get() + 1);
1012 }
1013 hit
1014 }
1015 fn insert(&self, offset: u64, object: Arc<EncodedObject>) {
1016 self.inserts.set(self.inserts.get() + 1);
1017 self.map.borrow_mut().insert(offset, object);
1018 }
1019 }
1020
1021 #[test]
1022 fn read_object_at_with_cache_matches_uncached_and_reuses_bases() {
1023 let mut objects = Vec::new();
1026 for idx in 0..8u32 {
1027 let mut body = vec![b'x'; 4096];
1028 body.extend_from_slice(format!("\nvariant {idx}\n").as_bytes());
1029 objects.push(EncodedObject::new(ObjectType::Blob, body));
1030 }
1031 let options = delta_pack_options(true);
1032 let written = PackFile::write_packed_with_options(&objects, ObjectFormat::Sha1, &options)
1033 .expect("test operation should succeed");
1034 let parsed = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
1035
1036 let cache = CountingDeltaCache::default();
1037 for _ in 0..2 {
1040 for po in &parsed.entries {
1041 let got = read_object_at_with_cache_arc(
1042 &written.pack,
1043 po.entry.offset,
1044 ObjectFormat::Sha1,
1045 |_| Ok(None),
1046 &cache,
1047 )
1048 .expect("test operation should succeed");
1049 assert_eq!(*got, po.object, "offset {}", po.entry.offset);
1050 }
1051 }
1052 assert!(cache.hits.get() > 0, "cache never served a warm object");
1055 }
1056
1057 #[test]
1058 fn writes_ofs_delta_pack_and_index_that_round_trip() {
1059 let (base, changed) = similar_blob_objects();
1060 let options = delta_pack_options(true);
1061 let written = PackFile::write_packed_with_options(
1062 &[base.clone(), changed.clone()],
1063 ObjectFormat::Sha1,
1064 &options,
1065 )
1066 .expect("test operation should succeed");
1067 let mut second_offset = written.entries[1].offset as usize;
1068 let header = parse_entry_header(&written.pack, &mut second_offset)
1069 .expect("test operation should succeed");
1070 assert_eq!(header.kind, PackObjectKind::OfsDelta);
1071
1072 let pack = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
1073 let index =
1074 PackIndex::parse_v2_sha1(&written.index).expect("test operation should succeed");
1075 let oid = changed
1076 .object_id(ObjectFormat::Sha1)
1077 .expect("test operation should succeed");
1078 assert_eq!(pack.entries[0].object, base);
1079 assert_eq!(pack.entries[1].object, changed);
1080 assert_eq!(index.pack_checksum, pack.checksum);
1081 assert_eq!(
1082 index
1083 .find(&oid)
1084 .expect("test operation should succeed")
1085 .offset,
1086 written.entries[1].offset
1087 );
1088 }
1089
1090 #[test]
1091 fn resolves_ofs_delta_pack_entry() {
1092 let base = b"hello";
1093 let result = b"hello world";
1094 let pack = two_object_delta_pack(ObjectFormat::Sha1, base, result, DeltaKind::Offset);
1095 let parsed = PackFile::parse_sha1(&pack).expect("test operation should succeed");
1096 assert_eq!(parsed.entries.len(), 2);
1097 assert_eq!(parsed.entries[0].object.body, base);
1098 assert_eq!(parsed.entries[1].object.body, result);
1099 assert_eq!(
1100 parsed.entries[1].entry.oid,
1101 sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", result)
1102 .expect("test operation should succeed")
1103 );
1104 }
1105
1106 #[test]
1107 fn resolves_ref_delta_pack_entry() {
1108 let base = b"hello";
1109 let result = b"hello world";
1110 let pack = two_object_delta_pack(ObjectFormat::Sha1, base, result, DeltaKind::Ref);
1111 let parsed = PackFile::parse_sha1(&pack).expect("test operation should succeed");
1112 assert_eq!(parsed.entries.len(), 2);
1113 assert_eq!(parsed.entries[0].object.body, base);
1114 assert_eq!(parsed.entries[1].object.body, result);
1115 assert_eq!(
1116 parsed.entries[1].entry.oid,
1117 sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", result)
1118 .expect("test operation should succeed")
1119 );
1120 }
1121
1122 #[test]
1123 fn resolves_thin_ref_delta_pack_entry_with_external_base() {
1124 let base = b"hello";
1125 let result = b"hello world";
1126 let pack = thin_ref_delta_pack(ObjectFormat::Sha1, base, result);
1127 assert!(PackFile::parse_sha1(&pack).is_err());
1128
1129 let base_oid = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", base)
1130 .expect("test operation should succeed");
1131 let parsed = PackFile::parse_thin(&pack, ObjectFormat::Sha1, |oid| {
1132 if oid == &base_oid {
1133 Ok(Some(EncodedObject::new(ObjectType::Blob, base.to_vec())))
1134 } else {
1135 Ok(None)
1136 }
1137 })
1138 .expect("test operation should succeed");
1139 assert_eq!(parsed.entries.len(), 1);
1140 assert_eq!(parsed.entries[0].object.body, result);
1141 assert_eq!(
1142 parsed.entries[0].entry.oid,
1143 sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", result)
1144 .expect("test operation should succeed")
1145 );
1146 }
1147
1148 #[test]
1149 fn fixes_thin_pack_and_indexes_it_without_external_bases() {
1150 for format in [ObjectFormat::Sha1, ObjectFormat::Sha256] {
1151 let base = b"hello";
1152 let result = b"hello world";
1153 let pack = thin_ref_delta_pack(format, base, result);
1154 let base_object = EncodedObject::new(ObjectType::Blob, base.to_vec());
1155 let base_oid = base_object
1156 .object_id(format)
1157 .expect("test operation should succeed");
1158
1159 let fixed = fix_thin_pack(&pack, format, |oid| {
1160 Ok((oid == &base_oid).then(|| base_object.clone()))
1161 })
1162 .expect("thin pack completion should succeed");
1163
1164 assert_eq!(u32_be(&fixed.pack[8..12]), 2);
1165 assert_eq!(fixed.appended_bases, vec![base_oid]);
1166 assert_eq!(fixed.index.entries.len(), 2);
1167 let parsed =
1168 PackFile::parse(&fixed.pack, format).expect("completed pack should stand alone");
1169 assert_eq!(parsed.entries.len(), 2);
1170 assert_eq!(parsed.entries[0].object.body, result);
1171 assert_eq!(parsed.entries[1].object, base_object);
1172 let rebuilt = PackIndex::write_v2_for_pack(&fixed.pack, format)
1173 .expect("completed pack should index without a resolver");
1174 assert_eq!(rebuilt, fixed.index);
1175 }
1176 }
1177
1178 #[test]
1179 fn fix_thin_is_idempotent_for_a_complete_pack() {
1180 let original = single_object_pack(ObjectFormat::Sha1, ObjectType::Blob, b"complete\n");
1181 let mut resolver_called = false;
1182 let fixed = fix_thin_pack(&original, ObjectFormat::Sha1, |_| {
1183 resolver_called = true;
1184 Ok(None)
1185 })
1186 .expect("complete pack should remain complete");
1187
1188 assert_eq!(fixed.pack, original);
1189 assert!(fixed.appended_bases.is_empty());
1190 assert!(!resolver_called);
1191 assert_eq!(fixed.index.entries.len(), 1);
1192 }
1193
1194 #[test]
1195 fn fix_thin_does_not_duplicate_a_base_already_in_the_body() {
1196 let pack =
1197 two_object_delta_pack(ObjectFormat::Sha1, b"hello", b"hello world", DeltaKind::Ref);
1198 let original_count = u32_be(&pack[8..12]);
1199 let mut resolver_called = false;
1200 let fixed = fix_thin_pack(&pack, ObjectFormat::Sha1, |_| {
1201 resolver_called = true;
1202 Ok(None)
1203 })
1204 .expect("in-pack ref base should remain self-contained");
1205
1206 assert_eq!(fixed.pack, pack);
1207 assert_eq!(u32_be(&fixed.pack[8..12]), original_count);
1208 assert_eq!(fixed.index.entries.len(), original_count as usize);
1209 assert!(fixed.appended_bases.is_empty());
1210 assert!(!resolver_called);
1211 }
1212
1213 #[test]
1214 fn fix_thin_appends_a_repeated_external_base_once() {
1215 let base = b"shared";
1216 let pack =
1217 thin_ref_delta_pack_many(ObjectFormat::Sha1, base, &[b"shared one", b"shared two"]);
1218 let base_object = EncodedObject::new(ObjectType::Blob, base.to_vec());
1219 let base_oid = base_object
1220 .object_id(ObjectFormat::Sha1)
1221 .expect("test operation should succeed");
1222 let mut calls = 0;
1223 let fixed = fix_thin_pack(&pack, ObjectFormat::Sha1, |oid| {
1224 calls += 1;
1225 Ok((oid == &base_oid).then(|| base_object.clone()))
1226 })
1227 .expect("shared base completion should succeed");
1228
1229 assert_eq!(calls, 1);
1230 assert_eq!(fixed.appended_bases, vec![base_oid]);
1231 assert_eq!(u32_be(&fixed.pack[8..12]), 3);
1232 assert_eq!(fixed.index.entries.len(), 3);
1233 }
1234
1235 #[test]
1236 fn fix_thin_rejects_missing_and_wrong_external_bases() {
1237 let base = b"hello";
1238 let pack = thin_ref_delta_pack(ObjectFormat::Sha1, base, b"hello world");
1239 assert!(fix_thin_pack(&pack, ObjectFormat::Sha1, |_| Ok(None)).is_err());
1240
1241 let wrong = EncodedObject::new(ObjectType::Blob, b"wrong".to_vec());
1242 let error = fix_thin_pack(&pack, ObjectFormat::Sha1, |_| Ok(Some(wrong.clone())))
1243 .expect_err("wrong base body must not be accepted for the requested oid");
1244 assert!(error.to_string().contains("resolved to object"));
1245 }
1246
1247 #[test]
1248 fn fix_thin_preserves_the_delta_depth_limit_boundary() {
1249 let base = b"hello";
1250 let pack = thin_ref_delta_pack(ObjectFormat::Sha1, base, b"hello world");
1251 let base_object = EncodedObject::new(ObjectType::Blob, base.to_vec());
1252 let base_oid = base_object
1253 .object_id(ObjectFormat::Sha1)
1254 .expect("test operation should succeed");
1255 let resolve = |oid: &ObjectId| Ok((oid == &base_oid).then(|| base_object.clone()));
1256
1257 fix_thin_pack_with_limits(
1258 &pack,
1259 ObjectFormat::Sha1,
1260 resolve,
1261 PackReadLimits {
1262 max_delta_depth: 1,
1263 ..PackReadLimits::default()
1264 },
1265 )
1266 .expect("one delta must pass at the exact limit");
1267
1268 let error = fix_thin_pack_with_limits(
1269 &pack,
1270 ObjectFormat::Sha1,
1271 resolve,
1272 PackReadLimits {
1273 max_delta_depth: 0,
1274 ..PackReadLimits::default()
1275 },
1276 )
1277 .expect_err("one delta must fail immediately above the configured limit");
1278 assert!(error.to_string().contains("observed depth 1"));
1279 }
1280
1281 #[test]
1282 fn streaming_indexer_resolves_thin_bases_but_rejects_missing_ones() {
1283 let base = b"hello";
1284 let result = b"hello stream";
1285 let pack = thin_ref_delta_pack(ObjectFormat::Sha1, base, result);
1286 let base_object = EncodedObject::new(ObjectType::Blob, base.to_vec());
1287 let base_oid = base_object
1288 .object_id(ObjectFormat::Sha1)
1289 .expect("test operation should succeed");
1290
1291 let mut reader = Cursor::new(&pack);
1292 let built =
1293 PackIndex::write_v2_for_pack_reader_with_base(&mut reader, ObjectFormat::Sha1, |oid| {
1294 Ok((oid == &base_oid).then(|| base_object.clone()))
1295 })
1296 .expect("streaming indexer should resolve the supplied base");
1297 assert_eq!(built.entries.len(), 1);
1298 assert_eq!(
1299 built.entries[0].oid,
1300 sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", result)
1301 .expect("test operation should succeed")
1302 );
1303
1304 let mut missing_reader = Cursor::new(&pack);
1305 let error = PackIndex::write_v2_for_pack_reader_with_base(
1306 &mut missing_reader,
1307 ObjectFormat::Sha1,
1308 |_| Ok(None),
1309 )
1310 .expect_err("an unresolved streamed base must remain an error");
1311 assert!(error.to_string().contains("unresolved delta base"));
1312 }
1313
1314 #[test]
1315 fn fixed_pack_passes_upstream_git_and_supports_clone() {
1316 let format = ObjectFormat::Sha1;
1317 let base = EncodedObject::new(
1318 ObjectType::Blob,
1319 b"the original contents are deliberately similar\n".repeat(32),
1320 );
1321 let mut target_body = base.body.clone();
1322 target_body.extend_from_slice(b"completed thin pack\n");
1323 let target = EncodedObject::new(ObjectType::Blob, target_body);
1324 let target_oid = target
1325 .object_id(format)
1326 .expect("test operation should succeed");
1327
1328 let mut tree_body = b"100644 payload\0".to_vec();
1329 tree_body.extend_from_slice(target_oid.as_bytes());
1330 let tree = EncodedObject::new(ObjectType::Tree, tree_body);
1331 let tree_oid = tree
1332 .object_id(format)
1333 .expect("test operation should succeed");
1334 let commit = EncodedObject::new(
1335 ObjectType::Commit,
1336 format!(
1337 "tree {tree_oid}\nauthor Sley <sley@example.com> 1 +0000\n\
1338 committer Sley <sley@example.com> 1 +0000\n\nthin pack fixture\n"
1339 )
1340 .into_bytes(),
1341 );
1342 let commit_oid = commit
1343 .object_id(format)
1344 .expect("test operation should succeed");
1345 let base_oid = base
1346 .object_id(format)
1347 .expect("test operation should succeed");
1348 let thin = PackFile::write_thin(
1349 &[commit, tree, target],
1350 format,
1351 HashMap::from([(base_oid, base.clone())]),
1352 )
1353 .expect("write thin pack");
1354 assert!(thin.delta_count > 0, "fixture must really be thin");
1355 assert!(PackIndex::write_v2_for_pack(&thin.pack, format).is_err());
1356
1357 let fixed = fix_thin_pack(&thin.pack, format, |oid| {
1358 Ok((oid == &base_oid).then(|| base.clone()))
1359 })
1360 .expect("fix thin pack");
1361 let root = unique_temp_dir("fix-thin-git");
1362 let strict_pack = root.join("strict.pack");
1363 fs::create_dir_all(&root).expect("create test root");
1364 fs::write(&strict_pack, &fixed.pack).expect("write strict pack");
1365 run_git_success(&root, &["index-pack", "--strict", "strict.pack"]);
1366
1367 let complete_repo = root.join("complete.git");
1368 run_git_success(
1369 &root,
1370 &[
1371 "init",
1372 "--bare",
1373 "-q",
1374 complete_repo.to_str().expect("utf8 complete repo path"),
1375 ],
1376 );
1377 let complete_pack_dir = complete_repo.join("objects/pack");
1378 let complete_stem = format!("pack-{}", fixed.index.pack_checksum);
1379 fs::write(
1380 complete_pack_dir.join(format!("{complete_stem}.pack")),
1381 &fixed.pack,
1382 )
1383 .expect("write completed repository pack");
1384 fs::write(
1385 complete_pack_dir.join(format!("{complete_stem}.idx")),
1386 &fixed.index.index,
1387 )
1388 .expect("write completed repository index");
1389 fs::create_dir_all(complete_repo.join("refs/heads")).expect("create refs");
1390 fs::write(
1391 complete_repo.join("refs/heads/main"),
1392 format!("{commit_oid}\n"),
1393 )
1394 .expect("write main ref");
1395 fs::write(complete_repo.join("HEAD"), b"ref: refs/heads/main\n").expect("write HEAD");
1396 run_git_success(&complete_repo, &["fsck", "--full"]);
1397
1398 let clone_path = root.join("clone");
1399 run_git_success(
1400 &root,
1401 &[
1402 "clone",
1403 "--no-local",
1404 "-q",
1405 complete_repo.to_str().expect("utf8 complete repo path"),
1406 clone_path.to_str().expect("utf8 clone path"),
1407 ],
1408 );
1409 run_git_success(&clone_path, &["fsck", "--full"]);
1410
1411 let thin_repo = root.join("thin.git");
1412 run_git_success(
1413 &root,
1414 &[
1415 "init",
1416 "--bare",
1417 "-q",
1418 thin_repo.to_str().expect("utf8 thin repo path"),
1419 ],
1420 );
1421 let thin_index = PackIndex::write_v2_for_pack_with_base(&thin.pack, format, |oid| {
1422 Ok((oid == &base_oid).then(|| base.clone()))
1423 })
1424 .expect("sley can build the deliberately permissive thin index");
1425 let thin_pack_dir = thin_repo.join("objects/pack");
1426 let thin_stem = format!("pack-{}", thin_index.pack_checksum);
1427 fs::write(thin_pack_dir.join(format!("{thin_stem}.pack")), &thin.pack)
1428 .expect("write thin repository pack");
1429 fs::write(
1430 thin_pack_dir.join(format!("{thin_stem}.idx")),
1431 &thin_index.index,
1432 )
1433 .expect("write thin repository index");
1434 fs::create_dir_all(thin_repo.join("refs/heads")).expect("create thin refs");
1435 fs::write(thin_repo.join("refs/heads/main"), format!("{commit_oid}\n"))
1436 .expect("write thin main ref");
1437 fs::write(thin_repo.join("HEAD"), b"ref: refs/heads/main\n").expect("write thin HEAD");
1438 let broken_clone_path = root.join("broken-clone");
1439 let failed_clone = Command::new("git")
1440 .current_dir(&root)
1441 .args([
1442 "clone",
1443 "--no-local",
1444 "-q",
1445 thin_repo.to_str().expect("utf8 thin repo path"),
1446 broken_clone_path.to_str().expect("utf8 broken clone path"),
1447 ])
1448 .output()
1449 .expect("run failing clone");
1450 assert!(
1451 !failed_clone.status.success(),
1452 "clone from uncompleted thin pack unexpectedly succeeded"
1453 );
1454
1455 fs::remove_dir_all(root).expect("remove git interoperability fixture");
1456 }
1457
1458 #[test]
1459 fn rejects_bad_pack_checksum() {
1460 let mut pack = single_object_pack(ObjectFormat::Sha1, ObjectType::Blob, b"hello\n");
1461 let last = pack.len() - 1;
1462 pack[last] ^= 1;
1463 assert!(PackFile::parse_sha1(&pack).is_err());
1464 }
1465
1466 #[test]
1467 fn raw_pack_index_rejects_bad_pack_checksum() {
1468 let mut pack = single_object_pack(ObjectFormat::Sha1, ObjectType::Blob, b"hello\n");
1469 let last = pack.len() - 1;
1470 pack[last] ^= 1;
1471 assert!(PackIndex::write_v2_for_pack_sha1(&pack).is_err());
1472 }
1473
1474 #[test]
1475 fn pack_index_writer_preserves_duplicate_object_ids() {
1476 let oid = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"same\n")
1477 .expect("test operation should succeed");
1478 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1479 .expect("test operation should succeed");
1480 let entries = vec![
1481 PackIndexEntry {
1482 oid,
1483 crc32: 1,
1484 offset: 12,
1485 },
1486 PackIndexEntry {
1487 oid,
1488 crc32: 2,
1489 offset: 24,
1490 },
1491 ];
1492 let index = PackIndex::write_v2(ObjectFormat::Sha1, &entries, &pack_checksum)
1493 .expect("duplicate objects are valid in a pack index");
1494 let parsed = PackIndex::parse(&index, ObjectFormat::Sha1)
1495 .expect("duplicate-object pack index should parse");
1496 assert_eq!(parsed.entries.len(), 2);
1497 assert!(parsed.entries.iter().all(|entry| entry.oid == oid));
1498 assert!(parsed.find(&oid).is_some());
1499 }
1500
1501 #[test]
1502 fn parses_single_entry_pack_index() {
1503 let oid = ObjectId::from_hex(
1504 ObjectFormat::Sha1,
1505 "ce013625030ba8dba906f756967f9e9ca394464a",
1506 )
1507 .expect("test operation should succeed");
1508 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1509 .expect("test operation should succeed");
1510 let index = single_entry_index(
1511 ObjectFormat::Sha1,
1512 oid,
1513 0x1234_5678,
1514 12,
1515 pack_checksum.clone(),
1516 );
1517 let parsed = PackIndex::parse_v2_sha1(&index).expect("test operation should succeed");
1518 assert_eq!(parsed.version, 2);
1519 assert_eq!(parsed.pack_checksum, pack_checksum);
1520 assert_eq!(parsed.entries.len(), 1);
1521 assert_eq!(
1522 parsed
1523 .find(&oid)
1524 .expect("test operation should succeed")
1525 .offset,
1526 12
1527 );
1528 assert_eq!(
1529 parsed
1530 .find(&oid)
1531 .expect("test operation should succeed")
1532 .crc32,
1533 0x1234_5678
1534 );
1535 assert_pack_index_view_matches_owned(&index, ObjectFormat::Sha1);
1536 }
1537
1538 #[test]
1539 fn parses_single_entry_pack_index_v1() {
1540 let oid = ObjectId::from_hex(
1541 ObjectFormat::Sha1,
1542 "ce013625030ba8dba906f756967f9e9ca394464a",
1543 )
1544 .expect("test operation should succeed");
1545 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1546 .expect("test operation should succeed");
1547 let index =
1548 single_entry_index_v1(ObjectFormat::Sha1, oid, 0x1234_5678, pack_checksum.clone());
1549 let parsed =
1550 PackIndex::parse(&index, ObjectFormat::Sha1).expect("test operation should succeed");
1551 assert_eq!(parsed.version, 1);
1552 assert_eq!(parsed.pack_checksum, pack_checksum);
1553 assert_eq!(parsed.entries.len(), 1);
1554 assert_eq!(
1555 parsed
1556 .find(&oid)
1557 .expect("test operation should succeed")
1558 .offset,
1559 0x1234_5678
1560 );
1561 assert_eq!(
1562 parsed
1563 .find(&oid)
1564 .expect("test operation should succeed")
1565 .crc32,
1566 0
1567 );
1568 assert_pack_index_view_matches_owned(&index, ObjectFormat::Sha1);
1569 }
1570
1571 #[test]
1572 fn rejects_bad_pack_index_v1_checksum() {
1573 let oid = ObjectId::from_hex(
1574 ObjectFormat::Sha1,
1575 "ce013625030ba8dba906f756967f9e9ca394464a",
1576 )
1577 .expect("test operation should succeed");
1578 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1579 .expect("test operation should succeed");
1580 let mut index = single_entry_index_v1(ObjectFormat::Sha1, oid, 12, pack_checksum);
1581 let last = index.len() - 1;
1582 index[last] ^= 1;
1583 assert!(PackIndex::parse(&index, ObjectFormat::Sha1).is_err());
1584 }
1585
1586 #[test]
1587 fn pack_index_view_reads_v2_large_offsets() {
1588 let first = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"large offset a\n")
1589 .expect("test operation should succeed");
1590 let second =
1591 sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"large offset b\n")
1592 .expect("test operation should succeed");
1593 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1594 .expect("test operation should succeed");
1595 let entries = vec![
1596 PackIndexEntry {
1597 oid: first,
1598 crc32: 0x1111_2222,
1599 offset: 0x8000_0000,
1600 },
1601 PackIndexEntry {
1602 oid: second,
1603 crc32: 0x3333_4444,
1604 offset: 0x1_0000_0042,
1605 },
1606 ];
1607 let index = PackIndex::write_v2(ObjectFormat::Sha1, &entries, &pack_checksum)
1608 .expect("test operation should succeed");
1609
1610 assert_pack_index_view_matches_owned(&index, ObjectFormat::Sha1);
1611 let view = PackIndexView::parse(&index, ObjectFormat::Sha1)
1612 .expect("test operation should succeed");
1613 for entry in entries {
1614 assert_eq!(
1615 view.find(&entry.oid),
1616 Some(PackIndexLookup {
1617 crc32: entry.crc32,
1618 offset: entry.offset,
1619 })
1620 );
1621 }
1622 }
1623
1624 #[test]
1625 fn pack_index_view_default_parse_checks_index_checksum() {
1626 let oid = ObjectId::from_hex(
1627 ObjectFormat::Sha1,
1628 "ce013625030ba8dba906f756967f9e9ca394464a",
1629 )
1630 .expect("test operation should succeed");
1631 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1632 .expect("test operation should succeed");
1633 let mut index = single_entry_index(ObjectFormat::Sha1, oid, 0x1234_5678, 12, pack_checksum);
1634 let last = index.len() - 1;
1635 index[last] ^= 1;
1636
1637 assert!(PackIndexView::parse(&index, ObjectFormat::Sha1).is_err());
1638 let view = PackIndexView::parse_without_checksum(&index, ObjectFormat::Sha1)
1639 .expect("test operation should succeed");
1640 let trusted_view = PackIndexViewData::parse_trusted_without_checksum(
1641 Arc::from(index.clone().into_boxed_slice()),
1642 ObjectFormat::Sha1,
1643 )
1644 .expect("test operation should succeed");
1645 assert_eq!(
1646 view.find(&oid),
1647 Some(PackIndexLookup {
1648 crc32: 0x1234_5678,
1649 offset: 12,
1650 })
1651 );
1652 assert_eq!(
1653 trusted_view.find(&oid),
1654 Some(PackIndexLookup {
1655 crc32: 0x1234_5678,
1656 offset: 12,
1657 })
1658 );
1659 }
1660
1661 #[test]
1662 fn reverse_index_resolves_oid_at_offset() {
1663 let objects = (0..3)
1664 .map(|idx| {
1665 EncodedObject::new(
1666 ObjectType::Blob,
1667 format!("reverse index lookup object {idx}\n").into_bytes(),
1668 )
1669 })
1670 .collect::<Vec<_>>();
1671 let written = PackFile::write_packed(&objects, ObjectFormat::Sha1)
1672 .expect("test operation should succeed");
1673 let index = PackIndex::parse(&written.index, ObjectFormat::Sha1)
1674 .expect("test operation should succeed");
1675 let view = PackIndexViewData::parse_trusted_without_checksum(
1676 Arc::from(written.index.clone().into_boxed_slice()),
1677 ObjectFormat::Sha1,
1678 )
1679 .expect("test operation should succeed");
1680 let positions = pack_order_index_positions(&index.entries);
1681 let reverse = PackReverseIndex::parse(
1682 &PackReverseIndex::write(ObjectFormat::Sha1, &positions, &index.pack_checksum)
1683 .expect("test operation should succeed"),
1684 ObjectFormat::Sha1,
1685 index.entries.len(),
1686 )
1687 .expect("test operation should succeed");
1688
1689 for entry in &index.entries {
1690 assert_eq!(
1691 reverse
1692 .oid_at_offset(&view, entry.offset)
1693 .expect("test operation should succeed"),
1694 entry.oid
1695 );
1696 }
1697 assert!(reverse.oid_at_offset(&view, 999).is_none());
1698 }
1699
1700 #[test]
1701 fn parses_pack_reverse_index() {
1702 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1703 .expect("test operation should succeed");
1704 let reverse_index = PackReverseIndex::write(ObjectFormat::Sha1, &[2, 0, 1], &pack_checksum)
1705 .expect("test operation should succeed");
1706 let parsed = PackReverseIndex::parse(&reverse_index, ObjectFormat::Sha1, 3)
1707 .expect("test operation should succeed");
1708 assert_eq!(parsed.version, 1);
1709 assert_eq!(parsed.format, ObjectFormat::Sha1);
1710 assert_eq!(parsed.positions, vec![2, 0, 1]);
1711 assert_eq!(parsed.pack_checksum, pack_checksum);
1712 assert_eq!(
1713 PackReverseIndex::write(ObjectFormat::Sha1, &parsed.positions, &parsed.pack_checksum)
1714 .expect("test operation should succeed"),
1715 reverse_index
1716 );
1717 }
1718
1719 #[test]
1720 fn rejects_bad_pack_reverse_index_checksum() {
1721 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1722 .expect("test operation should succeed");
1723 let mut reverse_index = PackReverseIndex::write(ObjectFormat::Sha1, &[0], &pack_checksum)
1724 .expect("test operation should succeed");
1725 let last = reverse_index.len() - 1;
1726 reverse_index[last] ^= 1;
1727 assert!(matches!(
1728 PackReverseIndex::parse(&reverse_index, ObjectFormat::Sha1, 1),
1729 Err(GitError::InvalidFormat(message)) if message == "invalid checksum"
1730 ));
1731 }
1732
1733 #[test]
1734 fn classifies_pack_reverse_index_corruption_for_fsck() {
1735 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1736 .expect("test operation should succeed");
1737 let valid = PackReverseIndex::write(ObjectFormat::Sha1, &[0], &pack_checksum)
1738 .expect("test operation should succeed");
1739 for (offset, value, expected) in [
1740 (1, 7, "unknown signature"),
1741 (7, 2, "unsupported version 2"),
1742 (11, 3, "unsupported hash id 3"),
1743 (14, 7, "invalid rev-index position 1792"),
1744 ] {
1745 let mut corrupt = valid.clone();
1746 corrupt[offset] = value;
1747 assert!(matches!(
1748 PackReverseIndex::parse(&corrupt, ObjectFormat::Sha1, 1),
1749 Err(GitError::InvalidFormat(message)) if message == expected
1750 ));
1751 }
1752 }
1753
1754 #[test]
1755 fn rejects_bad_pack_reverse_index_positions() {
1756 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1757 .expect("test operation should succeed");
1758 let duplicate = pack_reverse_index(ObjectFormat::Sha1, &[0, 0], pack_checksum.clone());
1759 assert!(PackReverseIndex::parse(&duplicate, ObjectFormat::Sha1, 2).is_err());
1760 let out_of_range = pack_reverse_index(ObjectFormat::Sha1, &[0, 2], pack_checksum);
1761 assert!(PackReverseIndex::parse(&out_of_range, ObjectFormat::Sha1, 2).is_err());
1762 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1763 .expect("test operation should succeed");
1764 assert!(PackReverseIndex::write(ObjectFormat::Sha1, &[0, 0], &pack_checksum).is_err());
1765 assert!(PackReverseIndex::write(ObjectFormat::Sha1, &[0, 2], &pack_checksum).is_err());
1766 }
1767
1768 #[test]
1769 fn parses_pack_mtimes() {
1770 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1771 .expect("test operation should succeed");
1772 let mtimes = PackMtimes::write(
1773 ObjectFormat::Sha1,
1774 &[1, 1_700_000_000, u32::MAX],
1775 &pack_checksum,
1776 )
1777 .expect("test operation should succeed");
1778 let parsed = PackMtimes::parse(&mtimes, ObjectFormat::Sha1, 3)
1779 .expect("test operation should succeed");
1780 assert_eq!(parsed.version, 1);
1781 assert_eq!(parsed.format, ObjectFormat::Sha1);
1782 assert_eq!(parsed.mtimes, vec![1, 1_700_000_000, u32::MAX]);
1783 assert_eq!(parsed.pack_checksum, pack_checksum);
1784 assert_eq!(
1785 PackMtimes::write(ObjectFormat::Sha1, &parsed.mtimes, &parsed.pack_checksum)
1786 .expect("test operation should succeed"),
1787 mtimes
1788 );
1789 }
1790
1791 #[test]
1792 fn rejects_bad_pack_mtimes_checksum() {
1793 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1794 .expect("test operation should succeed");
1795 let mut mtimes = PackMtimes::write(ObjectFormat::Sha1, &[1], &pack_checksum)
1796 .expect("test operation should succeed");
1797 let last = mtimes.len() - 1;
1798 mtimes[last] ^= 1;
1799 assert!(PackMtimes::parse(&mtimes, ObjectFormat::Sha1, 1).is_err());
1800 }
1801
1802 #[test]
1803 fn rejects_bad_pack_mtimes_shape() {
1804 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1805 .expect("test operation should succeed");
1806 let mtimes = pack_mtimes(ObjectFormat::Sha1, &[1, 2], pack_checksum.clone());
1807 assert!(PackMtimes::parse(&mtimes, ObjectFormat::Sha1, 1).is_err());
1808
1809 let mut wrong_hash = pack_mtimes(ObjectFormat::Sha1, &[1], pack_checksum);
1810 wrong_hash[11] = 2;
1811 let checksum_offset = wrong_hash.len() - ObjectFormat::Sha1.raw_len();
1812 let checksum = sley_core::digest_bytes(ObjectFormat::Sha1, &wrong_hash[..checksum_offset])
1813 .expect("test operation should succeed");
1814 wrong_hash[checksum_offset..].copy_from_slice(checksum.as_bytes());
1815 assert!(PackMtimes::parse(&wrong_hash, ObjectFormat::Sha1, 1).is_err());
1816 }
1817
1818 #[test]
1819 fn parses_multi_pack_index_header_and_chunk_lookup() {
1820 let first = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"first object\n")
1821 .expect("test operation should succeed");
1822 let second = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"second object\n")
1823 .expect("test operation should succeed");
1824 let chunks = midx_chunks_with_pack_names(
1825 ObjectFormat::Sha1,
1826 b"pack-a.idx\0pack-b.idx\0\0\0".to_vec(),
1827 &[(first.clone(), 0, 12), (second.clone(), 1, 0x1_0000_0000)],
1828 );
1829 let midx = multi_pack_index(ObjectFormat::Sha1, 2, 2, &chunks);
1830 let parsed = MultiPackIndex::parse(&midx, ObjectFormat::Sha1)
1831 .expect("test operation should succeed");
1832 assert_eq!(parsed.version, 2);
1833 assert_eq!(parsed.format, ObjectFormat::Sha1);
1834 assert_eq!(parsed.pack_count, 2);
1835 assert_eq!(parsed.pack_names, vec!["pack-a.idx", "pack-b.idx"]);
1836 assert_eq!(parsed.object_count, 2);
1837 assert_eq!(parsed.objects.len(), 2);
1838 assert_eq!(
1839 parsed
1840 .find(&first)
1841 .expect("test operation should succeed")
1842 .pack_int_id,
1843 0
1844 );
1845 assert_eq!(
1846 parsed
1847 .find(&first)
1848 .expect("test operation should succeed")
1849 .offset,
1850 12
1851 );
1852 assert_eq!(
1853 parsed
1854 .find(&second)
1855 .expect("test operation should succeed")
1856 .pack_int_id,
1857 1
1858 );
1859 assert_eq!(
1860 parsed
1861 .find(&second)
1862 .expect("test operation should succeed")
1863 .offset,
1864 0x1_0000_0000
1865 );
1866 assert_eq!(parsed.reverse_index, None);
1867 assert_eq!(parsed.bitmapped_packs, None);
1868 assert_eq!(parsed.chunks.len(), 5);
1869 assert_eq!(parsed.chunks[0].id, *b"PNAM");
1870 assert_eq!(parsed.chunks[0].offset, 84);
1871 assert_eq!(parsed.chunks[0].len, 24);
1872 assert_eq!(parsed.chunks[1].id, *b"OIDF");
1873 assert_eq!(parsed.chunks[1].offset, 108);
1874 assert_eq!(parsed.chunks[1].len, 1024);
1875 }
1876
1877 #[test]
1878 fn raw_multi_pack_index_lookup_finds_pack_and_offset() {
1879 let first = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"first object\n")
1880 .expect("test operation should succeed");
1881 let second = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"second object\n")
1882 .expect("test operation should succeed");
1883 let missing = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"missing\n")
1884 .expect("test operation should succeed");
1885 let chunks = midx_chunks_with_pack_names(
1886 ObjectFormat::Sha1,
1887 b"pack-a.idx\0pack-b.idx\0\0\0".to_vec(),
1888 &[(first.clone(), 0, 12), (second.clone(), 1, 0x1_0000_0000)],
1889 );
1890 let midx = Arc::new(multi_pack_index(ObjectFormat::Sha1, 2, 2, &chunks));
1891 let lookup = MultiPackIndexOidLookup::parse(midx, ObjectFormat::Sha1)
1892 .expect("test operation should succeed");
1893
1894 assert!(lookup.contains(&first));
1895 assert!(lookup.contains(&second));
1896 assert!(!lookup.contains(&missing));
1897
1898 let first_entry = lookup
1899 .find(&first)
1900 .expect("test operation should succeed")
1901 .expect("object should be present");
1902 assert_eq!(
1903 lookup.pack_name(first_entry.pack_int_id),
1904 Some("pack-a.idx")
1905 );
1906 assert_eq!(first_entry.offset, 12);
1907
1908 let second_entry = lookup
1909 .find(&second)
1910 .expect("test operation should succeed")
1911 .expect("object should be present");
1912 assert_eq!(
1913 lookup.pack_name(second_entry.pack_int_id),
1914 Some("pack-b.idx")
1915 );
1916 assert_eq!(second_entry.offset, 0x1_0000_0000);
1917 assert!(
1918 lookup
1919 .find(&missing)
1920 .expect("test operation should succeed")
1921 .is_none()
1922 );
1923 }
1924
1925 #[test]
1926 fn rejects_bad_multi_pack_index_checksum() {
1927 let chunks = midx_chunks_with_pack_names(ObjectFormat::Sha1, Vec::new(), &[]);
1928 let mut midx = multi_pack_index(ObjectFormat::Sha1, 1, 0, &chunks);
1929 let last = midx.len() - 1;
1930 midx[last] ^= 1;
1931 assert!(MultiPackIndex::parse(&midx, ObjectFormat::Sha1).is_err());
1932 }
1933
1934 #[test]
1935 fn rejects_bad_multi_pack_index_shape() {
1936 let chunks = midx_chunks_with_pack_names(ObjectFormat::Sha1, Vec::new(), &[]);
1937 let mut wrong_hash = multi_pack_index(ObjectFormat::Sha1, 1, 0, &chunks);
1938 wrong_hash[5] = 2;
1939 let checksum_offset = wrong_hash.len() - ObjectFormat::Sha1.raw_len();
1940 let checksum = sley_core::digest_bytes(ObjectFormat::Sha1, &wrong_hash[..checksum_offset])
1941 .expect("test operation should succeed");
1942 wrong_hash[checksum_offset..].copy_from_slice(checksum.as_bytes());
1943 assert!(MultiPackIndex::parse(&wrong_hash, ObjectFormat::Sha1).is_err());
1944
1945 let mut missing_terminator = multi_pack_index(ObjectFormat::Sha1, 1, 0, &chunks);
1946 missing_terminator[12] = b'B';
1947 let checksum_offset = missing_terminator.len() - ObjectFormat::Sha1.raw_len();
1948 let checksum =
1949 sley_core::digest_bytes(ObjectFormat::Sha1, &missing_terminator[..checksum_offset])
1950 .expect("test operation should succeed");
1951 missing_terminator[checksum_offset..].copy_from_slice(checksum.as_bytes());
1952 assert!(MultiPackIndex::parse(&missing_terminator, ObjectFormat::Sha1).is_err());
1953
1954 let mut bad_offset = multi_pack_index(
1955 ObjectFormat::Sha1,
1956 2,
1957 0,
1958 &midx_chunks_with_pack_names(ObjectFormat::Sha1, Vec::new(), &[]),
1959 );
1960 bad_offset[16..24].copy_from_slice(&0u64.to_be_bytes());
1961 let checksum_offset = bad_offset.len() - ObjectFormat::Sha1.raw_len();
1962 let checksum = sley_core::digest_bytes(ObjectFormat::Sha1, &bad_offset[..checksum_offset])
1963 .expect("test operation should succeed");
1964 bad_offset[checksum_offset..].copy_from_slice(checksum.as_bytes());
1965 assert!(MultiPackIndex::parse(&bad_offset, ObjectFormat::Sha1).is_err());
1966 }
1967
1968 #[test]
1969 fn rejects_bad_multi_pack_index_pack_names() {
1970 let missing = multi_pack_index(ObjectFormat::Sha1, 2, 1, &[]);
1971 assert!(MultiPackIndex::parse(&missing, ObjectFormat::Sha1).is_err());
1972
1973 let too_few = multi_pack_index(
1974 ObjectFormat::Sha1,
1975 2,
1976 2,
1977 &midx_chunks_with_pack_names(ObjectFormat::Sha1, b"pack-a.idx\0".to_vec(), &[]),
1978 );
1979 assert!(MultiPackIndex::parse(&too_few, ObjectFormat::Sha1).is_err());
1980
1981 let bad_padding = multi_pack_index(
1982 ObjectFormat::Sha1,
1983 2,
1984 1,
1985 &midx_chunks_with_pack_names(ObjectFormat::Sha1, b"pack-a.idx\0xxxx".to_vec(), &[]),
1986 );
1987 assert!(MultiPackIndex::parse(&bad_padding, ObjectFormat::Sha1).is_err());
1988
1989 let unsorted_v1 = multi_pack_index(
1990 ObjectFormat::Sha1,
1991 1,
1992 2,
1993 &midx_chunks_with_pack_names(
1994 ObjectFormat::Sha1,
1995 b"pack-b.idx\0pack-a.idx\0".to_vec(),
1996 &[],
1997 ),
1998 );
1999 assert!(MultiPackIndex::parse(&unsorted_v1, ObjectFormat::Sha1).is_err());
2000
2001 let unsorted_v2 = multi_pack_index(
2002 ObjectFormat::Sha1,
2003 2,
2004 2,
2005 &midx_chunks_with_pack_names(
2006 ObjectFormat::Sha1,
2007 b"pack-b.idx\0pack-a.idx\0".to_vec(),
2008 &[],
2009 ),
2010 );
2011 let parsed = MultiPackIndex::parse(&unsorted_v2, ObjectFormat::Sha1)
2012 .expect("test operation should succeed");
2013 assert_eq!(parsed.pack_names, vec!["pack-b.idx", "pack-a.idx"]);
2014 }
2015
2016 #[test]
2017 fn rejects_bad_multi_pack_index_object_tables() {
2018 let oid_a = ObjectId::from_hex(
2019 ObjectFormat::Sha1,
2020 "1111111111111111111111111111111111111111",
2021 )
2022 .expect("test operation should succeed");
2023 let oid_b = ObjectId::from_hex(
2024 ObjectFormat::Sha1,
2025 "2222222222222222222222222222222222222222",
2026 )
2027 .expect("test operation should succeed");
2028
2029 let missing_oidf = multi_pack_index(
2030 ObjectFormat::Sha1,
2031 2,
2032 1,
2033 &[(*b"PNAM", b"pack-a.idx\0\0".to_vec())],
2034 );
2035 assert!(MultiPackIndex::parse(&missing_oidf, ObjectFormat::Sha1).is_err());
2036
2037 let bad_fanout = vec![
2038 (*b"PNAM", b"pack-a.idx\0\0".to_vec()),
2039 (*b"OIDF", vec![0; 256 * 4]),
2040 (*b"OIDL", oid_a.as_bytes().to_vec()),
2041 (*b"OOFF", midx_ooff_entries(&[(0, 12)], &mut Vec::new())),
2042 ];
2043 let bad_fanout = multi_pack_index(ObjectFormat::Sha1, 2, 1, &bad_fanout);
2044 assert!(MultiPackIndex::parse(&bad_fanout, ObjectFormat::Sha1).is_err());
2045
2046 let mut unsorted = Vec::new();
2047 unsorted.push((*b"PNAM", b"pack-a.idx\0\0".to_vec()));
2048 unsorted.push((*b"OIDF", midx_oid_fanout(&[oid_a.clone(), oid_b.clone()])));
2049 let mut oid_lookup = Vec::new();
2050 oid_lookup.extend_from_slice(oid_b.as_bytes());
2051 oid_lookup.extend_from_slice(oid_a.as_bytes());
2052 unsorted.push((*b"OIDL", oid_lookup));
2053 unsorted.push((
2054 *b"OOFF",
2055 midx_ooff_entries(&[(0, 12), (0, 24)], &mut Vec::new()),
2056 ));
2057 let unsorted = multi_pack_index(ObjectFormat::Sha1, 2, 1, &unsorted);
2058 assert!(MultiPackIndex::parse(&unsorted, ObjectFormat::Sha1).is_err());
2059
2060 let bad_pack = multi_pack_index(
2061 ObjectFormat::Sha1,
2062 2,
2063 1,
2064 &midx_chunks_with_pack_names(
2065 ObjectFormat::Sha1,
2066 b"pack-a.idx\0\0".to_vec(),
2067 &[(oid_a.clone(), 1, 12)],
2068 ),
2069 );
2070 assert!(MultiPackIndex::parse(&bad_pack, ObjectFormat::Sha1).is_err());
2071
2072 let mut large_offsets = Vec::new();
2073 let missing_loff = vec![
2074 (*b"PNAM", b"pack-a.idx\0\0".to_vec()),
2075 (*b"OIDF", midx_oid_fanout(std::slice::from_ref(&oid_a))),
2076 (*b"OIDL", oid_a.as_bytes().to_vec()),
2077 (
2078 *b"OOFF",
2079 midx_ooff_entries(&[(0, 0x1_0000_0000)], &mut large_offsets),
2080 ),
2081 ];
2082 let missing_loff = multi_pack_index(ObjectFormat::Sha1, 2, 1, &missing_loff);
2083 assert!(MultiPackIndex::parse(&missing_loff, ObjectFormat::Sha1).is_err());
2084
2085 let mut bad_loff =
2086 midx_chunks_with_pack_names(ObjectFormat::Sha1, b"pack-a.idx\0\0".to_vec(), &[]);
2087 bad_loff.push((*b"LOFF", vec![0]));
2088 let bad_loff = multi_pack_index(ObjectFormat::Sha1, 2, 1, &bad_loff);
2089 assert!(MultiPackIndex::parse(&bad_loff, ObjectFormat::Sha1).is_err());
2090 }
2091
2092 #[test]
2093 fn parses_multi_pack_index_bitmap_chunks() {
2094 let first = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"first object\n")
2095 .expect("test operation should succeed");
2096 let second = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"second object\n")
2097 .expect("test operation should succeed");
2098 let mut chunks = midx_chunks_with_pack_names(
2099 ObjectFormat::Sha1,
2100 b"pack-a.idx\0pack-b.idx\0\0\0".to_vec(),
2101 &[(first, 0, 12), (second, 1, 24)],
2102 );
2103 chunks.push((*b"RIDX", midx_u32_table(&[1, 0])));
2104 chunks.push((*b"BTMP", midx_bitmap_packs(&[(0, 1), (1, 1)])));
2105 let midx = multi_pack_index(ObjectFormat::Sha1, 2, 2, &chunks);
2106
2107 let parsed = MultiPackIndex::parse(&midx, ObjectFormat::Sha1)
2108 .expect("test operation should succeed");
2109 assert_eq!(parsed.reverse_index, Some(vec![1, 0]));
2110 assert_eq!(
2111 parsed.bitmapped_packs,
2112 Some(vec![
2113 MultiPackBitmapPack {
2114 bitmap_pos: 0,
2115 bitmap_nr: 1,
2116 },
2117 MultiPackBitmapPack {
2118 bitmap_pos: 1,
2119 bitmap_nr: 1,
2120 },
2121 ])
2122 );
2123 }
2124
2125 #[test]
2126 fn writes_multi_pack_index_that_round_trips() {
2127 let first = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"first object\n")
2128 .expect("test operation should succeed");
2129 let second = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"second object\n")
2130 .expect("test operation should succeed");
2131 let bytes = MultiPackIndex::write(
2132 ObjectFormat::Sha1,
2133 2,
2134 &["pack-b.idx".into(), "pack-a.idx".into()],
2135 &[
2136 MultiPackIndexEntry {
2137 oid: second.clone(),
2138 pack_int_id: 0,
2139 offset: 0x1_0000_0000,
2140 force_large_offset: false,
2141 },
2142 MultiPackIndexEntry {
2143 oid: first.clone(),
2144 pack_int_id: 1,
2145 offset: 12,
2146 force_large_offset: false,
2147 },
2148 ],
2149 )
2150 .expect("test operation should succeed");
2151
2152 let parsed = MultiPackIndex::parse(&bytes, ObjectFormat::Sha1)
2153 .expect("test operation should succeed");
2154 assert_eq!(parsed.version, 2);
2155 assert_eq!(parsed.pack_names, vec!["pack-b.idx", "pack-a.idx"]);
2156 assert_eq!(parsed.object_count, 2);
2157 assert_eq!(
2158 parsed
2159 .find(&first)
2160 .expect("test operation should succeed")
2161 .pack_int_id,
2162 1
2163 );
2164 assert_eq!(
2165 parsed
2166 .find(&first)
2167 .expect("test operation should succeed")
2168 .offset,
2169 12
2170 );
2171 assert_eq!(
2172 parsed
2173 .find(&second)
2174 .expect("test operation should succeed")
2175 .pack_int_id,
2176 0
2177 );
2178 assert_eq!(
2179 parsed
2180 .find(&second)
2181 .expect("test operation should succeed")
2182 .offset,
2183 0x1_0000_0000
2184 );
2185 assert!(parsed.chunks.iter().any(|chunk| chunk.id == *b"LOFF"));
2186 }
2187
2188 #[test]
2189 fn write_multi_pack_index_rejects_invalid_inputs() {
2190 let oid = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"object\n")
2191 .expect("test operation should succeed");
2192 assert!(MultiPackIndex::write(ObjectFormat::Sha1, 3, &["pack-a.idx".into()], &[]).is_err());
2193 assert!(
2194 MultiPackIndex::write(
2195 ObjectFormat::Sha1,
2196 1,
2197 &["pack-b.idx".into(), "pack-a.idx".into()],
2198 &[],
2199 )
2200 .is_err()
2201 );
2202 assert!(MultiPackIndex::write(ObjectFormat::Sha1, 2, &["pack/a.idx".into()], &[]).is_err());
2203 assert!(
2204 MultiPackIndex::write(
2205 ObjectFormat::Sha1,
2206 2,
2207 &["pack-a.idx".into()],
2208 &[MultiPackIndexEntry {
2209 oid,
2210 pack_int_id: 1,
2211 offset: 12,
2212 force_large_offset: false,
2213 }],
2214 )
2215 .is_err()
2216 );
2217 assert!(
2218 MultiPackIndex::write(
2219 ObjectFormat::Sha1,
2220 2,
2221 &["pack-a.idx".into()],
2222 &[
2223 MultiPackIndexEntry {
2224 oid,
2225 pack_int_id: 0,
2226 offset: 12,
2227 force_large_offset: false,
2228 },
2229 MultiPackIndexEntry {
2230 oid,
2231 pack_int_id: 0,
2232 offset: 24,
2233 force_large_offset: false,
2234 },
2235 ],
2236 )
2237 .is_err()
2238 );
2239 }
2240
2241 #[test]
2242 fn rejects_bad_multi_pack_index_bitmap_chunks() {
2243 let oid_a = ObjectId::from_hex(
2244 ObjectFormat::Sha1,
2245 "1111111111111111111111111111111111111111",
2246 )
2247 .expect("test operation should succeed");
2248 let oid_b = ObjectId::from_hex(
2249 ObjectFormat::Sha1,
2250 "2222222222222222222222222222222222222222",
2251 )
2252 .expect("test operation should succeed");
2253
2254 let mut duplicate_ridx = midx_chunks_with_pack_names(
2255 ObjectFormat::Sha1,
2256 b"pack-a.idx\0\0".to_vec(),
2257 &[(oid_a.clone(), 0, 12), (oid_b.clone(), 0, 24)],
2258 );
2259 duplicate_ridx.push((*b"RIDX", midx_u32_table(&[0, 0])));
2260 let duplicate_ridx = multi_pack_index(ObjectFormat::Sha1, 2, 1, &duplicate_ridx);
2261 assert!(MultiPackIndex::parse(&duplicate_ridx, ObjectFormat::Sha1).is_err());
2262
2263 let mut short_btmp = midx_chunks_with_pack_names(
2264 ObjectFormat::Sha1,
2265 b"pack-a.idx\0pack-b.idx\0\0\0".to_vec(),
2266 &[(oid_a.clone(), 0, 12), (oid_b.clone(), 1, 24)],
2267 );
2268 short_btmp.push((*b"BTMP", midx_bitmap_packs(&[(0, 1)])));
2269 let short_btmp = multi_pack_index(ObjectFormat::Sha1, 2, 2, &short_btmp);
2270 assert!(MultiPackIndex::parse(&short_btmp, ObjectFormat::Sha1).is_err());
2271
2272 let mut out_of_range_btmp = midx_chunks_with_pack_names(
2273 ObjectFormat::Sha1,
2274 b"pack-a.idx\0\0".to_vec(),
2275 &[(oid_a, 0, 12), (oid_b, 0, 24)],
2276 );
2277 out_of_range_btmp.push((*b"BTMP", midx_bitmap_packs(&[(1, 2)])));
2278 let out_of_range_btmp = multi_pack_index(ObjectFormat::Sha1, 2, 1, &out_of_range_btmp);
2279 assert!(MultiPackIndex::parse(&out_of_range_btmp, ObjectFormat::Sha1).is_err());
2280 }
2281
2282 #[test]
2283 fn parses_pack_bitmap_index_with_hash_cache() {
2284 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
2285 .expect("test operation should succeed");
2286 let bitmap = pack_bitmap_index(
2287 ObjectFormat::Sha1,
2288 3,
2289 PackBitmapIndex::OPTION_FULL_DAG | PackBitmapIndex::OPTION_HASH_CACHE,
2290 &pack_checksum,
2291 &[(2, 0, 1, &[0b101])],
2292 Some(&[0x1111_1111, 0x2222_2222, 0x3333_3333]),
2293 );
2294
2295 let parsed = PackBitmapIndex::parse(&bitmap, ObjectFormat::Sha1, 3)
2296 .expect("test operation should succeed");
2297 assert_eq!(parsed.version, 1);
2298 assert_eq!(parsed.format, ObjectFormat::Sha1);
2299 assert_eq!(
2300 parsed.options,
2301 PackBitmapIndex::OPTION_FULL_DAG | PackBitmapIndex::OPTION_HASH_CACHE
2302 );
2303 assert_eq!(parsed.pack_checksum, pack_checksum);
2304 assert_eq!(parsed.type_bitmaps.commits.bit_size, 3);
2305 assert_eq!(parsed.type_bitmaps.trees.bit_size, 3);
2306 assert_eq!(parsed.entries.len(), 1);
2307 let entry = parsed
2308 .entry_for_index_position(2)
2309 .expect("test operation should succeed");
2310 assert_eq!(entry.xor_offset, 0);
2311 assert_eq!(entry.flags, 1);
2312 assert_eq!(entry.bitmap.words, ewah_literal_words(&[0b101]));
2313 assert_eq!(
2314 parsed.name_hash_cache,
2315 Some(vec![0x1111_1111, 0x2222_2222, 0x3333_3333])
2316 );
2317 }
2318
2319 #[test]
2320 fn parses_pack_bitmap_index_sha256() {
2321 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha256, b"pack")
2322 .expect("test operation should succeed");
2323 let bitmap = pack_bitmap_index(
2324 ObjectFormat::Sha256,
2325 2,
2326 PackBitmapIndex::OPTION_FULL_DAG,
2327 &pack_checksum,
2328 &[(0, 0, 0, &[0b11])],
2329 None,
2330 );
2331
2332 let parsed = PackBitmapIndex::parse(&bitmap, ObjectFormat::Sha256, 2)
2333 .expect("test operation should succeed");
2334 assert_eq!(parsed.version, 1);
2335 assert_eq!(parsed.format, ObjectFormat::Sha256);
2336 assert_eq!(parsed.pack_checksum, pack_checksum);
2337 assert_eq!(parsed.index_checksum.format(), ObjectFormat::Sha256);
2338 assert_eq!(parsed.entries[0].object_position, 0);
2339 assert_eq!(parsed.name_hash_cache, None);
2340 }
2341
2342 #[test]
2343 fn parses_upstream_git_written_pack_bitmap_index() {
2344 let root = unique_temp_dir("git-pack-bitmap-upstream");
2345 fs::create_dir_all(&root).expect("test operation should succeed");
2346 {
2347 run_git_success(&root, &["init", "-q", "-b", "main"]);
2348 run_git_success(
2349 &root,
2350 &[
2351 "-c",
2352 "user.name=Example User",
2353 "-c",
2354 "user.email=example@example.invalid",
2355 "commit",
2356 "--allow-empty",
2357 "-q",
2358 "-m",
2359 "one",
2360 ],
2361 );
2362 run_git_success(
2363 &root,
2364 &[
2365 "-c",
2366 "user.name=Example User",
2367 "-c",
2368 "user.email=example@example.invalid",
2369 "commit",
2370 "--allow-empty",
2371 "-q",
2372 "-m",
2373 "two",
2374 ],
2375 );
2376 run_git_success(&root, &["repack", "-adb"]);
2377 let pack_dir = root.join(".git").join("objects").join("pack");
2378 let idx_path = single_path_with_extension(&pack_dir, "idx");
2379 let bitmap_path = single_path_with_extension(&pack_dir, "bitmap");
2380 let index = PackIndex::parse(
2381 &fs::read(idx_path).expect("test operation should succeed"),
2382 ObjectFormat::Sha1,
2383 )
2384 .expect("test operation should succeed");
2385 let bitmap = PackBitmapIndex::parse(
2386 &fs::read(bitmap_path).expect("test operation should succeed"),
2387 ObjectFormat::Sha1,
2388 index.entries.len(),
2389 )
2390 .expect("test operation should succeed");
2391 assert_eq!(bitmap.pack_checksum, index.pack_checksum);
2392 assert!(!bitmap.entries.is_empty());
2393 };
2394 let _ = fs::remove_dir_all(&root);
2395 }
2396
2397 #[test]
2398 fn rejects_bad_pack_bitmap_index_header_and_checksum() {
2399 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
2400 .expect("test operation should succeed");
2401 let bitmap = pack_bitmap_index(
2402 ObjectFormat::Sha1,
2403 1,
2404 PackBitmapIndex::OPTION_FULL_DAG,
2405 &pack_checksum,
2406 &[(0, 0, 0, &[1])],
2407 None,
2408 );
2409
2410 let mut bad_signature = bitmap.clone();
2411 bad_signature[0] = b'X';
2412 assert!(PackBitmapIndex::parse(&bad_signature, ObjectFormat::Sha1, 1).is_err());
2413
2414 let mut bad_version = bitmap.clone();
2415 bad_version[5] = 2;
2416 refresh_trailing_checksum(ObjectFormat::Sha1, &mut bad_version);
2417 assert!(PackBitmapIndex::parse(&bad_version, ObjectFormat::Sha1, 1).is_err());
2418
2419 let mut bad_option = bitmap.clone();
2420 bad_option[7] = 0x20;
2421 refresh_trailing_checksum(ObjectFormat::Sha1, &mut bad_option);
2422 assert!(PackBitmapIndex::parse(&bad_option, ObjectFormat::Sha1, 1).is_err());
2423
2424 let mut bad_checksum = bitmap;
2425 let last = bad_checksum.len() - 1;
2426 bad_checksum[last] ^= 1;
2427 assert!(PackBitmapIndex::parse(&bad_checksum, ObjectFormat::Sha1, 1).is_err());
2428 }
2429
2430 #[test]
2431 fn rejects_bad_pack_bitmap_index_ewah_and_entries() {
2432 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
2433 .expect("test operation should succeed");
2434 let bitmap = pack_bitmap_index(
2435 ObjectFormat::Sha1,
2436 2,
2437 PackBitmapIndex::OPTION_FULL_DAG,
2438 &pack_checksum,
2439 &[(0, 0, 0, &[0b01]), (1, 1, 0, &[0b11])],
2440 None,
2441 );
2442
2443 let mut truncated = bitmap.clone();
2444 truncated.truncate(truncated.len() - ObjectFormat::Sha1.raw_len() - 1);
2445 refresh_trailing_checksum(ObjectFormat::Sha1, &mut truncated);
2446 assert!(PackBitmapIndex::parse(&truncated, ObjectFormat::Sha1, 2).is_err());
2447
2448 let mut out_of_range_position = pack_bitmap_index(
2449 ObjectFormat::Sha1,
2450 2,
2451 PackBitmapIndex::OPTION_FULL_DAG,
2452 &pack_checksum,
2453 &[(2, 0, 0, &[0b01])],
2454 None,
2455 );
2456 assert!(PackBitmapIndex::parse(&out_of_range_position, ObjectFormat::Sha1, 2).is_err());
2457 refresh_trailing_checksum(ObjectFormat::Sha1, &mut out_of_range_position);
2458 assert!(PackBitmapIndex::parse(&out_of_range_position, ObjectFormat::Sha1, 2).is_err());
2459
2460 let invalid_xor = pack_bitmap_index(
2461 ObjectFormat::Sha1,
2462 2,
2463 PackBitmapIndex::OPTION_FULL_DAG,
2464 &pack_checksum,
2465 &[(0, 1, 0, &[0b01])],
2466 None,
2467 );
2468 assert!(PackBitmapIndex::parse(&invalid_xor, ObjectFormat::Sha1, 2).is_err());
2469 }
2470
2471 #[test]
2472 fn parses_single_entry_pack_index_sha256() {
2473 let oid = sley_core::object_id_for_bytes(ObjectFormat::Sha256, "blob", b"hello sha256\n")
2474 .expect("test operation should succeed");
2475 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha256, b"pack")
2476 .expect("test operation should succeed");
2477 let index = single_entry_index(
2478 ObjectFormat::Sha256,
2479 oid,
2480 0x1234_5678,
2481 12,
2482 pack_checksum.clone(),
2483 );
2484 let parsed =
2485 PackIndex::parse(&index, ObjectFormat::Sha256).expect("test operation should succeed");
2486 assert_eq!(parsed.version, 2);
2487 assert_eq!(parsed.pack_checksum, pack_checksum);
2488 assert_eq!(parsed.entries.len(), 1);
2489 assert_eq!(
2490 parsed
2491 .find(&oid)
2492 .expect("test operation should succeed")
2493 .offset,
2494 12
2495 );
2496 assert_eq!(
2497 parsed
2498 .find(&oid)
2499 .expect("test operation should succeed")
2500 .crc32,
2501 0x1234_5678
2502 );
2503 assert_eq!(parsed.index_checksum.format(), ObjectFormat::Sha256);
2504 assert_pack_index_view_matches_owned(&index, ObjectFormat::Sha256);
2505 }
2506
2507 #[test]
2508 fn write_packed_deltifies_similar_blobs_and_round_trips_sha1() {
2509 write_packed_deltifies_similar_blobs_and_round_trips(ObjectFormat::Sha1);
2510 }
2511
2512 #[test]
2513 fn write_packed_deltifies_similar_blobs_and_round_trips_sha256() {
2514 write_packed_deltifies_similar_blobs_and_round_trips(ObjectFormat::Sha256);
2515 }
2516
2517 #[test]
2518 fn write_packed_rejects_duplicate_objects() {
2519 let object = EncodedObject::new(ObjectType::Blob, b"same\n".to_vec());
2520 assert!(PackFile::write_packed(&[object.clone(), object], ObjectFormat::Sha1,).is_err());
2521 }
2522
2523 #[test]
2524 fn write_packed_with_known_ids_validates_ids_before_trusting_them() {
2525 let object = EncodedObject::new(ObjectType::Blob, b"same\n".to_vec());
2526 let sha1 = object
2527 .object_id(ObjectFormat::Sha1)
2528 .expect("test operation should succeed");
2529 let sha256 = object
2530 .object_id(ObjectFormat::Sha256)
2531 .expect("test operation should succeed");
2532 let duplicate = [
2533 PackInput {
2534 oid: &sha1,
2535 object: &object,
2536 },
2537 PackInput {
2538 oid: &sha1,
2539 object: &object,
2540 },
2541 ];
2542 assert!(PackFile::write_packed_with_known_ids(&duplicate, ObjectFormat::Sha1).is_err());
2543
2544 let wrong_format = [PackInput {
2545 oid: &sha256,
2546 object: &object,
2547 }];
2548 assert!(PackFile::write_packed_with_known_ids(&wrong_format, ObjectFormat::Sha1).is_err());
2549 }
2550
2551 #[test]
2552 fn write_packed_with_known_ids_to_writer_matches_in_memory_pack() {
2553 let objects = similar_blob_family(6);
2554 let object_ids = objects
2555 .iter()
2556 .map(|object| {
2557 object
2558 .object_id(ObjectFormat::Sha1)
2559 .expect("test operation should succeed")
2560 })
2561 .collect::<Vec<_>>();
2562 let inputs = objects
2563 .iter()
2564 .zip(&object_ids)
2565 .map(|(object, oid)| PackInput { oid, object })
2566 .collect::<Vec<_>>();
2567 let options = PackWriteOptions::new();
2568 let in_memory = PackFile::write_packed_with_known_ids_and_options(
2569 &inputs,
2570 ObjectFormat::Sha1,
2571 &options,
2572 )
2573 .expect("test operation should succeed");
2574 let mut written = Vec::new();
2575 let streamed = PackFile::write_packed_with_known_ids_to_writer(
2576 &inputs,
2577 ObjectFormat::Sha1,
2578 &options,
2579 &mut written,
2580 )
2581 .expect("test operation should succeed");
2582
2583 assert_eq!(written, in_memory.pack);
2584 assert_eq!(streamed.index, in_memory.index);
2585 assert_eq!(streamed.checksum, in_memory.checksum);
2586 assert_eq!(streamed.entries, in_memory.entries);
2587 assert_eq!(streamed.delta_count, in_memory.delta_count);
2588 assert_eq!(streamed.pack_size, in_memory.pack.len() as u64);
2589 }
2590
2591 #[test]
2592 fn write_packed_from_source_to_writer_deltifies_across_windows() {
2593 let format = ObjectFormat::Sha1;
2594 let mut objects = Vec::new();
2595 for idx in 0..PACK_STREAM_COMPRESSION_WINDOW_OBJECTS - 1 {
2596 objects.push(EncodedObject::new(
2597 ObjectType::Blob,
2598 format!("unrelated streamed source object {idx:04}\n").into_bytes(),
2599 ));
2600 }
2601 let shared = b"cross-window base payload with enough shared anchors\n".repeat(64);
2605 let mut base_body = shared.clone();
2606 base_body.extend_from_slice(b"base\n");
2607 let mut target_body = shared;
2608 target_body.extend_from_slice(b"target\n");
2609 objects.push(EncodedObject::new(ObjectType::Blob, base_body));
2610 objects.push(EncodedObject::new(ObjectType::Blob, target_body));
2611
2612 let object_ids = objects
2613 .iter()
2614 .map(|object| {
2615 object
2616 .object_id(format)
2617 .expect("test operation should succeed")
2618 })
2619 .collect::<Vec<_>>();
2620 let base_oid = object_ids[PACK_STREAM_COMPRESSION_WINDOW_OBJECTS - 1];
2621 let target_oid = object_ids[PACK_STREAM_COMPRESSION_WINDOW_OBJECTS];
2622 let object_map = object_ids
2623 .iter()
2624 .copied()
2625 .zip(objects.into_iter().map(Arc::new))
2626 .collect::<HashMap<_, _>>();
2627
2628 let options = PackWriteOptions::new().with_reorder(false).with_window(10);
2629 let mut written = Vec::new();
2630 let summary = PackFile::write_packed_from_source_to_writer(
2631 &object_ids,
2632 format,
2633 &options,
2634 |oid| {
2635 object_map
2636 .get(oid)
2637 .cloned()
2638 .ok_or_else(|| GitError::not_found(format!("missing test object {oid}")))
2639 },
2640 &mut written,
2641 )
2642 .expect("test operation should succeed");
2643
2644 assert!(
2645 summary.delta_count > 0,
2646 "expected source-backed streaming writer to find deltas"
2647 );
2648 let stats =
2649 PackFile::verify_pack_stats(&written, format).expect("test operation should succeed");
2650 let target = stats
2651 .objects
2652 .iter()
2653 .find(|entry| entry.oid == target_oid)
2654 .expect("target object should be present");
2655 assert_eq!(target.base_oid, Some(base_oid));
2656 }
2657
2658 fn write_packed_deltifies_similar_blobs_and_round_trips(format: ObjectFormat) {
2659 let objects = similar_blob_family(8);
2660 let packed =
2661 PackFile::write_packed(&objects, format).expect("test operation should succeed");
2662 let undeltified =
2663 PackFile::write_undeltified(&objects, format).expect("test operation should succeed");
2664
2665 assert!(
2668 packed.pack.len() < undeltified.pack.len(),
2669 "expected delta pack ({}) smaller than undeltified pack ({})",
2670 packed.pack.len(),
2671 undeltified.pack.len()
2672 );
2673
2674 let kinds = pack_entry_kinds(&packed.pack, format);
2676 let delta_count = kinds
2677 .iter()
2678 .filter(|kind| matches!(kind, PackObjectKind::OfsDelta | PackObjectKind::RefDelta))
2679 .count();
2680 assert!(
2681 delta_count >= 1,
2682 "expected at least one delta entry, found kinds {kinds:?}"
2683 );
2684
2685 let parsed = PackFile::parse(&packed.pack, format).expect("test operation should succeed");
2687 assert_eq!(parsed.entries.len(), objects.len());
2688 for object in &objects {
2689 let oid = object
2690 .object_id(format)
2691 .expect("test operation should succeed");
2692 let found = parsed
2693 .entries
2694 .iter()
2695 .find(|entry| entry.entry.oid == oid)
2696 .unwrap_or_else(|| panic!("object {oid} missing from parsed pack"));
2697 assert_eq!(&found.object, object, "object {oid} did not round-trip");
2698 }
2699
2700 let index = PackIndex::parse(&packed.index, format).expect("test operation should succeed");
2702 assert_eq!(index.pack_checksum, packed.checksum);
2703 for object in &objects {
2704 let oid = object
2705 .object_id(format)
2706 .expect("test operation should succeed");
2707 assert!(index.find(&oid).is_some(), "index missing {oid}");
2708 }
2709 }
2710
2711 #[test]
2712 fn write_packed_emits_ofs_delta_by_default() {
2713 let objects = similar_blob_family(6);
2714 let packed = PackFile::write_packed(&objects, ObjectFormat::Sha1)
2715 .expect("test operation should succeed");
2716 let kinds = pack_entry_kinds(&packed.pack, ObjectFormat::Sha1);
2717 assert!(
2718 kinds.contains(&PackObjectKind::OfsDelta),
2719 "expected an ofs-delta entry by default, found {kinds:?}"
2720 );
2721 assert!(
2722 !kinds.contains(&PackObjectKind::RefDelta),
2723 "default self-contained pack must not use ref-delta, found {kinds:?}"
2724 );
2725 assert!(PackFile::parse(&packed.pack, ObjectFormat::Sha1).is_ok());
2727 }
2728
2729 #[test]
2730 fn write_packed_can_emit_ref_delta() {
2731 let objects = similar_blob_family(6);
2732 let options = PackWriteOptions::new().with_prefer_ofs_delta(false);
2733 let packed = PackFile::write_packed_with_options(&objects, ObjectFormat::Sha1, &options)
2734 .expect("test operation should succeed");
2735 let kinds = pack_entry_kinds(&packed.pack, ObjectFormat::Sha1);
2736 assert!(
2737 kinds.contains(&PackObjectKind::RefDelta),
2738 "expected a ref-delta entry, found {kinds:?}"
2739 );
2740 assert!(
2741 !kinds.contains(&PackObjectKind::OfsDelta),
2742 "ref-delta mode must not emit ofs-delta, found {kinds:?}"
2743 );
2744
2745 let parsed = PackFile::parse(&packed.pack, ObjectFormat::Sha1)
2748 .expect("test operation should succeed");
2749 assert_eq!(parsed.entries.len(), objects.len());
2750 }
2751
2752 #[test]
2753 fn write_packed_bounds_delta_chain_depth() {
2754 let objects = incremental_blob_chain(20);
2758 let format = ObjectFormat::Sha1;
2759
2760 for max_depth in [1usize, 2, 5] {
2761 let options = PackWriteOptions::new()
2762 .with_window(20)
2763 .with_depth(max_depth);
2764 let packed = PackFile::write_packed_with_options(&objects, format, &options)
2765 .expect("test operation should succeed");
2766
2767 let depths = pack_entry_depths(&packed.pack, format);
2768 let observed = depths.iter().copied().max().unwrap_or(0);
2769 assert!(
2770 observed <= max_depth,
2771 "max chain depth {observed} exceeded bound {max_depth}"
2772 );
2773
2774 let parsed =
2776 PackFile::parse(&packed.pack, format).expect("test operation should succeed");
2777 for object in &objects {
2778 let oid = object
2779 .object_id(format)
2780 .expect("test operation should succeed");
2781 let found = parsed
2782 .entries
2783 .iter()
2784 .find(|entry| entry.entry.oid == oid)
2785 .expect("test operation should succeed");
2786 assert_eq!(&found.object, object);
2787 }
2788 }
2789 }
2790
2791 #[test]
2792 fn write_packed_depth_zero_stores_everything_undeltified() {
2793 let objects = similar_blob_family(5);
2794 let options = PackWriteOptions::new().with_depth(0);
2795 let packed = PackFile::write_packed_with_options(&objects, ObjectFormat::Sha1, &options)
2796 .expect("test operation should succeed");
2797 let kinds = pack_entry_kinds(&packed.pack, ObjectFormat::Sha1);
2798 assert!(
2799 kinds
2800 .iter()
2801 .all(|kind| !matches!(kind, PackObjectKind::OfsDelta | PackObjectKind::RefDelta)),
2802 "depth 0 must disable deltas, found {kinds:?}"
2803 );
2804 }
2805
2806 #[test]
2807 fn write_thin_uses_external_base_and_round_trips_sha1() {
2808 write_thin_uses_external_base_and_round_trips(ObjectFormat::Sha1);
2809 }
2810
2811 #[test]
2812 fn write_thin_uses_external_base_and_round_trips_sha256() {
2813 write_thin_uses_external_base_and_round_trips(ObjectFormat::Sha256);
2814 }
2815
2816 fn write_thin_uses_external_base_and_round_trips(format: ObjectFormat) {
2817 let base = blob_with_marker("EXTERNAL-BASE");
2820 let target = blob_with_marker("EXTERNAL-TARGET");
2821 let base_oid = base
2822 .object_id(format)
2823 .expect("test operation should succeed");
2824
2825 let mut external = HashMap::new();
2826 external.insert(base_oid, base.clone());
2827 let packed = PackFile::write_thin(std::slice::from_ref(&target), format, external)
2828 .expect("test operation should succeed");
2829
2830 let kinds = pack_entry_kinds(&packed.pack, format);
2832 assert_eq!(kinds, vec![PackObjectKind::RefDelta]);
2833
2834 let mut offset = 12usize;
2836 let header =
2837 parse_entry_header(&packed.pack, &mut offset).expect("test operation should succeed");
2838 assert_eq!(header.kind, PackObjectKind::RefDelta);
2839 let referenced =
2840 ObjectId::from_raw(format, &packed.pack[offset..offset + format.raw_len()])
2841 .expect("test operation should succeed");
2842 assert_eq!(referenced, base_oid);
2843
2844 assert!(PackFile::parse(&packed.pack, format).is_err());
2846
2847 let parsed = PackFile::parse_thin(&packed.pack, format, |oid| {
2849 if oid == &base_oid {
2850 Ok(Some(base.clone()))
2851 } else {
2852 Ok(None)
2853 }
2854 })
2855 .expect("test operation should succeed");
2856 assert_eq!(parsed.entries.len(), 1);
2857 assert_eq!(parsed.entries[0].object, target);
2858 }
2859
2860 #[test]
2861 fn write_packed_preserves_distinct_objects_with_no_similarity() {
2862 let objects = vec![
2865 EncodedObject::new(ObjectType::Blob, b"alpha distinct\n".to_vec()),
2866 EncodedObject::new(ObjectType::Tree, vec![0u8; 0]),
2867 EncodedObject::new(ObjectType::Commit, b"tree 0000\n".to_vec()),
2868 ];
2869 let format = ObjectFormat::Sha1;
2870 let packed =
2871 PackFile::write_packed(&objects, format).expect("test operation should succeed");
2872 let parsed = PackFile::parse(&packed.pack, format).expect("test operation should succeed");
2873 assert_eq!(parsed.entries.len(), objects.len());
2874 for object in &objects {
2875 let oid = object
2876 .object_id(format)
2877 .expect("test operation should succeed");
2878 assert!(parsed.entries.iter().any(|entry| entry.entry.oid == oid));
2879 }
2880 }
2881
2882 fn similar_blob_family(count: usize) -> Vec<EncodedObject> {
2886 let mut common_head = Vec::new();
2887 for _ in 0..200 {
2888 common_head.extend_from_slice(b"shared header line for delta testing\n");
2889 }
2890 let mut common_tail = Vec::new();
2891 for _ in 0..200 {
2892 common_tail.extend_from_slice(b"shared trailer line for delta testing\n");
2893 }
2894 (0..count)
2895 .map(|idx| {
2896 let mut body = common_head.clone();
2897 body.extend_from_slice(format!("UNIQUE MIDDLE MARKER NUMBER {idx}\n").as_bytes());
2898 body.extend_from_slice(&common_tail);
2899 EncodedObject::new(ObjectType::Blob, body)
2900 })
2901 .collect()
2902 }
2903
2904 fn incremental_blob_chain(count: usize) -> Vec<EncodedObject> {
2907 let mut body = Vec::new();
2908 for _ in 0..100 {
2909 body.extend_from_slice(b"baseline content shared across the whole chain\n");
2910 }
2911 let mut objects = Vec::with_capacity(count);
2912 for idx in 0..count {
2913 body.extend_from_slice(format!("appended unique line {idx}\n").as_bytes());
2914 objects.push(EncodedObject::new(ObjectType::Blob, body.clone()));
2915 }
2916 objects
2917 }
2918
2919 fn blob_with_marker(marker: &str) -> EncodedObject {
2920 let mut body = Vec::new();
2921 for _ in 0..150 {
2922 body.extend_from_slice(b"common body shared between base and target\n");
2923 }
2924 body.extend_from_slice(marker.as_bytes());
2925 body.push(b'\n');
2926 for _ in 0..150 {
2927 body.extend_from_slice(b"more common body shared between objects\n");
2928 }
2929 EncodedObject::new(ObjectType::Blob, body)
2930 }
2931
2932 fn pack_entry_kinds(pack: &[u8], format: ObjectFormat) -> Vec<PackObjectKind> {
2934 pack_entry_descriptors(pack, format)
2935 .into_iter()
2936 .map(|descriptor| descriptor.kind)
2937 .collect()
2938 }
2939
2940 fn pack_entry_depths(pack: &[u8], format: ObjectFormat) -> Vec<usize> {
2944 let descriptors = pack_entry_descriptors(pack, format);
2945 let mut depth_by_offset: HashMap<u64, usize> = HashMap::new();
2946 let mut depths = Vec::with_capacity(descriptors.len());
2947 for descriptor in &descriptors {
2948 let depth = match &descriptor.base {
2949 EntryBase::None => 0,
2950 EntryBase::Offset(base_offset) => {
2951 depth_by_offset.get(base_offset).copied().unwrap_or(0) + 1
2952 }
2953 EntryBase::Ref => 1,
2957 };
2958 depth_by_offset.insert(descriptor.offset, depth);
2959 depths.push(depth);
2960 }
2961 depths
2962 }
2963
2964 struct EntryDescriptor {
2965 offset: u64,
2966 kind: PackObjectKind,
2967 base: EntryBase,
2968 }
2969
2970 enum EntryBase {
2971 None,
2972 Offset(u64),
2973 Ref,
2974 }
2975
2976 fn pack_entry_descriptors(pack: &[u8], format: ObjectFormat) -> Vec<EntryDescriptor> {
2977 let trailer_offset = pack.len() - format.raw_len();
2978 let count = u32_be(&pack[8..12]) as usize;
2979 let mut offset = 12usize;
2980 let mut descriptors = Vec::with_capacity(count);
2981 for _ in 0..count {
2982 let entry_offset = offset as u64;
2983 let header =
2984 parse_entry_header(pack, &mut offset).expect("test operation should succeed");
2985 let base = match header.kind {
2986 PackObjectKind::OfsDelta => {
2987 let base_offset = parse_ofs_delta_base_offset(pack, &mut offset, entry_offset)
2988 .expect("test operation should succeed");
2989 EntryBase::Offset(base_offset)
2990 }
2991 PackObjectKind::RefDelta => {
2992 offset += format.raw_len();
2993 EntryBase::Ref
2994 }
2995 _ => EntryBase::None,
2996 };
2997 let mut decoder = ZlibDecoder::new(&pack[offset..trailer_offset]);
2998 let mut body = Vec::new();
2999 decoder
3000 .read_to_end(&mut body)
3001 .expect("test operation should succeed");
3002 offset += decoder.total_in() as usize;
3003 descriptors.push(EntryDescriptor {
3004 offset: entry_offset,
3005 kind: header.kind,
3006 base,
3007 });
3008 }
3009 descriptors
3010 }
3011
3012 fn similar_blob_objects() -> (EncodedObject, EncodedObject) {
3013 let mut base = Vec::new();
3014 for _ in 0..300 {
3015 base.extend_from_slice(b"common payload\n");
3016 }
3017 base.extend_from_slice(b"base\n");
3018 let mut changed = Vec::new();
3019 for _ in 0..300 {
3020 changed.extend_from_slice(b"common payload\n");
3021 }
3022 changed.extend_from_slice(b"changed\n");
3023 (
3024 EncodedObject::new(ObjectType::Blob, base),
3025 EncodedObject::new(ObjectType::Blob, changed),
3026 )
3027 }
3028
3029 fn single_object_pack(format: ObjectFormat, object_type: ObjectType, body: &[u8]) -> Vec<u8> {
3030 let mut pack = Vec::new();
3031 pack.extend_from_slice(b"PACK");
3032 pack.extend_from_slice(&2u32.to_be_bytes());
3033 pack.extend_from_slice(&1u32.to_be_bytes());
3034 write_entry_header(&mut pack, object_type, body.len() as u64);
3035 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
3036 encoder
3037 .write_all(body)
3038 .expect("test operation should succeed");
3039 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
3040 let checksum =
3041 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
3042 pack.extend_from_slice(checksum.as_bytes());
3043 pack
3044 }
3045
3046 #[derive(Clone, Copy, Debug)]
3047 enum DeltaKind {
3048 Offset,
3049 Ref,
3050 }
3051
3052 fn two_object_delta_pack(
3053 format: ObjectFormat,
3054 base: &[u8],
3055 result: &[u8],
3056 delta_kind: DeltaKind,
3057 ) -> Vec<u8> {
3058 let mut pack = Vec::new();
3059 pack.extend_from_slice(b"PACK");
3060 pack.extend_from_slice(&2u32.to_be_bytes());
3061 pack.extend_from_slice(&2u32.to_be_bytes());
3062
3063 let base_offset = pack.len();
3064 write_entry_header(&mut pack, ObjectType::Blob, base.len() as u64);
3065 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
3066 encoder
3067 .write_all(base)
3068 .expect("test operation should succeed");
3069 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
3070
3071 let delta = append_suffix_delta(base, result);
3072 let delta_offset = pack.len();
3073 write_pack_entry_header_kind(
3074 &mut pack,
3075 match delta_kind {
3076 DeltaKind::Offset => 6,
3077 DeltaKind::Ref => 7,
3078 },
3079 delta.len() as u64,
3080 );
3081 match delta_kind {
3082 DeltaKind::Offset => write_ofs_delta_offset(&mut pack, delta_offset - base_offset),
3083 DeltaKind::Ref => {
3084 let base_oid = sley_core::object_id_for_bytes(format, "blob", base)
3085 .expect("test operation should succeed");
3086 pack.extend_from_slice(base_oid.as_bytes());
3087 }
3088 }
3089 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
3090 encoder
3091 .write_all(&delta)
3092 .expect("test operation should succeed");
3093 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
3094
3095 let checksum =
3096 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
3097 pack.extend_from_slice(checksum.as_bytes());
3098 pack
3099 }
3100
3101 fn thin_ref_delta_pack(format: ObjectFormat, base: &[u8], result: &[u8]) -> Vec<u8> {
3102 let mut pack = Vec::new();
3103 pack.extend_from_slice(b"PACK");
3104 pack.extend_from_slice(&2u32.to_be_bytes());
3105 pack.extend_from_slice(&1u32.to_be_bytes());
3106
3107 let delta = append_suffix_delta(base, result);
3108 write_pack_entry_header_kind(&mut pack, 7, delta.len() as u64);
3109 let base_oid = sley_core::object_id_for_bytes(format, "blob", base)
3110 .expect("test operation should succeed");
3111 pack.extend_from_slice(base_oid.as_bytes());
3112 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
3113 encoder
3114 .write_all(&delta)
3115 .expect("test operation should succeed");
3116 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
3117
3118 let checksum =
3119 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
3120 pack.extend_from_slice(checksum.as_bytes());
3121 pack
3122 }
3123
3124 fn thin_ref_delta_pack_many(format: ObjectFormat, base: &[u8], results: &[&[u8]]) -> Vec<u8> {
3125 let mut pack = Vec::new();
3126 pack.extend_from_slice(b"PACK");
3127 pack.extend_from_slice(&2u32.to_be_bytes());
3128 pack.extend_from_slice(&(results.len() as u32).to_be_bytes());
3129 let base_oid = sley_core::object_id_for_bytes(format, "blob", base)
3130 .expect("test operation should succeed");
3131 for result in results {
3132 let delta = append_suffix_delta(base, result);
3133 write_pack_entry_header_kind(&mut pack, 7, delta.len() as u64);
3134 pack.extend_from_slice(base_oid.as_bytes());
3135 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
3136 encoder
3137 .write_all(&delta)
3138 .expect("test operation should succeed");
3139 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
3140 }
3141 let checksum =
3142 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
3143 pack.extend_from_slice(checksum.as_bytes());
3144 pack
3145 }
3146
3147 fn unique_temp_dir(name: &str) -> PathBuf {
3148 let nanos = SystemTime::now()
3149 .duration_since(UNIX_EPOCH)
3150 .expect("test operation should succeed")
3151 .as_nanos();
3152 std::env::temp_dir().join(format!("sley-{name}-{}-{nanos}", std::process::id()))
3153 }
3154
3155 fn run_git_success(cwd: &Path, args: &[&str]) {
3156 let output = Command::new("git")
3157 .current_dir(cwd)
3158 .args(args)
3159 .output()
3160 .unwrap_or_else(|err| panic!("failed to run git {args:?}: {err}"));
3161 assert!(
3162 output.status.success(),
3163 "git {args:?} failed with status {:?}\nstdout:\n{}\nstderr:\n{}",
3164 output.status.code(),
3165 String::from_utf8_lossy(&output.stdout),
3166 String::from_utf8_lossy(&output.stderr)
3167 );
3168 }
3169
3170 fn single_path_with_extension(dir: &Path, extension: &str) -> PathBuf {
3171 let mut paths = fs::read_dir(dir)
3172 .expect("test operation should succeed")
3173 .map(|entry| entry.expect("test operation should succeed").path())
3174 .filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some(extension))
3175 .collect::<Vec<_>>();
3176 assert_eq!(paths.len(), 1, "expected one .{extension} file");
3177 paths.remove(0)
3178 }
3179
3180 fn pack_bitmap_index(
3181 format: ObjectFormat,
3182 object_count: u32,
3183 options: u16,
3184 pack_checksum: &ObjectId,
3185 entries: &[(u32, u8, u8, &[u64])],
3186 name_hash_cache: Option<&[u32]>,
3187 ) -> Vec<u8> {
3188 let mut out = Vec::new();
3189 out.extend_from_slice(b"BITM");
3190 out.extend_from_slice(&1u16.to_be_bytes());
3191 out.extend_from_slice(&options.to_be_bytes());
3192 out.extend_from_slice(&(entries.len() as u32).to_be_bytes());
3193 out.extend_from_slice(pack_checksum.as_bytes());
3194 write_test_ewah(&mut out, object_count, &[0b001]);
3195 write_test_ewah(&mut out, object_count, &[0b010]);
3196 write_test_ewah(&mut out, object_count, &[0b100]);
3197 write_test_ewah(&mut out, object_count, &[0]);
3198 for (position, xor_offset, flags, words) in entries {
3199 out.extend_from_slice(&position.to_be_bytes());
3200 out.push(*xor_offset);
3201 out.push(*flags);
3202 write_test_ewah(&mut out, object_count, words);
3203 }
3204 if let Some(cache) = name_hash_cache {
3205 for value in cache {
3206 out.extend_from_slice(&value.to_be_bytes());
3207 }
3208 }
3209 let checksum =
3210 sley_core::digest_bytes(format, &out).expect("test operation should succeed");
3211 out.extend_from_slice(checksum.as_bytes());
3212 out
3213 }
3214
3215 fn write_test_ewah(out: &mut Vec<u8>, bit_size: u32, literals: &[u64]) {
3216 out.extend_from_slice(&bit_size.to_be_bytes());
3217 let words = ewah_literal_words(literals);
3218 out.extend_from_slice(&(words.len() as u32).to_be_bytes());
3219 for word in words {
3220 out.extend_from_slice(&word.to_be_bytes());
3221 }
3222 out.extend_from_slice(&0u32.to_be_bytes());
3223 }
3224
3225 fn ewah_literal_words(literals: &[u64]) -> Vec<u64> {
3226 let rlw = (literals.len() as u64) << 33;
3227 let mut words = vec![rlw];
3228 words.extend_from_slice(literals);
3229 words
3230 }
3231
3232 fn refresh_trailing_checksum(format: ObjectFormat, bytes: &mut [u8]) {
3233 let checksum_offset = bytes.len() - format.raw_len();
3234 let checksum = sley_core::digest_bytes(format, &bytes[..checksum_offset])
3235 .expect("test operation should succeed");
3236 bytes[checksum_offset..].copy_from_slice(checksum.as_bytes());
3237 }
3238
3239 fn append_suffix_delta(base: &[u8], result: &[u8]) -> Vec<u8> {
3240 assert!(result.starts_with(base));
3241 let suffix = &result[base.len()..];
3242 assert!(base.len() < 0x10000);
3243 assert!(suffix.len() < 0x80);
3244 let mut delta = Vec::new();
3245 write_delta_varint(&mut delta, base.len() as u64);
3246 write_delta_varint(&mut delta, result.len() as u64);
3247 delta.push(0x90);
3248 delta.push(base.len() as u8);
3249 delta.push(suffix.len() as u8);
3250 delta.extend_from_slice(suffix);
3251 delta
3252 }
3253
3254 fn write_delta_varint(out: &mut Vec<u8>, mut value: u64) {
3255 loop {
3256 let mut byte = (value as u8) & 0x7f;
3257 value >>= 7;
3258 if value != 0 {
3259 byte |= 0x80;
3260 }
3261 out.push(byte);
3262 if value == 0 {
3263 break;
3264 }
3265 }
3266 }
3267
3268 fn write_pack_entry_header_kind(out: &mut Vec<u8>, type_code: u8, mut size: u64) {
3269 let mut byte = (type_code << 4) | ((size as u8) & 0x0f);
3270 size >>= 4;
3271 if size != 0 {
3272 byte |= 0x80;
3273 }
3274 out.push(byte);
3275 while size != 0 {
3276 let mut byte = (size as u8) & 0x7f;
3277 size >>= 7;
3278 if size != 0 {
3279 byte |= 0x80;
3280 }
3281 out.push(byte);
3282 }
3283 }
3284
3285 fn write_ofs_delta_offset(out: &mut Vec<u8>, relative: usize) {
3286 assert!(relative < 0x80);
3287 out.push(relative as u8);
3288 }
3289
3290 fn single_entry_index(
3291 format: ObjectFormat,
3292 oid: ObjectId,
3293 crc32: u32,
3294 offset: u32,
3295 pack_checksum: ObjectId,
3296 ) -> Vec<u8> {
3297 let mut index = Vec::new();
3298 index.extend_from_slice(&[0xff, b't', b'O', b'c']);
3299 index.extend_from_slice(&2u32.to_be_bytes());
3300 for idx in 0..256 {
3301 let count = if idx >= usize::from(oid.as_bytes()[0]) {
3302 1u32
3303 } else {
3304 0u32
3305 };
3306 index.extend_from_slice(&count.to_be_bytes());
3307 }
3308 index.extend_from_slice(oid.as_bytes());
3309 index.extend_from_slice(&crc32.to_be_bytes());
3310 index.extend_from_slice(&offset.to_be_bytes());
3311 index.extend_from_slice(pack_checksum.as_bytes());
3312 let checksum =
3313 sley_core::digest_bytes(format, &index).expect("test operation should succeed");
3314 index.extend_from_slice(checksum.as_bytes());
3315 index
3316 }
3317
3318 fn single_entry_index_v1(
3319 format: ObjectFormat,
3320 oid: ObjectId,
3321 offset: u32,
3322 pack_checksum: ObjectId,
3323 ) -> Vec<u8> {
3324 let mut index = Vec::new();
3325 for idx in 0..256 {
3326 let count = if idx >= usize::from(oid.as_bytes()[0]) {
3327 1u32
3328 } else {
3329 0u32
3330 };
3331 index.extend_from_slice(&count.to_be_bytes());
3332 }
3333 index.extend_from_slice(&offset.to_be_bytes());
3334 index.extend_from_slice(oid.as_bytes());
3335 index.extend_from_slice(pack_checksum.as_bytes());
3336 let checksum =
3337 sley_core::digest_bytes(format, &index).expect("test operation should succeed");
3338 index.extend_from_slice(checksum.as_bytes());
3339 index
3340 }
3341
3342 fn pack_reverse_index(
3343 format: ObjectFormat,
3344 positions: &[u32],
3345 pack_checksum: ObjectId,
3346 ) -> Vec<u8> {
3347 let mut reverse_index = Vec::new();
3348 reverse_index.extend_from_slice(b"RIDX");
3349 reverse_index.extend_from_slice(&1u32.to_be_bytes());
3350 reverse_index.extend_from_slice(&hash_function_id(format).to_be_bytes());
3351 for position in positions {
3352 reverse_index.extend_from_slice(&position.to_be_bytes());
3353 }
3354 reverse_index.extend_from_slice(pack_checksum.as_bytes());
3355 let checksum =
3356 sley_core::digest_bytes(format, &reverse_index).expect("test operation should succeed");
3357 reverse_index.extend_from_slice(checksum.as_bytes());
3358 reverse_index
3359 }
3360
3361 fn pack_mtimes(format: ObjectFormat, mtimes: &[u32], pack_checksum: ObjectId) -> Vec<u8> {
3362 let mut out = Vec::new();
3363 out.extend_from_slice(b"MTME");
3364 out.extend_from_slice(&1u32.to_be_bytes());
3365 out.extend_from_slice(&hash_function_id(format).to_be_bytes());
3366 for mtime in mtimes {
3367 out.extend_from_slice(&mtime.to_be_bytes());
3368 }
3369 out.extend_from_slice(pack_checksum.as_bytes());
3370 let checksum =
3371 sley_core::digest_bytes(format, &out).expect("test operation should succeed");
3372 out.extend_from_slice(checksum.as_bytes());
3373 out
3374 }
3375
3376 fn midx_chunks_with_pack_names(
3377 _format: ObjectFormat,
3378 pack_names: Vec<u8>,
3379 entries: &[(ObjectId, u32, u64)],
3380 ) -> Vec<([u8; 4], Vec<u8>)> {
3381 let mut entries = entries.to_vec();
3382 entries.sort_by(|left, right| left.0.as_bytes().cmp(right.0.as_bytes()));
3383 let object_ids: Vec<ObjectId> = entries.iter().map(|entry| entry.0).collect();
3384 let mut large_offsets = Vec::new();
3385 let mut chunks = vec![
3386 (*b"PNAM", pack_names),
3387 (*b"OIDF", midx_oid_fanout(&object_ids)),
3388 (*b"OIDL", midx_oid_lookup(&object_ids)),
3389 (
3390 *b"OOFF",
3391 midx_ooff_entries(
3392 &entries
3393 .iter()
3394 .map(|(_oid, pack_int_id, offset)| (*pack_int_id, *offset))
3395 .collect::<Vec<_>>(),
3396 &mut large_offsets,
3397 ),
3398 ),
3399 ];
3400 if !large_offsets.is_empty() {
3401 chunks.push((*b"LOFF", large_offsets));
3402 }
3403 chunks
3404 }
3405
3406 fn midx_oid_fanout(object_ids: &[ObjectId]) -> Vec<u8> {
3407 let mut counts = [0u32; 256];
3408 for oid in object_ids {
3409 counts[oid.as_bytes()[0] as usize] += 1;
3410 }
3411 let mut running = 0u32;
3412 let mut out = Vec::new();
3413 for count in counts {
3414 running += count;
3415 out.extend_from_slice(&running.to_be_bytes());
3416 }
3417 out
3418 }
3419
3420 fn midx_oid_lookup(object_ids: &[ObjectId]) -> Vec<u8> {
3421 let mut out = Vec::new();
3422 for oid in object_ids {
3423 out.extend_from_slice(oid.as_bytes());
3424 }
3425 out
3426 }
3427
3428 fn midx_ooff_entries(entries: &[(u32, u64)], large_offsets: &mut Vec<u8>) -> Vec<u8> {
3429 let mut out = Vec::new();
3430 for (pack_int_id, offset) in entries {
3431 out.extend_from_slice(&pack_int_id.to_be_bytes());
3432 if *offset < 0x8000_0000 {
3433 out.extend_from_slice(&(*offset as u32).to_be_bytes());
3434 } else {
3435 let large_idx = (large_offsets.len() / 8) as u32;
3436 out.extend_from_slice(&(0x8000_0000 | large_idx).to_be_bytes());
3437 large_offsets.extend_from_slice(&offset.to_be_bytes());
3438 }
3439 }
3440 out
3441 }
3442
3443 fn midx_u32_table(values: &[u32]) -> Vec<u8> {
3444 let mut out = Vec::new();
3445 for value in values {
3446 out.extend_from_slice(&value.to_be_bytes());
3447 }
3448 out
3449 }
3450
3451 fn midx_bitmap_packs(entries: &[(u32, u32)]) -> Vec<u8> {
3452 let mut out = Vec::new();
3453 for (bitmap_pos, bitmap_nr) in entries {
3454 out.extend_from_slice(&bitmap_pos.to_be_bytes());
3455 out.extend_from_slice(&bitmap_nr.to_be_bytes());
3456 }
3457 out
3458 }
3459
3460 fn multi_pack_index(
3461 format: ObjectFormat,
3462 version: u8,
3463 pack_count: u32,
3464 chunks: &[([u8; 4], Vec<u8>)],
3465 ) -> Vec<u8> {
3466 let lookup_len = (chunks.len() + 1) * 12;
3467 let mut out = Vec::new();
3468 out.extend_from_slice(b"MIDX");
3469 out.push(version);
3470 out.push(hash_function_id(format) as u8);
3471 out.push(chunks.len() as u8);
3472 out.push(0);
3473 out.extend_from_slice(&pack_count.to_be_bytes());
3474 let mut chunk_offset = (12 + lookup_len) as u64;
3475 for (id, data) in chunks {
3476 out.extend_from_slice(id);
3477 out.extend_from_slice(&chunk_offset.to_be_bytes());
3478 chunk_offset += data.len() as u64;
3479 }
3480 out.extend_from_slice(&[0, 0, 0, 0]);
3481 out.extend_from_slice(&chunk_offset.to_be_bytes());
3482 for (_id, data) in chunks {
3483 out.extend_from_slice(data);
3484 }
3485 let checksum =
3486 sley_core::digest_bytes(format, &out).expect("test operation should succeed");
3487 out.extend_from_slice(checksum.as_bytes());
3488 out
3489 }
3490
3491 fn pack_checksum_sha1() -> ObjectId {
3494 sley_core::digest_bytes(ObjectFormat::Sha1, b"pack").expect("test operation should succeed")
3495 }
3496
3497 fn parse_ewah_bytes(bytes: &[u8]) -> EwahBitmap {
3498 let mut offset = 0usize;
3501 let checksum_offset = bytes.len();
3502 parse_bitmap_ewah(bytes, &mut offset, checksum_offset, 0)
3503 .expect("test operation should succeed")
3504 }
3505
3506 #[test]
3507 fn ewah_encodes_single_literal_word_matching_helper() {
3508 let ewah = EwahBitmap::from_words(64, &[0b101]).expect("test operation should succeed");
3512 assert_eq!(ewah.words, ewah_literal_words(&[0b101]));
3513 assert_eq!(ewah.rlw_position, 0);
3514 assert_eq!(ewah.bit_size, 64);
3515 }
3516
3517 #[test]
3518 fn ewah_byte_layout_is_big_endian() {
3519 let ewah = EwahBitmap::from_words(64, &[0x0102_0304_0506_0708])
3520 .expect("test operation should succeed");
3521 let bytes = ewah.to_bytes();
3522 let mut expected = Vec::new();
3523 expected.extend_from_slice(&64u32.to_be_bytes()); expected.extend_from_slice(&2u32.to_be_bytes()); expected.extend_from_slice(&(1u64 << 33).to_be_bytes()); expected.extend_from_slice(&0x0102_0304_0506_0708u64.to_be_bytes());
3527 expected.extend_from_slice(&0u32.to_be_bytes()); assert_eq!(bytes, expected);
3529 }
3530
3531 #[test]
3532 fn ewah_empty_bitmap_serialises_like_git() {
3533 let ewah = EwahBitmap::empty();
3534 let bytes = ewah.to_bytes();
3535 assert_eq!(bytes, vec![0u8; 12]);
3537 let parsed = parse_ewah_bytes(&bytes);
3539 assert_eq!(parsed, ewah);
3540 assert!(
3541 parsed
3542 .to_positions()
3543 .expect("test operation should succeed")
3544 .is_empty()
3545 );
3546 }
3547
3548 #[test]
3549 fn ewah_compresses_clean_zero_run() {
3550 let ewah =
3553 EwahBitmap::from_words(256, &[0, 0, 0, 0b1]).expect("test operation should succeed");
3554 assert_eq!(ewah.words.len(), 2, "expected one RLW plus one literal");
3555 let rlw = ewah.words[0];
3556 assert_eq!(rlw & 1, 0, "run bit should be zero");
3557 assert_eq!((rlw >> 1) & 0xffff_ffff, 3, "run length should be 3");
3558 assert_eq!(rlw >> 33, 1, "literal length should be 1");
3559 assert_eq!(ewah.words[1], 0b1);
3560 }
3561
3562 #[test]
3563 fn ewah_compresses_clean_ones_run() {
3564 let ewah = EwahBitmap::from_words(192, &[u64::MAX, u64::MAX, u64::MAX])
3565 .expect("test operation should succeed");
3566 assert_eq!(ewah.words.len(), 1);
3568 let rlw = ewah.words[0];
3569 assert_eq!(rlw & 1, 1, "run bit should be one");
3570 assert_eq!((rlw >> 1) & 0xffff_ffff, 3, "run length should be 3");
3571 assert_eq!(rlw >> 33, 0, "no literals");
3572 }
3573
3574 #[test]
3575 fn ewah_run_then_literal_then_run_roundtrips() {
3576 let words = vec![0, 0, 0xdead_beef, u64::MAX, u64::MAX, 0, 0xabc];
3577 let bit_size = (words.len() * 64) as u32;
3578 let ewah = EwahBitmap::from_words(bit_size, &words).expect("test operation should succeed");
3579 assert_eq!(
3580 ewah.to_words().expect("test operation should succeed"),
3581 words
3582 );
3583 }
3584
3585 #[test]
3586 fn ewah_drops_trailing_clean_zero_words() {
3587 let words = vec![0b1, 0, 0, 0];
3590 let ewah = EwahBitmap::from_words(1, &words).expect("test operation should succeed");
3591 assert_eq!(ewah.bit_size, 1);
3593 assert_eq!(
3594 ewah.to_words().expect("test operation should succeed"),
3595 vec![0b1]
3596 );
3597 }
3598
3599 #[test]
3600 fn ewah_from_positions_roundtrips_via_positions() {
3601 let positions = [0u32, 1, 63, 64, 65, 200, 511];
3602 let ewah =
3603 EwahBitmap::from_positions(512, &positions).expect("test operation should succeed");
3604 let mut decoded = ewah.to_positions().expect("test operation should succeed");
3605 decoded.sort_unstable();
3606 assert_eq!(decoded, positions);
3607 }
3608
3609 #[test]
3610 fn ewah_from_positions_dedupes_and_orders() {
3611 let ewah = EwahBitmap::from_positions(128, &[100, 5, 100, 5, 5])
3612 .expect("test operation should succeed");
3613 assert_eq!(
3614 ewah.to_positions().expect("test operation should succeed"),
3615 vec![5, 100]
3616 );
3617 }
3618
3619 #[test]
3620 fn ewah_huge_zero_run_spans_multiple_rlws() {
3621 let mut builder = EwahBuilder::new(0);
3626 builder.add_empty_words(false, 0xffff_ffff);
3627 builder.add_empty_words(false, 5);
3628 let ewah = builder.finish().expect("test operation should succeed");
3629 assert_eq!(ewah.words.len(), 2, "run split across two RLWs");
3630 assert_eq!((ewah.words[0] >> 1) & 0xffff_ffff, 0xffff_ffff);
3631 assert_eq!(ewah.words[1] & 1, 0);
3632 assert_eq!((ewah.words[1] >> 1) & 0xffff_ffff, 5);
3633 assert_eq!(ewah.rlw_position, 1);
3634 }
3635
3636 #[test]
3637 fn ewah_from_words_rejects_oversized_bit_size() {
3638 assert!(EwahBitmap::from_words(65, &[0]).is_err());
3640 }
3641
3642 #[test]
3643 fn ewah_from_positions_rejects_out_of_range() {
3644 assert!(EwahBitmap::from_positions(64, &[64]).is_err());
3645 }
3646
3647 #[test]
3648 fn ewah_serialised_bytes_reparse_to_equal_bitmap() {
3649 let words = vec![0, u64::MAX, 0x1234_5678_9abc_def0, 0, 0, 0xff];
3652 let bit_size = (words.len() * 64) as u32;
3653 let ewah = EwahBitmap::from_words(bit_size, &words).expect("test operation should succeed");
3654 let bytes = ewah.to_bytes();
3655 let parsed = parse_ewah_bytes(&bytes);
3656 assert_eq!(parsed, ewah);
3657 assert_eq!(
3658 parsed.to_words().expect("test operation should succeed"),
3659 words
3660 );
3661 }
3662
3663 #[test]
3664 fn pack_bitmap_index_write_parse_roundtrip_sha1() {
3665 let object_types = [ObjectType::Commit, ObjectType::Tree, ObjectType::Blob];
3667 let bytes = write_bitmap(
3668 ObjectFormat::Sha1,
3669 pack_checksum_sha1(),
3670 &object_types,
3671 &[(0u32, 0u32, vec![1u32, 2u32])],
3672 None,
3673 )
3674 .expect("test operation should succeed");
3675 assert_eq!(&bytes[..4], b"BITM");
3676
3677 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, 3)
3678 .expect("test operation should succeed");
3679 assert_eq!(parsed.version, 1);
3680 assert_eq!(parsed.options, PackBitmapIndex::OPTION_FULL_DAG);
3681 assert_eq!(parsed.pack_checksum, pack_checksum_sha1());
3682 assert_eq!(
3683 parsed
3684 .type_bitmaps
3685 .commits
3686 .to_positions()
3687 .expect("test operation should succeed"),
3688 vec![0]
3689 );
3690 assert_eq!(
3691 parsed
3692 .type_bitmaps
3693 .trees
3694 .to_positions()
3695 .expect("test operation should succeed"),
3696 vec![1]
3697 );
3698 assert_eq!(
3699 parsed
3700 .type_bitmaps
3701 .blobs
3702 .to_positions()
3703 .expect("test operation should succeed"),
3704 vec![2]
3705 );
3706 assert!(
3707 parsed
3708 .type_bitmaps
3709 .tags
3710 .to_positions()
3711 .expect("test operation should succeed")
3712 .is_empty()
3713 );
3714 assert_eq!(parsed.entries.len(), 1);
3715 let entry = parsed
3716 .entry_for_index_position(0)
3717 .expect("test operation should succeed");
3718 assert_eq!(entry.xor_offset, 0);
3719 assert_eq!(entry.flags, 0);
3720 assert_eq!(
3721 entry
3722 .bitmap
3723 .to_positions()
3724 .expect("test operation should succeed"),
3725 vec![0, 1, 2]
3726 );
3727 assert_eq!(parsed.name_hash_cache, None);
3728 }
3729
3730 #[test]
3731 fn pack_bitmap_index_write_parse_roundtrip_sha256() {
3732 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha256, b"pack")
3733 .expect("test operation should succeed");
3734 let object_types = [ObjectType::Commit, ObjectType::Tree];
3735 let bytes = write_bitmap(
3736 ObjectFormat::Sha256,
3737 pack_checksum.clone(),
3738 &object_types,
3739 &[(0u32, 0u32, vec![1u32])],
3740 None,
3741 )
3742 .expect("test operation should succeed");
3743 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha256, 2)
3744 .expect("test operation should succeed");
3745 assert_eq!(parsed.format, ObjectFormat::Sha256);
3746 assert_eq!(parsed.pack_checksum, pack_checksum);
3747 assert_eq!(parsed.index_checksum.format(), ObjectFormat::Sha256);
3748 assert_eq!(
3749 parsed.entries[0]
3750 .bitmap
3751 .to_positions()
3752 .expect("test operation should succeed"),
3753 vec![0, 1]
3754 );
3755 }
3756
3757 #[test]
3758 fn pack_bitmap_index_write_includes_name_hash_cache() {
3759 let object_types = [ObjectType::Commit, ObjectType::Tree, ObjectType::Blob];
3760 let cache = vec![0x1111_1111u32, 0x2222_2222, 0x3333_3333];
3761 let bytes = write_bitmap(
3762 ObjectFormat::Sha1,
3763 pack_checksum_sha1(),
3764 &object_types,
3765 &[(0u32, 0u32, vec![1u32, 2u32])],
3766 Some(cache.clone()),
3767 )
3768 .expect("test operation should succeed");
3769 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, 3)
3770 .expect("test operation should succeed");
3771 assert_eq!(
3772 parsed.options,
3773 PackBitmapIndex::OPTION_FULL_DAG | PackBitmapIndex::OPTION_HASH_CACHE
3774 );
3775 assert_eq!(parsed.name_hash_cache, Some(cache));
3776 }
3777
3778 #[test]
3779 fn pack_bitmap_writer_supports_multiple_commits() {
3780 let object_types = [
3781 ObjectType::Commit,
3782 ObjectType::Commit,
3783 ObjectType::Tree,
3784 ObjectType::Blob,
3785 ];
3786 let mut writer =
3787 PackBitmapWriter::new(ObjectFormat::Sha1, pack_checksum_sha1(), &object_types)
3788 .expect("test operation should succeed");
3789 writer
3790 .add_commit(0, 0, &[2, 3])
3791 .expect("test operation should succeed");
3792 writer
3793 .add_commit(1, 1, &[2])
3794 .expect("test operation should succeed");
3795 let bytes = writer.write().expect("test operation should succeed");
3796 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, 4)
3797 .expect("test operation should succeed");
3798 assert_eq!(parsed.entries.len(), 2);
3799 assert_eq!(
3800 parsed
3801 .type_bitmaps
3802 .commits
3803 .to_positions()
3804 .expect("test operation should succeed"),
3805 vec![0, 1]
3806 );
3807 let first = parsed
3808 .entry_for_index_position(0)
3809 .expect("test operation should succeed");
3810 assert_eq!(
3811 first
3812 .bitmap
3813 .to_positions()
3814 .expect("test operation should succeed"),
3815 vec![0, 2, 3]
3816 );
3817 let second = parsed
3818 .entry_for_index_position(1)
3819 .expect("test operation should succeed");
3820 assert_eq!(
3821 second
3822 .bitmap
3823 .to_positions()
3824 .expect("test operation should succeed"),
3825 vec![1, 2]
3826 );
3827 }
3828
3829 #[test]
3830 fn pack_bitmap_writer_roundtrips_lookup_table() {
3831 let object_types = [ObjectType::Commit, ObjectType::Commit, ObjectType::Tree];
3832 let mut writer =
3833 PackBitmapWriter::new(ObjectFormat::Sha1, pack_checksum_sha1(), &object_types)
3834 .expect("test operation should succeed")
3835 .with_lookup_table(true);
3836 writer
3837 .add_commit(0, 1, &[2])
3838 .expect("test operation should succeed");
3839 writer
3840 .add_commit(1, 0, &[2])
3841 .expect("test operation should succeed");
3842 let bytes = writer.write().expect("test operation should succeed");
3843 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, 3)
3844 .expect("test operation should succeed");
3845 assert!(parsed.lookup_table);
3846 assert_ne!(parsed.options & PackBitmapIndex::OPTION_LOOKUP_TABLE, 0);
3847 assert_eq!(parsed.entries.len(), 2);
3848 }
3849
3850 #[test]
3851 fn pack_bitmap_index_recomputes_checksum_on_write() {
3852 let object_types = [ObjectType::Commit, ObjectType::Blob];
3855 let writer = PackBitmapWriter::new(ObjectFormat::Sha1, pack_checksum_sha1(), &object_types)
3856 .expect("test operation should succeed");
3857 let mut index = writer.build().expect("test operation should succeed");
3858 assert_eq!(index.index_checksum.as_bytes(), [0u8; 20]);
3860 index.entries.clear(); index.entries.push(PackBitmapEntry {
3862 object_position: 0,
3863 xor_offset: 0,
3864 flags: 0,
3865 bitmap: EwahBitmap::from_positions(2, &[0, 1]).expect("test operation should succeed"),
3866 });
3867 let bytes = index.write().expect("test operation should succeed");
3868 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, 2)
3870 .expect("test operation should succeed");
3871 assert_ne!(parsed.index_checksum.as_bytes(), [0u8; 20]);
3872 }
3873
3874 #[test]
3875 fn pack_bitmap_writer_rejects_non_commit_selection() {
3876 let object_types = [ObjectType::Commit, ObjectType::Blob];
3877 let mut writer =
3878 PackBitmapWriter::new(ObjectFormat::Sha1, pack_checksum_sha1(), &object_types)
3879 .expect("test operation should succeed");
3880 assert!(writer.add_commit(1, 1, &[]).is_err());
3882 assert!(writer.add_commit(5, 5, &[]).is_err());
3884 assert!(writer.add_commit(0, 5, &[]).is_err());
3886 assert!(writer.add_commit(0, 0, &[9]).is_err());
3888 }
3889
3890 #[test]
3891 fn pack_bitmap_writer_rejects_checksum_format_mismatch() {
3892 let sha256_checksum = sley_core::digest_bytes(ObjectFormat::Sha256, b"pack")
3893 .expect("test operation should succeed");
3894 assert!(
3895 PackBitmapWriter::new(ObjectFormat::Sha1, sha256_checksum, &[ObjectType::Commit])
3896 .is_err()
3897 );
3898 }
3899
3900 #[test]
3901 fn pack_bitmap_writer_rejects_bad_name_hash_cache_len() {
3902 let writer = PackBitmapWriter::new(
3903 ObjectFormat::Sha1,
3904 pack_checksum_sha1(),
3905 &[ObjectType::Commit],
3906 )
3907 .expect("test operation should succeed");
3908 assert!(writer.with_name_hash_cache(vec![1, 2]).is_err());
3909 }
3910
3911 #[test]
3912 fn pack_bitmap_index_write_rejects_inconsistent_cache_flag() {
3913 let mut index = PackBitmapWriter::new(
3914 ObjectFormat::Sha1,
3915 pack_checksum_sha1(),
3916 &[ObjectType::Commit],
3917 )
3918 .expect("test operation should succeed")
3919 .build()
3920 .expect("test operation should succeed");
3921 index.options |= PackBitmapIndex::OPTION_HASH_CACHE;
3923 assert!(index.write().is_err());
3924 index.options = PackBitmapIndex::OPTION_FULL_DAG;
3926 index.name_hash_cache = Some(vec![0]);
3927 assert!(index.write().is_err());
3928 }
3929
3930 #[test]
3931 fn write_bitmap_roundtrips_through_upstream_git_parser() {
3932 let root = unique_temp_dir("git-pack-bitmap-writer");
3936 fs::create_dir_all(&root).expect("test operation should succeed");
3937 {
3938 run_git_success(&root, &["init", "-q", "-b", "main"]);
3939 run_git_success(
3940 &root,
3941 &[
3942 "-c",
3943 "user.name=Example User",
3944 "-c",
3945 "user.email=example@example.invalid",
3946 "commit",
3947 "--allow-empty",
3948 "-q",
3949 "-m",
3950 "one",
3951 ],
3952 );
3953 run_git_success(&root, &["repack", "-adb"]);
3954 let pack_dir = root.join(".git").join("objects").join("pack");
3955 let idx_path = single_path_with_extension(&pack_dir, "idx");
3956 let index = PackIndex::parse(
3957 &fs::read(idx_path).expect("test operation should succeed"),
3958 ObjectFormat::Sha1,
3959 )
3960 .expect("test operation should succeed");
3961 let pack_path = single_path_with_extension(&pack_dir, "pack");
3963 let pack =
3964 PackFile::parse_sha1(&fs::read(pack_path).expect("test operation should succeed"))
3965 .expect("test operation should succeed");
3966 let mut offsets: Vec<u64> = index.entries.iter().map(|entry| entry.offset).collect();
3969 offsets.sort_unstable();
3970 let position_of = |offset: u64| -> u32 {
3971 offsets
3972 .iter()
3973 .position(|value| *value == offset)
3974 .expect("test operation should succeed") as u32
3975 };
3976 let mut object_types = vec![ObjectType::Blob; index.entries.len()];
3977 for entry in &index.entries {
3978 let position = position_of(entry.offset) as usize;
3979 if let Some(parsed) = pack
3981 .entries
3982 .iter()
3983 .find(|po| po.entry.offset == entry.offset)
3984 {
3985 object_types[position] = parsed.object.object_type;
3986 }
3987 }
3988 let commit_position = object_types
3990 .iter()
3991 .position(|ty| *ty == ObjectType::Commit)
3992 .expect("test operation should succeed") as u32;
3993 let commit_index_position = index
3995 .entries
3996 .iter()
3997 .position(|entry| position_of(entry.offset) == commit_position)
3998 .expect("test operation should succeed")
3999 as u32;
4000 let reachable: Vec<u32> = (0..index.entries.len() as u32).collect();
4001 let bytes = write_bitmap(
4002 ObjectFormat::Sha1,
4003 index.pack_checksum.clone(),
4004 &object_types,
4005 &[(commit_position, commit_index_position, reachable)],
4006 None,
4007 )
4008 .expect("test operation should succeed");
4009 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, index.entries.len())
4010 .expect("test operation should succeed");
4011 assert_eq!(parsed.pack_checksum, index.pack_checksum);
4012 assert_eq!(parsed.entries.len(), 1);
4013 assert_eq!(
4014 parsed.entries[0]
4015 .bitmap
4016 .to_positions()
4017 .expect("test operation should succeed")
4018 .len(),
4019 index.entries.len()
4020 );
4021 };
4022 let _ = fs::remove_dir_all(&root);
4023 }
4024
4025 fn zlib_compress(bytes: &[u8]) -> Vec<u8> {
4028 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
4029 encoder
4030 .write_all(bytes)
4031 .expect("test operation should succeed");
4032 encoder.finish().expect("test operation should succeed")
4033 }
4034
4035 fn pack_with_declared_object_count(declared: u32) -> Vec<u8> {
4038 let mut pack = Vec::new();
4039 pack.extend_from_slice(b"PACK");
4040 pack.extend_from_slice(&2u32.to_be_bytes());
4041 pack.extend_from_slice(&declared.to_be_bytes());
4042 let checksum = sley_core::digest_bytes(ObjectFormat::Sha1, &pack)
4043 .expect("test operation should succeed");
4044 pack.extend_from_slice(checksum.as_bytes());
4045 pack
4046 }
4047
4048 #[test]
4056 fn rejects_absurd_declared_object_count_without_preallocating() {
4057 const CHILD_ENV: &str = "SLEY_PACK_OBJECT_COUNT_BOMB_CHILD";
4058 const TEST_PATH: &str = "tests::rejects_absurd_declared_object_count_without_preallocating";
4059
4060 if std::env::var_os(CHILD_ENV).is_some() {
4061 for declared in [u32::MAX, 1 << 30, 1 << 24] {
4062 let pack = pack_with_declared_object_count(declared);
4063 assert!(
4064 PackFile::parse_sha1(&pack).is_err(),
4065 "a 32-byte pack declaring {declared} objects must be rejected"
4066 );
4067 assert!(
4068 PackIndex::write_v2_for_pack(&pack, ObjectFormat::Sha1).is_err(),
4069 "index-pack must reject {declared} objects in a 32-byte pack too"
4070 );
4071 assert!(
4072 PackFile::verify_pack_stats(&pack, ObjectFormat::Sha1).is_err(),
4073 "verify-pack must reject {declared} objects in a 32-byte pack too"
4074 );
4075 }
4076 return;
4077 }
4078
4079 let exe = std::env::current_exe().expect("test binary path should be available");
4080 let status = Command::new(exe)
4081 .args(["--exact", TEST_PATH, "--nocapture"])
4082 .env(CHILD_ENV, "1")
4083 .status()
4084 .expect("re-running the test binary should succeed");
4085 assert!(
4086 status.success(),
4087 "child process died ({status:?}) parsing packs with oversized declared \
4088 object counts — the declared count reached an allocation instead of a \
4089 bounds check"
4090 );
4091 }
4092
4093 #[test]
4097 fn rejects_declared_object_count_larger_than_pack_can_hold() {
4098 let pack = pack_with_declared_object_count(64);
4099 let error = PackFile::parse_sha1(&pack).expect_err("declared count must be rejected");
4100 assert!(
4101 format!("{error}").contains("only has room for"),
4102 "expected a declared-count bound error, got: {error}"
4103 );
4104 }
4105
4106 fn literal_delta(base_len: usize, result: &[u8]) -> Vec<u8> {
4110 assert!(result.len() <= 0x7f, "insert instruction size is one byte");
4111 let mut delta = Vec::new();
4112 write_delta_varint(&mut delta, base_len as u64);
4113 write_delta_varint(&mut delta, result.len() as u64);
4114 delta.push(result.len() as u8);
4115 delta.extend_from_slice(result);
4116 delta
4117 }
4118
4119 fn chain_bodies(depth: usize) -> Vec<Vec<u8>> {
4121 (0..=depth)
4122 .map(|idx| format!("{idx:08}").into_bytes())
4123 .collect()
4124 }
4125
4126 fn ofs_delta_chain_pack(format: ObjectFormat, depth: usize) -> Vec<u8> {
4129 let bodies = chain_bodies(depth);
4130 let mut pack = Vec::new();
4131 pack.extend_from_slice(b"PACK");
4132 pack.extend_from_slice(&2u32.to_be_bytes());
4133 pack.extend_from_slice(&(depth as u32 + 1).to_be_bytes());
4134
4135 let mut base_offset = pack.len();
4136 write_entry_header(&mut pack, ObjectType::Blob, bodies[0].len() as u64);
4137 pack.extend_from_slice(&zlib_compress(&bodies[0]));
4138
4139 for idx in 1..=depth {
4140 let delta = literal_delta(bodies[idx - 1].len(), &bodies[idx]);
4141 let entry_offset = pack.len();
4142 write_pack_entry_header_kind(&mut pack, 6, delta.len() as u64);
4143 write_ofs_delta_offset(&mut pack, entry_offset - base_offset);
4144 pack.extend_from_slice(&zlib_compress(&delta));
4145 base_offset = entry_offset;
4146 }
4147
4148 let checksum =
4149 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
4150 pack.extend_from_slice(checksum.as_bytes());
4151 pack
4152 }
4153
4154 fn ref_delta_chain_pack(format: ObjectFormat, depth: usize, reversed: bool) -> Vec<u8> {
4161 let bodies = chain_bodies(depth);
4162
4163 let mut base_entry = Vec::new();
4164 write_entry_header(&mut base_entry, ObjectType::Blob, bodies[0].len() as u64);
4165 base_entry.extend_from_slice(&zlib_compress(&bodies[0]));
4166
4167 let mut delta_entries = Vec::with_capacity(depth);
4168 for idx in 1..=depth {
4169 let delta = literal_delta(bodies[idx - 1].len(), &bodies[idx]);
4170 let base_oid = sley_core::object_id_for_bytes(format, "blob", &bodies[idx - 1])
4171 .expect("test operation should succeed");
4172 let mut entry = Vec::new();
4173 write_pack_entry_header_kind(&mut entry, 7, delta.len() as u64);
4174 entry.extend_from_slice(base_oid.as_bytes());
4175 entry.extend_from_slice(&zlib_compress(&delta));
4176 delta_entries.push(entry);
4177 }
4178 if reversed {
4179 delta_entries.reverse();
4180 }
4181
4182 let mut pack = Vec::new();
4183 pack.extend_from_slice(b"PACK");
4184 pack.extend_from_slice(&2u32.to_be_bytes());
4185 pack.extend_from_slice(&(depth as u32 + 1).to_be_bytes());
4186 if !reversed {
4187 pack.extend_from_slice(&base_entry);
4188 }
4189 for entry in &delta_entries {
4190 pack.extend_from_slice(entry);
4191 }
4192 if reversed {
4193 pack.extend_from_slice(&base_entry);
4194 }
4195
4196 let checksum =
4197 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
4198 pack.extend_from_slice(checksum.as_bytes());
4199 pack
4200 }
4201
4202 #[test]
4205 fn resolves_delta_chain_at_the_depth_ceiling() {
4206 for pack in [
4207 ofs_delta_chain_pack(ObjectFormat::Sha1, DEFAULT_PACK_DEPTH),
4208 ref_delta_chain_pack(ObjectFormat::Sha1, DEFAULT_PACK_DEPTH, false),
4209 ] {
4210 let parsed = PackFile::parse_sha1(&pack).expect("a chain at the ceiling must resolve");
4211 assert_eq!(parsed.entries.len(), DEFAULT_PACK_DEPTH + 1);
4212 }
4213 }
4214
4215 #[test]
4216 fn configured_delta_chain_depth_limit_allows_deeper_pack() {
4217 let limits = PackReadLimits {
4218 max_delta_depth: 60,
4219 ..PackReadLimits::default()
4220 };
4221 let pack = ofs_delta_chain_pack(ObjectFormat::Sha1, 60);
4222
4223 let parsed = PackFile::parse_with_limits(&pack, ObjectFormat::Sha1, limits)
4224 .expect("a chain at the configured ceiling must resolve");
4225 assert_eq!(parsed.entries.len(), 61);
4226
4227 let indexed = PackIndex::write_v2_for_pack_with_limits(&pack, ObjectFormat::Sha1, limits)
4228 .expect("the in-memory indexer must use the configured ceiling");
4229 assert_eq!(indexed.entries.len(), 61);
4230
4231 let streamed = PackIndex::write_v2_for_pack_reader_to_trailer_with_limits(
4232 &mut Cursor::new(pack),
4233 ObjectFormat::Sha1,
4234 limits,
4235 )
4236 .expect("the streaming indexer must use the configured ceiling");
4237 assert_eq!(streamed.entries.len(), 61);
4238 }
4239
4240 #[test]
4241 fn configured_delta_chain_depth_limit_remains_finite_and_actionable() {
4242 let limits = PackReadLimits {
4243 max_delta_depth: 60,
4244 ..PackReadLimits::default()
4245 };
4246 let pack = ofs_delta_chain_pack(ObjectFormat::Sha1, 61);
4247
4248 let error = PackFile::parse_with_limits(&pack, ObjectFormat::Sha1, limits)
4249 .expect_err("a chain past the configured ceiling must be rejected");
4250 let message = format!("{error}");
4251
4252 assert!(
4253 message.contains("observed depth 61"),
4254 "expected the observed depth in the error, got: {error}"
4255 );
4256 assert!(
4257 message.contains("configured limit 60"),
4258 "expected the configured limit in the error, got: {error}"
4259 );
4260 assert!(
4261 message.contains("PackReadLimits::max_delta_depth")
4262 && message.contains("git repack --depth="),
4263 "expected actionable remedies in the error, got: {error}"
4264 );
4265 }
4266
4267 #[test]
4270 fn rejects_delta_chain_deeper_than_the_ceiling() {
4271 for pack in [
4272 ofs_delta_chain_pack(ObjectFormat::Sha1, DEFAULT_PACK_DEPTH + 1),
4273 ref_delta_chain_pack(ObjectFormat::Sha1, DEFAULT_PACK_DEPTH + 1, false),
4274 ] {
4275 let error =
4276 PackFile::parse_sha1(&pack).expect_err("a chain past the ceiling must be rejected");
4277 assert!(
4278 format!("{error}").contains("exceeds maximum depth"),
4279 "expected a chain-depth error, got: {error}"
4280 );
4281 }
4282 }
4283
4284 #[test]
4291 fn rejects_adversarially_ordered_long_delta_chain_promptly() {
4292 let pack = ref_delta_chain_pack(ObjectFormat::Sha1, 5_000, true);
4293 let started = std::time::Instant::now();
4294 let error = PackFile::parse_sha1(&pack)
4295 .expect_err("a long adversarial chain must be rejected, not resolved");
4296 let elapsed = started.elapsed();
4297 assert!(
4298 format!("{error}").contains("exceeds maximum depth"),
4299 "expected a chain-depth error, got: {error}"
4300 );
4301 assert!(
4302 elapsed < std::time::Duration::from_secs(20),
4303 "rejection took {elapsed:?}; resolution is still doing work proportional \
4304 to the chain length"
4305 );
4306 }
4307
4308 #[test]
4311 fn index_pack_rejects_delta_chain_deeper_than_the_ceiling() {
4312 let pack = ofs_delta_chain_pack(ObjectFormat::Sha1, DEFAULT_PACK_DEPTH + 1);
4313 let error = PackIndex::write_v2_for_pack(&pack, ObjectFormat::Sha1)
4314 .expect_err("index-pack must reject a chain past the ceiling");
4315 assert!(
4316 format!("{error}").contains("exceeds maximum depth"),
4317 "expected a chain-depth error, got: {error}"
4318 );
4319 }
4320}