1#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))]
4
5use flate2::{Compress, Compression, FlushCompress, Status};
6use sley_core::{GitError, ObjectFormat, ObjectId, Result, StreamingDigest};
7use sley_formats::Bundle;
8use sley_object::{EncodedObject, ObjectType};
9use std::borrow::Borrow;
10use std::cell::RefCell;
11use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
12use std::fmt;
13use std::fs::File;
14use std::io::{Read, Seek, SeekFrom, Write};
15use std::ops::Range;
16use std::path::Path;
17use std::sync::Arc;
18
19mod delta;
26mod index;
27pub mod inflate;
28mod read;
29mod stream;
30mod write;
31
32pub(crate) use delta::*;
33pub use index::*;
34pub use read::*;
35pub(crate) use stream::*;
36pub use write::*;
37
38#[derive(Debug, Clone, PartialEq, Eq)]
39pub struct PackEntry {
40 pub oid: ObjectId,
41 pub compressed_size: u64,
42 pub uncompressed_size: u64,
43 pub offset: u64,
44}
45#[derive(Debug, Clone, PartialEq, Eq)]
46pub struct RepackPolicy {
47 pub write_bitmaps: bool,
48 pub cruft_packs: bool,
49 pub geometric_factor: Option<u8>,
50}
51
52#[derive(Debug, Clone, PartialEq, Eq)]
53pub struct PackFile {
54 pub version: u32,
55 pub entries: Vec<PackObject>,
56 pub checksum: ObjectId,
57}
58
59#[derive(Debug, Clone, PartialEq, Eq)]
60pub struct PackObject {
61 pub entry: PackEntry,
62 pub object: EncodedObject,
63}
64
65#[derive(Debug, Clone, PartialEq, Eq)]
68pub struct PackVerifyStat {
69 pub oid: ObjectId,
71 pub object_type: ObjectType,
73 pub size: u64,
75 pub size_in_pack: u64,
78 pub offset: u64,
80 pub delta_depth: u32,
82 pub base_oid: Option<ObjectId>,
85}
86
87#[derive(Debug, Clone, PartialEq, Eq)]
90pub struct PackVerifyStats {
91 pub objects: Vec<PackVerifyStat>,
92 pub checksum: ObjectId,
93}
94
95#[derive(Debug, Clone, PartialEq, Eq)]
96pub struct PackWrite {
97 pub pack: Vec<u8>,
98 pub index: Vec<u8>,
99 pub checksum: ObjectId,
100 pub entries: Vec<PackIndexEntry>,
101 pub delta_count: u32,
102}
103
104#[derive(Debug, Clone, PartialEq, Eq)]
105pub struct PackWriteSummary {
106 pub index: Vec<u8>,
107 pub checksum: ObjectId,
108 pub entries: Vec<PackIndexEntry>,
109 pub delta_count: u32,
110 pub pack_size: u64,
111}
112
113#[derive(Debug, Clone, Copy, PartialEq, Eq)]
114pub struct PackInput<'a> {
115 pub oid: &'a ObjectId,
116 pub object: &'a EncodedObject,
117}
118#[derive(Debug, Clone, Copy, PartialEq, Eq)]
119enum PackObjectKind {
120 Commit,
121 Tree,
122 Blob,
123 Tag,
124 OfsDelta,
125 RefDelta,
126}
127
128#[derive(Debug, Clone, PartialEq, Eq)]
129enum ParsedPackEntry {
130 Resolved(PackObject),
131 Delta {
132 base: DeltaBase,
133 compressed_size: u64,
134 delta_size: u64,
135 offset: u64,
136 delta: Vec<u8>,
137 },
138}
139
140#[derive(Debug, Clone, PartialEq, Eq)]
141enum DeltaBase {
142 Offset(u64),
143 Ref(ObjectId),
144}
145
146struct OnDiskEntry {
150 offset: u64,
151 base: Option<DeltaBase>,
152 stream_size: u64,
153}
154#[derive(Debug, Clone, Copy, PartialEq, Eq)]
155struct EntryHeader {
156 kind: PackObjectKind,
157 size: u64,
158}
159fn next_byte(bytes: &[u8], offset: &mut usize) -> Result<u8> {
160 let Some(byte) = bytes.get(*offset).copied() else {
161 return Err(GitError::InvalidFormat(
162 "truncated pack entry header".into(),
163 ));
164 };
165 *offset += 1;
166 Ok(byte)
167}
168
169fn u16_be(bytes: &[u8]) -> u16 {
170 u16::from_be_bytes([bytes[0], bytes[1]])
171}
172
173fn u32_be(bytes: &[u8]) -> u32 {
174 u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])
175}
176
177fn u64_be(bytes: &[u8]) -> u64 {
178 u64::from_be_bytes([
179 bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7],
180 ])
181}
182fn checked_range(
183 start: usize,
184 count: usize,
185 width: usize,
186 total: usize,
187) -> Result<std::ops::Range<usize>> {
188 let len = count
189 .checked_mul(width)
190 .ok_or_else(|| GitError::InvalidFormat("pack index table overflow".into()))?;
191 let end = start
192 .checked_add(len)
193 .ok_or_else(|| GitError::InvalidFormat("pack index table overflow".into()))?;
194 if end > total {
195 return Err(GitError::InvalidFormat("truncated pack index table".into()));
196 }
197 Ok(start..end)
198}
199
200fn validate_position_permutation(positions: &[u32]) -> Result<()> {
201 let mut seen = vec![false; positions.len()];
202 for position in positions {
203 let idx = *position as usize;
204 if idx >= positions.len() {
205 return Err(GitError::InvalidFormat(format!(
206 "invalid rev-index position {position}"
207 )));
208 }
209 if seen[idx] {
210 return Err(GitError::InvalidFormat(format!(
211 "invalid rev-index position {position}"
212 )));
213 }
214 seen[idx] = true;
215 }
216 Ok(())
217}
218
219thread_local! {
225 pub(crate) static INFLATE: RefCell<flate2::Decompress> = RefCell::new(flate2::Decompress::new(true));
226}
227
228#[cfg(test)]
229mod tests {
230 use super::*;
231 use flate2::Compression;
232 use flate2::read::ZlibDecoder;
233 use flate2::write::ZlibEncoder;
234 use std::fs;
235 use std::io::Read;
236 use std::io::Write;
237 use std::path::{Path, PathBuf};
238 use std::process::Command;
239 use std::time::{SystemTime, UNIX_EPOCH};
240
241 fn delta_pack_options(prefer_ofs_delta: bool) -> PackWriteOptions {
242 PackWriteOptions::new()
243 .with_prefer_ofs_delta(prefer_ofs_delta)
244 .with_reorder(false)
245 }
246
247 #[test]
248 fn parses_single_blob_pack() {
249 let pack = single_object_pack(ObjectFormat::Sha1, ObjectType::Blob, b"hello\n");
250 let parsed = PackFile::parse_sha1(&pack).expect("test operation should succeed");
251 assert_eq!(parsed.version, 2);
252 assert_eq!(parsed.entries.len(), 1);
253 let object = &parsed.entries[0].object;
254 assert_eq!(object.object_type, ObjectType::Blob);
255 assert_eq!(object.body, b"hello\n");
256 assert_eq!(
257 parsed.entries[0].entry.oid.to_hex(),
258 "ce013625030ba8dba906f756967f9e9ca394464a"
259 );
260 }
261
262 #[test]
263 fn parses_single_blob_pack_sha256() {
264 let pack = single_object_pack(ObjectFormat::Sha256, ObjectType::Blob, b"hello\n");
265 let parsed =
266 PackFile::parse(&pack, ObjectFormat::Sha256).expect("test operation should succeed");
267 assert_eq!(parsed.version, 2);
268 assert_eq!(parsed.entries.len(), 1);
269 let object = &parsed.entries[0].object;
270 assert_eq!(object.object_type, ObjectType::Blob);
271 assert_eq!(object.body, b"hello\n");
272 assert_eq!(
273 parsed.entries[0].entry.oid,
274 object
275 .object_id(ObjectFormat::Sha256)
276 .expect("test operation should succeed")
277 );
278 }
279
280 #[test]
281 fn parses_bundle_pack_payload_with_bundle_format() {
282 let pack = single_object_pack(ObjectFormat::Sha1, ObjectType::Blob, b"bundle\n");
283 let oid = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"bundle\n")
284 .expect("test operation should succeed");
285 let bundle_bytes = format!("# v2 git bundle\n{oid} refs/heads/main\n\n")
286 .into_bytes()
287 .into_iter()
288 .chain(pack)
289 .collect::<Vec<_>>();
290 let bundle = Bundle::parse(&bundle_bytes, ObjectFormat::Sha1)
291 .expect("test operation should succeed");
292
293 let parsed = PackFile::parse_bundle(&bundle).expect("test operation should succeed");
294 assert_eq!(parsed.entries.len(), 1);
295 assert_eq!(parsed.entries[0].object.object_type, ObjectType::Blob);
296 assert_eq!(parsed.entries[0].object.body, b"bundle\n");
297 }
298
299 fn lying_size_blob_pack(format: ObjectFormat, declared_size: u64, real_body: &[u8]) -> Vec<u8> {
305 let mut pack = Vec::new();
306 pack.extend_from_slice(b"PACK");
307 pack.extend_from_slice(&2u32.to_be_bytes());
308 pack.extend_from_slice(&1u32.to_be_bytes());
309 write_pack_entry_header_kind(&mut pack, 3, declared_size);
311 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
312 encoder
313 .write_all(real_body)
314 .expect("test operation should succeed");
315 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
316 let checksum =
317 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
318 pack.extend_from_slice(checksum.as_bytes());
319 pack
320 }
321
322 #[test]
335 fn rejects_decompression_bomb_header_without_oom() {
336 for &declared in &[u64::MAX, 100 * 1024 * 1024 * 1024, u64::from(u32::MAX) * 4] {
337 let pack = lying_size_blob_pack(ObjectFormat::Sha1, declared, b"tiny\n");
338 let handle = std::thread::spawn(move || PackFile::parse_sha1(&pack));
339 let result = handle.join();
340 assert!(
342 result.is_ok(),
343 "parsing a bomb header (declared={declared}) panicked instead of erroring cleanly"
344 );
345 let parse_result = result.expect("parse thread should not panic on a bomb header");
347 assert!(
348 parse_result.is_err(),
349 "bomb header (declared={declared}) should be rejected as invalid"
350 );
351 }
352 }
353
354 fn lying_result_size_delta_pack(
361 format: ObjectFormat,
362 declared_result_size: u64,
363 delta_kind: DeltaKind,
364 ) -> Vec<u8> {
365 let base = b"hello";
366 let result = b"hello world"; let mut delta = Vec::new();
370 write_delta_varint(&mut delta, base.len() as u64);
371 write_delta_varint(&mut delta, declared_result_size);
372 let suffix = &result[base.len()..];
374 delta.push(0x90); delta.push(base.len() as u8);
376 delta.push(suffix.len() as u8);
377 delta.extend_from_slice(suffix);
378
379 let mut pack = Vec::new();
380 pack.extend_from_slice(b"PACK");
381 pack.extend_from_slice(&2u32.to_be_bytes());
382 pack.extend_from_slice(&2u32.to_be_bytes());
383
384 let base_offset = pack.len();
385 write_entry_header(&mut pack, ObjectType::Blob, base.len() as u64);
386 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
387 encoder
388 .write_all(base)
389 .expect("test operation should succeed");
390 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
391
392 let delta_offset = pack.len();
393 write_pack_entry_header_kind(
394 &mut pack,
395 match delta_kind {
396 DeltaKind::Offset => 6,
397 DeltaKind::Ref => 7,
398 },
399 delta.len() as u64,
400 );
401 match delta_kind {
402 DeltaKind::Offset => write_ofs_delta_offset(&mut pack, delta_offset - base_offset),
403 DeltaKind::Ref => {
404 let base_oid = sley_core::object_id_for_bytes(format, "blob", base)
405 .expect("test operation should succeed");
406 pack.extend_from_slice(base_oid.as_bytes());
407 }
408 }
409 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
410 encoder
411 .write_all(&delta)
412 .expect("test operation should succeed");
413 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
414
415 let checksum =
416 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
417 pack.extend_from_slice(checksum.as_bytes());
418 pack
419 }
420
421 #[test]
431 fn rejects_delta_result_size_bomb_without_oom() {
432 let bombs: &[u64] = &[u64::MAX, 1024 * 1024 * 1024 * 1024];
433 for &declared in bombs {
434 for delta_kind in [DeltaKind::Ref, DeltaKind::Offset] {
435 let pack = lying_result_size_delta_pack(ObjectFormat::Sha1, declared, delta_kind);
436 let handle = std::thread::spawn(move || PackFile::parse_sha1(&pack));
437 let join_result = handle.join();
438 assert!(
439 join_result.is_ok(),
440 "delta bomb (declared={declared}, kind={delta_kind:?}) panicked/aborted \
441 instead of erroring cleanly"
442 );
443 let parse_result =
444 join_result.expect("parse thread should not panic on a delta bomb");
445 assert!(
446 parse_result.is_err(),
447 "delta bomb (declared={declared}, kind={delta_kind:?}) should be rejected \
448 as invalid (result.len() != declared)"
449 );
450 }
451 }
452 }
453
454 #[test]
458 fn applies_legitimate_delta_after_result_size_bound() {
459 for delta_kind in [DeltaKind::Ref, DeltaKind::Offset] {
460 let base = b"hello";
461 let result = b"hello world";
462 let pack = two_object_delta_pack(ObjectFormat::Sha1, base, result, delta_kind);
463 let parsed = PackFile::parse_sha1(&pack).expect("legitimate delta should resolve");
464 assert_eq!(parsed.entries.len(), 2);
465 assert_eq!(parsed.entries[0].object.body, base);
466 assert_eq!(parsed.entries[1].object.body, result);
467 }
468 }
469
470 #[test]
471 fn rejects_bundle_pack_payload_with_wrong_object_format() {
472 let pack = single_object_pack(ObjectFormat::Sha1, ObjectType::Blob, b"bundle\n");
473 let oid = sley_core::object_id_for_bytes(ObjectFormat::Sha256, "blob", b"bundle\n")
474 .expect("test operation should succeed");
475 let bundle_bytes =
476 format!("# v3 git bundle\n@object-format=sha256\n{oid} refs/heads/main\n\n")
477 .into_bytes()
478 .into_iter()
479 .chain(pack)
480 .collect::<Vec<_>>();
481 let bundle = Bundle::parse(&bundle_bytes, ObjectFormat::Sha1)
482 .expect("test operation should succeed");
483
484 assert!(PackFile::parse_bundle(&bundle).is_err());
485 }
486
487 fn assert_pack_index_view_matches_owned(index: &[u8], format: ObjectFormat) {
488 let owned = PackIndex::parse(index, format).expect("test operation should succeed");
489 let view = PackIndexView::parse(index, format).expect("test operation should succeed");
490 let owned_view =
491 PackIndexViewData::parse(Arc::from(index.to_vec().into_boxed_slice()), format)
492 .expect("test operation should succeed");
493
494 assert_eq!(view.version, owned.version);
495 assert_eq!(view.count, owned.entries.len());
496 assert_eq!(view.count(), owned.entries.len());
497 assert_eq!(view.fanout(), &owned.fanout);
498 assert_eq!(view.pack_checksum, owned.pack_checksum);
499 assert_eq!(view.index_checksum, owned.index_checksum);
500 assert_eq!(owned_view.version, owned.version);
501 assert_eq!(owned_view.count(), owned.entries.len());
502 assert_eq!(owned_view.fanout(), &owned.fanout);
503 assert_eq!(owned_view.pack_checksum, owned.pack_checksum);
504 assert_eq!(owned_view.index_checksum, owned.index_checksum);
505 for entry in &owned.entries {
506 let owned_found = owned
507 .find(&entry.oid)
508 .expect("test operation should succeed");
509 let expected = Some(PackIndexLookup {
510 crc32: owned_found.crc32,
511 offset: owned_found.offset,
512 });
513 assert_eq!(view.find(&entry.oid), expected);
514 assert_eq!(owned_view.find(&entry.oid), expected);
515 }
516 }
517
518 #[test]
519 fn writes_pack_and_index_that_round_trip() {
520 let object = EncodedObject::new(ObjectType::Blob, b"hello\n".to_vec());
521 let written = PackFile::write_undeltified_sha1(std::slice::from_ref(&object))
522 .expect("test operation should succeed");
523 let pack = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
524 let index =
525 PackIndex::parse_v2_sha1(&written.index).expect("test operation should succeed");
526 let oid = object
527 .object_id(ObjectFormat::Sha1)
528 .expect("test operation should succeed");
529 assert_eq!(pack.entries[0].object, object);
530 assert_eq!(index.pack_checksum, pack.checksum);
531 assert_eq!(
532 index
533 .find(&oid)
534 .expect("test operation should succeed")
535 .offset,
536 12
537 );
538 }
539
540 #[test]
541 fn pack_index_view_matches_owned_index_for_generated_sha1_pack() {
542 let objects = (0..8)
543 .map(|idx| {
544 EncodedObject::new(
545 ObjectType::Blob,
546 format!("borrowed pack index view sha1 object {idx}\n").into_bytes(),
547 )
548 })
549 .collect::<Vec<_>>();
550 let written = PackFile::write_packed(&objects, ObjectFormat::Sha1)
551 .expect("test operation should succeed");
552
553 assert_pack_index_view_matches_owned(&written.index, ObjectFormat::Sha1);
554
555 let view =
556 PackIndexView::parse_v2_sha1(&written.index).expect("test operation should succeed");
557 let missing = sley_core::object_id_for_bytes(
558 ObjectFormat::Sha1,
559 "blob",
560 b"not present in borrowed index\n",
561 )
562 .expect("test operation should succeed");
563 assert_eq!(view.find(&missing), None);
564 }
565
566 #[test]
567 fn writes_sha256_pack_and_index_that_round_trip() {
568 let object = EncodedObject::new(ObjectType::Blob, b"hello sha256\n".to_vec());
569 let written =
570 PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha256)
571 .expect("test operation should succeed");
572 let pack = PackFile::parse(&written.pack, ObjectFormat::Sha256)
573 .expect("test operation should succeed");
574 let index = PackIndex::parse(&written.index, ObjectFormat::Sha256)
575 .expect("test operation should succeed");
576 let oid = object
577 .object_id(ObjectFormat::Sha256)
578 .expect("test operation should succeed");
579 assert_eq!(pack.entries[0].object, object);
580 assert_eq!(index.pack_checksum, pack.checksum);
581 assert_eq!(index.pack_checksum.format(), ObjectFormat::Sha256);
582 assert_eq!(index.index_checksum.format(), ObjectFormat::Sha256);
583 assert_eq!(
584 index
585 .find(&oid)
586 .expect("test operation should succeed")
587 .offset,
588 12
589 );
590 }
591
592 #[test]
593 fn pack_index_view_matches_owned_index_for_generated_sha256_pack() {
594 let objects = (0..4)
595 .map(|idx| {
596 EncodedObject::new(
597 ObjectType::Blob,
598 format!("borrowed pack index view sha256 object {idx}\n").into_bytes(),
599 )
600 })
601 .collect::<Vec<_>>();
602 let written = PackFile::write_undeltified(&objects, ObjectFormat::Sha256)
603 .expect("test operation should succeed");
604
605 assert_pack_index_view_matches_owned(&written.index, ObjectFormat::Sha256);
606 }
607
608 #[test]
609 fn indexes_existing_sha256_pack_bytes() {
610 let object = EncodedObject::new(ObjectType::Blob, b"index raw sha256 pack\n".to_vec());
611 let written =
612 PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha256)
613 .expect("test operation should succeed");
614
615 let indexed = PackIndex::write_v2_for_pack(&written.pack, ObjectFormat::Sha256)
616 .expect("test operation should succeed");
617 let index = PackIndex::parse(&indexed.index, ObjectFormat::Sha256)
618 .expect("test operation should succeed");
619
620 assert_eq!(indexed.pack_checksum, written.checksum);
621 assert_eq!(indexed.entries, written.entries);
622 assert_eq!(index.pack_checksum, written.checksum);
623 assert_eq!(index.entries, written.entries);
624 }
625
626 #[test]
627 fn indexes_existing_delta_pack_bytes() {
628 let (base, changed) = similar_blob_objects();
629 let options = delta_pack_options(true);
630 let written = PackFile::write_packed_with_options(
631 &[base, changed.clone()],
632 ObjectFormat::Sha1,
633 &options,
634 )
635 .expect("test operation should succeed");
636
637 let indexed = PackIndex::write_v2_for_pack_sha1(&written.pack)
638 .expect("test operation should succeed");
639 let index =
640 PackIndex::parse_v2_sha1(&indexed.index).expect("test operation should succeed");
641 let changed_oid = changed
642 .object_id(ObjectFormat::Sha1)
643 .expect("test operation should succeed");
644
645 assert_eq!(indexed.pack_checksum, written.checksum);
646 assert_eq!(indexed.entries, written.entries);
647 assert_eq!(
648 index
649 .find(&changed_oid)
650 .expect("test operation should succeed")
651 .offset,
652 written.entries[1].offset
653 );
654 assert_eq!(
655 index
656 .find(&changed_oid)
657 .expect("test operation should succeed")
658 .crc32,
659 written.entries[1].crc32
660 );
661 }
662
663 #[test]
664 fn writes_ref_delta_pack_and_index_that_round_trip() {
665 let (base, changed) = similar_blob_objects();
666 let options = delta_pack_options(false);
667 let written = PackFile::write_packed_with_options(
668 &[base.clone(), changed.clone()],
669 ObjectFormat::Sha1,
670 &options,
671 )
672 .expect("test operation should succeed");
673 let mut second_offset = written.entries[1].offset as usize;
674 let header = parse_entry_header(&written.pack, &mut second_offset)
675 .expect("test operation should succeed");
676 assert_eq!(header.kind, PackObjectKind::RefDelta);
677
678 let pack = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
679 let index =
680 PackIndex::parse_v2_sha1(&written.index).expect("test operation should succeed");
681 let oid = changed
682 .object_id(ObjectFormat::Sha1)
683 .expect("test operation should succeed");
684 assert_eq!(pack.entries[0].object, base);
685 assert_eq!(pack.entries[1].object, changed);
686 assert_eq!(index.pack_checksum, pack.checksum);
687 assert_eq!(
688 index
689 .find(&oid)
690 .expect("test operation should succeed")
691 .offset,
692 written.entries[1].offset
693 );
694 }
695
696 #[test]
697 fn read_object_at_matches_full_parse_for_ofs_delta_pack() {
698 let (base, changed) = similar_blob_objects();
699 let options = delta_pack_options(true);
700 let written = PackFile::write_packed_with_options(
701 &[base, changed.clone()],
702 ObjectFormat::Sha1,
703 &options,
704 )
705 .expect("test operation should succeed");
706 let mut second = written.entries[1].offset as usize;
708 assert_eq!(
709 parse_entry_header(&written.pack, &mut second)
710 .expect("test operation should succeed")
711 .kind,
712 PackObjectKind::OfsDelta
713 );
714 let parsed = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
716 for po in &parsed.entries {
717 let got =
718 read_object_at_arc(&written.pack, po.entry.offset, ObjectFormat::Sha1, |_| {
719 Ok(None)
720 })
721 .expect("test operation should succeed");
722 assert_eq!(*got, po.object, "offset {}", po.entry.offset);
723 }
724 }
725
726 #[derive(Default)]
729 struct MapHeaderTypeCache(HashMap<u64, (ObjectType, u64)>);
730
731 impl HeaderTypeCache for MapHeaderTypeCache {
732 fn get(&self, pack_offset: u64) -> Option<(ObjectType, u64)> {
733 self.0.get(&pack_offset).copied()
734 }
735 fn put(&mut self, pack_offset: u64, header: (ObjectType, u64)) {
736 self.0.insert(pack_offset, header);
737 }
738 }
739
740 #[test]
741 fn read_object_header_at_cached_matches_uncached_cold_and_warm_for_ofs_delta() {
742 let (base, changed) = similar_blob_objects();
743 let options = delta_pack_options(true);
744 let written =
745 PackFile::write_packed_with_options(&[base, changed], ObjectFormat::Sha1, &options)
746 .expect("test operation should succeed");
747 let mut second = written.entries[1].offset as usize;
749 assert_eq!(
750 parse_entry_header(&written.pack, &mut second)
751 .expect("test operation should succeed")
752 .kind,
753 PackObjectKind::OfsDelta
754 );
755
756 let parsed = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
757 let mut cache = MapHeaderTypeCache::default();
758 for po in &parsed.entries {
759 let uncached =
760 read_object_header_at(&written.pack, po.entry.offset, ObjectFormat::Sha1, |_| {
761 Ok(None)
762 })
763 .expect("test operation should succeed");
764 assert_eq!(
766 uncached,
767 (po.object.object_type, po.object.body.len() as u64),
768 "uncached header at offset {}",
769 po.entry.offset
770 );
771 let cold = read_object_header_at_with_cache(
773 &written.pack,
774 po.entry.offset,
775 ObjectFormat::Sha1,
776 |_| Ok(None),
777 &mut cache,
778 )
779 .expect("test operation should succeed");
780 assert_eq!(cold, uncached, "cold cache at offset {}", po.entry.offset);
781 }
782 for po in &parsed.entries {
785 let warm = read_object_header_at_with_cache(
786 &written.pack,
787 po.entry.offset,
788 ObjectFormat::Sha1,
789 |_| panic!("warm cache must not re-walk the chain"),
790 &mut cache,
791 )
792 .expect("test operation should succeed");
793 assert_eq!(
794 warm,
795 (po.object.object_type, po.object.body.len() as u64),
796 "warm cache at offset {}",
797 po.entry.offset
798 );
799 }
800 }
801
802 #[test]
803 fn read_object_at_matches_full_parse_for_ref_delta_pack() {
804 let (base, changed) = similar_blob_objects();
805 let options = delta_pack_options(false);
806 let written = PackFile::write_packed_with_options(
807 &[base, changed.clone()],
808 ObjectFormat::Sha1,
809 &options,
810 )
811 .expect("test operation should succeed");
812 let parsed = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
813 let by_oid: HashMap<ObjectId, Arc<EncodedObject>> = parsed
814 .entries
815 .iter()
816 .map(|po| (po.entry.oid, Arc::new(po.object.clone())))
817 .collect();
818 for po in &parsed.entries {
819 let got =
820 read_object_at_arc(&written.pack, po.entry.offset, ObjectFormat::Sha1, |oid| {
821 Ok(by_oid.get(oid).cloned())
822 })
823 .expect("test operation should succeed");
824 assert_eq!(*got, po.object);
825 }
826 }
827
828 #[derive(Default)]
832 struct CountingDeltaCache {
833 map: std::cell::RefCell<HashMap<u64, Arc<EncodedObject>>>,
834 hits: std::cell::Cell<usize>,
835 inserts: std::cell::Cell<usize>,
836 }
837
838 impl PackDeltaCache for CountingDeltaCache {
839 fn get(&self, offset: u64) -> Option<Arc<EncodedObject>> {
840 let hit = self.map.borrow().get(&offset).cloned();
841 if hit.is_some() {
842 self.hits.set(self.hits.get() + 1);
843 }
844 hit
845 }
846 fn insert(&self, offset: u64, object: Arc<EncodedObject>) {
847 self.inserts.set(self.inserts.get() + 1);
848 self.map.borrow_mut().insert(offset, object);
849 }
850 }
851
852 #[test]
853 fn read_object_at_with_cache_matches_uncached_and_reuses_bases() {
854 let mut objects = Vec::new();
857 for idx in 0..8u32 {
858 let mut body = vec![b'x'; 4096];
859 body.extend_from_slice(format!("\nvariant {idx}\n").as_bytes());
860 objects.push(EncodedObject::new(ObjectType::Blob, body));
861 }
862 let options = delta_pack_options(true);
863 let written = PackFile::write_packed_with_options(&objects, ObjectFormat::Sha1, &options)
864 .expect("test operation should succeed");
865 let parsed = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
866
867 let cache = CountingDeltaCache::default();
868 for _ in 0..2 {
871 for po in &parsed.entries {
872 let got = read_object_at_with_cache_arc(
873 &written.pack,
874 po.entry.offset,
875 ObjectFormat::Sha1,
876 |_| Ok(None),
877 &cache,
878 )
879 .expect("test operation should succeed");
880 assert_eq!(*got, po.object, "offset {}", po.entry.offset);
881 }
882 }
883 assert!(cache.hits.get() > 0, "cache never served a warm object");
886 }
887
888 #[test]
889 fn writes_ofs_delta_pack_and_index_that_round_trip() {
890 let (base, changed) = similar_blob_objects();
891 let options = delta_pack_options(true);
892 let written = PackFile::write_packed_with_options(
893 &[base.clone(), changed.clone()],
894 ObjectFormat::Sha1,
895 &options,
896 )
897 .expect("test operation should succeed");
898 let mut second_offset = written.entries[1].offset as usize;
899 let header = parse_entry_header(&written.pack, &mut second_offset)
900 .expect("test operation should succeed");
901 assert_eq!(header.kind, PackObjectKind::OfsDelta);
902
903 let pack = PackFile::parse_sha1(&written.pack).expect("test operation should succeed");
904 let index =
905 PackIndex::parse_v2_sha1(&written.index).expect("test operation should succeed");
906 let oid = changed
907 .object_id(ObjectFormat::Sha1)
908 .expect("test operation should succeed");
909 assert_eq!(pack.entries[0].object, base);
910 assert_eq!(pack.entries[1].object, changed);
911 assert_eq!(index.pack_checksum, pack.checksum);
912 assert_eq!(
913 index
914 .find(&oid)
915 .expect("test operation should succeed")
916 .offset,
917 written.entries[1].offset
918 );
919 }
920
921 #[test]
922 fn resolves_ofs_delta_pack_entry() {
923 let base = b"hello";
924 let result = b"hello world";
925 let pack = two_object_delta_pack(ObjectFormat::Sha1, base, result, DeltaKind::Offset);
926 let parsed = PackFile::parse_sha1(&pack).expect("test operation should succeed");
927 assert_eq!(parsed.entries.len(), 2);
928 assert_eq!(parsed.entries[0].object.body, base);
929 assert_eq!(parsed.entries[1].object.body, result);
930 assert_eq!(
931 parsed.entries[1].entry.oid,
932 sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", result)
933 .expect("test operation should succeed")
934 );
935 }
936
937 #[test]
938 fn resolves_ref_delta_pack_entry() {
939 let base = b"hello";
940 let result = b"hello world";
941 let pack = two_object_delta_pack(ObjectFormat::Sha1, base, result, DeltaKind::Ref);
942 let parsed = PackFile::parse_sha1(&pack).expect("test operation should succeed");
943 assert_eq!(parsed.entries.len(), 2);
944 assert_eq!(parsed.entries[0].object.body, base);
945 assert_eq!(parsed.entries[1].object.body, result);
946 assert_eq!(
947 parsed.entries[1].entry.oid,
948 sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", result)
949 .expect("test operation should succeed")
950 );
951 }
952
953 #[test]
954 fn resolves_thin_ref_delta_pack_entry_with_external_base() {
955 let base = b"hello";
956 let result = b"hello world";
957 let pack = thin_ref_delta_pack(ObjectFormat::Sha1, base, result);
958 assert!(PackFile::parse_sha1(&pack).is_err());
959
960 let base_oid = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", base)
961 .expect("test operation should succeed");
962 let parsed = PackFile::parse_thin(&pack, ObjectFormat::Sha1, |oid| {
963 if oid == &base_oid {
964 Ok(Some(EncodedObject::new(ObjectType::Blob, base.to_vec())))
965 } else {
966 Ok(None)
967 }
968 })
969 .expect("test operation should succeed");
970 assert_eq!(parsed.entries.len(), 1);
971 assert_eq!(parsed.entries[0].object.body, result);
972 assert_eq!(
973 parsed.entries[0].entry.oid,
974 sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", result)
975 .expect("test operation should succeed")
976 );
977 }
978
979 #[test]
980 fn rejects_bad_pack_checksum() {
981 let mut pack = single_object_pack(ObjectFormat::Sha1, ObjectType::Blob, b"hello\n");
982 let last = pack.len() - 1;
983 pack[last] ^= 1;
984 assert!(PackFile::parse_sha1(&pack).is_err());
985 }
986
987 #[test]
988 fn raw_pack_index_rejects_bad_pack_checksum() {
989 let mut pack = single_object_pack(ObjectFormat::Sha1, ObjectType::Blob, b"hello\n");
990 let last = pack.len() - 1;
991 pack[last] ^= 1;
992 assert!(PackIndex::write_v2_for_pack_sha1(&pack).is_err());
993 }
994
995 #[test]
996 fn pack_index_writer_preserves_duplicate_object_ids() {
997 let oid = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"same\n")
998 .expect("test operation should succeed");
999 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1000 .expect("test operation should succeed");
1001 let entries = vec![
1002 PackIndexEntry {
1003 oid,
1004 crc32: 1,
1005 offset: 12,
1006 },
1007 PackIndexEntry {
1008 oid,
1009 crc32: 2,
1010 offset: 24,
1011 },
1012 ];
1013 let index = PackIndex::write_v2(ObjectFormat::Sha1, &entries, &pack_checksum)
1014 .expect("duplicate objects are valid in a pack index");
1015 let parsed = PackIndex::parse(&index, ObjectFormat::Sha1)
1016 .expect("duplicate-object pack index should parse");
1017 assert_eq!(parsed.entries.len(), 2);
1018 assert!(parsed.entries.iter().all(|entry| entry.oid == oid));
1019 assert!(parsed.find(&oid).is_some());
1020 }
1021
1022 #[test]
1023 fn parses_single_entry_pack_index() {
1024 let oid = ObjectId::from_hex(
1025 ObjectFormat::Sha1,
1026 "ce013625030ba8dba906f756967f9e9ca394464a",
1027 )
1028 .expect("test operation should succeed");
1029 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1030 .expect("test operation should succeed");
1031 let index = single_entry_index(
1032 ObjectFormat::Sha1,
1033 oid,
1034 0x1234_5678,
1035 12,
1036 pack_checksum.clone(),
1037 );
1038 let parsed = PackIndex::parse_v2_sha1(&index).expect("test operation should succeed");
1039 assert_eq!(parsed.version, 2);
1040 assert_eq!(parsed.pack_checksum, pack_checksum);
1041 assert_eq!(parsed.entries.len(), 1);
1042 assert_eq!(
1043 parsed
1044 .find(&oid)
1045 .expect("test operation should succeed")
1046 .offset,
1047 12
1048 );
1049 assert_eq!(
1050 parsed
1051 .find(&oid)
1052 .expect("test operation should succeed")
1053 .crc32,
1054 0x1234_5678
1055 );
1056 assert_pack_index_view_matches_owned(&index, ObjectFormat::Sha1);
1057 }
1058
1059 #[test]
1060 fn parses_single_entry_pack_index_v1() {
1061 let oid = ObjectId::from_hex(
1062 ObjectFormat::Sha1,
1063 "ce013625030ba8dba906f756967f9e9ca394464a",
1064 )
1065 .expect("test operation should succeed");
1066 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1067 .expect("test operation should succeed");
1068 let index =
1069 single_entry_index_v1(ObjectFormat::Sha1, oid, 0x1234_5678, pack_checksum.clone());
1070 let parsed =
1071 PackIndex::parse(&index, ObjectFormat::Sha1).expect("test operation should succeed");
1072 assert_eq!(parsed.version, 1);
1073 assert_eq!(parsed.pack_checksum, pack_checksum);
1074 assert_eq!(parsed.entries.len(), 1);
1075 assert_eq!(
1076 parsed
1077 .find(&oid)
1078 .expect("test operation should succeed")
1079 .offset,
1080 0x1234_5678
1081 );
1082 assert_eq!(
1083 parsed
1084 .find(&oid)
1085 .expect("test operation should succeed")
1086 .crc32,
1087 0
1088 );
1089 assert_pack_index_view_matches_owned(&index, ObjectFormat::Sha1);
1090 }
1091
1092 #[test]
1093 fn rejects_bad_pack_index_v1_checksum() {
1094 let oid = ObjectId::from_hex(
1095 ObjectFormat::Sha1,
1096 "ce013625030ba8dba906f756967f9e9ca394464a",
1097 )
1098 .expect("test operation should succeed");
1099 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1100 .expect("test operation should succeed");
1101 let mut index = single_entry_index_v1(ObjectFormat::Sha1, oid, 12, pack_checksum);
1102 let last = index.len() - 1;
1103 index[last] ^= 1;
1104 assert!(PackIndex::parse(&index, ObjectFormat::Sha1).is_err());
1105 }
1106
1107 #[test]
1108 fn pack_index_view_reads_v2_large_offsets() {
1109 let first = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"large offset a\n")
1110 .expect("test operation should succeed");
1111 let second =
1112 sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"large offset b\n")
1113 .expect("test operation should succeed");
1114 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1115 .expect("test operation should succeed");
1116 let entries = vec![
1117 PackIndexEntry {
1118 oid: first,
1119 crc32: 0x1111_2222,
1120 offset: 0x8000_0000,
1121 },
1122 PackIndexEntry {
1123 oid: second,
1124 crc32: 0x3333_4444,
1125 offset: 0x1_0000_0042,
1126 },
1127 ];
1128 let index = PackIndex::write_v2(ObjectFormat::Sha1, &entries, &pack_checksum)
1129 .expect("test operation should succeed");
1130
1131 assert_pack_index_view_matches_owned(&index, ObjectFormat::Sha1);
1132 let view = PackIndexView::parse(&index, ObjectFormat::Sha1)
1133 .expect("test operation should succeed");
1134 for entry in entries {
1135 assert_eq!(
1136 view.find(&entry.oid),
1137 Some(PackIndexLookup {
1138 crc32: entry.crc32,
1139 offset: entry.offset,
1140 })
1141 );
1142 }
1143 }
1144
1145 #[test]
1146 fn pack_index_view_default_parse_checks_index_checksum() {
1147 let oid = ObjectId::from_hex(
1148 ObjectFormat::Sha1,
1149 "ce013625030ba8dba906f756967f9e9ca394464a",
1150 )
1151 .expect("test operation should succeed");
1152 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1153 .expect("test operation should succeed");
1154 let mut index = single_entry_index(ObjectFormat::Sha1, oid, 0x1234_5678, 12, pack_checksum);
1155 let last = index.len() - 1;
1156 index[last] ^= 1;
1157
1158 assert!(PackIndexView::parse(&index, ObjectFormat::Sha1).is_err());
1159 let view = PackIndexView::parse_without_checksum(&index, ObjectFormat::Sha1)
1160 .expect("test operation should succeed");
1161 let trusted_view = PackIndexViewData::parse_trusted_without_checksum(
1162 Arc::from(index.clone().into_boxed_slice()),
1163 ObjectFormat::Sha1,
1164 )
1165 .expect("test operation should succeed");
1166 assert_eq!(
1167 view.find(&oid),
1168 Some(PackIndexLookup {
1169 crc32: 0x1234_5678,
1170 offset: 12,
1171 })
1172 );
1173 assert_eq!(
1174 trusted_view.find(&oid),
1175 Some(PackIndexLookup {
1176 crc32: 0x1234_5678,
1177 offset: 12,
1178 })
1179 );
1180 }
1181
1182 #[test]
1183 fn reverse_index_resolves_oid_at_offset() {
1184 let objects = (0..3)
1185 .map(|idx| {
1186 EncodedObject::new(
1187 ObjectType::Blob,
1188 format!("reverse index lookup object {idx}\n").into_bytes(),
1189 )
1190 })
1191 .collect::<Vec<_>>();
1192 let written = PackFile::write_packed(&objects, ObjectFormat::Sha1)
1193 .expect("test operation should succeed");
1194 let index = PackIndex::parse(&written.index, ObjectFormat::Sha1)
1195 .expect("test operation should succeed");
1196 let view = PackIndexViewData::parse_trusted_without_checksum(
1197 Arc::from(written.index.clone().into_boxed_slice()),
1198 ObjectFormat::Sha1,
1199 )
1200 .expect("test operation should succeed");
1201 let positions = pack_order_index_positions(&index.entries);
1202 let reverse = PackReverseIndex::parse(
1203 &PackReverseIndex::write(ObjectFormat::Sha1, &positions, &index.pack_checksum)
1204 .expect("test operation should succeed"),
1205 ObjectFormat::Sha1,
1206 index.entries.len(),
1207 )
1208 .expect("test operation should succeed");
1209
1210 for entry in &index.entries {
1211 assert_eq!(
1212 reverse
1213 .oid_at_offset(&view, entry.offset)
1214 .expect("test operation should succeed"),
1215 entry.oid
1216 );
1217 }
1218 assert!(reverse.oid_at_offset(&view, 999).is_none());
1219 }
1220
1221 #[test]
1222 fn parses_pack_reverse_index() {
1223 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1224 .expect("test operation should succeed");
1225 let reverse_index = PackReverseIndex::write(ObjectFormat::Sha1, &[2, 0, 1], &pack_checksum)
1226 .expect("test operation should succeed");
1227 let parsed = PackReverseIndex::parse(&reverse_index, ObjectFormat::Sha1, 3)
1228 .expect("test operation should succeed");
1229 assert_eq!(parsed.version, 1);
1230 assert_eq!(parsed.format, ObjectFormat::Sha1);
1231 assert_eq!(parsed.positions, vec![2, 0, 1]);
1232 assert_eq!(parsed.pack_checksum, pack_checksum);
1233 assert_eq!(
1234 PackReverseIndex::write(ObjectFormat::Sha1, &parsed.positions, &parsed.pack_checksum)
1235 .expect("test operation should succeed"),
1236 reverse_index
1237 );
1238 }
1239
1240 #[test]
1241 fn rejects_bad_pack_reverse_index_checksum() {
1242 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1243 .expect("test operation should succeed");
1244 let mut reverse_index = PackReverseIndex::write(ObjectFormat::Sha1, &[0], &pack_checksum)
1245 .expect("test operation should succeed");
1246 let last = reverse_index.len() - 1;
1247 reverse_index[last] ^= 1;
1248 assert!(matches!(
1249 PackReverseIndex::parse(&reverse_index, ObjectFormat::Sha1, 1),
1250 Err(GitError::InvalidFormat(message)) if message == "invalid checksum"
1251 ));
1252 }
1253
1254 #[test]
1255 fn classifies_pack_reverse_index_corruption_for_fsck() {
1256 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1257 .expect("test operation should succeed");
1258 let valid = PackReverseIndex::write(ObjectFormat::Sha1, &[0], &pack_checksum)
1259 .expect("test operation should succeed");
1260 for (offset, value, expected) in [
1261 (1, 7, "unknown signature"),
1262 (7, 2, "unsupported version 2"),
1263 (11, 3, "unsupported hash id 3"),
1264 (14, 7, "invalid rev-index position 1792"),
1265 ] {
1266 let mut corrupt = valid.clone();
1267 corrupt[offset] = value;
1268 assert!(matches!(
1269 PackReverseIndex::parse(&corrupt, ObjectFormat::Sha1, 1),
1270 Err(GitError::InvalidFormat(message)) if message == expected
1271 ));
1272 }
1273 }
1274
1275 #[test]
1276 fn rejects_bad_pack_reverse_index_positions() {
1277 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1278 .expect("test operation should succeed");
1279 let duplicate = pack_reverse_index(ObjectFormat::Sha1, &[0, 0], pack_checksum.clone());
1280 assert!(PackReverseIndex::parse(&duplicate, ObjectFormat::Sha1, 2).is_err());
1281 let out_of_range = pack_reverse_index(ObjectFormat::Sha1, &[0, 2], pack_checksum);
1282 assert!(PackReverseIndex::parse(&out_of_range, ObjectFormat::Sha1, 2).is_err());
1283 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1284 .expect("test operation should succeed");
1285 assert!(PackReverseIndex::write(ObjectFormat::Sha1, &[0, 0], &pack_checksum).is_err());
1286 assert!(PackReverseIndex::write(ObjectFormat::Sha1, &[0, 2], &pack_checksum).is_err());
1287 }
1288
1289 #[test]
1290 fn parses_pack_mtimes() {
1291 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1292 .expect("test operation should succeed");
1293 let mtimes = PackMtimes::write(
1294 ObjectFormat::Sha1,
1295 &[1, 1_700_000_000, u32::MAX],
1296 &pack_checksum,
1297 )
1298 .expect("test operation should succeed");
1299 let parsed = PackMtimes::parse(&mtimes, ObjectFormat::Sha1, 3)
1300 .expect("test operation should succeed");
1301 assert_eq!(parsed.version, 1);
1302 assert_eq!(parsed.format, ObjectFormat::Sha1);
1303 assert_eq!(parsed.mtimes, vec![1, 1_700_000_000, u32::MAX]);
1304 assert_eq!(parsed.pack_checksum, pack_checksum);
1305 assert_eq!(
1306 PackMtimes::write(ObjectFormat::Sha1, &parsed.mtimes, &parsed.pack_checksum)
1307 .expect("test operation should succeed"),
1308 mtimes
1309 );
1310 }
1311
1312 #[test]
1313 fn rejects_bad_pack_mtimes_checksum() {
1314 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1315 .expect("test operation should succeed");
1316 let mut mtimes = PackMtimes::write(ObjectFormat::Sha1, &[1], &pack_checksum)
1317 .expect("test operation should succeed");
1318 let last = mtimes.len() - 1;
1319 mtimes[last] ^= 1;
1320 assert!(PackMtimes::parse(&mtimes, ObjectFormat::Sha1, 1).is_err());
1321 }
1322
1323 #[test]
1324 fn rejects_bad_pack_mtimes_shape() {
1325 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1326 .expect("test operation should succeed");
1327 let mtimes = pack_mtimes(ObjectFormat::Sha1, &[1, 2], pack_checksum.clone());
1328 assert!(PackMtimes::parse(&mtimes, ObjectFormat::Sha1, 1).is_err());
1329
1330 let mut wrong_hash = pack_mtimes(ObjectFormat::Sha1, &[1], pack_checksum);
1331 wrong_hash[11] = 2;
1332 let checksum_offset = wrong_hash.len() - ObjectFormat::Sha1.raw_len();
1333 let checksum = sley_core::digest_bytes(ObjectFormat::Sha1, &wrong_hash[..checksum_offset])
1334 .expect("test operation should succeed");
1335 wrong_hash[checksum_offset..].copy_from_slice(checksum.as_bytes());
1336 assert!(PackMtimes::parse(&wrong_hash, ObjectFormat::Sha1, 1).is_err());
1337 }
1338
1339 #[test]
1340 fn parses_multi_pack_index_header_and_chunk_lookup() {
1341 let first = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"first object\n")
1342 .expect("test operation should succeed");
1343 let second = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"second object\n")
1344 .expect("test operation should succeed");
1345 let chunks = midx_chunks_with_pack_names(
1346 ObjectFormat::Sha1,
1347 b"pack-a.idx\0pack-b.idx\0\0\0".to_vec(),
1348 &[(first.clone(), 0, 12), (second.clone(), 1, 0x1_0000_0000)],
1349 );
1350 let midx = multi_pack_index(ObjectFormat::Sha1, 2, 2, &chunks);
1351 let parsed = MultiPackIndex::parse(&midx, ObjectFormat::Sha1)
1352 .expect("test operation should succeed");
1353 assert_eq!(parsed.version, 2);
1354 assert_eq!(parsed.format, ObjectFormat::Sha1);
1355 assert_eq!(parsed.pack_count, 2);
1356 assert_eq!(parsed.pack_names, vec!["pack-a.idx", "pack-b.idx"]);
1357 assert_eq!(parsed.object_count, 2);
1358 assert_eq!(parsed.objects.len(), 2);
1359 assert_eq!(
1360 parsed
1361 .find(&first)
1362 .expect("test operation should succeed")
1363 .pack_int_id,
1364 0
1365 );
1366 assert_eq!(
1367 parsed
1368 .find(&first)
1369 .expect("test operation should succeed")
1370 .offset,
1371 12
1372 );
1373 assert_eq!(
1374 parsed
1375 .find(&second)
1376 .expect("test operation should succeed")
1377 .pack_int_id,
1378 1
1379 );
1380 assert_eq!(
1381 parsed
1382 .find(&second)
1383 .expect("test operation should succeed")
1384 .offset,
1385 0x1_0000_0000
1386 );
1387 assert_eq!(parsed.reverse_index, None);
1388 assert_eq!(parsed.bitmapped_packs, None);
1389 assert_eq!(parsed.chunks.len(), 5);
1390 assert_eq!(parsed.chunks[0].id, *b"PNAM");
1391 assert_eq!(parsed.chunks[0].offset, 84);
1392 assert_eq!(parsed.chunks[0].len, 24);
1393 assert_eq!(parsed.chunks[1].id, *b"OIDF");
1394 assert_eq!(parsed.chunks[1].offset, 108);
1395 assert_eq!(parsed.chunks[1].len, 1024);
1396 }
1397
1398 #[test]
1399 fn raw_multi_pack_index_lookup_finds_pack_and_offset() {
1400 let first = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"first object\n")
1401 .expect("test operation should succeed");
1402 let second = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"second object\n")
1403 .expect("test operation should succeed");
1404 let missing = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"missing\n")
1405 .expect("test operation should succeed");
1406 let chunks = midx_chunks_with_pack_names(
1407 ObjectFormat::Sha1,
1408 b"pack-a.idx\0pack-b.idx\0\0\0".to_vec(),
1409 &[(first.clone(), 0, 12), (second.clone(), 1, 0x1_0000_0000)],
1410 );
1411 let midx = Arc::new(multi_pack_index(ObjectFormat::Sha1, 2, 2, &chunks));
1412 let lookup = MultiPackIndexOidLookup::parse(midx, ObjectFormat::Sha1)
1413 .expect("test operation should succeed");
1414
1415 assert!(lookup.contains(&first));
1416 assert!(lookup.contains(&second));
1417 assert!(!lookup.contains(&missing));
1418
1419 let first_entry = lookup
1420 .find(&first)
1421 .expect("test operation should succeed")
1422 .expect("object should be present");
1423 assert_eq!(
1424 lookup.pack_name(first_entry.pack_int_id),
1425 Some("pack-a.idx")
1426 );
1427 assert_eq!(first_entry.offset, 12);
1428
1429 let second_entry = lookup
1430 .find(&second)
1431 .expect("test operation should succeed")
1432 .expect("object should be present");
1433 assert_eq!(
1434 lookup.pack_name(second_entry.pack_int_id),
1435 Some("pack-b.idx")
1436 );
1437 assert_eq!(second_entry.offset, 0x1_0000_0000);
1438 assert!(
1439 lookup
1440 .find(&missing)
1441 .expect("test operation should succeed")
1442 .is_none()
1443 );
1444 }
1445
1446 #[test]
1447 fn rejects_bad_multi_pack_index_checksum() {
1448 let chunks = midx_chunks_with_pack_names(ObjectFormat::Sha1, Vec::new(), &[]);
1449 let mut midx = multi_pack_index(ObjectFormat::Sha1, 1, 0, &chunks);
1450 let last = midx.len() - 1;
1451 midx[last] ^= 1;
1452 assert!(MultiPackIndex::parse(&midx, ObjectFormat::Sha1).is_err());
1453 }
1454
1455 #[test]
1456 fn rejects_bad_multi_pack_index_shape() {
1457 let chunks = midx_chunks_with_pack_names(ObjectFormat::Sha1, Vec::new(), &[]);
1458 let mut wrong_hash = multi_pack_index(ObjectFormat::Sha1, 1, 0, &chunks);
1459 wrong_hash[5] = 2;
1460 let checksum_offset = wrong_hash.len() - ObjectFormat::Sha1.raw_len();
1461 let checksum = sley_core::digest_bytes(ObjectFormat::Sha1, &wrong_hash[..checksum_offset])
1462 .expect("test operation should succeed");
1463 wrong_hash[checksum_offset..].copy_from_slice(checksum.as_bytes());
1464 assert!(MultiPackIndex::parse(&wrong_hash, ObjectFormat::Sha1).is_err());
1465
1466 let mut missing_terminator = multi_pack_index(ObjectFormat::Sha1, 1, 0, &chunks);
1467 missing_terminator[12] = b'B';
1468 let checksum_offset = missing_terminator.len() - ObjectFormat::Sha1.raw_len();
1469 let checksum =
1470 sley_core::digest_bytes(ObjectFormat::Sha1, &missing_terminator[..checksum_offset])
1471 .expect("test operation should succeed");
1472 missing_terminator[checksum_offset..].copy_from_slice(checksum.as_bytes());
1473 assert!(MultiPackIndex::parse(&missing_terminator, ObjectFormat::Sha1).is_err());
1474
1475 let mut bad_offset = multi_pack_index(
1476 ObjectFormat::Sha1,
1477 2,
1478 0,
1479 &midx_chunks_with_pack_names(ObjectFormat::Sha1, Vec::new(), &[]),
1480 );
1481 bad_offset[16..24].copy_from_slice(&0u64.to_be_bytes());
1482 let checksum_offset = bad_offset.len() - ObjectFormat::Sha1.raw_len();
1483 let checksum = sley_core::digest_bytes(ObjectFormat::Sha1, &bad_offset[..checksum_offset])
1484 .expect("test operation should succeed");
1485 bad_offset[checksum_offset..].copy_from_slice(checksum.as_bytes());
1486 assert!(MultiPackIndex::parse(&bad_offset, ObjectFormat::Sha1).is_err());
1487 }
1488
1489 #[test]
1490 fn rejects_bad_multi_pack_index_pack_names() {
1491 let missing = multi_pack_index(ObjectFormat::Sha1, 2, 1, &[]);
1492 assert!(MultiPackIndex::parse(&missing, ObjectFormat::Sha1).is_err());
1493
1494 let too_few = multi_pack_index(
1495 ObjectFormat::Sha1,
1496 2,
1497 2,
1498 &midx_chunks_with_pack_names(ObjectFormat::Sha1, b"pack-a.idx\0".to_vec(), &[]),
1499 );
1500 assert!(MultiPackIndex::parse(&too_few, ObjectFormat::Sha1).is_err());
1501
1502 let bad_padding = multi_pack_index(
1503 ObjectFormat::Sha1,
1504 2,
1505 1,
1506 &midx_chunks_with_pack_names(ObjectFormat::Sha1, b"pack-a.idx\0xxxx".to_vec(), &[]),
1507 );
1508 assert!(MultiPackIndex::parse(&bad_padding, ObjectFormat::Sha1).is_err());
1509
1510 let unsorted_v1 = multi_pack_index(
1511 ObjectFormat::Sha1,
1512 1,
1513 2,
1514 &midx_chunks_with_pack_names(
1515 ObjectFormat::Sha1,
1516 b"pack-b.idx\0pack-a.idx\0".to_vec(),
1517 &[],
1518 ),
1519 );
1520 assert!(MultiPackIndex::parse(&unsorted_v1, ObjectFormat::Sha1).is_err());
1521
1522 let unsorted_v2 = multi_pack_index(
1523 ObjectFormat::Sha1,
1524 2,
1525 2,
1526 &midx_chunks_with_pack_names(
1527 ObjectFormat::Sha1,
1528 b"pack-b.idx\0pack-a.idx\0".to_vec(),
1529 &[],
1530 ),
1531 );
1532 let parsed = MultiPackIndex::parse(&unsorted_v2, ObjectFormat::Sha1)
1533 .expect("test operation should succeed");
1534 assert_eq!(parsed.pack_names, vec!["pack-b.idx", "pack-a.idx"]);
1535 }
1536
1537 #[test]
1538 fn rejects_bad_multi_pack_index_object_tables() {
1539 let oid_a = ObjectId::from_hex(
1540 ObjectFormat::Sha1,
1541 "1111111111111111111111111111111111111111",
1542 )
1543 .expect("test operation should succeed");
1544 let oid_b = ObjectId::from_hex(
1545 ObjectFormat::Sha1,
1546 "2222222222222222222222222222222222222222",
1547 )
1548 .expect("test operation should succeed");
1549
1550 let missing_oidf = multi_pack_index(
1551 ObjectFormat::Sha1,
1552 2,
1553 1,
1554 &[(*b"PNAM", b"pack-a.idx\0\0".to_vec())],
1555 );
1556 assert!(MultiPackIndex::parse(&missing_oidf, ObjectFormat::Sha1).is_err());
1557
1558 let bad_fanout = vec![
1559 (*b"PNAM", b"pack-a.idx\0\0".to_vec()),
1560 (*b"OIDF", vec![0; 256 * 4]),
1561 (*b"OIDL", oid_a.as_bytes().to_vec()),
1562 (*b"OOFF", midx_ooff_entries(&[(0, 12)], &mut Vec::new())),
1563 ];
1564 let bad_fanout = multi_pack_index(ObjectFormat::Sha1, 2, 1, &bad_fanout);
1565 assert!(MultiPackIndex::parse(&bad_fanout, ObjectFormat::Sha1).is_err());
1566
1567 let mut unsorted = Vec::new();
1568 unsorted.push((*b"PNAM", b"pack-a.idx\0\0".to_vec()));
1569 unsorted.push((*b"OIDF", midx_oid_fanout(&[oid_a.clone(), oid_b.clone()])));
1570 let mut oid_lookup = Vec::new();
1571 oid_lookup.extend_from_slice(oid_b.as_bytes());
1572 oid_lookup.extend_from_slice(oid_a.as_bytes());
1573 unsorted.push((*b"OIDL", oid_lookup));
1574 unsorted.push((
1575 *b"OOFF",
1576 midx_ooff_entries(&[(0, 12), (0, 24)], &mut Vec::new()),
1577 ));
1578 let unsorted = multi_pack_index(ObjectFormat::Sha1, 2, 1, &unsorted);
1579 assert!(MultiPackIndex::parse(&unsorted, ObjectFormat::Sha1).is_err());
1580
1581 let bad_pack = multi_pack_index(
1582 ObjectFormat::Sha1,
1583 2,
1584 1,
1585 &midx_chunks_with_pack_names(
1586 ObjectFormat::Sha1,
1587 b"pack-a.idx\0\0".to_vec(),
1588 &[(oid_a.clone(), 1, 12)],
1589 ),
1590 );
1591 assert!(MultiPackIndex::parse(&bad_pack, ObjectFormat::Sha1).is_err());
1592
1593 let mut large_offsets = Vec::new();
1594 let missing_loff = vec![
1595 (*b"PNAM", b"pack-a.idx\0\0".to_vec()),
1596 (*b"OIDF", midx_oid_fanout(std::slice::from_ref(&oid_a))),
1597 (*b"OIDL", oid_a.as_bytes().to_vec()),
1598 (
1599 *b"OOFF",
1600 midx_ooff_entries(&[(0, 0x1_0000_0000)], &mut large_offsets),
1601 ),
1602 ];
1603 let missing_loff = multi_pack_index(ObjectFormat::Sha1, 2, 1, &missing_loff);
1604 assert!(MultiPackIndex::parse(&missing_loff, ObjectFormat::Sha1).is_err());
1605
1606 let mut bad_loff =
1607 midx_chunks_with_pack_names(ObjectFormat::Sha1, b"pack-a.idx\0\0".to_vec(), &[]);
1608 bad_loff.push((*b"LOFF", vec![0]));
1609 let bad_loff = multi_pack_index(ObjectFormat::Sha1, 2, 1, &bad_loff);
1610 assert!(MultiPackIndex::parse(&bad_loff, ObjectFormat::Sha1).is_err());
1611 }
1612
1613 #[test]
1614 fn parses_multi_pack_index_bitmap_chunks() {
1615 let first = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"first object\n")
1616 .expect("test operation should succeed");
1617 let second = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"second object\n")
1618 .expect("test operation should succeed");
1619 let mut chunks = midx_chunks_with_pack_names(
1620 ObjectFormat::Sha1,
1621 b"pack-a.idx\0pack-b.idx\0\0\0".to_vec(),
1622 &[(first, 0, 12), (second, 1, 24)],
1623 );
1624 chunks.push((*b"RIDX", midx_u32_table(&[1, 0])));
1625 chunks.push((*b"BTMP", midx_bitmap_packs(&[(0, 1), (1, 1)])));
1626 let midx = multi_pack_index(ObjectFormat::Sha1, 2, 2, &chunks);
1627
1628 let parsed = MultiPackIndex::parse(&midx, ObjectFormat::Sha1)
1629 .expect("test operation should succeed");
1630 assert_eq!(parsed.reverse_index, Some(vec![1, 0]));
1631 assert_eq!(
1632 parsed.bitmapped_packs,
1633 Some(vec![
1634 MultiPackBitmapPack {
1635 bitmap_pos: 0,
1636 bitmap_nr: 1,
1637 },
1638 MultiPackBitmapPack {
1639 bitmap_pos: 1,
1640 bitmap_nr: 1,
1641 },
1642 ])
1643 );
1644 }
1645
1646 #[test]
1647 fn writes_multi_pack_index_that_round_trips() {
1648 let first = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"first object\n")
1649 .expect("test operation should succeed");
1650 let second = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"second object\n")
1651 .expect("test operation should succeed");
1652 let bytes = MultiPackIndex::write(
1653 ObjectFormat::Sha1,
1654 2,
1655 &["pack-b.idx".into(), "pack-a.idx".into()],
1656 &[
1657 MultiPackIndexEntry {
1658 oid: second.clone(),
1659 pack_int_id: 0,
1660 offset: 0x1_0000_0000,
1661 force_large_offset: false,
1662 },
1663 MultiPackIndexEntry {
1664 oid: first.clone(),
1665 pack_int_id: 1,
1666 offset: 12,
1667 force_large_offset: false,
1668 },
1669 ],
1670 )
1671 .expect("test operation should succeed");
1672
1673 let parsed = MultiPackIndex::parse(&bytes, ObjectFormat::Sha1)
1674 .expect("test operation should succeed");
1675 assert_eq!(parsed.version, 2);
1676 assert_eq!(parsed.pack_names, vec!["pack-b.idx", "pack-a.idx"]);
1677 assert_eq!(parsed.object_count, 2);
1678 assert_eq!(
1679 parsed
1680 .find(&first)
1681 .expect("test operation should succeed")
1682 .pack_int_id,
1683 1
1684 );
1685 assert_eq!(
1686 parsed
1687 .find(&first)
1688 .expect("test operation should succeed")
1689 .offset,
1690 12
1691 );
1692 assert_eq!(
1693 parsed
1694 .find(&second)
1695 .expect("test operation should succeed")
1696 .pack_int_id,
1697 0
1698 );
1699 assert_eq!(
1700 parsed
1701 .find(&second)
1702 .expect("test operation should succeed")
1703 .offset,
1704 0x1_0000_0000
1705 );
1706 assert!(parsed.chunks.iter().any(|chunk| chunk.id == *b"LOFF"));
1707 }
1708
1709 #[test]
1710 fn write_multi_pack_index_rejects_invalid_inputs() {
1711 let oid = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"object\n")
1712 .expect("test operation should succeed");
1713 assert!(MultiPackIndex::write(ObjectFormat::Sha1, 3, &["pack-a.idx".into()], &[]).is_err());
1714 assert!(
1715 MultiPackIndex::write(
1716 ObjectFormat::Sha1,
1717 1,
1718 &["pack-b.idx".into(), "pack-a.idx".into()],
1719 &[],
1720 )
1721 .is_err()
1722 );
1723 assert!(MultiPackIndex::write(ObjectFormat::Sha1, 2, &["pack/a.idx".into()], &[]).is_err());
1724 assert!(
1725 MultiPackIndex::write(
1726 ObjectFormat::Sha1,
1727 2,
1728 &["pack-a.idx".into()],
1729 &[MultiPackIndexEntry {
1730 oid,
1731 pack_int_id: 1,
1732 offset: 12,
1733 force_large_offset: false,
1734 }],
1735 )
1736 .is_err()
1737 );
1738 assert!(
1739 MultiPackIndex::write(
1740 ObjectFormat::Sha1,
1741 2,
1742 &["pack-a.idx".into()],
1743 &[
1744 MultiPackIndexEntry {
1745 oid,
1746 pack_int_id: 0,
1747 offset: 12,
1748 force_large_offset: false,
1749 },
1750 MultiPackIndexEntry {
1751 oid,
1752 pack_int_id: 0,
1753 offset: 24,
1754 force_large_offset: false,
1755 },
1756 ],
1757 )
1758 .is_err()
1759 );
1760 }
1761
1762 #[test]
1763 fn rejects_bad_multi_pack_index_bitmap_chunks() {
1764 let oid_a = ObjectId::from_hex(
1765 ObjectFormat::Sha1,
1766 "1111111111111111111111111111111111111111",
1767 )
1768 .expect("test operation should succeed");
1769 let oid_b = ObjectId::from_hex(
1770 ObjectFormat::Sha1,
1771 "2222222222222222222222222222222222222222",
1772 )
1773 .expect("test operation should succeed");
1774
1775 let mut duplicate_ridx = midx_chunks_with_pack_names(
1776 ObjectFormat::Sha1,
1777 b"pack-a.idx\0\0".to_vec(),
1778 &[(oid_a.clone(), 0, 12), (oid_b.clone(), 0, 24)],
1779 );
1780 duplicate_ridx.push((*b"RIDX", midx_u32_table(&[0, 0])));
1781 let duplicate_ridx = multi_pack_index(ObjectFormat::Sha1, 2, 1, &duplicate_ridx);
1782 assert!(MultiPackIndex::parse(&duplicate_ridx, ObjectFormat::Sha1).is_err());
1783
1784 let mut short_btmp = midx_chunks_with_pack_names(
1785 ObjectFormat::Sha1,
1786 b"pack-a.idx\0pack-b.idx\0\0\0".to_vec(),
1787 &[(oid_a.clone(), 0, 12), (oid_b.clone(), 1, 24)],
1788 );
1789 short_btmp.push((*b"BTMP", midx_bitmap_packs(&[(0, 1)])));
1790 let short_btmp = multi_pack_index(ObjectFormat::Sha1, 2, 2, &short_btmp);
1791 assert!(MultiPackIndex::parse(&short_btmp, ObjectFormat::Sha1).is_err());
1792
1793 let mut out_of_range_btmp = midx_chunks_with_pack_names(
1794 ObjectFormat::Sha1,
1795 b"pack-a.idx\0\0".to_vec(),
1796 &[(oid_a, 0, 12), (oid_b, 0, 24)],
1797 );
1798 out_of_range_btmp.push((*b"BTMP", midx_bitmap_packs(&[(1, 2)])));
1799 let out_of_range_btmp = multi_pack_index(ObjectFormat::Sha1, 2, 1, &out_of_range_btmp);
1800 assert!(MultiPackIndex::parse(&out_of_range_btmp, ObjectFormat::Sha1).is_err());
1801 }
1802
1803 #[test]
1804 fn parses_pack_bitmap_index_with_hash_cache() {
1805 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1806 .expect("test operation should succeed");
1807 let bitmap = pack_bitmap_index(
1808 ObjectFormat::Sha1,
1809 3,
1810 PackBitmapIndex::OPTION_FULL_DAG | PackBitmapIndex::OPTION_HASH_CACHE,
1811 &pack_checksum,
1812 &[(2, 0, 1, &[0b101])],
1813 Some(&[0x1111_1111, 0x2222_2222, 0x3333_3333]),
1814 );
1815
1816 let parsed = PackBitmapIndex::parse(&bitmap, ObjectFormat::Sha1, 3)
1817 .expect("test operation should succeed");
1818 assert_eq!(parsed.version, 1);
1819 assert_eq!(parsed.format, ObjectFormat::Sha1);
1820 assert_eq!(
1821 parsed.options,
1822 PackBitmapIndex::OPTION_FULL_DAG | PackBitmapIndex::OPTION_HASH_CACHE
1823 );
1824 assert_eq!(parsed.pack_checksum, pack_checksum);
1825 assert_eq!(parsed.type_bitmaps.commits.bit_size, 3);
1826 assert_eq!(parsed.type_bitmaps.trees.bit_size, 3);
1827 assert_eq!(parsed.entries.len(), 1);
1828 let entry = parsed
1829 .entry_for_index_position(2)
1830 .expect("test operation should succeed");
1831 assert_eq!(entry.xor_offset, 0);
1832 assert_eq!(entry.flags, 1);
1833 assert_eq!(entry.bitmap.words, ewah_literal_words(&[0b101]));
1834 assert_eq!(
1835 parsed.name_hash_cache,
1836 Some(vec![0x1111_1111, 0x2222_2222, 0x3333_3333])
1837 );
1838 }
1839
1840 #[test]
1841 fn parses_pack_bitmap_index_sha256() {
1842 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha256, b"pack")
1843 .expect("test operation should succeed");
1844 let bitmap = pack_bitmap_index(
1845 ObjectFormat::Sha256,
1846 2,
1847 PackBitmapIndex::OPTION_FULL_DAG,
1848 &pack_checksum,
1849 &[(0, 0, 0, &[0b11])],
1850 None,
1851 );
1852
1853 let parsed = PackBitmapIndex::parse(&bitmap, ObjectFormat::Sha256, 2)
1854 .expect("test operation should succeed");
1855 assert_eq!(parsed.version, 1);
1856 assert_eq!(parsed.format, ObjectFormat::Sha256);
1857 assert_eq!(parsed.pack_checksum, pack_checksum);
1858 assert_eq!(parsed.index_checksum.format(), ObjectFormat::Sha256);
1859 assert_eq!(parsed.entries[0].object_position, 0);
1860 assert_eq!(parsed.name_hash_cache, None);
1861 }
1862
1863 #[test]
1864 fn parses_upstream_git_written_pack_bitmap_index() {
1865 let root = unique_temp_dir("git-pack-bitmap-upstream");
1866 fs::create_dir_all(&root).expect("test operation should succeed");
1867 {
1868 run_git_success(&root, &["init", "-q", "-b", "main"]);
1869 run_git_success(
1870 &root,
1871 &[
1872 "-c",
1873 "user.name=Example User",
1874 "-c",
1875 "user.email=example@example.invalid",
1876 "commit",
1877 "--allow-empty",
1878 "-q",
1879 "-m",
1880 "one",
1881 ],
1882 );
1883 run_git_success(
1884 &root,
1885 &[
1886 "-c",
1887 "user.name=Example User",
1888 "-c",
1889 "user.email=example@example.invalid",
1890 "commit",
1891 "--allow-empty",
1892 "-q",
1893 "-m",
1894 "two",
1895 ],
1896 );
1897 run_git_success(&root, &["repack", "-adb"]);
1898 let pack_dir = root.join(".git").join("objects").join("pack");
1899 let idx_path = single_path_with_extension(&pack_dir, "idx");
1900 let bitmap_path = single_path_with_extension(&pack_dir, "bitmap");
1901 let index = PackIndex::parse(
1902 &fs::read(idx_path).expect("test operation should succeed"),
1903 ObjectFormat::Sha1,
1904 )
1905 .expect("test operation should succeed");
1906 let bitmap = PackBitmapIndex::parse(
1907 &fs::read(bitmap_path).expect("test operation should succeed"),
1908 ObjectFormat::Sha1,
1909 index.entries.len(),
1910 )
1911 .expect("test operation should succeed");
1912 assert_eq!(bitmap.pack_checksum, index.pack_checksum);
1913 assert!(!bitmap.entries.is_empty());
1914 };
1915 let _ = fs::remove_dir_all(&root);
1916 }
1917
1918 #[test]
1919 fn rejects_bad_pack_bitmap_index_header_and_checksum() {
1920 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1921 .expect("test operation should succeed");
1922 let bitmap = pack_bitmap_index(
1923 ObjectFormat::Sha1,
1924 1,
1925 PackBitmapIndex::OPTION_FULL_DAG,
1926 &pack_checksum,
1927 &[(0, 0, 0, &[1])],
1928 None,
1929 );
1930
1931 let mut bad_signature = bitmap.clone();
1932 bad_signature[0] = b'X';
1933 assert!(PackBitmapIndex::parse(&bad_signature, ObjectFormat::Sha1, 1).is_err());
1934
1935 let mut bad_version = bitmap.clone();
1936 bad_version[5] = 2;
1937 refresh_trailing_checksum(ObjectFormat::Sha1, &mut bad_version);
1938 assert!(PackBitmapIndex::parse(&bad_version, ObjectFormat::Sha1, 1).is_err());
1939
1940 let mut bad_option = bitmap.clone();
1941 bad_option[7] = 0x20;
1942 refresh_trailing_checksum(ObjectFormat::Sha1, &mut bad_option);
1943 assert!(PackBitmapIndex::parse(&bad_option, ObjectFormat::Sha1, 1).is_err());
1944
1945 let mut bad_checksum = bitmap;
1946 let last = bad_checksum.len() - 1;
1947 bad_checksum[last] ^= 1;
1948 assert!(PackBitmapIndex::parse(&bad_checksum, ObjectFormat::Sha1, 1).is_err());
1949 }
1950
1951 #[test]
1952 fn rejects_bad_pack_bitmap_index_ewah_and_entries() {
1953 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha1, b"pack")
1954 .expect("test operation should succeed");
1955 let bitmap = pack_bitmap_index(
1956 ObjectFormat::Sha1,
1957 2,
1958 PackBitmapIndex::OPTION_FULL_DAG,
1959 &pack_checksum,
1960 &[(0, 0, 0, &[0b01]), (1, 1, 0, &[0b11])],
1961 None,
1962 );
1963
1964 let mut truncated = bitmap.clone();
1965 truncated.truncate(truncated.len() - ObjectFormat::Sha1.raw_len() - 1);
1966 refresh_trailing_checksum(ObjectFormat::Sha1, &mut truncated);
1967 assert!(PackBitmapIndex::parse(&truncated, ObjectFormat::Sha1, 2).is_err());
1968
1969 let mut out_of_range_position = pack_bitmap_index(
1970 ObjectFormat::Sha1,
1971 2,
1972 PackBitmapIndex::OPTION_FULL_DAG,
1973 &pack_checksum,
1974 &[(2, 0, 0, &[0b01])],
1975 None,
1976 );
1977 assert!(PackBitmapIndex::parse(&out_of_range_position, ObjectFormat::Sha1, 2).is_err());
1978 refresh_trailing_checksum(ObjectFormat::Sha1, &mut out_of_range_position);
1979 assert!(PackBitmapIndex::parse(&out_of_range_position, ObjectFormat::Sha1, 2).is_err());
1980
1981 let invalid_xor = pack_bitmap_index(
1982 ObjectFormat::Sha1,
1983 2,
1984 PackBitmapIndex::OPTION_FULL_DAG,
1985 &pack_checksum,
1986 &[(0, 1, 0, &[0b01])],
1987 None,
1988 );
1989 assert!(PackBitmapIndex::parse(&invalid_xor, ObjectFormat::Sha1, 2).is_err());
1990 }
1991
1992 #[test]
1993 fn parses_single_entry_pack_index_sha256() {
1994 let oid = sley_core::object_id_for_bytes(ObjectFormat::Sha256, "blob", b"hello sha256\n")
1995 .expect("test operation should succeed");
1996 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha256, b"pack")
1997 .expect("test operation should succeed");
1998 let index = single_entry_index(
1999 ObjectFormat::Sha256,
2000 oid,
2001 0x1234_5678,
2002 12,
2003 pack_checksum.clone(),
2004 );
2005 let parsed =
2006 PackIndex::parse(&index, ObjectFormat::Sha256).expect("test operation should succeed");
2007 assert_eq!(parsed.version, 2);
2008 assert_eq!(parsed.pack_checksum, pack_checksum);
2009 assert_eq!(parsed.entries.len(), 1);
2010 assert_eq!(
2011 parsed
2012 .find(&oid)
2013 .expect("test operation should succeed")
2014 .offset,
2015 12
2016 );
2017 assert_eq!(
2018 parsed
2019 .find(&oid)
2020 .expect("test operation should succeed")
2021 .crc32,
2022 0x1234_5678
2023 );
2024 assert_eq!(parsed.index_checksum.format(), ObjectFormat::Sha256);
2025 assert_pack_index_view_matches_owned(&index, ObjectFormat::Sha256);
2026 }
2027
2028 #[test]
2029 fn write_packed_deltifies_similar_blobs_and_round_trips_sha1() {
2030 write_packed_deltifies_similar_blobs_and_round_trips(ObjectFormat::Sha1);
2031 }
2032
2033 #[test]
2034 fn write_packed_deltifies_similar_blobs_and_round_trips_sha256() {
2035 write_packed_deltifies_similar_blobs_and_round_trips(ObjectFormat::Sha256);
2036 }
2037
2038 #[test]
2039 fn write_packed_rejects_duplicate_objects() {
2040 let object = EncodedObject::new(ObjectType::Blob, b"same\n".to_vec());
2041 assert!(PackFile::write_packed(&[object.clone(), object], ObjectFormat::Sha1,).is_err());
2042 }
2043
2044 #[test]
2045 fn write_packed_with_known_ids_validates_ids_before_trusting_them() {
2046 let object = EncodedObject::new(ObjectType::Blob, b"same\n".to_vec());
2047 let sha1 = object
2048 .object_id(ObjectFormat::Sha1)
2049 .expect("test operation should succeed");
2050 let sha256 = object
2051 .object_id(ObjectFormat::Sha256)
2052 .expect("test operation should succeed");
2053 let duplicate = [
2054 PackInput {
2055 oid: &sha1,
2056 object: &object,
2057 },
2058 PackInput {
2059 oid: &sha1,
2060 object: &object,
2061 },
2062 ];
2063 assert!(PackFile::write_packed_with_known_ids(&duplicate, ObjectFormat::Sha1).is_err());
2064
2065 let wrong_format = [PackInput {
2066 oid: &sha256,
2067 object: &object,
2068 }];
2069 assert!(PackFile::write_packed_with_known_ids(&wrong_format, ObjectFormat::Sha1).is_err());
2070 }
2071
2072 #[test]
2073 fn write_packed_with_known_ids_to_writer_matches_in_memory_pack() {
2074 let objects = similar_blob_family(6);
2075 let object_ids = objects
2076 .iter()
2077 .map(|object| {
2078 object
2079 .object_id(ObjectFormat::Sha1)
2080 .expect("test operation should succeed")
2081 })
2082 .collect::<Vec<_>>();
2083 let inputs = objects
2084 .iter()
2085 .zip(&object_ids)
2086 .map(|(object, oid)| PackInput { oid, object })
2087 .collect::<Vec<_>>();
2088 let options = PackWriteOptions::new();
2089 let in_memory = PackFile::write_packed_with_known_ids_and_options(
2090 &inputs,
2091 ObjectFormat::Sha1,
2092 &options,
2093 )
2094 .expect("test operation should succeed");
2095 let mut written = Vec::new();
2096 let streamed = PackFile::write_packed_with_known_ids_to_writer(
2097 &inputs,
2098 ObjectFormat::Sha1,
2099 &options,
2100 &mut written,
2101 )
2102 .expect("test operation should succeed");
2103
2104 assert_eq!(written, in_memory.pack);
2105 assert_eq!(streamed.index, in_memory.index);
2106 assert_eq!(streamed.checksum, in_memory.checksum);
2107 assert_eq!(streamed.entries, in_memory.entries);
2108 assert_eq!(streamed.delta_count, in_memory.delta_count);
2109 assert_eq!(streamed.pack_size, in_memory.pack.len() as u64);
2110 }
2111
2112 #[test]
2113 fn write_packed_from_source_to_writer_deltifies_across_windows() {
2114 let format = ObjectFormat::Sha1;
2115 let mut objects = Vec::new();
2116 for idx in 0..PACK_STREAM_COMPRESSION_WINDOW_OBJECTS - 1 {
2117 objects.push(EncodedObject::new(
2118 ObjectType::Blob,
2119 format!("unrelated streamed source object {idx:04}\n").into_bytes(),
2120 ));
2121 }
2122 let shared = b"cross-window base payload with enough shared anchors\n".repeat(64);
2126 let mut base_body = shared.clone();
2127 base_body.extend_from_slice(b"base\n");
2128 let mut target_body = shared;
2129 target_body.extend_from_slice(b"target\n");
2130 objects.push(EncodedObject::new(ObjectType::Blob, base_body));
2131 objects.push(EncodedObject::new(ObjectType::Blob, target_body));
2132
2133 let object_ids = objects
2134 .iter()
2135 .map(|object| {
2136 object
2137 .object_id(format)
2138 .expect("test operation should succeed")
2139 })
2140 .collect::<Vec<_>>();
2141 let base_oid = object_ids[PACK_STREAM_COMPRESSION_WINDOW_OBJECTS - 1];
2142 let target_oid = object_ids[PACK_STREAM_COMPRESSION_WINDOW_OBJECTS];
2143 let object_map = object_ids
2144 .iter()
2145 .copied()
2146 .zip(objects.into_iter().map(Arc::new))
2147 .collect::<HashMap<_, _>>();
2148
2149 let options = PackWriteOptions::new().with_reorder(false).with_window(10);
2150 let mut written = Vec::new();
2151 let summary = PackFile::write_packed_from_source_to_writer(
2152 &object_ids,
2153 format,
2154 &options,
2155 |oid| {
2156 object_map
2157 .get(oid)
2158 .cloned()
2159 .ok_or_else(|| GitError::not_found(format!("missing test object {oid}")))
2160 },
2161 &mut written,
2162 )
2163 .expect("test operation should succeed");
2164
2165 assert!(
2166 summary.delta_count > 0,
2167 "expected source-backed streaming writer to find deltas"
2168 );
2169 let stats =
2170 PackFile::verify_pack_stats(&written, format).expect("test operation should succeed");
2171 let target = stats
2172 .objects
2173 .iter()
2174 .find(|entry| entry.oid == target_oid)
2175 .expect("target object should be present");
2176 assert_eq!(target.base_oid, Some(base_oid));
2177 }
2178
2179 fn write_packed_deltifies_similar_blobs_and_round_trips(format: ObjectFormat) {
2180 let objects = similar_blob_family(8);
2181 let packed =
2182 PackFile::write_packed(&objects, format).expect("test operation should succeed");
2183 let undeltified =
2184 PackFile::write_undeltified(&objects, format).expect("test operation should succeed");
2185
2186 assert!(
2189 packed.pack.len() < undeltified.pack.len(),
2190 "expected delta pack ({}) smaller than undeltified pack ({})",
2191 packed.pack.len(),
2192 undeltified.pack.len()
2193 );
2194
2195 let kinds = pack_entry_kinds(&packed.pack, format);
2197 let delta_count = kinds
2198 .iter()
2199 .filter(|kind| matches!(kind, PackObjectKind::OfsDelta | PackObjectKind::RefDelta))
2200 .count();
2201 assert!(
2202 delta_count >= 1,
2203 "expected at least one delta entry, found kinds {kinds:?}"
2204 );
2205
2206 let parsed = PackFile::parse(&packed.pack, format).expect("test operation should succeed");
2208 assert_eq!(parsed.entries.len(), objects.len());
2209 for object in &objects {
2210 let oid = object
2211 .object_id(format)
2212 .expect("test operation should succeed");
2213 let found = parsed
2214 .entries
2215 .iter()
2216 .find(|entry| entry.entry.oid == oid)
2217 .unwrap_or_else(|| panic!("object {oid} missing from parsed pack"));
2218 assert_eq!(&found.object, object, "object {oid} did not round-trip");
2219 }
2220
2221 let index = PackIndex::parse(&packed.index, format).expect("test operation should succeed");
2223 assert_eq!(index.pack_checksum, packed.checksum);
2224 for object in &objects {
2225 let oid = object
2226 .object_id(format)
2227 .expect("test operation should succeed");
2228 assert!(index.find(&oid).is_some(), "index missing {oid}");
2229 }
2230 }
2231
2232 #[test]
2233 fn write_packed_emits_ofs_delta_by_default() {
2234 let objects = similar_blob_family(6);
2235 let packed = PackFile::write_packed(&objects, ObjectFormat::Sha1)
2236 .expect("test operation should succeed");
2237 let kinds = pack_entry_kinds(&packed.pack, ObjectFormat::Sha1);
2238 assert!(
2239 kinds.contains(&PackObjectKind::OfsDelta),
2240 "expected an ofs-delta entry by default, found {kinds:?}"
2241 );
2242 assert!(
2243 !kinds.contains(&PackObjectKind::RefDelta),
2244 "default self-contained pack must not use ref-delta, found {kinds:?}"
2245 );
2246 assert!(PackFile::parse(&packed.pack, ObjectFormat::Sha1).is_ok());
2248 }
2249
2250 #[test]
2251 fn write_packed_can_emit_ref_delta() {
2252 let objects = similar_blob_family(6);
2253 let options = PackWriteOptions::new().with_prefer_ofs_delta(false);
2254 let packed = PackFile::write_packed_with_options(&objects, ObjectFormat::Sha1, &options)
2255 .expect("test operation should succeed");
2256 let kinds = pack_entry_kinds(&packed.pack, ObjectFormat::Sha1);
2257 assert!(
2258 kinds.contains(&PackObjectKind::RefDelta),
2259 "expected a ref-delta entry, found {kinds:?}"
2260 );
2261 assert!(
2262 !kinds.contains(&PackObjectKind::OfsDelta),
2263 "ref-delta mode must not emit ofs-delta, found {kinds:?}"
2264 );
2265
2266 let parsed = PackFile::parse(&packed.pack, ObjectFormat::Sha1)
2269 .expect("test operation should succeed");
2270 assert_eq!(parsed.entries.len(), objects.len());
2271 }
2272
2273 #[test]
2274 fn write_packed_bounds_delta_chain_depth() {
2275 let objects = incremental_blob_chain(20);
2279 let format = ObjectFormat::Sha1;
2280
2281 for max_depth in [1usize, 2, 5] {
2282 let options = PackWriteOptions::new()
2283 .with_window(20)
2284 .with_depth(max_depth);
2285 let packed = PackFile::write_packed_with_options(&objects, format, &options)
2286 .expect("test operation should succeed");
2287
2288 let depths = pack_entry_depths(&packed.pack, format);
2289 let observed = depths.iter().copied().max().unwrap_or(0);
2290 assert!(
2291 observed <= max_depth,
2292 "max chain depth {observed} exceeded bound {max_depth}"
2293 );
2294
2295 let parsed =
2297 PackFile::parse(&packed.pack, format).expect("test operation should succeed");
2298 for object in &objects {
2299 let oid = object
2300 .object_id(format)
2301 .expect("test operation should succeed");
2302 let found = parsed
2303 .entries
2304 .iter()
2305 .find(|entry| entry.entry.oid == oid)
2306 .expect("test operation should succeed");
2307 assert_eq!(&found.object, object);
2308 }
2309 }
2310 }
2311
2312 #[test]
2313 fn write_packed_depth_zero_stores_everything_undeltified() {
2314 let objects = similar_blob_family(5);
2315 let options = PackWriteOptions::new().with_depth(0);
2316 let packed = PackFile::write_packed_with_options(&objects, ObjectFormat::Sha1, &options)
2317 .expect("test operation should succeed");
2318 let kinds = pack_entry_kinds(&packed.pack, ObjectFormat::Sha1);
2319 assert!(
2320 kinds
2321 .iter()
2322 .all(|kind| !matches!(kind, PackObjectKind::OfsDelta | PackObjectKind::RefDelta)),
2323 "depth 0 must disable deltas, found {kinds:?}"
2324 );
2325 }
2326
2327 #[test]
2328 fn write_thin_uses_external_base_and_round_trips_sha1() {
2329 write_thin_uses_external_base_and_round_trips(ObjectFormat::Sha1);
2330 }
2331
2332 #[test]
2333 fn write_thin_uses_external_base_and_round_trips_sha256() {
2334 write_thin_uses_external_base_and_round_trips(ObjectFormat::Sha256);
2335 }
2336
2337 fn write_thin_uses_external_base_and_round_trips(format: ObjectFormat) {
2338 let base = blob_with_marker("EXTERNAL-BASE");
2341 let target = blob_with_marker("EXTERNAL-TARGET");
2342 let base_oid = base
2343 .object_id(format)
2344 .expect("test operation should succeed");
2345
2346 let mut external = HashMap::new();
2347 external.insert(base_oid, base.clone());
2348 let packed = PackFile::write_thin(std::slice::from_ref(&target), format, external)
2349 .expect("test operation should succeed");
2350
2351 let kinds = pack_entry_kinds(&packed.pack, format);
2353 assert_eq!(kinds, vec![PackObjectKind::RefDelta]);
2354
2355 let mut offset = 12usize;
2357 let header =
2358 parse_entry_header(&packed.pack, &mut offset).expect("test operation should succeed");
2359 assert_eq!(header.kind, PackObjectKind::RefDelta);
2360 let referenced =
2361 ObjectId::from_raw(format, &packed.pack[offset..offset + format.raw_len()])
2362 .expect("test operation should succeed");
2363 assert_eq!(referenced, base_oid);
2364
2365 assert!(PackFile::parse(&packed.pack, format).is_err());
2367
2368 let parsed = PackFile::parse_thin(&packed.pack, format, |oid| {
2370 if oid == &base_oid {
2371 Ok(Some(base.clone()))
2372 } else {
2373 Ok(None)
2374 }
2375 })
2376 .expect("test operation should succeed");
2377 assert_eq!(parsed.entries.len(), 1);
2378 assert_eq!(parsed.entries[0].object, target);
2379 }
2380
2381 #[test]
2382 fn write_packed_preserves_distinct_objects_with_no_similarity() {
2383 let objects = vec![
2386 EncodedObject::new(ObjectType::Blob, b"alpha distinct\n".to_vec()),
2387 EncodedObject::new(ObjectType::Tree, vec![0u8; 0]),
2388 EncodedObject::new(ObjectType::Commit, b"tree 0000\n".to_vec()),
2389 ];
2390 let format = ObjectFormat::Sha1;
2391 let packed =
2392 PackFile::write_packed(&objects, format).expect("test operation should succeed");
2393 let parsed = PackFile::parse(&packed.pack, format).expect("test operation should succeed");
2394 assert_eq!(parsed.entries.len(), objects.len());
2395 for object in &objects {
2396 let oid = object
2397 .object_id(format)
2398 .expect("test operation should succeed");
2399 assert!(parsed.entries.iter().any(|entry| entry.entry.oid == oid));
2400 }
2401 }
2402
2403 fn similar_blob_family(count: usize) -> Vec<EncodedObject> {
2407 let mut common_head = Vec::new();
2408 for _ in 0..200 {
2409 common_head.extend_from_slice(b"shared header line for delta testing\n");
2410 }
2411 let mut common_tail = Vec::new();
2412 for _ in 0..200 {
2413 common_tail.extend_from_slice(b"shared trailer line for delta testing\n");
2414 }
2415 (0..count)
2416 .map(|idx| {
2417 let mut body = common_head.clone();
2418 body.extend_from_slice(format!("UNIQUE MIDDLE MARKER NUMBER {idx}\n").as_bytes());
2419 body.extend_from_slice(&common_tail);
2420 EncodedObject::new(ObjectType::Blob, body)
2421 })
2422 .collect()
2423 }
2424
2425 fn incremental_blob_chain(count: usize) -> Vec<EncodedObject> {
2428 let mut body = Vec::new();
2429 for _ in 0..100 {
2430 body.extend_from_slice(b"baseline content shared across the whole chain\n");
2431 }
2432 let mut objects = Vec::with_capacity(count);
2433 for idx in 0..count {
2434 body.extend_from_slice(format!("appended unique line {idx}\n").as_bytes());
2435 objects.push(EncodedObject::new(ObjectType::Blob, body.clone()));
2436 }
2437 objects
2438 }
2439
2440 fn blob_with_marker(marker: &str) -> EncodedObject {
2441 let mut body = Vec::new();
2442 for _ in 0..150 {
2443 body.extend_from_slice(b"common body shared between base and target\n");
2444 }
2445 body.extend_from_slice(marker.as_bytes());
2446 body.push(b'\n');
2447 for _ in 0..150 {
2448 body.extend_from_slice(b"more common body shared between objects\n");
2449 }
2450 EncodedObject::new(ObjectType::Blob, body)
2451 }
2452
2453 fn pack_entry_kinds(pack: &[u8], format: ObjectFormat) -> Vec<PackObjectKind> {
2455 pack_entry_descriptors(pack, format)
2456 .into_iter()
2457 .map(|descriptor| descriptor.kind)
2458 .collect()
2459 }
2460
2461 fn pack_entry_depths(pack: &[u8], format: ObjectFormat) -> Vec<usize> {
2465 let descriptors = pack_entry_descriptors(pack, format);
2466 let mut depth_by_offset: HashMap<u64, usize> = HashMap::new();
2467 let mut depths = Vec::with_capacity(descriptors.len());
2468 for descriptor in &descriptors {
2469 let depth = match &descriptor.base {
2470 EntryBase::None => 0,
2471 EntryBase::Offset(base_offset) => {
2472 depth_by_offset.get(base_offset).copied().unwrap_or(0) + 1
2473 }
2474 EntryBase::Ref => 1,
2478 };
2479 depth_by_offset.insert(descriptor.offset, depth);
2480 depths.push(depth);
2481 }
2482 depths
2483 }
2484
2485 struct EntryDescriptor {
2486 offset: u64,
2487 kind: PackObjectKind,
2488 base: EntryBase,
2489 }
2490
2491 enum EntryBase {
2492 None,
2493 Offset(u64),
2494 Ref,
2495 }
2496
2497 fn pack_entry_descriptors(pack: &[u8], format: ObjectFormat) -> Vec<EntryDescriptor> {
2498 let trailer_offset = pack.len() - format.raw_len();
2499 let count = u32_be(&pack[8..12]) as usize;
2500 let mut offset = 12usize;
2501 let mut descriptors = Vec::with_capacity(count);
2502 for _ in 0..count {
2503 let entry_offset = offset as u64;
2504 let header =
2505 parse_entry_header(pack, &mut offset).expect("test operation should succeed");
2506 let base = match header.kind {
2507 PackObjectKind::OfsDelta => {
2508 let base_offset = parse_ofs_delta_base_offset(pack, &mut offset, entry_offset)
2509 .expect("test operation should succeed");
2510 EntryBase::Offset(base_offset)
2511 }
2512 PackObjectKind::RefDelta => {
2513 offset += format.raw_len();
2514 EntryBase::Ref
2515 }
2516 _ => EntryBase::None,
2517 };
2518 let mut decoder = ZlibDecoder::new(&pack[offset..trailer_offset]);
2519 let mut body = Vec::new();
2520 decoder
2521 .read_to_end(&mut body)
2522 .expect("test operation should succeed");
2523 offset += decoder.total_in() as usize;
2524 descriptors.push(EntryDescriptor {
2525 offset: entry_offset,
2526 kind: header.kind,
2527 base,
2528 });
2529 }
2530 descriptors
2531 }
2532
2533 fn similar_blob_objects() -> (EncodedObject, EncodedObject) {
2534 let mut base = Vec::new();
2535 for _ in 0..300 {
2536 base.extend_from_slice(b"common payload\n");
2537 }
2538 base.extend_from_slice(b"base\n");
2539 let mut changed = Vec::new();
2540 for _ in 0..300 {
2541 changed.extend_from_slice(b"common payload\n");
2542 }
2543 changed.extend_from_slice(b"changed\n");
2544 (
2545 EncodedObject::new(ObjectType::Blob, base),
2546 EncodedObject::new(ObjectType::Blob, changed),
2547 )
2548 }
2549
2550 fn single_object_pack(format: ObjectFormat, object_type: ObjectType, body: &[u8]) -> Vec<u8> {
2551 let mut pack = Vec::new();
2552 pack.extend_from_slice(b"PACK");
2553 pack.extend_from_slice(&2u32.to_be_bytes());
2554 pack.extend_from_slice(&1u32.to_be_bytes());
2555 write_entry_header(&mut pack, object_type, body.len() as u64);
2556 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
2557 encoder
2558 .write_all(body)
2559 .expect("test operation should succeed");
2560 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
2561 let checksum =
2562 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
2563 pack.extend_from_slice(checksum.as_bytes());
2564 pack
2565 }
2566
2567 #[derive(Clone, Copy, Debug)]
2568 enum DeltaKind {
2569 Offset,
2570 Ref,
2571 }
2572
2573 fn two_object_delta_pack(
2574 format: ObjectFormat,
2575 base: &[u8],
2576 result: &[u8],
2577 delta_kind: DeltaKind,
2578 ) -> Vec<u8> {
2579 let mut pack = Vec::new();
2580 pack.extend_from_slice(b"PACK");
2581 pack.extend_from_slice(&2u32.to_be_bytes());
2582 pack.extend_from_slice(&2u32.to_be_bytes());
2583
2584 let base_offset = pack.len();
2585 write_entry_header(&mut pack, ObjectType::Blob, base.len() as u64);
2586 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
2587 encoder
2588 .write_all(base)
2589 .expect("test operation should succeed");
2590 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
2591
2592 let delta = append_suffix_delta(base, result);
2593 let delta_offset = pack.len();
2594 write_pack_entry_header_kind(
2595 &mut pack,
2596 match delta_kind {
2597 DeltaKind::Offset => 6,
2598 DeltaKind::Ref => 7,
2599 },
2600 delta.len() as u64,
2601 );
2602 match delta_kind {
2603 DeltaKind::Offset => write_ofs_delta_offset(&mut pack, delta_offset - base_offset),
2604 DeltaKind::Ref => {
2605 let base_oid = sley_core::object_id_for_bytes(format, "blob", base)
2606 .expect("test operation should succeed");
2607 pack.extend_from_slice(base_oid.as_bytes());
2608 }
2609 }
2610 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
2611 encoder
2612 .write_all(&delta)
2613 .expect("test operation should succeed");
2614 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
2615
2616 let checksum =
2617 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
2618 pack.extend_from_slice(checksum.as_bytes());
2619 pack
2620 }
2621
2622 fn thin_ref_delta_pack(format: ObjectFormat, base: &[u8], result: &[u8]) -> Vec<u8> {
2623 let mut pack = Vec::new();
2624 pack.extend_from_slice(b"PACK");
2625 pack.extend_from_slice(&2u32.to_be_bytes());
2626 pack.extend_from_slice(&1u32.to_be_bytes());
2627
2628 let delta = append_suffix_delta(base, result);
2629 write_pack_entry_header_kind(&mut pack, 7, delta.len() as u64);
2630 let base_oid = sley_core::object_id_for_bytes(format, "blob", base)
2631 .expect("test operation should succeed");
2632 pack.extend_from_slice(base_oid.as_bytes());
2633 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
2634 encoder
2635 .write_all(&delta)
2636 .expect("test operation should succeed");
2637 pack.extend_from_slice(&encoder.finish().expect("test operation should succeed"));
2638
2639 let checksum =
2640 sley_core::digest_bytes(format, &pack).expect("test operation should succeed");
2641 pack.extend_from_slice(checksum.as_bytes());
2642 pack
2643 }
2644
2645 fn unique_temp_dir(name: &str) -> PathBuf {
2646 let nanos = SystemTime::now()
2647 .duration_since(UNIX_EPOCH)
2648 .expect("test operation should succeed")
2649 .as_nanos();
2650 std::env::temp_dir().join(format!("sley-{name}-{}-{nanos}", std::process::id()))
2651 }
2652
2653 fn run_git_success(cwd: &Path, args: &[&str]) {
2654 let output = Command::new("git")
2655 .current_dir(cwd)
2656 .args(args)
2657 .output()
2658 .unwrap_or_else(|err| panic!("failed to run git {args:?}: {err}"));
2659 assert!(
2660 output.status.success(),
2661 "git {args:?} failed with status {:?}\nstdout:\n{}\nstderr:\n{}",
2662 output.status.code(),
2663 String::from_utf8_lossy(&output.stdout),
2664 String::from_utf8_lossy(&output.stderr)
2665 );
2666 }
2667
2668 fn single_path_with_extension(dir: &Path, extension: &str) -> PathBuf {
2669 let mut paths = fs::read_dir(dir)
2670 .expect("test operation should succeed")
2671 .map(|entry| entry.expect("test operation should succeed").path())
2672 .filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some(extension))
2673 .collect::<Vec<_>>();
2674 assert_eq!(paths.len(), 1, "expected one .{extension} file");
2675 paths.remove(0)
2676 }
2677
2678 fn pack_bitmap_index(
2679 format: ObjectFormat,
2680 object_count: u32,
2681 options: u16,
2682 pack_checksum: &ObjectId,
2683 entries: &[(u32, u8, u8, &[u64])],
2684 name_hash_cache: Option<&[u32]>,
2685 ) -> Vec<u8> {
2686 let mut out = Vec::new();
2687 out.extend_from_slice(b"BITM");
2688 out.extend_from_slice(&1u16.to_be_bytes());
2689 out.extend_from_slice(&options.to_be_bytes());
2690 out.extend_from_slice(&(entries.len() as u32).to_be_bytes());
2691 out.extend_from_slice(pack_checksum.as_bytes());
2692 write_test_ewah(&mut out, object_count, &[0b001]);
2693 write_test_ewah(&mut out, object_count, &[0b010]);
2694 write_test_ewah(&mut out, object_count, &[0b100]);
2695 write_test_ewah(&mut out, object_count, &[0]);
2696 for (position, xor_offset, flags, words) in entries {
2697 out.extend_from_slice(&position.to_be_bytes());
2698 out.push(*xor_offset);
2699 out.push(*flags);
2700 write_test_ewah(&mut out, object_count, words);
2701 }
2702 if let Some(cache) = name_hash_cache {
2703 for value in cache {
2704 out.extend_from_slice(&value.to_be_bytes());
2705 }
2706 }
2707 let checksum =
2708 sley_core::digest_bytes(format, &out).expect("test operation should succeed");
2709 out.extend_from_slice(checksum.as_bytes());
2710 out
2711 }
2712
2713 fn write_test_ewah(out: &mut Vec<u8>, bit_size: u32, literals: &[u64]) {
2714 out.extend_from_slice(&bit_size.to_be_bytes());
2715 let words = ewah_literal_words(literals);
2716 out.extend_from_slice(&(words.len() as u32).to_be_bytes());
2717 for word in words {
2718 out.extend_from_slice(&word.to_be_bytes());
2719 }
2720 out.extend_from_slice(&0u32.to_be_bytes());
2721 }
2722
2723 fn ewah_literal_words(literals: &[u64]) -> Vec<u64> {
2724 let rlw = (literals.len() as u64) << 33;
2725 let mut words = vec![rlw];
2726 words.extend_from_slice(literals);
2727 words
2728 }
2729
2730 fn refresh_trailing_checksum(format: ObjectFormat, bytes: &mut [u8]) {
2731 let checksum_offset = bytes.len() - format.raw_len();
2732 let checksum = sley_core::digest_bytes(format, &bytes[..checksum_offset])
2733 .expect("test operation should succeed");
2734 bytes[checksum_offset..].copy_from_slice(checksum.as_bytes());
2735 }
2736
2737 fn append_suffix_delta(base: &[u8], result: &[u8]) -> Vec<u8> {
2738 assert!(result.starts_with(base));
2739 let suffix = &result[base.len()..];
2740 assert!(base.len() < 0x10000);
2741 assert!(suffix.len() < 0x80);
2742 let mut delta = Vec::new();
2743 write_delta_varint(&mut delta, base.len() as u64);
2744 write_delta_varint(&mut delta, result.len() as u64);
2745 delta.push(0x90);
2746 delta.push(base.len() as u8);
2747 delta.push(suffix.len() as u8);
2748 delta.extend_from_slice(suffix);
2749 delta
2750 }
2751
2752 fn write_delta_varint(out: &mut Vec<u8>, mut value: u64) {
2753 loop {
2754 let mut byte = (value as u8) & 0x7f;
2755 value >>= 7;
2756 if value != 0 {
2757 byte |= 0x80;
2758 }
2759 out.push(byte);
2760 if value == 0 {
2761 break;
2762 }
2763 }
2764 }
2765
2766 fn write_pack_entry_header_kind(out: &mut Vec<u8>, type_code: u8, mut size: u64) {
2767 let mut byte = (type_code << 4) | ((size as u8) & 0x0f);
2768 size >>= 4;
2769 if size != 0 {
2770 byte |= 0x80;
2771 }
2772 out.push(byte);
2773 while size != 0 {
2774 let mut byte = (size as u8) & 0x7f;
2775 size >>= 7;
2776 if size != 0 {
2777 byte |= 0x80;
2778 }
2779 out.push(byte);
2780 }
2781 }
2782
2783 fn write_ofs_delta_offset(out: &mut Vec<u8>, relative: usize) {
2784 assert!(relative < 0x80);
2785 out.push(relative as u8);
2786 }
2787
2788 fn single_entry_index(
2789 format: ObjectFormat,
2790 oid: ObjectId,
2791 crc32: u32,
2792 offset: u32,
2793 pack_checksum: ObjectId,
2794 ) -> Vec<u8> {
2795 let mut index = Vec::new();
2796 index.extend_from_slice(&[0xff, b't', b'O', b'c']);
2797 index.extend_from_slice(&2u32.to_be_bytes());
2798 for idx in 0..256 {
2799 let count = if idx >= usize::from(oid.as_bytes()[0]) {
2800 1u32
2801 } else {
2802 0u32
2803 };
2804 index.extend_from_slice(&count.to_be_bytes());
2805 }
2806 index.extend_from_slice(oid.as_bytes());
2807 index.extend_from_slice(&crc32.to_be_bytes());
2808 index.extend_from_slice(&offset.to_be_bytes());
2809 index.extend_from_slice(pack_checksum.as_bytes());
2810 let checksum =
2811 sley_core::digest_bytes(format, &index).expect("test operation should succeed");
2812 index.extend_from_slice(checksum.as_bytes());
2813 index
2814 }
2815
2816 fn single_entry_index_v1(
2817 format: ObjectFormat,
2818 oid: ObjectId,
2819 offset: u32,
2820 pack_checksum: ObjectId,
2821 ) -> Vec<u8> {
2822 let mut index = Vec::new();
2823 for idx in 0..256 {
2824 let count = if idx >= usize::from(oid.as_bytes()[0]) {
2825 1u32
2826 } else {
2827 0u32
2828 };
2829 index.extend_from_slice(&count.to_be_bytes());
2830 }
2831 index.extend_from_slice(&offset.to_be_bytes());
2832 index.extend_from_slice(oid.as_bytes());
2833 index.extend_from_slice(pack_checksum.as_bytes());
2834 let checksum =
2835 sley_core::digest_bytes(format, &index).expect("test operation should succeed");
2836 index.extend_from_slice(checksum.as_bytes());
2837 index
2838 }
2839
2840 fn pack_reverse_index(
2841 format: ObjectFormat,
2842 positions: &[u32],
2843 pack_checksum: ObjectId,
2844 ) -> Vec<u8> {
2845 let mut reverse_index = Vec::new();
2846 reverse_index.extend_from_slice(b"RIDX");
2847 reverse_index.extend_from_slice(&1u32.to_be_bytes());
2848 reverse_index.extend_from_slice(&hash_function_id(format).to_be_bytes());
2849 for position in positions {
2850 reverse_index.extend_from_slice(&position.to_be_bytes());
2851 }
2852 reverse_index.extend_from_slice(pack_checksum.as_bytes());
2853 let checksum =
2854 sley_core::digest_bytes(format, &reverse_index).expect("test operation should succeed");
2855 reverse_index.extend_from_slice(checksum.as_bytes());
2856 reverse_index
2857 }
2858
2859 fn pack_mtimes(format: ObjectFormat, mtimes: &[u32], pack_checksum: ObjectId) -> Vec<u8> {
2860 let mut out = Vec::new();
2861 out.extend_from_slice(b"MTME");
2862 out.extend_from_slice(&1u32.to_be_bytes());
2863 out.extend_from_slice(&hash_function_id(format).to_be_bytes());
2864 for mtime in mtimes {
2865 out.extend_from_slice(&mtime.to_be_bytes());
2866 }
2867 out.extend_from_slice(pack_checksum.as_bytes());
2868 let checksum =
2869 sley_core::digest_bytes(format, &out).expect("test operation should succeed");
2870 out.extend_from_slice(checksum.as_bytes());
2871 out
2872 }
2873
2874 fn midx_chunks_with_pack_names(
2875 _format: ObjectFormat,
2876 pack_names: Vec<u8>,
2877 entries: &[(ObjectId, u32, u64)],
2878 ) -> Vec<([u8; 4], Vec<u8>)> {
2879 let mut entries = entries.to_vec();
2880 entries.sort_by(|left, right| left.0.as_bytes().cmp(right.0.as_bytes()));
2881 let object_ids: Vec<ObjectId> = entries.iter().map(|entry| entry.0).collect();
2882 let mut large_offsets = Vec::new();
2883 let mut chunks = vec![
2884 (*b"PNAM", pack_names),
2885 (*b"OIDF", midx_oid_fanout(&object_ids)),
2886 (*b"OIDL", midx_oid_lookup(&object_ids)),
2887 (
2888 *b"OOFF",
2889 midx_ooff_entries(
2890 &entries
2891 .iter()
2892 .map(|(_oid, pack_int_id, offset)| (*pack_int_id, *offset))
2893 .collect::<Vec<_>>(),
2894 &mut large_offsets,
2895 ),
2896 ),
2897 ];
2898 if !large_offsets.is_empty() {
2899 chunks.push((*b"LOFF", large_offsets));
2900 }
2901 chunks
2902 }
2903
2904 fn midx_oid_fanout(object_ids: &[ObjectId]) -> Vec<u8> {
2905 let mut counts = [0u32; 256];
2906 for oid in object_ids {
2907 counts[oid.as_bytes()[0] as usize] += 1;
2908 }
2909 let mut running = 0u32;
2910 let mut out = Vec::new();
2911 for count in counts {
2912 running += count;
2913 out.extend_from_slice(&running.to_be_bytes());
2914 }
2915 out
2916 }
2917
2918 fn midx_oid_lookup(object_ids: &[ObjectId]) -> Vec<u8> {
2919 let mut out = Vec::new();
2920 for oid in object_ids {
2921 out.extend_from_slice(oid.as_bytes());
2922 }
2923 out
2924 }
2925
2926 fn midx_ooff_entries(entries: &[(u32, u64)], large_offsets: &mut Vec<u8>) -> Vec<u8> {
2927 let mut out = Vec::new();
2928 for (pack_int_id, offset) in entries {
2929 out.extend_from_slice(&pack_int_id.to_be_bytes());
2930 if *offset < 0x8000_0000 {
2931 out.extend_from_slice(&(*offset as u32).to_be_bytes());
2932 } else {
2933 let large_idx = (large_offsets.len() / 8) as u32;
2934 out.extend_from_slice(&(0x8000_0000 | large_idx).to_be_bytes());
2935 large_offsets.extend_from_slice(&offset.to_be_bytes());
2936 }
2937 }
2938 out
2939 }
2940
2941 fn midx_u32_table(values: &[u32]) -> Vec<u8> {
2942 let mut out = Vec::new();
2943 for value in values {
2944 out.extend_from_slice(&value.to_be_bytes());
2945 }
2946 out
2947 }
2948
2949 fn midx_bitmap_packs(entries: &[(u32, u32)]) -> Vec<u8> {
2950 let mut out = Vec::new();
2951 for (bitmap_pos, bitmap_nr) in entries {
2952 out.extend_from_slice(&bitmap_pos.to_be_bytes());
2953 out.extend_from_slice(&bitmap_nr.to_be_bytes());
2954 }
2955 out
2956 }
2957
2958 fn multi_pack_index(
2959 format: ObjectFormat,
2960 version: u8,
2961 pack_count: u32,
2962 chunks: &[([u8; 4], Vec<u8>)],
2963 ) -> Vec<u8> {
2964 let lookup_len = (chunks.len() + 1) * 12;
2965 let mut out = Vec::new();
2966 out.extend_from_slice(b"MIDX");
2967 out.push(version);
2968 out.push(hash_function_id(format) as u8);
2969 out.push(chunks.len() as u8);
2970 out.push(0);
2971 out.extend_from_slice(&pack_count.to_be_bytes());
2972 let mut chunk_offset = (12 + lookup_len) as u64;
2973 for (id, data) in chunks {
2974 out.extend_from_slice(id);
2975 out.extend_from_slice(&chunk_offset.to_be_bytes());
2976 chunk_offset += data.len() as u64;
2977 }
2978 out.extend_from_slice(&[0, 0, 0, 0]);
2979 out.extend_from_slice(&chunk_offset.to_be_bytes());
2980 for (_id, data) in chunks {
2981 out.extend_from_slice(data);
2982 }
2983 let checksum =
2984 sley_core::digest_bytes(format, &out).expect("test operation should succeed");
2985 out.extend_from_slice(checksum.as_bytes());
2986 out
2987 }
2988
2989 fn pack_checksum_sha1() -> ObjectId {
2992 sley_core::digest_bytes(ObjectFormat::Sha1, b"pack").expect("test operation should succeed")
2993 }
2994
2995 fn parse_ewah_bytes(bytes: &[u8]) -> EwahBitmap {
2996 let mut offset = 0usize;
2999 let checksum_offset = bytes.len();
3000 parse_bitmap_ewah(bytes, &mut offset, checksum_offset, 0)
3001 .expect("test operation should succeed")
3002 }
3003
3004 #[test]
3005 fn ewah_encodes_single_literal_word_matching_helper() {
3006 let ewah = EwahBitmap::from_words(64, &[0b101]).expect("test operation should succeed");
3010 assert_eq!(ewah.words, ewah_literal_words(&[0b101]));
3011 assert_eq!(ewah.rlw_position, 0);
3012 assert_eq!(ewah.bit_size, 64);
3013 }
3014
3015 #[test]
3016 fn ewah_byte_layout_is_big_endian() {
3017 let ewah = EwahBitmap::from_words(64, &[0x0102_0304_0506_0708])
3018 .expect("test operation should succeed");
3019 let bytes = ewah.to_bytes();
3020 let mut expected = Vec::new();
3021 expected.extend_from_slice(&64u32.to_be_bytes()); expected.extend_from_slice(&2u32.to_be_bytes()); expected.extend_from_slice(&(1u64 << 33).to_be_bytes()); expected.extend_from_slice(&0x0102_0304_0506_0708u64.to_be_bytes());
3025 expected.extend_from_slice(&0u32.to_be_bytes()); assert_eq!(bytes, expected);
3027 }
3028
3029 #[test]
3030 fn ewah_empty_bitmap_serialises_like_git() {
3031 let ewah = EwahBitmap::empty();
3032 let bytes = ewah.to_bytes();
3033 assert_eq!(bytes, vec![0u8; 12]);
3035 let parsed = parse_ewah_bytes(&bytes);
3037 assert_eq!(parsed, ewah);
3038 assert!(
3039 parsed
3040 .to_positions()
3041 .expect("test operation should succeed")
3042 .is_empty()
3043 );
3044 }
3045
3046 #[test]
3047 fn ewah_compresses_clean_zero_run() {
3048 let ewah =
3051 EwahBitmap::from_words(256, &[0, 0, 0, 0b1]).expect("test operation should succeed");
3052 assert_eq!(ewah.words.len(), 2, "expected one RLW plus one literal");
3053 let rlw = ewah.words[0];
3054 assert_eq!(rlw & 1, 0, "run bit should be zero");
3055 assert_eq!((rlw >> 1) & 0xffff_ffff, 3, "run length should be 3");
3056 assert_eq!(rlw >> 33, 1, "literal length should be 1");
3057 assert_eq!(ewah.words[1], 0b1);
3058 }
3059
3060 #[test]
3061 fn ewah_compresses_clean_ones_run() {
3062 let ewah = EwahBitmap::from_words(192, &[u64::MAX, u64::MAX, u64::MAX])
3063 .expect("test operation should succeed");
3064 assert_eq!(ewah.words.len(), 1);
3066 let rlw = ewah.words[0];
3067 assert_eq!(rlw & 1, 1, "run bit should be one");
3068 assert_eq!((rlw >> 1) & 0xffff_ffff, 3, "run length should be 3");
3069 assert_eq!(rlw >> 33, 0, "no literals");
3070 }
3071
3072 #[test]
3073 fn ewah_run_then_literal_then_run_roundtrips() {
3074 let words = vec![0, 0, 0xdead_beef, u64::MAX, u64::MAX, 0, 0xabc];
3075 let bit_size = (words.len() * 64) as u32;
3076 let ewah = EwahBitmap::from_words(bit_size, &words).expect("test operation should succeed");
3077 assert_eq!(
3078 ewah.to_words().expect("test operation should succeed"),
3079 words
3080 );
3081 }
3082
3083 #[test]
3084 fn ewah_drops_trailing_clean_zero_words() {
3085 let words = vec![0b1, 0, 0, 0];
3088 let ewah = EwahBitmap::from_words(1, &words).expect("test operation should succeed");
3089 assert_eq!(ewah.bit_size, 1);
3091 assert_eq!(
3092 ewah.to_words().expect("test operation should succeed"),
3093 vec![0b1]
3094 );
3095 }
3096
3097 #[test]
3098 fn ewah_from_positions_roundtrips_via_positions() {
3099 let positions = [0u32, 1, 63, 64, 65, 200, 511];
3100 let ewah =
3101 EwahBitmap::from_positions(512, &positions).expect("test operation should succeed");
3102 let mut decoded = ewah.to_positions().expect("test operation should succeed");
3103 decoded.sort_unstable();
3104 assert_eq!(decoded, positions);
3105 }
3106
3107 #[test]
3108 fn ewah_from_positions_dedupes_and_orders() {
3109 let ewah = EwahBitmap::from_positions(128, &[100, 5, 100, 5, 5])
3110 .expect("test operation should succeed");
3111 assert_eq!(
3112 ewah.to_positions().expect("test operation should succeed"),
3113 vec![5, 100]
3114 );
3115 }
3116
3117 #[test]
3118 fn ewah_huge_zero_run_spans_multiple_rlws() {
3119 let mut builder = EwahBuilder::new(0);
3124 builder.add_empty_words(false, 0xffff_ffff);
3125 builder.add_empty_words(false, 5);
3126 let ewah = builder.finish().expect("test operation should succeed");
3127 assert_eq!(ewah.words.len(), 2, "run split across two RLWs");
3128 assert_eq!((ewah.words[0] >> 1) & 0xffff_ffff, 0xffff_ffff);
3129 assert_eq!(ewah.words[1] & 1, 0);
3130 assert_eq!((ewah.words[1] >> 1) & 0xffff_ffff, 5);
3131 assert_eq!(ewah.rlw_position, 1);
3132 }
3133
3134 #[test]
3135 fn ewah_from_words_rejects_oversized_bit_size() {
3136 assert!(EwahBitmap::from_words(65, &[0]).is_err());
3138 }
3139
3140 #[test]
3141 fn ewah_from_positions_rejects_out_of_range() {
3142 assert!(EwahBitmap::from_positions(64, &[64]).is_err());
3143 }
3144
3145 #[test]
3146 fn ewah_serialised_bytes_reparse_to_equal_bitmap() {
3147 let words = vec![0, u64::MAX, 0x1234_5678_9abc_def0, 0, 0, 0xff];
3150 let bit_size = (words.len() * 64) as u32;
3151 let ewah = EwahBitmap::from_words(bit_size, &words).expect("test operation should succeed");
3152 let bytes = ewah.to_bytes();
3153 let parsed = parse_ewah_bytes(&bytes);
3154 assert_eq!(parsed, ewah);
3155 assert_eq!(
3156 parsed.to_words().expect("test operation should succeed"),
3157 words
3158 );
3159 }
3160
3161 #[test]
3162 fn pack_bitmap_index_write_parse_roundtrip_sha1() {
3163 let object_types = [ObjectType::Commit, ObjectType::Tree, ObjectType::Blob];
3165 let bytes = write_bitmap(
3166 ObjectFormat::Sha1,
3167 pack_checksum_sha1(),
3168 &object_types,
3169 &[(0u32, 0u32, vec![1u32, 2u32])],
3170 None,
3171 )
3172 .expect("test operation should succeed");
3173 assert_eq!(&bytes[..4], b"BITM");
3174
3175 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, 3)
3176 .expect("test operation should succeed");
3177 assert_eq!(parsed.version, 1);
3178 assert_eq!(parsed.options, PackBitmapIndex::OPTION_FULL_DAG);
3179 assert_eq!(parsed.pack_checksum, pack_checksum_sha1());
3180 assert_eq!(
3181 parsed
3182 .type_bitmaps
3183 .commits
3184 .to_positions()
3185 .expect("test operation should succeed"),
3186 vec![0]
3187 );
3188 assert_eq!(
3189 parsed
3190 .type_bitmaps
3191 .trees
3192 .to_positions()
3193 .expect("test operation should succeed"),
3194 vec![1]
3195 );
3196 assert_eq!(
3197 parsed
3198 .type_bitmaps
3199 .blobs
3200 .to_positions()
3201 .expect("test operation should succeed"),
3202 vec![2]
3203 );
3204 assert!(
3205 parsed
3206 .type_bitmaps
3207 .tags
3208 .to_positions()
3209 .expect("test operation should succeed")
3210 .is_empty()
3211 );
3212 assert_eq!(parsed.entries.len(), 1);
3213 let entry = parsed
3214 .entry_for_index_position(0)
3215 .expect("test operation should succeed");
3216 assert_eq!(entry.xor_offset, 0);
3217 assert_eq!(entry.flags, 0);
3218 assert_eq!(
3219 entry
3220 .bitmap
3221 .to_positions()
3222 .expect("test operation should succeed"),
3223 vec![0, 1, 2]
3224 );
3225 assert_eq!(parsed.name_hash_cache, None);
3226 }
3227
3228 #[test]
3229 fn pack_bitmap_index_write_parse_roundtrip_sha256() {
3230 let pack_checksum = sley_core::digest_bytes(ObjectFormat::Sha256, b"pack")
3231 .expect("test operation should succeed");
3232 let object_types = [ObjectType::Commit, ObjectType::Tree];
3233 let bytes = write_bitmap(
3234 ObjectFormat::Sha256,
3235 pack_checksum.clone(),
3236 &object_types,
3237 &[(0u32, 0u32, vec![1u32])],
3238 None,
3239 )
3240 .expect("test operation should succeed");
3241 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha256, 2)
3242 .expect("test operation should succeed");
3243 assert_eq!(parsed.format, ObjectFormat::Sha256);
3244 assert_eq!(parsed.pack_checksum, pack_checksum);
3245 assert_eq!(parsed.index_checksum.format(), ObjectFormat::Sha256);
3246 assert_eq!(
3247 parsed.entries[0]
3248 .bitmap
3249 .to_positions()
3250 .expect("test operation should succeed"),
3251 vec![0, 1]
3252 );
3253 }
3254
3255 #[test]
3256 fn pack_bitmap_index_write_includes_name_hash_cache() {
3257 let object_types = [ObjectType::Commit, ObjectType::Tree, ObjectType::Blob];
3258 let cache = vec![0x1111_1111u32, 0x2222_2222, 0x3333_3333];
3259 let bytes = write_bitmap(
3260 ObjectFormat::Sha1,
3261 pack_checksum_sha1(),
3262 &object_types,
3263 &[(0u32, 0u32, vec![1u32, 2u32])],
3264 Some(cache.clone()),
3265 )
3266 .expect("test operation should succeed");
3267 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, 3)
3268 .expect("test operation should succeed");
3269 assert_eq!(
3270 parsed.options,
3271 PackBitmapIndex::OPTION_FULL_DAG | PackBitmapIndex::OPTION_HASH_CACHE
3272 );
3273 assert_eq!(parsed.name_hash_cache, Some(cache));
3274 }
3275
3276 #[test]
3277 fn pack_bitmap_writer_supports_multiple_commits() {
3278 let object_types = [
3279 ObjectType::Commit,
3280 ObjectType::Commit,
3281 ObjectType::Tree,
3282 ObjectType::Blob,
3283 ];
3284 let mut writer =
3285 PackBitmapWriter::new(ObjectFormat::Sha1, pack_checksum_sha1(), &object_types)
3286 .expect("test operation should succeed");
3287 writer
3288 .add_commit(0, 0, &[2, 3])
3289 .expect("test operation should succeed");
3290 writer
3291 .add_commit(1, 1, &[2])
3292 .expect("test operation should succeed");
3293 let bytes = writer.write().expect("test operation should succeed");
3294 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, 4)
3295 .expect("test operation should succeed");
3296 assert_eq!(parsed.entries.len(), 2);
3297 assert_eq!(
3298 parsed
3299 .type_bitmaps
3300 .commits
3301 .to_positions()
3302 .expect("test operation should succeed"),
3303 vec![0, 1]
3304 );
3305 let first = parsed
3306 .entry_for_index_position(0)
3307 .expect("test operation should succeed");
3308 assert_eq!(
3309 first
3310 .bitmap
3311 .to_positions()
3312 .expect("test operation should succeed"),
3313 vec![0, 2, 3]
3314 );
3315 let second = parsed
3316 .entry_for_index_position(1)
3317 .expect("test operation should succeed");
3318 assert_eq!(
3319 second
3320 .bitmap
3321 .to_positions()
3322 .expect("test operation should succeed"),
3323 vec![1, 2]
3324 );
3325 }
3326
3327 #[test]
3328 fn pack_bitmap_writer_roundtrips_lookup_table() {
3329 let object_types = [ObjectType::Commit, ObjectType::Commit, ObjectType::Tree];
3330 let mut writer =
3331 PackBitmapWriter::new(ObjectFormat::Sha1, pack_checksum_sha1(), &object_types)
3332 .expect("test operation should succeed")
3333 .with_lookup_table(true);
3334 writer
3335 .add_commit(0, 1, &[2])
3336 .expect("test operation should succeed");
3337 writer
3338 .add_commit(1, 0, &[2])
3339 .expect("test operation should succeed");
3340 let bytes = writer.write().expect("test operation should succeed");
3341 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, 3)
3342 .expect("test operation should succeed");
3343 assert!(parsed.lookup_table);
3344 assert_ne!(parsed.options & PackBitmapIndex::OPTION_LOOKUP_TABLE, 0);
3345 assert_eq!(parsed.entries.len(), 2);
3346 }
3347
3348 #[test]
3349 fn pack_bitmap_index_recomputes_checksum_on_write() {
3350 let object_types = [ObjectType::Commit, ObjectType::Blob];
3353 let writer = PackBitmapWriter::new(ObjectFormat::Sha1, pack_checksum_sha1(), &object_types)
3354 .expect("test operation should succeed");
3355 let mut index = writer.build().expect("test operation should succeed");
3356 assert_eq!(index.index_checksum.as_bytes(), [0u8; 20]);
3358 index.entries.clear(); index.entries.push(PackBitmapEntry {
3360 object_position: 0,
3361 xor_offset: 0,
3362 flags: 0,
3363 bitmap: EwahBitmap::from_positions(2, &[0, 1]).expect("test operation should succeed"),
3364 });
3365 let bytes = index.write().expect("test operation should succeed");
3366 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, 2)
3368 .expect("test operation should succeed");
3369 assert_ne!(parsed.index_checksum.as_bytes(), [0u8; 20]);
3370 }
3371
3372 #[test]
3373 fn pack_bitmap_writer_rejects_non_commit_selection() {
3374 let object_types = [ObjectType::Commit, ObjectType::Blob];
3375 let mut writer =
3376 PackBitmapWriter::new(ObjectFormat::Sha1, pack_checksum_sha1(), &object_types)
3377 .expect("test operation should succeed");
3378 assert!(writer.add_commit(1, 1, &[]).is_err());
3380 assert!(writer.add_commit(5, 5, &[]).is_err());
3382 assert!(writer.add_commit(0, 5, &[]).is_err());
3384 assert!(writer.add_commit(0, 0, &[9]).is_err());
3386 }
3387
3388 #[test]
3389 fn pack_bitmap_writer_rejects_checksum_format_mismatch() {
3390 let sha256_checksum = sley_core::digest_bytes(ObjectFormat::Sha256, b"pack")
3391 .expect("test operation should succeed");
3392 assert!(
3393 PackBitmapWriter::new(ObjectFormat::Sha1, sha256_checksum, &[ObjectType::Commit])
3394 .is_err()
3395 );
3396 }
3397
3398 #[test]
3399 fn pack_bitmap_writer_rejects_bad_name_hash_cache_len() {
3400 let writer = PackBitmapWriter::new(
3401 ObjectFormat::Sha1,
3402 pack_checksum_sha1(),
3403 &[ObjectType::Commit],
3404 )
3405 .expect("test operation should succeed");
3406 assert!(writer.with_name_hash_cache(vec![1, 2]).is_err());
3407 }
3408
3409 #[test]
3410 fn pack_bitmap_index_write_rejects_inconsistent_cache_flag() {
3411 let mut index = PackBitmapWriter::new(
3412 ObjectFormat::Sha1,
3413 pack_checksum_sha1(),
3414 &[ObjectType::Commit],
3415 )
3416 .expect("test operation should succeed")
3417 .build()
3418 .expect("test operation should succeed");
3419 index.options |= PackBitmapIndex::OPTION_HASH_CACHE;
3421 assert!(index.write().is_err());
3422 index.options = PackBitmapIndex::OPTION_FULL_DAG;
3424 index.name_hash_cache = Some(vec![0]);
3425 assert!(index.write().is_err());
3426 }
3427
3428 #[test]
3429 fn write_bitmap_roundtrips_through_upstream_git_parser() {
3430 let root = unique_temp_dir("git-pack-bitmap-writer");
3434 fs::create_dir_all(&root).expect("test operation should succeed");
3435 {
3436 run_git_success(&root, &["init", "-q", "-b", "main"]);
3437 run_git_success(
3438 &root,
3439 &[
3440 "-c",
3441 "user.name=Example User",
3442 "-c",
3443 "user.email=example@example.invalid",
3444 "commit",
3445 "--allow-empty",
3446 "-q",
3447 "-m",
3448 "one",
3449 ],
3450 );
3451 run_git_success(&root, &["repack", "-adb"]);
3452 let pack_dir = root.join(".git").join("objects").join("pack");
3453 let idx_path = single_path_with_extension(&pack_dir, "idx");
3454 let index = PackIndex::parse(
3455 &fs::read(idx_path).expect("test operation should succeed"),
3456 ObjectFormat::Sha1,
3457 )
3458 .expect("test operation should succeed");
3459 let pack_path = single_path_with_extension(&pack_dir, "pack");
3461 let pack =
3462 PackFile::parse_sha1(&fs::read(pack_path).expect("test operation should succeed"))
3463 .expect("test operation should succeed");
3464 let mut offsets: Vec<u64> = index.entries.iter().map(|entry| entry.offset).collect();
3467 offsets.sort_unstable();
3468 let position_of = |offset: u64| -> u32 {
3469 offsets
3470 .iter()
3471 .position(|value| *value == offset)
3472 .expect("test operation should succeed") as u32
3473 };
3474 let mut object_types = vec![ObjectType::Blob; index.entries.len()];
3475 for entry in &index.entries {
3476 let position = position_of(entry.offset) as usize;
3477 if let Some(parsed) = pack
3479 .entries
3480 .iter()
3481 .find(|po| po.entry.offset == entry.offset)
3482 {
3483 object_types[position] = parsed.object.object_type;
3484 }
3485 }
3486 let commit_position = object_types
3488 .iter()
3489 .position(|ty| *ty == ObjectType::Commit)
3490 .expect("test operation should succeed") as u32;
3491 let commit_index_position = index
3493 .entries
3494 .iter()
3495 .position(|entry| position_of(entry.offset) == commit_position)
3496 .expect("test operation should succeed")
3497 as u32;
3498 let reachable: Vec<u32> = (0..index.entries.len() as u32).collect();
3499 let bytes = write_bitmap(
3500 ObjectFormat::Sha1,
3501 index.pack_checksum.clone(),
3502 &object_types,
3503 &[(commit_position, commit_index_position, reachable)],
3504 None,
3505 )
3506 .expect("test operation should succeed");
3507 let parsed = PackBitmapIndex::parse(&bytes, ObjectFormat::Sha1, index.entries.len())
3508 .expect("test operation should succeed");
3509 assert_eq!(parsed.pack_checksum, index.pack_checksum);
3510 assert_eq!(parsed.entries.len(), 1);
3511 assert_eq!(
3512 parsed.entries[0]
3513 .bitmap
3514 .to_positions()
3515 .expect("test operation should succeed")
3516 .len(),
3517 index.entries.len()
3518 );
3519 };
3520 let _ = fs::remove_dir_all(&root);
3521 }
3522}