1#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))]
4
5use flate2::Compression;
6use flate2::read::ZlibDecoder;
7use flate2::write::ZlibEncoder;
8use flate2::{Decompress, FlushDecompress};
9use sley_core::{GitError, MissingObjectContext, ObjectFormat, ObjectId, Result};
10use sley_formats::{Bundle, BundleReference};
11use sley_object::{
12 Commit, EncodedObject, ObjectType, Tag, TreeEntries, parse_framed_object,
13 tree_entry_object_type,
14};
15use sley_pack::{
16 MultiPackIndex, MultiPackIndexOidLookup, PackBitmapIndex, PackBitmapWriter, PackFile,
17 PackIndex, PackIndexByteSource, PackIndexEntry, PackIndexViewData, PackInput,
18 PackStreamIndexBuild, PackWrite, PackWriteOptions, PackWriteSummary,
19};
20use std::collections::{HashMap, HashSet};
21use std::io::{Read, Write};
22use std::path::{Path, PathBuf};
23use std::sync::atomic::{AtomicU64, Ordering};
24use std::sync::{Arc, Mutex, OnceLock};
25use std::{env, fs};
26
27static TEMPFILE_COUNTER: AtomicU64 = AtomicU64::new(0);
28
29pub trait ObjectReader {
30 fn read_object(&self, oid: &ObjectId) -> Result<Arc<EncodedObject>>;
31
32 fn is_shallow_graft(&self, _oid: &ObjectId) -> bool {
40 false
41 }
42
43 fn has_shallow_grafts(&self) -> bool {
47 false
48 }
49
50 fn is_promised_object(&self, _oid: &ObjectId) -> bool {
54 false
55 }
56}
57
58fn implied_empty_tree_object(format: ObjectFormat, oid: &ObjectId) -> Option<Arc<EncodedObject>> {
59 (*oid == ObjectId::empty_tree(format))
60 .then(|| Arc::new(EncodedObject::new(ObjectType::Tree, Vec::new())))
61}
62
63fn with_missing_object_context(
64 err: GitError,
65 oid: ObjectId,
66 context: MissingObjectContext,
67) -> GitError {
68 let kind = err
69 .not_found_kind()
70 .and_then(sley_core::NotFoundKind::missing_object_kind);
71 match kind {
72 Some(kind) => GitError::object_kind_not_found_in(oid, kind, context),
73 None => err,
74 }
75}
76
77pub fn grafted_parents<R: ObjectReader + ?Sized>(
81 reader: &R,
82 oid: &ObjectId,
83 parents: Vec<ObjectId>,
84) -> Vec<ObjectId> {
85 if reader.is_shallow_graft(oid) {
86 Vec::new()
87 } else {
88 parents
89 }
90}
91
92pub trait ObjectWriter {
93 fn write_object(&self, object: EncodedObject) -> Result<ObjectId>;
99}
100
101#[derive(Debug, Clone, PartialEq, Eq)]
102pub struct BundleUnbundleResult {
103 pub written_objects: Vec<ObjectId>,
104 pub references: Vec<BundleReference>,
105}
106
107#[derive(Debug, Clone, PartialEq, Eq)]
108pub struct PackUnpackResult {
109 pub written_objects: Vec<ObjectId>,
110}
111
112#[derive(Debug, Clone, PartialEq, Eq)]
113pub struct PackInstallResult {
114 pub pack_name: String,
115 pub pack_path: PathBuf,
116 pub index_path: PathBuf,
117 pub promisor_path: Option<PathBuf>,
118 pub object_ids: Vec<ObjectId>,
119}
120
121#[derive(Debug)]
122pub struct RawPackStreamingInstall {
123 format: ObjectFormat,
124 expected_pack_id: ObjectId,
125 expected_pack_size: u64,
126 options: RawPackInstallOptions,
127 pack_dir: PathBuf,
128 pack_name: String,
129 pack_path: PathBuf,
130 index_path: PathBuf,
131 temp_pack_path: PathBuf,
132 file: Option<fs::File>,
133 written: u64,
134 finished: bool,
135}
136
137#[derive(Debug, Clone, PartialEq, Eq)]
138pub struct RawPackInstallResult {
139 pub object_ids: Vec<ObjectId>,
140}
141
142#[derive(Debug, Clone, PartialEq, Eq)]
143pub struct RawPackIndexResult {
144 pub pack_id: ObjectId,
145 pub index: Vec<u8>,
146 pub objects: Vec<RawPackIndexedObject>,
147}
148
149#[derive(Debug, Clone, PartialEq, Eq)]
150pub struct RawPackIndexedObject {
151 pub oid: ObjectId,
152 pub object_type: ObjectType,
153 pub size: u64,
154 pub offset: u64,
155}
156
157struct PackInstallTeeReader<'a, R, W> {
158 reader: &'a mut R,
159 writer: &'a mut W,
160}
161
162impl<R, W> Read for PackInstallTeeReader<'_, R, W>
163where
164 R: Read,
165 W: Write,
166{
167 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
168 let len = self.reader.read(buf)?;
169 if len > 0 {
170 self.writer.write_all(&buf[..len])?;
171 }
172 Ok(len)
173 }
174}
175
176#[derive(Debug, Clone, PartialEq, Eq)]
177pub struct ReachablePackFile {
178 pub pack_path: PathBuf,
179 pub pack_size: u64,
180 pub checksum: ObjectId,
181 pub object_count: usize,
182 pub delta_count: u32,
183}
184
185#[derive(Debug, Clone, PartialEq, Eq)]
186pub struct ReachablePackWriteSummary {
187 pub index: Vec<u8>,
188 pub checksum: ObjectId,
189 pub object_count: usize,
190 pub delta_count: u32,
191 pub pack_size: u64,
192}
193
194#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
195pub struct RawPackInstallOptions {
196 pub promisor: bool,
197}
198
199pub trait RawPackInstaller {
200 fn install_raw_pack_from_reader<R>(&self, reader: &mut R) -> Result<RawPackInstallResult>
201 where
202 R: Read;
203}
204
205#[cfg(test)]
206const REACHABLE_PACK_STREAMING_MIN_OBJECTS: usize = 32;
207#[cfg(not(test))]
208const REACHABLE_PACK_STREAMING_MIN_OBJECTS: usize = 4096;
209
210#[derive(Debug, Clone, PartialEq, Eq)]
211pub enum ObjectPrefixResolution {
212 Missing,
213 Unique(ObjectId),
214 Ambiguous(Vec<ObjectId>),
215}
216
217#[derive(Debug, Clone, PartialEq, Eq)]
218pub struct ObjectStorageInfo {
219 pub disk_size: u64,
220 pub deltabase: ObjectId,
221}
222
223impl RawPackInstaller for FileObjectDatabase {
224 fn install_raw_pack_from_reader<R>(&self, reader: &mut R) -> Result<RawPackInstallResult>
225 where
226 R: Read,
227 {
228 let result = FileObjectDatabase::install_raw_pack_from_reader(self, reader)?;
229 Ok(RawPackInstallResult {
230 object_ids: result.object_ids,
231 })
232 }
233}
234
235impl RawPackInstaller for ObjectDatabase {
236 fn install_raw_pack_from_reader<R>(&self, reader: &mut R) -> Result<RawPackInstallResult>
237 where
238 R: Read,
239 {
240 let mut pack_bytes = Vec::new();
241 reader.read_to_end(&mut pack_bytes)?;
242 let result = unpack_packfile_objects(&pack_bytes, self.format, self)?;
243 Ok(RawPackInstallResult {
244 object_ids: result.written_objects,
245 })
246 }
247}
248
249impl RawPackStreamingInstall {
250 pub fn bytes_written(&self) -> u64 {
251 self.written
252 }
253
254 pub fn pack_path(&self) -> &Path {
255 &self.pack_path
256 }
257
258 pub fn index_path(&self) -> &Path {
259 &self.index_path
260 }
261
262 pub fn finish(mut self) -> Result<PackInstallResult> {
263 let result = (|| -> Result<PackInstallResult> {
264 let mut file = self.file.take().ok_or_else(|| {
265 GitError::InvalidFormat("raw pack stream already finished".into())
266 })?;
267 file.flush()?;
268 file.sync_all()?;
269 drop(file);
270
271 if self.written != self.expected_pack_size {
272 return Err(GitError::InvalidFormat(format!(
273 "raw pack stream length mismatch: expected {}, got {}",
274 self.expected_pack_size, self.written
275 )));
276 }
277
278 let built = PackIndex::write_v2_for_pack_path(&self.temp_pack_path, self.format)?;
279 if built.pack_checksum != self.expected_pack_id {
280 return Err(GitError::InvalidFormat(format!(
281 "raw pack stream checksum mismatch: expected {}, got {}",
282 self.expected_pack_id, built.pack_checksum
283 )));
284 }
285
286 match fs::rename(&self.temp_pack_path, &self.pack_path) {
287 Ok(()) => {}
288 Err(_) if self.pack_path.exists() => {
289 let _ = fs::remove_file(&self.temp_pack_path);
290 }
291 Err(err) => return Err(GitError::Io(err.to_string())),
292 }
293 write_pack_component(&self.index_path, &built.index)?;
294 let promisor_path = write_promisor_pack_sidecar(
295 &self.pack_dir,
296 &self.pack_name,
297 self.options.promisor,
298 )?;
299 Ok(PackInstallResult {
300 pack_name: self.pack_name.clone(),
301 pack_path: self.pack_path.clone(),
302 index_path: self.index_path.clone(),
303 promisor_path,
304 object_ids: built.entries.iter().map(|entry| entry.oid).collect(),
305 })
306 })();
307
308 if result.is_ok() {
309 self.finished = true;
310 } else {
311 let _ = fs::remove_file(&self.temp_pack_path);
312 }
313 result
314 }
315}
316
317impl Write for RawPackStreamingInstall {
318 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
319 let next_written = self.written.checked_add(buf.len() as u64).ok_or_else(|| {
320 std::io::Error::new(std::io::ErrorKind::InvalidData, "pack size overflow")
321 })?;
322 if next_written > self.expected_pack_size {
323 return Err(std::io::Error::new(
324 std::io::ErrorKind::InvalidData,
325 format!(
326 "raw pack stream exceeds expected size {}; got at least {}",
327 self.expected_pack_size, next_written
328 ),
329 ));
330 }
331 let file = self.file.as_mut().ok_or_else(|| {
332 std::io::Error::new(
333 std::io::ErrorKind::BrokenPipe,
334 "raw pack stream already finished",
335 )
336 })?;
337 let written = file.write(buf)?;
338 self.written = self.written.checked_add(written as u64).ok_or_else(|| {
339 std::io::Error::new(std::io::ErrorKind::InvalidData, "pack size overflow")
340 })?;
341 Ok(written)
342 }
343
344 fn flush(&mut self) -> std::io::Result<()> {
345 match self.file.as_mut() {
346 Some(file) => file.flush(),
347 None => Ok(()),
348 }
349 }
350}
351
352impl Drop for RawPackStreamingInstall {
353 fn drop(&mut self) {
354 if !self.finished {
355 let _ = self.file.take();
356 let _ = fs::remove_file(&self.temp_pack_path);
357 }
358 }
359}
360
361pub fn verify_bundle_prerequisites<R: ObjectReader>(bundle: &Bundle, reader: &R) -> Result<()> {
362 let mut missing = Vec::new();
363 for prerequisite in &bundle.prerequisites {
364 match reader.read_object(&prerequisite.oid) {
365 Ok(object) => {
366 let actual = object.object_id(bundle.format)?;
367 if actual != prerequisite.oid {
368 return Err(GitError::InvalidObject(format!(
369 "bundle prerequisite {} hashes to {actual}",
370 prerequisite.oid
371 )));
372 }
373 }
374 Err(GitError::NotFound(_)) => missing.push(prerequisite.oid),
375 Err(err) => return Err(err),
376 }
377 }
378 if missing.is_empty() {
379 return Ok(());
380 }
381 Err(GitError::object_not_found_in(
382 missing[0],
383 MissingObjectContext::PackInstall,
384 ))
385}
386
387pub fn unbundle_objects<R, W>(
388 bundle: &Bundle,
389 prerequisite_reader: &R,
390 writer: &mut W,
391) -> Result<BundleUnbundleResult>
392where
393 R: ObjectReader,
394 W: ObjectWriter,
395{
396 verify_bundle_prerequisites(bundle, prerequisite_reader)?;
397 let pack = PackFile::parse_bundle(bundle)?;
398 let written_objects = write_pack_objects(pack, writer, "bundle")?.written_objects;
399 Ok(BundleUnbundleResult {
400 written_objects,
401 references: bundle.references.clone(),
402 })
403}
404
405pub fn install_bundle_pack<R>(
406 bundle: &Bundle,
407 prerequisite_reader: &R,
408 destination: &impl RawPackInstaller,
409) -> Result<BundleUnbundleResult>
410where
411 R: ObjectReader,
412{
413 verify_bundle_prerequisites(bundle, prerequisite_reader)?;
414 let mut reader = bundle.pack.as_slice();
415 let install = destination.install_raw_pack_from_reader(&mut reader)?;
416 Ok(BundleUnbundleResult {
417 written_objects: install.object_ids,
418 references: bundle.references.clone(),
419 })
420}
421
422pub fn unpack_packfile_objects<W>(
423 pack_bytes: &[u8],
424 format: ObjectFormat,
425 writer: &W,
426) -> Result<PackUnpackResult>
427where
428 W: ObjectWriter,
429{
430 let pack = PackFile::parse(pack_bytes, format)?;
431 write_pack_objects(pack, writer, "pack")
432}
433
434pub fn index_raw_pack(pack_bytes: &[u8], format: ObjectFormat) -> Result<RawPackIndexResult> {
435 let pack = PackFile::parse(pack_bytes, format)?;
436 let built = PackIndex::write_v2_for_pack(pack_bytes, format)?;
437 if built.pack_checksum != pack.checksum {
438 return Err(GitError::InvalidFormat(
439 "pack index checksum does not match parsed pack checksum".to_string(),
440 ));
441 }
442
443 let offsets = built
444 .entries
445 .iter()
446 .map(|entry| (entry.oid, entry.offset))
447 .collect::<HashMap<_, _>>();
448 let mut objects = Vec::with_capacity(pack.entries.len());
449 for object in pack.entries {
450 let offset = offsets.get(&object.entry.oid).copied().ok_or_else(|| {
451 GitError::InvalidFormat(format!(
452 "pack index is missing object {}",
453 object.entry.oid.to_hex()
454 ))
455 })?;
456 objects.push(RawPackIndexedObject {
457 oid: object.entry.oid,
458 object_type: object.object.object_type,
459 size: object.object.body.len() as u64,
460 offset,
461 });
462 }
463
464 Ok(RawPackIndexResult {
465 pack_id: built.pack_checksum,
466 index: built.index,
467 objects,
468 })
469}
470
471pub fn index_raw_pack_from_reader<R>(
472 reader: &mut R,
473 format: ObjectFormat,
474) -> Result<RawPackIndexResult>
475where
476 R: Read,
477{
478 Ok(stream_index_build_to_raw_result(
479 PackIndex::write_v2_for_pack_reader_to_trailer(reader, format)?,
480 ))
481}
482
483pub fn index_raw_pack_from_reader_with_len<R>(
484 reader: &mut R,
485 format: ObjectFormat,
486 pack_len: u64,
487) -> Result<RawPackIndexResult>
488where
489 R: Read,
490{
491 Ok(stream_index_build_to_raw_result(
492 PackIndex::write_v2_for_pack_reader_with_len(reader, format, pack_len)?,
493 ))
494}
495
496pub fn index_raw_pack_file(
497 path: impl AsRef<Path>,
498 format: ObjectFormat,
499) -> Result<RawPackIndexResult> {
500 Ok(stream_index_build_to_raw_result(
501 PackIndex::write_v2_for_pack_path(path, format)?,
502 ))
503}
504
505fn stream_index_build_to_raw_result(built: PackStreamIndexBuild) -> RawPackIndexResult {
506 let objects = built
507 .objects
508 .into_iter()
509 .map(|object| RawPackIndexedObject {
510 oid: object.oid,
511 object_type: object.object_type,
512 size: object.size,
513 offset: object.offset,
514 })
515 .collect::<Vec<_>>();
516 RawPackIndexResult {
517 pack_id: built.pack_checksum,
518 index: built.index,
519 objects,
520 }
521}
522
523fn write_pack_objects<W>(pack: PackFile, writer: &W, source: &str) -> Result<PackUnpackResult>
524where
525 W: ObjectWriter,
526{
527 let mut written_objects = Vec::with_capacity(pack.entries.len());
528 for entry in pack.entries {
529 let expected = entry.entry.oid;
530 let actual = writer.write_object(entry.object)?;
531 if actual != expected {
532 return Err(GitError::InvalidObject(format!(
533 "{source} object id mismatch: expected {expected}, wrote {actual}"
534 )));
535 }
536 written_objects.push(actual);
537 }
538 Ok(PackUnpackResult { written_objects })
539}
540
541pub fn collect_reachable_object_ids<R, I>(
542 reader: &R,
543 format: ObjectFormat,
544 starts: I,
545) -> Result<HashSet<ObjectId>>
546where
547 R: ObjectReader,
548 I: IntoIterator<Item = ObjectId>,
549{
550 walk_reachable_objects(reader, format, starts, &HashSet::new(), |_, _| {})
551}
552
553pub fn collect_reachable_object_ids_tolerating_promised_missing<R, I>(
554 reader: &R,
555 format: ObjectFormat,
556 starts: I,
557) -> Result<HashSet<ObjectId>>
558where
559 R: ObjectReader,
560 I: IntoIterator<Item = ObjectId>,
561{
562 collect_reachable_object_ids_excluding_promised_missing(reader, format, starts, &HashSet::new())
563}
564
565pub fn collect_reachable_object_ids_tolerating_missing<R, I>(
566 reader: &R,
567 format: ObjectFormat,
568 starts: I,
569) -> Result<HashSet<ObjectId>>
570where
571 R: ObjectReader,
572 I: IntoIterator<Item = ObjectId>,
573{
574 walk_reachable_objects_tolerating_missing(reader, format, starts)
575}
576
577pub fn collect_reachable_object_ids_with_cut<R, I>(
582 reader: &R,
583 format: ObjectFormat,
584 starts: I,
585 cut: &HashSet<ObjectId>,
586) -> Result<HashSet<ObjectId>>
587where
588 R: ObjectReader,
589 I: IntoIterator<Item = ObjectId>,
590{
591 walk_reachable_objects_with_cut(reader, format, starts, &HashSet::new(), cut, |_, _| {})
592}
593
594pub fn collect_reachable_object_ids_excluding<R, I>(
598 reader: &R,
599 format: ObjectFormat,
600 starts: I,
601 excluded: &HashSet<ObjectId>,
602) -> Result<HashSet<ObjectId>>
603where
604 R: ObjectReader,
605 I: IntoIterator<Item = ObjectId>,
606{
607 walk_reachable_objects(reader, format, starts, excluded, |_, _| {})
608}
609
610fn collect_reachable_object_ids_excluding_promised_missing<R, I>(
611 reader: &R,
612 format: ObjectFormat,
613 starts: I,
614 excluded: &HashSet<ObjectId>,
615) -> Result<HashSet<ObjectId>>
616where
617 R: ObjectReader,
618 I: IntoIterator<Item = ObjectId>,
619{
620 let mut seen = HashSet::new();
621 let mut pending: Vec<ObjectId> = starts.into_iter().collect();
622 while let Some(oid) = pending.pop() {
623 if excluded.contains(&oid) || !seen.insert(oid) {
624 continue;
625 }
626 let object = match reader
627 .read_object(&oid)
628 .map_err(|err| with_missing_object_context(err, oid, MissingObjectContext::Traversal))
629 {
630 Ok(object) => object,
631 Err(GitError::NotFound(_)) if reader.is_promised_object(&oid) => continue,
632 Err(err) => return Err(err),
633 };
634 match object.object_type {
635 ObjectType::Commit => {
636 let commit = Commit::parse_ref(format, &object.body)?;
637 pending.extend(grafted_parents(reader, &oid, commit.parents));
638 pending.push(commit.tree);
639 }
640 ObjectType::Tree => {
641 for entry in TreeEntries::new(format, &object.body) {
642 let entry = entry?;
643 if !entry.is_gitlink() {
644 pending.push(entry.oid);
645 }
646 }
647 }
648 ObjectType::Tag => {
649 let tag = Tag::parse_ref(format, &object.body)?;
650 pending.push(tag.object);
651 }
652 ObjectType::Blob => {}
653 }
654 }
655 Ok(seen)
656}
657
658pub fn collect_reachable_objects<R, I>(
659 reader: &R,
660 format: ObjectFormat,
661 starts: I,
662 excluded: &HashSet<ObjectId>,
663) -> Result<Vec<Arc<EncodedObject>>>
664where
665 R: ObjectReader,
666 I: IntoIterator<Item = ObjectId>,
667{
668 let mut objects = Vec::new();
669 walk_reachable_objects(reader, format, starts, excluded, |_, object| {
670 objects.push(Arc::clone(object));
671 })?;
672 Ok(objects)
673}
674
675#[derive(Debug, Clone)]
676struct ReachablePackObject {
677 oid: ObjectId,
678 object: Arc<EncodedObject>,
679}
680
681#[derive(Debug, Clone, PartialEq, Eq)]
682struct ReachablePackObjectMeta {
683 oid: ObjectId,
684 object_type: ObjectType,
685 size: u64,
686}
687
688enum ReachablePackObjectsForWrite {
689 Buffered(Vec<ReachablePackObject>),
690 Streaming(Vec<ReachablePackObjectMeta>),
691}
692
693fn collect_reachable_pack_objects<R, I>(
694 reader: &R,
695 format: ObjectFormat,
696 starts: I,
697 excluded: &HashSet<ObjectId>,
698) -> Result<Vec<ReachablePackObject>>
699where
700 R: ObjectReader,
701 I: IntoIterator<Item = ObjectId>,
702{
703 let mut objects = Vec::new();
704 walk_reachable_objects(reader, format, starts, excluded, |oid, object| {
705 objects.push(ReachablePackObject {
706 oid: *oid,
707 object: Arc::clone(object),
708 });
709 })?;
710 Ok(objects)
711}
712
713fn collect_reachable_pack_objects_for_write<R, I>(
714 reader: &R,
715 format: ObjectFormat,
716 starts: I,
717 excluded: &HashSet<ObjectId>,
718) -> Result<ReachablePackObjectsForWrite>
719where
720 R: ObjectReader,
721 I: IntoIterator<Item = ObjectId>,
722{
723 let mut buffered = Some(Vec::new());
724 let mut metadata = Vec::new();
725 walk_reachable_objects(reader, format, starts, excluded, |oid, object| {
726 metadata.push(ReachablePackObjectMeta {
727 oid: *oid,
728 object_type: object.object_type,
729 size: object.body.len() as u64,
730 });
731 let should_stream = buffered
732 .as_ref()
733 .is_some_and(|objects| objects.len() + 1 >= REACHABLE_PACK_STREAMING_MIN_OBJECTS);
734 if should_stream {
735 buffered = None;
736 }
737 if let Some(objects) = buffered.as_mut() {
738 objects.push(ReachablePackObject {
739 oid: *oid,
740 object: Arc::clone(object),
741 });
742 }
743 })?;
744
745 match buffered {
746 Some(objects) => Ok(ReachablePackObjectsForWrite::Buffered(objects)),
747 None => {
748 sort_reachable_pack_metadata(&mut metadata);
749 Ok(ReachablePackObjectsForWrite::Streaming(metadata))
750 }
751 }
752}
753
754fn sort_reachable_pack_metadata(metadata: &mut [ReachablePackObjectMeta]) {
755 metadata.sort_by(|left, right| {
756 reachable_pack_type_rank(left.object_type)
757 .cmp(&reachable_pack_type_rank(right.object_type))
758 .then_with(|| right.size.cmp(&left.size))
759 .then_with(|| left.oid.as_bytes().cmp(right.oid.as_bytes()))
760 });
761}
762
763fn reachable_pack_type_rank(object_type: ObjectType) -> u8 {
764 match object_type {
765 ObjectType::Commit => 0,
766 ObjectType::Tree => 1,
767 ObjectType::Blob => 2,
768 ObjectType::Tag => 3,
769 }
770}
771
772fn pack_inputs(objects: &[ReachablePackObject]) -> Vec<PackInput<'_>> {
773 objects
774 .iter()
775 .map(|entry| PackInput {
776 oid: &entry.oid,
777 object: &entry.object,
778 })
779 .collect()
780}
781
782pub fn install_reachable_pack<I>(
783 source: &impl ObjectReader,
784 destination: &impl RawPackInstaller,
785 format: ObjectFormat,
786 starts: I,
787) -> Result<Option<RawPackInstallResult>>
788where
789 I: IntoIterator<Item = ObjectId>,
790{
791 install_reachable_pack_excluding(source, destination, format, starts, &HashSet::new())
792}
793
794pub fn install_reachable_pack_excluding<I>(
795 source: &impl ObjectReader,
796 destination: &impl RawPackInstaller,
797 format: ObjectFormat,
798 starts: I,
799 excluded: &HashSet<ObjectId>,
800) -> Result<Option<RawPackInstallResult>>
801where
802 I: IntoIterator<Item = ObjectId>,
803{
804 let pack = match build_reachable_pack(source, format, starts, excluded)? {
805 Some(pack) => pack,
806 None => return Ok(None),
807 };
808 let mut reader = pack.pack.as_slice();
809 destination
810 .install_raw_pack_from_reader(&mut reader)
811 .map(Some)
812}
813
814pub fn build_reachable_pack<R, I>(
815 reader: &R,
816 format: ObjectFormat,
817 starts: I,
818 excluded: &HashSet<ObjectId>,
819) -> Result<Option<PackWrite>>
820where
821 R: ObjectReader,
822 I: IntoIterator<Item = ObjectId>,
823{
824 let objects = collect_reachable_pack_objects(reader, format, starts, excluded)?;
825 if objects.is_empty() {
826 return Ok(None);
827 }
828 let inputs = pack_inputs(&objects);
833 PackFile::write_packed_with_known_ids(&inputs, format).map(Some)
834}
835
836pub fn build_reachable_pack_file<R, I>(
837 reader: &R,
838 format: ObjectFormat,
839 starts: I,
840 excluded: &HashSet<ObjectId>,
841 pack_path: impl AsRef<Path>,
842) -> Result<Option<ReachablePackFile>>
843where
844 R: ObjectReader,
845 I: IntoIterator<Item = ObjectId>,
846{
847 let objects = collect_reachable_pack_objects(reader, format, starts, excluded)?;
848 if objects.is_empty() {
849 return Ok(None);
850 }
851 let inputs = pack_inputs(&objects);
852 let pack_path = pack_path.as_ref();
853 if let Some(parent) = pack_path.parent() {
854 fs::create_dir_all(parent)?;
855 }
856 let mut file = fs::OpenOptions::new()
857 .write(true)
858 .create(true)
859 .truncate(true)
860 .open(pack_path)?;
861 let summary = PackFile::write_packed_with_known_ids_to_writer(
862 &inputs,
863 format,
864 &PackWriteOptions::new(),
865 &mut file,
866 )?;
867 file.sync_all()?;
868 Ok(Some(reachable_pack_file_result(pack_path, summary)))
869}
870
871pub fn write_reachable_pack_to_writer<R, I, W>(
872 reader: &R,
873 format: ObjectFormat,
874 starts: I,
875 excluded: &HashSet<ObjectId>,
876 writer: &mut W,
877) -> Result<Option<ReachablePackWriteSummary>>
878where
879 R: ObjectReader,
880 I: IntoIterator<Item = ObjectId>,
881 W: Write,
882{
883 match collect_reachable_pack_objects_for_write(reader, format, starts, excluded)? {
884 ReachablePackObjectsForWrite::Buffered(objects) => {
885 if objects.is_empty() {
886 return Ok(None);
887 }
888 let inputs = pack_inputs(&objects);
889 let summary = PackFile::write_packed_with_known_ids_to_writer(
890 &inputs,
891 format,
892 &PackWriteOptions::new(),
893 writer,
894 )?;
895 Ok(Some(reachable_pack_write_summary(summary)))
896 }
897 ReachablePackObjectsForWrite::Streaming(metadata) => {
898 if metadata.is_empty() {
899 return Ok(None);
900 }
901 let object_ids = metadata.iter().map(|meta| meta.oid).collect::<Vec<_>>();
902 write_object_id_pack_to_writer(reader, format, &object_ids, writer).map(Some)
903 }
904 }
905}
906
907pub fn write_object_id_pack_to_writer<R, W>(
908 reader: &R,
909 format: ObjectFormat,
910 object_ids: &[ObjectId],
911 writer: &mut W,
912) -> Result<ReachablePackWriteSummary>
913where
914 R: ObjectReader,
915 W: Write,
916{
917 let summary = PackFile::write_packed_from_source_to_writer(
918 object_ids,
919 format,
920 &PackWriteOptions::new(),
921 |oid| reader.read_object(oid),
922 writer,
923 )?;
924 Ok(reachable_pack_write_summary(summary))
925}
926
927fn reachable_pack_file_result(path: &Path, summary: PackWriteSummary) -> ReachablePackFile {
928 ReachablePackFile {
929 pack_path: path.to_path_buf(),
930 pack_size: summary.pack_size,
931 checksum: summary.checksum,
932 object_count: summary.entries.len(),
933 delta_count: summary.delta_count,
934 }
935}
936
937fn reachable_pack_write_summary(summary: PackWriteSummary) -> ReachablePackWriteSummary {
938 ReachablePackWriteSummary {
939 index: summary.index,
940 checksum: summary.checksum,
941 object_count: summary.entries.len(),
942 delta_count: summary.delta_count,
943 pack_size: summary.pack_size,
944 }
945}
946
947pub fn build_and_install_reachable_pack<R, I>(
948 source: &R,
949 destination: &FileObjectDatabase,
950 format: ObjectFormat,
951 starts: I,
952 excluded: &HashSet<ObjectId>,
953 options: RawPackInstallOptions,
954) -> Result<Option<PackInstallResult>>
955where
956 R: ObjectReader,
957 I: IntoIterator<Item = ObjectId>,
958{
959 build_and_install_reachable_pack_filtered(
960 source,
961 destination,
962 format,
963 starts,
964 excluded,
965 options,
966 None,
967 None,
968 )
969}
970
971#[derive(Debug, Clone, PartialEq, Eq)]
978pub enum PackObjectFilter {
979 BlobNone,
981 BlobLimit(u64),
983 TreeDepth(u32),
985 SparsePathSet(Vec<String>),
987}
988
989#[allow(clippy::too_many_arguments)]
993pub fn build_and_install_reachable_pack_filtered<R, I>(
994 source: &R,
995 destination: &FileObjectDatabase,
996 format: ObjectFormat,
997 starts: I,
998 excluded: &HashSet<ObjectId>,
999 options: RawPackInstallOptions,
1000 filter: Option<PackObjectFilter>,
1001 unpack_limit: Option<usize>,
1002) -> Result<Option<PackInstallResult>>
1003where
1004 R: ObjectReader,
1005 I: IntoIterator<Item = ObjectId>,
1006{
1007 let starts: Vec<ObjectId> = starts.into_iter().collect();
1008 let wanted: HashSet<ObjectId> = starts.iter().copied().collect();
1009 let mut objects = collect_reachable_pack_objects(source, format, starts, excluded)?;
1010 match filter {
1011 Some(PackObjectFilter::BlobNone) => {
1012 objects.retain(|entry| {
1013 entry.object.object_type != ObjectType::Blob || wanted.contains(&entry.oid)
1014 });
1015 }
1016 Some(PackObjectFilter::BlobLimit(limit)) => {
1017 objects.retain(|entry| {
1018 entry.object.object_type != ObjectType::Blob
1019 || wanted.contains(&entry.oid)
1020 || (entry.object.body.len() as u64) < limit
1021 });
1022 }
1023 Some(PackObjectFilter::TreeDepth(depth)) => {
1024 let tree_depths = collect_tree_filter_depths(source, format, &objects)?;
1025 objects.retain(|entry| {
1026 if wanted.contains(&entry.oid) {
1027 return true;
1028 }
1029 match entry.object.object_type {
1030 ObjectType::Blob => false,
1031 ObjectType::Tree => tree_depths
1032 .get(&entry.oid)
1033 .is_some_and(|tree_depth| *tree_depth < depth),
1034 _ => true,
1035 }
1036 });
1037 }
1038 Some(PackObjectFilter::SparsePathSet(paths)) => {
1039 let allowed_blobs = collect_sparse_filter_blobs(source, format, &objects, &paths)?;
1040 objects.retain(|entry| {
1041 entry.object.object_type != ObjectType::Blob
1042 || wanted.contains(&entry.oid)
1043 || allowed_blobs.contains(&entry.oid)
1044 });
1045 }
1046 None => {}
1047 }
1048 if objects.is_empty() {
1049 return Ok(None);
1050 }
1051 if let Some(limit) = unpack_limit
1055 && objects.len() < limit
1056 {
1057 for entry in &objects {
1058 destination.loose().write_object((*entry.object).clone())?;
1059 }
1060 return Ok(None);
1061 }
1062 let inputs = pack_inputs(&objects);
1063 let pack_dir = destination.objects_dir.join("pack");
1064 fs::create_dir_all(&pack_dir)?;
1065 let temp_pack_path = unique_temp_path(&pack_dir);
1066 let result = (|| -> Result<PackInstallResult> {
1067 let mut file = fs::OpenOptions::new()
1068 .write(true)
1069 .create_new(true)
1070 .open(&temp_pack_path)?;
1071 let summary = PackFile::write_packed_with_known_ids_to_writer(
1072 &inputs,
1073 format,
1074 &PackWriteOptions::new(),
1075 &mut file,
1076 )?;
1077 file.flush()?;
1078 file.sync_all()?;
1079 drop(file);
1080 trace_packfile_path(&temp_pack_path)?;
1081 destination.install_pack_file_from_temp(
1082 &temp_pack_path,
1083 summary.checksum,
1084 &summary.index,
1085 summary.entries.iter().map(|entry| entry.oid).collect(),
1086 options,
1087 )
1088 })();
1089 if result.is_err() {
1090 let _ = fs::remove_file(&temp_pack_path);
1091 }
1092 result.map(Some)
1093}
1094
1095fn trace_packfile_path(pack_path: &Path) -> Result<()> {
1096 let Some(path) = env::var_os("GIT_TRACE_PACKFILE").filter(|value| !value.is_empty()) else {
1097 return Ok(());
1098 };
1099 fs::copy(pack_path, path)?;
1100 Ok(())
1101}
1102
1103fn collect_tree_filter_depths<R>(
1104 reader: &R,
1105 format: ObjectFormat,
1106 objects: &[ReachablePackObject],
1107) -> Result<HashMap<ObjectId, u32>>
1108where
1109 R: ObjectReader,
1110{
1111 let available: HashSet<ObjectId> = objects.iter().map(|entry| entry.oid).collect();
1112 let mut depths = HashMap::new();
1113 let mut stack = Vec::new();
1114 for entry in objects {
1115 if entry.object.object_type != ObjectType::Commit {
1116 continue;
1117 }
1118 let commit = Commit::parse(format, &entry.object.body)?;
1119 if available.contains(&commit.tree) {
1120 stack.push((commit.tree, 0u32));
1121 }
1122 }
1123 while let Some((tree_oid, depth)) = stack.pop() {
1124 if depths
1125 .get(&tree_oid)
1126 .is_some_and(|old_depth| *old_depth <= depth)
1127 {
1128 continue;
1129 }
1130 depths.insert(tree_oid, depth);
1131 let tree = reader.read_object(&tree_oid)?;
1132 if tree.object_type != ObjectType::Tree {
1133 continue;
1134 }
1135 let child_depth = depth.saturating_add(1);
1136 for entry in TreeEntries::new(format, &tree.body) {
1137 let entry = entry?;
1138 if tree_entry_object_type(entry.mode) == ObjectType::Tree
1139 && available.contains(&entry.oid)
1140 {
1141 stack.push((entry.oid, child_depth));
1142 }
1143 }
1144 }
1145 Ok(depths)
1146}
1147
1148fn collect_sparse_filter_blobs<R>(
1149 reader: &R,
1150 format: ObjectFormat,
1151 objects: &[ReachablePackObject],
1152 paths: &[String],
1153) -> Result<HashSet<ObjectId>>
1154where
1155 R: ObjectReader,
1156{
1157 let wanted_paths: HashSet<&str> = paths.iter().map(String::as_str).collect();
1158 let mut allowed = HashSet::new();
1159 let mut seen_trees = HashSet::new();
1160 for entry in objects {
1161 if entry.object.object_type != ObjectType::Commit {
1162 continue;
1163 }
1164 let commit = Commit::parse(format, &entry.object.body)?;
1165 collect_sparse_tree_blobs(
1166 reader,
1167 format,
1168 &commit.tree,
1169 "",
1170 &wanted_paths,
1171 &mut seen_trees,
1172 &mut allowed,
1173 )?;
1174 }
1175 Ok(allowed)
1176}
1177
1178fn collect_sparse_tree_blobs<R>(
1179 reader: &R,
1180 format: ObjectFormat,
1181 tree_oid: &ObjectId,
1182 prefix: &str,
1183 wanted_paths: &HashSet<&str>,
1184 seen_trees: &mut HashSet<ObjectId>,
1185 allowed: &mut HashSet<ObjectId>,
1186) -> Result<()>
1187where
1188 R: ObjectReader,
1189{
1190 if !seen_trees.insert(*tree_oid) {
1191 return Ok(());
1192 }
1193 let tree = reader.read_object(tree_oid)?;
1194 if tree.object_type != ObjectType::Tree {
1195 return Ok(());
1196 }
1197 for entry in TreeEntries::new(format, &tree.body) {
1198 let entry = entry?;
1199 let name = String::from_utf8_lossy(entry.name);
1200 let path = if prefix.is_empty() {
1201 name.into_owned()
1202 } else {
1203 format!("{prefix}/{name}")
1204 };
1205 if tree_entry_object_type(entry.mode) == ObjectType::Tree {
1206 collect_sparse_tree_blobs(
1207 reader,
1208 format,
1209 &entry.oid,
1210 &path,
1211 wanted_paths,
1212 seen_trees,
1213 allowed,
1214 )?;
1215 } else if wanted_paths.contains(path.as_str()) {
1216 allowed.insert(entry.oid);
1217 }
1218 }
1219 Ok(())
1220}
1221
1222pub fn assemble_pack_with_verbatim_reuse(
1232 format: ObjectFormat,
1233 reused_pack_bytes: &[u8],
1234 appended: &[PackInput<'_>],
1235) -> Result<(Vec<u8>, u32)> {
1236 assemble_pack_with_verbatim_reuses(format, &[reused_pack_bytes], appended)
1237}
1238
1239pub fn assemble_pack_with_verbatim_reuses(
1242 format: ObjectFormat,
1243 reused_packs: &[&[u8]],
1244 appended: &[PackInput<'_>],
1245) -> Result<(Vec<u8>, u32)> {
1246 let hash_len = format.raw_len();
1247 let mut reused_count = 0u32;
1248 let mut capacity = 12 + hash_len + 64 * appended.len();
1249 for reused_pack_bytes in reused_packs {
1250 if reused_pack_bytes.len() < 12 + hash_len {
1251 return Err(GitError::InvalidFormat("reused pack too short".into()));
1252 }
1253 if &reused_pack_bytes[..4] != b"PACK" {
1254 return Err(GitError::InvalidFormat(
1255 "reused pack has no signature".into(),
1256 ));
1257 }
1258 let version = u32::from_be_bytes([
1259 reused_pack_bytes[4],
1260 reused_pack_bytes[5],
1261 reused_pack_bytes[6],
1262 reused_pack_bytes[7],
1263 ]);
1264 if version != 2 {
1265 return Err(GitError::Unsupported(format!(
1266 "reused pack version {version}"
1267 )));
1268 }
1269 let count = u32::from_be_bytes([
1270 reused_pack_bytes[8],
1271 reused_pack_bytes[9],
1272 reused_pack_bytes[10],
1273 reused_pack_bytes[11],
1274 ]);
1275 reused_count = reused_count
1276 .checked_add(count)
1277 .ok_or_else(|| GitError::InvalidFormat("too many pack objects".into()))?;
1278 capacity = capacity.saturating_add(reused_pack_bytes.len().saturating_sub(12 + hash_len));
1279 }
1280 let total = reused_count
1281 .checked_add(appended.len() as u32)
1282 .ok_or_else(|| GitError::InvalidFormat("too many pack objects".into()))?;
1283
1284 let mut out = Vec::with_capacity(capacity);
1285 out.extend_from_slice(b"PACK");
1286 out.extend_from_slice(&2u32.to_be_bytes());
1287 out.extend_from_slice(&total.to_be_bytes());
1288 for reused_pack_bytes in reused_packs {
1289 out.extend_from_slice(&reused_pack_bytes[12..reused_pack_bytes.len() - hash_len]);
1290 }
1291 for input in appended {
1292 write_undeltified_pack_entry(&mut out, input.object)?;
1293 }
1294 let checksum = sley_core::digest_bytes(format, &out)?;
1295 out.extend_from_slice(checksum.as_bytes());
1296 Ok((out, reused_count))
1297}
1298
1299pub fn assemble_pack_with_verbatim_entries(
1302 format: ObjectFormat,
1303 reused_entries: &[&[u8]],
1304 appended: &[PackInput<'_>],
1305) -> Result<(Vec<u8>, u32)> {
1306 let reused_count = u32::try_from(reused_entries.len())
1307 .map_err(|_| GitError::InvalidFormat("too many pack objects".into()))?;
1308 let total = reused_count
1309 .checked_add(appended.len() as u32)
1310 .ok_or_else(|| GitError::InvalidFormat("too many pack objects".into()))?;
1311
1312 let mut capacity = 12 + format.raw_len() + 64 * appended.len();
1313 for entry in reused_entries {
1314 capacity = capacity.saturating_add(entry.len());
1315 }
1316 let mut out = Vec::with_capacity(capacity);
1317 out.extend_from_slice(b"PACK");
1318 out.extend_from_slice(&2u32.to_be_bytes());
1319 out.extend_from_slice(&total.to_be_bytes());
1320 for entry in reused_entries {
1321 out.extend_from_slice(entry);
1322 }
1323 for input in appended {
1324 write_undeltified_pack_entry(&mut out, input.object)?;
1325 }
1326 let checksum = sley_core::digest_bytes(format, &out)?;
1327 out.extend_from_slice(checksum.as_bytes());
1328 Ok((out, reused_count))
1329}
1330
1331fn write_undeltified_pack_entry(out: &mut Vec<u8>, object: &EncodedObject) -> Result<()> {
1333 let type_bits: u8 = match object.object_type {
1334 ObjectType::Commit => 1,
1335 ObjectType::Tree => 2,
1336 ObjectType::Blob => 3,
1337 ObjectType::Tag => 4,
1338 };
1339 let mut size = object.body.len() as u64;
1340 let mut byte = (type_bits << 4) | (size & 0x0f) as u8;
1341 size >>= 4;
1342 while size > 0 {
1343 out.push(byte | 0x80);
1344 byte = (size & 0x7f) as u8;
1345 size >>= 7;
1346 }
1347 out.push(byte);
1348 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
1349 encoder.write_all(&object.body)?;
1350 out.extend_from_slice(&encoder.finish()?);
1351 Ok(())
1352}
1353
1354#[derive(Debug, Clone, PartialEq, Eq)]
1362pub struct RepackResult {
1363 pub pack: Vec<u8>,
1365 pub idx: Vec<u8>,
1367 pub object_count: usize,
1369 pub obsolete_packs: Vec<PathBuf>,
1372 pub packed_loose: Vec<ObjectId>,
1375 retained_pack_stems: Vec<String>,
1378 promisor: bool,
1380 pack_checksum: ObjectId,
1381 index_entries: Vec<PackIndexEntry>,
1382}
1383
1384#[derive(Debug, Clone, Default)]
1385pub struct RepackOptions {
1386 pub local: bool,
1388 pub pack_kept_objects: bool,
1390 pub keep_pack_stems: HashSet<String>,
1392}
1393
1394pub fn repack_reachable_objects(
1414 git_dir: &Path,
1415 format: ObjectFormat,
1416 roots: &[ObjectId],
1417) -> Result<Option<RepackResult>> {
1418 repack_reachable_objects_with_options(git_dir, format, roots, &RepackOptions::default())
1419}
1420
1421pub fn repack_reachable_objects_with_options(
1422 git_dir: &Path,
1423 format: ObjectFormat,
1424 roots: &[ObjectId],
1425 options: &RepackOptions,
1426) -> Result<Option<RepackResult>> {
1427 let objects_dir = repository_objects_dir(git_dir);
1428 let database = if options.local {
1429 FileObjectDatabase::without_alternates(objects_dir.clone(), format)
1430 } else {
1431 FileObjectDatabase::new(objects_dir.clone(), format)
1432 };
1433 let retained_pack_stems = repack_retained_pack_stems(
1434 &objects_dir.join("pack"),
1435 &options.keep_pack_stems,
1436 !options.pack_kept_objects,
1437 )?;
1438 let excluded_oids = if options.pack_kept_objects {
1439 HashSet::new()
1440 } else {
1441 pack_oids_for_stems(&objects_dir.join("pack"), format, &retained_pack_stems)?
1442 };
1443 let promisor_oids = promisor_pack_object_ids(&objects_dir, format)?;
1444
1445 let mut seen: HashSet<ObjectId> = HashSet::new();
1446 let mut objects: Vec<ReachablePackObject> = Vec::new();
1447 let mut pending: Vec<ObjectId> = roots.to_vec();
1448 while let Some(oid) = pending.pop() {
1449 if !seen.insert(oid) {
1450 continue;
1451 }
1452 if promisor_oids.contains(&oid) {
1453 continue;
1454 }
1455 let object = match database.read_object(&oid) {
1456 Ok(object) => object,
1457 Err(GitError::NotFound(_)) => continue,
1458 Err(err) => return Err(err),
1459 };
1460 match object.object_type {
1461 ObjectType::Commit => {
1462 let commit = Commit::parse_ref(format, &object.body)?;
1463 pending.extend(grafted_parents(&database, &oid, commit.parents));
1464 pending.push(commit.tree);
1465 }
1466 ObjectType::Tree => {
1467 for entry in TreeEntries::new(format, &object.body) {
1468 let entry = entry?;
1469 if !entry.is_gitlink() {
1470 pending.push(entry.oid);
1471 }
1472 }
1473 }
1474 ObjectType::Tag => {
1475 let tag = Tag::parse_ref(format, &object.body)?;
1476 pending.push(tag.object);
1477 }
1478 ObjectType::Blob => {}
1479 }
1480 if !excluded_oids.contains(&oid) {
1481 objects.push(ReachablePackObject { oid, object });
1482 }
1483 }
1484
1485 if !options.local {
1491 for (alternate, oid) in alternate_packed_object_ids(&objects_dir, format)? {
1492 if excluded_oids.contains(&oid) || !seen.insert(oid) {
1493 continue;
1494 }
1495 let alternate_db = FileObjectDatabase::without_alternates(alternate, format);
1496 match alternate_db.read_object(&oid) {
1497 Ok(object) => objects.push(ReachablePackObject { oid, object }),
1498 Err(GitError::NotFound(_)) => {}
1499 Err(err) => return Err(err),
1500 }
1501 }
1502 }
1503
1504 if objects.is_empty() {
1505 return Ok(None);
1506 }
1507
1508 let inputs = pack_inputs(&objects);
1509 let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
1510 let object_count = written.entries.len();
1511
1512 let new_pack_file_name = format!("pack-{}.pack", written.checksum.to_hex());
1515 let obsolete_packs = existing_pack_files(&objects_dir.join("pack"))?
1516 .into_iter()
1517 .filter(|path| path.file_name().and_then(|name| name.to_str()) != Some(&new_pack_file_name))
1518 .collect();
1519
1520 let packed_oid_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
1521 let mut packed_loose: Vec<ObjectId> = loose_object_ids(&objects_dir, format)?
1522 .into_iter()
1523 .filter(|oid| packed_oid_set.contains(oid))
1524 .collect();
1525 packed_loose.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
1526
1527 let pack_checksum = written.checksum;
1528 let index_entries = written.entries.clone();
1529 Ok(Some(RepackResult {
1530 pack: written.pack,
1531 idx: written.index,
1532 object_count,
1533 obsolete_packs,
1534 packed_loose,
1535 retained_pack_stems,
1536 promisor: false,
1537 pack_checksum,
1538 index_entries,
1539 }))
1540}
1541
1542fn repack_retained_pack_stems(
1543 pack_dir: &Path,
1544 explicit: &HashSet<String>,
1545 keep_dot_keep: bool,
1546) -> Result<Vec<String>> {
1547 let mut stems = explicit.clone();
1548 if keep_dot_keep {
1549 for pack_path in existing_pack_files(pack_dir)? {
1550 if pack_path.with_extension("keep").exists()
1551 && let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str())
1552 {
1553 stems.insert(stem.to_string());
1554 }
1555 }
1556 }
1557 let mut stems = stems.into_iter().collect::<Vec<_>>();
1558 stems.sort();
1559 Ok(stems)
1560}
1561
1562fn pack_oids_for_stems(
1563 pack_dir: &Path,
1564 format: ObjectFormat,
1565 stems: &[String],
1566) -> Result<HashSet<ObjectId>> {
1567 let wanted: HashSet<&str> = stems.iter().map(String::as_str).collect();
1568 if wanted.is_empty() {
1569 return Ok(HashSet::new());
1570 }
1571 let mut oids = HashSet::new();
1572 for pack_path in existing_pack_files(pack_dir)? {
1573 let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str()) else {
1574 continue;
1575 };
1576 if !wanted.contains(stem) {
1577 continue;
1578 }
1579 let index_path = pack_path.with_extension("idx");
1580 if !index_path.exists() {
1581 continue;
1582 }
1583 let index = PackIndex::parse(&fs::read(index_path)?, format)?;
1584 oids.extend(index.entries.into_iter().map(|entry| entry.oid));
1585 }
1586 Ok(oids)
1587}
1588
1589fn alternate_packed_object_ids(
1590 objects_dir: &Path,
1591 format: ObjectFormat,
1592) -> Result<Vec<(PathBuf, ObjectId)>> {
1593 let mut oids = Vec::new();
1594 for alternate in alternate_object_dirs(objects_dir) {
1595 let mut alternate_oids = HashSet::new();
1596 collect_packed_object_ids(&alternate.join("pack"), format, &mut alternate_oids)?;
1597 oids.extend(
1598 alternate_oids
1599 .into_iter()
1600 .map(|oid| (alternate.clone(), oid)),
1601 );
1602 }
1603 oids.sort_by(|left, right| {
1604 left.0
1605 .cmp(&right.0)
1606 .then(left.1.as_bytes().cmp(right.1.as_bytes()))
1607 });
1608 Ok(oids)
1609}
1610
1611pub fn repack_all_objects(git_dir: &Path, format: ObjectFormat) -> Result<Option<RepackResult>> {
1612 let objects_dir = repository_objects_dir(git_dir);
1613 let database = FileObjectDatabase::new(objects_dir.clone(), format);
1614
1615 let all_oids = object_ids_in_objects_dir(&objects_dir, format)?;
1619 if all_oids.is_empty() {
1620 return Ok(None);
1621 }
1622
1623 let mut objects = Vec::with_capacity(all_oids.len());
1627 for oid in &all_oids {
1628 objects.push(ReachablePackObject {
1629 oid: *oid,
1630 object: database.read_object(oid)?,
1631 });
1632 }
1633
1634 let inputs = pack_inputs(&objects);
1635 let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
1636 let object_count = written.entries.len();
1637
1638 let new_pack_file_name = format!("pack-{}.pack", written.checksum.to_hex());
1644 let obsolete_packs = existing_pack_files(&objects_dir.join("pack"))?
1645 .into_iter()
1646 .filter(|path| path.file_name().and_then(|name| name.to_str()) != Some(&new_pack_file_name))
1647 .collect();
1648
1649 let packed_oid_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
1652 let mut packed_loose: Vec<ObjectId> = loose_object_ids(&objects_dir, format)?
1653 .into_iter()
1654 .filter(|oid| packed_oid_set.contains(oid))
1655 .collect();
1656 packed_loose.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
1657
1658 Ok(Some(RepackResult {
1659 pack: written.pack,
1660 idx: written.index,
1661 object_count,
1662 obsolete_packs,
1663 packed_loose,
1664 retained_pack_stems: Vec::new(),
1665 promisor: false,
1666 pack_checksum: written.checksum,
1667 index_entries: written.entries,
1668 }))
1669}
1670
1671pub fn repack_promisor_objects(
1676 git_dir: &Path,
1677 format: ObjectFormat,
1678) -> Result<Option<RepackResult>> {
1679 let objects_dir = repository_objects_dir(git_dir);
1680 let pack_dir = objects_dir.join("pack");
1681 let promisor_packs = existing_pack_files(&pack_dir)?
1682 .into_iter()
1683 .filter(|path| path.with_extension("promisor").exists())
1684 .collect::<Vec<_>>();
1685 if promisor_packs.len() <= 1 {
1686 return Ok(None);
1687 }
1688
1689 let database = FileObjectDatabase::new(objects_dir.clone(), format);
1690 let mut seen = HashSet::new();
1691 let mut objects = Vec::new();
1692 for pack_path in &promisor_packs {
1693 let index_path = pack_path.with_extension("idx");
1694 if !index_path.exists() {
1695 continue;
1696 }
1697 let index = PackIndex::parse(&fs::read(index_path)?, format)?;
1698 for entry in index.entries {
1699 if !seen.insert(entry.oid) {
1700 continue;
1701 }
1702 objects.push(ReachablePackObject {
1703 oid: entry.oid,
1704 object: database.read_object(&entry.oid)?,
1705 });
1706 }
1707 }
1708 if objects.is_empty() {
1709 return Ok(None);
1710 }
1711 objects.sort_by(|left, right| left.oid.as_bytes().cmp(right.oid.as_bytes()));
1712
1713 let inputs = pack_inputs(&objects);
1714 let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
1715 let object_count = written.entries.len();
1716 let packed_oid_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
1717 let mut packed_loose: Vec<ObjectId> = loose_object_ids(&objects_dir, format)?
1718 .into_iter()
1719 .filter(|oid| packed_oid_set.contains(oid))
1720 .collect();
1721 packed_loose.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
1722
1723 let pack_checksum = written.checksum;
1724 let index_entries = written.entries.clone();
1725 Ok(Some(RepackResult {
1726 pack: written.pack,
1727 idx: written.index,
1728 object_count,
1729 obsolete_packs: promisor_packs,
1730 packed_loose,
1731 retained_pack_stems: Vec::new(),
1732 promisor: true,
1733 pack_checksum,
1734 index_entries,
1735 }))
1736}
1737
1738pub fn repack_loose_objects(git_dir: &Path, format: ObjectFormat) -> Result<Option<RepackResult>> {
1744 let objects_dir = repository_objects_dir(git_dir);
1745 let database = FileObjectDatabase::new(objects_dir.clone(), format);
1746 let loose_oids = loose_object_ids(&objects_dir, format)?;
1747 if loose_oids.is_empty() {
1748 return Ok(None);
1749 }
1750
1751 let mut objects = Vec::with_capacity(loose_oids.len());
1752 for oid in &loose_oids {
1753 objects.push(ReachablePackObject {
1754 oid: *oid,
1755 object: database.read_object(oid)?,
1756 });
1757 }
1758
1759 let inputs = pack_inputs(&objects);
1760 let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
1761 let object_count = written.entries.len();
1762 let packed_oid_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
1763 let mut packed_loose: Vec<ObjectId> = loose_oids
1764 .into_iter()
1765 .filter(|oid| packed_oid_set.contains(oid))
1766 .collect();
1767 packed_loose.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
1768
1769 let pack_checksum = written.checksum;
1770 let index_entries = written.entries.clone();
1771 Ok(Some(RepackResult {
1772 pack: written.pack,
1773 idx: written.index,
1774 object_count,
1775 obsolete_packs: Vec::new(),
1776 packed_loose,
1777 retained_pack_stems: Vec::new(),
1778 promisor: false,
1779 pack_checksum,
1780 index_entries,
1781 }))
1782}
1783
1784#[derive(Debug, Clone)]
1787struct GeometryPack {
1788 pack_path: PathBuf,
1790 oids: Vec<ObjectId>,
1792 weight: u64,
1794 is_promisor: bool,
1796}
1797
1798#[derive(Debug, Clone)]
1801pub struct GeometricRepackResult {
1802 pub result: Option<RepackResult>,
1804 pub rolled_up_packs: Vec<PathBuf>,
1806}
1807
1808#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1809pub struct GeometricRepackPlan {
1810 pub split: usize,
1811 pub pack_count: usize,
1812}
1813
1814fn collect_geometry_packs(
1817 objects_dir: &Path,
1818 format: ObjectFormat,
1819 kept_pack_stems: &HashSet<String>,
1820) -> Result<Vec<GeometryPack>> {
1821 let pack_dir = objects_dir.join("pack");
1822 let mut packs = Vec::new();
1823 for pack_path in existing_pack_files(&pack_dir)? {
1824 if pack_path.with_extension("mtimes").exists() {
1827 continue;
1828 }
1829 if pack_path.with_extension("keep").exists() {
1830 continue;
1831 }
1832 let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str()) else {
1833 continue;
1834 };
1835 if kept_pack_stems.contains(stem) {
1836 continue;
1837 }
1838 let index_path = pack_path.with_extension("idx");
1839 if !index_path.exists() {
1840 continue;
1841 }
1842 let index = PackIndex::parse(&fs::read(&index_path)?, format)?;
1843 let oids: Vec<ObjectId> = index.entries.iter().map(|entry| entry.oid).collect();
1844 let weight = oids.len() as u64;
1845 packs.push(GeometryPack {
1846 is_promisor: pack_path.with_extension("promisor").exists(),
1847 pack_path,
1848 oids,
1849 weight,
1850 });
1851 }
1852 packs.sort_by(|a, b| a.weight.cmp(&b.weight).then(a.pack_path.cmp(&b.pack_path)));
1854 Ok(packs)
1855}
1856
1857fn compute_geometry_split(packs: &[GeometryPack], split_factor: u64) -> usize {
1861 let pack_nr = packs.len();
1862 if pack_nr == 0 {
1863 return 0;
1864 }
1865 let mut i = pack_nr - 1;
1867 while i > 0 {
1868 let ours = packs[i].weight;
1869 let prev = packs[i - 1].weight;
1870 if ours < split_factor.saturating_mul(prev) {
1871 break;
1872 }
1873 i -= 1;
1874 }
1875 let mut split = i;
1876 if split != 0 {
1877 split += 1;
1879 }
1880
1881 let mut total_size: u64 = packs[..split].iter().map(|p| p.weight).sum();
1885 for pack in &packs[split..] {
1886 if pack.weight < split_factor.saturating_mul(total_size) {
1887 split += 1;
1888 total_size = total_size.saturating_add(pack.weight);
1889 } else {
1890 break;
1891 }
1892 }
1893 split
1894}
1895
1896pub fn geometric_repack_plan(
1897 git_dir: &Path,
1898 format: ObjectFormat,
1899 split_factor: u64,
1900 kept_pack_stems: &HashSet<String>,
1901) -> Result<GeometricRepackPlan> {
1902 let objects_dir = repository_objects_dir(git_dir);
1903 let packs: Vec<GeometryPack> = collect_geometry_packs(&objects_dir, format, kept_pack_stems)?
1904 .into_iter()
1905 .filter(|pack| !pack.is_promisor)
1906 .collect();
1907 Ok(GeometricRepackPlan {
1908 split: compute_geometry_split(&packs, split_factor),
1909 pack_count: packs.len(),
1910 })
1911}
1912
1913pub fn repack_geometric(
1923 git_dir: &Path,
1924 format: ObjectFormat,
1925 split_factor: u64,
1926 kept_pack_stems: &HashSet<String>,
1927) -> Result<GeometricRepackResult> {
1928 let objects_dir = repository_objects_dir(git_dir);
1929 let database = FileObjectDatabase::new(objects_dir.clone(), format);
1930
1931 let all_packs = collect_geometry_packs(&objects_dir, format, kept_pack_stems)?;
1935 let packs: Vec<GeometryPack> = all_packs
1936 .into_iter()
1937 .filter(|pack| !pack.is_promisor)
1938 .collect();
1939
1940 let split = compute_geometry_split(&packs, split_factor);
1941
1942 let loose_oids = loose_object_ids(&objects_dir, format)?;
1943
1944 let mut excluded_oids: HashSet<ObjectId> = HashSet::new();
1949 for pack in &packs[split..] {
1950 excluded_oids.extend(pack.oids.iter().copied());
1951 }
1952
1953 let mut included: Vec<ObjectId> = Vec::new();
1954 let mut seen: HashSet<ObjectId> = HashSet::new();
1955 for pack in &packs[..split] {
1956 for oid in &pack.oids {
1957 if excluded_oids.contains(oid) {
1958 continue;
1959 }
1960 if seen.insert(*oid) {
1961 included.push(*oid);
1962 }
1963 }
1964 }
1965 for oid in &loose_oids {
1966 if excluded_oids.contains(oid) {
1967 continue;
1968 }
1969 if seen.insert(*oid) {
1970 included.push(*oid);
1971 }
1972 }
1973
1974 if included.is_empty() {
1976 return Ok(GeometricRepackResult {
1977 result: None,
1978 rolled_up_packs: Vec::new(),
1979 });
1980 }
1981
1982 included.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
1983 let mut objects = Vec::with_capacity(included.len());
1984 for oid in &included {
1985 objects.push(ReachablePackObject {
1986 oid: *oid,
1987 object: database.read_object(oid)?,
1988 });
1989 }
1990
1991 let inputs = pack_inputs(&objects);
1992 let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
1993 let object_count = written.entries.len();
1994
1995 let packed_oid_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
1996 let mut packed_loose: Vec<ObjectId> = loose_oids
1997 .into_iter()
1998 .filter(|oid| packed_oid_set.contains(oid))
1999 .collect();
2000 packed_loose.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
2001
2002 let rolled_up_packs: Vec<PathBuf> = packs[..split]
2003 .iter()
2004 .map(|pack| pack.pack_path.clone())
2005 .collect();
2006
2007 let pack_checksum = written.checksum;
2008 let index_entries = written.entries.clone();
2009 Ok(GeometricRepackResult {
2010 result: Some(RepackResult {
2011 pack: written.pack,
2012 idx: written.index,
2013 object_count,
2014 obsolete_packs: rolled_up_packs.clone(),
2015 packed_loose,
2016 retained_pack_stems: Vec::new(),
2017 promisor: false,
2018 pack_checksum,
2019 index_entries,
2020 }),
2021 rolled_up_packs,
2022 })
2023}
2024
2025pub fn install_repack_result(
2040 git_dir: &Path,
2041 format: ObjectFormat,
2042 result: &RepackResult,
2043 prune: bool,
2044) -> Result<()> {
2045 install_repack_result_with_bitmap(git_dir, format, result, prune, None, None)
2046}
2047
2048pub fn install_repack_result_with_bitmap(
2054 git_dir: &Path,
2055 format: ObjectFormat,
2056 result: &RepackResult,
2057 prune: bool,
2058 bitmap_tips: Option<&HashSet<ObjectId>>,
2059 bitmap_pseudo_merge_groups: Option<&[BitmapPseudoMergeGroup]>,
2060) -> Result<()> {
2061 let objects_dir = repository_objects_dir(git_dir);
2062 let pack_dir = objects_dir.join("pack");
2063 fs::create_dir_all(&pack_dir)?;
2064
2065 validate_pack_checksum(&result.pack, format, &result.pack_checksum, "repack")?;
2070 let parsed_index = PackIndex::parse(&result.idx, format)?;
2071 if parsed_index.pack_checksum != result.pack_checksum {
2072 return Err(GitError::InvalidFormat(
2073 "repack index checksum does not match the new pack".into(),
2074 ));
2075 }
2076 if !pack_index_entries_match_writer(&parsed_index.entries, &result.index_entries) {
2077 return Err(GitError::InvalidFormat(
2078 "repack index does not match the new pack contents".into(),
2079 ));
2080 }
2081 let pack_name = format!("pack-{}", result.pack_checksum.to_hex());
2082 let new_pack_path = pack_dir.join(format!("{pack_name}.pack"));
2083 let new_rev_path = pack_dir.join(format!("{pack_name}.rev"));
2084 let new_index_path = pack_dir.join(format!("{pack_name}.idx"));
2085 let reverse_index = sley_pack::PackReverseIndex::write(
2089 format,
2090 &sley_pack::pack_order_index_positions(&parsed_index.entries),
2091 &result.pack_checksum,
2092 )?;
2093 write_pack_component(&new_pack_path, &result.pack)?;
2094 write_pack_component(&new_rev_path, &reverse_index)?;
2095 write_pack_component(&new_index_path, &result.idx)?;
2096 let new_promisor_path = write_promisor_pack_sidecar(&pack_dir, &pack_name, result.promisor)?;
2097
2098 if let Some(tips) = bitmap_tips {
2099 let database = FileObjectDatabase::new(objects_dir.clone(), format);
2102 if let Some(bitmap) = build_pack_bitmap(
2103 &database,
2104 format,
2105 &result.index_entries,
2106 &result.pack_checksum,
2107 tips,
2108 bitmap_pseudo_merge_groups.unwrap_or(&[]),
2109 )? {
2110 let bitmap_path = pack_dir.join(format!("{pack_name}.bitmap"));
2115 remove_file_if_exists(&bitmap_path)?;
2116 write_pack_component(&bitmap_path, &bitmap)?;
2117 }
2118 }
2119
2120 if !prune {
2121 return Ok(());
2122 }
2123
2124 let present: HashSet<ObjectId> = parsed_index.entries.iter().map(|entry| entry.oid).collect();
2128
2129 prune_obsolete_pack_paths(
2130 &objects_dir,
2131 format,
2132 &result.obsolete_packs,
2133 &new_pack_path,
2134 &result.retained_pack_stems,
2135 result.promisor,
2136 )?;
2137 prune_loose_objects(&objects_dir, format, result.packed_loose.iter(), &present)?;
2138 if result.promisor && new_promisor_path.is_none() {
2139 return Err(GitError::InvalidFormat(
2140 "promisor repack did not write sidecar".into(),
2141 ));
2142 }
2143 Ok(())
2144}
2145
2146pub fn install_geometric_repack_result(
2152 git_dir: &Path,
2153 format: ObjectFormat,
2154 geometric: &GeometricRepackResult,
2155 prune: bool,
2156 bitmap_tips: Option<&HashSet<ObjectId>>,
2157) -> Result<()> {
2158 let Some(result) = geometric.result.as_ref() else {
2159 return Ok(());
2160 };
2161 let objects_dir = repository_objects_dir(git_dir);
2162 let pack_dir = objects_dir.join("pack");
2163 fs::create_dir_all(&pack_dir)?;
2164
2165 validate_pack_checksum(&result.pack, format, &result.pack_checksum, "repack")?;
2166 let parsed_index = PackIndex::parse(&result.idx, format)?;
2167 if parsed_index.pack_checksum != result.pack_checksum {
2168 return Err(GitError::InvalidFormat(
2169 "repack index checksum does not match the new pack".into(),
2170 ));
2171 }
2172 if !pack_index_entries_match_writer(&parsed_index.entries, &result.index_entries) {
2173 return Err(GitError::InvalidFormat(
2174 "repack index does not match the new pack contents".into(),
2175 ));
2176 }
2177 let pack_name = format!("pack-{}", result.pack_checksum.to_hex());
2178 let new_pack_path = pack_dir.join(format!("{pack_name}.pack"));
2179 let new_rev_path = pack_dir.join(format!("{pack_name}.rev"));
2180 let new_index_path = pack_dir.join(format!("{pack_name}.idx"));
2181 let reverse_index = sley_pack::PackReverseIndex::write(
2182 format,
2183 &sley_pack::pack_order_index_positions(&parsed_index.entries),
2184 &result.pack_checksum,
2185 )?;
2186 write_pack_component(&new_pack_path, &result.pack)?;
2187 write_pack_component(&new_rev_path, &reverse_index)?;
2188 write_pack_component(&new_index_path, &result.idx)?;
2189
2190 if let Some(tips) = bitmap_tips {
2191 let database = FileObjectDatabase::new(objects_dir.clone(), format);
2192 if let Some(bitmap) = build_pack_bitmap(
2193 &database,
2194 format,
2195 &result.index_entries,
2196 &result.pack_checksum,
2197 tips,
2198 &[],
2199 )? {
2200 let bitmap_path = pack_dir.join(format!("{pack_name}.bitmap"));
2201 remove_file_if_exists(&bitmap_path)?;
2202 write_pack_component(&bitmap_path, &bitmap)?;
2203 }
2204 }
2205
2206 if !prune {
2207 return Ok(());
2208 }
2209
2210 for pack_path in &geometric.rolled_up_packs {
2213 if *pack_path == new_pack_path {
2214 continue;
2215 }
2216 if pack_path.with_extension("keep").exists() {
2217 continue;
2218 }
2219 remove_file_if_exists(pack_path)?;
2220 remove_file_if_exists(&pack_path.with_extension("idx"))?;
2221 for ext in ["rev", "mtimes", "bitmap", "promisor"] {
2222 remove_file_if_exists(&pack_path.with_extension(ext))?;
2223 }
2224 }
2225
2226 let present: HashSet<ObjectId> = parsed_index.entries.iter().map(|entry| entry.oid).collect();
2228 prune_loose_objects(&objects_dir, format, result.packed_loose.iter(), &present)?;
2229
2230 let removed_stems: HashSet<String> = geometric
2232 .rolled_up_packs
2233 .iter()
2234 .filter_map(|p| p.file_stem().map(|s| s.to_string_lossy().into_owned()))
2235 .collect();
2236 prune_stale_multi_pack_index(&pack_dir, format, &removed_stems)?;
2237 Ok(())
2238}
2239
2240fn validate_pack_checksum(
2241 pack: &[u8],
2242 format: ObjectFormat,
2243 expected: &ObjectId,
2244 context: &str,
2245) -> Result<()> {
2246 if expected.format() != format {
2247 return Err(GitError::InvalidObjectId(format!(
2248 "{context} checksum format does not match object format"
2249 )));
2250 }
2251 let hash_len = format.raw_len();
2252 if pack.len() < 12 + hash_len {
2253 return Err(GitError::InvalidFormat(format!(
2254 "{context} pack file too short"
2255 )));
2256 }
2257 if &pack[..4] != b"PACK" {
2258 return Err(GitError::InvalidFormat(format!(
2259 "{context} pack file missing PACK signature"
2260 )));
2261 }
2262 let trailer_offset = pack.len() - hash_len;
2263 let actual = sley_core::digest_bytes(format, &pack[..trailer_offset])?;
2264 let trailer = ObjectId::from_raw(format, &pack[trailer_offset..])?;
2265 if &actual != expected || trailer != *expected {
2266 return Err(GitError::InvalidFormat(format!(
2267 "{context} pack checksum does not match generated pack"
2268 )));
2269 }
2270 Ok(())
2271}
2272
2273fn path_mtime_secs(path: &Path) -> u32 {
2275 fs::metadata(path)
2276 .and_then(|metadata| metadata.modified())
2277 .ok()
2278 .and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok())
2279 .map(|dur| dur.as_secs() as u32)
2280 .unwrap_or(0)
2281}
2282
2283#[derive(Debug, Clone)]
2286pub struct CruftPack {
2287 pub pack: Vec<u8>,
2288 pub idx: Vec<u8>,
2289 pub rev: Vec<u8>,
2290 pub mtimes: Vec<u8>,
2291 pub checksum: ObjectId,
2292 pub oids: Vec<ObjectId>,
2294}
2295
2296#[derive(Debug, Clone)]
2299pub struct CruftRepackResult {
2300 pub reachable: Option<RepackResult>,
2302 pub cruft: Option<CruftPack>,
2304 pub obsolete_packs: Vec<PathBuf>,
2307 pub obsolete_cruft_packs: Vec<PathBuf>,
2310 retained_pack_stems: Vec<String>,
2311}
2312
2313pub fn object_mtimes_on_disk_pub(
2317 objects_dir: &Path,
2318 format: ObjectFormat,
2319) -> Result<HashMap<ObjectId, u32>> {
2320 object_mtimes_on_disk(objects_dir, format)
2321}
2322
2323fn object_mtimes_on_disk(
2324 objects_dir: &Path,
2325 format: ObjectFormat,
2326) -> Result<HashMap<ObjectId, u32>> {
2327 let mut mtimes: HashMap<ObjectId, u32> = HashMap::new();
2328 let mut record = |oid: ObjectId, mtime: u32| {
2329 mtimes
2330 .entry(oid)
2331 .and_modify(|existing| {
2332 if mtime > *existing {
2333 *existing = mtime;
2334 }
2335 })
2336 .or_insert(mtime);
2337 };
2338
2339 let pack_dir = objects_dir.join("pack");
2340 if let Ok(entries) = fs::read_dir(&pack_dir) {
2341 let mut idx_paths: Vec<PathBuf> = Vec::new();
2342 for entry in entries {
2343 let path = entry?.path();
2344 if path.extension().and_then(|ext| ext.to_str()) == Some("idx") {
2345 idx_paths.push(path);
2346 }
2347 }
2348 idx_paths.sort();
2349 for idx_path in idx_paths {
2350 let pack_path = idx_path.with_extension("pack");
2351 if !pack_path.exists() {
2352 continue;
2353 }
2354 let index = PackIndex::parse(&fs::read(&idx_path)?, format)?;
2355 let mtimes_path = idx_path.with_extension("mtimes");
2356 let pack_object_mtimes: Option<Vec<u32>> =
2357 fs::read(&mtimes_path).ok().and_then(|bytes| {
2358 sley_pack::PackMtimes::parse(&bytes, format, index.entries.len())
2359 .ok()
2360 .map(|parsed| parsed.mtimes)
2361 });
2362 let pack_mtime = path_mtime_secs(&pack_path);
2363 for (pos, entry) in index.entries.iter().enumerate() {
2364 let mtime = pack_object_mtimes
2365 .as_ref()
2366 .and_then(|table| table.get(pos).copied())
2367 .unwrap_or(pack_mtime);
2368 record(entry.oid, mtime);
2369 }
2370 }
2371 }
2372
2373 let store = LooseObjectStore::new(objects_dir.to_path_buf(), format);
2374 for oid in loose_object_ids(objects_dir, format)? {
2375 let path = store.object_path(&oid)?;
2376 record(oid, path_mtime_secs(&path));
2377 }
2378 Ok(mtimes)
2379}
2380
2381pub fn build_cruft_pack_pub(
2383 database: &FileObjectDatabase,
2384 format: ObjectFormat,
2385 survivors: &HashMap<ObjectId, u32>,
2386) -> Result<Option<CruftPack>> {
2387 build_cruft_pack(database, format, survivors)
2388}
2389
2390fn build_cruft_pack(
2393 database: &FileObjectDatabase,
2394 format: ObjectFormat,
2395 survivors: &HashMap<ObjectId, u32>,
2396) -> Result<Option<CruftPack>> {
2397 if survivors.is_empty() {
2398 return Ok(None);
2399 }
2400 let mut ordered: Vec<(ObjectId, u32)> = survivors.iter().map(|(o, m)| (*o, *m)).collect();
2401 ordered.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
2402
2403 let mut oids: Vec<ObjectId> = Vec::with_capacity(ordered.len());
2404 let mut objects: Vec<Arc<EncodedObject>> = Vec::with_capacity(ordered.len());
2405 let mut mtime_by_oid: HashMap<ObjectId, u32> = HashMap::with_capacity(ordered.len());
2406 for (oid, mtime) in ordered {
2407 match database.read_object(&oid) {
2408 Ok(object) => {
2409 oids.push(oid);
2410 objects.push(object);
2411 mtime_by_oid.insert(oid, mtime);
2412 }
2413 Err(GitError::NotFound(_)) => {}
2414 Err(err) => return Err(err),
2415 }
2416 }
2417 if oids.is_empty() {
2418 return Ok(None);
2419 }
2420
2421 let inputs: Vec<PackInput<'_>> = oids
2422 .iter()
2423 .zip(&objects)
2424 .map(|(oid, object)| PackInput {
2425 oid,
2426 object: object.as_ref(),
2427 })
2428 .collect();
2429 let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
2430
2431 let mut sorted_entries: Vec<&sley_pack::PackIndexEntry> = written.entries.iter().collect();
2433 sorted_entries.sort_by(|a, b| a.oid.as_bytes().cmp(b.oid.as_bytes()));
2434 let mtimes_table: Vec<u32> = sorted_entries
2435 .iter()
2436 .map(|entry| mtime_by_oid.get(&entry.oid).copied().unwrap_or(0))
2437 .collect();
2438 let positions = sley_pack::pack_order_index_positions(&written.entries);
2439 let rev = sley_pack::PackReverseIndex::write(format, &positions, &written.checksum)?;
2440 let mtimes = sley_pack::PackMtimes::write(format, &mtimes_table, &written.checksum)?;
2441
2442 let mut cruft_oids: Vec<ObjectId> = sorted_entries.iter().map(|e| e.oid).collect();
2443 cruft_oids.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
2444 Ok(Some(CruftPack {
2445 pack: written.pack,
2446 idx: written.index,
2447 rev,
2448 mtimes,
2449 checksum: written.checksum,
2450 oids: cruft_oids,
2451 }))
2452}
2453
2454pub fn repack_cruft(
2464 git_dir: &Path,
2465 format: ObjectFormat,
2466 roots: &[ObjectId],
2467 cruft_expiration: Option<u32>,
2468) -> Result<CruftRepackResult> {
2469 repack_cruft_with_options(
2470 git_dir,
2471 format,
2472 roots,
2473 cruft_expiration,
2474 &RepackOptions::default(),
2475 )
2476}
2477
2478pub fn repack_cruft_with_options(
2479 git_dir: &Path,
2480 format: ObjectFormat,
2481 roots: &[ObjectId],
2482 cruft_expiration: Option<u32>,
2483 options: &RepackOptions,
2484) -> Result<CruftRepackResult> {
2485 let objects_dir = repository_objects_dir(git_dir);
2486 let database = FileObjectDatabase::new(objects_dir.clone(), format);
2487 let pack_dir = objects_dir.join("pack");
2488 let retained_pack_stems = repack_retained_pack_stems(
2489 &pack_dir,
2490 &options.keep_pack_stems,
2491 !options.pack_kept_objects,
2492 )?;
2493 let excluded_oids = if options.pack_kept_objects {
2494 HashSet::new()
2495 } else {
2496 pack_oids_for_stems(&pack_dir, format, &retained_pack_stems)?
2497 };
2498 let promisor_oids = promisor_pack_object_ids(&objects_dir, format)?;
2499 let database = if promisor_oids.is_empty() {
2500 database
2501 } else {
2502 database.with_promisor_remote_present(true)
2503 };
2504
2505 let mut reachable_ids = collect_reachable_object_ids_excluding_promised_missing(
2507 &database,
2508 format,
2509 roots.iter().copied(),
2510 &promisor_oids,
2511 )?;
2512 reachable_ids.retain(|oid| !excluded_oids.contains(oid));
2513 let reachable_result = if reachable_ids.is_empty() {
2514 None
2515 } else {
2516 let mut ids: Vec<ObjectId> = reachable_ids.iter().copied().collect();
2517 ids.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
2518 let mut objects = Vec::with_capacity(ids.len());
2519 for oid in &ids {
2520 match database.read_object(oid) {
2521 Ok(object) => objects.push(ReachablePackObject { oid: *oid, object }),
2522 Err(GitError::NotFound(_)) => {}
2523 Err(err) => return Err(err),
2524 }
2525 }
2526 if objects.is_empty() {
2527 None
2528 } else {
2529 let inputs = pack_inputs(&objects);
2530 let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
2531 let packed_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
2532 let mut packed_loose: Vec<ObjectId> = loose_object_ids(&objects_dir, format)?
2533 .into_iter()
2534 .filter(|oid| packed_set.contains(oid))
2535 .collect();
2536 packed_loose.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
2537 Some(RepackResult {
2538 pack: written.pack,
2539 idx: written.index,
2540 object_count: written.entries.len(),
2541 obsolete_packs: Vec::new(),
2542 packed_loose,
2543 retained_pack_stems: Vec::new(),
2544 promisor: false,
2545 pack_checksum: written.checksum,
2546 index_entries: written.entries,
2547 })
2548 }
2549 };
2550
2551 let mut survivors: HashMap<ObjectId, u32> = object_mtimes_on_disk(&objects_dir, format)?
2554 .into_iter()
2555 .filter(|(oid, _)| {
2556 !reachable_ids.contains(oid)
2557 && !excluded_oids.contains(oid)
2558 && !promisor_oids.contains(oid)
2559 })
2560 .collect();
2561
2562 if let Some(expiration) = cruft_expiration {
2564 rescue_and_expire_cruft_objects(&database, format, &mut survivors, expiration)?;
2565 }
2566
2567 let cruft = build_cruft_pack(&database, format, &survivors)?;
2568
2569 let mut obsolete_packs = Vec::new();
2572 let mut obsolete_cruft_packs = Vec::new();
2573 for pack_path in existing_pack_files(&pack_dir)? {
2574 if let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str())
2575 && retained_pack_stems.iter().any(|retained| retained == stem)
2576 {
2577 continue;
2578 }
2579 if pack_path.with_extension("keep").exists() {
2580 continue;
2581 }
2582 if pack_path.with_extension("mtimes").exists() {
2583 obsolete_cruft_packs.push(pack_path);
2584 } else {
2585 obsolete_packs.push(pack_path);
2586 }
2587 }
2588
2589 Ok(CruftRepackResult {
2590 reachable: reachable_result,
2591 cruft,
2592 obsolete_packs,
2593 obsolete_cruft_packs,
2594 retained_pack_stems,
2595 })
2596}
2597
2598fn rescue_and_expire_cruft_objects(
2603 database: &FileObjectDatabase,
2604 format: ObjectFormat,
2605 survivors: &mut HashMap<ObjectId, u32>,
2606 expiration: u32,
2607) -> Result<()> {
2608 let recent: Vec<ObjectId> = survivors
2609 .iter()
2610 .filter(|(_, mtime)| **mtime > expiration)
2611 .map(|(oid, _)| *oid)
2612 .collect();
2613
2614 let mut keep: HashSet<ObjectId> = HashSet::new();
2615 let mut pending: Vec<ObjectId> = recent.clone();
2616 while let Some(oid) = pending.pop() {
2617 if !keep.insert(oid) {
2618 continue;
2619 }
2620 let Ok(object) = database.read_object(&oid) else {
2621 continue;
2622 };
2623 match object.object_type {
2624 ObjectType::Commit => {
2625 if let Ok(commit) = Commit::parse_ref(format, &object.body) {
2626 pending.extend(commit.parents.iter().copied());
2627 pending.push(commit.tree);
2628 }
2629 }
2630 ObjectType::Tree => {
2631 for entry in TreeEntries::new(format, &object.body).flatten() {
2632 if !entry.is_gitlink() {
2633 pending.push(entry.oid);
2634 }
2635 }
2636 }
2637 ObjectType::Tag => {
2638 if let Ok(tag) = Tag::parse_ref(format, &object.body) {
2639 pending.push(tag.object);
2640 }
2641 }
2642 ObjectType::Blob => {}
2643 }
2644 }
2645
2646 survivors.retain(|oid, mtime| *mtime > expiration || keep.contains(oid));
2649 Ok(())
2650}
2651
2652pub fn install_cruft_repack_result(
2656 git_dir: &Path,
2657 format: ObjectFormat,
2658 result: &CruftRepackResult,
2659 prune: bool,
2660) -> Result<()> {
2661 let objects_dir = repository_objects_dir(git_dir);
2662 let pack_dir = objects_dir.join("pack");
2663 fs::create_dir_all(&pack_dir)?;
2664
2665 let new_reachable_name = result
2667 .reachable
2668 .as_ref()
2669 .map(|r| format!("pack-{}.pack", r.pack_checksum.to_hex()));
2670 let new_cruft_name = result
2671 .cruft
2672 .as_ref()
2673 .map(|c| format!("pack-{}.pack", c.checksum.to_hex()));
2674
2675 if let Some(reachable) = result.reachable.as_ref() {
2677 let parsed_index = PackIndex::parse(&reachable.idx, format)?;
2678 let pack_name = format!("pack-{}", reachable.pack_checksum.to_hex());
2679 let reverse_index = sley_pack::PackReverseIndex::write(
2680 format,
2681 &sley_pack::pack_order_index_positions(&parsed_index.entries),
2682 &reachable.pack_checksum,
2683 )?;
2684 write_pack_component(&pack_dir.join(format!("{pack_name}.pack")), &reachable.pack)?;
2685 write_pack_component(&pack_dir.join(format!("{pack_name}.rev")), &reverse_index)?;
2686 write_pack_component(&pack_dir.join(format!("{pack_name}.idx")), &reachable.idx)?;
2687 }
2688
2689 if let Some(cruft) = result.cruft.as_ref() {
2691 let pack_name = format!("pack-{}", cruft.checksum.to_hex());
2692 write_pack_component(&pack_dir.join(format!("{pack_name}.pack")), &cruft.pack)?;
2693 write_pack_component(&pack_dir.join(format!("{pack_name}.rev")), &cruft.rev)?;
2694 write_pack_component(&pack_dir.join(format!("{pack_name}.mtimes")), &cruft.mtimes)?;
2695 write_pack_component(&pack_dir.join(format!("{pack_name}.idx")), &cruft.idx)?;
2696 }
2697
2698 if !prune {
2699 return Ok(());
2700 }
2701
2702 let mut present: HashSet<ObjectId> = HashSet::new();
2704 if let Some(reachable) = result.reachable.as_ref() {
2705 present.extend(reachable.index_entries.iter().map(|e| e.oid));
2706 }
2707 if let Some(cruft) = result.cruft.as_ref() {
2708 present.extend(cruft.oids.iter().copied());
2709 }
2710
2711 let mut removed_stems: HashSet<String> = HashSet::new();
2713 for pack_path in result
2714 .obsolete_packs
2715 .iter()
2716 .chain(result.obsolete_cruft_packs.iter())
2717 {
2718 let file_name = pack_path.file_name().and_then(|n| n.to_str());
2719 if file_name == new_reachable_name.as_deref() || file_name == new_cruft_name.as_deref() {
2720 continue;
2721 }
2722 if let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str())
2723 && result
2724 .retained_pack_stems
2725 .iter()
2726 .any(|retained| retained == stem)
2727 {
2728 continue;
2729 }
2730 if pack_path.with_extension("keep").exists() {
2731 continue;
2732 }
2733 if pack_path.with_extension("promisor").exists() {
2734 continue;
2735 }
2736 if let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str()) {
2737 removed_stems.insert(stem.to_string());
2738 }
2739 remove_file_if_exists(pack_path)?;
2740 remove_file_if_exists(&pack_path.with_extension("idx"))?;
2741 for ext in ["rev", "mtimes", "bitmap", "promisor"] {
2742 remove_file_if_exists(&pack_path.with_extension(ext))?;
2743 }
2744 }
2745
2746 let loose_now_packed: Vec<ObjectId> = loose_object_ids(&objects_dir, format)?
2748 .into_iter()
2749 .filter(|oid| present.contains(oid))
2750 .collect();
2751 prune_loose_objects(&objects_dir, format, loose_now_packed.iter(), &present)?;
2752
2753 prune_stale_multi_pack_index(&pack_dir, format, &removed_stems)?;
2754 Ok(())
2755}
2756
2757fn pack_index_entries_match_writer(
2758 parsed: &[PackIndexEntry],
2759 writer_entries: &[PackIndexEntry],
2760) -> bool {
2761 if parsed.len() != writer_entries.len() {
2762 return false;
2763 }
2764 let mut writer_entries = writer_entries.iter().collect::<Vec<_>>();
2765 writer_entries.sort_by(|left, right| left.oid.as_bytes().cmp(right.oid.as_bytes()));
2766 parsed.iter().zip(writer_entries).all(|(left, right)| {
2767 left.oid == right.oid && left.crc32 == right.crc32 && left.offset == right.offset
2768 })
2769}
2770
2771pub fn prune_unreachable_loose<I>(
2780 git_dir: &Path,
2781 format: ObjectFormat,
2782 roots: I,
2783 delete: bool,
2784) -> Result<Vec<ObjectId>>
2785where
2786 I: IntoIterator<Item = ObjectId>,
2787{
2788 prune_unreachable_loose_with_reachability(git_dir, format, roots, delete, false)
2789}
2790
2791pub fn prune_unreachable_loose_tolerating_missing<I>(
2796 git_dir: &Path,
2797 format: ObjectFormat,
2798 roots: I,
2799 delete: bool,
2800) -> Result<Vec<ObjectId>>
2801where
2802 I: IntoIterator<Item = ObjectId>,
2803{
2804 prune_unreachable_loose_with_reachability(git_dir, format, roots, delete, true)
2805}
2806
2807fn prune_unreachable_loose_with_reachability<I>(
2808 git_dir: &Path,
2809 format: ObjectFormat,
2810 roots: I,
2811 delete: bool,
2812 tolerate_missing: bool,
2813) -> Result<Vec<ObjectId>>
2814where
2815 I: IntoIterator<Item = ObjectId>,
2816{
2817 let objects_dir = repository_objects_dir(git_dir);
2818 let database = FileObjectDatabase::new(objects_dir.clone(), format);
2819 let reachable = if tolerate_missing {
2820 collect_reachable_object_ids_tolerating_missing(&database, format, roots)?
2821 } else {
2822 collect_reachable_object_ids(&database, format, roots)?
2823 };
2824
2825 let store = LooseObjectStore::new(objects_dir.clone(), format);
2826 let mut pruned: Vec<ObjectId> = loose_object_ids(&objects_dir, format)?
2827 .into_iter()
2828 .filter(|oid| !reachable.contains(oid))
2829 .collect();
2830 pruned.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
2831
2832 if delete {
2833 for oid in &pruned {
2834 let path = store.object_path(oid)?;
2835 match fs::remove_file(&path) {
2836 Ok(()) => {}
2837 Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
2838 Err(err) => return Err(GitError::Io(err.to_string())),
2839 }
2840 }
2841 }
2842 Ok(pruned)
2843}
2844
2845fn loose_object_ids(objects_dir: &Path, format: ObjectFormat) -> Result<Vec<ObjectId>> {
2848 let oids = loose_object_id_set(objects_dir, format)?;
2849 let mut oids = oids.into_iter().collect::<Vec<_>>();
2850 oids.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
2851 Ok(oids)
2852}
2853
2854fn loose_object_id_set(objects_dir: &Path, format: ObjectFormat) -> Result<HashSet<ObjectId>> {
2855 let mut oids = HashSet::new();
2856 collect_loose_object_ids(objects_dir, format, &mut oids)?;
2857 Ok(oids)
2858}
2859
2860fn existing_pack_files(pack_dir: &Path) -> Result<Vec<PathBuf>> {
2863 if !pack_dir.exists() {
2864 return Ok(Vec::new());
2865 }
2866 let mut packs = Vec::new();
2867 for entry in fs::read_dir(pack_dir)? {
2868 let path = entry?.path();
2869 if path.extension().and_then(|ext| ext.to_str()) == Some("pack") && path.is_file() {
2870 packs.push(path);
2871 }
2872 }
2873 packs.sort();
2874 Ok(packs)
2875}
2876
2877fn prune_obsolete_pack_paths(
2881 objects_dir: &Path,
2882 format: ObjectFormat,
2883 packs: &[PathBuf],
2884 keep: &Path,
2885 retained_pack_stems: &[String],
2886 prune_promisor: bool,
2887) -> Result<()> {
2888 prune_pack_paths_matching(
2889 objects_dir,
2890 format,
2891 packs.iter(),
2892 keep,
2893 retained_pack_stems,
2894 prune_promisor,
2895 |_| Ok(true),
2896 )
2897}
2898
2899fn prune_pack_paths_matching<'a>(
2900 objects_dir: &Path,
2901 format: ObjectFormat,
2902 packs: impl IntoIterator<Item = &'a PathBuf>,
2903 keep: &Path,
2904 retained_pack_stems: &[String],
2905 prune_promisor: bool,
2906 mut should_prune: impl FnMut(&Path) -> Result<bool>,
2907) -> Result<()> {
2908 let pack_dir = objects_dir.join("pack");
2909 let keep_stem = keep.file_stem().map(|stem| stem.to_owned());
2910 let retained_pack_stems: HashSet<&str> =
2911 retained_pack_stems.iter().map(String::as_str).collect();
2912 let mut removed_stems: HashSet<String> = HashSet::new();
2913
2914 for pack_path in packs {
2915 if pack_path == keep {
2916 continue;
2917 }
2918 let Some(stem) = pack_path.file_stem() else {
2919 continue;
2920 };
2921 if Some(stem) == keep_stem.as_deref() {
2922 continue;
2923 }
2924 if let Some(stem) = stem.to_str()
2925 && retained_pack_stems.contains(stem)
2926 {
2927 continue;
2928 }
2929 if pack_path.with_extension("keep").exists() {
2930 continue;
2931 }
2932 if pack_path.with_extension("promisor").exists() && !prune_promisor {
2933 continue;
2934 }
2935 if !should_prune(pack_path)? {
2936 continue;
2937 }
2938 remove_file_if_exists(pack_path)?;
2939 remove_file_if_exists(&pack_path.with_extension("idx"))?;
2940 for ext in ["rev", "mtimes", "bitmap", "promisor"] {
2941 remove_file_if_exists(&pack_path.with_extension(ext))?;
2942 }
2943 removed_stems.insert(stem.to_string_lossy().into_owned());
2944 }
2945
2946 prune_stale_multi_pack_index(&pack_dir, format, &removed_stems)?;
2947 Ok(())
2948}
2949
2950fn prune_stale_multi_pack_index(
2957 pack_dir: &Path,
2958 format: ObjectFormat,
2959 removed_stems: &HashSet<String>,
2960) -> Result<()> {
2961 if removed_stems.is_empty() {
2962 return Ok(());
2963 }
2964 let midx_path = pack_dir.join("multi-pack-index");
2965 if !midx_path.exists() {
2966 return Ok(());
2967 }
2968 let midx = MultiPackIndex::parse(&fs::read(&midx_path)?, format)?;
2969 let references_removed_pack = midx.pack_names.iter().any(|name| {
2970 let stem = name.strip_suffix(".idx").unwrap_or(name);
2971 removed_stems.contains(stem)
2972 });
2973 if references_removed_pack {
2974 remove_file_if_exists(&midx_path)?;
2975 }
2976 Ok(())
2977}
2978
2979fn prune_loose_objects<'a, I>(
2982 objects_dir: &Path,
2983 format: ObjectFormat,
2984 candidates: I,
2985 present: &HashSet<ObjectId>,
2986) -> Result<()>
2987where
2988 I: IntoIterator<Item = &'a ObjectId>,
2989{
2990 let store = LooseObjectStore::new(objects_dir.to_path_buf(), format);
2991 for oid in candidates {
2992 if !present.contains(oid) {
2993 continue;
2994 }
2995 remove_file_if_exists(&store.object_path(oid)?)?;
2996 }
2997 Ok(())
2998}
2999
3000enum PackDeltaBase {
3001 Offset(u64),
3002 Ref(ObjectId),
3003}
3004
3005struct PackIndexOffsetInfo {
3006 end_offset: u64,
3007 delta_base_oid: Option<ObjectId>,
3008}
3009
3010fn scan_pack_index_offsets(
3011 index: &PackIndex,
3012 target_offset: u64,
3013 trailer_offset: Option<u64>,
3014 delta_base_offset: Option<u64>,
3015) -> Result<PackIndexOffsetInfo> {
3016 let mut target_count = 0usize;
3017 let mut next_offset = None;
3018 let mut delta_base_oid = None;
3019
3020 for entry in &index.entries {
3021 if entry.offset == target_offset {
3022 target_count += 1;
3023 } else if entry.offset > target_offset {
3024 match next_offset {
3025 Some(current) if current <= entry.offset => {}
3026 _ => next_offset = Some(entry.offset),
3027 }
3028 }
3029 if Some(entry.offset) == delta_base_offset {
3030 delta_base_oid = Some(entry.oid);
3031 }
3032 }
3033
3034 if target_count == 0 {
3035 return Err(GitError::InvalidFormat(format!(
3036 "pack index offset {target_offset} not found"
3037 )));
3038 }
3039 if let Some(offset) = delta_base_offset
3040 && delta_base_oid.is_none()
3041 {
3042 return Err(GitError::InvalidFormat(format!(
3043 "ofs-delta base offset {offset} not found"
3044 )));
3045 }
3046
3047 Ok(PackIndexOffsetInfo {
3048 end_offset: if target_count > 1 {
3051 target_offset
3052 } else if let Some(offset) = next_offset {
3053 offset
3054 } else {
3055 trailer_offset.ok_or_else(|| {
3056 GitError::InvalidFormat("pack size unavailable for final indexed object".into())
3057 })?
3058 },
3059 delta_base_oid,
3060 })
3061}
3062
3063fn scan_pack_offsets_without_index(
3064 format: ObjectFormat,
3065 pack: &[u8],
3066 target_offset: u64,
3067) -> Result<Option<u64>> {
3068 let trailer_len = format.raw_len();
3069 if pack.len() < 12 + trailer_len {
3070 return Err(GitError::InvalidFormat("pack file too short".into()));
3071 }
3072 let trailer_offset = pack.len() - trailer_len;
3073 let checksum = sley_core::digest_bytes(format, &pack[..trailer_offset])?;
3074 let expected = ObjectId::from_raw(format, &pack[trailer_offset..])?;
3075 if checksum != expected {
3076 return Err(GitError::InvalidFormat(format!(
3077 "pack checksum mismatch: expected {expected}, got {checksum}"
3078 )));
3079 }
3080 if &pack[..4] != b"PACK" {
3081 return Err(GitError::InvalidFormat("missing PACK signature".into()));
3082 }
3083 let version = u32_be(&pack[4..8]);
3084 if version != 2 && version != 3 {
3085 return Err(GitError::Unsupported(format!("pack version {version}")));
3086 }
3087
3088 let count = u32_be(&pack[8..12]);
3089 let mut cursor = 12usize;
3090 for _ in 0..count {
3091 let entry_offset = cursor as u64;
3092 let first = pack_next_byte(pack, &mut cursor)?;
3093 let kind = (first >> 4) & 0x07;
3094 let mut byte = first;
3095 while byte & 0x80 != 0 {
3096 byte = pack_next_byte(pack, &mut cursor)?;
3097 }
3098 match kind {
3099 1..=4 => {}
3100 6 => {
3101 parse_ofs_delta_base_offset(pack, &mut cursor, entry_offset)?;
3102 }
3103 7 => {
3104 parse_ref_delta_base_oid(format, pack, &mut cursor)?;
3105 }
3106 _ => {
3107 return Err(GitError::InvalidFormat(format!(
3108 "invalid pack object kind {kind}"
3109 )));
3110 }
3111 }
3112 if cursor > trailer_offset {
3113 return Err(GitError::InvalidFormat(
3114 "pack entry extends past checksum".into(),
3115 ));
3116 }
3117 let consumed = inflate_pack_member_len(&pack[cursor..trailer_offset])?;
3118 if consumed == 0 {
3119 return Err(GitError::InvalidFormat(
3120 "empty compressed pack entry".into(),
3121 ));
3122 }
3123 cursor = cursor
3124 .checked_add(consumed)
3125 .ok_or_else(|| GitError::InvalidFormat("pack offset overflow".into()))?;
3126 if cursor > trailer_offset {
3127 return Err(GitError::InvalidFormat(
3128 "pack entry extends past checksum".into(),
3129 ));
3130 }
3131 if entry_offset == target_offset {
3132 return Ok(Some(cursor as u64));
3133 }
3134 }
3135 if cursor != trailer_offset {
3136 return Err(GitError::InvalidFormat(format!(
3137 "pack has {} trailing bytes before checksum",
3138 trailer_offset - cursor
3139 )));
3140 }
3141 Ok(None)
3142}
3143
3144fn u32_be(bytes: &[u8]) -> u32 {
3145 u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])
3146}
3147
3148fn inflate_pack_member_len(compressed: &[u8]) -> Result<usize> {
3149 let mut decompress = Decompress::new(true);
3150 let mut input = compressed;
3151 let mut consumed_total = 0usize;
3152 let mut out = [0u8; 8192];
3153 loop {
3154 let before_in = decompress.total_in();
3155 let before_out = decompress.total_out();
3156 let status = decompress
3157 .decompress(input, &mut out, FlushDecompress::None)
3158 .map_err(|err| GitError::InvalidObject(format!("zlib inflate failed: {err}")))?;
3159 let consumed = (decompress.total_in() - before_in) as usize;
3160 let produced = decompress.total_out() - before_out;
3161 input = &input[consumed..];
3162 consumed_total += consumed;
3163 match status {
3164 flate2::Status::StreamEnd => return Ok(consumed_total),
3165 _ if consumed == 0 && produced == 0 => {
3166 return Err(GitError::InvalidObject("truncated zlib stream".into()));
3167 }
3168 _ => {}
3169 }
3170 }
3171}
3172
3173fn pack_entry_delta_base(
3174 format: ObjectFormat,
3175 pack: &[u8],
3176 entry_offset: u64,
3177) -> Result<Option<PackDeltaBase>> {
3178 let mut cursor = usize::try_from(entry_offset)
3179 .map_err(|_| GitError::InvalidFormat("pack entry offset overflows usize".into()))?;
3180 let first = pack_next_byte(pack, &mut cursor)?;
3181 let kind = (first >> 4) & 0x07;
3182 let mut byte = first;
3183 while byte & 0x80 != 0 {
3184 byte = pack_next_byte(pack, &mut cursor)?;
3185 }
3186 match kind {
3187 6 => Ok(Some(PackDeltaBase::Offset(parse_ofs_delta_base_offset(
3188 pack,
3189 &mut cursor,
3190 entry_offset,
3191 )?))),
3192 7 => Ok(Some(PackDeltaBase::Ref(parse_ref_delta_base_oid(
3193 format,
3194 pack,
3195 &mut cursor,
3196 )?))),
3197 _ => Ok(None),
3198 }
3199}
3200
3201fn parse_ref_delta_base_oid(
3202 format: ObjectFormat,
3203 pack: &[u8],
3204 cursor: &mut usize,
3205) -> Result<ObjectId> {
3206 let raw_len = format.raw_len();
3207 if *cursor + raw_len > pack.len() {
3208 return Err(GitError::InvalidFormat(
3209 "truncated ref-delta base object id".into(),
3210 ));
3211 }
3212 let oid = ObjectId::from_raw(format, &pack[*cursor..*cursor + raw_len])?;
3213 *cursor += raw_len;
3214 Ok(oid)
3215}
3216
3217fn parse_ofs_delta_base_offset(pack: &[u8], cursor: &mut usize, entry_offset: u64) -> Result<u64> {
3218 let mut byte = pack_next_byte(pack, cursor)?;
3219 let mut relative = u64::from(byte & 0x7f);
3220 while byte & 0x80 != 0 {
3221 byte = pack_next_byte(pack, cursor)?;
3222 relative = relative
3223 .checked_add(1)
3224 .and_then(|value| value.checked_shl(7))
3225 .and_then(|value| value.checked_add(u64::from(byte & 0x7f)))
3226 .ok_or_else(|| GitError::InvalidFormat("ofs-delta offset overflow".into()))?;
3227 }
3228 entry_offset
3229 .checked_sub(relative)
3230 .ok_or_else(|| GitError::InvalidFormat("ofs-delta points before pack start".into()))
3231}
3232
3233fn pack_next_byte(pack: &[u8], cursor: &mut usize) -> Result<u8> {
3234 let Some(byte) = pack.get(*cursor).copied() else {
3235 return Err(GitError::InvalidFormat("truncated pack entry".into()));
3236 };
3237 *cursor += 1;
3238 Ok(byte)
3239}
3240
3241fn zero_oid(format: ObjectFormat) -> Result<ObjectId> {
3242 Ok(ObjectId::null(format))
3243}
3244
3245fn remove_file_if_exists(path: &Path) -> Result<()> {
3247 match fs::remove_file(path) {
3248 Ok(()) => Ok(()),
3249 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
3250 Err(err) => Err(GitError::Io(err.to_string())),
3251 }
3252}
3253
3254fn walk_reachable_objects<R, I, F>(
3255 reader: &R,
3256 format: ObjectFormat,
3257 starts: I,
3258 excluded: &HashSet<ObjectId>,
3259 visit: F,
3260) -> Result<HashSet<ObjectId>>
3261where
3262 R: ObjectReader,
3263 I: IntoIterator<Item = ObjectId>,
3264 F: FnMut(&ObjectId, &Arc<EncodedObject>),
3265{
3266 walk_reachable_objects_with_cut(reader, format, starts, excluded, &HashSet::new(), visit)
3267}
3268
3269fn walk_reachable_objects_with_cut<R, I, F>(
3273 reader: &R,
3274 format: ObjectFormat,
3275 starts: I,
3276 excluded: &HashSet<ObjectId>,
3277 cut: &HashSet<ObjectId>,
3278 mut visit: F,
3279) -> Result<HashSet<ObjectId>>
3280where
3281 R: ObjectReader,
3282 I: IntoIterator<Item = ObjectId>,
3283 F: FnMut(&ObjectId, &Arc<EncodedObject>),
3284{
3285 let mut seen = HashSet::new();
3286 let mut pending = Vec::new();
3287 for start in starts {
3288 pending.push(start);
3289 while let Some(oid) = pending.pop() {
3290 if excluded.contains(&oid) {
3291 continue;
3292 }
3293 if !seen.insert(oid) {
3294 continue;
3295 }
3296 let object = reader.read_object(&oid).map_err(|err| {
3297 with_missing_object_context(err, oid, MissingObjectContext::Traversal)
3298 })?;
3299 match object.object_type {
3300 ObjectType::Commit => {
3301 let (tree, parents) = {
3302 let commit = Commit::parse_ref(format, &object.body)?;
3303 (commit.tree, commit.parents)
3304 };
3305 visit(&oid, &object);
3306 if !cut.contains(&oid) {
3307 for parent in grafted_parents(reader, &oid, parents).into_iter().rev() {
3308 pending.push(parent);
3309 }
3310 }
3311 pending.push(tree);
3312 }
3313 ObjectType::Tree => {
3314 let mut child_oids = Vec::new();
3315 for entry in TreeEntries::new(format, &object.body) {
3316 let entry = entry?;
3317 if entry.is_gitlink() {
3318 continue;
3319 }
3320 child_oids.push(entry.oid);
3321 }
3322 visit(&oid, &object);
3323 pending.extend(child_oids.into_iter().rev());
3324 }
3325 ObjectType::Tag => {
3326 let target = {
3327 let tag = Tag::parse_ref(format, &object.body)?;
3328 tag.object
3329 };
3330 visit(&oid, &object);
3331 pending.push(target);
3332 }
3333 ObjectType::Blob => visit(&oid, &object),
3334 }
3335 }
3336 }
3337 Ok(seen)
3338}
3339
3340fn walk_reachable_objects_tolerating_missing<R, I>(
3341 reader: &R,
3342 format: ObjectFormat,
3343 starts: I,
3344) -> Result<HashSet<ObjectId>>
3345where
3346 R: ObjectReader,
3347 I: IntoIterator<Item = ObjectId>,
3348{
3349 let mut seen = HashSet::new();
3350 let mut pending: Vec<ObjectId> = starts.into_iter().collect();
3351 while let Some(oid) = pending.pop() {
3352 if !seen.insert(oid) {
3353 continue;
3354 }
3355 let object = match reader
3356 .read_object(&oid)
3357 .map_err(|err| with_missing_object_context(err, oid, MissingObjectContext::Traversal))
3358 {
3359 Ok(object) => object,
3360 Err(GitError::NotFound(_)) => continue,
3361 Err(err) => return Err(err),
3362 };
3363 match object.object_type {
3364 ObjectType::Commit => {
3365 let commit = Commit::parse_ref(format, &object.body)?;
3366 pending.extend(grafted_parents(reader, &oid, commit.parents));
3367 pending.push(commit.tree);
3368 }
3369 ObjectType::Tree => {
3370 for entry in TreeEntries::new(format, &object.body) {
3371 let entry = entry?;
3372 if !entry.is_gitlink() {
3373 pending.push(entry.oid);
3374 }
3375 }
3376 }
3377 ObjectType::Tag => {
3378 let tag = Tag::parse_ref(format, &object.body)?;
3379 pending.push(tag.object);
3380 }
3381 ObjectType::Blob => {}
3382 }
3383 }
3384 Ok(seen)
3385}
3386
3387#[derive(Debug, Clone)]
3390pub struct BitmapPseudoMergeGroup {
3391 pub commits: Vec<ObjectId>,
3392 pub exclude_selected: bool,
3393 pub partition: Option<BitmapPseudoMergePartition>,
3394}
3395
3396#[derive(Debug, Clone)]
3397pub struct BitmapPseudoMergePartition {
3398 pub max_merges: usize,
3399 pub decay: f64,
3400 pub sample_rate: f64,
3401}
3402
3403fn bitset_get(words: &[u64], position: u32) -> bool {
3406 let word = (position / 64) as usize;
3407 word < words.len() && words[word] & (1u64 << (position % 64)) != 0
3408}
3409
3410fn bitset_set(words: &mut [u64], position: u32) {
3411 let word = (position / 64) as usize;
3412 if word < words.len() {
3413 words[word] |= 1u64 << (position % 64);
3414 }
3415}
3416
3417fn bitset_or(acc: &mut [u64], other: &[u64]) {
3418 for (dst, src) in acc.iter_mut().zip(other) {
3419 *dst |= *src;
3420 }
3421}
3422
3423fn bitset_is_subset(needles: &[u64], haystack: &[u64]) -> bool {
3424 needles
3425 .iter()
3426 .zip(haystack)
3427 .all(|(needle, hay)| needle & !hay == 0)
3428}
3429
3430fn bitset_positions(words: &[u64]) -> Vec<u32> {
3432 let mut positions = Vec::new();
3433 for (word_index, word) in words.iter().enumerate() {
3434 let mut remaining = *word;
3435 while remaining != 0 {
3436 let bit = remaining.trailing_zeros();
3437 positions.push(word_index as u32 * 64 + bit);
3438 remaining &= remaining - 1;
3439 }
3440 }
3441 positions
3442}
3443
3444fn commit_identity_timestamp(identity: &[u8]) -> i64 {
3448 let mut fields = identity.rsplitn(3, |byte| *byte == b' ');
3449 let _tz = fields.next();
3450 fields
3451 .next()
3452 .and_then(|raw| std::str::from_utf8(raw).ok())
3453 .and_then(|raw| raw.parse::<i64>().ok())
3454 .unwrap_or(0)
3455}
3456
3457fn bitmap_next_commit_index(idx: u32) -> u32 {
3460 const MIN_COMMITS: u32 = 100;
3461 const MAX_COMMITS: u32 = 5000;
3462 const MUST_REGION: u32 = 100;
3463 const MIN_REGION: u32 = 20000;
3464
3465 if idx <= MUST_REGION {
3466 return 0;
3467 }
3468 if idx <= MIN_REGION {
3469 let offset = idx - MUST_REGION;
3470 return offset.min(MIN_COMMITS);
3471 }
3472 let offset = idx - MIN_REGION;
3473 offset.clamp(MIN_COMMITS, MAX_COMMITS)
3474}
3475
3476pub fn build_pack_bitmap(
3490 db: &FileObjectDatabase,
3491 format: ObjectFormat,
3492 index_entries: &[PackIndexEntry],
3493 pack_checksum: &ObjectId,
3494 preferred_tips: &HashSet<ObjectId>,
3495 pseudo_merge_groups: &[BitmapPseudoMergeGroup],
3496) -> Result<Option<Vec<u8>>> {
3497 let mut by_offset: Vec<usize> = (0..index_entries.len()).collect();
3500 by_offset.sort_by_key(|&slot| index_entries[slot].offset);
3501 let bit_order: Vec<ObjectId> = by_offset
3502 .into_iter()
3503 .map(|slot| index_entries[slot].oid)
3504 .collect();
3505 build_reachability_bitmap(
3506 db,
3507 format,
3508 pack_checksum,
3509 &bit_order,
3510 preferred_tips,
3511 pseudo_merge_groups,
3512 )
3513}
3514
3515pub fn build_midx_bitmap(
3521 db: &FileObjectDatabase,
3522 format: ObjectFormat,
3523 midx_entries: &[sley_pack::MultiPackIndexEntry],
3524 midx_checksum: &ObjectId,
3525 preferred_pack: u32,
3526 preferred_tips: &HashSet<ObjectId>,
3527 pseudo_merge_groups: &[BitmapPseudoMergeGroup],
3528) -> Result<Option<Vec<u8>>> {
3529 let mut pseudo: Vec<usize> = (0..midx_entries.len()).collect();
3530 pseudo.sort_by_key(|&slot| {
3531 let entry = &midx_entries[slot];
3532 (
3533 entry.pack_int_id != preferred_pack,
3534 entry.pack_int_id,
3535 entry.offset,
3536 )
3537 });
3538 let bit_order: Vec<ObjectId> = pseudo
3539 .into_iter()
3540 .map(|slot| midx_entries[slot].oid)
3541 .collect();
3542 build_reachability_bitmap(
3543 db,
3544 format,
3545 midx_checksum,
3546 &bit_order,
3547 preferred_tips,
3548 pseudo_merge_groups,
3549 )
3550}
3551
3552fn bitmap_num_maximal_commits(
3560 db: &FileObjectDatabase,
3561 format: ObjectFormat,
3562 selected: &[ObjectId],
3563) -> Result<usize> {
3564 let mut first_parent: HashMap<ObjectId, Option<ObjectId>> = HashMap::new();
3566 let mut stack: Vec<ObjectId> = selected.to_vec();
3567 while let Some(oid) = stack.pop() {
3568 if first_parent.contains_key(&oid) {
3569 continue;
3570 }
3571 let object = db.read_object(&oid)?;
3572 let commit = Commit::parse_ref(format, &object.body)?;
3573 let parent = grafted_parents(db, &oid, commit.parents).first().copied();
3574 first_parent.insert(oid, parent);
3575 if let Some(parent) = parent {
3576 stack.push(parent);
3577 }
3578 }
3579 let mut pending_children: HashMap<ObjectId, usize> = HashMap::new();
3581 for parent in first_parent.values().flatten() {
3582 *pending_children.entry(*parent).or_default() += 1;
3583 }
3584 let word_count = selected.len().div_ceil(64);
3585 struct MaximalEnt {
3586 mask: Vec<u64>,
3587 maximal: bool,
3588 }
3589 let mut ents: HashMap<ObjectId, MaximalEnt> = HashMap::new();
3590 for (bit, oid) in selected.iter().enumerate() {
3591 let ent = ents.entry(*oid).or_insert_with(|| MaximalEnt {
3592 mask: vec![0u64; word_count],
3593 maximal: true,
3594 });
3595 ent.mask[bit / 64] |= 1u64 << (bit % 64);
3596 ent.maximal = true;
3597 }
3598 let mut queue: Vec<ObjectId> = first_parent
3599 .keys()
3600 .filter(|oid| pending_children.get(*oid).copied().unwrap_or(0) == 0)
3601 .copied()
3602 .collect();
3603 let mut num_maximal = 0usize;
3604 while let Some(oid) = queue.pop() {
3605 if let Some(ent) = ents.remove(&oid) {
3606 if ent.maximal {
3607 num_maximal += 1;
3608 }
3609 if let Some(Some(parent)) = first_parent.get(&oid) {
3610 match ents.entry(*parent) {
3611 std::collections::hash_map::Entry::Vacant(vacant) => {
3612 vacant.insert(MaximalEnt {
3614 mask: ent.mask.clone(),
3615 maximal: false,
3616 });
3617 }
3618 std::collections::hash_map::Entry::Occupied(mut occupied) => {
3619 let parent_ent = occupied.get_mut();
3620 let c_not_p = ent
3621 .mask
3622 .iter()
3623 .zip(&parent_ent.mask)
3624 .any(|(child, parent)| child & !parent != 0);
3625 if c_not_p {
3626 let p_not_c = parent_ent
3627 .mask
3628 .iter()
3629 .zip(&ent.mask)
3630 .any(|(parent, child)| parent & !child != 0);
3631 for (parent, child) in parent_ent.mask.iter_mut().zip(&ent.mask) {
3632 *parent |= child;
3633 }
3634 parent_ent.maximal = p_not_c;
3635 }
3636 }
3637 }
3638 }
3639 }
3640 if let Some(Some(parent)) = first_parent.get(&oid)
3641 && let Some(remaining) = pending_children.get_mut(parent)
3642 {
3643 *remaining -= 1;
3644 if *remaining == 0 {
3645 queue.push(*parent);
3646 }
3647 }
3648 }
3649 Ok(num_maximal)
3650}
3651
3652fn build_reachability_bitmap(
3656 db: &FileObjectDatabase,
3657 format: ObjectFormat,
3658 checksum: &ObjectId,
3659 bit_order: &[ObjectId],
3660 preferred_tips: &HashSet<ObjectId>,
3661 pseudo_merge_groups: &[BitmapPseudoMergeGroup],
3662) -> Result<Option<Vec<u8>>> {
3663 if bit_order.is_empty() || bit_order.len() > u32::MAX as usize {
3664 return Ok(None);
3665 }
3666 let object_count = bit_order.len();
3667
3668 let mut oid_sorted: Vec<u32> = (0..object_count as u32).collect();
3671 oid_sorted.sort_by(|&left, &right| {
3672 bit_order[left as usize]
3673 .as_bytes()
3674 .cmp(bit_order[right as usize].as_bytes())
3675 });
3676 let mut index_position = vec![0u32; object_count];
3677 for (position, &slot) in oid_sorted.iter().enumerate() {
3678 index_position[slot as usize] = position as u32;
3679 }
3680 let mut oid_to_pack = HashMap::with_capacity(object_count);
3681 for (pack_pos, oid) in bit_order.iter().enumerate() {
3682 oid_to_pack.insert(*oid, pack_pos as u32);
3683 }
3684
3685 let mut object_types = Vec::with_capacity(object_count);
3687 struct IndexedCommit {
3688 oid: ObjectId,
3689 pack_pos: u32,
3690 index_pos: u32,
3691 date: i64,
3692 parent_count: usize,
3693 }
3694 let mut indexed_commits = Vec::new();
3695 for (pack_pos, oid) in bit_order.iter().enumerate() {
3696 let object_type = match db.read_object_header(oid)? {
3699 Some((object_type, _)) => object_type,
3700 None => db.read_object(oid)?.object_type,
3701 };
3702 object_types.push(object_type);
3703 if object_type == ObjectType::Commit {
3704 let object = db.read_object(oid)?;
3705 let commit = Commit::parse_ref(format, &object.body)?;
3706 indexed_commits.push(IndexedCommit {
3707 oid: *oid,
3708 pack_pos: pack_pos as u32,
3709 index_pos: index_position[pack_pos],
3710 date: commit_identity_timestamp(commit.committer),
3711 parent_count: grafted_parents(db, oid, commit.parents).len(),
3712 });
3713 }
3714 }
3715
3716 indexed_commits.sort_by_key(|commit| std::cmp::Reverse(commit.date));
3718 let mut selected: Vec<&IndexedCommit> = Vec::new();
3719 let commit_count = indexed_commits.len() as u32;
3720 if commit_count < 100 {
3721 selected.extend(indexed_commits.iter());
3722 } else {
3723 let mut i = 0u32;
3724 loop {
3725 let next = bitmap_next_commit_index(i);
3726 if i + next >= commit_count {
3727 break;
3728 }
3729 let mut chosen = &indexed_commits[(i + next) as usize];
3730 if next > 0 {
3731 for j in 0..=next {
3732 let candidate = &indexed_commits[(i + j) as usize];
3733 if preferred_tips.contains(&candidate.oid) {
3734 chosen = candidate;
3735 break;
3736 }
3737 if candidate.parent_count >= 2 {
3738 chosen = candidate;
3739 }
3740 }
3741 }
3742 selected.push(chosen);
3743 i += next + 1;
3744 }
3745 }
3746
3747 if std::env::var_os("GIT_TRACE2_EVENT").is_some() {
3752 let selected_oids: Vec<ObjectId> = selected.iter().map(|commit| commit.oid).collect();
3753 let num_maximal = bitmap_num_maximal_commits(db, format, &selected_oids)?;
3754 sley_core::trace2::data("pack-bitmap-write", "num_selected_commits", selected.len());
3755 sley_core::trace2::data("pack-bitmap-write", "num_maximal_commits", num_maximal);
3756 let reusable_pseudo_merges = pseudo_merge_groups
3757 .iter()
3758 .filter(|group| !group.exclude_selected)
3759 .count();
3760 sley_core::trace2::data(
3761 "pack-bitmap-write",
3762 "building_bitmaps_pseudo_merge_reused",
3763 reusable_pseudo_merges,
3764 );
3765 }
3766
3767 let word_count = object_count.div_ceil(64);
3770 let mut memo: HashMap<ObjectId, Arc<Vec<u64>>> = HashMap::new();
3771 for commit in selected.iter().rev() {
3772 let Some(acc) =
3773 bitmap_commit_closure(db, format, &[commit.oid], &oid_to_pack, word_count, &memo)?
3774 else {
3775 return Ok(None);
3776 };
3777 memo.insert(commit.oid, Arc::new(acc));
3778 }
3779
3780 let mut writer = PackBitmapWriter::new(format, *checksum, &object_types)?;
3781 for commit in &selected {
3782 let words = match memo.get(&commit.oid) {
3783 Some(words) => words,
3784 None => continue,
3785 };
3786 writer.add_commit(commit.pack_pos, commit.index_pos, &bitset_positions(words))?;
3787 }
3788 if !pseudo_merge_groups.is_empty() {
3789 let selected_oids: HashSet<ObjectId> = selected.iter().map(|commit| commit.oid).collect();
3790 for group in pseudo_merge_groups {
3791 let mut commits = Vec::new();
3792 for oid in &group.commits {
3793 if group.exclude_selected && selected_oids.contains(oid) {
3794 continue;
3795 }
3796 let Some(&pack_pos) = oid_to_pack.get(oid) else {
3797 continue;
3798 };
3799 if object_types.get(pack_pos as usize) != Some(&ObjectType::Commit) {
3800 continue;
3801 }
3802 commits.push((*oid, pack_pos));
3803 }
3804 if commits.is_empty() {
3805 continue;
3806 }
3807 if let Some(partition) = &group.partition {
3808 let mut start = 0usize;
3809 for merge_index in 0..partition.max_merges {
3810 if start >= commits.len() {
3811 break;
3812 }
3813 let size = bitmap_pseudo_merge_group_size(
3814 partition.max_merges,
3815 partition.decay,
3816 commits.len(),
3817 merge_index,
3818 );
3819 let end = if size < 8 {
3820 commits.len()
3821 } else {
3822 start.saturating_add(size).min(commits.len())
3823 };
3824 let sample_stride = if partition.sample_rate <= 0.0 {
3825 usize::MAX
3826 } else {
3827 ((1.0 / partition.sample_rate) as usize).max(1)
3828 };
3829 let sampled: Vec<(ObjectId, u32)> = commits[start..end]
3830 .iter()
3831 .enumerate()
3832 .filter(|(offset, _candidate)| *offset % sample_stride == 0)
3833 .map(|(_offset, candidate)| *candidate)
3834 .collect();
3835 if !sampled.is_empty()
3836 && !bitmap_add_pseudo_merge(
3837 &mut writer,
3838 db,
3839 format,
3840 &sampled,
3841 &oid_to_pack,
3842 word_count,
3843 &memo,
3844 )?
3845 {
3846 return Ok(None);
3847 }
3848 start = end;
3849 if end >= commits.len() {
3850 break;
3851 }
3852 }
3853 } else if !bitmap_add_pseudo_merge(
3854 &mut writer,
3855 db,
3856 format,
3857 &commits,
3858 &oid_to_pack,
3859 word_count,
3860 &memo,
3861 )? {
3862 return Ok(None);
3863 }
3864 }
3865 }
3866 writer.write().map(Some)
3867}
3868
3869fn bitmap_pseudo_merge_group_size(
3870 max_merges: usize,
3871 decay: f64,
3872 unstable_len: usize,
3873 index: usize,
3874) -> usize {
3875 let mut scale = 0.0;
3876 for n in 0..max_merges {
3877 scale += 1.0 / ((n + 1) as f64).powf(decay);
3878 }
3879 if scale == 0.0 {
3880 return 0;
3881 }
3882 ((unstable_len as f64 / scale) / ((index + 1) as f64).powf(decay) + 0.5) as usize
3883}
3884
3885fn bitmap_add_pseudo_merge(
3886 writer: &mut PackBitmapWriter,
3887 db: &FileObjectDatabase,
3888 format: ObjectFormat,
3889 commits: &[(ObjectId, u32)],
3890 oid_to_pack: &HashMap<ObjectId, u32>,
3891 word_count: usize,
3892 memo: &HashMap<ObjectId, Arc<Vec<u64>>>,
3893) -> Result<bool> {
3894 let roots: Vec<ObjectId> = commits.iter().map(|(oid, _position)| *oid).collect();
3895 let Some(words) = bitmap_commit_closure(db, format, &roots, oid_to_pack, word_count, memo)?
3896 else {
3897 return Ok(false);
3898 };
3899 let commit_positions: Vec<u32> = commits.iter().map(|(_oid, position)| *position).collect();
3900 writer.add_pseudo_merge(&commit_positions, &bitset_positions(&words))?;
3901 Ok(true)
3902}
3903
3904fn bitmap_commit_closure(
3905 db: &impl ObjectReader,
3906 format: ObjectFormat,
3907 roots: &[ObjectId],
3908 oid_to_pack: &HashMap<ObjectId, u32>,
3909 word_count: usize,
3910 memo: &HashMap<ObjectId, Arc<Vec<u64>>>,
3911) -> Result<Option<Vec<u64>>> {
3912 let mut acc = vec![0u64; word_count];
3913 let mut pending = roots.to_vec();
3914 while let Some(oid) = pending.pop() {
3915 let Some(&pack_pos) = oid_to_pack.get(&oid) else {
3916 eprintln!(
3917 "warning: Failed to write bitmap index. Packfile doesn't have full closure (object {oid} is missing)"
3918 );
3919 return Ok(None);
3920 };
3921 if bitset_get(&acc, pack_pos) {
3922 continue;
3923 }
3924 if let Some(stored) = memo.get(&oid) {
3925 bitset_or(&mut acc, stored);
3926 continue;
3927 }
3928 bitset_set(&mut acc, pack_pos);
3929 let object = db.read_object(&oid)?;
3930 let parsed = Commit::parse_ref(format, &object.body)?;
3931 pending.extend(grafted_parents(db, &oid, parsed.parents));
3932 if !bitmap_mark_tree(db, format, &parsed.tree, oid_to_pack, &mut acc)? {
3933 return Ok(None);
3934 }
3935 }
3936 Ok(Some(acc))
3937}
3938
3939fn bitmap_mark_tree(
3943 db: &impl ObjectReader,
3944 format: ObjectFormat,
3945 tree: &ObjectId,
3946 oid_to_pack: &HashMap<ObjectId, u32>,
3947 acc: &mut [u64],
3948) -> Result<bool> {
3949 let Some(&pack_pos) = oid_to_pack.get(tree) else {
3950 eprintln!(
3951 "warning: Failed to write bitmap index. Packfile doesn't have full closure (object {tree} is missing)"
3952 );
3953 return Ok(false);
3954 };
3955 if bitset_get(acc, pack_pos) {
3956 return Ok(true);
3957 }
3958 bitset_set(acc, pack_pos);
3959 let object = db.read_object(tree)?;
3960 for entry in TreeEntries::new(format, &object.body) {
3961 let entry = entry?;
3962 if entry.is_gitlink() {
3963 continue;
3964 }
3965 if entry.is_tree() {
3966 if !bitmap_mark_tree(db, format, &entry.oid, oid_to_pack, acc)? {
3967 return Ok(false);
3968 }
3969 } else {
3970 let Some(&blob_pos) = oid_to_pack.get(&entry.oid) else {
3971 eprintln!(
3972 "warning: Failed to write bitmap index. Packfile doesn't have full closure (object {} is missing)",
3973 entry.oid
3974 );
3975 return Ok(false);
3976 };
3977 bitset_set(acc, blob_pos);
3978 }
3979 }
3980 Ok(true)
3981}
3982
3983pub struct LoadedPackBitmap {
3987 object_count: u32,
3988 oid_to_pack: HashMap<ObjectId, u32>,
3989 pack_to_oid: Vec<ObjectId>,
3990 commit_words: HashMap<ObjectId, Arc<Vec<u64>>>,
3991 pseudo_merges: Vec<LoadedPseudoMerge>,
3992 commits: Vec<u64>,
3993 trees: Vec<u64>,
3994 blobs: Vec<u64>,
3995 tags: Vec<u64>,
3996}
3997
3998struct LoadedPseudoMerge {
3999 commits: Arc<Vec<u64>>,
4000 bitmap: Arc<Vec<u64>>,
4001}
4002
4003impl LoadedPackBitmap {
4004 pub fn object_count(&self) -> u32 {
4005 self.object_count
4006 }
4007
4008 pub fn pack_position(&self, oid: &ObjectId) -> Option<u32> {
4010 self.oid_to_pack.get(oid).copied()
4011 }
4012
4013 pub fn oid_at(&self, position: u32) -> Option<&ObjectId> {
4014 self.pack_to_oid.get(position as usize)
4015 }
4016
4017 pub fn bitmap_for_commit(&self, oid: &ObjectId) -> Option<&Arc<Vec<u64>>> {
4020 self.commit_words.get(oid)
4021 }
4022
4023 pub fn bitmapped_commits(&self) -> impl Iterator<Item = &ObjectId> {
4025 self.commit_words.keys()
4026 }
4027
4028 pub fn pseudo_merge_count(&self) -> usize {
4029 self.pseudo_merges.len()
4030 }
4031
4032 pub fn pseudo_merge_words(&self, index: usize) -> Option<(&[u64], &[u64])> {
4033 self.pseudo_merges
4034 .get(index)
4035 .map(|merge| (merge.commits.as_slice(), merge.bitmap.as_slice()))
4036 }
4037
4038 pub fn type_words(&self, object_type: ObjectType) -> &[u64] {
4040 match object_type {
4041 ObjectType::Commit => &self.commits,
4042 ObjectType::Tree => &self.trees,
4043 ObjectType::Blob => &self.blobs,
4044 ObjectType::Tag => &self.tags,
4045 }
4046 }
4047
4048 fn word_count(&self) -> usize {
4049 (self.object_count as usize).div_ceil(64)
4050 }
4051}
4052
4053pub fn load_pack_bitmap(
4060 objects_dir: &Path,
4061 format: ObjectFormat,
4062) -> Result<Option<LoadedPackBitmap>> {
4063 let pack_dir = objects_dir.join("pack");
4064 if !pack_dir.exists() {
4065 return Ok(None);
4066 }
4067 if let Some(bitmap) = load_incremental_midx_bitmap(&pack_dir, format)? {
4070 return Ok(Some(bitmap));
4071 }
4072 if let Some(bitmap) = load_midx_bitmap(&pack_dir, format)? {
4073 return Ok(Some(bitmap));
4074 }
4075 let mut bitmap_paths = Vec::new();
4076 for entry in fs::read_dir(&pack_dir)? {
4077 let path = entry?.path();
4078 if path.extension().and_then(|ext| ext.to_str()) == Some("bitmap")
4079 && path
4080 .file_name()
4081 .and_then(|name| name.to_str())
4082 .is_some_and(|name| name.starts_with("pack-"))
4083 {
4084 bitmap_paths.push(path);
4085 }
4086 }
4087 bitmap_paths.sort();
4088 for bitmap_path in bitmap_paths {
4089 match load_pack_bitmap_file(&bitmap_path, format) {
4090 Ok(Some(bitmap)) => return Ok(Some(bitmap)),
4091 Ok(None) | Err(_) => continue,
4092 }
4093 }
4094 Ok(None)
4095}
4096
4097fn load_incremental_midx_bitmap(
4098 pack_dir: &Path,
4099 format: ObjectFormat,
4100) -> Result<Option<LoadedPackBitmap>> {
4101 let chain = read_incremental_midx_chain(pack_dir)?;
4102 if chain.is_empty() {
4103 return Ok(None);
4104 }
4105 let midx_dir = pack_dir.join("multi-pack-index.d");
4106 if !chain.iter().any(|checksum| {
4107 midx_dir
4108 .join(format!("multi-pack-index-{checksum}.bitmap"))
4109 .exists()
4110 }) {
4111 return Ok(None);
4112 }
4113
4114 let mut pack_to_oid = Vec::new();
4115 for checksum in &chain {
4116 let path = midx_dir.join(format!("multi-pack-index-{checksum}.midx"));
4117 let Ok(bytes) = fs::read(path) else {
4118 return Ok(None);
4119 };
4120 let Ok(midx) = MultiPackIndex::parse(&bytes, format) else {
4121 return Ok(None);
4122 };
4123 let mut positions: Vec<usize> = match &midx.reverse_index {
4124 Some(reverse) => reverse.iter().map(|position| *position as usize).collect(),
4125 None => {
4126 let mut positions: Vec<usize> = (0..midx.objects.len()).collect();
4127 positions.sort_by_key(|&position| {
4128 let entry = &midx.objects[position];
4129 (entry.pack_int_id, entry.offset)
4130 });
4131 positions
4132 }
4133 };
4134 for position in positions.drain(..) {
4135 let Some(entry) = midx.objects.get(position) else {
4136 return Ok(None);
4137 };
4138 pack_to_oid.push(entry.oid);
4139 }
4140 }
4141
4142 let object_count = pack_to_oid.len();
4143 if object_count == 0 || object_count > u32::MAX as usize {
4144 return Ok(None);
4145 }
4146 let mut oid_to_pack = HashMap::with_capacity(object_count);
4147 for (position, oid) in pack_to_oid.iter().enumerate() {
4148 oid_to_pack.insert(*oid, position as u32);
4149 }
4150
4151 let Some(objects_dir) = pack_dir.parent() else {
4152 return Ok(None);
4153 };
4154 let db = FileObjectDatabase::new(objects_dir.to_path_buf(), format);
4155 let word_count = object_count.div_ceil(64);
4156 let mut commits = vec![0u64; word_count];
4157 let mut trees = vec![0u64; word_count];
4158 let mut blobs = vec![0u64; word_count];
4159 let mut tags = vec![0u64; word_count];
4160 let mut commit_oids = Vec::new();
4161 for (position, oid) in pack_to_oid.iter().enumerate() {
4162 let Ok(Some((object_type, _size))) = db.read_object_header(oid) else {
4163 return Ok(None);
4164 };
4165 let position = position as u32;
4166 match object_type {
4167 ObjectType::Commit => {
4168 bitset_set(&mut commits, position);
4169 commit_oids.push(*oid);
4170 }
4171 ObjectType::Tree => bitset_set(&mut trees, position),
4172 ObjectType::Blob => bitset_set(&mut blobs, position),
4173 ObjectType::Tag => bitset_set(&mut tags, position),
4174 }
4175 }
4176
4177 let mut loaded = LoadedPackBitmap {
4178 object_count: object_count as u32,
4179 oid_to_pack,
4180 pack_to_oid,
4181 commit_words: HashMap::new(),
4182 pseudo_merges: Vec::new(),
4183 commits,
4184 trees,
4185 blobs,
4186 tags,
4187 };
4188 for oid in commit_oids {
4189 let result = bitmap_reachable(&loaded, &db, format, &[oid], true)?;
4190 if result.extended.is_empty() {
4191 loaded.commit_words.insert(oid, Arc::new(result.words));
4192 }
4193 }
4194 Ok(Some(loaded))
4195}
4196
4197fn load_midx_bitmap(pack_dir: &Path, format: ObjectFormat) -> Result<Option<LoadedPackBitmap>> {
4202 let midx_path = pack_dir.join("multi-pack-index");
4203 if !midx_path.exists() {
4204 return Ok(None);
4205 }
4206 let Ok(midx_bytes) = fs::read(&midx_path) else {
4207 return Ok(None);
4208 };
4209 if midx_has_bad_ridx_chunk(&midx_bytes, format) {
4210 eprintln!("error: multi-pack-index reverse-index chunk is the wrong size");
4211 eprintln!("warning: multi-pack bitmap is missing required reverse index");
4212 return Ok(None);
4213 }
4214 let midx = match MultiPackIndex::parse(&midx_bytes, format) {
4215 Ok(midx) => midx,
4216 Err(GitError::InvalidFormat(message))
4217 if message == "multi-pack-index reverse-index chunk is the wrong size" =>
4218 {
4219 eprintln!("error: {message}");
4220 eprintln!("warning: multi-pack bitmap is missing required reverse index");
4221 return Ok(None);
4222 }
4223 Err(_) => return Ok(None),
4224 };
4225 let bitmap_path = pack_dir.join(format!(
4226 "multi-pack-index-{}.bitmap",
4227 midx.checksum.to_hex()
4228 ));
4229 if !bitmap_path.exists() {
4230 return Ok(None);
4231 }
4232 let object_count = midx.objects.len();
4233 let read_ridx_chunk = env::var("GIT_TEST_MIDX_READ_RIDX")
4238 .map(|value| value != "0" && !value.eq_ignore_ascii_case("false"))
4239 .unwrap_or(true);
4240 let reverse_index: Vec<u32> = match (&midx.reverse_index, read_ridx_chunk) {
4241 (Some(chunk), true) => {
4242 sley_core::trace2::data("load_midx_revindex", "source", "midx");
4243 chunk.clone()
4244 }
4245 _ => {
4246 let rev_path =
4247 pack_dir.join(format!("multi-pack-index-{}.rev", midx.checksum.to_hex()));
4248 let Ok(rev_bytes) = fs::read(&rev_path) else {
4249 return Ok(None);
4251 };
4252 let Ok(parsed_rev) =
4253 sley_pack::PackReverseIndex::parse(&rev_bytes, format, object_count)
4254 else {
4255 return Ok(None);
4256 };
4257 sley_core::trace2::data("load_midx_revindex", "source", "rev");
4258 parsed_rev.positions
4259 }
4260 };
4261 let Ok(bitmap_bytes) = fs::read(&bitmap_path) else {
4262 return Ok(None);
4263 };
4264 let parsed = match PackBitmapIndex::parse(&bitmap_bytes, format, object_count) {
4265 Ok(parsed) => parsed,
4266 Err(_) => return Ok(None),
4267 };
4268 if parsed.pack_checksum != midx.checksum {
4269 return Ok(None);
4270 }
4271
4272 let mut pack_to_oid = Vec::with_capacity(object_count);
4275 for &midx_pos in &reverse_index {
4276 let Some(entry) = midx.objects.get(midx_pos as usize) else {
4277 return Ok(None);
4278 };
4279 pack_to_oid.push(entry.oid);
4280 }
4281 let mut oid_to_pack = HashMap::with_capacity(object_count);
4282 for (pack_pos, oid) in pack_to_oid.iter().enumerate() {
4283 oid_to_pack.insert(*oid, pack_pos as u32);
4284 }
4285 match assemble_loaded_bitmap(parsed, object_count, pack_to_oid, oid_to_pack, |position| {
4286 midx.objects.get(position).map(|entry| entry.oid)
4287 }) {
4288 Ok(loaded) => Ok(Some(loaded)),
4289 Err(_) => Ok(None),
4290 }
4291}
4292
4293fn midx_has_bad_ridx_chunk(bytes: &[u8], format: ObjectFormat) -> bool {
4294 let hash_len = format.raw_len();
4295 if bytes.len() < 12 + 12 + hash_len || &bytes[..4] != b"MIDX" {
4296 return false;
4297 }
4298 let chunk_count = bytes[6] as usize;
4299 let table_len = match (chunk_count + 1).checked_mul(12) {
4300 Some(table_len) => table_len,
4301 None => return false,
4302 };
4303 let table_end = match 12usize.checked_add(table_len) {
4304 Some(table_end) if table_end <= bytes.len().saturating_sub(hash_len) => table_end,
4305 _ => return false,
4306 };
4307 let mut entries = Vec::with_capacity(chunk_count + 1);
4308 let mut cursor = 12usize;
4309 while cursor < table_end {
4310 let id = [
4311 bytes[cursor],
4312 bytes[cursor + 1],
4313 bytes[cursor + 2],
4314 bytes[cursor + 3],
4315 ];
4316 let mut raw_offset = [0u8; 8];
4317 raw_offset.copy_from_slice(&bytes[cursor + 4..cursor + 12]);
4318 entries.push((id, u64::from_be_bytes(raw_offset) as usize));
4319 cursor += 12;
4320 }
4321 let mut oidf = None;
4322 let mut ridx = None;
4323 for pair in entries.windows(2) {
4324 let start = pair[0].1;
4325 let end = pair[1].1;
4326 if end < start || end > bytes.len().saturating_sub(hash_len) {
4327 return false;
4328 }
4329 match &pair[0].0 {
4330 b"OIDF" => oidf = Some((start, end)),
4331 b"RIDX" => ridx = Some((start, end)),
4332 _ => {}
4333 }
4334 }
4335 let Some((oidf_start, oidf_end)) = oidf else {
4336 return false;
4337 };
4338 let Some((ridx_start, ridx_end)) = ridx else {
4339 return false;
4340 };
4341 if oidf_end.saturating_sub(oidf_start) != 256 * 4 {
4342 return false;
4343 }
4344 let object_count_start = oidf_end - 4;
4345 let object_count = u32::from_be_bytes([
4346 bytes[object_count_start],
4347 bytes[object_count_start + 1],
4348 bytes[object_count_start + 2],
4349 bytes[object_count_start + 3],
4350 ]) as usize;
4351 ridx_end.saturating_sub(ridx_start) != object_count.saturating_mul(4)
4352}
4353
4354fn load_pack_bitmap_file(
4355 bitmap_path: &Path,
4356 format: ObjectFormat,
4357) -> Result<Option<LoadedPackBitmap>> {
4358 let index_path = bitmap_path.with_extension("idx");
4359 if !index_path.exists() {
4360 return Ok(None);
4361 }
4362 let index = PackIndex::parse(&fs::read(&index_path)?, format)?;
4363 let object_count = index.entries.len();
4364 let parsed = PackBitmapIndex::parse(&fs::read(bitmap_path)?, format, object_count)?;
4365 if parsed.pack_checksum != index.pack_checksum {
4366 return Ok(None);
4367 }
4368
4369 let mut pack_order: Vec<u32> = (0..object_count as u32).collect();
4370 pack_order.sort_by_key(|index_pos| index.entries[*index_pos as usize].offset);
4371 let mut pack_to_oid = Vec::with_capacity(object_count);
4372 for index_pos in &pack_order {
4373 pack_to_oid.push(index.entries[*index_pos as usize].oid);
4374 }
4375 let mut oid_to_pack = HashMap::with_capacity(object_count);
4376 for (pack_pos, oid) in pack_to_oid.iter().enumerate() {
4377 oid_to_pack.insert(*oid, pack_pos as u32);
4378 }
4379
4380 assemble_loaded_bitmap(parsed, object_count, pack_to_oid, oid_to_pack, |position| {
4381 index.entries.get(position).map(|entry| entry.oid)
4382 })
4383 .map(Some)
4384}
4385
4386fn assemble_loaded_bitmap(
4391 parsed: PackBitmapIndex,
4392 object_count: usize,
4393 pack_to_oid: Vec<ObjectId>,
4394 oid_to_pack: HashMap<ObjectId, u32>,
4395 lookup_oid: impl Fn(usize) -> Option<ObjectId>,
4396) -> Result<LoadedPackBitmap> {
4397 let word_count = object_count.div_ceil(64);
4398 let expand = |bitmap: &sley_pack::EwahBitmap| -> Result<Vec<u64>> {
4399 let mut words = bitmap.to_words()?;
4400 words.resize(word_count, 0);
4401 Ok(words)
4402 };
4403
4404 let mut resolved: Vec<Arc<Vec<u64>>> = Vec::with_capacity(parsed.entries.len());
4405 let mut commit_words = HashMap::with_capacity(parsed.entries.len());
4406 for (entry_index, entry) in parsed.entries.iter().enumerate() {
4407 let mut words = expand(&entry.bitmap)?;
4408 if entry.xor_offset > 0 {
4409 let base_index = entry_index - entry.xor_offset as usize;
4410 let base = &resolved[base_index];
4411 for (dst, src) in words.iter_mut().zip(base.iter()) {
4412 *dst ^= *src;
4413 }
4414 }
4415 let words = Arc::new(words);
4416 resolved.push(Arc::clone(&words));
4417 let commit_oid = lookup_oid(entry.object_position as usize)
4418 .ok_or_else(|| GitError::InvalidFormat("bitmap entry position out of range".into()))?;
4419 commit_words.insert(commit_oid, words);
4420 }
4421 let mut pseudo_merges = Vec::with_capacity(parsed.pseudo_merges.len());
4422 for merge in &parsed.pseudo_merges {
4423 pseudo_merges.push(LoadedPseudoMerge {
4424 commits: Arc::new(expand(&merge.commits)?),
4425 bitmap: Arc::new(expand(&merge.bitmap)?),
4426 });
4427 }
4428
4429 Ok(LoadedPackBitmap {
4430 object_count: object_count as u32,
4431 oid_to_pack,
4432 pack_to_oid,
4433 commit_words,
4434 pseudo_merges,
4435 commits: expand(&parsed.type_bitmaps.commits)?,
4436 trees: expand(&parsed.type_bitmaps.trees)?,
4437 blobs: expand(&parsed.type_bitmaps.blobs)?,
4438 tags: expand(&parsed.type_bitmaps.tags)?,
4439 })
4440}
4441
4442pub struct BitmapWalkResult {
4446 pub words: Vec<u64>,
4447 pub extended: Vec<(ObjectId, ObjectType)>,
4448 pub pseudo_merges_satisfied: usize,
4449 pub pseudo_merges_cascades: usize,
4450}
4451
4452impl BitmapWalkResult {
4453 pub fn subtract(&mut self, haves: &BitmapWalkResult) {
4455 for (dst, src) in self.words.iter_mut().zip(haves.words.iter()) {
4456 *dst &= !*src;
4457 }
4458 let have_ext: HashSet<ObjectId> = haves.extended.iter().map(|(oid, _)| *oid).collect();
4459 self.extended.retain(|(oid, _)| !have_ext.contains(oid));
4460 }
4461}
4462
4463pub fn bitmap_reachable(
4474 bitmap: &LoadedPackBitmap,
4475 db: &impl ObjectReader,
4476 format: ObjectFormat,
4477 roots: &[ObjectId],
4478 include_objects: bool,
4479) -> Result<BitmapWalkResult> {
4480 let mut walk = BitmapFillWalk {
4481 bitmap,
4482 words: vec![0u64; bitmap.word_count()],
4483 extended: Vec::new(),
4484 extended_seen: HashSet::new(),
4485 };
4486 let mut commit_stack: Vec<ObjectId> = Vec::new();
4487
4488 for root in roots {
4489 let mut oid = *root;
4490 loop {
4492 let object = db.read_object(&oid)?;
4493 match object.object_type {
4494 ObjectType::Tag => {
4495 walk.mark(&oid, ObjectType::Tag);
4496 let tag = Tag::parse_ref(format, &object.body)?;
4497 oid = tag.object;
4498 }
4499 ObjectType::Commit => {
4500 commit_stack.push(oid);
4501 break;
4502 }
4503 ObjectType::Tree => {
4504 walk.mark_tree_closure(db, format, &oid)?;
4505 break;
4506 }
4507 ObjectType::Blob => {
4508 walk.mark(&oid, ObjectType::Blob);
4509 break;
4510 }
4511 }
4512 }
4513 }
4514
4515 while let Some(oid) = commit_stack.pop() {
4516 if let Some(position) = bitmap.pack_position(&oid) {
4517 if bitset_get(&walk.words, position) {
4518 continue;
4519 }
4520 if let Some(stored) = bitmap.bitmap_for_commit(&oid) {
4521 bitset_or(&mut walk.words, stored);
4522 continue;
4523 }
4524 bitset_set(&mut walk.words, position);
4525 } else {
4526 if walk.extended_seen.contains(&oid) {
4527 continue;
4528 }
4529 walk.extended_seen.insert(oid);
4530 walk.extended.push((oid, ObjectType::Commit));
4531 }
4532 let object = db.read_object(&oid)?;
4533 let commit = Commit::parse_ref(format, &object.body)?;
4534 commit_stack.extend(grafted_parents(db, &oid, commit.parents));
4535 if include_objects {
4536 walk.mark_tree_closure(db, format, &commit.tree)?;
4537 }
4538 }
4539
4540 let (pseudo_merges_satisfied, pseudo_merges_cascades) =
4541 bitmap_cascade_pseudo_merges(bitmap, &mut walk.words);
4542
4543 Ok(BitmapWalkResult {
4544 words: walk.words,
4545 extended: walk.extended,
4546 pseudo_merges_satisfied,
4547 pseudo_merges_cascades,
4548 })
4549}
4550
4551fn bitmap_cascade_pseudo_merges(bitmap: &LoadedPackBitmap, words: &mut [u64]) -> (usize, usize) {
4552 if bitmap.pseudo_merges.is_empty() {
4553 return (0, 0);
4554 }
4555 let mut satisfied = vec![false; bitmap.pseudo_merges.len()];
4556 let mut total = 0usize;
4557 loop {
4558 let mut any = false;
4559 for (index, merge) in bitmap.pseudo_merges.iter().enumerate() {
4560 if satisfied[index] || !bitset_is_subset(merge.commits.as_slice(), words) {
4561 continue;
4562 }
4563 bitset_or(words, merge.bitmap.as_slice());
4564 satisfied[index] = true;
4565 any = true;
4566 total += 1;
4567 }
4568 if !any {
4569 break;
4570 }
4571 }
4572 (total, usize::from(total > 0))
4573}
4574
4575struct BitmapFillWalk<'a> {
4576 bitmap: &'a LoadedPackBitmap,
4577 words: Vec<u64>,
4578 extended: Vec<(ObjectId, ObjectType)>,
4579 extended_seen: HashSet<ObjectId>,
4580}
4581
4582impl BitmapFillWalk<'_> {
4583 fn mark(&mut self, oid: &ObjectId, object_type: ObjectType) -> bool {
4585 if let Some(position) = self.bitmap.pack_position(oid) {
4586 if bitset_get(&self.words, position) {
4587 return false;
4588 }
4589 bitset_set(&mut self.words, position);
4590 true
4591 } else {
4592 if !self.extended_seen.insert(*oid) {
4593 return false;
4594 }
4595 self.extended.push((*oid, object_type));
4596 true
4597 }
4598 }
4599
4600 fn mark_tree_closure(
4604 &mut self,
4605 db: &impl ObjectReader,
4606 format: ObjectFormat,
4607 tree: &ObjectId,
4608 ) -> Result<()> {
4609 if !self.mark(tree, ObjectType::Tree) {
4610 return Ok(());
4611 }
4612 let object = db.read_object(tree)?;
4613 for entry in TreeEntries::new(format, &object.body) {
4614 let entry = entry?;
4615 if entry.is_gitlink() {
4616 continue;
4617 }
4618 if entry.is_tree() {
4619 self.mark_tree_closure(db, format, &entry.oid)?;
4620 } else {
4621 self.mark(&entry.oid, ObjectType::Blob);
4622 }
4623 }
4624 Ok(())
4625 }
4626}
4627
4628#[derive(Debug)]
4629pub struct ObjectDatabase {
4630 format: ObjectFormat,
4631 objects: Mutex<HashMap<ObjectId, Arc<EncodedObject>>>,
4637 promisor: bool,
4638}
4639
4640impl ObjectDatabase {
4641 pub fn new(format: ObjectFormat) -> Self {
4642 Self {
4643 format,
4644 objects: Mutex::new(HashMap::new()),
4645 promisor: false,
4646 }
4647 }
4648
4649 pub fn with_promisor(mut self, promisor: bool) -> Self {
4650 self.promisor = promisor;
4651 self
4652 }
4653
4654 pub fn contains(&self, oid: &ObjectId) -> bool {
4655 self.objects
4656 .lock()
4657 .map(|objects| objects.contains_key(oid))
4658 .unwrap_or(false)
4659 }
4660
4661 pub fn validate(&self, oid: &ObjectId) -> Result<()> {
4662 let object = self.read_object(oid)?;
4663 let actual = object.object_id(self.format)?;
4664 if &actual == oid {
4665 Ok(())
4666 } else {
4667 Err(GitError::InvalidObject(format!(
4668 "object id mismatch: expected {oid}, got {actual}"
4669 )))
4670 }
4671 }
4672}
4673
4674impl ObjectReader for ObjectDatabase {
4675 fn read_object(&self, oid: &ObjectId) -> Result<Arc<EncodedObject>> {
4676 self.objects
4677 .lock()
4678 .map_err(|_| GitError::object_not_found_in(*oid, MissingObjectContext::Read))?
4679 .get(oid)
4680 .map(Arc::clone)
4681 .or_else(|| implied_empty_tree_object(self.format, oid))
4682 .ok_or_else(|| GitError::object_not_found_in(*oid, MissingObjectContext::Read))
4683 }
4684}
4685
4686impl ObjectWriter for ObjectDatabase {
4687 fn write_object(&self, object: EncodedObject) -> Result<ObjectId> {
4688 let oid = object.object_id(self.format)?;
4689 self.objects
4690 .lock()
4691 .map_err(|_| GitError::Io("object cache lock poisoned".into()))?
4692 .entry(oid)
4693 .or_insert_with(|| Arc::new(object));
4694 Ok(oid)
4695 }
4696}
4697
4698#[derive(Debug, Clone, PartialEq, Eq)]
4699pub struct Alternate {
4700 pub path: std::path::PathBuf,
4701}
4702
4703#[derive(Debug, Clone, PartialEq, Eq)]
4704pub struct PartialClonePolicy {
4705 pub promisor_remote: Option<String>,
4706 pub allow_missing_promised_objects: bool,
4707}
4708
4709type PackBytesCache = Arc<Mutex<HashMap<PathBuf, Arc<PackData>>>>;
4713
4714#[derive(Debug)]
4717enum PackData {
4718 #[cfg(feature = "mmap")]
4719 Mapped(sley_mmap::MappedFile),
4720 Heap(Vec<u8>),
4721}
4722
4723impl std::ops::Deref for PackData {
4724 type Target = [u8];
4725
4726 fn deref(&self) -> &[u8] {
4727 match self {
4728 #[cfg(feature = "mmap")]
4729 Self::Mapped(mapped) => mapped,
4730 Self::Heap(bytes) => bytes,
4731 }
4732 }
4733}
4734
4735#[cfg(feature = "mmap")]
4738fn load_pack_data(pack_path: &Path) -> Result<PackData> {
4739 match sley_mmap::MappedFile::open_pack(pack_path) {
4740 Ok(mapped) => Ok(PackData::Mapped(mapped)),
4741 Err(_) => Ok(PackData::Heap(fs::read(pack_path)?)),
4742 }
4743}
4744
4745#[cfg(not(feature = "mmap"))]
4746fn load_pack_data(pack_path: &Path) -> Result<PackData> {
4747 Ok(PackData::Heap(fs::read(pack_path)?))
4748}
4749
4750#[cfg(feature = "mmap")]
4751fn load_pack_index_data(index_path: &Path) -> Result<Arc<dyn PackIndexByteSource>> {
4752 match sley_mmap::MappedFile::open_pack(index_path) {
4753 Ok(mapped) => Ok(Arc::new(mapped)),
4754 Err(_) => Ok(Arc::new(fs::read(index_path)?)),
4755 }
4756}
4757
4758#[cfg(not(feature = "mmap"))]
4759fn load_pack_index_data(index_path: &Path) -> Result<Arc<dyn PackIndexByteSource>> {
4760 Ok(Arc::new(fs::read(index_path)?))
4761}
4762
4763#[cfg(feature = "mmap")]
4764fn load_multi_pack_index_lookup_data(midx_path: &Path) -> Result<Arc<dyn PackIndexByteSource>> {
4765 match sley_mmap::MappedFile::open_multi_pack_index(midx_path) {
4766 Ok(mapped) => Ok(Arc::new(mapped)),
4767 Err(_) => Ok(Arc::new(fs::read(midx_path)?)),
4768 }
4769}
4770
4771#[cfg(not(feature = "mmap"))]
4772fn load_multi_pack_index_lookup_data(midx_path: &Path) -> Result<Arc<dyn PackIndexByteSource>> {
4773 Ok(Arc::new(fs::read(midx_path)?))
4774}
4775
4776type DecodedObjectCache = Arc<Mutex<LruObjectCache>>;
4782
4783type PackDeltaCaches = Arc<Mutex<HashMap<PathBuf, Arc<Mutex<LruOffsetCache>>>>>;
4788
4789type PackHeaderTypeCache = Arc<Mutex<HashMap<u64, (ObjectType, u64)>>>;
4799
4800type PackHeaderTypeCaches = Arc<Mutex<HashMap<PathBuf, PackHeaderTypeCache>>>;
4801
4802const DEFAULT_OBJECT_CACHE_BYTES: usize = 96 * 1024 * 1024;
4808
4809const DEFAULT_DELTA_BASE_CACHE_BYTES: usize = 96 * 1024 * 1024;
4813
4814fn cached_object_cost(object: &EncodedObject) -> usize {
4818 object.body.len().saturating_add(64)
4819}
4820
4821fn cache_budget_from_env(var: &str, default: usize) -> usize {
4824 match env::var(var) {
4825 Ok(value) => value.trim().parse::<usize>().unwrap_or(default),
4826 Err(_) => default,
4827 }
4828}
4829
4830fn object_cache_budget() -> usize {
4837 static BUDGET: OnceLock<usize> = OnceLock::new();
4838 *BUDGET.get_or_init(|| {
4839 cache_budget_from_env("SLEY_OBJECT_CACHE_BYTES", DEFAULT_OBJECT_CACHE_BYTES)
4840 })
4841}
4842
4843fn delta_base_cache_budget() -> usize {
4847 static BUDGET: OnceLock<usize> = OnceLock::new();
4848 *BUDGET.get_or_init(|| {
4849 cache_budget_from_env(
4850 "SLEY_DELTA_BASE_CACHE_BYTES",
4851 DEFAULT_DELTA_BASE_CACHE_BYTES,
4852 )
4853 })
4854}
4855
4856fn verify_reads_enabled() -> bool {
4867 static VERIFY: OnceLock<bool> = OnceLock::new();
4868 *VERIFY.get_or_init(|| match env::var("SLEY_VERIFY_READS") {
4869 Ok(value) => !matches!(value.trim(), "" | "0"),
4870 Err(_) => false,
4871 })
4872}
4873
4874#[derive(Debug)]
4882struct LruCache<K: std::hash::Hash + Eq + Clone> {
4883 budget: usize,
4884 used: usize,
4885 map: HashMap<K, LruEntry<K>>,
4886 head: Option<K>,
4887 tail: Option<K>,
4888}
4889
4890#[derive(Debug)]
4891struct LruEntry<K> {
4892 object: Arc<EncodedObject>,
4893 prev: Option<K>,
4894 next: Option<K>,
4895}
4896
4897impl<K: std::hash::Hash + Eq + Clone> LruCache<K> {
4898 fn new(budget: usize) -> Self {
4899 Self {
4900 budget,
4901 used: 0,
4902 map: HashMap::new(),
4903 head: None,
4904 tail: None,
4905 }
4906 }
4907
4908 fn get(&mut self, key: &K) -> Option<Arc<EncodedObject>> {
4909 let object = Arc::clone(&self.map.get(key)?.object);
4910 self.touch(key);
4911 Some(object)
4912 }
4913
4914 fn touch(&mut self, key: &K) {
4916 if self.tail.as_ref() == Some(key) {
4917 return;
4918 }
4919 if self.map.contains_key(key) {
4920 self.detach(key);
4921 self.attach_back(key.clone());
4922 }
4923 }
4924
4925 fn remove(&mut self, key: &K) {
4927 if let Some(entry) = self.map.get(key) {
4928 self.used = self.used.saturating_sub(cached_object_cost(&entry.object));
4929 }
4930 self.detach(key);
4931 self.map.remove(key);
4932 }
4933
4934 fn detach(&mut self, key: &K) {
4935 let Some((prev, next)) = self.map.get_mut(key).map(|entry| {
4936 let prev = entry.prev.take();
4937 let next = entry.next.take();
4938 (prev, next)
4939 }) else {
4940 return;
4941 };
4942
4943 match &prev {
4944 Some(prev_key) => {
4945 if let Some(prev_entry) = self.map.get_mut(prev_key) {
4946 prev_entry.next = next.clone();
4947 }
4948 }
4949 None => self.head = next.clone(),
4950 }
4951 match &next {
4952 Some(next_key) => {
4953 if let Some(next_entry) = self.map.get_mut(next_key) {
4954 next_entry.prev = prev.clone();
4955 }
4956 }
4957 None => self.tail = prev.clone(),
4958 }
4959 }
4960
4961 fn attach_back(&mut self, key: K) {
4962 let previous_tail = self.tail.replace(key.clone());
4963 match previous_tail {
4964 Some(tail_key) => {
4965 if let Some(tail_entry) = self.map.get_mut(&tail_key) {
4966 tail_entry.next = Some(key.clone());
4967 }
4968 if let Some(entry) = self.map.get_mut(&key) {
4969 entry.prev = Some(tail_key);
4970 entry.next = None;
4971 }
4972 }
4973 None => {
4974 self.head = Some(key.clone());
4975 if let Some(entry) = self.map.get_mut(&key) {
4976 entry.prev = None;
4977 entry.next = None;
4978 }
4979 }
4980 }
4981 }
4982
4983 fn clear(&mut self) {
4984 self.map.clear();
4985 self.head = None;
4986 self.tail = None;
4987 self.used = 0;
4988 }
4989
4990 fn put(&mut self, key: K, object: Arc<EncodedObject>) {
4991 if self.budget == 0 {
4992 return;
4993 }
4994 let cost = cached_object_cost(&object);
4995 if cost > self.budget {
4999 self.remove(&key);
5000 return;
5001 }
5002 if let Some(entry) = self.map.get_mut(&key) {
5003 let previous = std::mem::replace(&mut entry.object, object);
5004 self.used = self
5006 .used
5007 .saturating_sub(cached_object_cost(&previous))
5008 .saturating_add(cost);
5009 self.touch(&key);
5010 } else {
5011 self.used = self.used.saturating_add(cost);
5012 self.map.insert(
5013 key.clone(),
5014 LruEntry {
5015 object,
5016 prev: None,
5017 next: None,
5018 },
5019 );
5020 self.attach_back(key);
5021 }
5022 while self.used > self.budget {
5023 let Some(evicted) = self.head.clone() else {
5024 break;
5025 };
5026 self.remove(&evicted);
5027 }
5028 }
5029}
5030
5031type LruObjectCache = LruCache<ObjectId>;
5033type LruOffsetCache = LruCache<u64>;
5035
5036struct PackDeltaCacheAdapter<'a>(&'a Arc<Mutex<LruOffsetCache>>);
5041
5042impl sley_pack::PackDeltaCache for PackDeltaCacheAdapter<'_> {
5043 fn get(&self, offset: u64) -> Option<Arc<EncodedObject>> {
5044 self.0.lock().ok()?.get(&offset)
5045 }
5046
5047 fn insert(&self, offset: u64, object: Arc<EncodedObject>) {
5048 if let Ok(mut cache) = self.0.lock() {
5049 cache.put(offset, object);
5050 }
5051 }
5052}
5053
5054struct PackHeaderTypeCacheAdapter<'a>(&'a PackHeaderTypeCache);
5058
5059impl sley_pack::HeaderTypeCache for PackHeaderTypeCacheAdapter<'_> {
5060 fn get(&self, pack_offset: u64) -> Option<(ObjectType, u64)> {
5061 self.0.lock().ok()?.get(&pack_offset).copied()
5062 }
5063
5064 fn put(&mut self, pack_offset: u64, header: (ObjectType, u64)) {
5065 if let Ok(mut cache) = self.0.lock() {
5066 cache.insert(pack_offset, header);
5067 }
5068 }
5069}
5070
5071type PackIndexCache = Arc<Mutex<HashMap<PathBuf, Arc<PackIndex>>>>;
5076
5077type MultiPackIndexCache = Arc<Mutex<HashMap<PathBuf, Arc<MultiPackIndex>>>>;
5081
5082type MultiPackIndexOidLookupCache = Arc<Mutex<HashMap<PathBuf, Arc<MultiPackIndexOidLookup>>>>;
5086
5087#[derive(Debug)]
5092struct RegisteredPack {
5093 idx: PathBuf,
5094 pack: PathBuf,
5095 index: Mutex<Option<Arc<PackIndexViewData>>>,
5096 data: Mutex<Option<Arc<PackData>>>,
5097 delta_cache: Arc<Mutex<LruOffsetCache>>,
5098 header_type_cache: PackHeaderTypeCache,
5099}
5100
5101impl RegisteredPack {
5102 fn new(idx: PathBuf, pack: PathBuf) -> Self {
5103 Self {
5104 idx,
5105 pack,
5106 index: Mutex::new(None),
5107 data: Mutex::new(None),
5108 delta_cache: Arc::new(Mutex::new(LruOffsetCache::new(delta_base_cache_budget()))),
5109 header_type_cache: Arc::new(Mutex::new(HashMap::new())),
5110 }
5111 }
5112
5113 fn index(&self, format: ObjectFormat) -> Result<Arc<PackIndexViewData>> {
5114 if let Ok(cache) = self.index.lock()
5115 && let Some(index) = cache.as_ref()
5116 {
5117 return Ok(Arc::clone(index));
5118 }
5119 let index_bytes = load_pack_index_data(&self.idx)?;
5120 let index = Arc::new(PackIndexViewData::parse_trusted_source_without_checksum(
5121 index_bytes,
5122 format,
5123 )?);
5124 if let Ok(mut cache) = self.index.lock() {
5125 *cache = Some(Arc::clone(&index));
5126 }
5127 Ok(index)
5128 }
5129
5130 fn bytes(&self, pack_bytes: &PackBytesCache) -> Result<Arc<PackData>> {
5131 if let Ok(cache) = self.data.lock()
5132 && let Some(bytes) = cache.as_ref()
5133 {
5134 return Ok(Arc::clone(bytes));
5135 }
5136 if let Ok(cache) = pack_bytes.lock()
5137 && let Some(bytes) = cache.get(&self.pack)
5138 {
5139 let bytes = Arc::clone(bytes);
5140 if let Ok(mut local_cache) = self.data.lock() {
5141 *local_cache = Some(Arc::clone(&bytes));
5142 }
5143 return Ok(bytes);
5144 }
5145 let bytes = Arc::new(load_pack_data(&self.pack)?);
5146 if let Ok(mut local_cache) = self.data.lock() {
5147 *local_cache = Some(Arc::clone(&bytes));
5148 }
5149 if let Ok(mut cache) = pack_bytes.lock() {
5150 cache.insert(self.pack.clone(), Arc::clone(&bytes));
5151 }
5152 Ok(bytes)
5153 }
5154}
5155
5156#[derive(Debug, Clone, PartialEq, Eq)]
5157struct PackDirFingerprint {
5158 modified: Option<std::time::SystemTime>,
5159 idx_count: usize,
5160 pack_count: usize,
5161}
5162
5163#[derive(Debug)]
5168struct PackRegistrySnapshot {
5169 fingerprint: PackDirFingerprint,
5170 packs: Vec<Arc<RegisteredPack>>,
5171 recent_pack: Mutex<Option<usize>>,
5172}
5173
5174impl PackRegistrySnapshot {
5175 fn new(fingerprint: PackDirFingerprint, packs: Vec<Arc<RegisteredPack>>) -> Self {
5176 Self {
5177 fingerprint,
5178 packs,
5179 recent_pack: Mutex::new(None),
5180 }
5181 }
5182
5183 fn cached_hint(&self) -> Option<usize> {
5184 self.recent_pack
5185 .lock()
5186 .ok()
5187 .and_then(|hint| *hint)
5188 .filter(|pack_index| *pack_index < self.packs.len())
5189 }
5190
5191 fn remember_hint(&self, pack_index: usize) {
5192 if let Ok(mut hint) = self.recent_pack.lock() {
5193 *hint = Some(pack_index);
5194 }
5195 }
5196}
5197
5198type PackRegistryCache = Arc<Mutex<Option<Arc<PackRegistrySnapshot>>>>;
5202
5203#[derive(Debug, Clone)]
5204struct PackLookup {
5205 pack: PathBuf,
5206 registered: Option<Arc<RegisteredPack>>,
5207 offset: u64,
5208}
5209
5210impl PackLookup {
5211 fn from_registered(pack: Arc<RegisteredPack>, offset: u64) -> Self {
5212 Self {
5213 pack: pack.pack.clone(),
5214 registered: Some(pack),
5215 offset,
5216 }
5217 }
5218
5219 fn from_path(pack: PathBuf, offset: u64) -> Self {
5220 Self {
5221 pack,
5222 registered: None,
5223 offset,
5224 }
5225 }
5226
5227 fn pack_path(&self) -> &Path {
5228 &self.pack
5229 }
5230
5231 fn pack_bytes(&self, database: &FileObjectDatabase) -> Result<Arc<PackData>> {
5232 match &self.registered {
5233 Some(pack) => pack.bytes(&database.pack_bytes),
5234 None => database.cached_pack_bytes(&self.pack),
5235 }
5236 }
5237
5238 fn pack_index(&self, database: &FileObjectDatabase) -> Result<Arc<PackIndex>> {
5239 match &self.registered {
5240 Some(pack) => database.cached_pack_index(&pack.idx),
5241 None => database.cached_pack_index(&self.pack.with_extension("idx")),
5242 }
5243 }
5244
5245 fn delta_cache(&self, database: &FileObjectDatabase) -> Option<Arc<Mutex<LruOffsetCache>>> {
5246 match &self.registered {
5247 Some(pack) => Some(Arc::clone(&pack.delta_cache)),
5248 None => database.pack_delta_cache(&self.pack),
5249 }
5250 }
5251
5252 fn header_type_cache(&self, database: &FileObjectDatabase) -> Option<PackHeaderTypeCache> {
5253 match &self.registered {
5254 Some(pack) => Some(Arc::clone(&pack.header_type_cache)),
5255 None => database.pack_header_type_cache(&self.pack),
5256 }
5257 }
5258}
5259
5260#[derive(Debug, Clone)]
5261pub struct FileObjectDatabase {
5262 loose: LooseObjectStore,
5263 objects_dir: PathBuf,
5264 alternates: Vec<PathBuf>,
5265 format: ObjectFormat,
5266 pack_bytes: PackBytesCache,
5267 pack_indexes: PackIndexCache,
5268 multi_pack_indexes: MultiPackIndexCache,
5269 multi_pack_oid_lookups: MultiPackIndexOidLookupCache,
5270 pack_registry: PackRegistryCache,
5271 decoded: DecodedObjectCache,
5272 pack_deltas: PackDeltaCaches,
5273 pack_header_types: PackHeaderTypeCaches,
5274 promisor_objects: Arc<OnceLock<HashSet<ObjectId>>>,
5275 promisor_remote_present: bool,
5283 shallow_grafts: Arc<std::sync::OnceLock<HashSet<ObjectId>>>,
5287}
5288
5289#[derive(Debug)]
5290pub struct ObjectPresenceChecker {
5291 db: FileObjectDatabase,
5292 pack_dir: PathBuf,
5293 midx: Option<Arc<MultiPackIndexOidLookup>>,
5294 registry: Option<Arc<PackRegistrySnapshot>>,
5295 registry_indexes: Vec<Option<Arc<PackIndexViewData>>>,
5296 recent_pack: Option<usize>,
5297 prepared_packs: bool,
5298 prepared_registry: bool,
5299}
5300
5301impl ObjectPresenceChecker {
5302 fn new(db: FileObjectDatabase) -> Self {
5303 let pack_dir = db.objects_dir.join("pack");
5304 Self {
5305 db,
5306 pack_dir,
5307 midx: None,
5308 registry: None,
5309 registry_indexes: Vec::new(),
5310 recent_pack: None,
5311 prepared_packs: false,
5312 prepared_registry: false,
5313 }
5314 }
5315
5316 pub fn contains(&mut self, oid: &ObjectId) -> Result<bool> {
5317 if oid.format() != self.db.format {
5318 return Err(GitError::InvalidObjectId(format!(
5319 "object {oid} uses {}, store uses {}",
5320 oid.format().name(),
5321 self.db.format.name()
5322 )));
5323 }
5324 if self.db.loose.exists(oid)? {
5325 return Ok(true);
5326 }
5327 if self.find_packed(oid, false)? {
5328 return Ok(true);
5329 }
5330 if self.find_packed(oid, true)? {
5331 return Ok(true);
5332 }
5333 for alternate in &self.db.alternates {
5334 if FileObjectDatabase::without_alternates(alternate, self.db.format).contains(oid)? {
5335 return Ok(true);
5336 }
5337 }
5338 self.db.loose.invalidate_cache();
5341 self.db.loose.exists(oid)
5342 }
5343
5344 fn find_packed(&mut self, oid: &ObjectId, force_rescan: bool) -> Result<bool> {
5345 self.prepare_packs(force_rescan)?;
5346 if let Some(midx) = &self.midx
5347 && midx.contains(oid)
5348 {
5349 return Ok(true);
5350 }
5351 self.prepare_registry(force_rescan)?;
5352 self.find_in_registry(oid)
5353 }
5354
5355 fn prepare_packs(&mut self, force_rescan: bool) -> Result<()> {
5356 if self.prepared_packs && !force_rescan {
5357 return Ok(());
5358 }
5359 let midx_path = self.pack_dir.join("multi-pack-index");
5360 self.midx = self.db.cached_multi_pack_index_oid_lookup(&midx_path)?;
5361 self.prepared_packs = true;
5362 Ok(())
5363 }
5364
5365 fn prepare_registry(&mut self, force_rescan: bool) -> Result<()> {
5366 if self.prepared_registry && !force_rescan {
5367 return Ok(());
5368 }
5369 let registry = self.db.cached_pack_registry(&self.pack_dir, force_rescan)?;
5370 let registry_changed = match self.registry.as_ref() {
5371 Some(cached) => !Arc::ptr_eq(cached, ®istry),
5372 None => true,
5373 };
5374 if registry_changed {
5375 self.registry_indexes = vec![None; registry.packs.len()];
5376 self.recent_pack = None;
5377 self.registry = Some(registry);
5378 }
5379 self.prepared_registry = true;
5380 Ok(())
5381 }
5382
5383 fn find_in_registry(&mut self, oid: &ObjectId) -> Result<bool> {
5384 let Some(registry) = self.registry.as_ref().map(Arc::clone) else {
5385 return Ok(false);
5386 };
5387 if let Some(pack_index) = self
5388 .recent_pack
5389 .filter(|pack_index| *pack_index < registry.packs.len())
5390 {
5391 let index = self.registry_index(®istry, pack_index)?;
5392 if index.find(oid).is_some() {
5393 return Ok(true);
5394 }
5395 }
5396 for pack_index in 0..registry.packs.len() {
5397 if Some(pack_index) == self.recent_pack {
5398 continue;
5399 }
5400 let index = self.registry_index(®istry, pack_index)?;
5401 if index.find(oid).is_some() {
5402 self.recent_pack = Some(pack_index);
5403 return Ok(true);
5404 }
5405 }
5406 Ok(false)
5407 }
5408
5409 fn registry_index(
5410 &mut self,
5411 registry: &PackRegistrySnapshot,
5412 pack_index: usize,
5413 ) -> Result<Arc<PackIndexViewData>> {
5414 if self.registry_indexes.len() != registry.packs.len() {
5415 self.registry_indexes = vec![None; registry.packs.len()];
5416 self.recent_pack = None;
5417 }
5418 if let Some(index) = self
5419 .registry_indexes
5420 .get(pack_index)
5421 .and_then(|index| index.as_ref())
5422 {
5423 return Ok(Arc::clone(index));
5424 }
5425 let index = registry.packs[pack_index].index(self.db.format)?;
5426 if let Some(slot) = self.registry_indexes.get_mut(pack_index) {
5427 *slot = Some(Arc::clone(&index));
5428 }
5429 Ok(index)
5430 }
5431}
5432
5433fn read_shallow_grafts(shallow_file: &Path, format: ObjectFormat) -> HashSet<ObjectId> {
5437 let Ok(contents) = std::fs::read_to_string(shallow_file) else {
5438 return HashSet::new();
5439 };
5440 contents
5441 .lines()
5442 .filter_map(|line| ObjectId::from_hex(format, line.trim()).ok())
5443 .collect()
5444}
5445
5446pub fn repository_objects_dir(git_dir: impl AsRef<Path>) -> PathBuf {
5447 env::var_os("GIT_OBJECT_DIRECTORY")
5448 .map(PathBuf::from)
5449 .unwrap_or_else(|| repository_common_dir(git_dir).join("objects"))
5450}
5451
5452pub fn repository_common_dir(git_dir: impl AsRef<Path>) -> PathBuf {
5453 if let Some(common_dir) = env::var_os("GIT_COMMON_DIR") {
5454 return PathBuf::from(common_dir);
5455 }
5456 let git_dir = git_dir.as_ref();
5457 let commondir = git_dir.join("commondir");
5458 if let Ok(value) = fs::read_to_string(&commondir) {
5459 let path = PathBuf::from(value.trim());
5460 let common = if path.is_absolute() {
5461 path
5462 } else {
5463 git_dir.join(path)
5464 };
5465 return fs::canonicalize(&common).unwrap_or(common);
5466 }
5467 git_dir.to_path_buf()
5468}
5469
5470pub fn repository_object_ids(
5471 git_dir: impl AsRef<Path>,
5472 format: ObjectFormat,
5473) -> Result<Vec<ObjectId>> {
5474 object_ids_in_objects_dir(repository_objects_dir(git_dir), format)
5475}
5476
5477pub fn object_ids_in_objects_dir(
5478 objects_dir: impl AsRef<Path>,
5479 format: ObjectFormat,
5480) -> Result<Vec<ObjectId>> {
5481 let objects_dir = objects_dir.as_ref();
5482 let mut oids = HashSet::new();
5483 collect_loose_object_ids(objects_dir, format, &mut oids)?;
5484 collect_packed_object_ids(&objects_dir.join("pack"), format, &mut oids)?;
5485 let mut oids = oids.into_iter().collect::<Vec<_>>();
5486 oids.sort_by_key(ObjectId::to_hex);
5487 Ok(oids)
5488}
5489
5490fn collect_loose_object_ids(
5491 objects_dir: &Path,
5492 format: ObjectFormat,
5493 oids: &mut HashSet<ObjectId>,
5494) -> Result<()> {
5495 if !objects_dir.exists() {
5496 return Ok(());
5497 }
5498 let hex_len = format.hex_len();
5499 for entry in fs::read_dir(objects_dir)? {
5500 let entry = entry?;
5501 if !entry.file_type()?.is_dir() {
5502 continue;
5503 }
5504 let name = entry.file_name();
5505 let Some(fanout) = name.to_str() else {
5506 continue;
5507 };
5508 if fanout.len() != 2 || !fanout.bytes().all(|byte| byte.is_ascii_hexdigit()) {
5509 continue;
5510 }
5511 for object_entry in fs::read_dir(entry.path())? {
5512 let object_entry = object_entry?;
5513 if !object_entry.file_type()?.is_file() {
5514 continue;
5515 }
5516 let name = object_entry.file_name();
5517 let Some(suffix) = name.to_str() else {
5518 continue;
5519 };
5520 if suffix.len() != hex_len - 2 || !suffix.bytes().all(|byte| byte.is_ascii_hexdigit()) {
5521 continue;
5522 }
5523 oids.insert(ObjectId::from_hex(format, &format!("{fanout}{suffix}"))?);
5524 }
5525 }
5526 Ok(())
5527}
5528
5529fn collect_loose_fanout_object_ids(
5530 objects_dir: &Path,
5531 format: ObjectFormat,
5532 fanout: u8,
5533 oids: &mut HashSet<ObjectId>,
5534) -> Result<()> {
5535 let fanout_hex = format!("{fanout:02x}");
5536 let fanout_dir = objects_dir.join(&fanout_hex);
5537 let entries = match fs::read_dir(&fanout_dir) {
5538 Ok(entries) => entries,
5539 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
5540 Err(err) => return Err(GitError::Io(err.to_string())),
5541 };
5542 let hex_len = format.hex_len();
5543 for object_entry in entries {
5544 let object_entry = object_entry?;
5545 let name = object_entry.file_name();
5546 let Some(suffix) = name.to_str() else {
5547 continue;
5548 };
5549 if suffix.len() != hex_len - 2 || !suffix.bytes().all(|byte| byte.is_ascii_hexdigit()) {
5550 continue;
5551 }
5552 oids.insert(ObjectId::from_hex(
5553 format,
5554 &format!("{fanout_hex}{suffix}"),
5555 )?);
5556 }
5557 Ok(())
5558}
5559
5560fn present_loose_fanouts(objects_dir: &Path) -> Result<HashSet<u8>> {
5568 let mut present = HashSet::new();
5569 let entries = match fs::read_dir(objects_dir) {
5570 Ok(entries) => entries,
5571 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(present),
5572 Err(err) => return Err(GitError::Io(err.to_string())),
5573 };
5574 for entry in entries {
5575 let entry = entry?;
5576 let name = entry.file_name();
5577 let Some(name) = name.to_str() else {
5578 continue;
5579 };
5580 if name.len() != 2 {
5581 continue;
5582 }
5583 let mut bytes = name.bytes();
5584 let (Some(hi), Some(lo)) = (bytes.next(), bytes.next()) else {
5585 continue;
5586 };
5587 let (Some(hi), Some(lo)) = ((hi as char).to_digit(16), (lo as char).to_digit(16)) else {
5588 continue;
5589 };
5590 if entry.file_type().map(|ft| ft.is_dir()).unwrap_or(false) {
5594 present.insert(((hi << 4) | lo) as u8);
5595 }
5596 }
5597 Ok(present)
5598}
5599
5600#[derive(Debug, Default)]
5601struct LoosePresenceCache {
5602 loaded_fanouts: HashSet<u8>,
5604 objects: HashSet<ObjectId>,
5605 present_fanouts: Option<HashSet<u8>>,
5610}
5611
5612pub fn packed_object_ids(
5617 objects_dir: impl AsRef<Path>,
5618 format: ObjectFormat,
5619) -> Result<HashSet<ObjectId>> {
5620 let mut oids = HashSet::new();
5621 collect_packed_object_ids(&objects_dir.as_ref().join("pack"), format, &mut oids)?;
5622 Ok(oids)
5623}
5624
5625pub fn kept_pack_object_ids(
5626 objects_dir: impl AsRef<Path>,
5627 format: ObjectFormat,
5628) -> Result<HashSet<ObjectId>> {
5629 let pack_dir = objects_dir.as_ref().join("pack");
5630 let mut oids = HashSet::new();
5631 if !pack_dir.exists() {
5632 return Ok(oids);
5633 }
5634 for entry in fs::read_dir(pack_dir)? {
5635 let path = entry?.path();
5636 if path.extension().and_then(|ext| ext.to_str()) != Some("idx") {
5637 continue;
5638 }
5639 if !path.with_extension("pack").exists() || !path.with_extension("keep").exists() {
5640 continue;
5641 }
5642 let index = PackIndex::parse(&fs::read(path)?, format)?;
5643 oids.extend(index.entries.into_iter().map(|entry| entry.oid));
5644 }
5645 Ok(oids)
5646}
5647
5648fn collect_packed_object_ids(
5649 pack_dir: &Path,
5650 format: ObjectFormat,
5651 oids: &mut HashSet<ObjectId>,
5652) -> Result<()> {
5653 if !pack_dir.exists() {
5654 return Ok(());
5655 }
5656 let mut midx_pack_names = HashSet::new();
5657 let midx_path = pack_dir.join("multi-pack-index");
5658 if midx_path.exists() {
5659 let midx = MultiPackIndex::parse_without_checksum(&fs::read(&midx_path)?, format)?;
5660 midx_pack_names.extend(midx.pack_names.iter().cloned());
5661 oids.extend(midx.objects.into_iter().map(|entry| entry.oid));
5662 }
5663 collect_incremental_midx_object_ids(pack_dir, format, oids)?;
5664 for entry in fs::read_dir(pack_dir)? {
5665 let path = entry?.path();
5666 if path.extension().and_then(|ext| ext.to_str()) != Some("idx") {
5667 continue;
5668 }
5669 if !path.with_extension("pack").exists() {
5670 continue;
5671 }
5672 let index = match PackIndex::parse(&fs::read(&path)?, format) {
5673 Ok(index) => index,
5674 Err(_err)
5675 if path
5676 .file_name()
5677 .and_then(|name| name.to_str())
5678 .is_some_and(|name| midx_pack_names.contains(name)) =>
5679 {
5680 eprintln!(
5681 "error: packfile {} index unavailable",
5682 path.with_extension("pack").display()
5683 );
5684 continue;
5685 }
5686 Err(err) => return Err(err),
5687 };
5688 oids.extend(index.entries.into_iter().map(|entry| entry.oid));
5689 }
5690 Ok(())
5691}
5692
5693fn read_incremental_midx_chain(pack_dir: &Path) -> Result<Vec<String>> {
5694 let path = pack_dir
5695 .join("multi-pack-index.d")
5696 .join("multi-pack-index-chain");
5697 let Ok(contents) = fs::read_to_string(path) else {
5698 return Ok(Vec::new());
5699 };
5700 Ok(contents
5701 .lines()
5702 .map(str::trim)
5703 .filter(|line| !line.is_empty())
5704 .map(ToString::to_string)
5705 .collect())
5706}
5707
5708fn collect_incremental_midx_object_ids(
5709 pack_dir: &Path,
5710 format: ObjectFormat,
5711 oids: &mut HashSet<ObjectId>,
5712) -> Result<()> {
5713 let chain = read_incremental_midx_chain(pack_dir)?;
5714 if chain.is_empty() {
5715 return Ok(());
5716 }
5717 let midx_dir = pack_dir.join("multi-pack-index.d");
5718 for checksum in chain {
5719 let path = midx_dir.join(format!("multi-pack-index-{checksum}.midx"));
5720 let midx = MultiPackIndex::parse_without_checksum(&fs::read(path)?, format)?;
5721 oids.extend(midx.objects.into_iter().map(|entry| entry.oid));
5722 }
5723 Ok(())
5724}
5725
5726impl FileObjectDatabase {
5727 pub fn object_format(&self) -> ObjectFormat {
5729 self.format
5730 }
5731
5732 pub fn objects_dir(&self) -> &Path {
5734 &self.objects_dir
5735 }
5736
5737 pub fn new(objects_dir: impl Into<PathBuf>, format: ObjectFormat) -> Self {
5738 let objects_dir = objects_dir.into();
5739 Self {
5740 loose: LooseObjectStore::new(objects_dir.clone(), format),
5741 alternates: alternate_object_dirs(&objects_dir),
5742 objects_dir,
5743 format,
5744 pack_bytes: Arc::new(Mutex::new(HashMap::new())),
5745 pack_indexes: Arc::new(Mutex::new(HashMap::new())),
5746 multi_pack_indexes: Arc::new(Mutex::new(HashMap::new())),
5747 multi_pack_oid_lookups: Arc::new(Mutex::new(HashMap::new())),
5748 pack_registry: Arc::new(Mutex::new(None)),
5749 decoded: Arc::new(Mutex::new(LruObjectCache::new(object_cache_budget()))),
5750 pack_deltas: Arc::new(Mutex::new(HashMap::new())),
5751 pack_header_types: Arc::new(Mutex::new(HashMap::new())),
5752 promisor_objects: Arc::new(OnceLock::new()),
5753 promisor_remote_present: false,
5754 shallow_grafts: Arc::new(std::sync::OnceLock::new()),
5755 }
5756 }
5757
5758 fn without_alternates(objects_dir: impl Into<PathBuf>, format: ObjectFormat) -> Self {
5759 let objects_dir = objects_dir.into();
5760 Self {
5761 loose: LooseObjectStore::new(objects_dir.clone(), format),
5762 alternates: Vec::new(),
5763 objects_dir,
5764 format,
5765 pack_bytes: Arc::new(Mutex::new(HashMap::new())),
5766 pack_indexes: Arc::new(Mutex::new(HashMap::new())),
5767 multi_pack_indexes: Arc::new(Mutex::new(HashMap::new())),
5768 multi_pack_oid_lookups: Arc::new(Mutex::new(HashMap::new())),
5769 pack_registry: Arc::new(Mutex::new(None)),
5770 decoded: Arc::new(Mutex::new(LruObjectCache::new(object_cache_budget()))),
5771 pack_deltas: Arc::new(Mutex::new(HashMap::new())),
5772 pack_header_types: Arc::new(Mutex::new(HashMap::new())),
5773 promisor_objects: Arc::new(OnceLock::new()),
5774 promisor_remote_present: false,
5775 shallow_grafts: Arc::new(std::sync::OnceLock::new()),
5776 }
5777 }
5778
5779 pub fn from_git_dir(git_dir: impl AsRef<Path>, format: ObjectFormat) -> Self {
5780 Self::new(repository_objects_dir(git_dir), format)
5781 }
5782
5783 pub fn with_promisor_remote_present(mut self, present: bool) -> Self {
5792 self.promisor_remote_present = present;
5793 self
5794 }
5795
5796 pub fn refresh_read_cache(&self) {
5801 if let Ok(mut cache) = self.pack_registry.lock() {
5802 *cache = None;
5803 }
5804 if let Ok(mut cache) = self.pack_indexes.lock() {
5805 cache.clear();
5806 }
5807 if let Ok(mut cache) = self.multi_pack_indexes.lock() {
5808 cache.clear();
5809 }
5810 if let Ok(mut cache) = self.multi_pack_oid_lookups.lock() {
5811 cache.clear();
5812 }
5813 if let Ok(mut cache) = self.pack_bytes.lock() {
5814 cache.clear();
5815 }
5816 if let Ok(mut cache) = self.pack_deltas.lock() {
5817 cache.clear();
5818 }
5819 if let Ok(mut cache) = self.pack_header_types.lock() {
5820 cache.clear();
5821 }
5822 if let Ok(mut cache) = self.decoded.lock() {
5823 cache.clear();
5824 }
5825 self.loose.invalidate_cache();
5826 }
5827
5828 pub fn loose(&self) -> &LooseObjectStore {
5829 &self.loose
5830 }
5831
5832 pub fn presence_checker(&self) -> ObjectPresenceChecker {
5833 ObjectPresenceChecker::new(self.clone())
5834 }
5835
5836 pub fn install_pack(&self, pack: &PackWrite) -> Result<PackInstallResult> {
5837 self.install_pack_with_options(pack, RawPackInstallOptions::default())
5838 }
5839
5840 pub fn write_blob_as_pack(
5841 &self,
5842 oid: ObjectId,
5843 object: &EncodedObject,
5844 compression_level: u32,
5845 ) -> Result<ObjectId> {
5846 if object.object_type != ObjectType::Blob {
5847 return Err(GitError::InvalidObject(
5848 "write_blob_as_pack requires a blob object".into(),
5849 ));
5850 }
5851 if oid.format() != self.format {
5852 return Err(GitError::InvalidObjectId(format!(
5853 "object {oid} uses {}, store uses {}",
5854 oid.format().name(),
5855 self.format.name()
5856 )));
5857 }
5858 if self.contains(&oid)? {
5859 return Ok(oid);
5860 }
5861 let input = [PackInput { oid: &oid, object }];
5862 let options = PackWriteOptions::new()
5863 .with_window(0)
5864 .with_depth(0)
5865 .with_reorder(false)
5866 .with_compression_level(compression_level);
5867 let pack =
5868 PackFile::write_packed_with_known_ids_and_options(&input, self.format, &options)?;
5869 self.install_pack(&pack)?;
5870 Ok(oid)
5871 }
5872
5873 pub fn write_blobs_as_pack(
5874 &self,
5875 objects: &[(ObjectId, EncodedObject)],
5876 compression_level: u32,
5877 ) -> Result<()> {
5878 let mut seen = HashSet::with_capacity(objects.len());
5879 let mut inputs = Vec::new();
5880 for (oid, object) in objects {
5881 if object.object_type != ObjectType::Blob {
5882 return Err(GitError::InvalidObject(
5883 "write_blobs_as_pack requires blob objects".into(),
5884 ));
5885 }
5886 if oid.format() != self.format {
5887 return Err(GitError::InvalidObjectId(format!(
5888 "object {oid} uses {}, store uses {}",
5889 oid.format().name(),
5890 self.format.name()
5891 )));
5892 }
5893 if seen.insert(*oid) && !self.contains(oid)? {
5894 inputs.push(PackInput { oid, object });
5895 }
5896 }
5897 if inputs.is_empty() {
5898 return Ok(());
5899 }
5900 let options = PackWriteOptions::new()
5901 .with_window(0)
5902 .with_depth(0)
5903 .with_reorder(false)
5904 .with_compression_level(compression_level);
5905 let pack =
5906 PackFile::write_packed_with_known_ids_and_options(&inputs, self.format, &options)?;
5907 self.install_pack(&pack)?;
5908 Ok(())
5909 }
5910
5911 pub fn install_pack_with_options(
5912 &self,
5913 pack: &PackWrite,
5914 options: RawPackInstallOptions,
5915 ) -> Result<PackInstallResult> {
5916 if pack.checksum.format() != self.format {
5917 return Err(GitError::InvalidObjectId(format!(
5918 "pack checksum uses {}, store uses {}",
5919 pack.checksum.format().name(),
5920 self.format.name()
5921 )));
5922 }
5923 for entry in &pack.entries {
5924 if entry.oid.format() != self.format {
5925 return Err(GitError::InvalidObjectId(format!(
5926 "pack entry {} uses {}, store uses {}",
5927 entry.oid,
5928 entry.oid.format().name(),
5929 self.format.name()
5930 )));
5931 }
5932 }
5933 let canonical_index = PackIndex::write_v2_for_pack(&pack.pack, self.format)?;
5934 let parsed_index = PackIndex::parse(&pack.index, self.format)?;
5935 if canonical_index.pack_checksum != pack.checksum
5936 || parsed_index.pack_checksum != pack.checksum
5937 {
5938 return Err(GitError::InvalidFormat(
5939 "pack and index checksums do not match pack write".into(),
5940 ));
5941 }
5942 if pack.index != canonical_index.index {
5943 return Err(GitError::InvalidFormat(
5944 "pack index does not match pack contents".into(),
5945 ));
5946 }
5947
5948 let pack_dir = self.objects_dir.join("pack");
5949 fs::create_dir_all(&pack_dir)?;
5950 let pack_name = format!("pack-{}", pack.checksum.to_hex());
5951 let pack_path = pack_dir.join(format!("{pack_name}.pack"));
5952 let index_path = pack_dir.join(format!("{pack_name}.idx"));
5953 if !pack_path.exists() || !index_path.exists() {
5954 write_pack_component(&pack_path, &pack.pack)?;
5955 write_pack_component(&index_path, &pack.index)?;
5956 }
5957 let promisor_path = write_promisor_pack_sidecar(&pack_dir, &pack_name, options.promisor)?;
5958 Ok(PackInstallResult {
5959 pack_name,
5960 pack_path,
5961 index_path,
5962 promisor_path,
5963 object_ids: canonical_index
5964 .entries
5965 .iter()
5966 .map(|entry| entry.oid)
5967 .collect(),
5968 })
5969 }
5970
5971 pub fn install_written_pack(&self, pack: &PackWrite) -> Result<PackInstallResult> {
5979 self.install_written_pack_with_options(pack, RawPackInstallOptions::default())
5980 }
5981
5982 pub fn install_written_pack_with_options(
5983 &self,
5984 pack: &PackWrite,
5985 options: RawPackInstallOptions,
5986 ) -> Result<PackInstallResult> {
5987 validate_pack_checksum(&pack.pack, self.format, &pack.checksum, "pack write")?;
5988 let parsed_index = PackIndex::parse(&pack.index, self.format)?;
5989 if parsed_index.pack_checksum != pack.checksum {
5990 return Err(GitError::InvalidFormat(
5991 "pack write index checksum does not match pack".into(),
5992 ));
5993 }
5994 if !pack_index_entries_match_writer(&parsed_index.entries, &pack.entries) {
5995 return Err(GitError::InvalidFormat(
5996 "pack write index does not match generated entries".into(),
5997 ));
5998 }
5999 self.install_generated_pack_unchecked(pack, options)
6000 }
6001
6002 fn install_generated_pack_unchecked(
6003 &self,
6004 pack: &PackWrite,
6005 options: RawPackInstallOptions,
6006 ) -> Result<PackInstallResult> {
6007 let pack_dir = self.objects_dir.join("pack");
6008 fs::create_dir_all(&pack_dir)?;
6009 let pack_name = format!("pack-{}", pack.checksum.to_hex());
6010 let pack_path = pack_dir.join(format!("{pack_name}.pack"));
6011 let index_path = pack_dir.join(format!("{pack_name}.idx"));
6012 if !pack_path.exists() || !index_path.exists() {
6013 write_pack_component(&pack_path, &pack.pack)?;
6014 write_pack_component(&index_path, &pack.index)?;
6015 }
6016 let promisor_path = write_promisor_pack_sidecar(&pack_dir, &pack_name, options.promisor)?;
6017 Ok(PackInstallResult {
6018 pack_name,
6019 pack_path,
6020 index_path,
6021 promisor_path,
6022 object_ids: pack.entries.iter().map(|entry| entry.oid).collect(),
6023 })
6024 }
6025
6026 fn install_pack_file_from_temp(
6027 &self,
6028 temp_pack_path: &Path,
6029 pack_checksum: ObjectId,
6030 index: &[u8],
6031 object_ids: Vec<ObjectId>,
6032 options: RawPackInstallOptions,
6033 ) -> Result<PackInstallResult> {
6034 let pack_dir = self.objects_dir.join("pack");
6035 fs::create_dir_all(&pack_dir)?;
6036 let pack_name = format!("pack-{}", pack_checksum.to_hex());
6037 let pack_path = pack_dir.join(format!("{pack_name}.pack"));
6038 let index_path = pack_dir.join(format!("{pack_name}.idx"));
6039 match fs::rename(temp_pack_path, &pack_path) {
6040 Ok(()) => {}
6041 Err(_) if pack_path.exists() => {
6042 let _ = fs::remove_file(temp_pack_path);
6043 }
6044 Err(err) => return Err(GitError::Io(err.to_string())),
6045 }
6046 write_pack_component(&index_path, index)?;
6047 let promisor_path = write_promisor_pack_sidecar(&pack_dir, &pack_name, options.promisor)?;
6048 Ok(PackInstallResult {
6049 pack_name,
6050 pack_path,
6051 index_path,
6052 promisor_path,
6053 object_ids,
6054 })
6055 }
6056
6057 pub fn install_raw_pack_from_reader<R>(&self, reader: &mut R) -> Result<PackInstallResult>
6058 where
6059 R: Read,
6060 {
6061 self.install_raw_pack_from_reader_with_options(reader, RawPackInstallOptions::default())
6062 }
6063
6064 pub fn begin_raw_pack_install(
6065 &self,
6066 expected_pack_id: ObjectId,
6067 expected_pack_size: u64,
6068 ) -> Result<RawPackStreamingInstall> {
6069 self.begin_raw_pack_install_with_options(
6070 expected_pack_id,
6071 expected_pack_size,
6072 RawPackInstallOptions::default(),
6073 )
6074 }
6075
6076 pub fn begin_raw_pack_install_with_options(
6077 &self,
6078 expected_pack_id: ObjectId,
6079 expected_pack_size: u64,
6080 options: RawPackInstallOptions,
6081 ) -> Result<RawPackStreamingInstall> {
6082 if expected_pack_id.format() != self.format {
6083 return Err(GitError::InvalidObjectId(format!(
6084 "pack checksum uses {}, store uses {}",
6085 expected_pack_id.format().name(),
6086 self.format.name()
6087 )));
6088 }
6089 let pack_dir = self.objects_dir.join("pack");
6090 fs::create_dir_all(&pack_dir)?;
6091 let pack_name = format!("pack-{}", expected_pack_id.to_hex());
6092 let pack_path = pack_dir.join(format!("{pack_name}.pack"));
6093 let index_path = pack_dir.join(format!("{pack_name}.idx"));
6094 let temp_pack_path = unique_temp_path(&pack_dir);
6095 let file = fs::OpenOptions::new()
6096 .write(true)
6097 .create_new(true)
6098 .open(&temp_pack_path)?;
6099 Ok(RawPackStreamingInstall {
6100 format: self.format,
6101 expected_pack_id,
6102 expected_pack_size,
6103 options,
6104 pack_dir,
6105 pack_name,
6106 pack_path,
6107 index_path,
6108 temp_pack_path,
6109 file: Some(file),
6110 written: 0,
6111 finished: false,
6112 })
6113 }
6114
6115 pub fn install_raw_pack_from_reader_with_options<R>(
6116 &self,
6117 reader: &mut R,
6118 options: RawPackInstallOptions,
6119 ) -> Result<PackInstallResult>
6120 where
6121 R: Read,
6122 {
6123 let pack_dir = self.objects_dir.join("pack");
6124 fs::create_dir_all(&pack_dir)?;
6125 let temp_pack_path = unique_temp_path(&pack_dir);
6126 let result = (|| -> Result<PackInstallResult> {
6127 let mut file = fs::OpenOptions::new()
6130 .write(true)
6131 .create_new(true)
6132 .open(&temp_pack_path)?;
6133 let built = {
6134 let mut tee = PackInstallTeeReader {
6135 reader,
6136 writer: &mut file,
6137 };
6138 PackIndex::write_v2_for_pack_reader_to_trailer(&mut tee, self.format)?
6139 };
6140 file.flush()?;
6141 file.sync_all()?;
6142 drop(file);
6143
6144 self.install_pack_file_from_temp(
6145 &temp_pack_path,
6146 built.pack_checksum,
6147 &built.index,
6148 built.entries.iter().map(|entry| entry.oid).collect(),
6149 options,
6150 )
6151 })();
6152 if result.is_err() {
6153 let _ = fs::remove_file(&temp_pack_path);
6154 }
6155 result
6156 }
6157
6158 pub fn contains(&self, oid: &ObjectId) -> Result<bool> {
6159 if self.loose.exists(oid)? {
6160 return Ok(true);
6161 }
6162 if self.find_pack_containing(oid)?.is_some() {
6163 return Ok(true);
6164 }
6165 for alternate in &self.alternates {
6166 if Self::without_alternates(alternate, self.format).contains(oid)? {
6167 return Ok(true);
6168 }
6169 }
6170 self.loose.invalidate_cache();
6173 self.loose.exists(oid)
6174 }
6175
6176 pub fn object_ids(&self) -> Result<Vec<ObjectId>> {
6177 let mut oids = object_ids_in_objects_dir(&self.objects_dir, self.format)?
6178 .into_iter()
6179 .collect::<HashSet<_>>();
6180 for alternate in &self.alternates {
6181 oids.extend(Self::without_alternates(alternate, self.format).object_ids()?);
6182 }
6183 let mut oids = oids.into_iter().collect::<Vec<_>>();
6184 oids.sort_by_key(ObjectId::to_hex);
6185 Ok(oids)
6186 }
6187
6188 pub fn object_storage_info(&self, oid: &ObjectId) -> Result<Option<ObjectStorageInfo>> {
6189 if let Some(disk_size) = self.loose.disk_size(oid)? {
6190 return Ok(Some(ObjectStorageInfo {
6191 disk_size,
6192 deltabase: zero_oid(self.format)?,
6193 }));
6194 }
6195 if let Some(info) = self.packed_object_storage_info(oid)? {
6196 return Ok(Some(info));
6197 }
6198 for alternate in &self.alternates {
6199 if let Some(info) =
6200 Self::without_alternates(alternate, self.format).object_storage_info(oid)?
6201 {
6202 return Ok(Some(info));
6203 }
6204 }
6205 self.loose.invalidate_cache();
6208 if let Some(disk_size) = self.loose.disk_size(oid)? {
6209 return Ok(Some(ObjectStorageInfo {
6210 disk_size,
6211 deltabase: zero_oid(self.format)?,
6212 }));
6213 }
6214 Ok(None)
6215 }
6216
6217 pub fn resolve_prefix(&self, prefix: &str) -> Result<ObjectPrefixResolution> {
6218 let mut matches = self.object_ids_with_prefix(prefix)?;
6219 Ok(match matches.len() {
6220 0 => ObjectPrefixResolution::Missing,
6221 1 => ObjectPrefixResolution::Unique(matches.remove(0)),
6222 _ => ObjectPrefixResolution::Ambiguous(matches),
6223 })
6224 }
6225
6226 pub fn object_ids_with_prefix(&self, prefix: &str) -> Result<Vec<ObjectId>> {
6227 validate_object_id_prefix(self.format, prefix)?;
6228 let mut matches = Vec::new();
6229 for oid in self.object_ids()? {
6230 if object_id_matches_prefix(&oid, prefix) {
6231 matches.push(oid);
6232 }
6233 }
6234 Ok(matches)
6235 }
6236
6237 pub fn read_object_header(&self, oid: &ObjectId) -> Result<Option<(ObjectType, u64)>> {
6247 if implied_empty_tree_object(self.format, oid).is_some() {
6248 return Ok(Some((ObjectType::Tree, 0)));
6249 }
6250 if let Ok(mut cache) = self.decoded.lock()
6251 && let Some(object) = cache.get(oid)
6252 {
6253 return Ok(Some((object.object_type, object.body.len() as u64)));
6254 }
6255 if let Some(header) = self.loose.read_header(oid)? {
6256 return Ok(Some(header));
6257 }
6258 if let Some(pack_lookup) = self.find_pack_containing(oid)? {
6259 let bytes = pack_lookup.pack_bytes(self)?;
6260 let type_cache = pack_lookup.header_type_cache(self);
6265 let resolve_ref_base = |base: &ObjectId| {
6266 self.read_object_header(base)
6267 .map(|header| header.map(|(t, _)| t))
6268 };
6269 let header = match &type_cache {
6270 Some(cache) => {
6271 let mut adapter = PackHeaderTypeCacheAdapter(cache);
6272 sley_pack::read_object_header_at_with_cache(
6273 &bytes,
6274 pack_lookup.offset,
6275 self.format,
6276 resolve_ref_base,
6277 &mut adapter,
6278 )?
6279 }
6280 None => sley_pack::read_object_header_at(
6281 &bytes,
6282 pack_lookup.offset,
6283 self.format,
6284 resolve_ref_base,
6285 )?,
6286 };
6287 return Ok(Some(header));
6288 }
6289 for alternate in &self.alternates {
6290 if let Some(header) =
6291 Self::without_alternates(alternate, self.format).read_object_header(oid)?
6292 {
6293 return Ok(Some(header));
6294 }
6295 }
6296 self.loose.invalidate_cache();
6299 if let Some(header) = self.loose.read_header(oid)? {
6300 return Ok(Some(header));
6301 }
6302 Ok(None)
6303 }
6304
6305 fn read_packed_object(&self, oid: &ObjectId) -> Result<Option<Arc<EncodedObject>>> {
6306 if let Ok(mut cache) = self.decoded.lock()
6309 && let Some(object) = cache.get(oid)
6310 {
6311 return Ok(Some(object));
6312 }
6313 let Some(pack_lookup) = self.find_pack_containing(oid)? else {
6314 return Ok(None);
6315 };
6316 self.read_packed_object_at_lookup(oid, &pack_lookup)
6317 .map(Some)
6318 }
6319
6320 fn read_packed_object_at_lookup(
6321 &self,
6322 oid: &ObjectId,
6323 pack_lookup: &PackLookup,
6324 ) -> Result<Arc<EncodedObject>> {
6325 if let Ok(mut cache) = self.decoded.lock()
6326 && let Some(object) = cache.get(oid)
6327 {
6328 return Ok(object);
6329 }
6330 let bytes = pack_lookup.pack_bytes(self)?;
6331 let delta_cache = pack_lookup.delta_cache(self);
6336 let delta_adapter = delta_cache.as_ref().map(PackDeltaCacheAdapter);
6337 let resolve_ref_base = |base: &ObjectId| self.read_object(base).map(Some);
6343 let resolve_ofs_base =
6344 |base_offset| self.read_ofs_delta_base_from_other_sources(pack_lookup, base_offset);
6345 let object = match &delta_adapter {
6346 Some(adapter) => sley_pack::read_object_at_with_cache_and_ofs_base_arc(
6347 &bytes,
6348 pack_lookup.offset,
6349 self.format,
6350 resolve_ref_base,
6351 resolve_ofs_base,
6352 adapter,
6353 )?,
6354 None => sley_pack::read_object_at_with_ofs_base_arc(
6355 &bytes,
6356 pack_lookup.offset,
6357 self.format,
6358 resolve_ref_base,
6359 resolve_ofs_base,
6360 )?,
6361 };
6362 if verify_reads_enabled() {
6366 let actual = object.object_id(self.format)?;
6367 if actual != *oid {
6368 return Err(GitError::InvalidObject(format!(
6369 "pack object id mismatch: index says {oid}, decoded {actual}"
6370 )));
6371 }
6372 }
6373 if let Ok(mut cache) = self.decoded.lock() {
6374 cache.put(*oid, Arc::clone(&object));
6375 }
6376 Ok(object)
6377 }
6378
6379 fn pack_delta_cache(&self, pack_path: &Path) -> Option<Arc<Mutex<LruOffsetCache>>> {
6383 let mut caches = self.pack_deltas.lock().ok()?;
6384 let cache = caches.entry(pack_path.to_path_buf()).or_insert_with(|| {
6385 Arc::new(Mutex::new(LruOffsetCache::new(delta_base_cache_budget())))
6386 });
6387 Some(Arc::clone(cache))
6388 }
6389
6390 fn pack_header_type_cache(&self, pack_path: &Path) -> Option<PackHeaderTypeCache> {
6394 let mut caches = self.pack_header_types.lock().ok()?;
6395 let cache = caches
6396 .entry(pack_path.to_path_buf())
6397 .or_insert_with(|| Arc::new(Mutex::new(HashMap::new())));
6398 Some(Arc::clone(cache))
6399 }
6400
6401 fn cached_pack_bytes(&self, pack_path: &Path) -> Result<Arc<PackData>> {
6406 if let Ok(cache) = self.pack_bytes.lock()
6407 && let Some(bytes) = cache.get(pack_path)
6408 {
6409 return Ok(Arc::clone(bytes));
6410 }
6411 let bytes = Arc::new(load_pack_data(pack_path)?);
6412 if let Ok(mut cache) = self.pack_bytes.lock() {
6413 cache.insert(pack_path.to_path_buf(), Arc::clone(&bytes));
6414 }
6415 Ok(bytes)
6416 }
6417
6418 fn cached_pack_index(&self, index_path: &Path) -> Result<Arc<PackIndex>> {
6422 if let Ok(cache) = self.pack_indexes.lock()
6423 && let Some(index) = cache.get(index_path)
6424 {
6425 return Ok(Arc::clone(index));
6426 }
6427 let index = Arc::new(PackIndex::parse(&fs::read(index_path)?, self.format)?);
6428 if let Ok(mut cache) = self.pack_indexes.lock() {
6429 cache.insert(index_path.to_path_buf(), Arc::clone(&index));
6430 }
6431 Ok(index)
6432 }
6433
6434 fn cached_multi_pack_index_oid_lookup(
6435 &self,
6436 midx_path: &Path,
6437 ) -> Result<Option<Arc<MultiPackIndexOidLookup>>> {
6438 if !midx_path.exists() {
6439 return Ok(None);
6440 }
6441 if let Ok(cache) = self.multi_pack_oid_lookups.lock()
6442 && let Some(midx) = cache.get(midx_path)
6443 {
6444 return Ok(Some(Arc::clone(midx)));
6445 }
6446 let bytes = load_multi_pack_index_lookup_data(midx_path)?;
6447 let midx = match MultiPackIndexOidLookup::parse(bytes, self.format) {
6448 Ok(midx) => Arc::new(midx),
6449 Err(GitError::InvalidFormat(message))
6450 if message.starts_with("multi-pack-index hash id ") =>
6451 {
6452 let actual = message
6453 .strip_prefix("multi-pack-index hash id ")
6454 .and_then(|rest| rest.split_whitespace().next())
6455 .unwrap_or("0");
6456 let expected = match self.format {
6457 ObjectFormat::Sha1 => 1,
6458 ObjectFormat::Sha256 => 2,
6459 };
6460 eprintln!(
6461 "error: multi-pack-index hash version {actual} does not match version {expected}"
6462 );
6463 return Ok(None);
6464 }
6465 Err(err) => return Err(err),
6466 };
6467 if let Ok(mut cache) = self.multi_pack_oid_lookups.lock() {
6468 cache.insert(midx_path.to_path_buf(), Arc::clone(&midx));
6469 }
6470 Ok(Some(midx))
6471 }
6472
6473 fn cached_multi_pack_index(&self, midx_path: &Path) -> Result<Option<Arc<MultiPackIndex>>> {
6474 if !midx_path.exists() {
6475 return Ok(None);
6476 }
6477 if let Ok(cache) = self.multi_pack_indexes.lock()
6478 && let Some(midx) = cache.get(midx_path)
6479 {
6480 return Ok(Some(Arc::clone(midx)));
6481 }
6482 let bytes = load_multi_pack_index_lookup_data(midx_path)?;
6483 let midx = match MultiPackIndex::parse(bytes.as_bytes(), self.format) {
6484 Ok(midx) => Arc::new(midx),
6485 Err(GitError::InvalidFormat(message))
6486 if message.starts_with("multi-pack-index hash id ") =>
6487 {
6488 let actual = message
6489 .strip_prefix("multi-pack-index hash id ")
6490 .and_then(|rest| rest.split_whitespace().next())
6491 .unwrap_or("0");
6492 let expected = match self.format {
6493 ObjectFormat::Sha1 => 1,
6494 ObjectFormat::Sha256 => 2,
6495 };
6496 eprintln!(
6497 "error: multi-pack-index hash version {actual} does not match version {expected}"
6498 );
6499 return Ok(None);
6500 }
6501 Err(err) => return Err(err),
6502 };
6503 if let Ok(mut cache) = self.multi_pack_indexes.lock() {
6504 cache.insert(midx_path.to_path_buf(), Arc::clone(&midx));
6505 }
6506 Ok(Some(midx))
6507 }
6508
6509 fn cached_pack_registry(
6514 &self,
6515 pack_dir: &Path,
6516 force_rescan: bool,
6517 ) -> Result<Arc<PackRegistrySnapshot>> {
6518 if !force_rescan && let Some(registry) = self.cached_loaded_pack_registry(pack_dir)? {
6519 return Ok(registry);
6520 }
6521 let scanned = Arc::new(scan_pack_registry(pack_dir, self.format)?);
6522 if let Ok(mut cache) = self.pack_registry.lock() {
6523 match cache.as_ref() {
6524 Some(existing)
6525 if existing.fingerprint == scanned.fingerprint
6526 && same_registered_pack_set(&existing.packs, &scanned.packs) =>
6527 {
6528 return Ok(Arc::clone(existing));
6529 }
6530 _ => {
6531 *cache = Some(Arc::clone(&scanned));
6532 }
6533 }
6534 }
6535 Ok(scanned)
6536 }
6537
6538 fn find_in_pack_registry(
6539 &self,
6540 registry: Arc<PackRegistrySnapshot>,
6541 oid: &ObjectId,
6542 ) -> Result<Option<PackLookup>> {
6543 let hinted_pack_index = registry.cached_hint();
6544 if let Some(pack_index) = hinted_pack_index {
6545 let pack = ®istry.packs[pack_index];
6546 match pack.index(self.format) {
6547 Ok(index) => {
6548 if let Some(entry) = index.find(oid) {
6549 return Ok(Some(PackLookup::from_registered(
6550 Arc::clone(pack),
6551 entry.offset,
6552 )));
6553 }
6554 }
6555 Err(_) => {
6556 eprintln!("error: packfile {} index unavailable", pack.pack.display());
6557 }
6558 }
6559 }
6560 for (pack_index, pack) in registry.packs.iter().enumerate() {
6561 if Some(pack_index) == hinted_pack_index {
6562 continue;
6563 }
6564 let index = match pack.index(self.format) {
6565 Ok(index) => index,
6566 Err(_) => {
6567 eprintln!("error: packfile {} index unavailable", pack.pack.display());
6568 continue;
6569 }
6570 };
6571 if let Some(entry) = index.find(oid) {
6572 registry.remember_hint(pack_index);
6573 return Ok(Some(PackLookup::from_registered(
6574 Arc::clone(pack),
6575 entry.offset,
6576 )));
6577 }
6578 }
6579 Ok(None)
6580 }
6581
6582 fn read_packed_object_from_other_packs(
6588 &self,
6589 oid: &ObjectId,
6590 exclude: &PackLookup,
6591 ) -> Result<Option<Arc<EncodedObject>>> {
6592 let pack_dir = self.objects_dir.join("pack");
6593 let Ok(entries) = fs::read_dir(&pack_dir) else {
6594 return Ok(None);
6595 };
6596 let excluded_pack = exclude.pack_path().to_path_buf();
6597 for entry in entries {
6598 let idx_path = entry?.path();
6599 if idx_path.extension().and_then(|ext| ext.to_str()) != Some("idx") {
6600 continue;
6601 }
6602 let pack_path = idx_path.with_extension("pack");
6603 if pack_path == excluded_pack {
6604 continue;
6605 }
6606 let Ok(idx_bytes) = fs::read(&idx_path) else {
6607 continue;
6608 };
6609 let Ok(index) = PackIndex::parse(&idx_bytes, self.format) else {
6610 continue;
6611 };
6612 let Some(entry) = index.find(oid) else {
6613 continue;
6614 };
6615 let candidate = PackLookup::from_path(pack_path, entry.offset);
6616 if let Ok(object) = self.read_packed_object_at_lookup(oid, &candidate) {
6617 return Ok(Some(object));
6618 }
6619 }
6620 Ok(None)
6621 }
6622
6623 fn pack_oid_at_offset(
6624 &self,
6625 pack_lookup: &PackLookup,
6626 offset: u64,
6627 ) -> Result<Option<ObjectId>> {
6628 match pack_lookup.pack_index(self) {
6629 Ok(index) => Ok(index
6630 .entries
6631 .iter()
6632 .find(|entry| entry.offset == offset)
6633 .map(|entry| entry.oid)),
6634 Err(_) => self.midx_oid_for_pack_offset(pack_lookup, offset),
6635 }
6636 }
6637
6638 fn read_ofs_delta_base_from_other_sources(
6639 &self,
6640 pack_lookup: &PackLookup,
6641 base_offset: u64,
6642 ) -> Result<Option<Arc<EncodedObject>>> {
6643 let Some(base_oid) = self.pack_oid_at_offset(pack_lookup, base_offset)? else {
6644 return Ok(None);
6645 };
6646 if let Ok(mut cache) = self.decoded.lock()
6647 && let Some(object) = cache.get(&base_oid)
6648 {
6649 return Ok(Some(object));
6650 }
6651 if let Ok(object) = self.loose.read_object(&base_oid) {
6652 return Ok(Some(object));
6653 }
6654 if let Some(object) = self.read_packed_object_from_other_packs(&base_oid, pack_lookup)? {
6655 return Ok(Some(object));
6656 }
6657 for alternate in &self.alternates {
6658 if let Ok(object) =
6659 Self::without_alternates(alternate, self.format).read_object(&base_oid)
6660 {
6661 return Ok(Some(object));
6662 }
6663 }
6664 Ok(None)
6665 }
6666
6667 fn find_pack_containing(&self, oid: &ObjectId) -> Result<Option<PackLookup>> {
6668 if oid.format() != self.format {
6669 return Err(GitError::InvalidObjectId(format!(
6670 "object {oid} uses {}, store uses {}",
6671 oid.format().name(),
6672 self.format.name()
6673 )));
6674 }
6675 let pack_dir = self.objects_dir.join("pack");
6676 if let Some(midx) = self.cached_loaded_multi_pack_index_oid_lookup()
6681 && let Some(pack_paths) = self.midx_oid_lookup_pack_paths(&pack_dir, &midx, oid)?
6682 {
6683 return Ok(Some(pack_paths));
6684 }
6685 if let Some(registry) = self.cached_loaded_pack_registry(&pack_dir)?
6686 && let Some(pack_paths) = self.find_in_pack_registry(registry, oid)?
6687 {
6688 return Ok(Some(pack_paths));
6689 }
6690
6691 if !pack_dir.exists() {
6692 return Ok(None);
6693 }
6694 if let Some(pack_paths) = self.find_midx_pack_containing(&pack_dir, oid)? {
6695 return Ok(Some(pack_paths));
6696 }
6697 let registry = self.cached_pack_registry(&pack_dir, false)?;
6701 if let Some(pack_paths) = self.find_in_pack_registry(Arc::clone(®istry), oid)? {
6702 return Ok(Some(pack_paths));
6703 }
6704 let refreshed = self.cached_pack_registry(&pack_dir, true)?;
6705 if Arc::ptr_eq(®istry, &refreshed) {
6706 return Ok(None);
6708 }
6709 self.find_in_pack_registry(refreshed, oid)
6710 }
6711
6712 fn packed_object_storage_info(&self, oid: &ObjectId) -> Result<Option<ObjectStorageInfo>> {
6713 let Some(pack_lookup) = self.find_pack_containing(oid)? else {
6714 return Ok(None);
6715 };
6716 let index = pack_lookup.pack_index(self).ok();
6717 let pack = match pack_lookup.pack_bytes(self) {
6718 Ok(pack) => Some(pack),
6719 Err(_err) if index.is_some() => None,
6720 Err(err) => return Err(err),
6721 };
6722 let trailer_offset = pack
6723 .as_ref()
6724 .map(|pack| {
6725 (pack.len() as u64)
6726 .checked_sub(self.format.raw_len() as u64)
6727 .ok_or_else(|| {
6728 GitError::InvalidFormat("pack file shorter than checksum".into())
6729 })
6730 })
6731 .transpose()?;
6732 let delta_base = match &pack {
6733 Some(pack) => pack_entry_delta_base(self.format, pack, pack_lookup.offset)?,
6734 None => None,
6735 };
6736 let delta_base_offset = match &delta_base {
6737 Some(PackDeltaBase::Offset(offset)) => Some(*offset),
6738 Some(PackDeltaBase::Ref(_)) | None => None,
6739 };
6740 let offset_info = if let Some(index) = &index {
6741 scan_pack_index_offsets(index, pack_lookup.offset, trailer_offset, delta_base_offset)?
6742 } else if let Some(pack) = &pack {
6743 let end_offset =
6744 scan_pack_offsets_without_index(self.format, pack, pack_lookup.offset)?
6745 .ok_or_else(|| {
6746 GitError::InvalidFormat(format!(
6747 "pack offset {} not found",
6748 pack_lookup.offset
6749 ))
6750 })?;
6751 let delta_base_oid = match delta_base_offset {
6752 Some(offset) => self
6753 .midx_oid_for_pack_offset(&pack_lookup, offset)?
6754 .ok_or_else(|| {
6755 GitError::InvalidFormat(format!("ofs-delta base offset {offset} not found"))
6756 })?,
6757 None => zero_oid(self.format)?,
6758 };
6759 PackIndexOffsetInfo {
6760 end_offset,
6761 delta_base_oid: delta_base_offset.map(|_| delta_base_oid),
6762 }
6763 } else {
6764 return Err(GitError::InvalidFormat(
6765 "packed object metadata source unavailable".into(),
6766 ));
6767 };
6768 let disk_size = offset_info
6769 .end_offset
6770 .checked_sub(pack_lookup.offset)
6771 .ok_or_else(|| GitError::InvalidFormat("pack index offsets are not sorted".into()))?;
6772 let deltabase = match delta_base {
6773 Some(PackDeltaBase::Offset(_)) => offset_info.delta_base_oid.ok_or_else(|| {
6774 GitError::InvalidFormat("ofs-delta base oid missing from pack index".into())
6780 })?,
6781 Some(PackDeltaBase::Ref(oid)) => oid,
6782 None => zero_oid(self.format)?,
6783 };
6784 Ok(Some(ObjectStorageInfo {
6785 disk_size,
6786 deltabase,
6787 }))
6788 }
6789
6790 fn midx_oid_for_pack_offset(
6791 &self,
6792 pack_lookup: &PackLookup,
6793 offset: u64,
6794 ) -> Result<Option<ObjectId>> {
6795 let pack_dir = self.objects_dir.join("pack");
6796 let midx_path = pack_dir.join("multi-pack-index");
6797 let Some(midx) = self.cached_multi_pack_index(&midx_path)? else {
6798 return Ok(None);
6799 };
6800 let Some(pack_name) = pack_lookup
6801 .pack_path()
6802 .file_name()
6803 .and_then(|name| name.to_str())
6804 else {
6805 return Ok(None);
6806 };
6807 let idx_name = pack_name
6808 .strip_suffix(".pack")
6809 .map(|stem| format!("{stem}.idx"))
6810 .unwrap_or_else(|| pack_name.to_string());
6811 let Some(pack_int_id) = midx
6812 .pack_names
6813 .iter()
6814 .position(|candidate| candidate == &idx_name)
6815 else {
6816 return Ok(None);
6817 };
6818 Ok(midx
6819 .objects
6820 .iter()
6821 .find(|entry| entry.pack_int_id == pack_int_id as u32 && entry.offset == offset)
6822 .map(|entry| entry.oid))
6823 }
6824
6825 fn find_midx_pack_containing(
6826 &self,
6827 pack_dir: &Path,
6828 oid: &ObjectId,
6829 ) -> Result<Option<PackLookup>> {
6830 let midx_path = pack_dir.join("multi-pack-index");
6831 if let Some(midx) = self.cached_multi_pack_index_oid_lookup(&midx_path)?
6832 && let Some(pack_lookup) = self.midx_oid_lookup_pack_paths(pack_dir, &midx, oid)?
6833 {
6834 return Ok(Some(pack_lookup));
6835 }
6836 self.find_incremental_midx_pack_containing(pack_dir, oid)
6837 }
6838
6839 fn midx_oid_lookup_pack_paths(
6840 &self,
6841 pack_dir: &Path,
6842 midx: &MultiPackIndexOidLookup,
6843 oid: &ObjectId,
6844 ) -> Result<Option<PackLookup>> {
6845 let Some(entry) = midx.find(oid)? else {
6846 return Ok(None);
6847 };
6848 let Some(pack_name) = midx.pack_name(entry.pack_int_id) else {
6849 return Err(GitError::InvalidFormat(
6850 "multi-pack-index object points past pack table".into(),
6851 ));
6852 };
6853 let pack_file_name = pack_name
6854 .strip_suffix(".idx")
6855 .map(|stem| format!("{stem}.pack"))
6856 .unwrap_or_else(|| pack_name.to_string());
6857 let pack = pack_dir.join(pack_file_name);
6858 Ok(Some(PackLookup::from_path(pack, entry.offset)))
6859 }
6860
6861 fn find_incremental_midx_pack_containing(
6862 &self,
6863 pack_dir: &Path,
6864 oid: &ObjectId,
6865 ) -> Result<Option<PackLookup>> {
6866 let chain = read_incremental_midx_chain(pack_dir)?;
6867 if chain.is_empty() {
6868 return Ok(None);
6869 }
6870 let midx_dir = pack_dir.join("multi-pack-index.d");
6871 for checksum in chain.iter().rev() {
6872 let path = midx_dir.join(format!("multi-pack-index-{checksum}.midx"));
6873 if !path.exists() {
6874 continue;
6875 }
6876 let bytes = load_multi_pack_index_lookup_data(&path)?;
6877 let midx = match MultiPackIndexOidLookup::parse(bytes, self.format) {
6878 Ok(midx) => midx,
6879 Err(_) => continue,
6880 };
6881 if let Some(pack_lookup) = self.midx_oid_lookup_pack_paths(pack_dir, &midx, oid)? {
6882 return Ok(Some(pack_lookup));
6883 }
6884 }
6885 Ok(None)
6886 }
6887
6888 fn cached_loaded_multi_pack_index_oid_lookup(&self) -> Option<Arc<MultiPackIndexOidLookup>> {
6889 let midx_path = self.objects_dir.join("pack").join("multi-pack-index");
6890 let cache = self.multi_pack_oid_lookups.lock().ok()?;
6891 cache.get(&midx_path).map(Arc::clone)
6892 }
6893
6894 fn cached_loaded_pack_registry(
6900 &self,
6901 _pack_dir: &Path,
6902 ) -> Result<Option<Arc<PackRegistrySnapshot>>> {
6903 let cache = match self.pack_registry.lock() {
6904 Ok(cache) => cache,
6905 Err(_) => return Ok(None),
6906 };
6907 Ok(cache.as_ref().map(Arc::clone))
6908 }
6909}
6910
6911fn validate_object_id_prefix(format: ObjectFormat, prefix: &str) -> Result<()> {
6912 if prefix.len() < 4 || prefix.len() > format.hex_len() {
6913 return Err(GitError::InvalidObjectId(format!(
6914 "expected 4 to {} hex digits for {}, got {}",
6915 format.hex_len(),
6916 format.name(),
6917 prefix.len()
6918 )));
6919 }
6920 if !prefix.bytes().all(|byte| byte.is_ascii_hexdigit()) {
6921 return Err(GitError::InvalidObjectId(format!(
6922 "non-hex object id prefix {prefix}"
6923 )));
6924 }
6925 Ok(())
6926}
6927
6928fn object_id_matches_prefix(oid: &ObjectId, prefix: &str) -> bool {
6929 oid.to_hex()
6930 .as_bytes()
6931 .iter()
6932 .zip(prefix.as_bytes())
6933 .all(|(actual, expected)| actual.eq_ignore_ascii_case(expected))
6934}
6935
6936fn pack_dir_modified(pack_dir: &Path) -> Result<Option<std::time::SystemTime>> {
6937 match fs::metadata(pack_dir) {
6938 Ok(metadata) => Ok(metadata.modified().ok()),
6939 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
6940 Err(err) => Err(GitError::Io(err.to_string())),
6941 }
6942}
6943
6944fn scan_pack_registry(pack_dir: &Path, _format: ObjectFormat) -> Result<PackRegistrySnapshot> {
6949 let modified = pack_dir_modified(pack_dir)?;
6950 let entries = match fs::read_dir(pack_dir) {
6951 Ok(entries) => entries,
6952 Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
6953 return Ok(PackRegistrySnapshot::new(
6954 PackDirFingerprint {
6955 modified,
6956 idx_count: 0,
6957 pack_count: 0,
6958 },
6959 Vec::new(),
6960 ));
6961 }
6962 Err(err) => return Err(GitError::Io(err.to_string())),
6963 };
6964
6965 let mut idx_paths = Vec::new();
6966 let mut idx_count = 0;
6967 let mut pack_count = 0;
6968 for entry in entries {
6969 let entry = entry?;
6970 let path = entry.path();
6971 match path.extension().and_then(|ext| ext.to_str()) {
6972 Some("idx") => {
6973 idx_count += 1;
6974 idx_paths.push(path);
6975 }
6976 Some("pack") => {
6977 pack_count += 1;
6978 }
6979 _ => {}
6980 }
6981 }
6982
6983 let mut packs = Vec::new();
6984 for idx in idx_paths {
6985 let pack = idx.with_extension("pack");
6986 let Ok(metadata) = fs::metadata(&pack) else {
6987 continue;
6988 };
6989 let modified = pack_sort_modified(&metadata);
6990 packs.push((
6991 modified,
6992 metadata.len(),
6993 Arc::new(RegisteredPack::new(idx, pack)),
6994 ));
6995 }
6996 packs.sort_by(|left, right| {
7001 right
7002 .0
7003 .cmp(&left.0)
7004 .then_with(|| right.1.cmp(&left.1))
7005 .then_with(|| left.2.idx.cmp(&right.2.idx))
7006 });
7007 let packs = packs.into_iter().map(|(_, _, pack)| pack).collect();
7008 Ok(PackRegistrySnapshot::new(
7009 PackDirFingerprint {
7010 modified,
7011 idx_count,
7012 pack_count,
7013 },
7014 packs,
7015 ))
7016}
7017
7018fn pack_sort_modified(metadata: &fs::Metadata) -> (u64, u32) {
7019 metadata
7020 .modified()
7021 .ok()
7022 .and_then(|modified| {
7023 modified
7024 .duration_since(std::time::UNIX_EPOCH)
7025 .ok()
7026 .map(|duration| (duration.as_secs(), duration.subsec_nanos()))
7027 })
7028 .unwrap_or((0, 0))
7029}
7030
7031fn same_registered_pack_set(left: &[Arc<RegisteredPack>], right: &[Arc<RegisteredPack>]) -> bool {
7034 left.len() == right.len()
7035 && left
7036 .iter()
7037 .zip(right.iter())
7038 .all(|(a, b)| a.idx == b.idx && a.pack == b.pack)
7039}
7040
7041fn alternate_object_dirs(objects_dir: &Path) -> Vec<PathBuf> {
7042 let mut alternates = Vec::new();
7043 if let Some(value) = env::var_os("GIT_ALTERNATE_OBJECT_DIRECTORIES") {
7044 for raw in value.to_string_lossy().split(':') {
7045 if !raw.is_empty() {
7046 alternates.push(PathBuf::from(raw));
7047 }
7048 }
7049 }
7050 let alternates_path = objects_dir.join("info").join("alternates");
7051 if let Ok(contents) = fs::read(&alternates_path) {
7052 for raw in contents.split(|byte| *byte == b'\n') {
7053 let line = raw.strip_suffix(b"\r").unwrap_or(raw);
7054 if line.is_empty() || line.starts_with(b"#") {
7055 continue;
7056 }
7057 let Ok(value) = std::str::from_utf8(line) else {
7058 continue;
7059 };
7060 let path = Path::new(value);
7061 let absolute = if path.is_absolute() {
7062 path.to_path_buf()
7063 } else {
7064 objects_dir.join(path)
7065 };
7066 alternates.push(absolute);
7067 }
7068 }
7069 alternates
7070}
7071
7072impl ObjectReader for FileObjectDatabase {
7073 fn is_promised_object(&self, oid: &ObjectId) -> bool {
7074 self.promisor_remote_present && self.promisor_objects().contains(oid)
7080 }
7081
7082 fn has_shallow_grafts(&self) -> bool {
7083 !self
7084 .shallow_grafts
7085 .get_or_init(|| {
7086 let shallow_file = self
7087 .objects_dir
7088 .parent()
7089 .map(|git_dir| git_dir.join("shallow"));
7090 match shallow_file {
7091 Some(path) => read_shallow_grafts(&path, self.format),
7092 None => HashSet::new(),
7093 }
7094 })
7095 .is_empty()
7096 }
7097
7098 fn is_shallow_graft(&self, oid: &ObjectId) -> bool {
7099 self.shallow_grafts
7100 .get_or_init(|| {
7101 let shallow_file = self
7102 .objects_dir
7103 .parent()
7104 .map(|git_dir| git_dir.join("shallow"));
7105 match shallow_file {
7106 Some(path) => read_shallow_grafts(&path, self.format),
7107 None => HashSet::new(),
7108 }
7109 })
7110 .contains(oid)
7111 }
7112
7113 fn read_object(&self, oid: &ObjectId) -> Result<Arc<EncodedObject>> {
7114 if let Some(object) = implied_empty_tree_object(self.format, oid) {
7115 return Ok(object);
7116 }
7117 if let Some(pack_lookup) = self.find_pack_containing(oid)? {
7125 match self.read_packed_object_at_lookup(oid, &pack_lookup) {
7126 Ok(object) => return Ok(object),
7127 Err(GitError::NotFound(_)) => {}
7128 Err(packed_err) => {
7134 if let Ok(object) = self.loose.read_object(oid) {
7135 return Ok(object);
7136 }
7137 if let Some(object) =
7140 self.read_packed_object_from_other_packs(oid, &pack_lookup)?
7141 {
7142 return Ok(object);
7143 }
7144 for alternate in &self.alternates {
7145 if let Ok(object) =
7146 Self::without_alternates(alternate, self.format).read_object(oid)
7147 {
7148 return Ok(object);
7149 }
7150 }
7151 return Err(packed_err);
7152 }
7153 }
7154 }
7155 let loose_err = match self.loose.read_object(oid) {
7156 Ok(object) => return Ok(object),
7157 Err(GitError::NotFound(_)) => None,
7158 Err(err) => Some(err),
7159 };
7160 if let Some(object) = self.read_packed_object(oid)? {
7161 return Ok(object);
7162 }
7163 for alternate in &self.alternates {
7164 match Self::without_alternates(alternate, self.format).read_object(oid) {
7165 Ok(object) => return Ok(object),
7166 Err(GitError::NotFound(_)) => {}
7167 Err(err) => return Err(err),
7168 }
7169 }
7170 self.loose.invalidate_cache();
7176 match self.loose.read_object(oid) {
7177 Ok(object) => return Ok(object),
7178 Err(GitError::NotFound(_)) => {}
7179 Err(err) => return Err(err),
7180 }
7181 if let Some(err) = loose_err {
7185 return Err(err);
7186 }
7187 Err(GitError::object_not_found_in(
7188 *oid,
7189 MissingObjectContext::Read,
7190 ))
7191 }
7192}
7193
7194impl FileObjectDatabase {
7195 fn promisor_objects(&self) -> &HashSet<ObjectId> {
7196 self.promisor_objects.get_or_init(|| {
7197 let mut promised =
7198 promisor_pack_object_ids(&self.objects_dir, self.format).unwrap_or_default();
7199 let mut pending = promised.iter().copied().collect::<Vec<_>>();
7200 while let Some(oid) = pending.pop() {
7201 let Ok(object) = self.read_object(&oid) else {
7202 continue;
7203 };
7204 for link in promisor_object_links(self.format, &object) {
7205 if promised.insert(link) {
7206 pending.push(link);
7207 }
7208 }
7209 }
7210 promised
7211 })
7212 }
7213
7214 fn freshen_existing_object(&self, oid: &ObjectId) -> Result<bool> {
7215 if self.freshen_loose_object(oid)? {
7216 return Ok(true);
7217 }
7218 if self.freshen_packed_object(oid)? {
7219 return Ok(true);
7220 }
7221 for alternate in &self.alternates {
7222 if Self::without_alternates(alternate, self.format).freshen_existing_object(oid)? {
7223 return Ok(true);
7224 }
7225 }
7226 self.loose.invalidate_cache();
7228 self.freshen_loose_object(oid)
7229 }
7230
7231 fn freshen_loose_object(&self, oid: &ObjectId) -> Result<bool> {
7232 let path = self.loose.object_path(oid)?;
7233 freshen_file_mtime(&path)
7234 }
7235
7236 fn freshen_packed_object(&self, oid: &ObjectId) -> Result<bool> {
7237 let Some(pack_lookup) = self.find_pack_containing(oid)? else {
7238 return Ok(false);
7239 };
7240 freshen_file_mtime(pack_lookup.pack_path())
7241 }
7242}
7243
7244fn freshen_file_mtime(path: &Path) -> Result<bool> {
7245 let file = match fs::OpenOptions::new().read(true).open(path) {
7246 Ok(file) => file,
7247 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(false),
7248 Err(err) => return Err(GitError::Io(err.to_string())),
7249 };
7250 file.set_modified(std::time::SystemTime::now())
7251 .map_err(|err| GitError::Io(err.to_string()))?;
7252 Ok(true)
7253}
7254
7255fn promisor_pack_object_ids(objects_dir: &Path, format: ObjectFormat) -> Result<HashSet<ObjectId>> {
7256 let pack_dir = objects_dir.join("pack");
7257 let mut oids = HashSet::new();
7258 if !pack_dir.exists() {
7259 return Ok(oids);
7260 }
7261 for entry in fs::read_dir(pack_dir)? {
7262 let path = entry?.path();
7263 if path.extension().and_then(|ext| ext.to_str()) != Some("idx") {
7264 continue;
7265 }
7266 if !path.with_extension("pack").exists() || !path.with_extension("promisor").exists() {
7267 continue;
7268 }
7269 let index = PackIndex::parse(&fs::read(path)?, format)?;
7270 oids.extend(index.entries.into_iter().map(|entry| entry.oid));
7271 }
7272 Ok(oids)
7273}
7274
7275fn promisor_object_links(format: ObjectFormat, object: &EncodedObject) -> Vec<ObjectId> {
7276 match object.object_type {
7277 ObjectType::Commit => Commit::parse_ref(format, &object.body)
7278 .map(|commit| {
7279 let mut links = Vec::with_capacity(commit.parents.len() + 1);
7280 links.push(commit.tree);
7281 links.extend(commit.parents);
7282 links
7283 })
7284 .unwrap_or_default(),
7285 ObjectType::Tree => TreeEntries::new(format, &object.body)
7286 .filter_map(|entry| entry.ok().map(|entry| entry.oid))
7287 .collect(),
7288 ObjectType::Tag => Tag::parse_ref(format, &object.body)
7289 .map(|tag| vec![tag.object])
7290 .unwrap_or_default(),
7291 ObjectType::Blob => Vec::new(),
7292 }
7293}
7294
7295impl ObjectWriter for FileObjectDatabase {
7296 fn write_object(&self, object: EncodedObject) -> Result<ObjectId> {
7297 let oid = object.object_id(self.format)?;
7302 if self.freshen_existing_object(&oid)? {
7303 return Ok(oid);
7304 }
7305 self.loose.write_object(object)
7306 }
7307}
7308
7309fn write_pack_component(path: &Path, bytes: &[u8]) -> Result<()> {
7310 if path.exists() {
7311 return Ok(());
7312 }
7313 let parent = path
7314 .parent()
7315 .ok_or_else(|| GitError::InvalidPath("pack component path has no parent".into()))?;
7316 fs::create_dir_all(parent)?;
7317 let temp_path = unique_temp_path(parent);
7318 let write_result = (|| -> Result<()> {
7319 {
7320 let mut file = fs::OpenOptions::new()
7321 .write(true)
7322 .create_new(true)
7323 .open(&temp_path)?;
7324 file.write_all(bytes)?;
7325 file.sync_all()?;
7326 }
7327 match fs::rename(&temp_path, path) {
7328 Ok(()) => Ok(()),
7329 Err(_) if path.exists() => {
7330 let _ = fs::remove_file(&temp_path);
7331 Ok(())
7332 }
7333 Err(err) => Err(GitError::Io(err.to_string())),
7334 }
7335 })();
7336 if write_result.is_err() {
7337 let _ = fs::remove_file(&temp_path);
7338 }
7339 write_result
7340}
7341
7342fn write_promisor_pack_sidecar(
7343 pack_dir: &Path,
7344 pack_name: &str,
7345 promisor: bool,
7346) -> Result<Option<PathBuf>> {
7347 if !promisor {
7348 return Ok(None);
7349 }
7350 let path = pack_dir.join(format!("{pack_name}.promisor"));
7351 write_pack_component(&path, b"")?;
7352 Ok(Some(path))
7353}
7354
7355const MAX_LOOSE_HEADER_LEN: usize = 32;
7360
7361fn loose_header_too_long(oid: &ObjectId) -> GitError {
7366 GitError::InvalidObject(format!(
7367 "header for {oid} too long, exceeds {MAX_LOOSE_HEADER_LEN} bytes"
7368 ))
7369}
7370
7371fn loose_unpack_header_failed(oid: &ObjectId) -> GitError {
7375 GitError::InvalidObject(format!("unable to unpack {oid} header"))
7376}
7377
7378fn inflate_header_diagnostic(input: &[u8]) -> Option<&'static str> {
7386 let [cmf, flg, ..] = *input else { return None };
7387 if ((u16::from(cmf) << 8) | u16::from(flg)) % 31 != 0 {
7388 return Some("inflate: data stream error (incorrect header check)");
7389 }
7390 if cmf & 0x0f != 8 {
7391 return Some("inflate: data stream error (unknown compression method)");
7392 }
7393 if cmf >> 4 > 7 {
7394 return Some("inflate: data stream error (invalid window size)");
7395 }
7396 if flg & 0x20 != 0 {
7397 return Some("inflate: needs dictionary (no message)");
7398 }
7399 None
7400}
7401
7402fn emit_inflate_diagnostic(input: &[u8]) {
7405 if let Some(diagnostic) = inflate_header_diagnostic(input) {
7406 eprintln!("error: {diagnostic}");
7407 }
7408}
7409
7410#[derive(Debug, Clone, PartialEq, Eq)]
7413pub enum LooseObjectIntegrity {
7414 Ok,
7416 HashMismatch { actual: ObjectId },
7419 Corrupt,
7422}
7423
7424#[derive(Debug, Clone)]
7425pub struct LooseObjectStore {
7426 objects_dir: PathBuf,
7427 format: ObjectFormat,
7428 loose_cache: Arc<Mutex<LoosePresenceCache>>,
7437}
7438
7439impl LooseObjectStore {
7440 pub fn new(objects_dir: impl Into<PathBuf>, format: ObjectFormat) -> Self {
7441 Self {
7442 objects_dir: objects_dir.into(),
7443 format,
7444 loose_cache: Arc::new(Mutex::new(LoosePresenceCache::default())),
7445 }
7446 }
7447
7448 fn cached_loose_presence(&self, oid: &ObjectId) -> Option<bool> {
7453 let mut guard = self.loose_cache.lock().ok()?;
7454 let fanout = oid.as_bytes()[0];
7455 if !guard.loaded_fanouts.contains(&fanout) {
7456 if guard.present_fanouts.is_none() {
7463 guard.present_fanouts = Some(present_loose_fanouts(&self.objects_dir).ok()?);
7464 }
7465 let fanout_present = guard
7466 .present_fanouts
7467 .as_ref()
7468 .is_some_and(|present| present.contains(&fanout));
7469 if fanout_present {
7470 collect_loose_fanout_object_ids(
7471 &self.objects_dir,
7472 self.format,
7473 fanout,
7474 &mut guard.objects,
7475 )
7476 .ok()?;
7477 }
7478 guard.loaded_fanouts.insert(fanout);
7485 }
7486 Some(guard.objects.contains(oid))
7487 }
7488
7489 fn loose_object_ids_cached(&self) -> Result<Vec<ObjectId>> {
7493 if let Ok(mut guard) = self.loose_cache.lock() {
7494 guard.objects = loose_object_id_set(&self.objects_dir, self.format)?;
7495 guard.loaded_fanouts = (0..=u8::MAX).collect();
7496 let mut ids = guard.objects.iter().copied().collect::<Vec<_>>();
7497 ids.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
7498 return Ok(ids);
7499 }
7500 loose_object_ids(&self.objects_dir, self.format)
7501 }
7502
7503 fn note_loose_write(&self, oid: ObjectId) {
7507 if let Ok(mut guard) = self.loose_cache.lock() {
7508 let fanout = oid.as_bytes()[0];
7513 if let Some(present) = guard.present_fanouts.as_mut() {
7514 present.insert(fanout);
7515 }
7516 guard.objects.insert(oid);
7517 }
7518 }
7519
7520 pub(crate) fn invalidate_cache(&self) {
7523 if let Ok(mut guard) = self.loose_cache.lock() {
7524 *guard = LoosePresenceCache::default();
7525 }
7526 }
7527
7528 pub fn from_git_dir(git_dir: impl AsRef<Path>, format: ObjectFormat) -> Self {
7529 Self::new(repository_objects_dir(git_dir), format)
7530 }
7531
7532 fn validate_oid_format(&self, oid: &ObjectId) -> Result<()> {
7533 if oid.format() != self.format {
7534 return Err(GitError::InvalidObjectId(format!(
7535 "object {oid} uses {}, store uses {}",
7536 oid.format().name(),
7537 self.format.name()
7538 )));
7539 }
7540 Ok(())
7541 }
7542
7543 pub fn object_path(&self, oid: &ObjectId) -> Result<PathBuf> {
7544 self.validate_oid_format(oid)?;
7545 let hex = oid.to_hex();
7546 Ok(self.objects_dir.join(&hex[..2]).join(&hex[2..]))
7547 }
7548
7549 pub fn exists(&self, oid: &ObjectId) -> Result<bool> {
7550 self.validate_oid_format(oid)?;
7551 if self.cached_loose_presence(oid) == Some(false) {
7552 return Ok(false);
7553 }
7554 let path = self.object_path(oid)?;
7555 Ok(path.exists())
7556 }
7557
7558 pub fn disk_size(&self, oid: &ObjectId) -> Result<Option<u64>> {
7559 self.validate_oid_format(oid)?;
7560 if self.cached_loose_presence(oid) == Some(false) {
7561 return Ok(None);
7562 }
7563 let path = self.object_path(oid)?;
7564 match fs::metadata(path) {
7565 Ok(metadata) => Ok(Some(metadata.len())),
7566 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
7567 Err(err) => Err(GitError::Io(err.to_string())),
7568 }
7569 }
7570
7571 pub fn read_header(&self, oid: &ObjectId) -> Result<Option<(ObjectType, u64)>> {
7576 self.validate_oid_format(oid)?;
7577 if self.cached_loose_presence(oid) == Some(false) {
7578 return Ok(None);
7579 }
7580 let path = self.object_path(oid)?;
7581 let compressed = match fs::read(&path) {
7582 Ok(compressed) => compressed,
7583 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
7584 Err(err) => return Err(GitError::Io(err.to_string())),
7585 };
7586 match inflate_loose_header(&compressed)? {
7587 LooseHeader::Ok(header) => {
7588 let header = std::str::from_utf8(&header)
7589 .map_err(|err| GitError::InvalidObject(err.to_string()))?;
7590 let (kind, size) = header
7591 .split_once(' ')
7592 .ok_or_else(|| GitError::InvalidObject("missing object size".into()))?;
7593 let object_type = kind.parse::<ObjectType>()?;
7594 let size = size
7595 .parse::<u64>()
7596 .map_err(|_| GitError::InvalidObject("invalid object size".into()))?;
7597 Ok(Some((object_type, size)))
7598 }
7599 LooseHeader::Bad => {
7600 emit_inflate_diagnostic(compressed.get(..2).unwrap_or(&compressed));
7603 Err(loose_unpack_header_failed(oid))
7604 }
7605 LooseHeader::TooLong => {
7606 Err(loose_header_too_long(oid))
7611 }
7612 }
7613 }
7614
7615 pub fn object_ids(&self) -> Result<Vec<ObjectId>> {
7617 self.loose_object_ids_cached()
7618 }
7619
7620 pub fn verify_object(
7628 &self,
7629 oid: &ObjectId,
7630 display_path: &str,
7631 ) -> Result<Option<LooseObjectIntegrity>> {
7632 let path = self.object_path(oid)?;
7633 let compressed = match fs::read(&path) {
7634 Ok(compressed) => compressed,
7635 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
7636 Err(err) => return Err(GitError::Io(err.to_string())),
7637 };
7638 let mut decoder = ZlibDecoder::new(compressed.as_slice());
7639 let mut framed = Vec::new();
7640 if decoder.read_to_end(&mut framed).is_err() {
7641 emit_inflate_diagnostic(&compressed);
7642 if framed_loose_header_terminated(&framed) {
7650 eprintln!("error: corrupt loose object '{oid}'");
7651 eprintln!("error: unable to unpack contents of {display_path}");
7652 } else {
7653 eprintln!("error: unable to unpack header of {display_path}");
7654 }
7655 return Ok(Some(LooseObjectIntegrity::Corrupt));
7656 }
7657 if !framed_loose_header_terminated(&framed) {
7658 eprintln!("error: unable to unpack header of {display_path}");
7661 return Ok(Some(LooseObjectIntegrity::Corrupt));
7662 }
7663 if (decoder.total_in() as usize) < compressed.len() {
7670 eprintln!("error: garbage at end of loose object '{oid}'");
7674 eprintln!("error: unable to unpack contents of {display_path}");
7675 return Ok(Some(LooseObjectIntegrity::Corrupt));
7676 }
7677 if let Some(declared) = loose_header_declared_size(&framed) {
7684 let nul = framed.iter().position(|&b| b == 0).unwrap_or(framed.len());
7685 let body_len = framed.len() - (nul + 1).min(framed.len());
7686 if body_len < declared {
7687 eprintln!("error: corrupt loose object '{oid}'");
7688 eprintln!("error: unable to unpack contents of {display_path}");
7689 return Ok(Some(LooseObjectIntegrity::Corrupt));
7690 }
7691 }
7692 let Ok(object) = parse_framed_object(&framed) else {
7693 if let Some(header) = loose_header_with_unknown_type(&framed) {
7698 eprintln!("error: unable to parse type from header '{header}' of {display_path}");
7699 } else {
7700 eprintln!("error: unable to parse header of {display_path}");
7701 }
7702 return Ok(Some(LooseObjectIntegrity::Corrupt));
7703 };
7704 let actual = object.object_id(self.format)?;
7705 if &actual != oid {
7706 return Ok(Some(LooseObjectIntegrity::HashMismatch { actual }));
7707 }
7708 Ok(Some(LooseObjectIntegrity::Ok))
7709 }
7710}
7711
7712fn framed_loose_header_terminated(framed: &[u8]) -> bool {
7716 framed
7717 .iter()
7718 .take(MAX_LOOSE_HEADER_LEN)
7719 .any(|byte| *byte == 0)
7720}
7721
7722fn loose_header_with_unknown_type(framed: &[u8]) -> Option<String> {
7727 let nul = framed.iter().position(|&b| b == 0)?;
7728 let header = std::str::from_utf8(&framed[..nul]).ok()?;
7729 let (kind, size) = header.split_once(' ')?;
7730 let size: usize = size.parse().ok()?;
7731 if framed.len() - (nul + 1) != size {
7734 return None;
7735 }
7736 if kind.parse::<ObjectType>().is_ok() {
7739 return None;
7740 }
7741 Some(header.to_string())
7742}
7743
7744fn loose_header_declared_size(framed: &[u8]) -> Option<usize> {
7748 let nul = framed.iter().position(|&b| b == 0)?;
7749 let header = std::str::from_utf8(&framed[..nul]).ok()?;
7750 let (_kind, size) = header.split_once(' ')?;
7751 size.parse::<usize>().ok()
7752}
7753
7754enum LooseHeader {
7760 Ok(Vec<u8>),
7763 Bad,
7765 TooLong,
7767}
7768
7769fn inflate_loose_header(compressed: &[u8]) -> Result<LooseHeader> {
7783 let mut out = [0u8; MAX_LOOSE_HEADER_LEN];
7784 let mut decompress = Decompress::new(true);
7785 let status = decompress.decompress(compressed, &mut out, FlushDecompress::None);
7789 let produced = decompress.total_out() as usize;
7790 match status {
7791 Ok(_) => {
7792 let window = &out[..produced.min(MAX_LOOSE_HEADER_LEN)];
7793 match window.iter().position(|&byte| byte == 0) {
7794 Some(nul) => Ok(LooseHeader::Ok(window[..nul].to_vec())),
7795 None => Ok(LooseHeader::TooLong),
7799 }
7800 }
7801 Err(_) => Ok(LooseHeader::Bad),
7803 }
7804}
7805
7806impl ObjectReader for LooseObjectStore {
7807 fn read_object(&self, oid: &ObjectId) -> Result<Arc<EncodedObject>> {
7808 self.validate_oid_format(oid)?;
7809 if self.cached_loose_presence(oid) == Some(false) {
7813 return Err(GitError::object_not_found_in(
7814 *oid,
7815 MissingObjectContext::Read,
7816 ));
7817 }
7818 let path = self.object_path(oid)?;
7819 let compressed = match fs::read(&path) {
7820 Ok(compressed) => compressed,
7821 Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
7822 return Err(GitError::object_not_found_in(
7823 *oid,
7824 MissingObjectContext::Read,
7825 ));
7826 }
7827 Err(err) => return Err(GitError::Io(err.to_string())),
7828 };
7829 let mut decoder = ZlibDecoder::new(compressed.as_slice());
7830 let mut framed = Vec::new();
7831 if decoder.read_to_end(&mut framed).is_err() {
7832 emit_inflate_diagnostic(&compressed);
7833 if !framed_loose_header_terminated(&framed) {
7838 return Err(loose_unpack_header_failed(oid));
7839 }
7840 return Err(GitError::InvalidObject(format!(
7841 "corrupt loose object '{oid}'"
7842 )));
7843 }
7844 if framed
7849 .iter()
7850 .take(MAX_LOOSE_HEADER_LEN)
7851 .all(|byte| *byte != 0)
7852 {
7853 return Err(loose_header_too_long(oid));
7854 }
7855 let object = parse_framed_object(&framed)?;
7856 if verify_reads_enabled() {
7860 let actual = object.object_id(self.format)?;
7861 if &actual != oid {
7862 return Err(GitError::InvalidObject(format!(
7863 "loose object {} hashes to {actual}",
7864 path.display()
7865 )));
7866 }
7867 }
7868 Ok(Arc::new(object))
7869 }
7870}
7871
7872impl ObjectWriter for LooseObjectStore {
7873 fn write_object(&self, object: EncodedObject) -> Result<ObjectId> {
7874 let oid = object.object_id(self.format)?;
7875 let path = self.object_path(&oid)?;
7876 if path.exists() {
7877 self.note_loose_write(oid);
7878 return Ok(oid);
7879 }
7880 let parent = path
7881 .parent()
7882 .ok_or_else(|| GitError::InvalidPath("loose object path has no parent".into()))?;
7883 fs::create_dir_all(parent)?;
7884 let temp_path = unique_temp_path(parent);
7885 let write_result = (|| -> Result<()> {
7886 let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
7887 encoder.write_all(&object.framed_bytes())?;
7888 let compressed = encoder.finish()?;
7889 {
7890 let mut file = fs::OpenOptions::new()
7891 .write(true)
7892 .create_new(true)
7893 .open(&temp_path)?;
7894 file.write_all(&compressed)?;
7895 }
7905 match fs::rename(&temp_path, &path) {
7906 Ok(()) => Ok(()),
7907 Err(_) if path.exists() => {
7908 let _ = fs::remove_file(&temp_path);
7909 Ok(())
7910 }
7911 Err(err) => Err(GitError::Io(err.to_string())),
7912 }
7913 })();
7914 if write_result.is_err() {
7915 let _ = fs::remove_file(&temp_path);
7916 }
7917 write_result?;
7918 self.note_loose_write(oid);
7919 Ok(oid)
7920 }
7921}
7922
7923fn unique_temp_path(parent: &Path) -> PathBuf {
7924 let id = TEMPFILE_COUNTER.fetch_add(1, Ordering::Relaxed);
7925 parent.join(format!("tmp_obj_{}_{}", std::process::id(), id))
7926}
7927
7928#[cfg(test)]
7929mod tests {
7930 use super::*;
7931 use sley_core::BString;
7932 use sley_object::{Commit, EncodedObject, ObjectType, Tag, Tree, TreeEntry};
7933 use sley_pack::{PackFile, PackWriteOptions};
7934
7935 fn blob_of(byte: u8, len: usize) -> EncodedObject {
7936 EncodedObject::new(ObjectType::Blob, vec![byte; len])
7937 }
7938
7939 fn cached_blob_of(byte: u8, len: usize) -> Arc<EncodedObject> {
7940 Arc::new(blob_of(byte, len))
7941 }
7942
7943 fn read_object_for_assert(reader: &impl ObjectReader, oid: &ObjectId) -> EncodedObject {
7944 reader
7945 .read_object(oid)
7946 .expect("test operation should succeed")
7947 .as_ref()
7948 .clone()
7949 }
7950
7951 #[test]
7952 fn lru_cache_evicts_by_byte_budget_least_recently_used_first() {
7953 let one = cached_object_cost(&blob_of(0, 1000));
7955 let mut cache = LruCache::<u32>::new(one * 2 + 8);
7956 cache.put(1, cached_blob_of(b'a', 1000));
7957 cache.put(2, cached_blob_of(b'b', 1000));
7958 assert!(cache.get(&1).is_some());
7960 cache.put(3, cached_blob_of(b'c', 1000));
7961 assert!(cache.get(&1).is_some());
7963 assert!(cache.get(&2).is_none());
7964 assert!(cache.get(&3).is_some());
7965 }
7966
7967 #[test]
7968 fn lru_cache_zero_budget_is_inert() {
7969 let mut cache = LruCache::<u32>::new(0);
7970 cache.put(1, cached_blob_of(b'a', 16));
7971 assert!(cache.get(&1).is_none());
7972 }
7973
7974 #[test]
7975 fn lru_cache_skips_object_larger_than_budget_and_clears_stale_entry() {
7976 let mut cache = LruCache::<u32>::new(cached_object_cost(&blob_of(0, 100)));
7977 cache.put(1, cached_blob_of(b'a', 50));
7978 assert!(cache.get(&1).is_some());
7979 cache.put(1, cached_blob_of(b'b', 10_000));
7982 assert!(cache.get(&1).is_none());
7983 cache.put(2, cached_blob_of(b'c', 50));
7986 assert!(cache.get(&2).is_some());
7987 }
7988
7989 #[test]
7990 fn lru_cache_replacing_entry_updates_byte_accounting() {
7991 let small = cached_object_cost(&blob_of(0, 500));
7994 let mut cache = LruCache::<u32>::new(small * 2 + 200);
7995 cache.put(1, cached_blob_of(b'a', 500));
7996 cache.put(2, cached_blob_of(b'b', 500));
7997 assert!(cache.get(&1).is_some());
7998 assert!(cache.get(&2).is_some());
7999 cache.put(2, cached_blob_of(b'b', 1000));
8004 assert!(cache.get(&2).is_some());
8005 assert!(cache.get(&1).is_none());
8006 }
8007
8008 #[test]
8009 fn write_and_validate_blob() {
8010 let db = ObjectDatabase::new(ObjectFormat::Sha1);
8011 let oid = db
8012 .write_object(EncodedObject::new(ObjectType::Blob, b"hello\n".to_vec()))
8013 .expect("test operation should succeed");
8014 assert_eq!(oid.to_hex(), "ce013625030ba8dba906f756967f9e9ca394464a");
8015 db.validate(&oid).expect("test operation should succeed");
8016 }
8017
8018 #[test]
8019 fn loose_store_writes_and_reads_object() {
8020 let root = std::env::temp_dir().join(format!(
8021 "sley-loose-store-{}-{}",
8022 std::process::id(),
8023 TEMPFILE_COUNTER.fetch_add(1, Ordering::Relaxed)
8024 ));
8025 let store = LooseObjectStore::new(root.join("objects"), ObjectFormat::Sha1);
8026 let object = EncodedObject::new(ObjectType::Blob, b"hello\n".to_vec());
8027 let oid = store
8028 .write_object(object.clone())
8029 .expect("test operation should succeed");
8030 assert_eq!(read_object_for_assert(&store, &oid), object);
8031 assert!(
8032 store
8033 .object_path(&oid)
8034 .expect("test operation should succeed")
8035 .exists()
8036 );
8037 fs::remove_dir_all(root).expect("test operation should succeed");
8038 }
8039
8040 #[test]
8041 fn read_header_detects_corruption_within_gits_header_window() {
8042 let root = temp_root("sley-loose-header-corrupt");
8050 let store = LooseObjectStore::new(root.join("objects"), ObjectFormat::Sha1);
8051 let object = EncodedObject::new(ObjectType::Blob, b"content\n".to_vec());
8052 let oid = store
8053 .write_object(object)
8054 .expect("test operation should succeed");
8055 let path = store
8056 .object_path(&oid)
8057 .expect("test operation should succeed");
8058 let mut bytes = fs::read(&path).expect("test operation should succeed");
8059 bytes[10] = 0;
8063 fs::write(&path, &bytes).expect("test operation should succeed");
8064 store.invalidate_cache();
8065 let err = store
8066 .read_header(&oid)
8067 .expect_err("corrupt loose header must fail like git's ULHR_BAD");
8068 let msg = err.to_string();
8069 assert!(
8070 msg.contains("unable to unpack") && msg.contains(&oid.to_hex()),
8071 "expected git's ULHR_BAD message, got: {msg}"
8072 );
8073 fs::remove_dir_all(root).expect("test operation should succeed");
8074 }
8075
8076 #[test]
8077 fn read_header_ignores_corruption_past_gits_header_window() {
8078 let root = temp_root("sley-loose-header-deep-corrupt");
8083 let store = LooseObjectStore::new(root.join("objects"), ObjectFormat::Sha1);
8084 let body: Vec<u8> = (0..4096u32)
8087 .map(|i| (i.wrapping_mul(2654435761)) as u8)
8088 .collect();
8089 let object = EncodedObject::new(ObjectType::Blob, body.clone());
8090 let oid = store
8091 .write_object(object)
8092 .expect("test operation should succeed");
8093 let path = store
8094 .object_path(&oid)
8095 .expect("test operation should succeed");
8096 let mut bytes = fs::read(&path).expect("test operation should succeed");
8097 let deep = bytes.len() / 2;
8098 bytes[deep] ^= 0xff;
8099 fs::write(&path, &bytes).expect("test operation should succeed");
8100 store.invalidate_cache();
8101 let header = store
8102 .read_header(&oid)
8103 .expect("header-only read must still succeed for deep body corruption");
8104 assert_eq!(header, Some((ObjectType::Blob, body.len() as u64)));
8105 fs::remove_dir_all(root).expect("test operation should succeed");
8106 }
8107
8108 #[test]
8109 fn file_database_reads_object_from_pack_index() {
8110 let root = temp_root("sley-file-odb-pack");
8111 let git_dir = root.join(".git");
8112 let pack_dir = git_dir.join("objects").join("pack");
8113 fs::create_dir_all(&pack_dir).expect("test operation should succeed");
8114 let object = EncodedObject::new(ObjectType::Blob, b"packed\n".to_vec());
8115 let oid = object
8116 .object_id(ObjectFormat::Sha1)
8117 .expect("test operation should succeed");
8118 let written = PackFile::write_undeltified_sha1(std::slice::from_ref(&object))
8119 .expect("test operation should succeed");
8120 let pack_name = written.checksum.to_hex();
8121 fs::write(
8122 pack_dir.join(format!("pack-{pack_name}.pack")),
8123 written.pack,
8124 )
8125 .expect("test operation should succeed");
8126 fs::write(
8127 pack_dir.join(format!("pack-{pack_name}.idx")),
8128 written.index,
8129 )
8130 .expect("test operation should succeed");
8131
8132 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8133 assert!(db.contains(&oid).expect("test operation should succeed"));
8134 assert_eq!(read_object_for_assert(&db, &oid), object);
8135 fs::remove_dir_all(root).expect("test operation should succeed");
8136 }
8137
8138 #[test]
8139 fn file_database_loose_cache_observes_same_process_write_after_miss() {
8140 let root = temp_root("sley-file-odb-loose-cache-write");
8141 let git_dir = root.join(".git");
8142 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8143 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8144
8145 let object = EncodedObject::new(ObjectType::Blob, b"written after miss\n".to_vec());
8146 let oid = object
8147 .object_id(ObjectFormat::Sha1)
8148 .expect("test operation should succeed");
8149
8150 assert!(matches!(db.read_object(&oid), Err(GitError::NotFound(_))));
8151 db.loose()
8152 .write_object(object.clone())
8153 .expect("test operation should succeed");
8154
8155 assert_eq!(read_object_for_assert(&db, &oid), object);
8156 fs::remove_dir_all(root).expect("test operation should succeed");
8157 }
8158
8159 #[test]
8160 fn present_loose_fanouts_lists_only_existing_two_hex_dirs() {
8161 let root = temp_root("sley-present-fanouts");
8162 let objects = root.join("objects");
8163 fs::create_dir_all(objects.join("ab")).expect("test operation should succeed");
8164 fs::create_dir_all(objects.join("0f")).expect("test operation should succeed");
8165 fs::create_dir_all(objects.join("pack")).expect("test operation should succeed");
8167 fs::create_dir_all(objects.join("info")).expect("test operation should succeed");
8168 fs::create_dir_all(objects.join("zz")).expect("test operation should succeed");
8171 fs::write(objects.join("ff"), b"not a dir").expect("test operation should succeed");
8172
8173 let present = present_loose_fanouts(&objects).expect("test operation should succeed");
8174 assert_eq!(
8175 present,
8176 HashSet::from([0xab, 0x0f]),
8177 "only the genuine two-hex fanout directories should be reported"
8178 );
8179 fs::remove_dir_all(root).expect("test operation should succeed");
8180 }
8181
8182 #[test]
8183 fn present_loose_fanouts_empty_when_objects_dir_absent() {
8184 let root = temp_root("sley-present-fanouts-absent");
8185 fs::create_dir_all(&root).expect("test operation should succeed");
8186 let present =
8189 present_loose_fanouts(&root.join("objects")).expect("test operation should succeed");
8190 assert!(present.is_empty());
8191 fs::remove_dir_all(root).expect("test operation should succeed");
8192 }
8193
8194 #[test]
8195 fn packed_only_repo_read_does_not_probe_loose_fanout_dirs() {
8196 let root = temp_root("sley-packed-no-loose-probe");
8200 let git_dir = root.join(".git");
8201 let pack_dir = git_dir.join("objects").join("pack");
8202 fs::create_dir_all(&pack_dir).expect("test operation should succeed");
8203 let object = EncodedObject::new(ObjectType::Blob, b"packed only\n".to_vec());
8204 let oid = object
8205 .object_id(ObjectFormat::Sha1)
8206 .expect("test operation should succeed");
8207 let written = PackFile::write_undeltified_sha1(std::slice::from_ref(&object))
8208 .expect("test operation should succeed");
8209 let pack_name = written.checksum.to_hex();
8210 fs::write(
8211 pack_dir.join(format!("pack-{pack_name}.pack")),
8212 written.pack,
8213 )
8214 .expect("test operation should succeed");
8215 fs::write(
8216 pack_dir.join(format!("pack-{pack_name}.idx")),
8217 written.index,
8218 )
8219 .expect("test operation should succeed");
8220
8221 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8222 assert_eq!(
8225 db.read_object_header(&oid)
8226 .expect("test operation should succeed"),
8227 Some((ObjectType::Blob, object.body.len() as u64))
8228 );
8229 assert_eq!(read_object_for_assert(&db, &oid), object);
8230
8231 let fanout_hex = format!("{:02x}", oid.as_bytes()[0]);
8234 assert!(
8235 !git_dir.join("objects").join(&fanout_hex).exists(),
8236 "reading a packed object must not create its loose fanout dir"
8237 );
8238 if let Ok(guard) = db.loose().loose_cache.lock() {
8239 assert_eq!(
8240 guard.present_fanouts.as_ref(),
8241 Some(&HashSet::new()),
8242 "an all-packed repo must learn zero present fanouts"
8243 );
8244 }
8245 fs::remove_dir_all(root).expect("test operation should succeed");
8246 }
8247
8248 #[test]
8249 fn loose_object_in_existing_fanout_still_resolves_through_cache() {
8250 let root = temp_root("sley-loose-resolves");
8253 let git_dir = root.join(".git");
8254 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8255 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8256 let object = EncodedObject::new(ObjectType::Blob, b"a genuine loose object\n".to_vec());
8257 let oid = db
8258 .write_object(object.clone())
8259 .expect("test operation should succeed");
8260 db.refresh_read_cache();
8262 assert_eq!(
8263 db.read_object_header(&oid)
8264 .expect("test operation should succeed"),
8265 Some((ObjectType::Blob, object.body.len() as u64))
8266 );
8267 assert_eq!(read_object_for_assert(&db, &oid), object);
8268 fs::remove_dir_all(root).expect("test operation should succeed");
8269 }
8270
8271 #[test]
8272 fn loose_write_into_previously_absent_fanout_is_found_after_cache_built() {
8273 let root = temp_root("sley-new-loose-after-cache");
8277 let git_dir = root.join(".git");
8278 let pack_dir = git_dir.join("objects").join("pack");
8279 fs::create_dir_all(&pack_dir).expect("test operation should succeed");
8280 let packed = EncodedObject::new(ObjectType::Blob, b"seed packed\n".to_vec());
8283 let written = PackFile::write_undeltified_sha1(std::slice::from_ref(&packed))
8284 .expect("test operation should succeed");
8285 let pack_name = written.checksum.to_hex();
8286 fs::write(
8287 pack_dir.join(format!("pack-{pack_name}.pack")),
8288 written.pack,
8289 )
8290 .expect("test operation should succeed");
8291 fs::write(
8292 pack_dir.join(format!("pack-{pack_name}.idx")),
8293 written.index,
8294 )
8295 .expect("test operation should succeed");
8296 let packed_oid = packed
8297 .object_id(ObjectFormat::Sha1)
8298 .expect("test operation should succeed");
8299
8300 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8301 assert_eq!(read_object_for_assert(&db, &packed_oid), packed);
8303
8304 let loose = EncodedObject::new(ObjectType::Blob, b"new loose into empty fanout\n".to_vec());
8308 let loose_oid = db
8309 .write_object(loose.clone())
8310 .expect("test operation should succeed");
8311 assert_eq!(
8312 db.read_object_header(&loose_oid)
8313 .expect("test operation should succeed"),
8314 Some((ObjectType::Blob, loose.body.len() as u64)),
8315 "a loose object written after the present-fanout cache was built must be found"
8316 );
8317 assert_eq!(read_object_for_assert(&db, &loose_oid), loose);
8318 assert_eq!(read_object_for_assert(&db, &packed_oid), packed);
8320 fs::remove_dir_all(root).expect("test operation should succeed");
8321 }
8322
8323 #[test]
8324 fn loose_copy_is_consulted_when_packed_copy_is_corrupt() {
8325 let root = temp_root("sley-loose-shadows-corrupt-pack");
8330 let git_dir = root.join(".git");
8331 let pack_dir = git_dir.join("objects").join("pack");
8332 fs::create_dir_all(&pack_dir).expect("test operation should succeed");
8333 let object = EncodedObject::new(ObjectType::Blob, b"shadow me\n".to_vec());
8334 let oid = object
8335 .object_id(ObjectFormat::Sha1)
8336 .expect("test operation should succeed");
8337
8338 let written = PackFile::write_undeltified_sha1(std::slice::from_ref(&object))
8340 .expect("test operation should succeed");
8341 let pack_name = written.checksum.to_hex();
8342 let pack_path = pack_dir.join(format!("pack-{pack_name}.pack"));
8343 fs::write(&pack_path, written.pack).expect("test operation should succeed");
8344 fs::write(
8345 pack_dir.join(format!("pack-{pack_name}.idx")),
8346 written.index,
8347 )
8348 .expect("test operation should succeed");
8349 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8350 db.loose()
8351 .write_object(object.clone())
8352 .expect("test operation should succeed");
8353
8354 let mut pack_bytes = fs::read(&pack_path).expect("test operation should succeed");
8357 let mid = pack_bytes.len() / 2;
8358 pack_bytes[mid] ^= 0xff;
8359 fs::write(&pack_path, &pack_bytes).expect("test operation should succeed");
8360 db.refresh_read_cache();
8361
8362 assert_eq!(
8363 read_object_for_assert(&db, &oid),
8364 object,
8365 "a good loose copy must shadow a corrupt packed copy"
8366 );
8367 fs::remove_dir_all(root).expect("test operation should succeed");
8368 }
8369
8370 #[test]
8371 fn loose_ofs_delta_base_recovers_child_from_corrupt_pack_base() {
8372 let root = temp_root("sley-ofs-delta-base-corrupt-loose-recovery");
8373 let git_dir = root.join(".git");
8374 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8375 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8376
8377 let mut base_body = Vec::new();
8378 for idx in 0..256u16 {
8379 base_body.extend_from_slice(format!("shared line {idx:03}\n").as_bytes());
8380 }
8381 let base = EncodedObject::new(ObjectType::Blob, base_body.clone());
8382 let mut first_body = base_body;
8383 first_body.extend_from_slice(b"first delta payload\n");
8384 let first = EncodedObject::new(ObjectType::Blob, first_body.clone());
8385 let mut second_body = first_body;
8386 second_body.extend_from_slice(b"second delta payload\n");
8387 let second = EncodedObject::new(ObjectType::Blob, second_body);
8388 let base_oid = base
8389 .object_id(ObjectFormat::Sha1)
8390 .expect("test operation should succeed");
8391 let first_oid = first
8392 .object_id(ObjectFormat::Sha1)
8393 .expect("test operation should succeed");
8394 let second_oid = second
8395 .object_id(ObjectFormat::Sha1)
8396 .expect("test operation should succeed");
8397
8398 let options = PackWriteOptions::new()
8399 .with_prefer_ofs_delta(true)
8400 .with_reorder(false);
8401 let written = PackFile::write_packed_with_options(
8402 &[base, first.clone(), second.clone()],
8403 ObjectFormat::Sha1,
8404 &options,
8405 )
8406 .expect("test operation should succeed");
8407 let stats = PackFile::verify_pack_stats(&written.pack, ObjectFormat::Sha1)
8408 .expect("test operation should succeed");
8409 let first_stat = stats
8410 .objects
8411 .iter()
8412 .find(|stat| stat.oid == first_oid)
8413 .expect("first object should be packed");
8414 let second_stat = stats
8415 .objects
8416 .iter()
8417 .find(|stat| stat.oid == second_oid)
8418 .expect("second object should be packed");
8419 assert_eq!(first_stat.base_oid, Some(base_oid));
8420 assert_eq!(second_stat.base_oid, Some(first_oid));
8421
8422 let installed = db
8423 .install_pack(&written)
8424 .expect("test operation should succeed");
8425 db.loose()
8426 .write_object(first.clone())
8427 .expect("test operation should succeed");
8428
8429 let mut corrupt_pack = written.pack;
8430 let base_reference = ofs_delta_base_reference_position(&corrupt_pack, first_stat.offset);
8431 corrupt_pack[base_reference] = if corrupt_pack[base_reference] == 1 {
8432 2
8433 } else {
8434 1
8435 };
8436 assert!(PackFile::verify_pack_stats(&corrupt_pack, ObjectFormat::Sha1).is_err());
8437 fs::write(&installed.pack_path, &corrupt_pack).expect("test operation should succeed");
8438 db.refresh_read_cache();
8439
8440 assert_eq!(read_object_for_assert(&db, &first_oid), first);
8441 assert_eq!(read_object_for_assert(&db, &second_oid), second);
8442 fs::remove_dir_all(root).expect("test operation should succeed");
8443 }
8444
8445 fn ofs_delta_base_reference_position(pack: &[u8], offset: u64) -> usize {
8446 let mut cursor = usize::try_from(offset).expect("test operation should succeed");
8447 let first = pack[cursor];
8448 cursor += 1;
8449 let kind = (first >> 4) & 0x07;
8450 let mut byte = first;
8451 while byte & 0x80 != 0 {
8452 byte = pack[cursor];
8453 cursor += 1;
8454 }
8455 assert_eq!(kind, 6, "expected an ofs-delta entry");
8456 cursor
8457 }
8458
8459 #[test]
8460 fn object_presence_checker_observes_same_process_loose_write_after_miss() {
8461 let root = temp_root("sley-presence-checker-loose-cache-write");
8462 let git_dir = root.join(".git");
8463 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8464 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8465 let mut checker = db.presence_checker();
8466
8467 let object = EncodedObject::new(ObjectType::Blob, b"checker loose after miss\n".to_vec());
8468 let oid = object
8469 .object_id(ObjectFormat::Sha1)
8470 .expect("test operation should succeed");
8471
8472 assert!(
8473 !checker
8474 .contains(&oid)
8475 .expect("test operation should succeed")
8476 );
8477 db.loose()
8478 .write_object(object)
8479 .expect("test operation should succeed");
8480
8481 assert!(
8482 checker
8483 .contains(&oid)
8484 .expect("test operation should succeed")
8485 );
8486 fs::remove_dir_all(root).expect("test operation should succeed");
8487 }
8488
8489 #[test]
8490 fn read_object_header_matches_full_read_for_loose_and_packed_and_delta() {
8491 let root = temp_root("sley-read-object-header");
8492 let git_dir = root.join(".git");
8493 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8494 let format = ObjectFormat::Sha1;
8495 let db = FileObjectDatabase::from_git_dir(&git_dir, format);
8496
8497 let loose = EncodedObject::new(ObjectType::Blob, b"loose header object\n".to_vec());
8499 let loose_oid = db
8500 .write_object(loose.clone())
8501 .expect("test operation should succeed");
8502
8503 let base = EncodedObject::new(ObjectType::Blob, vec![b'a'; 4096]);
8507 let mut child_body = vec![b'a'; 4096];
8508 child_body.extend_from_slice(b" plus a deltified tail\n");
8509 let child = EncodedObject::new(ObjectType::Blob, child_body);
8510 let commitish =
8511 EncodedObject::new(ObjectType::Commit, b"header-only type probe\n".to_vec());
8512 let base_oid = base
8513 .object_id(format)
8514 .expect("test operation should succeed");
8515 let child_oid = child
8516 .object_id(format)
8517 .expect("test operation should succeed");
8518 let commit_oid = commitish
8519 .object_id(format)
8520 .expect("test operation should succeed");
8521 let options = PackWriteOptions::new()
8522 .with_prefer_ofs_delta(true)
8523 .with_reorder(false);
8524 let pack = PackFile::write_packed_with_options(
8525 &[base.clone(), child.clone(), commitish.clone()],
8526 format,
8527 &options,
8528 )
8529 .expect("test operation should succeed");
8530 db.install_pack(&pack)
8531 .expect("test operation should succeed");
8532
8533 for (oid, want_type, want_len) in [
8536 (&loose_oid, ObjectType::Blob, loose.body.len()),
8537 (&base_oid, ObjectType::Blob, base.body.len()),
8538 (&child_oid, ObjectType::Blob, child.body.len()),
8539 (&commit_oid, ObjectType::Commit, commitish.body.len()),
8540 ] {
8541 assert_eq!(
8542 db.read_object_header(oid)
8543 .expect("test operation should succeed"),
8544 Some((want_type, want_len as u64)),
8545 "header for {oid}"
8546 );
8547 let full = db.read_object(oid).expect("test operation should succeed");
8548 assert_eq!(
8549 db.read_object_header(oid)
8550 .expect("test operation should succeed"),
8551 Some((full.object_type, full.body.len() as u64))
8552 );
8553 }
8554
8555 let missing = ObjectId::from_hex(format, "0000000000000000000000000000000000000001")
8556 .expect("test operation should succeed");
8557 assert_eq!(
8558 db.read_object_header(&missing)
8559 .expect("test operation should succeed"),
8560 None
8561 );
8562 fs::remove_dir_all(root).expect("test operation should succeed");
8563 }
8564
8565 #[test]
8566 fn object_storage_info_reports_loose_packed_and_delta_metadata() {
8567 let root = temp_root("sley-object-storage-info");
8568 let git_dir = root.join(".git");
8569 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8570 let format = ObjectFormat::Sha1;
8571 let db = FileObjectDatabase::from_git_dir(&git_dir, format);
8572
8573 let loose = EncodedObject::new(ObjectType::Blob, b"loose storage object\n".to_vec());
8574 let loose_oid = db
8575 .write_object(loose)
8576 .expect("test operation should succeed");
8577 let loose_size = fs::metadata(
8578 db.loose()
8579 .object_path(&loose_oid)
8580 .expect("test operation should succeed"),
8581 )
8582 .expect("test operation should succeed")
8583 .len();
8584 let loose_info = db
8585 .object_storage_info(&loose_oid)
8586 .expect("test operation should succeed")
8587 .expect("test operation should succeed");
8588 assert_eq!(loose_info.disk_size, loose_size);
8589 assert_eq!(
8590 loose_info.deltabase,
8591 zero_oid(format).expect("test operation should succeed")
8592 );
8593
8594 let base = EncodedObject::new(ObjectType::Blob, vec![b'a'; 4096]);
8595 let mut child_body = vec![b'a'; 4096];
8596 child_body.extend_from_slice(b" changed tail\n");
8597 let child = EncodedObject::new(ObjectType::Blob, child_body);
8598 let base_oid = base
8599 .object_id(format)
8600 .expect("test operation should succeed");
8601 let child_oid = child
8602 .object_id(format)
8603 .expect("test operation should succeed");
8604 let options = PackWriteOptions::new()
8605 .with_prefer_ofs_delta(true)
8606 .with_reorder(false);
8607 let pack = PackFile::write_packed_with_options(&[base, child], format, &options)
8608 .expect("test operation should succeed");
8609 db.install_pack(&pack)
8610 .expect("test operation should succeed");
8611
8612 let base_info = db
8613 .object_storage_info(&base_oid)
8614 .expect("test operation should succeed")
8615 .expect("test operation should succeed");
8616 assert!(base_info.disk_size > 0);
8617 assert_eq!(
8618 base_info.deltabase,
8619 zero_oid(format).expect("test operation should succeed")
8620 );
8621
8622 let child_info = db
8623 .object_storage_info(&child_oid)
8624 .expect("test operation should succeed")
8625 .expect("test operation should succeed");
8626 assert!(child_info.disk_size > 0);
8627 assert_eq!(child_info.deltabase, base_oid);
8628
8629 let missing = ObjectId::from_hex(format, "0000000000000000000000000000000000000001")
8630 .expect("test operation should succeed");
8631 assert_eq!(
8632 db.object_storage_info(&missing)
8633 .expect("test operation should succeed"),
8634 None
8635 );
8636 fs::remove_dir_all(root).expect("test operation should succeed");
8637 }
8638
8639 #[test]
8640 fn object_storage_info_uses_midx_when_pack_sidecar_is_missing() {
8641 let root = temp_root("sley-object-storage-midx-fallback");
8642 let git_dir = root.join(".git");
8643 let pack_dir = git_dir.join("objects").join("pack");
8644 fs::create_dir_all(&pack_dir).expect("test operation should succeed");
8645 let format = ObjectFormat::Sha1;
8646 let first = EncodedObject::new(ObjectType::Blob, b"first packed object\n".to_vec());
8647 let second = EncodedObject::new(ObjectType::Blob, b"second packed object\n".to_vec());
8648 let written = PackFile::write_undeltified_sha1(&[first, second])
8649 .expect("test operation should succeed");
8650 let pack_name = written.checksum.to_hex();
8651 let pack_path = pack_dir.join(format!("pack-{pack_name}.pack"));
8652 let idx_path = pack_dir.join(format!("pack-{pack_name}.idx"));
8653 fs::write(&pack_path, &written.pack).expect("test operation should succeed");
8654 fs::write(&idx_path, &written.index).expect("test operation should succeed");
8655
8656 let idx_name = idx_path
8657 .file_name()
8658 .and_then(|name| name.to_str())
8659 .expect("test operation should succeed")
8660 .to_string();
8661 let midx_objects = written
8662 .entries
8663 .iter()
8664 .map(|entry| sley_pack::MultiPackIndexEntry {
8665 oid: entry.oid,
8666 pack_int_id: 0,
8667 offset: entry.offset,
8668 force_large_offset: false,
8669 })
8670 .collect::<Vec<_>>();
8671 let midx = MultiPackIndex::write(format, 1, &[idx_name], &midx_objects)
8672 .expect("test operation should succeed");
8673 fs::write(pack_dir.join("multi-pack-index"), midx).expect("test operation should succeed");
8674
8675 let target = written
8676 .entries
8677 .iter()
8678 .min_by_key(|entry| entry.offset)
8679 .expect("test operation should succeed")
8680 .oid;
8681 let indexed_info = FileObjectDatabase::from_git_dir(&git_dir, format)
8682 .object_storage_info(&target)
8683 .expect("test operation should succeed")
8684 .expect("test operation should succeed");
8685
8686 fs::remove_file(&idx_path).expect("test operation should succeed");
8687 let missing_idx_info = FileObjectDatabase::from_git_dir(&git_dir, format)
8688 .object_storage_info(&target)
8689 .expect("test operation should succeed")
8690 .expect("test operation should succeed");
8691 assert_eq!(missing_idx_info, indexed_info);
8692
8693 fs::write(&idx_path, &written.index).expect("test operation should succeed");
8694 fs::remove_file(&pack_path).expect("test operation should succeed");
8695 let missing_pack_info = FileObjectDatabase::from_git_dir(&git_dir, format)
8696 .object_storage_info(&target)
8697 .expect("test operation should succeed")
8698 .expect("test operation should succeed");
8699 assert_eq!(missing_pack_info.disk_size, indexed_info.disk_size);
8700 assert_eq!(
8701 missing_pack_info.deltabase,
8702 zero_oid(format).expect("test operation should succeed")
8703 );
8704
8705 fs::remove_dir_all(root).expect("test operation should succeed");
8706 }
8707
8708 #[test]
8709 fn file_database_resolves_unique_loose_object_prefix() {
8710 let root = temp_root("sley-file-odb-prefix-loose");
8711 let git_dir = root.join(".git");
8712 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8713 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8714 let object = EncodedObject::new(ObjectType::Blob, b"prefix loose\n".to_vec());
8715 let oid = db
8716 .write_object(object)
8717 .expect("test operation should succeed");
8718 let prefix = &oid.to_hex()[..8];
8719
8720 assert_eq!(
8721 db.resolve_prefix(prefix)
8722 .expect("test operation should succeed"),
8723 ObjectPrefixResolution::Unique(oid)
8724 );
8725 assert!(
8726 db.object_ids()
8727 .expect("test operation should succeed")
8728 .contains(&oid)
8729 );
8730 fs::remove_dir_all(root).expect("test operation should succeed");
8731 }
8732
8733 #[test]
8734 fn file_database_resolves_unique_packed_object_prefix() {
8735 let root = temp_root("sley-file-odb-prefix-packed");
8736 let git_dir = root.join(".git");
8737 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8738 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8739 let object = EncodedObject::new(ObjectType::Blob, b"prefix packed\n".to_vec());
8740 let oid = object
8741 .object_id(ObjectFormat::Sha1)
8742 .expect("test operation should succeed");
8743 let pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&object))
8744 .expect("test operation should succeed");
8745 db.install_pack(&pack)
8746 .expect("test operation should succeed");
8747 let prefix = &oid.to_hex()[..8];
8748
8749 assert_eq!(
8750 db.resolve_prefix(prefix)
8751 .expect("test operation should succeed"),
8752 ObjectPrefixResolution::Unique(oid)
8753 );
8754 fs::remove_dir_all(root).expect("test operation should succeed");
8755 }
8756
8757 #[test]
8758 fn file_database_reports_ambiguous_object_prefix() {
8759 let root = temp_root("sley-file-odb-prefix-ambiguous");
8760 let git_dir = root.join(".git");
8761 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8762 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8763 let mut seen = HashMap::new();
8764 let (prefix, first, second) = (0..10_000)
8765 .find_map(|idx| {
8766 let object =
8767 EncodedObject::new(ObjectType::Blob, format!("ambiguous {idx}\n").into_bytes());
8768 let oid = db
8769 .write_object(object)
8770 .expect("test operation should succeed");
8771 let prefix = oid.to_hex()[..4].to_string();
8772 seen.insert(prefix.clone(), oid)
8773 .map(|first| (prefix, first, oid))
8774 })
8775 .expect("test should find a 4-hex collision");
8776
8777 let ObjectPrefixResolution::Ambiguous(mut matches) = db
8778 .resolve_prefix(&prefix)
8779 .expect("test operation should succeed")
8780 else {
8781 panic!("expected ambiguous prefix {prefix}");
8782 };
8783 matches.sort_by_key(ObjectId::to_hex);
8784 let mut expected = vec![first, second];
8785 expected.sort_by_key(ObjectId::to_hex);
8786 assert_eq!(matches, expected);
8787 fs::remove_dir_all(root).expect("test operation should succeed");
8788 }
8789
8790 #[test]
8791 fn file_database_rejects_too_short_object_prefix() {
8792 let root = temp_root("sley-file-odb-prefix-short");
8793 let git_dir = root.join(".git");
8794 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8795 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8796
8797 assert!(matches!(
8798 db.resolve_prefix("abc"),
8799 Err(GitError::InvalidObjectId(_))
8800 ));
8801 fs::remove_dir_all(root).expect("test operation should succeed");
8802 }
8803
8804 #[test]
8805 fn file_database_reads_sha256_object_from_pack_index() {
8806 let root = temp_root("sley-file-odb-pack-sha256");
8807 let git_dir = root.join(".git");
8808 let pack_dir = git_dir.join("objects").join("pack");
8809 fs::create_dir_all(&pack_dir).expect("test operation should succeed");
8810 let object = EncodedObject::new(ObjectType::Blob, b"packed sha256\n".to_vec());
8811 let oid = object
8812 .object_id(ObjectFormat::Sha256)
8813 .expect("test operation should succeed");
8814 let written =
8815 PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha256)
8816 .expect("test operation should succeed");
8817 let pack_name = written.checksum.to_hex();
8818 fs::write(
8819 pack_dir.join(format!("pack-{pack_name}.pack")),
8820 written.pack,
8821 )
8822 .expect("test operation should succeed");
8823 fs::write(
8824 pack_dir.join(format!("pack-{pack_name}.idx")),
8825 written.index,
8826 )
8827 .expect("test operation should succeed");
8828
8829 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha256);
8830 assert!(db.contains(&oid).expect("test operation should succeed"));
8831 assert_eq!(read_object_for_assert(&db, &oid), object);
8832 fs::remove_dir_all(root).expect("test operation should succeed");
8833 }
8834
8835 #[test]
8836 fn file_database_installs_sha256_pack_without_loose_objects() {
8837 let root = temp_root("sley-file-odb-install-pack");
8838 let git_dir = root.join(".git");
8839 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8840 let object = EncodedObject::new(ObjectType::Blob, b"installed sha256 pack\n".to_vec());
8841 let oid = object
8842 .object_id(ObjectFormat::Sha256)
8843 .expect("test operation should succeed");
8844 let pack = PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha256)
8845 .expect("test operation should succeed");
8846 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha256);
8847
8848 let result = db
8849 .install_pack(&pack)
8850 .expect("test operation should succeed");
8851
8852 assert_eq!(result.pack_name, format!("pack-{}", pack.checksum.to_hex()));
8853 assert_eq!(result.object_ids, vec![oid]);
8854 assert!(result.pack_path.exists());
8855 assert!(result.index_path.exists());
8856 assert_eq!(result.promisor_path, None);
8857 assert!(
8858 !db.loose()
8859 .object_path(&oid)
8860 .expect("test operation should succeed")
8861 .exists()
8862 );
8863 assert!(db.contains(&oid).expect("test operation should succeed"));
8864 assert_eq!(read_object_for_assert(&db, &oid), object);
8865 fs::remove_dir_all(root).expect("test operation should succeed");
8866 }
8867
8868 #[test]
8869 fn file_database_installs_raw_sha256_pack_without_loose_objects() {
8870 let root = temp_root("sley-file-odb-install-raw-pack");
8871 let git_dir = root.join(".git");
8872 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8873 let object = EncodedObject::new(ObjectType::Blob, b"installed raw sha256 pack\n".to_vec());
8874 let oid = object
8875 .object_id(ObjectFormat::Sha256)
8876 .expect("test operation should succeed");
8877 let pack = PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha256)
8878 .expect("test operation should succeed");
8879 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha256);
8880 let mut reader = pack.pack.as_slice();
8881
8882 let result = db
8883 .install_raw_pack_from_reader(&mut reader)
8884 .expect("test operation should succeed");
8885
8886 assert_eq!(result.pack_name, format!("pack-{}", pack.checksum.to_hex()));
8887 assert_eq!(result.object_ids, vec![oid]);
8888 assert!(result.pack_path.exists());
8889 assert!(result.index_path.exists());
8890 assert_eq!(result.promisor_path, None);
8891 assert!(
8892 !db.loose()
8893 .object_path(&oid)
8894 .expect("test operation should succeed")
8895 .exists()
8896 );
8897 assert!(db.contains(&oid).expect("test operation should succeed"));
8898 assert_eq!(read_object_for_assert(&db, &oid), object);
8899 fs::remove_dir_all(root).expect("test operation should succeed");
8900 }
8901
8902 #[test]
8903 fn file_database_streams_raw_pack_install_to_packfile() {
8904 use std::io::Write as _;
8905
8906 let root = temp_root("sley-file-odb-stream-raw-pack");
8907 let git_dir = root.join(".git");
8908 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8909 let object = EncodedObject::new(ObjectType::Blob, b"streamed raw pack\n".to_vec());
8910 let oid = object
8911 .object_id(ObjectFormat::Sha1)
8912 .expect("test operation should succeed");
8913 let pack = PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha1)
8914 .expect("test operation should succeed");
8915 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8916
8917 let mut install = db
8918 .begin_raw_pack_install(pack.checksum, pack.pack.len() as u64)
8919 .expect("test operation should succeed");
8920 for chunk in pack.pack.chunks(5) {
8921 install
8922 .write_all(chunk)
8923 .expect("test operation should succeed");
8924 }
8925 let result = install.finish().expect("test operation should succeed");
8926
8927 assert_eq!(result.pack_name, format!("pack-{}", pack.checksum.to_hex()));
8928 assert_eq!(result.object_ids, vec![oid]);
8929 assert_eq!(
8930 fs::read(&result.pack_path).expect("test operation should succeed"),
8931 pack.pack
8932 );
8933 assert!(result.index_path.exists());
8934 assert!(db.contains(&oid).expect("test operation should succeed"));
8935 assert_eq!(read_object_for_assert(&db, &oid), object);
8936
8937 let bad_id = ObjectId::from_raw(ObjectFormat::Sha1, &[0x42; 20])
8938 .expect("test operation should succeed");
8939 let mut bad_install = db
8940 .begin_raw_pack_install(bad_id, pack.pack.len() as u64)
8941 .expect("test operation should succeed");
8942 bad_install
8943 .write_all(&pack.pack)
8944 .expect("test operation should succeed");
8945 assert!(
8946 bad_install.finish().is_err(),
8947 "checksum mismatch should reject the streamed pack"
8948 );
8949 assert!(
8950 !git_dir
8951 .join("objects")
8952 .join("pack")
8953 .join(format!("pack-{}.pack", bad_id.to_hex()))
8954 .exists()
8955 );
8956
8957 fs::remove_dir_all(root).expect("test operation should succeed");
8958 }
8959
8960 #[test]
8961 fn file_database_installs_unknown_length_raw_pack_from_reader() {
8962 let root = temp_root("sley-file-odb-install-raw-pack-reader");
8963 let git_dir = root.join(".git");
8964 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8965 let object = EncodedObject::new(ObjectType::Blob, b"reader streamed raw pack\n".to_vec());
8966 let oid = object
8967 .object_id(ObjectFormat::Sha1)
8968 .expect("test operation should succeed");
8969 let pack = PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha1)
8970 .expect("test operation should succeed");
8971 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8972 let mut reader = pack.pack.as_slice();
8973
8974 let result = db
8975 .install_raw_pack_from_reader(&mut reader)
8976 .expect("test operation should succeed");
8977
8978 assert_eq!(result.pack_name, format!("pack-{}", pack.checksum.to_hex()));
8979 assert_eq!(result.object_ids, vec![oid]);
8980 assert_eq!(
8981 fs::read(&result.pack_path).expect("test operation should succeed"),
8982 pack.pack
8983 );
8984 assert!(result.index_path.exists());
8985 assert!(db.contains(&oid).expect("test operation should succeed"));
8986 assert_eq!(read_object_for_assert(&db, &oid), object);
8987 fs::remove_dir_all(root).expect("test operation should succeed");
8988 }
8989
8990 #[test]
8991 fn file_database_rejects_unknown_length_raw_pack_with_trailing_bytes() {
8992 let root = temp_root("sley-file-odb-install-raw-pack-reader-trailing");
8993 let git_dir = root.join(".git");
8994 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
8995 let object = EncodedObject::new(ObjectType::Blob, b"trailing streamed raw pack\n".to_vec());
8996 let pack = PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha1)
8997 .expect("test operation should succeed");
8998 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
8999 let mut bytes = pack.pack;
9000 bytes.extend_from_slice(b"not part of the pack");
9001 let mut reader = bytes.as_slice();
9002
9003 let err = db
9004 .install_raw_pack_from_reader(&mut reader)
9005 .expect_err("trailing bytes should be rejected");
9006
9007 assert!(err.to_string().contains("trailing bytes after checksum"));
9008 let pack_dir = git_dir.join("objects").join("pack");
9009 let pack_entries = fs::read_dir(&pack_dir)
9010 .map(|entries| entries.count())
9011 .unwrap_or_default();
9012 assert_eq!(pack_entries, 0);
9013 fs::remove_dir_all(root).expect("test operation should succeed");
9014 }
9015
9016 #[test]
9017 fn file_database_rejects_noncanonical_pack_index() {
9018 let root = temp_root("sley-file-odb-install-bad-index");
9019 let git_dir = root.join(".git");
9020 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9021 let object = EncodedObject::new(ObjectType::Blob, b"bad index crc\n".to_vec());
9022 let pack = PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha1)
9023 .expect("test operation should succeed");
9024 let mut entries = pack.entries.clone();
9025 entries[0].crc32 ^= 1;
9026 let mut bad_pack = pack.clone();
9027 bad_pack.index = PackIndex::write_v2(ObjectFormat::Sha1, &entries, &pack.checksum)
9028 .expect("test operation should succeed");
9029 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
9030
9031 assert!(db.install_pack(&bad_pack).is_err());
9032
9033 fs::remove_dir_all(root).expect("test operation should succeed");
9034 }
9035
9036 #[test]
9037 fn file_database_installs_raw_promisor_pack_with_sidecar() {
9038 let root = temp_root("sley-file-odb-install-raw-promisor-pack");
9039 let git_dir = root.join(".git");
9040 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9041 let object = EncodedObject::new(ObjectType::Blob, b"installed promisor pack\n".to_vec());
9042 let oid = object
9043 .object_id(ObjectFormat::Sha1)
9044 .expect("test operation should succeed");
9045 let pack = PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha1)
9046 .expect("test operation should succeed");
9047 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
9048 let mut reader = pack.pack.as_slice();
9049
9050 let result = db
9051 .install_raw_pack_from_reader_with_options(
9052 &mut reader,
9053 RawPackInstallOptions { promisor: true },
9054 )
9055 .expect("test operation should succeed");
9056
9057 let promisor_path = result.promisor_path.expect("promisor sidecar");
9058 assert_eq!(promisor_path.file_stem(), result.pack_path.file_stem());
9059 assert_eq!(
9060 promisor_path.extension().and_then(|ext| ext.to_str()),
9061 Some("promisor")
9062 );
9063 assert!(promisor_path.exists());
9064 assert_eq!(
9065 fs::read(&promisor_path).expect("test operation should succeed"),
9066 b""
9067 );
9068 assert!(result.pack_path.exists());
9069 assert!(result.index_path.exists());
9070 assert!(
9071 !db.loose()
9072 .object_path(&oid)
9073 .expect("test operation should succeed")
9074 .exists()
9075 );
9076 assert_eq!(read_object_for_assert(&db, &oid), object);
9077 fs::remove_dir_all(root).expect("test operation should succeed");
9078 }
9079
9080 #[test]
9081 fn repository_objects_dir_uses_linked_worktree_common_dir() {
9082 let root = temp_root("sley-odb-common-dir");
9083 let common = root.join(".git");
9084 let admin = common.join("worktrees").join("linked");
9085 fs::create_dir_all(&admin).expect("test operation should succeed");
9086 fs::write(admin.join("commondir"), "../..\n").expect("test operation should succeed");
9087
9088 let common = fs::canonicalize(common).expect("test operation should succeed");
9089 assert_eq!(repository_common_dir(&admin), common);
9090 assert_eq!(repository_objects_dir(&admin), common.join("objects"));
9091
9092 fs::remove_dir_all(root).expect("test operation should succeed");
9093 }
9094
9095 #[test]
9096 fn reachable_object_helpers_walk_graph_and_install_pack() {
9097 let root = temp_root("sley-reachable-pack");
9098 let source_git_dir = root.join("source.git");
9099 let destination_git_dir = root.join("destination.git");
9100 fs::create_dir_all(source_git_dir.join("objects")).expect("test operation should succeed");
9101 fs::create_dir_all(destination_git_dir.join("objects"))
9102 .expect("test operation should succeed");
9103 let format = ObjectFormat::Sha1;
9104 let source = FileObjectDatabase::from_git_dir(&source_git_dir, format);
9105 let destination = FileObjectDatabase::from_git_dir(&destination_git_dir, format);
9106
9107 let blob = EncodedObject::new(ObjectType::Blob, b"reachable payload\n".to_vec());
9108 let blob_oid = source
9109 .write_object(blob.clone())
9110 .expect("test operation should succeed");
9111 let tree = EncodedObject::new(
9112 ObjectType::Tree,
9113 Tree {
9114 entries: vec![TreeEntry {
9115 mode: 0o100644,
9116 name: BString::from(b"payload.txt"),
9117 oid: blob_oid,
9118 }],
9119 }
9120 .write(),
9121 );
9122 let tree_oid = source
9123 .write_object(tree.clone())
9124 .expect("test operation should succeed");
9125 let identity = b"Example <example@example.invalid> 0 +0000".to_vec();
9126 let commit = EncodedObject::new(
9127 ObjectType::Commit,
9128 Commit {
9129 tree: tree_oid,
9130 parents: Vec::new(),
9131 author: identity.clone(),
9132 committer: identity,
9133 encoding: None,
9134 message: b"initial\n".to_vec(),
9135 }
9136 .write(),
9137 );
9138 let commit_oid = source
9139 .write_object(commit.clone())
9140 .expect("test operation should succeed");
9141
9142 let reachable = collect_reachable_object_ids(&source, format, std::iter::once(commit_oid))
9143 .expect("test operation should succeed");
9144 assert!(reachable.contains(&commit_oid));
9145 assert!(reachable.contains(&tree_oid));
9146 assert!(reachable.contains(&blob_oid));
9147
9148 let install =
9149 install_reachable_pack(&source, &destination, format, std::iter::once(commit_oid))
9150 .expect("test operation should succeed")
9151 .expect("reachable pack should be written");
9152 assert_eq!(install.object_ids.len(), 3);
9153 for (oid, object) in [
9154 (&commit_oid, &commit),
9155 (&tree_oid, &tree),
9156 (&blob_oid, &blob),
9157 ] {
9158 assert!(
9159 !destination
9160 .loose()
9161 .object_path(oid)
9162 .expect("test operation should succeed")
9163 .exists()
9164 );
9165 assert!(
9166 destination
9167 .contains(oid)
9168 .expect("test operation should succeed")
9169 );
9170 assert_eq!(read_object_for_assert(&destination, oid), *object);
9171 }
9172 fs::remove_dir_all(root).expect("test operation should succeed");
9173 }
9174
9175 #[test]
9176 fn reachable_object_helpers_respect_exclusions_and_duplicate_starts() {
9177 let root = temp_root("sley-reachable-exclusions");
9178 let git_dir = root.join("repo.git");
9179 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9180 let format = ObjectFormat::Sha1;
9181 let db = FileObjectDatabase::from_git_dir(&git_dir, format);
9182
9183 let blob = EncodedObject::new(ObjectType::Blob, b"excluded payload\n".to_vec());
9184 let blob_oid = db
9185 .write_object(blob)
9186 .expect("test operation should succeed");
9187 let tree = EncodedObject::new(
9188 ObjectType::Tree,
9189 Tree {
9190 entries: vec![TreeEntry {
9191 mode: 0o100644,
9192 name: BString::from(b"payload.txt"),
9193 oid: blob_oid,
9194 }],
9195 }
9196 .write(),
9197 );
9198 let tree_oid = db
9199 .write_object(tree)
9200 .expect("test operation should succeed");
9201 let identity = b"Example <example@example.invalid> 0 +0000".to_vec();
9202 let commit = EncodedObject::new(
9203 ObjectType::Commit,
9204 Commit {
9205 tree: tree_oid,
9206 parents: Vec::new(),
9207 author: identity.clone(),
9208 committer: identity,
9209 encoding: None,
9210 message: b"initial\n".to_vec(),
9211 }
9212 .write(),
9213 );
9214 let commit_oid = db
9215 .write_object(commit)
9216 .expect("test operation should succeed");
9217 let excluded = HashSet::from([tree_oid]);
9218
9219 let objects = collect_reachable_objects(&db, format, [commit_oid, commit_oid], &excluded)
9220 .expect("test operation should succeed");
9221
9222 assert_eq!(objects.len(), 1);
9223 assert_eq!(
9224 objects[0]
9225 .object_id(format)
9226 .expect("test operation should succeed"),
9227 commit_oid
9228 );
9229 fs::remove_dir_all(root).expect("test operation should succeed");
9230 }
9231
9232 #[test]
9233 fn build_reachable_pack_returns_raw_pack_and_respects_empty_exclusions() {
9234 let root = temp_root("sley-build-reachable-pack");
9235 let git_dir = root.join("repo.git");
9236 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9237 let format = ObjectFormat::Sha1;
9238 let db = FileObjectDatabase::from_git_dir(&git_dir, format);
9239
9240 let object = EncodedObject::new(ObjectType::Blob, b"raw reachable pack\n".to_vec());
9241 let oid = db
9242 .write_object(object.clone())
9243 .expect("test operation should succeed");
9244 let pack = build_reachable_pack(&db, format, std::iter::once(oid), &HashSet::new())
9245 .expect("test operation should succeed")
9246 .expect("reachable pack should be built");
9247 assert!(pack.pack.starts_with(b"PACK"));
9248 assert_eq!(pack.entries.len(), 1);
9249 assert_eq!(pack.entries[0].oid, oid);
9250
9251 let pack_path = root.join("reachable.pack");
9252 let pack_file = build_reachable_pack_file(
9253 &db,
9254 format,
9255 std::iter::once(oid),
9256 &HashSet::new(),
9257 &pack_path,
9258 )
9259 .expect("test operation should succeed")
9260 .expect("reachable pack file should be built");
9261 assert_eq!(pack_file.checksum, pack.checksum);
9262 assert_eq!(pack_file.pack_size, pack.pack.len() as u64);
9263 assert_eq!(pack_file.object_count, 1);
9264 assert_eq!(
9265 fs::read(&pack_file.pack_path).expect("test operation should succeed"),
9266 pack.pack
9267 );
9268
9269 let mut streamed_pack = Vec::new();
9270 let streamed = write_reachable_pack_to_writer(
9271 &db,
9272 format,
9273 std::iter::once(oid),
9274 &HashSet::new(),
9275 &mut streamed_pack,
9276 )
9277 .expect("test operation should succeed")
9278 .expect("reachable pack should be streamed");
9279 assert_eq!(streamed.checksum, pack.checksum);
9280 assert_eq!(streamed.pack_size, pack.pack.len() as u64);
9281 assert_eq!(streamed.object_count, 1);
9282 assert_eq!(streamed_pack, pack.pack);
9283
9284 let mut sink = std::io::sink();
9285 let dry_run = write_reachable_pack_to_writer(
9286 &db,
9287 format,
9288 std::iter::once(oid),
9289 &HashSet::new(),
9290 &mut sink,
9291 )
9292 .expect("test operation should succeed")
9293 .expect("reachable pack should stream to sink");
9294 assert_eq!(dry_run.checksum, pack.checksum);
9295 assert_eq!(dry_run.pack_size, pack.pack.len() as u64);
9296 assert_eq!(dry_run.object_count, 1);
9297
9298 let excluded = HashSet::from([oid]);
9299 assert!(
9300 build_reachable_pack(
9301 &db,
9302 format,
9303 pack.entries.into_iter().map(|entry| entry.oid),
9304 &excluded
9305 )
9306 .expect("test operation should succeed")
9307 .is_none()
9308 );
9309 fs::remove_dir_all(root).expect("test operation should succeed");
9310 }
9311
9312 #[test]
9313 fn large_reachable_pack_streams_objects_by_id_windows() {
9314 let root = temp_root("sley-reachable-pack-streamed-large");
9315 let git_dir = root.join(".git");
9316 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9317 let format = ObjectFormat::Sha1;
9318 let db = FileObjectDatabase::from_git_dir(&git_dir, format);
9319
9320 let mut roots = Vec::new();
9321 for idx in 0..(REACHABLE_PACK_STREAMING_MIN_OBJECTS + 5) {
9322 let object = EncodedObject::new(
9323 ObjectType::Blob,
9324 format!("streamed reachable blob {idx:04}\n").into_bytes(),
9325 );
9326 roots.push(
9327 db.write_object(object)
9328 .expect("test operation should succeed"),
9329 );
9330 }
9331
9332 let mut pack_bytes = Vec::new();
9333 let summary = write_reachable_pack_to_writer(
9334 &db,
9335 format,
9336 roots.iter().copied(),
9337 &HashSet::new(),
9338 &mut pack_bytes,
9339 )
9340 .expect("test operation should succeed")
9341 .expect("reachable pack should be streamed");
9342 assert_eq!(summary.object_count, roots.len());
9343 assert!(
9344 summary.delta_count > 0,
9345 "streamed large packs should still find deltas"
9346 );
9347 assert_eq!(summary.pack_size, pack_bytes.len() as u64);
9348
9349 let parsed = PackFile::parse(&pack_bytes, format).expect("test operation should succeed");
9350 let expected_oids = roots.iter().copied().collect::<HashSet<_>>();
9351 let parsed_oids = parsed
9352 .entries
9353 .iter()
9354 .map(|entry| entry.entry.oid)
9355 .collect::<HashSet<_>>();
9356 assert_eq!(parsed.checksum, summary.checksum);
9357 assert_eq!(parsed_oids, expected_oids);
9358
9359 fs::remove_dir_all(root).expect("test operation should succeed");
9360 }
9361
9362 #[test]
9363 fn index_raw_pack_returns_validated_pack_metadata() {
9364 let root = temp_root("sley-index-raw-pack");
9365 let git_dir = root.join("repo.git");
9366 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9367 let format = ObjectFormat::Sha1;
9368 let mut db = FileObjectDatabase::from_git_dir(&git_dir, format);
9369 let graph = write_commit_graph(&mut db, b"pack indexed\n");
9370 let commit_oid = graph[0].0;
9371 let expected = graph
9372 .iter()
9373 .map(|(oid, object)| (*oid, (object.object_type, object.body.len() as u64)))
9374 .collect::<HashMap<_, _>>();
9375 let pack = build_reachable_pack(&db, format, std::iter::once(commit_oid), &HashSet::new())
9376 .expect("test operation should succeed")
9377 .expect("reachable pack should be built");
9378
9379 let indexed = index_raw_pack(&pack.pack, format).expect("test operation should succeed");
9380 let mut cursor = std::io::Cursor::new(pack.pack.clone());
9381 let streamed = index_raw_pack_from_reader(&mut cursor, format)
9382 .expect("streamed pack indexing should match in-memory indexing");
9383 assert_eq!(streamed, indexed);
9384 let pack_path = root.join("reachable.pack");
9385 fs::write(&pack_path, &pack.pack).expect("test operation should succeed");
9386 let file_indexed = index_raw_pack_file(&pack_path, format)
9387 .expect("file-backed pack indexing should match in-memory indexing");
9388 assert_eq!(file_indexed, indexed);
9389
9390 assert_eq!(indexed.pack_id, pack.checksum);
9391 assert_eq!(indexed.index, pack.index);
9392 assert_eq!(indexed.objects.len(), 3);
9393 for object in indexed.objects {
9394 let (expected_type, expected_size) = expected
9395 .get(&object.oid)
9396 .copied()
9397 .expect("indexed object should be reachable");
9398 assert_eq!(object.object_type, expected_type);
9399 assert_eq!(object.size, expected_size);
9400 assert!(object.offset > 0);
9401 }
9402 fs::remove_dir_all(root).expect("test operation should succeed");
9403 }
9404
9405 #[test]
9406 fn reachable_object_helpers_follow_tags_and_report_missing_objects() {
9407 let root = temp_root("sley-reachable-tags");
9408 let git_dir = root.join("repo.git");
9409 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9410 let format = ObjectFormat::Sha1;
9411 let db = FileObjectDatabase::from_git_dir(&git_dir, format);
9412
9413 let blob = EncodedObject::new(ObjectType::Blob, b"tagged payload\n".to_vec());
9414 let blob_oid = db
9415 .write_object(blob)
9416 .expect("test operation should succeed");
9417 let tag = EncodedObject::new(
9418 ObjectType::Tag,
9419 Tag {
9420 object: blob_oid,
9421 object_type: ObjectType::Blob,
9422 name: b"v1".to_vec(),
9423 tagger: Some(b"Example <example@example.invalid> 0 +0000".to_vec()),
9424 message: b"tag message\n".to_vec(),
9425 raw_body: None,
9426 }
9427 .write(),
9428 );
9429 let tag_oid = db.write_object(tag).expect("test operation should succeed");
9430
9431 let reachable = collect_reachable_object_ids(&db, format, std::iter::once(tag_oid))
9432 .expect("test operation should succeed");
9433 assert!(reachable.contains(&tag_oid));
9434 assert!(reachable.contains(&blob_oid));
9435
9436 let missing = ObjectId::from_hex(format, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
9437 .expect("test operation should succeed");
9438 let err = collect_reachable_object_ids(&db, format, std::iter::once(missing))
9439 .expect_err("missing traversal root should error");
9440 let kind = err.not_found_kind().expect("typed not found");
9441 assert_eq!(kind.object_id(), Some(missing));
9442 assert_eq!(
9443 kind.missing_object_context(),
9444 Some(MissingObjectContext::Traversal)
9445 );
9446 fs::remove_dir_all(root).expect("test operation should succeed");
9447 }
9448
9449 #[test]
9450 fn install_reachable_pack_empty_starts_create_no_pack() {
9451 let root = temp_root("sley-reachable-empty");
9452 let source_git_dir = root.join("source.git");
9453 let destination_git_dir = root.join("destination.git");
9454 fs::create_dir_all(source_git_dir.join("objects")).expect("test operation should succeed");
9455 fs::create_dir_all(destination_git_dir.join("objects"))
9456 .expect("test operation should succeed");
9457 let format = ObjectFormat::Sha1;
9458 let source = FileObjectDatabase::from_git_dir(&source_git_dir, format);
9459 let destination = FileObjectDatabase::from_git_dir(&destination_git_dir, format);
9460
9461 let result = install_reachable_pack(&source, &destination, format, Vec::<ObjectId>::new())
9462 .expect("test operation should succeed");
9463
9464 assert!(result.is_none());
9465 assert!(!destination_git_dir.join("objects").join("pack").exists());
9466 fs::remove_dir_all(root).expect("test operation should succeed");
9467 }
9468
9469 #[test]
9470 fn install_reachable_pack_excluding_skips_fully_excluded_starts() {
9471 let root = temp_root("sley-reachable-install-excluding");
9472 let source_git_dir = root.join("source.git");
9473 let destination_git_dir = root.join("destination.git");
9474 fs::create_dir_all(source_git_dir.join("objects")).expect("test operation should succeed");
9475 fs::create_dir_all(destination_git_dir.join("objects"))
9476 .expect("test operation should succeed");
9477 let format = ObjectFormat::Sha1;
9478 let source = FileObjectDatabase::from_git_dir(&source_git_dir, format);
9479 let destination = FileObjectDatabase::from_git_dir(&destination_git_dir, format);
9480 let object = EncodedObject::new(ObjectType::Blob, b"excluded install\n".to_vec());
9481 let oid = source
9482 .write_object(object)
9483 .expect("test operation should succeed");
9484 let excluded = HashSet::from([oid]);
9485
9486 let result = install_reachable_pack_excluding(
9487 &source,
9488 &destination,
9489 format,
9490 std::iter::once(oid),
9491 &excluded,
9492 )
9493 .expect("test operation should succeed");
9494
9495 assert!(result.is_none());
9496 assert!(!destination_git_dir.join("objects").join("pack").exists());
9497 fs::remove_dir_all(root).expect("test operation should succeed");
9498 }
9499
9500 #[test]
9501 fn install_reachable_pack_supports_sha256() {
9502 let root = temp_root("sley-reachable-pack-sha256");
9503 let source_git_dir = root.join("source.git");
9504 let destination_git_dir = root.join("destination.git");
9505 fs::create_dir_all(source_git_dir.join("objects")).expect("test operation should succeed");
9506 fs::create_dir_all(destination_git_dir.join("objects"))
9507 .expect("test operation should succeed");
9508 let format = ObjectFormat::Sha256;
9509 let source = FileObjectDatabase::from_git_dir(&source_git_dir, format);
9510 let destination = FileObjectDatabase::from_git_dir(&destination_git_dir, format);
9511 let object = EncodedObject::new(ObjectType::Blob, b"sha256 reachable pack\n".to_vec());
9512 let oid = source
9513 .write_object(object.clone())
9514 .expect("test operation should succeed");
9515
9516 let pack = build_reachable_pack(&source, format, std::iter::once(oid), &HashSet::new())
9517 .expect("test operation should succeed")
9518 .expect("sha256 reachable pack should be built");
9519 assert!(pack.pack.starts_with(b"PACK"));
9520 assert_eq!(pack.entries[0].oid, oid);
9521
9522 let result = install_reachable_pack(&source, &destination, format, std::iter::once(oid))
9523 .expect("test operation should succeed")
9524 .expect("sha256 reachable pack should be written");
9525
9526 assert_eq!(result.object_ids, vec![oid]);
9527 assert!(
9528 !destination
9529 .loose()
9530 .object_path(&oid)
9531 .expect("test operation should succeed")
9532 .exists()
9533 );
9534 assert_eq!(read_object_for_assert(&destination, &oid), object);
9535 fs::remove_dir_all(root).expect("test operation should succeed");
9536 }
9537
9538 #[test]
9539 fn install_helpers_accept_custom_raw_pack_installer() {
9540 #[derive(Default)]
9541 struct RecordingInstaller {
9542 packs: std::cell::RefCell<Vec<Vec<u8>>>,
9543 installed: std::cell::RefCell<Vec<ObjectId>>,
9544 }
9545
9546 impl RawPackInstaller for RecordingInstaller {
9547 fn install_raw_pack_from_reader<R>(
9548 &self,
9549 reader: &mut R,
9550 ) -> Result<RawPackInstallResult>
9551 where
9552 R: Read,
9553 {
9554 let mut pack_bytes = Vec::new();
9555 reader.read_to_end(&mut pack_bytes)?;
9556 self.packs.borrow_mut().push(pack_bytes.to_vec());
9557 let object_ids = self.installed.borrow().clone();
9558 Ok(RawPackInstallResult { object_ids })
9559 }
9560 }
9561
9562 let format = ObjectFormat::Sha1;
9563 let source = ObjectDatabase::new(format);
9564 let object = EncodedObject::new(ObjectType::Blob, b"custom raw installer\n".to_vec());
9565 let oid = source
9566 .write_object(object)
9567 .expect("test operation should succeed");
9568 let installer = RecordingInstaller::default();
9569 installer.installed.borrow_mut().push(oid);
9570
9571 let result = install_reachable_pack(&source, &installer, format, std::iter::once(oid))
9572 .expect("test operation should succeed")
9573 .expect("custom installer should receive pack");
9574
9575 assert_eq!(result.object_ids, installer.installed.into_inner());
9576 let packs = installer.packs.into_inner();
9577 assert_eq!(packs.len(), 1);
9578 assert!(packs[0].starts_with(b"PACK"));
9579 }
9580
9581 #[test]
9582 fn file_database_reads_object_from_multi_pack_index() {
9583 let root = temp_root("sley-file-odb-midx");
9584 let git_dir = root.join(".git");
9585 let pack_dir = git_dir.join("objects").join("pack");
9586 fs::create_dir_all(&pack_dir).expect("test operation should succeed");
9587 let first = EncodedObject::new(ObjectType::Blob, b"first packed\n".to_vec());
9588 let second = EncodedObject::new(ObjectType::Blob, b"second packed\n".to_vec());
9589 let first_oid = first
9590 .object_id(ObjectFormat::Sha1)
9591 .expect("test operation should succeed");
9592 let second_oid = second
9593 .object_id(ObjectFormat::Sha1)
9594 .expect("test operation should succeed");
9595 let first_pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&first))
9596 .expect("test operation should succeed");
9597 let second_pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&second))
9598 .expect("test operation should succeed");
9599 let first_pack_name = format!("pack-{}.idx", first_pack.checksum.to_hex());
9600 let second_pack_name = format!("pack-{}.idx", second_pack.checksum.to_hex());
9601 fs::write(
9602 pack_dir.join(first_pack_name.replace(".idx", ".pack")),
9603 first_pack.pack,
9604 )
9605 .expect("test operation should succeed");
9606 fs::write(
9607 pack_dir.join(second_pack_name.replace(".idx", ".pack")),
9608 second_pack.pack,
9609 )
9610 .expect("test operation should succeed");
9611 let midx = MultiPackIndex::write(
9612 ObjectFormat::Sha1,
9613 2,
9614 &[first_pack_name, second_pack_name],
9615 &[
9616 sley_pack::MultiPackIndexEntry {
9617 oid: first_oid,
9618 pack_int_id: 0,
9619 offset: first_pack.entries[0].offset,
9620 force_large_offset: false,
9621 },
9622 sley_pack::MultiPackIndexEntry {
9623 oid: second_oid,
9624 pack_int_id: 1,
9625 offset: second_pack.entries[0].offset,
9626 force_large_offset: false,
9627 },
9628 ],
9629 )
9630 .expect("test operation should succeed");
9631 fs::write(pack_dir.join("multi-pack-index"), midx).expect("test operation should succeed");
9632
9633 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
9634 assert!(
9635 db.contains(&second_oid)
9636 .expect("test operation should succeed")
9637 );
9638 assert_eq!(
9639 db.resolve_prefix(&second_oid.to_hex()[..8])
9640 .expect("test operation should succeed"),
9641 ObjectPrefixResolution::Unique(second_oid)
9642 );
9643 assert_eq!(read_object_for_assert(&db, &second_oid), second);
9644 assert_eq!(read_object_for_assert(&db, &first_oid), first);
9645 fs::remove_dir_all(root).expect("test operation should succeed");
9646 }
9647
9648 #[test]
9649 fn file_database_finds_pack_added_after_registry_was_cached() {
9650 let root = temp_root("sley-file-odb-pack-added-late");
9654 let git_dir = root.join(".git");
9655 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9656 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
9657
9658 let first = EncodedObject::new(ObjectType::Blob, b"first late\n".to_vec());
9660 let first_oid = first
9661 .object_id(ObjectFormat::Sha1)
9662 .expect("test operation should succeed");
9663 let first_pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&first))
9664 .expect("test operation should succeed");
9665 db.install_pack(&first_pack)
9666 .expect("test operation should succeed");
9667 assert_eq!(read_object_for_assert(&db, &first_oid), first);
9668
9669 let second = EncodedObject::new(ObjectType::Blob, b"second late\n".to_vec());
9671 let second_oid = second
9672 .object_id(ObjectFormat::Sha1)
9673 .expect("test operation should succeed");
9674 assert!(matches!(
9676 db.read_object(&second_oid),
9677 Err(GitError::NotFound(_))
9678 ));
9679
9680 let second_pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&second))
9683 .expect("test operation should succeed");
9684 db.install_pack(&second_pack)
9685 .expect("test operation should succeed");
9686 assert!(
9687 db.contains(&second_oid)
9688 .expect("test operation should succeed")
9689 );
9690 assert_eq!(read_object_for_assert(&db, &second_oid), second);
9691 assert_eq!(read_object_for_assert(&db, &first_oid), first);
9693
9694 fs::remove_dir_all(root).expect("test operation should succeed");
9695 }
9696
9697 #[test]
9698 fn object_presence_checker_finds_pack_added_after_registry_was_cached() {
9699 let root = temp_root("sley-presence-checker-pack-added-late");
9700 let git_dir = root.join(".git");
9701 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9702 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
9703
9704 let first = EncodedObject::new(ObjectType::Blob, b"checker first late\n".to_vec());
9705 let first_oid = first
9706 .object_id(ObjectFormat::Sha1)
9707 .expect("test operation should succeed");
9708 let first_pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&first))
9709 .expect("test operation should succeed");
9710 db.install_pack(&first_pack)
9711 .expect("test operation should succeed");
9712
9713 let second = EncodedObject::new(ObjectType::Blob, b"checker second late\n".to_vec());
9714 let second_oid = second
9715 .object_id(ObjectFormat::Sha1)
9716 .expect("test operation should succeed");
9717 let mut checker = db.presence_checker();
9718 assert!(
9719 checker
9720 .contains(&first_oid)
9721 .expect("test operation should succeed")
9722 );
9723 assert!(
9724 !checker
9725 .contains(&second_oid)
9726 .expect("test operation should succeed")
9727 );
9728
9729 let second_pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&second))
9730 .expect("test operation should succeed");
9731 db.install_pack(&second_pack)
9732 .expect("test operation should succeed");
9733
9734 assert!(
9735 checker
9736 .contains(&second_oid)
9737 .expect("test operation should succeed")
9738 );
9739 fs::remove_dir_all(root).expect("test operation should succeed");
9740 }
9741
9742 #[test]
9743 fn file_database_pack_registry_loads_indexes_lazily_and_refreshes_after_count_change() {
9744 let root = temp_root("sley-file-odb-pack-registry-refresh");
9745 let git_dir = root.join(".git");
9746 let pack_dir = git_dir.join("objects").join("pack");
9747 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9748 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
9749
9750 let first = EncodedObject::new(ObjectType::Blob, b"registry first\n".to_vec());
9751 let first_oid = first
9752 .object_id(ObjectFormat::Sha1)
9753 .expect("test operation should succeed");
9754 let first_pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&first))
9755 .expect("test operation should succeed");
9756 db.install_pack(&first_pack)
9757 .expect("test operation should succeed");
9758
9759 let first_registry = db
9760 .cached_pack_registry(&pack_dir, false)
9761 .expect("test operation should succeed");
9762 assert_eq!(first_registry.fingerprint.idx_count, 1);
9763 assert_eq!(first_registry.fingerprint.pack_count, 1);
9764 assert_eq!(first_registry.packs.len(), 1);
9765 assert!(
9766 first_registry.packs[0]
9767 .index
9768 .lock()
9769 .expect("test operation should succeed")
9770 .is_none()
9771 );
9772 assert!(
9773 first_registry.packs[0]
9774 .data
9775 .lock()
9776 .expect("test operation should succeed")
9777 .is_none()
9778 );
9779
9780 assert!(
9783 db.contains(&first_oid)
9784 .expect("test operation should succeed")
9785 );
9786 assert!(
9787 first_registry.packs[0]
9788 .index
9789 .lock()
9790 .expect("test operation should succeed")
9791 .is_some()
9792 );
9793 assert!(
9794 first_registry.packs[0]
9795 .data
9796 .lock()
9797 .expect("test operation should succeed")
9798 .is_none()
9799 );
9800 assert_eq!(read_object_for_assert(&db, &first_oid), first);
9801 assert!(
9802 first_registry.packs[0]
9803 .data
9804 .lock()
9805 .expect("test operation should succeed")
9806 .is_some()
9807 );
9808
9809 let second = EncodedObject::new(ObjectType::Blob, b"registry second\n".to_vec());
9810 let second_oid = second
9811 .object_id(ObjectFormat::Sha1)
9812 .expect("test operation should succeed");
9813 let second_pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&second))
9814 .expect("test operation should succeed");
9815 db.install_pack(&second_pack)
9816 .expect("test operation should succeed");
9817
9818 let refreshed = db
9819 .cached_pack_registry(&pack_dir, true)
9820 .expect("test operation should succeed");
9821 assert!(!Arc::ptr_eq(&first_registry, &refreshed));
9822 assert_eq!(refreshed.fingerprint.idx_count, 2);
9823 assert_eq!(refreshed.fingerprint.pack_count, 2);
9824 assert_eq!(refreshed.packs.len(), 2);
9825 assert_eq!(read_object_for_assert(&db, &second_oid), second);
9826
9827 fs::remove_dir_all(root).expect("test operation should succeed");
9828 }
9829
9830 #[test]
9831 fn file_database_pack_search_hint_rebuilds_after_pack_added() {
9832 let root = temp_root("sley-file-odb-pack-lookup-added-late");
9836 let git_dir = root.join(".git");
9837 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9838 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
9839
9840 let first = EncodedObject::new(ObjectType::Blob, b"first lookup\n".to_vec());
9841 let second = EncodedObject::new(ObjectType::Blob, b"second lookup\n".to_vec());
9842 let third = EncodedObject::new(ObjectType::Blob, b"third lookup\n".to_vec());
9843 let first_oid = first
9844 .object_id(ObjectFormat::Sha1)
9845 .expect("test operation should succeed");
9846 let second_oid = second
9847 .object_id(ObjectFormat::Sha1)
9848 .expect("test operation should succeed");
9849 let third_oid = third
9850 .object_id(ObjectFormat::Sha1)
9851 .expect("test operation should succeed");
9852
9853 let first_pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&first))
9854 .expect("test operation should succeed");
9855 let second_pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&second))
9856 .expect("test operation should succeed");
9857 db.install_pack(&first_pack)
9858 .expect("test operation should succeed");
9859 db.install_pack(&second_pack)
9860 .expect("test operation should succeed");
9861
9862 assert_eq!(read_object_for_assert(&db, &first_oid), first);
9864 assert_eq!(read_object_for_assert(&db, &second_oid), second);
9865 assert!(matches!(
9866 db.read_object(&third_oid),
9867 Err(GitError::NotFound(_))
9868 ));
9869
9870 let third_pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&third))
9871 .expect("test operation should succeed");
9872 db.install_pack(&third_pack)
9873 .expect("test operation should succeed");
9874
9875 assert_eq!(read_object_for_assert(&db, &third_oid), third);
9876 assert_eq!(read_object_for_assert(&db, &first_oid), first);
9877
9878 fs::remove_dir_all(root).expect("test operation should succeed");
9879 }
9880
9881 #[test]
9882 fn file_database_prefers_loose_object_over_packed_object() {
9883 let root = temp_root("sley-file-odb-prefer-loose");
9884 let git_dir = root.join(".git");
9885 let pack_dir = git_dir.join("objects").join("pack");
9886 fs::create_dir_all(&pack_dir).expect("test operation should succeed");
9887 let object = EncodedObject::new(ObjectType::Blob, b"same\n".to_vec());
9888 let written = PackFile::write_undeltified_sha1(std::slice::from_ref(&object))
9889 .expect("test operation should succeed");
9890 let pack_name = written.checksum.to_hex();
9891 fs::write(
9892 pack_dir.join(format!("pack-{pack_name}.pack")),
9893 written.pack,
9894 )
9895 .expect("test operation should succeed");
9896 fs::write(
9897 pack_dir.join(format!("pack-{pack_name}.idx")),
9898 written.index,
9899 )
9900 .expect("test operation should succeed");
9901
9902 let db = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
9903 let oid = db
9904 .write_object(object.clone())
9905 .expect("test operation should succeed");
9906 assert_eq!(read_object_for_assert(&db, &oid), object);
9907 fs::remove_dir_all(root).expect("test operation should succeed");
9908 }
9909
9910 #[test]
9911 fn bundle_prerequisite_verification_reads_existing_objects() {
9912 let db = ObjectDatabase::new(ObjectFormat::Sha1);
9913 let oid = db
9914 .write_object(EncodedObject::new(ObjectType::Blob, b"base\n".to_vec()))
9915 .expect("test operation should succeed");
9916 let bundle_bytes = format!("# v2 git bundle\n-{oid} base\n\n").into_bytes();
9917 let bundle = Bundle::parse(&bundle_bytes, ObjectFormat::Sha1)
9918 .expect("test operation should succeed");
9919
9920 verify_bundle_prerequisites(&bundle, &db).expect("test operation should succeed");
9921 }
9922
9923 #[test]
9924 fn bundle_prerequisite_verification_reports_missing_objects() {
9925 let db = ObjectDatabase::new(ObjectFormat::Sha1);
9926 let missing = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"missing\n")
9927 .expect("test operation should succeed");
9928 let bundle_bytes = format!("# v2 git bundle\n-{missing} missing\n\n").into_bytes();
9929 let bundle = Bundle::parse(&bundle_bytes, ObjectFormat::Sha1)
9930 .expect("test operation should succeed");
9931
9932 assert!(verify_bundle_prerequisites(&bundle, &db).is_err());
9933 }
9934
9935 #[test]
9936 fn unbundle_objects_writes_pack_entries_and_returns_refs() {
9937 let prerequisite_reader = ObjectDatabase::new(ObjectFormat::Sha1);
9938 let mut writer = ObjectDatabase::new(ObjectFormat::Sha1);
9939 let object = EncodedObject::new(ObjectType::Blob, b"bundle object\n".to_vec());
9940 let oid = object
9941 .object_id(ObjectFormat::Sha1)
9942 .expect("test operation should succeed");
9943 let pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&object))
9944 .expect("test operation should succeed");
9945 let bundle_bytes = format!("# v2 git bundle\n{oid} refs/heads/main\n\n")
9946 .into_bytes()
9947 .into_iter()
9948 .chain(pack.pack)
9949 .collect::<Vec<_>>();
9950 let bundle = Bundle::parse(&bundle_bytes, ObjectFormat::Sha1)
9951 .expect("test operation should succeed");
9952
9953 let result = unbundle_objects(&bundle, &prerequisite_reader, &mut writer)
9954 .expect("test operation should succeed");
9955 assert_eq!(result.written_objects, vec![oid]);
9956 assert_eq!(result.references, bundle.references);
9957 assert_eq!(read_object_for_assert(&writer, &oid), object);
9958 }
9959
9960 #[test]
9961 fn install_bundle_pack_writes_pack_and_returns_refs() {
9962 let root = temp_root("sley-install-bundle-pack");
9963 let git_dir = root.join(".git");
9964 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
9965 let prerequisite_reader = ObjectDatabase::new(ObjectFormat::Sha1);
9966 let database = FileObjectDatabase::from_git_dir(&git_dir, ObjectFormat::Sha1);
9967 let object = EncodedObject::new(ObjectType::Blob, b"bundle pack object\n".to_vec());
9968 let oid = object
9969 .object_id(ObjectFormat::Sha1)
9970 .expect("test operation should succeed");
9971 let pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&object))
9972 .expect("test operation should succeed");
9973 let bundle_bytes = format!("# v2 git bundle\n{oid} refs/heads/main\n\n")
9974 .into_bytes()
9975 .into_iter()
9976 .chain(pack.pack)
9977 .collect::<Vec<_>>();
9978 let bundle = Bundle::parse(&bundle_bytes, ObjectFormat::Sha1)
9979 .expect("test operation should succeed");
9980
9981 let result = install_bundle_pack(&bundle, &prerequisite_reader, &database)
9982 .expect("test operation should succeed");
9983
9984 assert_eq!(result.written_objects, vec![oid]);
9985 assert_eq!(result.references, bundle.references);
9986 assert!(
9987 database
9988 .contains(&oid)
9989 .expect("test operation should succeed")
9990 );
9991 assert_eq!(read_object_for_assert(&database, &oid), object);
9992 assert!(
9993 !database
9994 .loose()
9995 .object_path(&oid)
9996 .expect("test operation should succeed")
9997 .exists()
9998 );
9999 fs::remove_dir_all(root).expect("test operation should succeed");
10000 }
10001
10002 #[test]
10003 fn unpack_packfile_objects_writes_sha256_pack_entries() {
10004 let writer = ObjectDatabase::new(ObjectFormat::Sha256);
10005 let object = EncodedObject::new(ObjectType::Blob, b"transport pack object\n".to_vec());
10006 let oid = object
10007 .object_id(ObjectFormat::Sha256)
10008 .expect("test operation should succeed");
10009 let pack = PackFile::write_undeltified(std::slice::from_ref(&object), ObjectFormat::Sha256)
10010 .expect("test operation should succeed");
10011
10012 let result = unpack_packfile_objects(&pack.pack, ObjectFormat::Sha256, &writer)
10013 .expect("test operation should succeed");
10014
10015 assert_eq!(result.written_objects, vec![oid]);
10016 assert_eq!(read_object_for_assert(&writer, &oid), object);
10017 }
10018
10019 #[test]
10020 fn unbundle_objects_rejects_missing_prerequisites_before_writing() {
10021 let prerequisite_reader = ObjectDatabase::new(ObjectFormat::Sha1);
10022 let mut writer = ObjectDatabase::new(ObjectFormat::Sha1);
10023 let missing = sley_core::object_id_for_bytes(ObjectFormat::Sha1, "blob", b"missing\n")
10024 .expect("test operation should succeed");
10025 let object = EncodedObject::new(ObjectType::Blob, b"bundle object\n".to_vec());
10026 let oid = object
10027 .object_id(ObjectFormat::Sha1)
10028 .expect("test operation should succeed");
10029 let pack = PackFile::write_undeltified_sha1(std::slice::from_ref(&object))
10030 .expect("test operation should succeed");
10031 let bundle_bytes =
10032 format!("# v2 git bundle\n-{missing} missing\n{oid} refs/heads/main\n\n")
10033 .into_bytes()
10034 .into_iter()
10035 .chain(pack.pack)
10036 .collect::<Vec<_>>();
10037 let bundle = Bundle::parse(&bundle_bytes, ObjectFormat::Sha1)
10038 .expect("test operation should succeed");
10039
10040 assert!(unbundle_objects(&bundle, &prerequisite_reader, &mut writer).is_err());
10041 assert!(!writer.contains(&oid));
10042 }
10043
10044 fn write_commit_graph(
10047 db: &mut FileObjectDatabase,
10048 payload: &[u8],
10049 ) -> Vec<(ObjectId, EncodedObject)> {
10050 let blob = EncodedObject::new(ObjectType::Blob, payload.to_vec());
10051 let blob_oid = db
10052 .write_object(blob.clone())
10053 .expect("test operation should succeed");
10054 let tree = EncodedObject::new(
10055 ObjectType::Tree,
10056 Tree {
10057 entries: vec![TreeEntry {
10058 mode: 0o100644,
10059 name: BString::from(b"payload.txt"),
10060 oid: blob_oid,
10061 }],
10062 }
10063 .write(),
10064 );
10065 let tree_oid = db
10066 .write_object(tree.clone())
10067 .expect("test operation should succeed");
10068 let identity = b"Example <example@example.invalid> 0 +0000".to_vec();
10069 let commit = EncodedObject::new(
10070 ObjectType::Commit,
10071 Commit {
10072 tree: tree_oid,
10073 parents: Vec::new(),
10074 author: identity.clone(),
10075 committer: identity,
10076 encoding: None,
10077 message: b"initial\n".to_vec(),
10078 }
10079 .write(),
10080 );
10081 let commit_oid = db
10082 .write_object(commit.clone())
10083 .expect("test operation should succeed");
10084 vec![(commit_oid, commit), (tree_oid, tree), (blob_oid, blob)]
10085 }
10086
10087 fn repack_all_objects_consolidates_loose_and_pack(format: ObjectFormat) {
10088 let root = temp_root("sley-repack-all");
10089 let git_dir = root.join(".git");
10090 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
10091 let mut db = FileObjectDatabase::from_git_dir(&git_dir, format);
10092
10093 let packed_blob = EncodedObject::new(ObjectType::Blob, b"already packed\n".to_vec());
10095 let packed_oid = packed_blob
10096 .object_id(format)
10097 .expect("test operation should succeed");
10098 let existing_pack = PackFile::write_undeltified(std::slice::from_ref(&packed_blob), format)
10099 .expect("test operation should succeed");
10100 let existing = db
10101 .install_pack(&existing_pack)
10102 .expect("test operation should succeed");
10103
10104 let graph = write_commit_graph(&mut db, b"repack payload\n");
10105
10106 let mut expected: HashMap<ObjectId, EncodedObject> = graph.iter().cloned().collect();
10107 expected.insert(packed_oid, packed_blob.clone());
10108
10109 let result = repack_all_objects(&git_dir, format)
10110 .expect("test operation should succeed")
10111 .expect("repository has objects");
10112
10113 assert_eq!(result.object_count, expected.len());
10115 let parsed = PackFile::parse(&result.pack, format).expect("test operation should succeed");
10116 assert_eq!(parsed.entries.len(), expected.len());
10117 for entry in &parsed.entries {
10118 let want = expected
10119 .get(&entry.entry.oid)
10120 .expect("packed object was in the repository");
10121 assert_eq!(&entry.object, want);
10122 assert_eq!(
10123 entry
10124 .object
10125 .object_id(format)
10126 .expect("test operation should succeed"),
10127 entry.entry.oid
10128 );
10129 }
10130 let idx = PackIndex::parse(&result.idx, format).expect("test operation should succeed");
10132 assert_eq!(idx.pack_checksum, parsed.checksum);
10133 assert_eq!(idx.entries.len(), expected.len());
10134
10135 assert_eq!(result.obsolete_packs, vec![existing.pack_path.clone()]);
10137 let mut want_loose: Vec<ObjectId> = graph.iter().map(|(oid, _)| *oid).collect();
10139 want_loose.sort_by_key(ObjectId::to_hex);
10140 assert_eq!(result.packed_loose, want_loose);
10141 assert!(!result.packed_loose.contains(&packed_oid));
10142
10143 fs::remove_dir_all(root).expect("test operation should succeed");
10144 }
10145
10146 #[test]
10147 fn repack_all_objects_consolidates_loose_and_pack_sha1() {
10148 repack_all_objects_consolidates_loose_and_pack(ObjectFormat::Sha1);
10149 }
10150
10151 #[test]
10152 fn repack_all_objects_consolidates_loose_and_pack_sha256() {
10153 repack_all_objects_consolidates_loose_and_pack(ObjectFormat::Sha256);
10154 }
10155
10156 #[test]
10157 fn repack_all_objects_returns_none_for_empty_repository() {
10158 let root = temp_root("sley-repack-empty");
10159 let git_dir = root.join(".git");
10160 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
10161
10162 assert!(
10163 repack_all_objects(&git_dir, ObjectFormat::Sha1)
10164 .expect("test operation should succeed")
10165 .is_none()
10166 );
10167
10168 fs::remove_dir_all(root).expect("test operation should succeed");
10169 }
10170
10171 #[test]
10172 fn install_repack_result_writes_pack_without_pruning_by_default() {
10173 let root = temp_root("sley-repack-install-nodelete");
10174 let git_dir = root.join(".git");
10175 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
10176 let format = ObjectFormat::Sha1;
10177 let mut db = FileObjectDatabase::from_git_dir(&git_dir, format);
10178 let graph = write_commit_graph(&mut db, b"install no prune\n");
10179
10180 let result = repack_all_objects(&git_dir, format)
10181 .expect("test operation should succeed")
10182 .expect("test operation should succeed");
10183 install_repack_result(&git_dir, format, &result, false)
10184 .expect("test operation should succeed");
10185
10186 let parsed = PackFile::parse(&result.pack, format).expect("test operation should succeed");
10188 let pack_dir = git_dir.join("objects").join("pack");
10189 let pack_path = pack_dir.join(format!("pack-{}.pack", parsed.checksum.to_hex()));
10190 let idx_path = pack_dir.join(format!("pack-{}.idx", parsed.checksum.to_hex()));
10191 assert!(pack_path.exists());
10192 assert!(idx_path.exists());
10193 for (oid, object) in &graph {
10195 assert!(
10196 db.loose()
10197 .object_path(oid)
10198 .expect("test operation should succeed")
10199 .exists()
10200 );
10201 assert_eq!(read_object_for_assert(&db, oid), *object);
10202 }
10203
10204 fs::remove_dir_all(root).expect("test operation should succeed");
10205 }
10206
10207 #[test]
10208 fn install_repack_result_prunes_obsolete_packs_and_loose_objects() {
10209 let root = temp_root("sley-repack-install-prune");
10210 let git_dir = root.join(".git");
10211 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
10212 let format = ObjectFormat::Sha1;
10213 let mut db = FileObjectDatabase::from_git_dir(&git_dir, format);
10214
10215 let packed_blob = EncodedObject::new(ObjectType::Blob, b"prune packed\n".to_vec());
10216 let existing_pack = PackFile::write_undeltified(std::slice::from_ref(&packed_blob), format)
10217 .expect("test operation should succeed");
10218 let existing = db
10219 .install_pack(&existing_pack)
10220 .expect("test operation should succeed");
10221 let graph = write_commit_graph(&mut db, b"prune payload\n");
10222
10223 let result = repack_all_objects(&git_dir, format)
10224 .expect("test operation should succeed")
10225 .expect("test operation should succeed");
10226 let new_pack_checksum = PackFile::parse(&result.pack, format)
10227 .expect("test operation should succeed")
10228 .checksum;
10229 install_repack_result(&git_dir, format, &result, true)
10230 .expect("test operation should succeed");
10231
10232 assert!(!existing.pack_path.exists());
10234 assert!(!existing.index_path.exists());
10235 for (oid, _) in &graph {
10237 assert!(
10238 !db.loose()
10239 .object_path(oid)
10240 .expect("test operation should succeed")
10241 .exists()
10242 );
10243 }
10244 let pack_dir = git_dir.join("objects").join("pack");
10246 assert!(
10247 pack_dir
10248 .join(format!("pack-{}.pack", new_pack_checksum.to_hex()))
10249 .exists()
10250 );
10251 let reopened = FileObjectDatabase::from_git_dir(&git_dir, format);
10252 for (oid, object) in &graph {
10253 assert!(
10254 reopened
10255 .contains(oid)
10256 .expect("test operation should succeed")
10257 );
10258 assert_eq!(read_object_for_assert(&reopened, oid), *object);
10259 }
10260 let packed_oid = packed_blob
10261 .object_id(format)
10262 .expect("test operation should succeed");
10263 assert_eq!(read_object_for_assert(&reopened, &packed_oid), packed_blob);
10264
10265 fs::remove_dir_all(root).expect("test operation should succeed");
10266 }
10267
10268 #[test]
10269 fn install_repack_result_preserves_keep_and_promisor_packs() {
10270 let root = temp_root("sley-repack-install-keep-promisor");
10271 let git_dir = root.join(".git");
10272 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
10273 let format = ObjectFormat::Sha1;
10274 let mut db = FileObjectDatabase::from_git_dir(&git_dir, format);
10275
10276 let keep_blob = EncodedObject::new(ObjectType::Blob, b"keep protected\n".to_vec());
10277 let keep_pack = PackFile::write_undeltified(std::slice::from_ref(&keep_blob), format)
10278 .expect("test operation should succeed");
10279 let keep_install = db
10280 .install_pack(&keep_pack)
10281 .expect("test operation should succeed");
10282 let keep_sidecar = keep_install.pack_path.with_extension("keep");
10283 fs::write(&keep_sidecar, b"").expect("test operation should succeed");
10284
10285 let promisor_blob = EncodedObject::new(ObjectType::Blob, b"promisor protected\n".to_vec());
10286 let promisor_pack =
10287 PackFile::write_undeltified(std::slice::from_ref(&promisor_blob), format)
10288 .expect("test operation should succeed");
10289 let promisor_install = db
10290 .install_pack_with_options(&promisor_pack, RawPackInstallOptions { promisor: true })
10291 .expect("test operation should succeed");
10292 let promisor_sidecar = promisor_install
10293 .promisor_path
10294 .clone()
10295 .expect("promisor sidecar");
10296
10297 let graph = write_commit_graph(&mut db, b"new consolidated payload\n");
10298 let result = repack_all_objects(&git_dir, format)
10299 .expect("test operation should succeed")
10300 .expect("test operation should succeed");
10301 assert!(result.obsolete_packs.contains(&keep_install.pack_path));
10302 assert!(result.obsolete_packs.contains(&promisor_install.pack_path));
10303
10304 install_repack_result(&git_dir, format, &result, true)
10305 .expect("test operation should succeed");
10306
10307 for path in [
10308 &keep_install.pack_path,
10309 &keep_install.index_path,
10310 &keep_sidecar,
10311 &promisor_install.pack_path,
10312 &promisor_install.index_path,
10313 &promisor_sidecar,
10314 ] {
10315 assert!(path.exists(), "{} should be preserved", path.display());
10316 }
10317 for (oid, _) in &graph {
10318 assert!(
10319 !db.loose()
10320 .object_path(oid)
10321 .expect("test operation should succeed")
10322 .exists()
10323 );
10324 }
10325
10326 fs::remove_dir_all(root).expect("test operation should succeed");
10327 }
10328
10329 #[test]
10330 fn install_repack_result_keeps_loose_object_absent_from_new_pack() {
10331 let root = temp_root("sley-repack-install-safety");
10334 let git_dir = root.join(".git");
10335 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
10336 let format = ObjectFormat::Sha1;
10337 let mut db = FileObjectDatabase::from_git_dir(&git_dir, format);
10338 let graph = write_commit_graph(&mut db, b"safety packed\n");
10339
10340 let mut result = repack_all_objects(&git_dir, format)
10341 .expect("test operation should succeed")
10342 .expect("test operation should succeed");
10343
10344 let stray = EncodedObject::new(ObjectType::Blob, b"never packed\n".to_vec());
10346 let stray_oid = db
10347 .write_object(stray.clone())
10348 .expect("test operation should succeed");
10349 assert!(!result.packed_loose.contains(&stray_oid));
10350 result.packed_loose.push(stray_oid);
10351
10352 install_repack_result(&git_dir, format, &result, true)
10353 .expect("test operation should succeed");
10354
10355 assert!(
10357 db.loose()
10358 .object_path(&stray_oid)
10359 .expect("test operation should succeed")
10360 .exists()
10361 );
10362 assert_eq!(read_object_for_assert(&db, &stray_oid), stray);
10363 for (oid, _) in &graph {
10365 assert!(
10366 !db.loose()
10367 .object_path(oid)
10368 .expect("test operation should succeed")
10369 .exists()
10370 );
10371 }
10372
10373 fs::remove_dir_all(root).expect("test operation should succeed");
10374 }
10375
10376 #[test]
10377 fn prune_unreachable_loose_reports_and_deletes_only_unreachable() {
10378 let root = temp_root("sley-prune-unreachable");
10379 let git_dir = root.join(".git");
10380 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
10381 let format = ObjectFormat::Sha1;
10382 let mut db = FileObjectDatabase::from_git_dir(&git_dir, format);
10383 let graph = write_commit_graph(&mut db, b"reachable payload\n");
10384 let commit_oid = graph[0].0.clone();
10385
10386 let dangling = EncodedObject::new(ObjectType::Blob, b"dangling\n".to_vec());
10388 let dangling_oid = db
10389 .write_object(dangling)
10390 .expect("test operation should succeed");
10391
10392 let reported = prune_unreachable_loose(&git_dir, format, [commit_oid], false)
10394 .expect("test operation should succeed");
10395 assert_eq!(reported, vec![dangling_oid]);
10396 assert!(
10397 db.loose()
10398 .object_path(&dangling_oid)
10399 .expect("test operation should succeed")
10400 .exists()
10401 );
10402
10403 let deleted = prune_unreachable_loose(&git_dir, format, [commit_oid], true)
10405 .expect("test operation should succeed");
10406 assert_eq!(deleted, vec![dangling_oid]);
10407 assert!(
10408 !db.loose()
10409 .object_path(&dangling_oid)
10410 .expect("test operation should succeed")
10411 .exists()
10412 );
10413 for (oid, object) in &graph {
10414 assert!(
10415 db.loose()
10416 .object_path(oid)
10417 .expect("test operation should succeed")
10418 .exists()
10419 );
10420 assert_eq!(read_object_for_assert(&db, oid), *object);
10421 }
10422
10423 fs::remove_dir_all(root).expect("test operation should succeed");
10424 }
10425
10426 #[test]
10427 fn prune_unreachable_loose_ignores_gitlink_targets() {
10428 let root = temp_root("sley-prune-gitlink");
10429 let git_dir = root.join(".git");
10430 fs::create_dir_all(git_dir.join("objects")).expect("test operation should succeed");
10431 let format = ObjectFormat::Sha1;
10432 let db = FileObjectDatabase::from_git_dir(&git_dir, format);
10433
10434 let submodule_oid = ObjectId::from_hex(format, "1111111111111111111111111111111111111111")
10435 .expect("test operation should succeed");
10436 let tree = EncodedObject::new(
10437 ObjectType::Tree,
10438 Tree {
10439 entries: vec![TreeEntry {
10440 mode: 0o160000,
10441 name: BString::from(b"submodule"),
10442 oid: submodule_oid,
10443 }],
10444 }
10445 .write(),
10446 );
10447 let tree_oid = db
10448 .write_object(tree)
10449 .expect("test operation should succeed");
10450 let identity = b"Example <example@example.invalid> 0 +0000".to_vec();
10451 let commit = EncodedObject::new(
10452 ObjectType::Commit,
10453 Commit {
10454 tree: tree_oid,
10455 parents: Vec::new(),
10456 author: identity.clone(),
10457 committer: identity,
10458 encoding: None,
10459 message: b"gitlink\n".to_vec(),
10460 }
10461 .write(),
10462 );
10463 let commit_oid = db
10464 .write_object(commit)
10465 .expect("test operation should succeed");
10466 let dangling = EncodedObject::new(ObjectType::Blob, b"dangling with gitlink\n".to_vec());
10467 let dangling_oid = db
10468 .write_object(dangling)
10469 .expect("test operation should succeed");
10470
10471 let deleted = prune_unreachable_loose(&git_dir, format, [commit_oid], true)
10472 .expect("test operation should succeed");
10473
10474 assert_eq!(deleted, vec![dangling_oid]);
10475 assert!(
10476 !db.loose()
10477 .object_path(&dangling_oid)
10478 .expect("test operation should succeed")
10479 .exists()
10480 );
10481
10482 fs::remove_dir_all(root).expect("test operation should succeed");
10483 }
10484
10485 fn temp_root(prefix: &str) -> PathBuf {
10486 std::env::temp_dir().join(format!(
10487 "{prefix}-{}-{}",
10488 std::process::id(),
10489 TEMPFILE_COUNTER.fetch_add(1, Ordering::Relaxed)
10490 ))
10491 }
10492}