1use crate::error::ObjectError;
8use std::collections::HashSet;
9use std::ops::Deref;
10use std::path::{Path, PathBuf};
11#[cfg(any(test, feature = "test-support"))]
12use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
13
14use tempfile::TempPath;
15
16use crate::error::{Error, Result};
17use crate::fs_publish::publish_no_clobber;
18use crate::hash::{verify_trailer, Parallelism};
19use crate::objects::{ObjectId, ObjectKind};
20use crate::odb::store::touch_pack_mtime;
21use crate::odb::Odb;
22use crate::pack::{verify_pack_and_collect, write_v2_pack_index_sorting};
23use crate::pack_index_build::{build_mapped_pack_index, RESIDENT_PACK_BYTES};
24use crate::pack_map::{PackData, ResidentBudget};
25use crate::progress::{poll_cancel, CancelToken, ProgressSink};
26use crate::remote::RemoteError;
27use crate::unpack_objects::{IngestedObjects, PackIndexRecord};
28
29pub use crate::pack_index_build::build_pack_index_records;
30
31#[cfg(any(test, feature = "test-support"))]
32static INGEST_THREAD_RECORDING: AtomicBool = AtomicBool::new(false);
33#[cfg(any(test, feature = "test-support"))]
34static LAST_RESOLVED_INGEST_THREADS: AtomicUsize = AtomicUsize::new(0);
35
36#[cfg(any(test, feature = "test-support"))]
38pub(crate) fn set_ingest_thread_recording_for_tests(enabled: bool) {
39 INGEST_THREAD_RECORDING.store(enabled, Ordering::SeqCst);
40 if !enabled {
41 LAST_RESOLVED_INGEST_THREADS.store(0, Ordering::SeqCst);
42 }
43}
44
45#[cfg(any(test, feature = "test-support"))]
47pub(crate) fn last_resolved_ingest_threads_for_tests() -> Option<usize> {
48 if !INGEST_THREAD_RECORDING.load(Ordering::SeqCst) {
49 return None;
50 }
51 let raw = LAST_RESOLVED_INGEST_THREADS.load(Ordering::SeqCst);
52 (raw > 0).then(|| raw - 1)
53}
54
55fn record_resolved_ingest_threads(#[allow(unused_variables)] threads: usize) {
56 #[cfg(any(test, feature = "test-support"))]
57 if INGEST_THREAD_RECORDING.load(Ordering::SeqCst) {
58 LAST_RESOLVED_INGEST_THREADS.store(threads.saturating_add(1), Ordering::SeqCst);
59 }
60}
61
62#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
64#[cfg_attr(feature = "serde-models", derive(serde::Serialize))]
65pub struct ObjectKindCounts {
66 pub commits: u64,
68 pub trees: u64,
70 pub blobs: u64,
72 pub tags: u64,
74}
75
76impl ObjectKindCounts {
77 pub fn add(&mut self, kind: ObjectKind) {
79 match kind {
80 ObjectKind::Commit => self.commits += 1,
81 ObjectKind::Tree => self.trees += 1,
82 ObjectKind::Blob => self.blobs += 1,
83 ObjectKind::Tag => self.tags += 1,
84 }
85 }
86
87 #[must_use]
89 pub fn total(&self) -> u64 {
90 self.commits + self.trees + self.blobs + self.tags
91 }
92}
93
94impl std::ops::AddAssign for ObjectKindCounts {
95 fn add_assign(&mut self, other: Self) {
96 self.commits += other.commits;
97 self.trees += other.trees;
98 self.blobs += other.blobs;
99 self.tags += other.tags;
100 }
101}
102
103impl FromIterator<ObjectKind> for ObjectKindCounts {
104 fn from_iter<I: IntoIterator<Item = ObjectKind>>(iter: I) -> Self {
105 let mut counts = Self::default();
106 for kind in iter {
107 counts.add(kind);
108 }
109 counts
110 }
111}
112
113#[derive(Debug, Clone, Default)]
115pub struct IngestedPack {
116 pub object_ids: HashSet<ObjectId>,
118 pub pack_path: Option<PathBuf>,
124 pub object_counts: ObjectKindCounts,
126 pub pack_bytes: u64,
129}
130
131impl From<IngestedObjects> for IngestedPack {
132 fn from(ingested: IngestedObjects) -> Self {
133 Self {
134 object_ids: ingested.object_ids,
135 pack_path: None,
136 object_counts: ingested.object_counts,
137 pack_bytes: ingested.pack_bytes,
138 }
139 }
140}
141
142#[derive(Debug, Clone)]
144pub struct IngestPackOptions {
145 pub fix_thin: bool,
148 pub threads: Option<usize>,
150 pub skip_post_index_verify: bool,
153 pub merged_config: Option<crate::config::ConfigSet>,
155 pub environment: crate::environment::Environment,
157 pub mark_promisor_pack: bool,
159 pub keep_message: Option<String>,
166 pub trusted_index_rows: Option<Vec<(ObjectId, u64, u32)>>,
171}
172
173impl Default for IngestPackOptions {
174 fn default() -> Self {
175 Self {
176 fix_thin: false,
177 threads: None,
178 skip_post_index_verify: false,
179 merged_config: None,
180 environment: crate::environment::Environment::empty(),
181 mark_promisor_pack: false,
182 keep_message: None,
183 trusted_index_rows: None,
184 }
185 }
186}
187
188struct IngestPackPublishPolicy {
189 shared_repo: i32,
190 write_rev: bool,
191}
192
193fn resolve_ingest_pack_publish_policy(
194 opts: &IngestPackOptions,
195 odb: &Odb,
196) -> Result<IngestPackPublishPolicy> {
197 let cfg = if let Some(c) = &opts.merged_config {
198 Some(c.clone())
199 } else if let Some(git_dir) = odb.config_git_dir() {
200 Some(crate::config::ConfigSet::load(
201 &opts.environment,
202 Some(git_dir),
203 &crate::config::LoadConfigOptions::standard(&opts.environment, Some(git_dir)),
204 )?)
205 } else {
206 None
207 };
208 let shared_raw = cfg.as_ref().and_then(|c| c.value("core.sharedRepository"));
209 let shared_repo =
210 crate::shared_repo::shared_repository_from_config_value(shared_raw.as_deref())
211 .map_err(|msg| Error::Io(std::io::Error::new(std::io::ErrorKind::InvalidInput, msg)))?;
212 let write_rev = cfg
213 .as_ref()
214 .is_none_or(|c| c.pack_write_reverse_index_default());
215 Ok(IngestPackPublishPolicy {
216 shared_repo,
217 write_rev,
218 })
219}
220
221impl IngestPackOptions {
222 fn index_parallelism(&self, odb: &Odb) -> Result<Parallelism> {
223 if let Some(n) = self.threads {
224 return Ok(Parallelism::resolve(Some(n)));
225 }
226 if let Some(cfg) = &self.merged_config {
227 return crate::parallel::ThreadPolicy::default()
228 .try_resolve(crate::parallel::ParallelOp::IndexPack, cfg)
229 .map(|o| o.parallelism())
230 .map_err(Error::Config);
231 }
232 if let Some(git_dir) = odb.config_git_dir() {
233 let cfg = crate::config::ConfigSet::load(
234 &self.environment,
235 Some(git_dir),
236 &crate::config::LoadConfigOptions::standard(&self.environment, Some(git_dir)),
237 )?;
238 return crate::parallel::ThreadPolicy::default()
239 .try_resolve(crate::parallel::ParallelOp::IndexPack, &cfg)
240 .map(|o| o.parallelism())
241 .map_err(Error::Config);
242 }
243 Ok(Parallelism::resolve(None))
244 }
245}
246
247pub fn ingest_received_pack(
263 pack: Vec<u8>,
264 odb: &Odb,
265 opts: &IngestPackOptions,
266) -> Result<IngestedPack> {
267 ingest_received_pack_with_progress(pack, odb, opts, None, None)
268}
269
270pub fn ingest_received_pack_with_progress(
272 pack: Vec<u8>,
273 odb: &Odb,
274 opts: &IngestPackOptions,
275 progress: Option<&mut dyn ProgressSink>,
276 cancel: Option<&CancelToken>,
277) -> Result<IngestedPack> {
278 if pack.is_empty() {
279 return Ok(IngestedPack::default());
280 }
281 if pack.len() < 12 || &pack[0..4] != b"PACK" {
282 return Err(Error::Object(ObjectError::Corrupt {
283 detail: "received data is not a pack stream".to_owned(),
284 }));
285 }
286 verify_trailer(odb.hash_algo(), &pack).map_err(|e| {
287 Error::Object(ObjectError::Corrupt {
288 detail: format!("received pack checksum mismatch (download may be incomplete): {e}"),
289 })
290 })?;
291 if !odb.uses_files_primary() {
292 return ingest_into_custom_primary(&mut pack.as_slice(), odb, progress, cancel);
293 }
294 install_pack_bytes_with_progress(pack, odb, opts, progress, cancel)
295}
296
297fn ingest_into_custom_primary(
299 reader: &mut dyn std::io::Read,
300 odb: &Odb,
301 progress: Option<&mut dyn ProgressSink>,
302 cancel: Option<&CancelToken>,
303) -> Result<IngestedPack> {
304 crate::unpack_objects::ingest_pack_into_odb(
305 reader,
306 odb,
307 &crate::unpack_objects::UnpackOptions::default(),
308 progress,
309 cancel,
310 )
311 .map(IngestedPack::from)
312}
313
314pub fn ingest_received_pack_path(
324 pack_path: PathBuf,
325 odb: &Odb,
326 opts: &IngestPackOptions,
327 progress: &mut dyn ProgressSink,
328 cancel: &CancelToken,
329) -> Result<IngestedPack> {
330 ingest_received_pack_path_with_progress(pack_path, odb, opts, Some(progress), Some(cancel))
331}
332
333fn wait_for_received_pack_path(path: PathBuf) -> Result<PathBuf> {
334 use std::io::ErrorKind;
335 use std::thread;
336 use std::time::Duration;
337
338 const MAX_ATTEMPTS: u32 = if cfg!(windows) { 12 } else { 4 };
339 for attempt in 0..MAX_ATTEMPTS {
340 if attempt > 0 {
341 thread::sleep(Duration::from_millis(25_u64.saturating_mul(attempt as u64)));
342 }
343 match std::fs::metadata(&path) {
344 Ok(meta) if meta.is_file() => return Ok(path),
345 Ok(_) => break,
346 Err(e) if e.kind() == ErrorKind::NotFound => continue,
347 Err(e) => return Err(Error::Io(e)),
348 }
349 }
350 Err(RemoteError::ReceivedPackTempMissing { path }.into())
351}
352
353pub(crate) fn ingest_received_pack_path_with_progress(
359 pack_path: PathBuf,
360 odb: &Odb,
361 opts: &IngestPackOptions,
362 progress: Option<&mut dyn ProgressSink>,
363 cancel: Option<&CancelToken>,
364) -> Result<IngestedPack> {
365 crate::trace::trace_span!("grit.pack.index");
366 let pack_path = wait_for_received_pack_path(pack_path)?;
367 if !odb.uses_files_primary() {
368 let ingested = std::fs::File::open(&pack_path)
369 .map_err(Error::Io)
370 .and_then(|file| {
371 let mut reader = std::io::BufReader::with_capacity(1 << 20, file);
372 ingest_into_custom_primary(&mut reader, odb, progress, cancel)
373 });
374 let _ = std::fs::remove_file(&pack_path);
375 return ingested;
376 }
377 install_pack_path_with_progress(&pack_path, odb, opts, progress, cancel)
378}
379
380pub fn install_pack_bytes(
384 pack: Vec<u8>,
385 odb: &Odb,
386 opts: &IngestPackOptions,
387) -> Result<IngestedPack> {
388 install_pack_bytes_with_progress(pack, odb, opts, None, None)
389}
390
391pub fn install_pack_bytes_with_progress(
393 pack: Vec<u8>,
394 odb: &Odb,
395 opts: &IngestPackOptions,
396 progress: Option<&mut dyn ProgressSink>,
397 cancel: Option<&CancelToken>,
398) -> Result<IngestedPack> {
399 odb.primary_store()
400 .require_files_primary(crate::repo::error::FilesOnlyOperation::PackInstall)?;
401 let pack_dir = odb.objects_dir().join("pack");
402 std::fs::create_dir_all(&pack_dir).map_err(Error::Io)?;
403 let tmp = pack_dir.join(format!("tmp_install_{}", unique_install_suffix()));
404 std::fs::write(&tmp, &pack).map_err(Error::Io)?;
405 match install_pack_path_with_progress(&tmp, odb, opts, progress, cancel) {
406 Ok(ingested) => Ok(ingested),
407 Err(e) => {
408 let _ = std::fs::remove_file(&tmp);
409 Err(e)
410 }
411 }
412}
413
414pub fn install_pack_path(
431 pack_path: &Path,
432 odb: &Odb,
433 opts: &IngestPackOptions,
434 progress: &mut dyn ProgressSink,
435 cancel: &CancelToken,
436) -> Result<IngestedPack> {
437 install_pack_path_with_progress(pack_path, odb, opts, Some(progress), Some(cancel))
438}
439
440pub(crate) fn install_pack_path_with_progress(
446 pack_path: &Path,
447 odb: &Odb,
448 opts: &IngestPackOptions,
449 mut progress: Option<&mut dyn ProgressSink>,
450 cancel: Option<&CancelToken>,
451) -> Result<IngestedPack> {
452 odb.primary_store()
453 .require_files_primary(crate::repo::error::FilesOnlyOperation::PackInstall)?;
454 let publish_policy = resolve_ingest_pack_publish_policy(opts, odb)?;
455 let owned = pack_path.to_path_buf();
456 let hb = odb.hash_algo().len();
457 let meta = std::fs::metadata(&owned).map_err(Error::Io)?;
458 let len = meta.len();
459 if len < 12 + hb as u64 {
460 return Err(Error::Object(ObjectError::Corrupt {
461 detail: "pack too small".to_owned(),
462 }));
463 }
464 let mut header = [0u8; 12];
465 {
466 use std::io::Read;
467 let mut f = std::fs::File::open(&owned).map_err(Error::Io)?;
468 f.read_exact(&mut header).map_err(Error::Io)?;
469 }
470 if &header[0..4] != b"PACK" {
471 return Err(Error::Object(ObjectError::Corrupt {
472 detail: "received data is not a pack stream".to_owned(),
473 }));
474 }
475 let mut trailer = vec![0u8; hb];
476 {
477 use std::io::{Read, Seek, SeekFrom};
478 let mut f = std::fs::File::open(&owned).map_err(Error::Io)?;
479 f.seek(SeekFrom::End(-(hb as i64))).map_err(Error::Io)?;
480 f.read_exact(&mut trailer).map_err(Error::Io)?;
481 }
482 let pack_hash = ObjectId::from_bytes(&trailer)?;
483 let stem = format!("pack-{}", pack_hash.to_hex());
484 let pack_dir = odb.objects_dir().join("pack");
485 std::fs::create_dir_all(&pack_dir).map_err(Error::Io)?;
486
487 let stage = pack_dir.join(format!(".{stem}-install-{}", unique_install_suffix()));
489 let stage_pack = stage.join(format!("{stem}.pack"));
490 let stage_idx = stage.join(format!("{stem}.idx"));
491
492 let install_result = (|| -> Result<IngestedPack> {
493 poll_ingest_cancel(&mut progress, cancel)?;
494 std::fs::create_dir_all(&stage).map_err(Error::Io)?;
495 if std::fs::rename(&owned, &stage_pack).is_err() {
496 std::fs::copy(&owned, &stage_pack).map_err(Error::Io)?;
497 let _ = std::fs::remove_file(&owned);
498 }
499 poll_ingest_cancel(&mut progress, cancel)?;
500 let parallelism = opts.index_parallelism(odb)?;
501 record_resolved_ingest_threads(parallelism.threads().get());
502 let sort_threads = parallelism.threads().get();
503 let trusted_as_written = opts
506 .trusted_index_rows
507 .as_deref()
508 .filter(|_| opts.skip_post_index_verify);
509 let mut trailer = trailer;
510 let (mut entries, object_counts) = if let Some(rows) = trusted_as_written {
511 if rows.len()
512 != u32::from_be_bytes([header[8], header[9], header[10], header[11]]) as usize
513 {
514 return Err(Error::Object(ObjectError::Corrupt {
515 detail: "trusted index row count does not match pack header".to_owned(),
516 }));
517 }
518 (rows.to_vec(), ObjectKindCounts::default())
519 } else {
520 let indexed = PackData::open(&stage_pack)?;
521 let resident = ResidentBudget::new(&indexed, RESIDENT_PACK_BYTES);
522 let pack_bytes: &[u8] = indexed.deref();
523 let body_len = pack_bytes.len() - hb;
524 if resident.checksum(odb.hash_algo(), &pack_bytes[..body_len]) != pack_bytes[body_len..]
525 {
526 return Err(Error::Object(ObjectError::Corrupt {
527 detail: "received pack checksum mismatch (download may be incomplete)"
528 .to_owned(),
529 }));
530 }
531 poll_ingest_cancel(&mut progress, cancel)?;
532 trailer = pack_bytes[body_len..].to_vec();
533 if let Some(rows) = &opts.trusted_index_rows {
534 if rows.len()
535 != u32::from_be_bytes([
536 pack_bytes[8],
537 pack_bytes[9],
538 pack_bytes[10],
539 pack_bytes[11],
540 ]) as usize
541 {
542 return Err(Error::Object(ObjectError::Corrupt {
543 detail: "trusted index row count does not match pack header".to_owned(),
544 }));
545 }
546 drop(indexed);
547 (rows.clone(), ObjectKindCounts::default())
548 } else {
549 let file = std::fs::File::open(&stage_pack).map_err(Error::Io)?;
550 let build = build_mapped_pack_index(
551 &indexed,
552 &file,
553 odb,
554 parallelism,
555 false,
556 progress.as_mut().map(|p| p as &mut dyn ProgressSink),
557 cancel,
558 )?;
559 drop(file);
560 drop(indexed);
561 let mut records = build.records;
562 if opts.fix_thin && !build.thin_bases.is_empty() {
563 poll_ingest_cancel(&mut progress, cancel)?;
564 trailer = append_thin_bases(&stage_pack, odb, &build.thin_bases, &mut records)?;
565 }
566 let object_counts: ObjectKindCounts = records.iter().map(|r| r.kind).collect();
567 let entries: Vec<(ObjectId, u64, u32)> = records
568 .into_iter()
569 .map(
570 |PackIndexRecord {
571 oid, offset, crc32, ..
572 }| (oid, offset, crc32),
573 )
574 .collect();
575 (entries, object_counts)
576 }
577 };
578 poll_ingest_cancel(&mut progress, cancel)?;
579 write_v2_pack_index_sorting(&stage_idx, &mut entries, &trailer, hb, Some(sort_threads))?;
580 drop(entries);
583 let index = crate::pack::read_pack_index_no_verify(&stage_idx)?;
584 let mut oids: HashSet<ObjectId> = HashSet::with_capacity(index.len());
585 for i in 0..index.len() {
586 oids.insert(ObjectId::from_bytes(index.oid_at(i))?);
587 }
588 drop(index);
589 poll_ingest_cancel(&mut progress, cancel)?;
590 if !opts.skip_post_index_verify {
591 verify_pack_and_collect(&stage_idx)?;
592 }
593 poll_ingest_cancel(&mut progress, cancel)?;
594 let stem = format!("pack-{}", ObjectId::from_bytes(&trailer)?.to_hex());
595 let final_pack = pack_dir.join(format!("{stem}.pack"));
596 let idx_path = pack_dir.join(format!("{stem}.idx"));
597 let publish = |staged: &Path, final_path: &Path| -> Result<bool> {
603 let staged = TempPath::try_from_path(staged).map_err(Error::Io)?;
604 publish_no_clobber(staged, final_path).map_err(Error::Io)
605 };
606 let shared_repo = publish_policy.shared_repo;
607 let keep = write_keep_marker(&final_pack, opts)?;
609 let published = (|| -> Result<()> {
610 if !publish(&stage_pack, &final_pack)? {
611 touch_pack_mtime(&final_pack);
612 } else {
613 crate::shared_repo::finalize_pack_file_mode(shared_repo, &final_pack)?;
614 }
615 if publish(&stage_idx, &idx_path)? {
616 crate::shared_repo::finalize_pack_file_mode(shared_repo, &idx_path)?;
617 }
618 if publish_policy.write_rev {
619 crate::pack_rev::write_pack_rev_sidecar_for_index_path_with_shared(
620 &idx_path,
621 shared_repo,
622 )?;
623 }
624 Ok(())
625 })();
626 if let Err(err) = published {
627 if let Some(keep) = keep.filter(|_| !final_pack.is_file()) {
628 let _ = std::fs::remove_file(keep);
629 }
630 return Err(err);
631 }
632 if opts.mark_promisor_pack {
633 std::fs::write(final_pack.with_extension("promisor"), b"").map_err(Error::Io)?;
634 }
635 let _ = std::fs::remove_dir(&stage);
636 let pack_bytes = std::fs::metadata(&final_pack).map_err(Error::Io)?.len();
637 Ok(IngestedPack {
638 object_ids: oids,
639 pack_path: Some(final_pack.clone()),
640 object_counts,
641 pack_bytes,
642 })
643 })();
644
645 if install_result.is_err() {
646 cleanup_install_stage(&stage);
647 let _ = std::fs::remove_file(&owned);
648 }
649 let ingested = install_result?;
650 odb.invalidate_packs();
651 Ok(ingested)
652}
653
654pub(crate) fn sync_installed_pack(pack_path: &Path) -> Result<()> {
667 let sync = |path: &Path| std::fs::File::open(path).and_then(|f| f.sync_all());
668 sync(pack_path).map_err(Error::Io)?;
669 sync(&pack_path.with_extension("idx")).map_err(Error::Io)?;
670 for ext in ["rev", "keep", "promisor"] {
671 let sidecar = pack_path.with_extension(ext);
672 if sidecar.is_file() {
673 sync(&sidecar).map_err(Error::Io)?;
674 }
675 }
676 #[cfg(unix)]
678 if let Some(dir) = pack_path.parent() {
679 sync(dir).map_err(Error::Io)?;
680 }
681 Ok(())
682}
683
684fn write_keep_marker(final_pack: &Path, opts: &IngestPackOptions) -> Result<Option<PathBuf>> {
690 let Some(message) = &opts.keep_message else {
691 return Ok(None);
692 };
693 let path = final_pack.with_extension("keep");
694 let created = !path.exists();
695 let mut contents = message.trim_end_matches('\n').as_bytes().to_vec();
696 if !contents.is_empty() {
697 contents.push(b'\n');
698 }
699 std::fs::write(&path, contents).map_err(Error::Io)?;
700 Ok(created.then_some(path))
701}
702
703fn append_thin_bases(
717 pack_path: &Path,
718 odb: &Odb,
719 bases: &[ObjectId],
720 records: &mut Vec<PackIndexRecord>,
721) -> Result<Vec<u8>> {
722 use std::io::{Read, Seek, SeekFrom, Write};
723
724 let algo = odb.hash_algo();
725 let mut file = std::fs::OpenOptions::new()
726 .read(true)
727 .write(true)
728 .open(pack_path)
729 .map_err(Error::Io)?;
730 let mut header = [0u8; 12];
731 file.read_exact(&mut header).map_err(Error::Io)?;
732 let added = u32::try_from(bases.len()).ok();
733 let count = added
734 .and_then(|added| {
735 u32::from_be_bytes([header[8], header[9], header[10], header[11]]).checked_add(added)
736 })
737 .ok_or_else(|| {
738 Error::Object(ObjectError::Corrupt {
739 detail: "thin pack fix: object count overflow".to_owned(),
740 })
741 })?;
742 header[8..12].copy_from_slice(&count.to_be_bytes());
743
744 let len = file.metadata().map_err(Error::Io)?.len();
745 let mut offset = len - algo.len() as u64;
746 file.seek(SeekFrom::Start(offset)).map_err(Error::Io)?;
747 let mut object = Vec::new();
748 for oid in bases {
749 object.clear();
750 let kind = crate::pack_objects::append_whole_object_from_odb(&mut object, odb, *oid)?;
751 file.write_all(&object).map_err(Error::Io)?;
752 records.push(PackIndexRecord {
753 oid: *oid,
754 offset,
755 crc32: crc32fast::hash(&object),
756 kind,
757 });
758 offset += object.len() as u64;
759 }
760 file.seek(SeekFrom::Start(0)).map_err(Error::Io)?;
761 file.write_all(&header).map_err(Error::Io)?;
762
763 file.seek(SeekFrom::Start(0)).map_err(Error::Io)?;
764 let mut hasher = algo.checksum_hasher();
765 let mut buf = vec![0u8; 1024 * 1024];
766 let mut remaining = offset;
767 while remaining > 0 {
768 let want = usize::try_from(remaining).map_or(buf.len(), |r| r.min(buf.len()));
769 file.read_exact(&mut buf[..want]).map_err(Error::Io)?;
770 hasher.update(&buf[..want]);
771 remaining -= want as u64;
772 }
773 let trailer = hasher.finalize();
774 file.write_all(&trailer).map_err(Error::Io)?;
775 Ok(trailer)
776}
777
778fn poll_ingest_cancel(
779 progress: &mut Option<&mut dyn ProgressSink>,
780 cancel: Option<&CancelToken>,
781) -> Result<()> {
782 let Some(c) = cancel else {
783 return Ok(());
784 };
785 if let Some(p) = progress.as_mut() {
786 poll_cancel(p, c)
787 } else if c.is_cancelled() {
788 Err(Error::Cancelled)
789 } else {
790 Ok(())
791 }
792}
793
794fn unique_install_suffix() -> String {
796 static INSTALL_SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
797 let seq = INSTALL_SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
798 format!("{}_{seq}", std::process::id())
799}
800
801fn cleanup_install_stage(stage: &Path) {
802 if !stage.exists() {
803 return;
804 }
805 if let Ok(entries) = std::fs::read_dir(stage) {
806 for entry in entries.flatten() {
807 let _ = std::fs::remove_file(entry.path());
808 }
809 }
810 let _ = std::fs::remove_dir(stage);
811}
812
813#[cfg(test)]
814mod tests {
815 use super::*;
816 use crate::objects::ObjectKind;
817 use crate::odb::Odb;
818 use crate::pack::read_pack_index;
819 use crate::unpack_objects::pack_index_records;
820 use std::process::Command;
821 use std::sync::Arc;
822
823 fn git_index_pack(pack: &[u8]) -> (tempfile::TempDir, Vec<u8>) {
824 let dir = tempfile::tempdir().expect("tempdir");
825 let pack_path = dir.path().join("in.pack");
826 std::fs::write(&pack_path, pack).expect("write");
827 let out = Command::new("git") .current_dir(dir.path())
829 .args(["index-pack", "-v", &pack_path.to_string_lossy()])
830 .env("GIT_CONFIG_GLOBAL", "/dev/null")
831 .env("GIT_CONFIG_SYSTEM", "/dev/null")
832 .output()
833 .expect("git index-pack");
834 assert!(
835 out.status.success(),
836 "git index-pack: {}",
837 String::from_utf8_lossy(&out.stderr)
838 );
839 let indexed = std::fs::read(dir.path().join("in.pack")).expect("read indexed pack");
840 (dir, indexed)
841 }
842
843 fn single_blob_pack(data: &[u8]) -> Vec<u8> {
844 use crate::transfer::{build_pack, PackBuildOptions};
845 let tmp = tempfile::tempdir().expect("tempdir");
846 let odb = Odb::new(tmp.path());
847 let oid = odb
848 .write(ObjectKind::Blob, data)
849 .expect("write loose blob for pack build");
850 build_pack(
851 &odb,
852 &[oid],
853 &[],
854 &PackBuildOptions {
855 delta: false,
856 ..Default::default()
857 },
858 )
859 .expect("build pack")
860 }
861
862 #[test]
863 fn object_kind_counts_tally_and_total() {
864 let counts: ObjectKindCounts = [
865 ObjectKind::Commit,
866 ObjectKind::Blob,
867 ObjectKind::Blob,
868 ObjectKind::Tree,
869 ObjectKind::Tag,
870 ObjectKind::Commit,
871 ]
872 .into_iter()
873 .collect();
874 assert_eq!(
875 counts,
876 ObjectKindCounts {
877 commits: 2,
878 trees: 1,
879 blobs: 2,
880 tags: 1,
881 }
882 );
883 assert_eq!(counts.total(), 6);
884 assert_eq!(ObjectKindCounts::default().total(), 0);
885 }
886
887 #[test]
888 fn ingest_counts_objects_by_resolved_kind() {
889 use grit_test_support::objects::{HashAlgo, ObjectKind as PK, PackBuilder};
890 let mut pb = PackBuilder::new(HashAlgo::Sha1);
891 pb.add_full(
892 PK::Commit,
893 b"tree 0000000000000000000000000000000000000000\n\nc\n",
894 );
895 pb.add_full(PK::Tree, b"");
896 let base = b"blob base contents\n";
897 let base_idx = pb.add_full(PK::Blob, base);
898 let target = b"blob base contents, edited\n";
899 let delta = crate::delta_encode::encode_lcp_delta(base, target).expect("delta");
900 pb.add_ofs_delta(base_idx, &delta, delta.len());
902 pb.add_full(
903 PK::Tag,
904 b"object 0000000000000000000000000000000000000000\n",
905 );
906 let pack = pb.build().bytes;
907
908 let tmp = tempfile::tempdir().expect("tempdir");
909 let objects = tmp.path().join("objects");
910 std::fs::create_dir_all(objects.join("pack")).expect("pack dir");
911 let odb = Odb::new(&objects);
912 let ingested =
913 ingest_received_pack(pack, &odb, &IngestPackOptions::default()).expect("ingest");
914 assert_eq!(
915 ingested.object_counts,
916 ObjectKindCounts {
917 commits: 1,
918 trees: 1,
919 blobs: 2,
920 tags: 1,
921 }
922 );
923 assert_eq!(
924 ingested.object_counts.total(),
925 u64::try_from(ingested.object_ids.len()).expect("count")
926 );
927 }
928
929 #[test]
932 fn parallel_index_pack_zero_byte_blob_among_many() {
933 use crate::hash::Parallelism;
934 use crate::unpack_objects::pack_index_records;
935 use grit_test_support::objects::{HashAlgo as FixtureAlgo, ObjectKind as PK, PackBuilder};
936
937 let mut pb = PackBuilder::new(FixtureAlgo::Sha1);
938 for i in 0..200 {
939 pb.add_full(PK::Blob, format!("payload-{i}\n").as_bytes());
940 }
941 pb.add_full(PK::Blob, b"");
942 for i in 200..400 {
943 pb.add_full(PK::Blob, format!("after-empty-{i}\n").as_bytes());
944 }
945 let pack = pb.build().bytes;
946 let tmp = tempfile::tempdir().expect("tempdir");
947 let odb = Odb::new(tmp.path());
948 for threads in [1_usize, 8] {
949 let records = pack_index_records(&pack, &odb, Parallelism::resolve(Some(threads)))
950 .unwrap_or_else(|e| panic!("threads={threads}: {e}"));
951 assert_eq!(records.len(), 401, "threads={threads}");
952 }
953 }
954
955 #[test]
956 fn install_pack_cancelled_during_indexing_cleans_stage() {
957 use crate::progress::{CancelToken, ProgressPhase, ProgressSink, ProgressUnit};
958 use grit_test_support::objects::{HashAlgo, ObjectKind as PK, PackBuilder};
959 use std::sync::atomic::{AtomicBool, Ordering};
960 use std::sync::Arc;
961
962 struct CancelAtIndexing {
963 flag: CancelToken,
964 saw_indexing: Arc<AtomicBool>,
965 }
966
967 impl ProgressSink for CancelAtIndexing {
968 fn start(&mut self, phase: ProgressPhase, total: Option<u64>) {
969 if phase == ProgressPhase::IndexingObjects {
970 self.saw_indexing.store(true, Ordering::Relaxed);
971 self.flag.cancel();
972 }
973 let _ = total;
974 }
975 fn start_with_unit(
976 &mut self,
977 phase: ProgressPhase,
978 unit: ProgressUnit,
979 total: Option<u64>,
980 ) {
981 self.start(phase, total);
982 let _ = unit;
983 }
984 }
985
986 let mut pb = PackBuilder::new(HashAlgo::Sha1);
987 for i in 0..400 {
988 pb.add_full(PK::Blob, format!("blob-{i}\n").as_bytes());
989 }
990 let pack = pb.build().bytes;
991 let tmp = tempfile::tempdir().expect("tempdir");
992 let pack_path = tmp.path().join("incoming.pack");
993 std::fs::write(&pack_path, &pack).expect("write pack");
994 let odb = Odb::new(tmp.path());
995 let flag = CancelToken::new();
996 let saw = Arc::new(AtomicBool::new(false));
997 let mut progress = CancelAtIndexing {
998 flag: flag.clone(),
999 saw_indexing: Arc::clone(&saw),
1000 };
1001 let err = install_pack_path_with_progress(
1002 &pack_path,
1003 &odb,
1004 &IngestPackOptions {
1005 threads: Some(4),
1006 ..Default::default()
1007 },
1008 Some(&mut progress),
1009 Some(&flag),
1010 )
1011 .expect_err("cancel during index-pack");
1012 assert!(matches!(err, Error::Cancelled), "got {err:?}");
1013 assert!(saw.load(Ordering::Relaxed), "expected indexing phase");
1014
1015 let pack_dir = tmp.path().join("pack");
1016 if pack_dir.exists() {
1017 for entry in pack_dir_entries(&pack_dir) {
1018 let name = entry
1019 .file_name()
1020 .and_then(|s| s.to_str())
1021 .unwrap_or_default();
1022 assert!(
1023 !name.ends_with(".pack") && !name.ends_with(".idx"),
1024 "leftover install artifact: {}",
1025 entry.display()
1026 );
1027 assert!(
1028 !entry.is_dir() || !name.contains("-install"),
1029 "leftover stage dir: {}",
1030 entry.display()
1031 );
1032 }
1033 }
1034 }
1035
1036 #[test]
1037 fn install_rejects_truncated_pack_before_object_scan() {
1038 let tmp = tempfile::tempdir().expect("tempdir");
1039 let odb = Odb::new(tmp.path());
1040 let mut pack = single_blob_pack(b"truncated-before-index");
1041 pack.truncate(pack.len().saturating_sub(8));
1042 let err = install_pack_bytes(
1043 pack,
1044 &odb,
1045 &IngestPackOptions {
1046 fix_thin: false,
1047 ..Default::default()
1048 },
1049 )
1050 .expect_err("truncated pack");
1051 let msg = err.to_string();
1052 assert!(
1053 msg.contains("checksum mismatch") || msg.contains("truncated"),
1054 "expected checksum/truncated error, got {msg}"
1055 );
1056 }
1057
1058 fn pack_dir_entries(pack_dir: &Path) -> Vec<std::path::PathBuf> {
1059 std::fs::read_dir(pack_dir)
1060 .expect("read pack dir")
1061 .filter_map(|e| e.ok())
1062 .map(|e| e.path())
1063 .collect()
1064 }
1065
1066 #[test]
1068 #[ignore = "manual RSS harness (GRIT_CARGO_PACK)"]
1069 fn rss_mmap_index_records() {
1070 use crate::hash::Parallelism;
1071 use std::env;
1072 let path = env::var("GRIT_CARGO_PACK").expect("GRIT_CARGO_PACK"); let mapped = PackData::open(Path::new(&path)).expect("mmap");
1074 let tmp = tempfile::tempdir().expect("tempdir");
1075 let odb = Odb::new(tmp.path());
1076 let _records = pack_index_records(mapped.deref(), &odb, Parallelism::resolve(Some(4)))
1077 .expect("index records");
1078 }
1079
1080 #[cfg_attr(
1081 not(feature = "full-tests"),
1082 ignore = "starts system git or is heavy; run with --features full-tests"
1083 )]
1084 #[test]
1085 fn install_pack_matches_git_index_pack_layout() {
1086 let blob = b"clone pack retention fixture\n";
1087 let pack = single_blob_pack(blob);
1088 let (_git_dir, git_pack) = git_index_pack(&pack);
1089
1090 let tmp = tempfile::tempdir().expect("tempdir");
1091 let odb = Odb::new(tmp.path());
1092 install_pack_bytes(
1093 git_pack,
1094 &odb,
1095 &IngestPackOptions {
1096 fix_thin: false,
1097 ..Default::default()
1098 },
1099 )
1100 .expect("install");
1101
1102 let pack_dir = tmp.path().join("pack");
1103 let packs: Vec<_> = std::fs::read_dir(&pack_dir)
1104 .expect("read pack dir")
1105 .filter_map(|e| e.ok())
1106 .filter(|e| e.path().extension().is_some_and(|x| x == "pack"))
1107 .collect();
1108 assert_eq!(packs.len(), 1, "expected one pack file");
1109 let loose_count = std::fs::read_dir(tmp.path())
1110 .expect("read objects")
1111 .filter_map(|e| e.ok())
1112 .filter(|e| {
1113 e.file_name()
1114 .to_str()
1115 .is_some_and(|s| s.len() == 2 && s.chars().all(|c| c.is_ascii_hexdigit()))
1116 })
1117 .count();
1118 assert_eq!(loose_count, 0, "install must not create loose objects");
1119
1120 let oid = odb.hash(ObjectKind::Blob, blob).expect("hash");
1121 let obj = odb.read(&oid).expect("read from pack");
1122 assert_eq!(obj.data.as_slice(), blob);
1123 }
1124
1125 fn run_git(cwd: &Path, args: &[&str]) {
1126 let out = Command::new("/usr/bin/git") .args(args)
1128 .current_dir(cwd)
1129 .env("GIT_CONFIG_GLOBAL", "/dev/null")
1130 .env("GIT_CONFIG_SYSTEM", "/dev/null")
1131 .env("GIT_AUTHOR_NAME", "t")
1132 .env("GIT_AUTHOR_EMAIL", "t@t")
1133 .env("GIT_COMMITTER_NAME", "t")
1134 .env("GIT_COMMITTER_EMAIL", "t@t")
1135 .output()
1136 .expect("spawn git");
1137 assert!(
1138 out.status.success(),
1139 "git {args:?} in {}: {}",
1140 cwd.display(),
1141 String::from_utf8_lossy(&out.stderr)
1142 );
1143 }
1144
1145 #[cfg_attr(
1146 not(feature = "full-tests"),
1147 ignore = "starts system git or is heavy; run with --features full-tests"
1148 )]
1149 #[test]
1150 fn local_fetch_ingest_with_fix_thin_from_bare_remote() {
1151 use crate::objects::ObjectId;
1152 use crate::transfer::{build_pack, open_odb, PackBuildOptions};
1153
1154 let tmp = tempfile::tempdir().expect("tempdir");
1155 let work = tmp.path().join("work");
1156 std::fs::create_dir_all(&work).expect("work");
1157 run_git(&work, &["init", "-q", "-b", "main", "."]);
1158 run_git(&work, &["commit", "-q", "--allow-empty", "-m", "c1"]);
1159 let bare = tmp.path().join("repo.git");
1160 run_git(
1161 tmp.path(),
1162 &[
1163 "clone",
1164 "-q",
1165 "--bare",
1166 work.to_str().unwrap(),
1167 bare.to_str().unwrap(),
1168 ],
1169 );
1170
1171 let consumer = tmp.path().join("consumer");
1172 std::fs::create_dir_all(&consumer).expect("consumer");
1173 run_git(&consumer, &["init", "-q", "-b", "main", "."]);
1174 let local_git = consumer.join(".git");
1175 let local_odb = open_odb(&local_git);
1176 let remote_odb = open_odb(&bare);
1177
1178 let tip_out = Command::new("/usr/bin/git") .args(["rev-parse", "refs/heads/main"])
1180 .current_dir(&bare)
1181 .output()
1182 .expect("rev-parse");
1183 assert!(tip_out.status.success());
1184 let tip = ObjectId::from_hex(String::from_utf8_lossy(&tip_out.stdout).trim()).expect("oid");
1185 let pack =
1186 build_pack(&remote_odb, &[tip], &[], &PackBuildOptions::default()).expect("build_pack");
1187 assert!(
1188 !crate::unpack_objects::pack_is_thin(&pack, remote_odb.hash_algo()),
1189 "whole-object local fetch pack must not be classified as thin"
1190 );
1191 let (_git_dir, _indexed) = git_index_pack(&pack);
1192 install_pack_bytes(
1193 pack,
1194 &local_odb,
1195 &IngestPackOptions {
1196 fix_thin: true,
1197 threads: Some(1),
1198 ..Default::default()
1199 },
1200 )
1201 .expect("ingest like fetch_local serial index-pack");
1202 assert!(local_odb.exists(&tip));
1203 }
1204
1205 #[test]
1206 fn fetch_ingest_keeps_pack_despite_receive_unpacklimit_zero() {
1207 let blob = b"unpacklimit must not apply to fetch ingest\n";
1208 let pack = single_blob_pack(blob);
1209 let tmp = tempfile::tempdir().expect("tempdir");
1210 let git_dir = tmp.path().join(".git");
1211 std::fs::create_dir_all(git_dir.join("objects")).expect("objects");
1212 std::fs::write(git_dir.join("config"), "[receive]\n\tunpackLimit = 0\n").expect("config");
1213 let objects = git_dir.join("objects");
1214 let odb = Odb::new(&objects).with_config_git_dir(git_dir.clone());
1215 install_pack_bytes(
1216 pack,
1217 &odb,
1218 &IngestPackOptions {
1219 fix_thin: false,
1220 ..Default::default()
1221 },
1222 )
1223 .expect("install");
1224 let pack_dir = git_dir.join("objects").join("pack");
1225 let packs: Vec<_> = std::fs::read_dir(&pack_dir)
1226 .expect("pack dir")
1227 .filter_map(|e| e.ok())
1228 .filter(|e| e.path().extension().is_some_and(|x| x == "pack"))
1229 .collect();
1230 assert_eq!(packs.len(), 1, "fetch ingest must index pack, not unpack");
1231 }
1232
1233 #[test]
1234 fn v2_index_large_offset_roundtrip() {
1235 let tmp = tempfile::tempdir().expect("tempdir");
1236 let pack_path = tmp.path().join("large.pack");
1237 let pack = single_blob_pack(b"large offset idx fixture");
1238 std::fs::write(&pack_path, &pack).expect("write pack");
1239 let idx_path = pack_path.with_extension("idx");
1240 let fake_oid = ObjectId::from_hex("aabbccddeeff00112233445566778899aabbccdd").unwrap();
1241 let huge_off = 1u64 << 31;
1242 crate::pack::write_v2_pack_index_from_trailer(
1243 &idx_path,
1244 &[(fake_oid, huge_off, 0x1234_5678)],
1245 &pack[pack.len() - 20..],
1246 20,
1247 None,
1248 )
1249 .expect("write idx");
1250 let idx = read_pack_index(&idx_path).expect("read idx");
1251 assert_eq!(idx.len(), 1);
1252 assert_eq!(idx.offset_at(0).expect("off"), huge_off);
1253 }
1254
1255 fn pack_with_junk_before_trailer(valid: &[u8], junk: &[u8]) -> Vec<u8> {
1256 use crate::objects::HashAlgo;
1257
1258 let algo = HashAlgo::Sha1;
1259 let hb = algo.len();
1260 let body = &valid[..valid.len().checked_sub(hb).expect("pack trailer")];
1261 let mut body_with_junk = body.to_vec();
1262 body_with_junk.extend_from_slice(junk);
1263 let trailer = algo.checksum_digest(&body_with_junk);
1264 body_with_junk.extend_from_slice(&trailer);
1265 body_with_junk
1266 }
1267
1268 fn git_index_pack_strict_rejects(pack: &[u8]) {
1269 let dir = tempfile::tempdir().expect("tempdir");
1270 let pack_path = dir.path().join("in.pack");
1271 std::fs::write(&pack_path, pack).expect("write pack");
1272 let out = Command::new("/usr/bin/git") .current_dir(dir.path())
1274 .args([
1275 "index-pack",
1276 "--strict",
1277 pack_path.to_string_lossy().as_ref(),
1278 ])
1279 .env("GIT_CONFIG_GLOBAL", "/dev/null")
1280 .env("GIT_CONFIG_SYSTEM", "/dev/null")
1281 .output()
1282 .expect("git index-pack --strict");
1283 assert!(
1284 !out.status.success(),
1285 "git index-pack --strict must reject junk before trailer: {}",
1286 String::from_utf8_lossy(&out.stderr)
1287 );
1288 }
1289
1290 #[test]
1291 fn rejects_extra_bytes_before_pack_trailer() {
1292 use crate::hash::Parallelism;
1293 use crate::unpack_objects::pack_index_records;
1294
1295 let valid = single_blob_pack(b"one blob\n");
1296 let corrupted = pack_with_junk_before_trailer(&valid, b"JUNK");
1297 git_index_pack_strict_rejects(&corrupted);
1298
1299 let tmp = tempfile::tempdir().expect("tempdir");
1300 let odb = Odb::new(tmp.path());
1301
1302 for threads in [1_usize, 4] {
1303 let err = pack_index_records(&corrupted, &odb, Parallelism::resolve(Some(threads)))
1304 .expect_err("index-pack hashing");
1305 assert!(
1306 matches!(err, Error::Object(ObjectError::Corrupt { .. })),
1307 "threads={threads}: {err:?}"
1308 );
1309 }
1310
1311 for threads in [1_usize, 4] {
1312 let err = install_pack_bytes(
1313 corrupted.clone(),
1314 &odb,
1315 &IngestPackOptions {
1316 fix_thin: false,
1317 threads: Some(threads),
1318 ..Default::default()
1319 },
1320 )
1321 .expect_err("install pack");
1322 assert!(
1323 matches!(err, Error::Object(ObjectError::Corrupt { .. })),
1324 "install threads={threads}: {err:?}"
1325 );
1326 }
1327 }
1328
1329 fn pack_with_whole_objects_and_ofs_deltas() -> (Vec<u8>, usize, usize) {
1330 use grit_test_support::objects::{HashAlgo as FixtureAlgo, ObjectKind as PK, PackBuilder};
1331
1332 let mut pb = PackBuilder::new(FixtureAlgo::Sha1);
1333 for i in 0..6 {
1334 pb.add_full(PK::Blob, format!("standalone-{i}\n").as_bytes());
1335 }
1336 let mut base_indices = Vec::new();
1337 for i in 0..4 {
1338 let data = format!("delta-base-{i}\n");
1339 base_indices.push(pb.add_full(PK::Blob, data.as_bytes()));
1340 }
1341 for (i, base_idx) in base_indices.into_iter().enumerate() {
1342 let base = format!("delta-base-{i}\n");
1343 let target = format!("delta-base-{i}-extended\n");
1344 let delta = crate::delta_encode::encode_lcp_delta(base.as_bytes(), target.as_bytes())
1345 .expect("delta");
1346 pb.add_ofs_delta(base_idx, &delta, delta.len());
1347 }
1348 let pack = pb.build().bytes;
1349 let nr_objects = 6 + 4 + 4;
1350 let nr_deltas = 4;
1351 (pack, nr_objects, nr_deltas)
1352 }
1353
1354 #[derive(Default)]
1355 struct IndexPackProgressCapture {
1356 active: Option<crate::progress::ProgressPhase>,
1357 phase_totals: Vec<(crate::progress::ProgressPhase, Option<u64>)>,
1358 phase_max: std::collections::HashMap<crate::progress::ProgressPhase, u64>,
1359 set_samples: Vec<u64>,
1360 }
1361
1362 impl crate::progress::ProgressSink for IndexPackProgressCapture {
1363 fn start(&mut self, phase: crate::progress::ProgressPhase, total: Option<u64>) {
1364 self.active = Some(phase);
1365 self.phase_totals.push((phase, total));
1366 }
1367
1368 fn set(&mut self, current: u64) {
1369 self.set_samples.push(current);
1370 if let Some(phase) = self.active {
1371 let entry = self.phase_max.entry(phase).or_insert(0);
1372 *entry = (*entry).max(current);
1373 }
1374 }
1375
1376 fn finish(&mut self) {
1377 self.active = None;
1378 }
1379 }
1380
1381 #[test]
1382 fn progress_reports_object_and_delta_totals() {
1383 use crate::hash::Parallelism;
1384 use crate::unpack_objects::pack_index_records_with_progress;
1385
1386 let (pack, nr_objects, nr_deltas) = pack_with_whole_objects_and_ofs_deltas();
1387 let tmp = tempfile::tempdir().expect("tempdir");
1388 let odb = Odb::new(tmp.path());
1389
1390 for threads in [1_usize, 4] {
1391 let mut cap = IndexPackProgressCapture::default();
1392 let records = pack_index_records_with_progress(
1393 &pack,
1394 &odb,
1395 Parallelism::resolve(Some(threads)),
1396 Some(&mut cap),
1397 )
1398 .unwrap_or_else(|e| panic!("threads={threads}: {e}"));
1399 assert_eq!(records.len(), nr_objects, "threads={threads}");
1400 assert_eq!(
1401 cap.phase_max
1402 .get(&crate::progress::ProgressPhase::IndexingObjects)
1403 .copied(),
1404 Some(nr_objects as u64),
1405 "threads={threads}"
1406 );
1407 assert_eq!(
1408 cap.phase_max
1409 .get(&crate::progress::ProgressPhase::ResolvingDeltas)
1410 .copied(),
1411 Some(nr_deltas as u64),
1412 "threads={threads}"
1413 );
1414 let indexing_total = cap
1415 .phase_totals
1416 .iter()
1417 .find(|(p, _)| *p == crate::progress::ProgressPhase::IndexingObjects)
1418 .and_then(|(_, t)| *t);
1419 assert_eq!(indexing_total, Some(nr_objects as u64), "threads={threads}");
1420 let delta_total = cap
1421 .phase_totals
1422 .iter()
1423 .find(|(p, _)| *p == crate::progress::ProgressPhase::ResolvingDeltas)
1424 .and_then(|(_, t)| *t);
1425 assert_eq!(delta_total, Some(nr_deltas as u64), "threads={threads}");
1426 }
1427 }
1428
1429 #[derive(Default)]
1430 struct MonotonicPhaseCapture {
1431 active: Option<crate::progress::ProgressPhase>,
1432 sets_by_phase: std::collections::HashMap<crate::progress::ProgressPhase, Vec<u64>>,
1433 }
1434
1435 impl crate::progress::ProgressSink for MonotonicPhaseCapture {
1436 fn start(&mut self, phase: crate::progress::ProgressPhase, _total: Option<u64>) {
1437 self.active = Some(phase);
1438 self.sets_by_phase.entry(phase).or_default();
1439 }
1440
1441 fn set(&mut self, current: u64) {
1442 if let Some(phase) = self.active {
1443 self.sets_by_phase.entry(phase).or_default().push(current);
1444 }
1445 }
1446
1447 fn finish(&mut self) {
1448 self.active = None;
1449 }
1450 }
1451
1452 #[test]
1453 fn progress_counts_monotonic() {
1454 use crate::hash::Parallelism;
1455 use crate::unpack_objects::pack_index_records_with_progress;
1456
1457 let (pack, _, _) = pack_with_whole_objects_and_ofs_deltas();
1458 let tmp = tempfile::tempdir().expect("tempdir");
1459 let odb = Odb::new(tmp.path());
1460 let mut cap = MonotonicPhaseCapture::default();
1461 pack_index_records_with_progress(
1462 &pack,
1463 &odb,
1464 Parallelism::resolve(Some(4)),
1465 Some(&mut cap),
1466 )
1467 .expect("index with progress");
1468
1469 for (phase, samples) in &cap.sets_by_phase {
1470 let mut last = 0_u64;
1471 for &cur in samples {
1472 assert!(
1473 cur >= last,
1474 "phase {phase:?} regressed from {last} to {cur}: {samples:?}"
1475 );
1476 last = cur;
1477 }
1478 }
1479 }
1480
1481 fn git_in(dir: &Path, args: &[&str]) -> Vec<u8> {
1482 let out = Command::new("git") .current_dir(dir)
1484 .args(args)
1485 .env("GIT_CONFIG_GLOBAL", "/dev/null")
1486 .env("GIT_CONFIG_SYSTEM", "/dev/null")
1487 .env("GIT_AUTHOR_NAME", "T")
1488 .env("GIT_AUTHOR_EMAIL", "t@example.com")
1489 .env("GIT_COMMITTER_NAME", "T")
1490 .env("GIT_COMMITTER_EMAIL", "t@example.com")
1491 .output()
1492 .expect("git");
1493 assert!(
1494 out.status.success(),
1495 "git {args:?}: {}",
1496 String::from_utf8_lossy(&out.stderr)
1497 );
1498 out.stdout
1499 }
1500
1501 fn commit_pack() -> (Vec<u8>, String) {
1503 use std::io::Write;
1504 let src = tempfile::tempdir().expect("src");
1505 git_in(src.path(), &["init", "-q", "-b", "main"]);
1506 std::fs::write(src.path().join("f"), b"shared pack\n").expect("write");
1507 git_in(src.path(), &["add", "f"]);
1508 git_in(src.path(), &["commit", "-qm", "c"]);
1509 let tip = String::from_utf8(git_in(src.path(), &["rev-parse", "HEAD"])).expect("hex");
1510 let tip = tip.trim().to_owned();
1511 let mut child = Command::new("git") .current_dir(src.path())
1513 .args(["pack-objects", "--revs", "--stdout", "-q"])
1514 .stdin(std::process::Stdio::piped())
1515 .stdout(std::process::Stdio::piped())
1516 .spawn()
1517 .expect("pack-objects");
1518 child
1519 .stdin
1520 .take()
1521 .expect("stdin")
1522 .write_all(format!("{tip}\n").as_bytes())
1523 .expect("revs");
1524 let out = child.wait_with_output().expect("pack-objects output");
1525 assert!(out.status.success());
1526 (out.stdout, tip)
1527 }
1528
1529 fn published_files(pack_dir: &Path, ext: &str) -> usize {
1530 pack_dir_entries(pack_dir)
1531 .iter()
1532 .filter(|p| p.extension().is_some_and(|e| e == ext))
1533 .count()
1534 }
1535
1536 fn assert_repo_fsck_clean(repo: &Path, tip: &str) {
1537 git_in(repo, &["update-ref", "refs/heads/main", tip]);
1538 git_in(repo, &["fsck", "--strict"]);
1539 }
1540
1541 #[cfg_attr(
1542 not(feature = "full-tests"),
1543 ignore = "starts system git or is heavy; run with --features full-tests"
1544 )]
1545 #[test]
1546 fn cancel_once_pack_published_never_strands_a_peer_install() {
1547 use std::sync::atomic::{AtomicBool, Ordering};
1548 use std::sync::Arc;
1549 use std::thread;
1550 use std::time::Duration;
1551
1552 use crate::progress::CancelToken;
1553
1554 let (pack, tip) = commit_pack();
1555 let repo = tempfile::tempdir().expect("repo");
1556 git_in(repo.path(), &["init", "-q", "-b", "main"]);
1557 let objects = repo.path().join(".git/objects");
1558 let pack_dir = objects.join("pack");
1559 let cancel = CancelToken::new();
1560 let peer_ran = Arc::new(AtomicBool::new(false));
1561 let watch_cancel = cancel.clone();
1562 let watch_pack_dir = pack_dir.clone();
1563 let watch_objects = objects.clone();
1564 let watch_pack = pack.clone();
1565 let watch_peer = Arc::clone(&peer_ran);
1566 thread::spawn(move || {
1567 while !watch_cancel.is_cancelled() {
1568 if published_files(&watch_pack_dir, "pack") > 0 {
1569 watch_peer.store(true, Ordering::Relaxed);
1570 let peer_odb = Odb::new(&watch_objects);
1571 install_pack_bytes(watch_pack, &peer_odb, &IngestPackOptions::default())
1572 .expect("peer install succeeds");
1573 watch_cancel.cancel();
1574 return;
1575 }
1576 thread::sleep(Duration::from_millis(1));
1577 }
1578 });
1579 let odb = Odb::new(&objects);
1580 let result = install_pack_bytes_with_progress(
1581 pack.clone(),
1582 &odb,
1583 &IngestPackOptions::default(),
1584 None,
1585 Some(&cancel),
1586 );
1587 match result {
1588 Ok(_) => {}
1589 Err(Error::Cancelled) => assert!(peer_ran.load(Ordering::Relaxed)),
1590 Err(e) => panic!("unexpected install error: {e:?}"),
1591 }
1592 assert_eq!(published_files(&pack_dir, "pack"), 1);
1593 assert_eq!(published_files(&pack_dir, "idx"), 1);
1594 assert_repo_fsck_clean(repo.path(), &tip);
1595 }
1596
1597 #[cfg_attr(
1598 not(feature = "full-tests"),
1599 ignore = "starts system git or is heavy; run with --features full-tests"
1600 )]
1601 #[test]
1602 fn error_after_publish_keeps_published_pack_and_index() {
1603 let (pack, tip) = commit_pack();
1604 let repo = tempfile::tempdir().expect("repo");
1605 git_in(repo.path(), &["init", "-q", "-b", "main"]);
1606 let objects = repo.path().join(".git/objects");
1607 let pack_dir = objects.join("pack");
1608 let trailer = ObjectId::from_bytes(&pack[pack.len() - 20..]).expect("trailer");
1609 std::fs::create_dir_all(pack_dir.join(format!("pack-{}.promisor", trailer.to_hex())))
1612 .expect("promisor obstacle");
1613 let odb = Odb::new(&objects);
1614 let err = install_pack_bytes(
1615 pack,
1616 &odb,
1617 &IngestPackOptions {
1618 mark_promisor_pack: true,
1619 ..Default::default()
1620 },
1621 )
1622 .expect_err("promisor write fails");
1623 assert!(matches!(err, Error::Io(_)), "got {err:?}");
1624 assert_eq!(published_files(&pack_dir, "pack"), 1);
1625 assert_eq!(published_files(&pack_dir, "idx"), 1);
1626 assert!(
1627 pack_dir_entries(&pack_dir).iter().all(|p| {
1628 let name = p.file_name().and_then(|n| n.to_str()).unwrap_or_default();
1629 !name.starts_with('.') && !name.starts_with("tmp_")
1630 }),
1631 "staging left behind"
1632 );
1633 assert_repo_fsck_clean(repo.path(), &tip);
1634 }
1635
1636 #[test]
1637 fn keep_message_is_written_next_to_the_pack() {
1638 let tmp = tempfile::tempdir().expect("tempdir");
1639 let odb = Odb::new(tmp.path());
1640 let ingested = install_pack_bytes(
1641 single_blob_pack(b"kept blob"),
1642 &odb,
1643 &IngestPackOptions {
1644 keep_message: Some("large-object promisor".to_owned()),
1645 mark_promisor_pack: true,
1646 ..Default::default()
1647 },
1648 )
1649 .expect("install");
1650 let pack = ingested.pack_path.expect("pack path");
1651 assert_eq!(
1652 std::fs::read_to_string(pack.with_extension("keep")).expect("keep"),
1653 "large-object promisor\n"
1654 );
1655 assert!(pack.with_extension("promisor").is_file());
1656
1657 let plain = install_pack_bytes(
1659 single_blob_pack(b"plain blob"),
1660 &odb,
1661 &IngestPackOptions::default(),
1662 )
1663 .expect("install")
1664 .pack_path
1665 .expect("pack path");
1666 assert!(!plain.with_extension("keep").exists());
1667 }
1668
1669 #[test]
1670 fn install_failure_leaves_no_published_pack_or_index() {
1671 let tmp = tempfile::tempdir().expect("tempdir");
1672 let odb = Odb::new(tmp.path());
1673 let mut pack = single_blob_pack(b"truncated");
1674 pack.truncate(pack.len().saturating_sub(8));
1675 let err = install_pack_bytes(
1676 pack,
1677 &odb,
1678 &IngestPackOptions {
1679 fix_thin: false,
1680 ..Default::default()
1681 },
1682 )
1683 .expect_err("truncated pack must fail");
1684 assert!(
1685 matches!(err, Error::Object(ObjectError::Corrupt { .. })),
1686 "expected corrupt object, got {err:?}"
1687 );
1688 let pack_dir = tmp.path().join("pack");
1689 if pack_dir.exists() {
1690 let entries = pack_dir_entries(&pack_dir);
1691 assert!(
1692 entries
1693 .iter()
1694 .all(|p| p.extension().is_some_and(|e| e == "pack" || e == "idx")
1695 || p.file_name()
1696 .and_then(|n| n.to_str())
1697 .is_some_and(|n| n.starts_with('.') && n.contains("-install-"))),
1698 "must not publish partial .pack/.idx, got {entries:?}"
1699 );
1700 }
1701 }
1702
1703 #[test]
1704 fn ingest_missing_received_pack_path_is_typed() {
1705 let tmp = tempfile::tempdir().expect("tempdir");
1706 let odb = Odb::new(tmp.path());
1707 let missing = tmp.path().join("no-such-pack");
1708 let err = ingest_received_pack_path_with_progress(
1709 missing.clone(),
1710 &odb,
1711 &IngestPackOptions::default(),
1712 None,
1713 None,
1714 )
1715 .expect_err("missing spool");
1716 assert!(
1717 matches!(
1718 err,
1719 Error::Remote(RemoteError::ReceivedPackTempMissing { .. })
1720 ),
1721 "expected typed missing-pack error, got {err:?}"
1722 );
1723 }
1724
1725 #[cfg(unix)]
1726 fn pack_sidecar_mode(path: &Path) -> u32 {
1727 use std::os::unix::fs::PermissionsExt;
1728 std::fs::metadata(path)
1729 .expect("metadata")
1730 .permissions()
1731 .mode()
1732 & 0o777
1733 }
1734
1735 #[cfg(unix)]
1736 fn large_blob_pack_for_install_test() -> Vec<u8> {
1737 use grit_test_support::objects::{HashAlgo, ObjectKind as PK, PackBuilder};
1738 let mut pb = PackBuilder::new(HashAlgo::Sha1);
1739 for i in 0..120 {
1740 pb.add_full(PK::Blob, format!("install-mode-{i}\n").as_bytes());
1741 }
1742 pb.build().bytes
1743 }
1744
1745 #[cfg(unix)]
1746 #[test]
1747 fn install_pack_finalizes_modes_for_shared_group_and_writes_rev() {
1748 use std::os::unix::fs::OpenOptionsExt;
1749
1750 let tmp = tempfile::tempdir().expect("tempdir");
1751 let git_dir = tmp.path().join("remote.git");
1752 std::fs::create_dir_all(git_dir.join("objects/pack")).expect("pack dir");
1753 std::fs::write(
1754 git_dir.join("config"),
1755 "[core]\n\trepositoryformatversion = 0\n\tsharedRepository = group\n",
1756 )
1757 .expect("config");
1758 let objects = git_dir.join("objects");
1759 let odb = Odb::new(objects.as_path()).with_config_git_dir(git_dir.clone());
1760
1761 let pack_bytes = large_blob_pack_for_install_test();
1762 let spool = git_dir.join("objects/pack/incoming.pack");
1763 {
1764 use std::io::Write;
1765 let mut f = std::fs::OpenOptions::new()
1766 .write(true)
1767 .create_new(true)
1768 .mode(0o600)
1769 .open(&spool)
1770 .expect("0600 spool");
1771 f.write_all(&pack_bytes).expect("write pack");
1772 }
1773 assert_eq!(pack_sidecar_mode(&spool), 0o600);
1774
1775 install_pack_path_with_progress(&spool, &odb, &IngestPackOptions::default(), None, None)
1776 .expect("install");
1777
1778 let pack_dir = git_dir.join("objects/pack");
1779 let pack_path = std::fs::read_dir(&pack_dir)
1780 .expect("read pack dir")
1781 .filter_map(|e| e.ok())
1782 .map(|e| e.path())
1783 .find(|p| p.extension().is_some_and(|x| x == "pack"))
1784 .expect("installed pack");
1785 let idx_path = pack_path.with_extension("idx");
1786 let rev_path = pack_path.with_extension("rev");
1787 assert_eq!(pack_sidecar_mode(&pack_path), 0o444);
1788 assert_eq!(pack_sidecar_mode(&idx_path), 0o444);
1789 assert!(rev_path.is_file(), "expected .rev sidecar");
1790 assert_eq!(pack_sidecar_mode(&rev_path), 0o444);
1791 }
1792
1793 #[cfg(unix)]
1794 #[test]
1795 fn install_pack_finalizes_modes_for_shared_0660_config() {
1796 let tmp = tempfile::tempdir().expect("tempdir");
1797 let git_dir = tmp.path().join("remote.git");
1798 std::fs::create_dir_all(git_dir.join("objects/pack")).expect("pack dir");
1799 std::fs::write(
1800 git_dir.join("config"),
1801 "[core]\n\trepositoryformatversion = 0\n\tsharedRepository = 0660\n",
1802 )
1803 .expect("config");
1804 let objects = git_dir.join("objects");
1805 let odb = Odb::new(objects.as_path()).with_config_git_dir(git_dir.clone());
1806 let pack_bytes = large_blob_pack_for_install_test();
1807 install_pack_bytes(pack_bytes, &odb, &IngestPackOptions::default()).expect("install");
1808 let pack_path = std::fs::read_dir(git_dir.join("objects/pack"))
1809 .expect("read pack dir")
1810 .filter_map(|e| e.ok())
1811 .map(|e| e.path())
1812 .find(|p| p.extension().is_some_and(|x| x == "pack"))
1813 .expect("pack");
1814 assert_eq!(pack_sidecar_mode(&pack_path), 0o440);
1815 assert_eq!(pack_sidecar_mode(&pack_path.with_extension("idx")), 0o440);
1816 }
1817}