1pub mod builder;
25mod commit;
26mod peel;
27mod read_store;
28pub mod store;
29mod stream;
30mod tree;
31
32pub(crate) use stream::deflate_stream_to_writer;
33
34#[doc(inline)]
35pub use builder::OdbBuilder;
36
37pub(crate) use store::loose::{
38 decompress_zlib_loose_bytes, for_each_loose_object_id, parse_object_bytes,
39 read_loose_object_info,
40};
41#[doc(inline)]
42pub use store::LooseStore;
43pub use store::{CompositeStore, FilesSource, ObjectStream};
44#[doc(inline)]
45pub use store::{MemoryStore, ObjectStore, WritableObjectStore};
46
47use crate::git_path::CanonicalizeExt;
48use std::collections::{HashMap, HashSet};
49use std::fs;
50use std::hash::{Hash, Hasher};
51use std::ops::ControlFlow;
52use std::path::{Path, PathBuf};
53#[cfg(test)]
54use std::sync::atomic::AtomicUsize;
55use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
56use std::sync::{Arc, Mutex, OnceLock, RwLock};
57
58use crate::config::ConfigSet;
59use crate::diagnostics::NullDiagnostics;
60use crate::error::{Error, Result};
61use crate::hash;
62use crate::midx::validate_midx_referenced_packs_with_diagnostics;
63use crate::objects::{
64 oid_hash_set, HashAlgo, Object, ObjectId, ObjectInfo, ObjectKind, OidHashMap, OidHashSet,
65};
66use crate::pack;
67use crate::pack_store::PackStore;
68use filetime::{set_file_mtime, FileTime};
69use flate2::Compression;
70
71pub(crate) use store::BoundedRead;
72use store::{LocalSizedRead, PackedObjects};
73
74#[derive(Debug, Clone, Copy, PartialEq, Eq)]
76pub(crate) enum UnreachablePackRetention {
77 KeepAll,
79 KeepGraceProtected(std::time::SystemTime),
81 DropAll,
83}
84
85pub(crate) struct RepackPackInventorySnapshot {
87 pub pack_paths: HashSet<PathBuf>,
88 pub kept_at_inventory: HashSet<PathBuf>,
89}
90
91type MemOdbOverlayStore = Arc<RwLock<Option<Arc<MemoryStore>>>>;
92
93struct AlternateSourcesCache {
95 file_generation: u64,
96 env_fingerprint: u64,
97 submodule_generation: u64,
98 sources: Option<Arc<CompositeStore>>,
99}
100
101struct AlternateSourcesKey {
102 file_generation: u64,
103 env_fingerprint: u64,
104 submodule_generation: u64,
105}
106
107struct FileAlternatesCache {
109 generation: u64,
110 snapshot: Option<Arc<Vec<PathBuf>>>,
111}
112
113#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
115pub struct WriteOptions {
116 pub silent: bool,
118 pub assume_loose_only_existence: bool,
124 pub trust_new_loose: bool,
127 pub loose_prefix_dirs_ready: bool,
130 pub freshen_existing: bool,
140}
141
142pub const BULK_NEW_LOOSE_WRITE: WriteOptions = WriteOptions {
147 assume_loose_only_existence: true,
148 trust_new_loose: true,
149 silent: false,
150 loose_prefix_dirs_ready: true,
151 freshen_existing: false,
152};
153
154pub const BULK_TRACKED_RESTAGE_LOOSE_WRITE: WriteOptions = WriteOptions {
156 assume_loose_only_existence: true,
157 trust_new_loose: false,
158 silent: false,
159 loose_prefix_dirs_ready: true,
160 freshen_existing: false,
161};
162
163fn exists_materialized_in_objects_dir(objects_dir: &Path, oid: &ObjectId) -> bool {
169 store::local_object_materialized(objects_dir, oid)
170}
171
172fn filter_oids(
174 oids: impl IntoIterator<Item = ObjectId>,
175 mut keep: impl FnMut(&ObjectId) -> Result<bool>,
176) -> Result<Vec<ObjectId>> {
177 oids.into_iter()
178 .filter_map(|oid| keep(&oid).map(|k| k.then_some(oid)).transpose())
179 .collect()
180}
181
182fn objects_dir_has_pack_index_files(objects_dir: &Path) -> bool {
183 let pack_dir = objects_dir.join("pack");
184 let Ok(entries) = fs::read_dir(pack_dir) else {
185 return false;
186 };
187 entries
188 .filter_map(|e| e.ok())
189 .any(|ent| ent.path().extension().is_some_and(|ext| ext == "idx"))
190}
191
192#[derive(Clone)]
212pub struct Odb {
213 objects_dir: PathBuf,
214 work_tree: Option<PathBuf>,
216 submodule_alternate_dirs: Arc<Mutex<Vec<PathBuf>>>,
218 config_git_dir: Option<PathBuf>,
220 core_multi_pack_index_cache: Arc<OnceLock<bool>>,
226 local_pack_indexes_seen: Arc<OnceLock<bool>>,
228 file_alternate_dirs_cache: Arc<RwLock<FileAlternatesCache>>,
230 #[cfg(test)]
231 exists_probe: Arc<AtomicUsize>,
232 #[cfg(test)]
233 pub(crate) hot_path_test_metrics: Arc<crate::hot_path_test_metrics::HotPathTestMetrics>,
234 env_alternate_dirs: Vec<PathBuf>,
236 env_alternate_lazy: Arc<OnceLock<Arc<Vec<PathBuf>>>>,
238 mem_overlay: MemOdbOverlayStore,
243 mem_overlay_active: Arc<AtomicBool>,
245 primary_source: Arc<OnceLock<Arc<FilesSource>>>,
247 alternate_sources_cache: Arc<RwLock<AlternateSourcesCache>>,
249 midx_packs_validated: Arc<OnceLock<()>>,
251 hash_algo_cache: Arc<OnceLock<HashAlgo>>,
255 loose_zlib_cache: Arc<OnceLock<Compression>>,
257 pack_zlib_cache: Arc<OnceLock<Compression>>,
259 big_file_threshold_cache: Arc<OnceLock<u64>>,
261 shared_config_state: Option<crate::repo::RepositoryConfigSnapshot>,
263 delta_base_cache_sync: Arc<OnceLock<()>>,
265 shared_environment: Option<Arc<crate::environment::Environment>>,
267 empty_config_env: Arc<OnceLock<crate::environment::Environment>>,
269 freshened_packs: Arc<Mutex<HashSet<PathBuf>>>,
272 pack_store: Arc<PackStore>,
274 alternate_pack_stores: Arc<Mutex<HashMap<PathBuf, Arc<PackStore>>>>,
276 submodule_sources_generation: Arc<AtomicU64>,
278 custom_primary: Option<Arc<dyn WritableObjectStore>>,
280 extra_read_sources: Vec<Arc<dyn ObjectStore>>,
282 alternates_enabled: bool,
284 attached_commit_graph: Option<Arc<crate::commit_graph_file::CommitGraphChain>>,
286 session_loose_reads: Arc<std::sync::Mutex<SessionLooseReads>>,
288 session_tree_entries: Arc<Mutex<SessionTreeEntries>>,
290}
291
292const SESSION_BOUNDED_LOOSE_CACHE_MAX: usize = 64 * 1024;
296
297const SESSION_READ_BLOB_CACHE_MAX: usize = 4 * 1024;
301
302const SESSION_LOOSE_CACHE_MAX_ENTRIES: usize = 256;
304
305const SESSION_LOOSE_CACHE_MAX_BYTES: usize = 16 * 1024 * 1024;
308
309#[derive(Default)]
312struct SessionLooseReads {
313 objects: OidHashMap<Object>,
314 bytes: usize,
315}
316
317impl SessionLooseReads {
318 fn get(&self, oid: &ObjectId) -> Option<&Object> {
319 self.objects.get(oid)
320 }
321
322 fn insert(&mut self, oid: ObjectId, obj: &Object) {
323 let len = obj.data.len();
324 if len > SESSION_LOOSE_CACHE_MAX_BYTES {
325 return;
326 }
327 if self.objects.len() > SESSION_LOOSE_CACHE_MAX_ENTRIES
328 || self.bytes.saturating_add(len) > SESSION_LOOSE_CACHE_MAX_BYTES
329 {
330 self.objects.clear();
331 self.bytes = 0;
332 }
333 if let Some(old) = self.objects.insert(oid, obj.clone()) {
334 self.bytes = self.bytes.saturating_sub(old.data.len());
335 }
336 self.bytes += len;
337 }
338
339 #[cfg(test)]
340 fn is_empty(&self) -> bool {
341 self.objects.is_empty()
342 }
343}
344
345#[derive(Clone)]
348pub(crate) enum SessionTree {
349 Parsed(Arc<Vec<crate::objects::TreeEntry>>),
351 Raw(Arc<Vec<u8>>),
353}
354
355impl SessionTree {
356 const PARSED_ENTRY_BYTES: usize = 80;
358
359 fn cost(&self) -> usize {
360 match self {
361 Self::Parsed(entries) => entries.len().saturating_mul(Self::PARSED_ENTRY_BYTES),
362 Self::Raw(data) => data.len(),
363 }
364 }
365}
366
367#[derive(Default)]
370struct SessionTreeEntries {
371 trees: OidHashMap<SessionTree>,
372 bytes: usize,
373}
374
375impl SessionTreeEntries {
376 const BYTE_BUDGET: usize = (1 << 16) * SessionTree::PARSED_ENTRY_BYTES;
380
381 fn insert(&mut self, oid: ObjectId, tree: SessionTree) {
382 let cost = tree.cost();
383 if cost > Self::BYTE_BUDGET {
384 return;
385 }
386 if let Some(old) = self.trees.remove(&oid) {
387 self.bytes -= old.cost();
388 }
389 if self.bytes + cost > Self::BYTE_BUDGET {
390 self.trees.clear();
391 self.bytes = 0;
392 }
393 self.bytes += cost;
394 self.trees.insert(oid, tree);
395 }
396}
397
398#[cfg(test)]
399mod session_tree_entries_tests {
400 use super::*;
401 use crate::objects::TreeEntry;
402
403 fn tree(entries: usize) -> SessionTree {
404 let entry = TreeEntry {
405 mode: 0o100644,
406 name: b"f".to_vec(),
407 oid: ObjectId::zero(),
408 };
409 SessionTree::Parsed(Arc::new(vec![entry; entries]))
410 }
411
412 fn raw(bytes: usize) -> SessionTree {
413 SessionTree::Raw(Arc::new(vec![0u8; bytes]))
414 }
415
416 fn oid(n: u8) -> ObjectId {
417 ObjectId::from_bytes(&[n; 20]).expect("oid")
418 }
419
420 const ENTRY_BUDGET: usize = SessionTreeEntries::BYTE_BUDGET / SessionTree::PARSED_ENTRY_BYTES;
421
422 #[test]
423 fn cache_starts_over_when_the_budget_is_exceeded() {
424 let mut cache = SessionTreeEntries::default();
425 let half = ENTRY_BUDGET / 2;
426 cache.insert(oid(1), tree(half));
427 cache.insert(oid(2), tree(half));
428 assert_eq!(cache.bytes, SessionTreeEntries::BYTE_BUDGET);
429 assert_eq!(cache.trees.len(), 2);
430
431 cache.insert(oid(3), tree(1));
432 assert_eq!(cache.trees.len(), 1);
433 assert!(cache.trees.contains_key(&oid(3)));
434 assert_eq!(cache.bytes, SessionTree::PARSED_ENTRY_BYTES);
435
436 cache.insert(oid(4), tree(ENTRY_BUDGET + 1));
437 assert!(
438 !cache.trees.contains_key(&oid(4)),
439 "an oversized tree is not cached"
440 );
441 assert!(cache.trees.contains_key(&oid(3)));
442 assert!(cache.bytes <= SessionTreeEntries::BYTE_BUDGET);
443 }
444
445 #[test]
446 fn raw_trees_count_their_payload_bytes() {
447 let mut cache = SessionTreeEntries::default();
448 cache.insert(oid(1), raw(1000));
449 cache.insert(oid(2), tree(10));
450 assert_eq!(cache.bytes, 1000 + 10 * SessionTree::PARSED_ENTRY_BYTES);
451
452 cache.insert(oid(3), raw(SessionTreeEntries::BYTE_BUDGET + 1));
453 assert!(
454 !cache.trees.contains_key(&oid(3)),
455 "an oversized payload is not cached"
456 );
457 assert_eq!(cache.trees.len(), 2);
458 }
459
460 #[test]
461 fn replacing_a_tree_releases_its_old_cost() {
462 let mut cache = SessionTreeEntries::default();
463 cache.insert(oid(1), raw(4000));
464 cache.insert(oid(1), tree(2));
465 assert_eq!(cache.trees.len(), 1);
466 assert_eq!(cache.bytes, 2 * SessionTree::PARSED_ENTRY_BYTES);
467 assert!(matches!(
468 cache.trees.get(&oid(1)),
469 Some(SessionTree::Parsed(_))
470 ));
471 }
472}
473
474const _: () = {
475 fn assert_send_sync<T: Send + Sync>() {}
476 let _ = assert_send_sync::<Odb>;
477};
478
479impl std::fmt::Debug for Odb {
480 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
481 f.debug_struct("Odb")
482 .field("objects_dir", &self.objects_dir)
483 .field("work_tree", &self.work_tree)
484 .field("submodule_alternate_dirs", &"<mutex>")
485 .field("config_git_dir", &self.config_git_dir)
486 .finish()
487 }
488}
489
490#[must_use]
495pub fn hash_algo_for_git_dir(env: &crate::environment::Environment, git_dir: &Path) -> HashAlgo {
496 ConfigSet::load(env, Some(git_dir), true)
497 .ok()
498 .and_then(|cfg| cfg.get("extensions.objectformat"))
499 .and_then(|v| HashAlgo::from_name(&v))
500 .unwrap_or(HashAlgo::Sha1)
501}
502
503#[must_use]
505pub fn hash_algo_for_objects_dir(
506 env: &crate::environment::Environment,
507 objects_dir: &Path,
508) -> HashAlgo {
509 objects_dir
510 .parent()
511 .map(|d| hash_algo_for_git_dir(env, d))
512 .unwrap_or(HashAlgo::Sha1)
513}
514
515impl Odb {
516 #[must_use]
523 pub fn new(objects_dir: &Path) -> Self {
524 Self {
525 objects_dir: objects_dir.to_path_buf(),
526 work_tree: None,
527 submodule_alternate_dirs: Arc::new(Mutex::new(Vec::new())),
528 config_git_dir: None,
529 core_multi_pack_index_cache: Arc::new(OnceLock::new()),
530 local_pack_indexes_seen: Arc::new(OnceLock::new()),
531 file_alternate_dirs_cache: Arc::new(RwLock::new(FileAlternatesCache {
532 generation: 0,
533 snapshot: None,
534 })),
535 env_alternate_dirs: Vec::new(),
536 env_alternate_lazy: Arc::new(OnceLock::new()),
537 mem_overlay: Arc::new(RwLock::new(None)),
538 mem_overlay_active: Arc::new(AtomicBool::new(false)),
539 primary_source: Arc::new(OnceLock::new()),
540 alternate_sources_cache: Arc::new(RwLock::new(AlternateSourcesCache {
541 file_generation: 0,
542 env_fingerprint: 0,
543 submodule_generation: 0,
544 sources: None,
545 })),
546 midx_packs_validated: Arc::new(OnceLock::new()),
547 hash_algo_cache: Arc::new(OnceLock::new()),
548 loose_zlib_cache: Arc::new(OnceLock::new()),
549 pack_zlib_cache: Arc::new(OnceLock::new()),
550 big_file_threshold_cache: Arc::new(OnceLock::new()),
551 shared_config_state: None,
552 delta_base_cache_sync: Arc::new(OnceLock::new()),
553 shared_environment: None,
554 empty_config_env: Arc::new(OnceLock::new()),
555 freshened_packs: Arc::new(Mutex::new(HashSet::new())),
556 pack_store: Arc::new(PackStore::new(objects_dir.to_path_buf())),
557 alternate_pack_stores: Arc::new(Mutex::new(HashMap::new())),
558 submodule_sources_generation: Arc::new(AtomicU64::new(0)),
559 custom_primary: None,
560 extra_read_sources: Vec::new(),
561 alternates_enabled: true,
562 attached_commit_graph: None,
563 session_loose_reads: Arc::new(Mutex::new(SessionLooseReads::default())),
564 session_tree_entries: Arc::new(Mutex::new(SessionTreeEntries::default())),
565 #[cfg(test)]
566 exists_probe: Arc::new(AtomicUsize::new(0)),
567 #[cfg(test)]
568 hot_path_test_metrics: crate::hot_path_test_metrics::HotPathTestMetrics::new(),
569 }
570 }
571
572 #[must_use]
577 pub fn with_work_tree(objects_dir: &Path, work_tree: &Path) -> Self {
578 Self {
579 objects_dir: objects_dir.to_path_buf(),
580 work_tree: Some(work_tree.to_path_buf()),
581 submodule_alternate_dirs: Arc::new(Mutex::new(Vec::new())),
582 config_git_dir: None,
583 core_multi_pack_index_cache: Arc::new(OnceLock::new()),
584 local_pack_indexes_seen: Arc::new(OnceLock::new()),
585 file_alternate_dirs_cache: Arc::new(RwLock::new(FileAlternatesCache {
586 generation: 0,
587 snapshot: None,
588 })),
589 env_alternate_dirs: Vec::new(),
590 env_alternate_lazy: Arc::new(OnceLock::new()),
591 mem_overlay: Arc::new(RwLock::new(None)),
592 mem_overlay_active: Arc::new(AtomicBool::new(false)),
593 primary_source: Arc::new(OnceLock::new()),
594 alternate_sources_cache: Arc::new(RwLock::new(AlternateSourcesCache {
595 file_generation: 0,
596 env_fingerprint: 0,
597 submodule_generation: 0,
598 sources: None,
599 })),
600 midx_packs_validated: Arc::new(OnceLock::new()),
601 hash_algo_cache: Arc::new(OnceLock::new()),
602 loose_zlib_cache: Arc::new(OnceLock::new()),
603 pack_zlib_cache: Arc::new(OnceLock::new()),
604 big_file_threshold_cache: Arc::new(OnceLock::new()),
605 shared_config_state: None,
606 delta_base_cache_sync: Arc::new(OnceLock::new()),
607 shared_environment: None,
608 empty_config_env: Arc::new(OnceLock::new()),
609 freshened_packs: Arc::new(Mutex::new(HashSet::new())),
610 pack_store: Arc::new(PackStore::new(objects_dir.to_path_buf())),
611 alternate_pack_stores: Arc::new(Mutex::new(HashMap::new())),
612 submodule_sources_generation: Arc::new(AtomicU64::new(0)),
613 custom_primary: None,
614 extra_read_sources: Vec::new(),
615 alternates_enabled: true,
616 attached_commit_graph: None,
617 session_loose_reads: Arc::new(Mutex::new(SessionLooseReads::default())),
618 session_tree_entries: Arc::new(Mutex::new(SessionTreeEntries::default())),
619 #[cfg(test)]
620 exists_probe: Arc::new(AtomicUsize::new(0)),
621 #[cfg(test)]
622 hot_path_test_metrics: crate::hot_path_test_metrics::HotPathTestMetrics::new(),
623 }
624 }
625
626 fn session_tree(&self, oid: &ObjectId) -> Option<SessionTree> {
627 self.session_tree_entries
628 .lock()
629 .ok()?
630 .trees
631 .get(oid)
632 .cloned()
633 }
634
635 fn remember_session_tree(&self, oid: &ObjectId, tree: SessionTree) {
636 if let Ok(mut guard) = self.session_tree_entries.lock() {
637 guard.insert(*oid, tree);
638 }
639 }
640
641 fn read_tree_payload(&self, oid: &ObjectId) -> Result<Vec<u8>> {
642 let obj = self.read_impl(oid, false)?;
643 if obj.kind != ObjectKind::Tree {
644 return Err(Error::CorruptObject(format!(
645 "expected tree, got {}",
646 obj.kind.as_str()
647 )));
648 }
649 Ok(obj.data)
650 }
651
652 pub(crate) fn cached_tree_entries(
654 &self,
655 oid: &ObjectId,
656 ) -> Result<Arc<Vec<crate::objects::TreeEntry>>> {
657 let parsed = match self.session_tree(oid) {
658 Some(SessionTree::Parsed(entries)) => return Ok(entries),
659 Some(SessionTree::Raw(data)) => {
660 crate::objects::parse_tree_with_oid_len(&data, self.hash_algo().len())?
661 }
662 None => crate::objects::parse_tree_with_oid_len(
663 &self.read_tree_payload(oid)?,
664 self.hash_algo().len(),
665 )?,
666 };
667 let arc = Arc::new(parsed);
668 self.remember_session_tree(oid, SessionTree::Parsed(Arc::clone(&arc)));
669 Ok(arc)
670 }
671
672 pub(crate) fn cached_tree_for_walk(&self, oid: &ObjectId) -> Result<SessionTree> {
677 if let Some(tree) = self.session_tree(oid) {
678 return Ok(tree);
679 }
680 let tree = SessionTree::Raw(Arc::new(self.read_tree_payload(oid)?));
681 self.remember_session_tree(oid, tree.clone());
682 Ok(tree)
683 }
684
685 fn session_cached_loose_read(&self, oid: &ObjectId) -> Option<Object> {
686 self.session_loose_reads.lock().ok()?.get(oid).cloned()
687 }
688
689 #[cfg(test)]
691 pub(crate) fn session_read_is_cached(&self, oid: &ObjectId) -> bool {
692 self.session_cached_loose_read(oid).is_some()
693 }
694
695 fn remember_session_loose_read(&self, oid: &ObjectId, obj: &Object) {
698 if let Ok(mut guard) = self.session_loose_reads.lock() {
699 guard.insert(*oid, obj);
700 }
701 }
702
703 fn finish_session_loose_read(
711 &self,
712 oid: &ObjectId,
713 obj: Object,
714 remember: bool,
715 ) -> Result<Object> {
716 if remember
717 && (obj.kind != ObjectKind::Blob || obj.data.len() <= SESSION_READ_BLOB_CACHE_MAX)
718 {
719 self.remember_session_loose_read(oid, &obj);
720 }
721 Ok(obj)
722 }
723
724 #[cfg(test)]
726 #[must_use]
727 pub(crate) fn hot_path_metrics(&self) -> &crate::hot_path_test_metrics::HotPathTestMetrics {
728 &self.hot_path_test_metrics
729 }
730
731 #[cfg(test)]
732 #[must_use]
733 pub(crate) fn hot_path_metrics_arc(
734 &self,
735 ) -> Arc<crate::hot_path_test_metrics::HotPathTestMetrics> {
736 Arc::clone(&self.hot_path_test_metrics)
737 }
738
739 #[must_use]
741 pub(crate) fn with_shared_config_state(
742 mut self,
743 state: crate::repo::RepositoryConfigSnapshot,
744 env: Arc<crate::environment::Environment>,
745 ) -> Self {
746 self.pack_store.set_repo_caches(Arc::clone(&state));
747 self.shared_config_state = Some(state);
748 self.shared_environment = Some(env);
749 self
750 }
751
752 fn diagnostics_handle(&self) -> crate::diagnostics::DiagnosticsHandle {
753 self.shared_config_state
754 .as_ref()
755 .map(|state| state.diagnostics_handle())
756 .unwrap_or_else(|| Arc::new(NullDiagnostics))
757 }
758
759 pub(crate) fn load_config_cascade(&self) -> Result<Arc<ConfigSet>> {
765 let git_dir = self
766 .config_git_dir
767 .as_deref()
768 .or_else(|| self.objects_dir.parent());
769 if let (Some(caches), Some(env)) = (&self.shared_config_state, &self.shared_environment) {
770 let opts = crate::config::LoadConfigOptions::for_repository(
771 env,
772 git_dir,
773 Some(Arc::clone(caches)),
774 );
775 return caches.load_config_with_options(env.as_ref(), git_dir, &opts);
776 }
777 match git_dir {
778 Some(git_dir) => {
779 ConfigSet::load(self.config_environment(), Some(git_dir), true).map(Arc::new)
780 }
781 None => Ok(Arc::new(ConfigSet::new())),
782 }
783 }
784
785 fn config_environment(&self) -> &crate::environment::Environment {
786 if let Some(env) = self.shared_environment.as_deref() {
787 return env;
788 }
789 self.empty_config_env
790 .get_or_init(crate::environment::Environment::empty)
791 }
792
793 fn env_alternate_dirs_snapshot(&self) -> Arc<Vec<PathBuf>> {
794 if !self.env_alternate_dirs.is_empty() {
795 return Arc::new(self.env_alternate_dirs.clone());
796 }
797 Arc::clone(self.env_alternate_lazy.get_or_init(|| {
798 Arc::new(Self::env_alternate_dirs_from_var(
799 self.config_environment(),
800 self.work_tree.as_deref(),
801 ))
802 }))
803 }
804
805 #[must_use]
809 pub fn env_alternate_dirs_from_var(
810 env: &crate::environment::Environment,
811 resolve_base: Option<&Path>,
812 ) -> Vec<PathBuf> {
813 match env.var("GIT_ALTERNATE_OBJECT_DIRECTORIES") {
814 Some(val) if !val.is_empty() => {
815 let mut dirs = parse_alternate_env(&val);
816 if let Some(base) = resolve_base {
817 for dir in &mut dirs {
818 if dir.is_relative() {
819 *dir = base.join(&dir);
820 }
821 }
822 }
823 dirs
824 }
825 _ => Vec::new(),
826 }
827 }
828
829 #[must_use]
831 pub fn with_env_alternate_dirs(mut self, dirs: Vec<PathBuf>) -> Self {
832 self.env_alternate_dirs = dirs;
833 self.clear_alternate_sources_cache();
834 self
835 }
836
837 fn clear_alternate_sources_cache(&self) {
838 if let Ok(mut guard) = self.alternate_sources_cache.write() {
839 guard.sources = None;
840 }
841 }
842
843 fn alternate_sources_key(&self) -> AlternateSourcesKey {
844 AlternateSourcesKey {
845 file_generation: self
846 .file_alternate_dirs_cache
847 .read()
848 .map(|g| g.generation)
849 .unwrap_or(0),
850 env_fingerprint: self.env_alternate_fingerprint(),
851 submodule_generation: self.submodule_sources_generation.load(Ordering::Relaxed),
852 }
853 }
854
855 fn env_alternate_fingerprint(&self) -> u64 {
856 let mut hasher = std::collections::hash_map::DefaultHasher::new();
857 for path in self.env_alternate_dirs_snapshot().iter() {
858 path.hash(&mut hasher);
859 }
860 hasher.finish()
861 }
862
863 pub fn refresh_file_alternates_from_disk(&self) -> Result<()> {
869 let snapshot = Arc::new(pack::read_alternates_recursive(&self.objects_dir)?);
870 if let Ok(mut guard) = self.file_alternate_dirs_cache.write() {
871 guard.generation = guard.generation.wrapping_add(1);
872 guard.snapshot = Some(snapshot);
873 }
874 self.clear_alternate_sources_cache();
875 Ok(())
876 }
877
878 pub fn append_file_alternate(&self, alternate_objects_dir: &Path) -> Result<()> {
884 Self::append_alternate_objects_line(&self.objects_dir, alternate_objects_dir)?;
885 self.refresh_file_alternates_from_disk()?;
886 Ok(())
887 }
888
889 pub fn append_alternate_objects_line(
894 objects_dir: &Path,
895 alternate_objects_dir: &Path,
896 ) -> Result<()> {
897 let info = objects_dir.join("info");
898 fs::create_dir_all(&info)?;
899 let alt_path = info.join("alternates");
900 let alt_abs = alternate_objects_dir
901 .canonicalize_simplified()
902 .unwrap_or_else(|_| alternate_objects_dir.to_path_buf());
903 let line = format!("{}\n", alt_abs.display());
904 let existing = match fs::read_to_string(&alt_path) {
905 Ok(s) => s,
906 Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(),
907 Err(e) => return Err(Error::Io(e)),
908 };
909 if existing
910 .lines()
911 .any(|l| l.trim() == alt_abs.to_string_lossy())
912 {
913 return Ok(());
914 }
915 let mut out = existing;
916 out.push_str(&line);
917 fs::write(alt_path, out).map_err(Error::Io)
918 }
919
920 fn file_alternate_dirs_snapshot(&self) -> Arc<Vec<PathBuf>> {
921 loop {
922 let observed_gen = self
923 .file_alternate_dirs_cache
924 .read()
925 .map(|g| g.generation)
926 .unwrap_or(0);
927 if let Ok(guard) = self.file_alternate_dirs_cache.read() {
928 if guard.generation == observed_gen {
929 if let Some(snapshot) = &guard.snapshot {
930 return Arc::clone(snapshot);
931 }
932 }
933 }
934 if let Ok(mut guard) = self.file_alternate_dirs_cache.write() {
935 if guard.generation != observed_gen {
936 continue;
937 }
938 if let Some(snapshot) = &guard.snapshot {
939 return Arc::clone(snapshot);
940 }
941 let snapshot = Arc::new(
942 pack::read_alternates_recursive(&self.objects_dir).unwrap_or_default(),
943 );
944 guard.snapshot = Some(Arc::clone(&snapshot));
945 return snapshot;
946 }
947 break;
948 }
949 Arc::new(Vec::new())
950 }
951
952 #[cfg(test)]
953 pub(crate) fn exists_probe_count(&self) -> usize {
954 self.exists_probe.load(Ordering::Relaxed)
955 }
956
957 #[cfg(test)]
958 pub(crate) fn reset_exists_probe(&self) {
959 self.exists_probe.store(0, Ordering::Relaxed);
960 }
961
962 pub fn enable_mem_overlay(&self) {
967 self.mem_overlay_active.store(true, Ordering::Release);
968 if let Ok(mut guard) = self.mem_overlay.write() {
969 if guard.is_none() {
970 *guard = Some(Arc::new(MemoryStore::new(self.hash_algo())));
971 }
972 }
973 }
974
975 pub fn disable_mem_overlay(&self) {
977 self.mem_overlay_active.store(false, Ordering::Release);
978 if let Ok(mut guard) = self.mem_overlay.write() {
979 *guard = None;
980 }
981 }
982
983 #[must_use]
985 pub(crate) fn is_mem_overlay_active(&self) -> bool {
986 self.mem_overlay_active.load(Ordering::Acquire)
987 }
988
989 fn overlay_active(&self) -> bool {
990 self.mem_overlay
991 .read()
992 .ok()
993 .and_then(|g| g.as_ref().map(|_| ()))
994 .is_some()
995 }
996
997 fn overlay_store(&self) -> Option<Arc<MemoryStore>> {
998 if !self.mem_overlay_active.load(Ordering::Acquire) {
999 return None;
1000 }
1001 self.mem_overlay.read().ok().and_then(|g| g.clone())
1002 }
1003
1004 #[cfg(test)]
1006 pub(crate) fn mem_overlay_len_for_tests(&self) -> Option<usize> {
1007 let store = self.overlay_store()?;
1008 let mut count = 0usize;
1009 store
1010 .for_each_object(&mut |_| {
1011 count += 1;
1012 ControlFlow::Continue(())
1013 })
1014 .ok()?;
1015 Some(count)
1016 }
1017
1018 fn overlay_store_object(&self, oid: ObjectId, kind: ObjectKind, data: &[u8]) -> bool {
1021 let Some(store) = self.overlay_store() else {
1022 return false;
1023 };
1024 store.insert_object(oid, kind, Arc::from(data.to_owned()));
1025 true
1026 }
1027
1028 fn overlay_read(&self, oid: &ObjectId) -> Option<Object> {
1030 let store = self.overlay_store()?;
1031 store.read(oid).ok().flatten()
1032 }
1033
1034 pub fn primary(&self) -> Result<Arc<FilesSource>> {
1046 self.require_files_primary("primary")?;
1047 if let Some(src) = self.primary_source.get() {
1048 return Ok(Arc::clone(src));
1049 }
1050 let src = self.build_files_source(
1051 &self.objects_dir,
1052 self.multi_pack_index_reads_enabled(),
1053 self.midx_tip_listing_enabled(),
1054 )?;
1055 match self.primary_source.set(Arc::clone(&src)) {
1056 Ok(()) => Ok(src),
1057 Err(existing) => Ok(existing),
1058 }
1059 }
1060
1061 #[must_use]
1066 pub(crate) fn has_secondary_object_stores(&self) -> bool {
1067 if !self.extra_read_sources.is_empty() {
1068 return true;
1069 }
1070 self.alternates_enabled && self.alternate_object_lookup_needed()
1071 }
1072
1073 #[must_use]
1075 fn alternate_object_lookup_needed(&self) -> bool {
1076 if !self.file_alternate_dirs_snapshot().is_empty() {
1077 return true;
1078 }
1079 if !self.env_alternate_dirs_snapshot().is_empty() {
1080 return true;
1081 }
1082 self.submodule_alternate_dirs
1083 .lock()
1084 .ok()
1085 .is_some_and(|guard| !guard.is_empty())
1086 }
1087
1088 pub fn sources(&self) -> Result<Arc<CompositeStore>> {
1098 if !self.alternates_enabled {
1099 return Ok(Arc::new(CompositeStore::new(Vec::new(), self.hash_algo())));
1100 }
1101 let key = self.alternate_sources_key();
1102 if let Ok(guard) = self.alternate_sources_cache.read() {
1103 if guard.file_generation == key.file_generation
1104 && guard.env_fingerprint == key.env_fingerprint
1105 && guard.submodule_generation == key.submodule_generation
1106 {
1107 if let Some(sources) = guard.sources.as_ref() {
1108 return Ok(Arc::clone(sources));
1109 }
1110 }
1111 }
1112 let use_midx = self.multi_pack_index_reads_enabled();
1113 let midx_tip = self.midx_tip_listing_enabled();
1114 let mut stores: Vec<Arc<dyn ObjectStore>> = Vec::new();
1115 for alt_dir in self.file_alternate_dirs_snapshot().iter() {
1116 stores.push(self.build_files_source(alt_dir, use_midx, midx_tip)?);
1117 }
1118 for alt_dir in self.env_alternate_dirs_snapshot().iter() {
1119 stores.push(self.build_files_source(alt_dir, use_midx, midx_tip)?);
1120 }
1121 if let Ok(guard) = self.submodule_alternate_dirs.lock() {
1122 for alt_dir in guard.iter() {
1123 stores.push(self.build_files_source(alt_dir, false, false)?);
1124 }
1125 }
1126 let composite = Arc::new(CompositeStore::new(stores, self.hash_algo()));
1127 if let Ok(mut guard) = self.alternate_sources_cache.write() {
1128 guard.file_generation = key.file_generation;
1129 guard.env_fingerprint = key.env_fingerprint;
1130 guard.submodule_generation = key.submodule_generation;
1131 guard.sources = Some(Arc::clone(&composite));
1132 }
1133 Ok(composite)
1134 }
1135
1136 fn midx_tip_listing_enabled(&self) -> bool {
1137 self.config_git_dir.is_some() && self.core_multi_pack_index_enabled()
1138 }
1139
1140 fn build_files_source(
1141 &self,
1142 objects_dir: &Path,
1143 use_midx: bool,
1144 midx_tip_listing_for_exists: bool,
1145 ) -> Result<Arc<FilesSource>> {
1146 let loose = self.loose_at(objects_dir)?;
1147 let pack_store = self.pack_store_for(objects_dir);
1148 let diagnostics = if objects_dir == self.objects_dir.as_path() {
1149 self.diagnostics_handle()
1150 } else {
1151 Arc::new(NullDiagnostics)
1152 };
1153 FilesSource::open(
1154 pack_store,
1155 loose,
1156 use_midx,
1157 midx_tip_listing_for_exists,
1158 diagnostics,
1159 )
1160 .map(Arc::new)
1161 }
1162
1163 pub fn register_submodule_object_directories_from_index(
1167 &self,
1168 work_tree: &Path,
1169 index: &crate::index::Index,
1170 ) {
1171 use crate::diff::submodule_embedded_git_dir;
1172
1173 let Ok(mut dirs) = self.submodule_alternate_dirs.lock() else {
1174 return;
1175 };
1176 dirs.clear();
1177 for e in &index.entries {
1178 if e.stage() != 0 || e.mode != crate::index::MODE_GITLINK {
1179 continue;
1180 }
1181 let path_str = String::from_utf8_lossy(&e.path);
1182 let abs = work_tree.join(path_str.as_ref());
1183 let Some(sub_git) = submodule_embedded_git_dir(&abs) else {
1184 continue;
1185 };
1186 let objects = sub_git.join("objects");
1187 if !objects.is_dir() {
1188 continue;
1189 }
1190 let canon = objects.canonicalize_simplified().unwrap_or(objects);
1191 if !dirs.iter().any(|p| p == &canon) {
1192 dirs.push(canon);
1193 }
1194 }
1195 self.submodule_sources_generation
1196 .fetch_add(1, Ordering::Relaxed);
1197 self.clear_alternate_sources_cache();
1198 }
1199
1200 #[must_use]
1202 pub fn with_config_git_dir(mut self, git_dir: PathBuf) -> Self {
1203 self.config_git_dir = Some(git_dir);
1204 self
1205 }
1206
1207 #[must_use]
1219 pub fn hash_algo(&self) -> HashAlgo {
1220 if let Some(primary) = &self.custom_primary {
1221 return primary.hash_algo();
1222 }
1223 *self.hash_algo_cache.get_or_init(|| {
1224 if self.config_git_dir.is_none() && self.objects_dir.parent().is_none() {
1225 return HashAlgo::Sha1;
1226 }
1227 let cfg = self.load_config_cascade().unwrap_or_default();
1228 cfg.get("extensions.objectformat")
1229 .and_then(|v| HashAlgo::from_name(&v))
1230 .unwrap_or(HashAlgo::Sha1)
1231 })
1232 }
1233
1234 fn resolve_loose_zlib_level(&self) -> Result<u32> {
1235 if self.config_git_dir.is_none() && self.objects_dir.parent().is_none() {
1236 return Ok(ConfigSet::LOOSE_OBJECTS_ZLIB_DEFAULT as u32);
1237 }
1238 let cfg = self.load_config_cascade()?;
1239 Ok(cfg.loose_objects_zlib_level()? as u32)
1240 }
1241
1242 pub(crate) fn loose_compression(&self) -> Result<Compression> {
1249 if let Some(cached) = self.loose_zlib_cache.get() {
1250 return Ok(*cached);
1251 }
1252 let level = self.resolve_loose_zlib_level()?;
1253 let compression = Compression::new(level);
1254 let _ = self.loose_zlib_cache.set(compression);
1255 Ok(compression)
1256 }
1257
1258 fn resolve_pack_zlib_level(&self) -> Result<i32> {
1259 if self.config_git_dir.is_none() && self.objects_dir.parent().is_none() {
1260 return Ok(6);
1261 }
1262 let cfg = self.load_config_cascade()?;
1263 cfg.pack_objects_zlib_level()
1264 }
1265
1266 pub(crate) fn pack_compression(&self) -> Result<Compression> {
1272 if let Some(cached) = self.pack_zlib_cache.get() {
1273 return Ok(*cached);
1274 }
1275 let level = self.resolve_pack_zlib_level()?;
1276 let compression = crate::pack_zlib::flate2_compression_for_pack_level(level);
1277 let _ = self.pack_zlib_cache.set(compression);
1278 Ok(compression)
1279 }
1280
1281 pub fn big_file_threshold(&self) -> Result<u64> {
1292 if let Some(cached) = self.big_file_threshold_cache.get() {
1293 return Ok(*cached);
1294 }
1295 let threshold = if self.config_git_dir.is_none() && self.objects_dir.parent().is_none() {
1296 ConfigSet::BIG_FILE_THRESHOLD_DEFAULT
1297 } else {
1298 self.load_config_cascade()?.big_file_threshold()?
1299 };
1300 let _ = self.big_file_threshold_cache.set(threshold);
1301 Ok(threshold)
1302 }
1303
1304 pub(crate) fn with_bounded_pack_cache(&self) -> Self {
1308 let mut odb = self.clone();
1309 odb.pack_store = Arc::new(PackStore::for_bounded_reads(self.objects_dir.clone()));
1310 odb.primary_source = Arc::new(OnceLock::new());
1311 odb.session_loose_reads = Arc::new(Mutex::new(SessionLooseReads::default()));
1312 odb.alternate_pack_stores = Arc::new(Mutex::new(HashMap::new()));
1313 odb.alternate_sources_cache = Arc::new(RwLock::new(AlternateSourcesCache {
1314 file_generation: 0,
1315 env_fingerprint: 0,
1316 submodule_generation: 0,
1317 sources: None,
1318 }));
1319 odb
1320 }
1321
1322 fn sync_delta_base_cache_limit(&self) {
1323 if self.pack_store.bounded_reads() {
1324 return;
1325 }
1326 self.delta_base_cache_sync.get_or_init(|| {
1327 if self.config_git_dir.is_some() {
1328 let cfg = self.load_config_cascade().unwrap_or_default();
1329 self.pack_store
1330 .configure_delta_base_from_config(Some(cfg.as_ref()));
1331 }
1332 });
1333 }
1334
1335 pub(crate) fn pack_store_for(&self, objects_dir: &Path) -> Arc<PackStore> {
1336 if objects_dir == self.objects_dir.as_path() {
1337 return Arc::clone(&self.pack_store);
1338 }
1339 let new_store = || {
1340 Arc::new(if self.pack_store.bounded_reads() {
1341 PackStore::for_bounded_reads(objects_dir.to_path_buf())
1342 } else {
1343 PackStore::new(objects_dir.to_path_buf())
1344 })
1345 };
1346 if let Ok(mut guard) = self.alternate_pack_stores.lock() {
1347 return Arc::clone(
1348 guard
1349 .entry(objects_dir.to_path_buf())
1350 .or_insert_with(new_store),
1351 );
1352 }
1353 new_store()
1354 }
1355
1356 pub fn pack_store(&self) -> Result<&Arc<PackStore>> {
1358 self.require_files_primary("pack_store")?;
1359 Ok(&self.pack_store)
1360 }
1361
1362 pub fn largest_pack_index_path(&self) -> Result<Option<PathBuf>> {
1371 self.require_files_primary("largest_pack_index_path")?;
1372 let mut indexes = pack::read_local_pack_indexes_cached(self.objects_dir())?;
1373 indexes.sort_by_key(|idx| std::cmp::Reverse(idx.len()));
1374 Ok(indexes.first().map(|idx| idx.idx_path.clone()))
1375 }
1376
1377 pub(crate) fn clear_delta_base_cache(&self) {
1381 self.pack_store.delta_bases().clear();
1382 }
1383
1384 pub fn invalidate_packs(&self) {
1391 self.pack_store.invalidate_all();
1392 PackStore::invalidate_legacy_for_objects_dir(&self.objects_dir);
1393 if let Ok(guard) = self.alternate_pack_stores.lock() {
1394 for store in guard.values() {
1395 store.invalidate_all();
1396 PackStore::invalidate_legacy_for_objects_dir(store.objects_dir());
1397 }
1398 }
1399 self.invalidate_change_id_header_probe();
1400 }
1401
1402 fn invalidate_change_id_header_probe(&self) {
1403 if let Some(caches) = self.shared_config_state.as_ref() {
1404 crate::change_id::invalidate_stored_change_id_header_probe(caches, self.objects_dir());
1405 }
1406 }
1407
1408 fn note_commit_object_written(&self, kind: ObjectKind) {
1409 if kind == ObjectKind::Commit {
1410 self.invalidate_change_id_header_probe();
1411 }
1412 }
1413
1414 pub(crate) fn loosen_unreachable_from_superseded_packs(
1428 &self,
1429 pack_dir: &Path,
1430 keep_pack: &Path,
1431 reachable: &crate::gc::LazyReachable<'_>,
1432 retention: UnreachablePackRetention,
1433 retained_packs: &HashSet<PathBuf>,
1434 inventory: &RepackPackInventorySnapshot,
1435 ) -> Result<usize> {
1436 if matches!(retention, UnreachablePackRetention::DropAll) {
1437 return Ok(0);
1438 }
1439 self.require_files_primary("repack")?;
1440 let keep_idx = keep_pack.with_extension("idx");
1441 let keep_pack_oids = if keep_idx.is_file() {
1442 PackedObjects::object_ids_from_index_path(&keep_idx)?
1443 } else {
1444 HashSet::new()
1445 };
1446 let kept_pack_oids =
1447 oids_in_inventory_kept_packs(pack_dir, keep_pack, &inventory.kept_at_inventory)?;
1448 let mut loosened = 0usize;
1449 for (idx_path, pack_mtime) in
1450 superseded_pack_indexes(pack_dir, keep_pack, retained_packs, inventory)?
1451 {
1452 let oids = PackedObjects::object_ids_from_index_path(&idx_path)?;
1453 for oid in oids {
1454 let keep = match retention {
1455 UnreachablePackRetention::KeepAll => true,
1456 UnreachablePackRetention::KeepGraceProtected(cutoff) => pack_mtime >= cutoff,
1457 UnreachablePackRetention::DropAll => false,
1458 };
1459 if !keep {
1460 continue;
1461 }
1462 if keep_pack_oids.contains(&oid) || kept_pack_oids.contains(&oid) {
1463 continue;
1464 }
1465 if reachable.get()?.contains(&oid) {
1466 continue;
1467 }
1468 let loose_path = oid.loose_path_in(&self.objects_dir);
1469 let pack_ft = system_time_to_file_time(pack_mtime);
1470 match retention {
1471 UnreachablePackRetention::KeepGraceProtected(cutoff) => {
1472 if loose_path.is_file() {
1473 let loose_mtime = fs::metadata(&loose_path)
1474 .and_then(|m| m.modified())
1475 .unwrap_or(std::time::SystemTime::UNIX_EPOCH);
1476 if loose_mtime >= cutoff {
1477 continue;
1478 }
1479 set_file_mtime(&loose_path, pack_ft).map_err(Error::Io)?;
1480 loosened += 1;
1481 continue;
1482 }
1483 }
1484 UnreachablePackRetention::KeepAll => {
1485 if loose_path.is_file() {
1486 continue;
1487 }
1488 }
1489 UnreachablePackRetention::DropAll => {}
1490 }
1491 let obj = self.read(&oid)?;
1492 self.write_loose_materialize(obj.kind, &obj.data)?;
1493 let primary = self.primary()?;
1494 let loose_path = primary.loose_store().object_path(&oid);
1495 set_file_mtime(&loose_path, pack_ft).map_err(Error::Io)?;
1496 loosened += 1;
1497 }
1498 }
1499 Ok(loosened)
1500 }
1501
1502 fn core_multi_pack_index_enabled(&self) -> bool {
1503 *self.core_multi_pack_index_cache.get_or_init(|| {
1507 if self.config_git_dir.is_none() {
1508 return false;
1509 }
1510 let cfg = self.load_config_cascade().unwrap_or_default();
1511 match cfg.get_bool("core.multiPackIndex") {
1512 Some(Ok(b)) => b,
1513 Some(Err(_)) => true,
1514 None => true,
1515 }
1516 })
1517 }
1518
1519 fn multi_pack_index_reads_enabled(&self) -> bool {
1520 if self.custom_primary.is_some() {
1521 return false;
1522 }
1523 if self.config_git_dir.is_none() {
1524 return false;
1525 }
1526 if !*self
1527 .local_pack_indexes_seen
1528 .get_or_init(|| objects_dir_has_pack_index_files(&self.objects_dir))
1529 {
1530 return false;
1531 }
1532 self.core_multi_pack_index_enabled()
1533 }
1534
1535 fn ensure_midx_prepared(&self) {
1536 if !self.multi_pack_index_reads_enabled() {
1537 return;
1538 }
1539 let _ = self.midx_packs_validated.get_or_init(|| {
1540 let diagnostics = self.diagnostics_handle();
1541 self.with_pack_store_for(&self.objects_dir, || {
1542 validate_midx_referenced_packs_with_diagnostics(
1543 &self.objects_dir,
1544 diagnostics.as_ref(),
1545 );
1546 });
1547 });
1548 }
1549
1550 fn try_read_packed_in_batch_context(&self, oid: &ObjectId) -> Result<Option<Object>> {
1555 if self.custom_primary.is_some()
1556 || !PackStore::read_context_matches_objects_dir(&self.objects_dir)
1557 || self.multi_pack_index_reads_enabled()
1558 {
1559 return Ok(None);
1560 }
1561 self.sync_delta_base_cache_limit();
1562 match pack::try_read_object_from_packs_with_options(
1563 &self.objects_dir,
1564 oid,
1565 pack::PackLookupOptions::ALL_LOCAL_PACKS,
1566 ) {
1567 Ok(obj) => Ok(Some(obj)),
1568 Err(Error::ObjectNotFound(_)) => Ok(None),
1569 Err(err) => Err(err),
1570 }
1571 }
1572
1573 fn try_read_info_packed_in_batch_context(&self, oid: &ObjectId) -> Result<Option<ObjectInfo>> {
1574 if self.custom_primary.is_some()
1575 || !PackStore::read_context_matches_objects_dir(&self.objects_dir)
1576 || self.multi_pack_index_reads_enabled()
1577 {
1578 return Ok(None);
1579 }
1580 match pack::try_read_object_info_from_packs_with_options(
1581 &self.objects_dir,
1582 oid,
1583 pack::PackLookupOptions::ALL_LOCAL_PACKS,
1584 ) {
1585 Ok(info) => Ok(Some(info)),
1586 Err(Error::ObjectNotFound(_)) => Ok(None),
1587 Err(err) => Err(err),
1588 }
1589 }
1590
1591 #[must_use]
1598 pub fn objects_dir(&self) -> &Path {
1599 &self.objects_dir
1600 }
1601
1602 pub(crate) fn commit_graph_chain(
1613 &self,
1614 ) -> Option<Arc<crate::commit_graph_file::CommitGraphChain>> {
1615 if let Some(chain) = &self.attached_commit_graph {
1616 return Some(Arc::clone(chain));
1617 }
1618 if self.custom_primary.is_some() {
1619 return None;
1620 }
1621 self.shared_config_state
1622 .as_ref()?
1623 .commit_graph_chain(self.objects_dir())
1624 }
1625
1626 fn try_read_extra_sources(&self, oid: &ObjectId) -> Result<Option<Object>> {
1627 for src in &self.extra_read_sources {
1628 if let Some(obj) = src.read(oid)? {
1629 return Ok(Some(obj));
1630 }
1631 }
1632 Ok(None)
1633 }
1634
1635 fn try_read_info_extra_sources(&self, oid: &ObjectId) -> Result<Option<ObjectInfo>> {
1636 for src in &self.extra_read_sources {
1637 if let Some(info) = src.read_info(oid)? {
1638 return Ok(Some(info));
1639 }
1640 }
1641 Ok(None)
1642 }
1643
1644 fn try_open_stream_extra_sources(
1645 &self,
1646 oid: &ObjectId,
1647 ) -> Result<Option<store::ObjectStream<'_>>> {
1648 for src in &self.extra_read_sources {
1649 if let Some(stream) = src.open_stream(oid)? {
1650 return Ok(Some(stream));
1651 }
1652 }
1653 Ok(None)
1654 }
1655
1656 #[must_use]
1662 pub fn config_git_dir(&self) -> Option<&Path> {
1663 self.config_git_dir.as_deref()
1664 }
1665
1666 #[must_use]
1668 pub fn object_path(&self, oid: &ObjectId) -> PathBuf {
1669 oid.loose_path_in(&self.objects_dir)
1670 }
1671
1672 pub(crate) fn local_pack_indexes(&self) -> Result<Vec<Arc<pack::PackIndex>>> {
1679 self.with_primary_read_context(|| pack::read_local_pack_indexes_cached(&self.objects_dir))
1682 }
1683
1684 pub(crate) fn contained_in_alternate_pack(&self, oid: &ObjectId) -> Result<bool> {
1692 if !self.alternates_enabled {
1693 return Ok(false);
1694 }
1695 for alt_dir in self.file_alternate_dirs_snapshot().iter() {
1696 if store::object_in_non_promisor_pack(alt_dir, oid) {
1697 return Ok(true);
1698 }
1699 }
1700 for alt_dir in self.env_alternate_dirs_snapshot().iter() {
1701 if store::object_in_non_promisor_pack(alt_dir, oid) {
1702 return Ok(true);
1703 }
1704 }
1705 if let Ok(guard) = self.submodule_alternate_dirs.lock() {
1706 for alt_dir in guard.iter() {
1707 if store::object_in_non_promisor_pack(alt_dir, oid) {
1708 return Ok(true);
1709 }
1710 }
1711 }
1712 Ok(false)
1713 }
1714
1715 pub(crate) fn remove_local_loose(&self, oid: &ObjectId) -> Result<bool> {
1724 match fs::remove_file(self.object_path(oid)) {
1725 Ok(()) => Ok(true),
1726 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(false),
1727 Err(err) => Err(Error::Io(err)),
1728 }
1729 }
1730
1731 pub(crate) fn local_loose_file_size(&self, oid: &ObjectId) -> Option<u64> {
1734 fs::metadata(self.object_path(oid))
1735 .ok()
1736 .filter(fs::Metadata::is_file)
1737 .map(|meta| meta.len())
1738 }
1739
1740 #[must_use]
1756 pub fn exists_local(&self, oid: &ObjectId) -> bool {
1757 self.try_exists_local(oid).unwrap_or(false)
1758 }
1759
1760 pub(crate) fn try_exists_local(&self, oid: &ObjectId) -> Result<bool> {
1762 if self.is_empty_tree(oid) {
1763 return Ok(true);
1764 }
1765 if let Some(primary) = &self.custom_primary {
1766 return primary.contains(oid);
1767 }
1768 Ok(self.with_pack_store_for(&self.objects_dir, || {
1769 exists_materialized_in_objects_dir(&self.objects_dir, oid)
1770 }))
1771 }
1772
1773 #[must_use]
1781 pub fn exists_stored(&self, oid: &ObjectId) -> bool {
1782 self.try_exists_stored(oid).unwrap_or(false)
1783 }
1784
1785 pub(crate) fn try_exists_stored(&self, oid: &ObjectId) -> Result<bool> {
1787 if oid.is_canonical_empty_tree() || self.overlay_read(oid).is_some() {
1788 return Ok(true);
1789 }
1790 if let Some(primary) = &self.custom_primary {
1791 return primary.contains(oid);
1792 }
1793 Ok(self.with_pack_store_for(&self.objects_dir, || {
1794 store::local_object_stored(&self.objects_dir, oid)
1795 }))
1796 }
1797
1798 #[must_use]
1808 pub fn exists_materialized_local(&self, oid: &ObjectId) -> bool {
1809 if let Some(primary) = &self.custom_primary {
1810 return primary.contains(oid).unwrap_or(false);
1811 }
1812 self.with_pack_store_for(&self.objects_dir, || {
1813 exists_materialized_in_objects_dir(&self.objects_dir, oid)
1814 })
1815 }
1816
1817 pub(crate) fn reprepare_local_packs(&self) {
1822 if self.custom_primary.is_some() {
1823 return;
1824 }
1825 self.with_pack_store_for(&self.objects_dir, || {
1826 let _ = crate::pack::reprepare_pack_directory_on_miss(&self.objects_dir);
1827 });
1828 }
1829
1830 pub(crate) fn exists_materialized_local_without_rescan(&self, oid: &ObjectId) -> bool {
1838 if self.custom_primary.is_some() {
1839 return self.exists_materialized_local(oid);
1840 }
1841 self.with_pack_store_for(&self.objects_dir, || {
1842 store::local_object_materialized_without_rescan(&self.objects_dir, oid)
1843 })
1844 }
1845
1846 pub fn contains(&self, oid: &ObjectId) -> Result<bool> {
1858 self.probe_sources(oid, Err)
1859 }
1860
1861 fn probe_sources(
1864 &self,
1865 oid: &ObjectId,
1866 mut on_err: impl FnMut(Error) -> Result<()>,
1867 ) -> Result<bool> {
1868 #[cfg(test)]
1869 self.exists_probe.fetch_add(1, Ordering::Relaxed);
1870 if self.is_empty_tree(oid) || self.overlay_read(oid).is_some() {
1871 return Ok(true);
1872 }
1873 let mut found = |probe: Result<bool>| match probe {
1874 Ok(present) => Ok(present),
1875 Err(err) => on_err(err).map(|()| false),
1876 };
1877 if found(self.primary_writable().and_then(|p| p.contains(oid)))? {
1878 return Ok(true);
1879 }
1880 for src in &self.extra_read_sources {
1881 if found(src.contains(oid))? {
1882 return Ok(true);
1883 }
1884 }
1885 if !self.alternate_object_lookup_needed() {
1886 return Ok(false);
1887 }
1888 found(self.sources().and_then(|s| s.contains(oid)))
1889 }
1890
1891 pub(crate) fn contains_many(&self, oids: &[ObjectId]) -> Result<Vec<bool>> {
1901 #[cfg(test)]
1902 self.exists_probe.fetch_add(oids.len(), Ordering::Relaxed);
1903 let mut present: Vec<bool> = oids
1904 .iter()
1905 .map(|oid| self.is_empty_tree(oid) || self.overlay_read(oid).is_some())
1906 .collect();
1907 let mut pending: Vec<usize> = (0..oids.len()).filter(|&i| !present[i]).collect();
1908 store::resolve_pending(oids, &mut present, &mut pending, |batch| {
1909 self.primary_writable()?.contains_many(batch)
1910 })?;
1911 for src in &self.extra_read_sources {
1912 store::resolve_pending(oids, &mut present, &mut pending, |batch| {
1913 src.contains_many(batch)
1914 })?;
1915 }
1916 if !pending.is_empty() && self.alternate_object_lookup_needed() {
1917 store::resolve_pending(oids, &mut present, &mut pending, |batch| {
1918 self.sources()?.contains_many(batch)
1919 })?;
1920 }
1921 Ok(present)
1922 }
1923
1924 pub(crate) fn present_oids(
1930 &self,
1931 oids: impl IntoIterator<Item = ObjectId>,
1932 ) -> Result<Vec<ObjectId>> {
1933 self.partition_oids(oids, true)
1934 }
1935
1936 pub(crate) fn missing_oids(
1942 &self,
1943 oids: impl IntoIterator<Item = ObjectId>,
1944 ) -> Result<Vec<ObjectId>> {
1945 self.partition_oids(oids, false)
1946 }
1947
1948 fn partition_oids(
1950 &self,
1951 oids: impl IntoIterator<Item = ObjectId>,
1952 want_present: bool,
1953 ) -> Result<Vec<ObjectId>> {
1954 let oids: Vec<ObjectId> = oids.into_iter().collect();
1955 let present = self.contains_many(&oids)?;
1956 Ok(oids
1957 .into_iter()
1958 .zip(present)
1959 .filter_map(|(oid, has)| (has == want_present).then_some(oid))
1960 .collect())
1961 }
1962
1963 pub(crate) fn missing_stored_oids(
1973 &self,
1974 oids: impl IntoIterator<Item = ObjectId>,
1975 ) -> Result<Vec<ObjectId>> {
1976 if self.custom_primary.is_some() {
1977 return filter_oids(oids, |oid| self.try_exists_stored(oid).map(|has| !has));
1978 }
1979 self.reprepare_local_packs();
1980 self.with_pack_store_for(&self.objects_dir, || {
1981 filter_oids(oids, |oid| {
1982 Ok(!(oid.is_canonical_empty_tree()
1983 || self.overlay_read(oid).is_some()
1984 || store::local_object_stored_without_rescan(&self.objects_dir, oid)))
1985 })
1986 })
1987 }
1988
1989 #[must_use]
1996 pub fn exists(&self, oid: &ObjectId) -> bool {
1997 self.probe_sources(oid, |_| Ok(())).unwrap_or(false)
1998 }
1999
2000 fn with_pack_store_for<R>(&self, objects_dir: &Path, f: impl FnOnce() -> R) -> R {
2002 let store = self.pack_store_for(objects_dir);
2003 PackStore::with_context(store, f)
2004 }
2005
2006 fn with_primary_read_context<R>(&self, f: impl FnOnce() -> R) -> R {
2012 if self.custom_primary.is_some()
2013 || PackStore::read_context_matches_objects_dir(&self.objects_dir)
2014 {
2015 f()
2016 } else {
2017 self.with_pack_store_for(&self.objects_dir, f)
2018 }
2019 }
2020
2021 pub fn with_pack_read_context<R>(&self, f: impl FnOnce() -> R) -> R {
2026 self.with_pack_store_for(&self.objects_dir, f)
2027 }
2028
2029 pub(crate) fn pack_read_context_guard(&self) -> crate::pack_store::PackReadContextGuard {
2032 crate::pack_store::PackReadContextGuard::install(self.pack_store_for(&self.objects_dir))
2033 }
2034
2035 pub(crate) fn open_session_pack_reader(&self) -> Result<Option<pack::SingleLocalPackReader>> {
2043 if self.custom_primary.is_some()
2044 || !PackStore::read_context_matches_objects_dir(&self.objects_dir)
2045 {
2046 return Ok(None);
2047 }
2048 self.sync_delta_base_cache_limit();
2049 pack::SingleLocalPackReader::open_allow_loose(&self.objects_dir)
2050 }
2051
2052 #[must_use]
2057 pub fn freshen_object(&self, oid: &ObjectId) -> bool {
2058 if self.is_empty_tree(oid) {
2059 return false;
2060 }
2061
2062 let loose = self.object_path(oid);
2063 if loose.is_file() {
2064 return self.touch_object_mtime(&loose).is_some();
2065 }
2066 self.freshen_packed_or_alternate(oid)
2067 }
2068
2069 fn freshen_packed_or_alternate(&self, oid: &ObjectId) -> bool {
2073 if self.is_empty_tree(oid) {
2074 return false;
2075 }
2076 if self.freshen_packed_in_objects_dir(&self.objects_dir, oid) {
2077 return true;
2078 }
2079
2080 let file_alts = self.file_alternate_dirs_snapshot();
2081 for alt_dir in file_alts.iter() {
2082 if self.freshen_object_in_objects_dir(alt_dir, oid) {
2083 return true;
2084 }
2085 }
2086
2087 for alt_dir in self.env_alternate_dirs_snapshot().iter() {
2088 if self.freshen_object_in_objects_dir(alt_dir, oid) {
2089 return true;
2090 }
2091 }
2092
2093 if let Ok(guard) = self.submodule_alternate_dirs.lock() {
2094 for alt_dir in guard.iter() {
2095 if self.freshen_object_in_objects_dir(alt_dir, oid) {
2096 return true;
2097 }
2098 }
2099 }
2100
2101 false
2102 }
2103
2104 fn freshen_object_in_objects_dir(&self, objects_dir: &Path, oid: &ObjectId) -> bool {
2105 let loose = objects_dir
2106 .join(oid.loose_prefix())
2107 .join(oid.loose_suffix());
2108 if loose.is_file() {
2109 return self.touch_object_mtime(&loose).is_some();
2110 }
2111 self.freshen_packed_in_objects_dir(objects_dir, oid)
2112 }
2113
2114 fn freshen_packed_in_objects_dir(&self, objects_dir: &Path, oid: &ObjectId) -> bool {
2115 self.with_pack_store_for(objects_dir, || {
2116 let Ok(indexes) = pack::read_local_pack_indexes_cached(objects_dir) else {
2117 return false;
2118 };
2119 for idx in &indexes {
2120 if idx.contains(oid).unwrap_or(false) {
2121 return self.freshen_pack_once(&idx.pack_path, idx.is_cruft);
2122 }
2123 }
2124 false
2125 })
2126 }
2127
2128 fn freshen_pack_once(&self, pack_path: &Path, is_cruft: bool) -> bool {
2133 let Ok(mut guard) = self.freshened_packs.lock() else {
2134 return false;
2135 };
2136 if guard.contains(pack_path) {
2137 return true;
2138 }
2139 if is_cruft {
2140 return false;
2141 }
2142 let Some(touched_at) = self.touch_object_mtime(pack_path) else {
2143 return false;
2144 };
2145 pack::refresh_pack_bytes_signature(pack_path, touched_at);
2146 guard.insert(pack_path.to_path_buf());
2147 true
2148 }
2149
2150 #[must_use]
2159 pub fn bulk_stage_write_options(&self) -> WriteOptions {
2160 let stored_beyond_loose = self.custom_primary.is_none()
2161 && (self.alternate_object_lookup_needed()
2162 || self.with_pack_read_context(|| {
2163 pack::read_local_pack_indexes_cached(&self.objects_dir)
2164 .is_ok_and(|indexes| !indexes.is_empty())
2165 }));
2166 WriteOptions {
2167 freshen_existing: stored_beyond_loose,
2168 ..BULK_NEW_LOOSE_WRITE
2169 }
2170 }
2171
2172 fn try_freshen_existing(&self, oid: &ObjectId) -> Option<ObjectId> {
2175 if let Ok(primary) = self.primary() {
2176 if let Ok(Some(existing)) =
2177 primary.try_finish_duplicate_write(oid, WriteOptions::default())
2178 {
2179 return Some(existing);
2180 }
2181 }
2182 if !self.exists(oid) {
2183 return None;
2184 }
2185 if self.freshen_object(oid) {
2186 return Some(*oid);
2187 }
2188 None
2189 }
2190
2191 fn try_freshen_existing_local(&self, oid: &ObjectId) -> Option<ObjectId> {
2192 let primary = self.primary().ok()?;
2193 primary
2194 .try_finish_duplicate_write(
2195 oid,
2196 WriteOptions {
2197 assume_loose_only_existence: true,
2198 ..WriteOptions::default()
2199 },
2200 )
2201 .ok()
2202 .flatten()
2203 }
2204
2205 fn object_exists_for_write_skip(&self, oid: &ObjectId) -> bool {
2210 if self.is_empty_tree(oid) {
2211 self.exists_materialized_local(oid)
2212 } else {
2213 self.exists(oid)
2214 }
2215 }
2216
2217 pub fn read_loose_verify_oid(path: &Path, expected_oid: &ObjectId) -> Result<Object> {
2228 LooseStore::read_verify_oid(path, expected_oid)
2229 }
2230
2231 fn local_loose(&self) -> Result<LooseStore> {
2232 self.require_files_primary("loose_write")?;
2233 Ok(LooseStore::new(
2234 self.objects_dir.clone(),
2235 self.hash_algo(),
2236 self.loose_compression()?,
2237 ))
2238 }
2239
2240 fn loose_at(&self, objects_dir: &Path) -> Result<LooseStore> {
2241 Ok(LooseStore::new(
2242 objects_dir.to_path_buf(),
2243 self.hash_algo(),
2244 self.loose_compression()?,
2245 ))
2246 }
2247
2248 pub fn read(&self, oid: &ObjectId) -> Result<Object> {
2256 self.read_impl(oid, true)
2257 }
2258
2259 fn read_impl(&self, oid: &ObjectId, remember: bool) -> Result<Object> {
2262 if self.is_empty_tree(oid) {
2263 return Ok(Object {
2264 kind: ObjectKind::Tree,
2265 data: Vec::new(),
2266 });
2267 }
2268
2269 if let Some(obj) = self.session_cached_loose_read(oid) {
2270 return Ok(obj);
2271 }
2272
2273 if let Some(obj) = self.overlay_read(oid) {
2274 return Ok(obj);
2275 }
2276 if let Some(obj) = self.try_read_packed_in_batch_context(oid)? {
2277 return Ok(obj);
2278 }
2279
2280 let use_midx = self.multi_pack_index_reads_enabled();
2281 if use_midx {
2282 self.ensure_midx_prepared();
2283 }
2284
2285 self.sync_delta_base_cache_limit();
2286
2287 let primary = self.primary_writable()?;
2288 let files_primary = self.primary().ok();
2289 let mut unreadable_local_loose = None;
2290 let read_primary = || primary.read(oid);
2291 match self.with_primary_read_context(read_primary) {
2292 Ok(Some(obj)) => return self.finish_session_loose_read(oid, obj, remember),
2293 Ok(None) => {}
2294 Err(err)
2295 if files_primary
2296 .as_ref()
2297 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2298 {
2299 unreadable_local_loose = Some(err);
2300 }
2301 Err(err) => return Err(err),
2302 }
2303
2304 if let Some(obj) = self.try_read_extra_sources(oid)? {
2305 return self.finish_session_loose_read(oid, obj, remember);
2306 }
2307
2308 if self.alternates_enabled {
2309 match self.sources()?.read(oid) {
2310 Ok(Some(obj)) => return self.finish_session_loose_read(oid, obj, remember),
2311 Ok(None) => {}
2312 Err(err) => return Err(err),
2313 }
2314 }
2315
2316 if let Some(err) = unreadable_local_loose {
2317 return Err(err);
2318 }
2319
2320 self.read_after_promisor_miss(oid)
2321 }
2322
2323 #[cold]
2324 #[inline(never)]
2325 fn read_after_promisor_miss(&self, oid: &ObjectId) -> Result<Object> {
2326 let mut lazy_retries = 0u32;
2327 loop {
2328 let Some(res) = self.resolve_promisor_miss(oid, lazy_retries)? else {
2329 return Err(Error::ObjectNotFound(oid.to_hex()));
2330 };
2331 match res {
2332 crate::promisor_fetch::PromisorMissResolution::RetryAfterFetch => {
2333 lazy_retries += 1;
2334 if let Some(obj) = self.try_read_once(oid)? {
2335 return Ok(obj);
2336 }
2337 }
2338 crate::promisor_fetch::PromisorMissResolution::NotPromisorRepo => {
2339 return Err(Error::ObjectNotFound(oid.to_hex()));
2340 }
2341 crate::promisor_fetch::PromisorMissResolution::PromisedMissing(err) => {
2342 return Err(err);
2343 }
2344 }
2345 }
2346 }
2347
2348 pub(crate) fn try_read_once(&self, oid: &ObjectId) -> Result<Option<Object>> {
2351 if self.is_empty_tree(oid) {
2352 return Ok(Some(Object {
2353 kind: ObjectKind::Tree,
2354 data: Vec::new(),
2355 }));
2356 }
2357
2358 if let Some(obj) = self.session_cached_loose_read(oid) {
2359 return Ok(Some(obj));
2360 }
2361
2362 if let Some(obj) = self.overlay_read(oid) {
2363 return Ok(Some(obj));
2364 }
2365 if let Some(obj) = self.try_read_packed_in_batch_context(oid)? {
2366 return Ok(Some(self.finish_session_loose_read(oid, obj, true)?));
2367 }
2368
2369 let use_midx = self.multi_pack_index_reads_enabled();
2370 if use_midx {
2371 self.ensure_midx_prepared();
2372 }
2373
2374 self.sync_delta_base_cache_limit();
2375
2376 let primary = self.primary_writable()?;
2377 let files_primary = self.primary().ok();
2378 let mut unreadable_local_loose = None;
2379 let read_primary = || primary.read(oid);
2380 match self.with_primary_read_context(read_primary) {
2381 Ok(Some(obj)) => return Ok(Some(self.finish_session_loose_read(oid, obj, true)?)),
2382 Ok(None) => {}
2383 Err(err)
2384 if files_primary
2385 .as_ref()
2386 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2387 {
2388 unreadable_local_loose = Some(err);
2389 }
2390 Err(err) => return Err(err),
2391 }
2392
2393 if let Some(obj) = self.try_read_extra_sources(oid)? {
2394 return Ok(Some(self.finish_session_loose_read(oid, obj, true)?));
2395 }
2396
2397 if self.alternates_enabled {
2398 match self.sources()?.read(oid) {
2399 Ok(Some(obj)) => return Ok(Some(self.finish_session_loose_read(oid, obj, true)?)),
2400 Ok(None) => {}
2401 Err(err) => return Err(err),
2402 }
2403 }
2404
2405 if let Some(err) = unreadable_local_loose {
2406 return Err(err);
2407 }
2408
2409 Ok(None)
2410 }
2411
2412 fn resolve_promisor_miss(
2413 &self,
2414 oid: &ObjectId,
2415 lazy_retries: u32,
2416 ) -> Result<Option<crate::promisor_fetch::PromisorMissResolution>> {
2417 let (Some(caches), Some(env)) = (&self.shared_config_state, &self.shared_environment)
2418 else {
2419 return Ok(None);
2420 };
2421 Ok(Some(crate::promisor_fetch::resolve_promisor_read_miss(
2422 caches,
2423 self,
2424 oid,
2425 env.as_ref(),
2426 lazy_retries,
2427 )?))
2428 }
2429
2430 pub(crate) fn prefetch_promisor_read_batch(
2438 &self,
2439 oids: impl IntoIterator<Item = ObjectId>,
2440 ) -> Result<()> {
2441 let (Some(caches), Some(env)) = (&self.shared_config_state, &self.shared_environment)
2442 else {
2443 return Ok(());
2444 };
2445 crate::promisor_fetch::prefetch_promisor_read_batch(caches, self, env.as_ref(), oids)
2446 }
2447
2448 pub fn read_into(&self, oid: &ObjectId, buf: &mut Vec<u8>) -> Result<ObjectKind> {
2457 if self.is_empty_tree(oid) {
2458 buf.clear();
2459 return Ok(ObjectKind::Tree);
2460 }
2461
2462 if let Some(obj) = self.session_cached_loose_read(oid) {
2463 buf.clear();
2464 buf.extend_from_slice(&obj.data);
2465 return Ok(obj.kind);
2466 }
2467
2468 if let Some(obj) = self.overlay_read(oid) {
2469 buf.clear();
2470 buf.extend_from_slice(&obj.data);
2471 return Ok(obj.kind);
2472 }
2473
2474 if let Some(kind) = self.try_read_packed_in_batch_context_into(oid, false, buf)? {
2475 return Ok(kind);
2476 }
2477
2478 let use_midx = self.multi_pack_index_reads_enabled();
2479 if use_midx {
2480 self.ensure_midx_prepared();
2481 }
2482
2483 self.sync_delta_base_cache_limit();
2484
2485 let primary = self.primary_writable()?;
2486 let files_primary = self.primary().ok();
2487 let mut unreadable_local_loose = None;
2488 let read_primary = || primary.read_into(oid, buf);
2489 match self.with_primary_read_context(read_primary) {
2490 Ok(Some(kind)) => return Ok(kind),
2491 Ok(None) => {}
2492 Err(err)
2493 if files_primary
2494 .as_ref()
2495 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2496 {
2497 unreadable_local_loose = Some(err);
2498 }
2499 Err(err) => return Err(err),
2500 }
2501
2502 if let Some(kind) = self.try_read_into_extra_sources(oid, buf)? {
2503 return Ok(kind);
2504 }
2505
2506 if self.alternates_enabled {
2507 match self.sources()?.read_into(oid, buf) {
2508 Ok(Some(kind)) => return Ok(kind),
2509 Ok(None) => {}
2510 Err(err) => return Err(err),
2511 }
2512 }
2513
2514 if let Some(err) = unreadable_local_loose {
2515 return Err(err);
2516 }
2517
2518 self.read_into_after_promisor_miss(oid, buf)
2519 }
2520
2521 pub(crate) fn read_into_caching_delta_bases(
2535 &self,
2536 oid: &ObjectId,
2537 buf: &mut Vec<u8>,
2538 ) -> Result<ObjectKind> {
2539 if let Some(kind) = self.try_read_packed_in_batch_context_into(oid, true, buf)? {
2540 return Ok(kind);
2541 }
2542 if self.custom_primary.is_none() && !self.is_empty_tree(oid) {
2543 if let Ok(files) = self.primary() {
2544 self.ensure_midx_prepared();
2545 self.sync_delta_base_cache_limit();
2546 let mut read = || files.read_into_with_delta_policy(oid, true, buf);
2547 let local = if PackStore::read_context_matches_objects_dir(&self.objects_dir) {
2548 read()
2549 } else {
2550 self.with_pack_store_for(&self.objects_dir, read)
2551 };
2552 if let Ok(Some(kind)) = local {
2553 return Ok(kind);
2554 }
2555 }
2556 }
2557 if self.custom_primary.is_some() {
2558 return self.read_into(oid, buf);
2559 }
2560 let object = self.read(oid)?;
2561 buf.clear();
2562 buf.extend_from_slice(&object.data);
2563 Ok(object.kind)
2564 }
2565
2566 pub(crate) fn read_packed_commit_header_block_into(
2579 &self,
2580 oid: &ObjectId,
2581 buf: &mut Vec<u8>,
2582 ) -> Result<bool> {
2583 if self.custom_primary.is_some() {
2584 return Ok(false);
2585 }
2586 let Ok(files) = self.primary() else {
2587 return Ok(false);
2588 };
2589 files.read_packed_commit_header_block_into(oid, buf)
2590 }
2591
2592 #[cold]
2593 #[inline(never)]
2594 fn read_into_after_promisor_miss(
2595 &self,
2596 oid: &ObjectId,
2597 buf: &mut Vec<u8>,
2598 ) -> Result<ObjectKind> {
2599 let mut lazy_retries = 0u32;
2600 loop {
2601 let Some(res) = self.resolve_promisor_miss(oid, lazy_retries)? else {
2602 return Err(Error::ObjectNotFound(oid.to_hex()));
2603 };
2604 match res {
2605 crate::promisor_fetch::PromisorMissResolution::RetryAfterFetch => {
2606 lazy_retries += 1;
2607 if let Some(kind) = self.try_read_into_once(oid, buf)? {
2608 return Ok(kind);
2609 }
2610 }
2611 crate::promisor_fetch::PromisorMissResolution::NotPromisorRepo => {
2612 return Err(Error::ObjectNotFound(oid.to_hex()));
2613 }
2614 crate::promisor_fetch::PromisorMissResolution::PromisedMissing(err) => {
2615 return Err(err);
2616 }
2617 }
2618 }
2619 }
2620
2621 fn try_read_into_once(&self, oid: &ObjectId, buf: &mut Vec<u8>) -> Result<Option<ObjectKind>> {
2622 if oid.is_well_known_empty_tree() {
2623 buf.clear();
2624 return Ok(Some(ObjectKind::Tree));
2625 }
2626
2627 if let Some(obj) = self.session_cached_loose_read(oid) {
2628 buf.clear();
2629 buf.extend_from_slice(&obj.data);
2630 return Ok(Some(obj.kind));
2631 }
2632
2633 if let Some(obj) = self.overlay_read(oid) {
2634 buf.clear();
2635 buf.extend_from_slice(&obj.data);
2636 return Ok(Some(obj.kind));
2637 }
2638
2639 if let Some(kind) = self.try_read_packed_in_batch_context_into(oid, false, buf)? {
2640 return Ok(Some(kind));
2641 }
2642
2643 let use_midx = self.multi_pack_index_reads_enabled();
2644 if use_midx {
2645 self.ensure_midx_prepared();
2646 }
2647
2648 self.sync_delta_base_cache_limit();
2649
2650 let primary = self.primary_writable()?;
2651 let files_primary = self.primary().ok();
2652 let mut unreadable_local_loose = None;
2653 let read_primary = || primary.read_into(oid, buf);
2654 match self.with_primary_read_context(read_primary) {
2655 Ok(Some(kind)) => return Ok(Some(kind)),
2656 Ok(None) => {}
2657 Err(err)
2658 if files_primary
2659 .as_ref()
2660 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2661 {
2662 unreadable_local_loose = Some(err);
2663 }
2664 Err(err) => return Err(err),
2665 }
2666
2667 if let Some(kind) = self.try_read_into_extra_sources(oid, buf)? {
2668 return Ok(Some(kind));
2669 }
2670
2671 if self.alternates_enabled {
2672 match self.sources()?.read_into(oid, buf) {
2673 Ok(Some(kind)) => return Ok(Some(kind)),
2674 Ok(None) => {}
2675 Err(err) => return Err(err),
2676 }
2677 }
2678
2679 if let Some(err) = unreadable_local_loose {
2680 return Err(err);
2681 }
2682
2683 Ok(None)
2684 }
2685
2686 fn try_read_into_extra_sources(
2687 &self,
2688 oid: &ObjectId,
2689 buf: &mut Vec<u8>,
2690 ) -> Result<Option<ObjectKind>> {
2691 for src in &self.extra_read_sources {
2692 if let Some(kind) = src.read_into(oid, buf)? {
2693 return Ok(Some(kind));
2694 }
2695 }
2696 Ok(None)
2697 }
2698
2699 fn try_read_packed_in_batch_context_into(
2700 &self,
2701 oid: &ObjectId,
2702 populate_delta_lru: bool,
2703 buf: &mut Vec<u8>,
2704 ) -> Result<Option<ObjectKind>> {
2705 if self.custom_primary.is_some()
2706 || !PackStore::read_context_matches_objects_dir(&self.objects_dir)
2707 || self.multi_pack_index_reads_enabled()
2708 {
2709 return Ok(None);
2710 }
2711 self.sync_delta_base_cache_limit();
2712 match pack::try_read_object_from_packs_with_options_into(
2713 &self.objects_dir,
2714 oid,
2715 pack::PackLookupOptions::ALL_LOCAL_PACKS,
2716 populate_delta_lru,
2717 buf,
2718 ) {
2719 Ok(kind) => Ok(Some(kind)),
2720 Err(Error::ObjectNotFound(_)) => Ok(None),
2721 Err(err) => Err(err),
2722 }
2723 }
2724
2725 pub fn read_info(&self, oid: &ObjectId) -> Result<ObjectInfo> {
2737 if self.is_empty_tree(oid) {
2738 return Ok(ObjectInfo {
2739 kind: ObjectKind::Tree,
2740 size: 0,
2741 });
2742 }
2743
2744 if let Some(obj) = self.overlay_read(oid) {
2745 return Ok(ObjectInfo {
2746 kind: obj.kind,
2747 size: u64::try_from(obj.data.len())
2748 .map_err(|_| Error::CorruptObject("object size overflow".to_owned()))?,
2749 });
2750 }
2751
2752 if let Some(info) = self.try_read_info_packed_in_batch_context(oid)? {
2753 return Ok(info);
2754 }
2755
2756 let use_midx = self.multi_pack_index_reads_enabled();
2757 if use_midx {
2758 self.ensure_midx_prepared();
2759 }
2760
2761 let primary = self.primary_writable()?;
2762 let files_primary = self.primary().ok();
2763 let mut unreadable_local_loose = None;
2764 match primary.read_info(oid) {
2765 Ok(Some(info)) => return Ok(info),
2766 Ok(None) => {}
2767 Err(err)
2768 if files_primary
2769 .as_ref()
2770 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2771 {
2772 unreadable_local_loose = Some(err);
2773 }
2774 Err(err) => return Err(err),
2775 }
2776
2777 if let Some(info) = self.try_read_info_extra_sources(oid)? {
2778 return Ok(info);
2779 }
2780
2781 if self.alternates_enabled {
2782 match self.sources()?.read_info(oid) {
2783 Ok(Some(info)) => return Ok(info),
2784 Ok(None) => {}
2785 Err(err) => return Err(err),
2786 }
2787 }
2788
2789 if let Some(err) = unreadable_local_loose {
2790 return Err(err);
2791 }
2792
2793 self.read_info_after_promisor_miss(oid)
2794 }
2795
2796 pub(crate) fn read_info_without_fetch(&self, oid: &ObjectId) -> Result<ObjectInfo> {
2801 self.try_read_info_once(oid)?
2802 .ok_or_else(|| Error::ObjectNotFound(oid.to_hex()))
2803 }
2804
2805 pub(crate) fn stored_object_kind(&self, oid: &ObjectId) -> Result<Option<ObjectKind>> {
2817 if oid.is_well_known_empty_tree() && !self.empty_tree_stored(oid) {
2818 return Ok(None);
2819 }
2820 match self.try_read_info_once(oid) {
2821 Ok(info) => Ok(info.map(|info| info.kind)),
2822 Err(Error::ObjectNotFound(_)) => Ok(None),
2823 Err(err) => Err(err),
2824 }
2825 }
2826
2827 fn empty_tree_stored(&self, oid: &ObjectId) -> bool {
2828 if self.overlay_read(oid).is_some() || self.exists_materialized_local(oid) {
2829 return true;
2830 }
2831 if !self.alternates_enabled {
2832 return false;
2833 }
2834 let file_dirs = self.file_alternate_dirs_snapshot();
2835 let env_dirs = self.env_alternate_dirs_snapshot();
2836 file_dirs
2837 .iter()
2838 .chain(env_dirs.iter())
2839 .any(|dir| self.with_pack_store_for(dir, || store::local_object_materialized(dir, oid)))
2840 }
2841
2842 #[cold]
2843 #[inline(never)]
2844 fn read_info_after_promisor_miss(&self, oid: &ObjectId) -> Result<ObjectInfo> {
2845 let mut lazy_retries = 0u32;
2846 loop {
2847 let Some(res) = self.resolve_promisor_miss(oid, lazy_retries)? else {
2848 return Err(Error::ObjectNotFound(oid.to_hex()));
2849 };
2850 match res {
2851 crate::promisor_fetch::PromisorMissResolution::PromisedMissing(err) => {
2852 return Err(err);
2853 }
2854 crate::promisor_fetch::PromisorMissResolution::NotPromisorRepo => {
2855 return Err(Error::ObjectNotFound(oid.to_hex()));
2856 }
2857 crate::promisor_fetch::PromisorMissResolution::RetryAfterFetch => {
2858 lazy_retries += 1;
2859 if let Some(info) = self.try_read_info_once(oid)? {
2860 return Ok(info);
2861 }
2862 }
2863 }
2864 }
2865 }
2866
2867 fn try_read_info_once(&self, oid: &ObjectId) -> Result<Option<ObjectInfo>> {
2868 if self.is_empty_tree(oid) {
2869 return Ok(Some(ObjectInfo {
2870 kind: ObjectKind::Tree,
2871 size: 0,
2872 }));
2873 }
2874
2875 if let Some(obj) = self.overlay_read(oid) {
2876 return Ok(Some(ObjectInfo {
2877 kind: obj.kind,
2878 size: u64::try_from(obj.data.len())
2879 .map_err(|_| Error::CorruptObject("object size overflow".to_owned()))?,
2880 }));
2881 }
2882
2883 if let Some(info) = self.try_read_info_packed_in_batch_context(oid)? {
2884 return Ok(Some(info));
2885 }
2886
2887 let use_midx = self.multi_pack_index_reads_enabled();
2888 if use_midx {
2889 self.ensure_midx_prepared();
2890 }
2891
2892 let primary = self.primary_writable()?;
2893 let files_primary = self.primary().ok();
2894 let mut unreadable_local_loose = None;
2895 match primary.read_info(oid) {
2896 Ok(Some(info)) => return Ok(Some(info)),
2897 Ok(None) => {}
2898 Err(err)
2899 if files_primary
2900 .as_ref()
2901 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2902 {
2903 unreadable_local_loose = Some(err);
2904 }
2905 Err(err) => return Err(err),
2906 }
2907
2908 if let Some(info) = self.try_read_info_extra_sources(oid)? {
2909 return Ok(Some(info));
2910 }
2911
2912 if self.alternates_enabled {
2913 match self.sources()?.read_info(oid) {
2914 Ok(Some(info)) => return Ok(Some(info)),
2915 Ok(None) => {}
2916 Err(err) => return Err(err),
2917 }
2918 }
2919
2920 if let Some(err) = unreadable_local_loose {
2921 return Err(err);
2922 }
2923
2924 Ok(None)
2925 }
2926
2927 pub(crate) fn read_unless_larger(
2943 &self,
2944 oid: &ObjectId,
2945 limit: u64,
2946 cache_small: bool,
2947 ) -> Result<BoundedRead> {
2948 if cache_small {
2949 if let Some(read) = self.session_loose_reads.lock().ok().and_then(|cache| {
2950 cache.get(oid).map(|obj| {
2951 if obj.data.len() as u64 > limit {
2952 BoundedRead::Larger
2953 } else {
2954 BoundedRead::Object(obj.clone())
2955 }
2956 })
2957 }) {
2958 return Ok(read);
2959 }
2960 }
2961 if self.custom_primary.is_none() && !self.multi_pack_index_reads_enabled() {
2962 if let Some(read) =
2963 pack::pack_cache::try_read_object_via_single_pack_reader_unless_whole_larger(
2964 &self.objects_dir,
2965 oid,
2966 limit,
2967 )?
2968 {
2969 return Ok(read);
2970 }
2971 }
2972 if let Ok(files) = self.primary() {
2973 if self.multi_pack_index_reads_enabled() {
2974 self.ensure_midx_prepared();
2975 }
2976 match files.read_unless_larger(oid, limit)? {
2977 LocalSizedRead::Packed(Some(size)) if size > limit => {
2978 return Ok(BoundedRead::Larger);
2979 }
2980 LocalSizedRead::Packed(_) => return self.read(oid).map(BoundedRead::Object),
2981 LocalSizedRead::Loose(BoundedRead::Object(obj)) => {
2982 if cache_small && obj.data.len() <= SESSION_BOUNDED_LOOSE_CACHE_MAX {
2983 self.remember_session_loose_read(oid, &obj);
2984 }
2985 return Ok(BoundedRead::Object(obj));
2986 }
2987 LocalSizedRead::Loose(read) => return Ok(read),
2988 LocalSizedRead::Missing => {}
2989 }
2990 }
2991 let info = self.read_info(oid)?;
2992 if info.size > limit {
2993 return Ok(BoundedRead::Larger);
2994 }
2995 self.read(oid).map(BoundedRead::Object)
2996 }
2997
2998 pub fn open_stream(&self, oid: &ObjectId) -> Result<Option<ObjectStream<'static>>> {
3007 self.with_stream(oid, materialize_stream)
3008 }
3009
3010 pub fn with_stream<R>(
3027 &self,
3028 oid: &ObjectId,
3029 f: impl FnOnce(ObjectStream<'_>) -> Result<R>,
3030 ) -> Result<Option<R>> {
3031 use std::io::Cursor;
3032 if self.is_empty_tree(oid) {
3033 return f(ObjectStream {
3034 kind: ObjectKind::Tree,
3035 size: 0,
3036 reader: Box::new(std::io::empty()),
3037 })
3038 .map(Some);
3039 }
3040 if let Some(obj) = self.overlay_read(oid) {
3041 let size = u64::try_from(obj.data.len()).map_err(|_| {
3042 Error::CorruptObject(format!("object size overflow for {}", oid.to_hex()))
3043 })?;
3044 return f(ObjectStream {
3045 kind: obj.kind,
3046 size,
3047 reader: Box::new(Cursor::new(obj.data)),
3048 })
3049 .map(Some);
3050 }
3051 if self.multi_pack_index_reads_enabled() {
3052 self.ensure_midx_prepared();
3053 }
3054 self.sync_delta_base_cache_limit();
3055 let primary = self.primary_writable()?;
3056 let files_primary = self.primary().ok();
3057 let mut unreadable_local_loose = None;
3058 match primary.open_stream(oid) {
3059 Ok(Some(stream)) => return f(stream).map(Some),
3060 Ok(None) => {}
3061 Err(err)
3062 if files_primary
3063 .as_ref()
3064 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
3065 {
3066 unreadable_local_loose = Some(err);
3067 }
3068 Err(err) => return Err(err),
3069 }
3070 if let Some(stream) = self.try_open_stream_extra_sources(oid)? {
3071 return f(stream).map(Some);
3072 }
3073 if self.alternates_enabled {
3074 let sources = self.sources()?;
3075 let stream = sources.open_stream(oid)?;
3076 if let Some(stream) = stream {
3077 return f(stream).map(Some);
3078 }
3079 }
3080 if let Some(err) = unreadable_local_loose {
3081 return Err(err);
3082 }
3083 Ok(None)
3084 }
3085
3086 pub fn for_each_object(&self, f: &mut dyn FnMut(&ObjectId) -> ControlFlow<()>) -> Result<()> {
3092 let mut seen = oid_hash_set();
3093 let mut visit = |store: &dyn ObjectStore| -> Result<bool> {
3094 store::for_each_propagate_break(store, &mut |oid| {
3095 if !seen.insert(*oid) {
3096 return ControlFlow::Continue(());
3097 }
3098 f(oid)
3099 })
3100 };
3101 if let Some(overlay) = self.overlay_store() {
3102 if visit(overlay.as_ref())? {
3103 return Ok(());
3104 }
3105 }
3106 if visit(self.primary_writable()?.as_ref())? {
3107 return Ok(());
3108 }
3109 for src in &self.extra_read_sources {
3110 if visit(src.as_ref())? {
3111 return Ok(());
3112 }
3113 }
3114 if self.alternates_enabled {
3115 let _ = visit(self.sources()?.as_ref())?;
3116 }
3117 Ok(())
3118 }
3119
3120 pub fn lookup_prefix(&self, prefix: &str, limit: usize, out: &mut Vec<ObjectId>) -> Result<()> {
3126 let mut seen: OidHashSet = out.iter().copied().collect();
3127 if let Some(overlay) = self.overlay_store() {
3128 append_lookup_prefix_layer(overlay.as_ref(), prefix, limit, out, &mut seen)?;
3129 if limit != 0 && out.len() >= limit {
3130 return Ok(());
3131 }
3132 }
3133 append_lookup_prefix_layer(
3134 self.primary_writable()?.as_ref(),
3135 prefix,
3136 limit,
3137 out,
3138 &mut seen,
3139 )?;
3140 if limit != 0 && out.len() >= limit {
3141 return Ok(());
3142 }
3143 for src in &self.extra_read_sources {
3144 append_lookup_prefix_layer(src.as_ref(), prefix, limit, out, &mut seen)?;
3145 if limit != 0 && out.len() >= limit {
3146 return Ok(());
3147 }
3148 }
3149 if self.alternates_enabled {
3150 append_lookup_prefix_layer(self.sources()?.as_ref(), prefix, limit, out, &mut seen)?;
3151 }
3152 Ok(())
3153 }
3154
3155 pub fn enumerate_local_loose_objects(&self) -> Result<Vec<(ObjectId, PathBuf)>> {
3163 let primary = self.primary()?;
3164 let loose = primary.loose_store();
3165 let mut out = Vec::new();
3166 loose.for_each_object(&mut |oid| {
3167 out.push((*oid, loose.object_path(oid)));
3168 ControlFlow::Continue(())
3169 })?;
3170 Ok(out)
3171 }
3172
3173 pub fn hash(&self, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
3181 hash_object_data_with(self.hash_algo(), kind, data)
3182 }
3183
3184 pub fn hash_stream(
3195 &self,
3196 kind: ObjectKind,
3197 size: u64,
3198 reader: &mut dyn std::io::Read,
3199 ) -> Result<ObjectId> {
3200 #[cfg(test)]
3201 self.hot_path_metrics().record_stream_hash();
3202 stream::hash_stream(self.hash_algo(), kind, size, reader)
3203 }
3204
3205 pub fn write_stream(
3226 &self,
3227 kind: ObjectKind,
3228 size: u64,
3229 reader: &mut dyn std::io::Read,
3230 options: WriteOptions,
3231 ) -> Result<ObjectId> {
3232 #[cfg(test)]
3233 self.hot_path_metrics().record_stream_write();
3234 if let Some(primary) = &self.custom_primary {
3235 if !self.overlay_active() {
3236 let oid = primary.write_stream(kind, size, reader, options)?;
3237 self.note_commit_object_written(kind);
3238 return Ok(oid);
3239 }
3240 }
3241 if self.overlay_active() || size < self.big_file_threshold()? {
3242 let len = usize::try_from(size).map_err(|_| {
3243 Error::CorruptObject(format!("object size {size} exceeds address space"))
3244 })?;
3245 let mut data = Vec::with_capacity(len);
3246 let read =
3247 std::io::Read::read_to_end(&mut std::io::Read::take(reader, size), &mut data)?;
3248 if read != len {
3249 return Err(Error::ObjectStreamTruncated {
3250 expected: size,
3251 actual: read as u64,
3252 });
3253 }
3254 return self.write_with_options(kind, &data, options);
3255 }
3256 let staged = stream::StagedPack::write(
3257 &self.objects_dir.join("pack"),
3258 self.hash_algo(),
3259 kind,
3260 size,
3261 reader,
3262 self.pack_compression()?,
3263 )?;
3264 let oid = staged.oid();
3265 if self.object_exists_for_write_skip(&oid) {
3266 if !options.silent {
3267 let _ = self.freshen_object(&oid);
3268 }
3269 return Ok(oid);
3270 }
3271 staged.install()?;
3272 self.invalidate_packs();
3273 self.note_commit_object_written(kind);
3274 Ok(oid)
3275 }
3276
3277 pub fn empty_tree_oid(&self) -> Result<ObjectId> {
3285 self.hash(ObjectKind::Tree, b"")
3286 }
3287
3288 #[must_use]
3290 pub fn is_empty_tree(&self, oid: &ObjectId) -> bool {
3291 empty_tree_oid_matches(self.hash_algo(), oid)
3292 }
3293
3294 pub fn write(&self, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
3307 self.write_with_options(kind, data, WriteOptions::default())
3308 }
3309
3310 pub fn ensure_all_loose_prefix_dirs(&self) -> Result<()> {
3319 if self.custom_primary.is_some() {
3320 return Ok(());
3321 }
3322 self.local_loose()?.ensure_all_loose_prefix_dirs()
3323 }
3324
3325 #[cfg(test)]
3327 #[must_use]
3328 pub fn test_loose_object_path(&self, oid: &ObjectId) -> PathBuf {
3329 self.object_path(oid)
3330 }
3331
3332 #[cfg(test)]
3334 pub fn test_remove_loose_object(&self, oid: &ObjectId) {
3335 let _ = fs::remove_file(self.object_path(oid));
3336 }
3337
3338 pub fn count_loose_objects(&self) -> Result<usize> {
3344 let mut count = 0usize;
3345 for_each_loose_object_id(&self.objects_dir, self.hash_algo(), &mut |_| {
3346 count += 1;
3347 ControlFlow::Continue(())
3348 })?;
3349 Ok(count)
3350 }
3351
3352 pub fn approximate_object_count(&self) -> Result<usize> {
3362 let pack_objects: Result<usize> = self.with_pack_read_context(|| {
3363 let indexes = pack::read_local_pack_indexes_cached(self.objects_dir())?;
3364 Ok(indexes.iter().map(|idx| idx.len()).sum())
3365 });
3366 let mut count = pack_objects?;
3367 count += self.count_loose_objects()?;
3368 Ok(count)
3369 }
3370
3371 pub fn zlib_compress_loose_store(&self, store_bytes: &[u8]) -> Result<Vec<u8>> {
3377 self.local_loose()?.zlib_compress_store_bytes(store_bytes)
3378 }
3379
3380 pub fn write_loose_zlib_prehashed(
3394 &self,
3395 oid: &ObjectId,
3396 zlib_store: &[u8],
3397 options: WriteOptions,
3398 ) -> Result<ObjectId> {
3399 if self.custom_primary.is_some() {
3400 let raw = decompress_zlib_loose_bytes(zlib_store)?;
3401 let obj = parse_object_bytes(&raw)?;
3402 return self.write_with_options(obj.kind, &obj.data, options);
3403 }
3404 let primary = self.primary()?;
3405 if let Some(existing) = primary.try_finish_duplicate_write(oid, options)? {
3406 return Ok(existing);
3407 }
3408 if options.freshen_existing && self.freshen_packed_or_alternate(oid) {
3409 return Ok(*oid);
3410 }
3411
3412 let already_exists = !(options.trust_new_loose || options.assume_loose_only_existence)
3413 && self.object_exists_for_write_skip(oid);
3414
3415 if self.overlay_active() && !already_exists {
3416 if let Ok(raw) = decompress_zlib_loose_bytes(zlib_store) {
3417 if let Ok(obj) = parse_object_bytes(&raw) {
3418 if self.overlay_store_object(*oid, obj.kind, &obj.data) {
3419 return Ok(*oid);
3420 }
3421 }
3422 }
3423 }
3424
3425 if already_exists {
3426 if !options.silent {
3427 let _ = self.freshen_object(oid);
3428 }
3429 return Ok(*oid);
3430 }
3431
3432 primary.write_zlib_prehashed(oid, zlib_store, options)
3433 }
3434
3435 pub(crate) fn write_blob_prehashed(
3441 &self,
3442 oid: &ObjectId,
3443 data: &[u8],
3444 options: WriteOptions,
3445 ) -> Result<ObjectId> {
3446 if self.custom_primary.is_some() {
3447 return self.write_with_options(ObjectKind::Blob, data, options);
3448 }
3449 let primary = self.primary()?;
3450 if let Some(existing) = primary.try_finish_duplicate_write(oid, options)? {
3451 return Ok(existing);
3452 }
3453 if options.freshen_existing && self.freshen_packed_or_alternate(oid) {
3454 return Ok(*oid);
3455 }
3456 let already_exists = !(options.trust_new_loose || options.assume_loose_only_existence)
3457 && self.object_exists_for_write_skip(oid);
3458 if self.overlay_active()
3459 && !already_exists
3460 && self.overlay_store_object(*oid, ObjectKind::Blob, data)
3461 {
3462 return Ok(*oid);
3463 }
3464 if already_exists {
3465 if !options.silent {
3466 let _ = self.freshen_object(oid);
3467 }
3468 return Ok(*oid);
3469 }
3470 primary
3471 .loose_store()
3472 .write_prehashed(oid, ObjectKind::Blob, data, options)
3473 }
3474 pub fn write_with_options(
3476 &self,
3477 kind: ObjectKind,
3478 data: &[u8],
3479 options: WriteOptions,
3480 ) -> Result<ObjectId> {
3481 let oid = hash_object_data_with(self.hash_algo(), kind, data)?;
3482
3483 if let Some(primary) = &self.custom_primary {
3484 let already_exists = if options.trust_new_loose {
3485 false
3486 } else if options.assume_loose_only_existence {
3487 primary.contains(&oid)?
3488 } else {
3489 self.object_exists_for_write_skip(&oid)
3490 };
3491 if self.overlay_active()
3492 && !already_exists
3493 && self.overlay_store_object(oid, kind, data)
3494 {
3495 return Ok(oid);
3496 }
3497 if already_exists {
3498 return Ok(oid);
3499 }
3500 let oid = WritableObjectStore::write(primary.as_ref(), kind, data, options)?;
3501 self.note_commit_object_written(kind);
3502 return Ok(oid);
3503 }
3504
3505 let primary = self.primary()?;
3506 if let Some(existing) = primary.try_finish_duplicate_write(&oid, options)? {
3507 return Ok(existing);
3508 }
3509 if options.freshen_existing && self.freshen_packed_or_alternate(&oid) {
3510 return Ok(oid);
3511 }
3512
3513 let already_exists = !(options.trust_new_loose || options.assume_loose_only_existence)
3514 && self.object_exists_for_write_skip(&oid);
3515
3516 if self.overlay_active() && !already_exists && self.overlay_store_object(oid, kind, data) {
3519 return Ok(oid);
3520 }
3521
3522 if already_exists {
3523 if !options.silent {
3524 let _ = self.freshen_object(&oid);
3525 }
3526 return Ok(oid);
3527 }
3528
3529 let oid = WritableObjectStore::write(primary.as_ref(), kind, data, options)?;
3530 self.note_commit_object_written(kind);
3531 Ok(oid)
3532 }
3533
3534 pub fn write_local(&self, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
3550 if self.custom_primary.is_some() {
3551 return Err(Error::UnsupportedObjectStore {
3552 operation: "write_local",
3553 });
3554 }
3555 let oid = hash_object_data_with(self.hash_algo(), kind, data)?;
3556
3557 if let Some(existing) = self.try_freshen_existing_local(&oid) {
3558 return Ok(existing);
3559 }
3560
3561 let oid = self
3562 .local_loose()?
3563 .write(kind, data, WriteOptions::default())?;
3564 self.note_commit_object_written(kind);
3565 Ok(oid)
3566 }
3567
3568 pub(crate) fn write_local_stream_with(
3570 &self,
3571 kind: ObjectKind,
3572 size: u64,
3573 supply: impl FnMut(&mut dyn FnMut(&[u8]) -> Result<()>) -> Result<()>,
3574 ) -> Result<ObjectId> {
3575 if self.custom_primary.is_some() {
3576 return Err(Error::UnsupportedObjectStore {
3577 operation: "write_local_stream",
3578 });
3579 }
3580 let loose = self.local_loose()?;
3581 let oid = loose.write_payload_stream_with(kind, size, WriteOptions::default(), supply)?;
3582 self.note_commit_object_written(kind);
3583 Ok(oid)
3584 }
3585
3586 pub fn write_loose_materialize(&self, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
3593 if self.custom_primary.is_some() {
3594 return Err(Error::UnsupportedObjectStore {
3595 operation: "write_loose_materialize",
3596 });
3597 }
3598 let oid = hash_object_data_with(self.hash_algo(), kind, data)?;
3599 if let Ok(primary) = self.primary() {
3600 if let Ok(Some(existing)) = primary
3601 .loose_store()
3602 .try_finish_if_loose_present(&oid, WriteOptions::default())
3603 {
3604 return Ok(existing);
3605 }
3606 }
3607
3608 self.local_loose()?
3609 .write(kind, data, WriteOptions::default())
3610 }
3611
3612 pub fn write_raw(&self, store_bytes: &[u8]) -> Result<ObjectId> {
3622 let obj = parse_object_bytes(store_bytes)?;
3623 let oid = hash_object_data_with(self.hash_algo(), obj.kind, &obj.data)?;
3624
3625 if let Some(primary) = &self.custom_primary {
3626 if self.exists(&oid) {
3627 return Ok(oid);
3628 }
3629 if self.overlay_active() && self.overlay_store_object(oid, obj.kind, &obj.data) {
3630 return Ok(oid);
3631 }
3632 return WritableObjectStore::write(
3633 primary.as_ref(),
3634 obj.kind,
3635 &obj.data,
3636 WriteOptions::default(),
3637 );
3638 }
3639
3640 if let Some(existing) = self.try_freshen_existing(&oid) {
3641 return Ok(existing);
3642 }
3643
3644 self.local_loose()?
3645 .write_store_prehashed(&oid, store_bytes, WriteOptions::default())
3646 }
3647
3648 pub fn write_raw_local(&self, store_bytes: &[u8]) -> Result<ObjectId> {
3656 if self.custom_primary.is_some() {
3657 return Err(Error::UnsupportedObjectStore {
3658 operation: "write_raw_local",
3659 });
3660 }
3661 let obj = parse_object_bytes(store_bytes)?;
3662 let oid = hash_object_data_with(self.hash_algo(), obj.kind, &obj.data)?;
3663 if let Some(existing) = self.try_freshen_existing_local(&oid) {
3664 return Ok(existing);
3665 }
3666
3667 self.local_loose()?
3668 .write_store_prehashed(&oid, store_bytes, WriteOptions::default())
3669 }
3670
3671 fn touch_object_mtime(&self, path: &Path) -> Option<std::time::SystemTime> {
3673 #[cfg(test)]
3674 self.hot_path_test_metrics.record_freshen();
3675 crate::fs_readonly::set_file_times_now(path)
3676 }
3677}
3678
3679fn hash_object_data_with(algo: HashAlgo, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
3682 hash::hash_object(algo, kind, data)
3683}
3684
3685#[must_use]
3687pub(crate) fn empty_tree_oid_matches(algo: HashAlgo, oid: &ObjectId) -> bool {
3688 oid.is_empty_tree_for(algo)
3689}
3690
3691pub(crate) fn zlib_compress_blob_loose(
3694 data: &[u8],
3695 compression: flate2::Compression,
3696) -> Result<Vec<u8>> {
3697 use std::io::Write as _;
3698 let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), compression);
3699 let zlib_err = |e: std::io::Error| Error::Zlib(e.to_string());
3700 write!(encoder, "{} {}\0", ObjectKind::Blob, data.len()).map_err(zlib_err)?;
3701 encoder.write_all(data).map_err(zlib_err)?;
3702 encoder.finish().map_err(zlib_err)
3703}
3704
3705fn append_lookup_prefix_layer(
3706 store: &dyn ObjectStore,
3707 prefix: &str,
3708 limit: usize,
3709 out: &mut Vec<ObjectId>,
3710 seen: &mut OidHashSet,
3711) -> Result<()> {
3712 let mut scratch = Vec::new();
3713 store.lookup_prefix(prefix, 0, &mut scratch)?;
3714 for oid in scratch {
3715 if seen.insert(oid) {
3716 out.push(oid);
3717 if limit != 0 && out.len() >= limit {
3718 return Ok(());
3719 }
3720 }
3721 }
3722 Ok(())
3723}
3724
3725fn materialize_stream(stream: ObjectStream<'_>) -> Result<ObjectStream<'static>> {
3726 use std::io::{Cursor, Read};
3727 let ObjectStream {
3728 kind,
3729 size,
3730 mut reader,
3731 } = stream;
3732 let mut payload = Vec::new();
3733 reader.read_to_end(&mut payload).map_err(Error::Io)?;
3734 Ok(ObjectStream {
3735 kind,
3736 size,
3737 reader: Box::new(Cursor::new(payload)),
3738 })
3739}
3740
3741fn parse_alternate_env(val: &str) -> Vec<PathBuf> {
3744 let mut result = Vec::new();
3745 let mut chars = val.chars().peekable();
3746 while chars.peek().is_some() {
3747 if chars.peek() == Some(&':') {
3748 chars.next();
3749 continue;
3750 }
3751 if chars.peek() == Some(&'"') {
3752 chars.next(); let saved: Vec<char> = chars.clone().collect();
3756 let mut path = String::new();
3757 let mut properly_closed = false;
3758 loop {
3759 match chars.next() {
3760 None => break,
3761 Some('"') => {
3762 properly_closed = true;
3763 break;
3764 }
3765 Some('\\') => match chars.peek() {
3766 Some(c) if c.is_ascii_digit() => {
3767 let mut oct = String::new();
3768 for _ in 0..3 {
3769 if let Some(&c) = chars.peek() {
3770 if c.is_ascii_digit() {
3771 oct.push(c);
3772 chars.next();
3773 } else {
3774 break;
3775 }
3776 } else {
3777 break;
3778 }
3779 }
3780 if let Ok(byte) = u8::from_str_radix(&oct, 8) {
3781 path.push(byte as char);
3782 }
3783 }
3784 Some(_) => {
3785 if let Some(c) = chars.next() {
3786 match c {
3787 'n' => path.push('\n'),
3788 't' => path.push('\t'),
3789 'r' => path.push('\r'),
3790 _ => path.push(c),
3791 }
3792 }
3793 }
3794 None => {}
3795 },
3796 Some(c) => path.push(c),
3797 }
3798 }
3799 if !properly_closed {
3800 let raw: String = std::iter::once('"').chain(saved).collect();
3803 let raw_path = raw.split(':').next().unwrap_or(&raw);
3805 if !raw_path.is_empty() {
3806 result.push(PathBuf::from(raw_path));
3807 }
3808 let remainder = &raw[raw_path.len()..];
3812 if let Some(rest) = remainder.strip_prefix(':') {
3813 result.extend(parse_alternate_env(rest));
3815 }
3816 return result;
3817 } else if !path.is_empty() {
3818 result.push(PathBuf::from(path));
3819 }
3820 } else {
3821 let mut path = String::new();
3822 while let Some(&c) = chars.peek() {
3823 if c == ':' {
3824 break;
3825 }
3826 path.push(c);
3827 chars.next();
3828 }
3829 if !path.is_empty() {
3830 result.push(PathBuf::from(path));
3831 }
3832 }
3833 }
3834 result
3835}
3836
3837fn system_time_to_file_time(t: std::time::SystemTime) -> FileTime {
3838 let dur = t
3839 .duration_since(std::time::SystemTime::UNIX_EPOCH)
3840 .unwrap_or_default();
3841 FileTime::from_unix_time(
3842 i64::try_from(dur.as_secs()).unwrap_or(i64::MAX),
3843 dur.subsec_nanos(),
3844 )
3845}
3846
3847fn superseded_pack_indexes(
3848 pack_dir: &Path,
3849 keep_pack: &Path,
3850 retained_packs: &HashSet<PathBuf>,
3851 inventory: &RepackPackInventorySnapshot,
3852) -> Result<Vec<(PathBuf, std::time::SystemTime)>> {
3853 let keep_stem = keep_pack
3854 .file_stem()
3855 .and_then(|s| s.to_str())
3856 .ok_or_else(|| {
3857 Error::Io(std::io::Error::new(
3858 std::io::ErrorKind::InvalidInput,
3859 "bad pack path",
3860 ))
3861 })?;
3862 if !pack_dir.is_dir() {
3863 return Ok(Vec::new());
3864 }
3865 let mut out = Vec::new();
3866 for entry in fs::read_dir(pack_dir).map_err(Error::Io)? {
3867 let entry = entry.map_err(Error::Io)?;
3868 let name = entry.file_name();
3869 let name_str = name.to_string_lossy();
3870 if name_str == keep_stem || name_str.starts_with(&format!("{keep_stem}.")) {
3871 continue;
3872 }
3873 if !name_str.ends_with(".idx") {
3874 continue;
3875 }
3876 let idx_path = entry.path();
3877 let pack_path = idx_path.with_extension("pack");
3878 if !pack_path.is_file() {
3879 continue;
3880 }
3881 if inventory.kept_at_inventory.contains(&pack_path)
3882 || pack_path.with_extension("promisor").is_file()
3883 || retained_packs.contains(&pack_path)
3884 || !inventory.pack_paths.contains(&pack_path)
3885 {
3886 continue;
3887 }
3888 let pack_mtime = fs::metadata(&pack_path)
3889 .and_then(|m| m.modified())
3890 .unwrap_or(std::time::SystemTime::UNIX_EPOCH);
3891 out.push((idx_path, pack_mtime));
3892 }
3893 Ok(out)
3894}
3895
3896fn oids_in_inventory_kept_packs(
3897 pack_dir: &Path,
3898 keep_pack: &Path,
3899 inventory_kept_packs: &HashSet<PathBuf>,
3900) -> Result<OidHashSet> {
3901 let keep_stem = keep_pack.file_stem().and_then(|s| s.to_str()).unwrap_or("");
3902 let mut oids = oid_hash_set();
3903 for pack_path in inventory_kept_packs {
3904 let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str()) else {
3905 continue;
3906 };
3907 if stem == keep_stem {
3908 continue;
3909 }
3910 if pack_path.parent().is_some_and(|p| p != pack_dir) {
3911 continue;
3912 }
3913 let idx_path = pack_path.with_extension("idx");
3914 if idx_path.is_file() {
3915 oids.extend(PackedObjects::object_ids_from_index_path(&idx_path)?);
3916 }
3917 }
3918 Ok(oids)
3919}
3920
3921#[cfg(test)]
3922mod tests {
3923 #![allow(clippy::expect_used, clippy::unwrap_used)]
3924
3925 use super::*;
3926 use tempfile::TempDir;
3927
3928 #[test]
3929 fn round_trip_blob() {
3930 let dir = TempDir::new().unwrap();
3931 let odb = Odb::new(dir.path());
3932 let data = b"hello world";
3933 let oid = odb.write(ObjectKind::Blob, data).unwrap();
3934 let obj = odb.read(&oid).unwrap();
3935 assert_eq!(obj.kind, ObjectKind::Blob);
3936 assert_eq!(obj.data, data);
3937 }
3938
3939 #[test]
3940 fn known_blob_hash() {
3941 let oid = HashAlgo::Sha1
3944 .hash_object(ObjectKind::Blob, b"hello")
3945 .expect("hash");
3946 assert_eq!(oid.to_hex(), "b6fc4c620b67d95f953a5c1c1230aaab5db5a1b0");
3947 }
3948
3949 fn pack_objects(objects: &Path, oids: &[ObjectId]) {
3951 let odb = Odb::new(objects);
3952 let pack = crate::pack_objects::build_pack_from_send_list(
3953 &odb,
3954 oids,
3955 &HashSet::new(),
3956 &crate::pack_objects::PackBuildOptions::default(),
3957 &HashMap::new(),
3958 )
3959 .unwrap();
3960 for oid in oids {
3961 fs::remove_file(oid.loose_path_in(objects)).unwrap();
3962 }
3963 crate::index_pack::install_pack_bytes(
3964 pack,
3965 &odb,
3966 &crate::index_pack::IngestPackOptions::default(),
3967 )
3968 .unwrap();
3969 }
3970
3971 #[test]
3972 fn general_reads_cache_trees_and_tiny_blobs_but_not_larger_blobs() {
3973 let dir = TempDir::new().unwrap();
3974 let odb = Odb::new(dir.path());
3975 let blob = odb
3976 .write(
3977 ObjectKind::Blob,
3978 &vec![b'x'; SESSION_READ_BLOB_CACHE_MAX + 1],
3979 )
3980 .unwrap();
3981 let tiny = odb.write(ObjectKind::Blob, b"tiny\n").unwrap();
3982 let tree = odb
3983 .write(
3984 ObjectKind::Tree,
3985 &crate::objects::serialize_tree(&[crate::objects::TreeEntry {
3986 mode: 0o100644,
3987 name: b"f".to_vec(),
3988 oid: blob,
3989 }]),
3990 )
3991 .unwrap();
3992 assert_eq!(
3993 odb.read(&blob).unwrap().data.len(),
3994 SESSION_READ_BLOB_CACHE_MAX + 1
3995 );
3996 assert!(odb.session_loose_reads.lock().unwrap().is_empty());
3997 let tree_bytes = odb.read(&tree).unwrap().data;
3998 assert_eq!(odb.read(&tiny).unwrap().data, b"tiny\n");
3999 for oid in [blob, tree, tiny] {
4000 let hex = oid.to_hex();
4001 fs::remove_file(dir.path().join(&hex[..2]).join(&hex[2..])).unwrap();
4002 }
4003 assert_eq!(odb.read(&tree).unwrap().data, tree_bytes, "tree reused");
4004 assert_eq!(odb.read(&tiny).unwrap().data, b"tiny\n", "tiny blob reused");
4005 assert!(odb.read(&blob).is_err(), "blob was not retained");
4006 }
4007
4008 #[test]
4009 fn session_read_cache_is_bounded_by_bytes_and_entries() {
4010 let mut cache = SessionLooseReads::default();
4011 let chunk = SESSION_LOOSE_CACHE_MAX_BYTES / 4;
4012 let object = |len: usize| Object {
4013 kind: ObjectKind::Tree,
4014 data: vec![0; len],
4015 };
4016 let oid = |n: u32| {
4017 let mut raw = [0u8; 20];
4018 raw[..4].copy_from_slice(&n.to_be_bytes());
4019 ObjectId::from_bytes(&raw).unwrap()
4020 };
4021 for n in 0..4 {
4022 cache.insert(oid(n), &object(chunk));
4023 }
4024 assert_eq!(cache.bytes, SESSION_LOOSE_CACHE_MAX_BYTES);
4025 cache.insert(oid(4), &object(1));
4026 assert_eq!(cache.bytes, 1, "exceeding the byte budget starts over");
4027 assert!(cache.get(&oid(0)).is_none());
4028 cache.insert(oid(4), &object(10));
4029 assert_eq!(cache.bytes, 10, "replacing an entry drops its old size");
4030 cache.insert(oid(5), &object(SESSION_LOOSE_CACHE_MAX_BYTES + 1));
4031 assert!(
4032 cache.get(&oid(5)).is_none(),
4033 "oversized objects are never kept"
4034 );
4035 assert!(cache.get(&oid(4)).is_some());
4036 for n in 10..(10 + SESSION_LOOSE_CACHE_MAX_ENTRIES as u32 + 2) {
4037 cache.insert(oid(n), &object(1));
4038 }
4039 assert!(cache.objects.len() <= SESSION_LOOSE_CACHE_MAX_ENTRIES + 1);
4040 assert!(cache.bytes <= SESSION_LOOSE_CACHE_MAX_BYTES);
4041 }
4042
4043 fn bounded_read_fixture() -> (TempDir, PathBuf, ObjectId, ObjectId) {
4044 let dir = TempDir::new().unwrap();
4045 let objects = dir.path().to_path_buf();
4046 let odb = Odb::new(&objects);
4047 let small = odb.write(ObjectKind::Blob, b"small blob\n").unwrap();
4048 let big: Vec<u8> = (0..300_000u32).flat_map(|i| i.to_le_bytes()).collect();
4049 let big = odb.write(ObjectKind::Blob, &big).unwrap();
4050 (dir, objects, small, big)
4051 }
4052
4053 fn assert_bounded_reads(odb: &Odb, small: &ObjectId, big: &ObjectId) {
4054 match odb.read_unless_larger(small, 1000, false).unwrap() {
4055 BoundedRead::Object(obj) => assert_eq!(obj.data, b"small blob\n"),
4056 BoundedRead::Larger => panic!("small blob is within the limit"),
4057 }
4058 assert!(matches!(
4059 odb.read_unless_larger(big, 1000, false).unwrap(),
4060 BoundedRead::Larger
4061 ));
4062 match odb.read_unless_larger(big, 1_200_000, false).unwrap() {
4063 BoundedRead::Object(obj) => assert_eq!(obj.data.len(), 1_200_000),
4064 BoundedRead::Larger => panic!("limit equal to the size reads the object"),
4065 }
4066 let missing = ObjectId::from_hex("1234567890123456789012345678901234567890").unwrap();
4067 assert!(matches!(
4068 odb.read_unless_larger(&missing, 1000, false),
4069 Err(Error::ObjectNotFound(_))
4070 ));
4071 }
4072
4073 #[test]
4074 fn read_unless_larger_sizes_loose_objects_from_their_header() {
4075 let (_dir, objects, small, big) = bounded_read_fixture();
4076 assert_bounded_reads(&Odb::new(&objects), &small, &big);
4077 }
4078
4079 #[test]
4080 fn read_unless_larger_sizes_packed_objects_in_and_out_of_read_context() {
4081 let (_dir, objects, small, big) = bounded_read_fixture();
4082 pack_objects(&objects, &[small, big]);
4083 let odb = Odb::new(&objects);
4084 assert_bounded_reads(&odb, &small, &big);
4085 odb.with_pack_read_context(|| assert_bounded_reads(&odb, &small, &big));
4086 }
4087
4088 #[test]
4089 fn parsed_tree_reads_keep_no_raw_copy_in_the_session_cache() {
4090 let (_dir, objects, small, _big) = bounded_read_fixture();
4091 let odb = Odb::new(&objects);
4092 let entries = [crate::objects::TreeEntry {
4093 mode: 0o100644,
4094 name: b"small".to_vec(),
4095 oid: small,
4096 }];
4097 let tree = odb
4098 .write(ObjectKind::Tree, &crate::objects::serialize_tree(&entries))
4099 .unwrap();
4100 for packed in [false, true] {
4101 if packed {
4102 pack_objects(&objects, &[tree]);
4103 }
4104 let odb = Odb::new(&objects);
4105 assert_eq!(*odb.cached_tree_entries(&tree).unwrap(), entries);
4106 assert!(odb.session_cached_loose_read(&tree).is_none());
4107 assert!(Arc::ptr_eq(
4108 &odb.cached_tree_entries(&tree).unwrap(),
4109 &odb.cached_tree_entries(&tree).unwrap()
4110 ));
4111 assert_eq!(odb.read(&tree).unwrap().kind, ObjectKind::Tree);
4112 assert!(
4113 odb.session_cached_loose_read(&tree).is_some(),
4114 "plain reads still use the session cache (packed: {packed})"
4115 );
4116 }
4117 }
4118
4119 #[test]
4120 fn read_unless_larger_reuses_small_loose_reads_but_never_caches_big_ones() {
4121 let (_dir, objects, small, big) = bounded_read_fixture();
4122 let odb = Odb::new(&objects);
4123 assert_bounded_reads(&odb, &small, &big);
4124 assert!(odb.session_loose_reads.lock().unwrap().is_empty());
4125 assert!(matches!(
4126 odb.read_unless_larger(&small, 1000, true).unwrap(),
4127 BoundedRead::Object(_)
4128 ));
4129 assert!(matches!(
4130 odb.read_unless_larger(&big, 1_200_000, true).unwrap(),
4131 BoundedRead::Object(_)
4132 ));
4133 for oid in [small, big] {
4134 let hex = oid.to_hex();
4135 fs::remove_file(objects.join(&hex[..2]).join(&hex[2..])).unwrap();
4136 }
4137 match odb.read_unless_larger(&small, 1000, true).unwrap() {
4138 BoundedRead::Object(obj) => assert_eq!(obj.data, b"small blob\n"),
4139 BoundedRead::Larger => panic!("small blob is within the limit"),
4140 }
4141 assert!(matches!(
4142 odb.read_unless_larger(&small, 4, true),
4143 Ok(BoundedRead::Larger)
4144 ));
4145 assert!(matches!(
4146 odb.read_unless_larger(&big, 1_200_000, true),
4147 Err(Error::ObjectNotFound(_))
4148 ));
4149 }
4150
4151 #[test]
4152 fn read_unless_larger_finds_objects_across_several_packs() {
4153 let (_dir, objects, small, big) = bounded_read_fixture();
4154 pack_objects(&objects, &[small]);
4155 pack_objects(&objects, &[big]);
4156 let odb = Odb::new(&objects);
4157 odb.with_pack_read_context(|| assert_bounded_reads(&odb, &small, &big));
4158 }
4159
4160 #[test]
4161 fn parse_object_header_prefix_unknown_kind_is_typed_error() {
4162 let err = crate::odb::store::loose::parse_object_header_prefix(b"not-a-git-kind 0\0")
4163 .unwrap_err();
4164 assert!(matches!(err, Error::UnknownObjectType(_)));
4165 }
4166
4167 fn git_dir_with_config(config: &str) -> (TempDir, PathBuf, PathBuf) {
4168 let dir = TempDir::new().unwrap();
4169 let git_dir = dir.path().join(".git");
4170 let objects = git_dir.join("objects");
4171 fs::create_dir_all(&objects).unwrap();
4172 fs::write(git_dir.join("config"), config).unwrap();
4173 (dir, git_dir, objects)
4174 }
4175
4176 fn zlib_flevel(compressed: &[u8]) -> u8 {
4178 assert!(compressed.len() >= 2);
4179 (compressed[1] >> 6) & 3
4180 }
4181
4182 fn tmp_loose_object_paths(objects: &Path) -> Vec<PathBuf> {
4183 let mut tmp = Vec::new();
4184 let Ok(shards) = fs::read_dir(objects) else {
4185 return tmp;
4186 };
4187 for shard in shards.flatten() {
4188 if !shard.file_type().is_ok_and(|t| t.is_dir()) {
4189 continue;
4190 }
4191 let Ok(files) = fs::read_dir(shard.path()) else {
4192 continue;
4193 };
4194 for file in files.flatten() {
4195 let name = file.file_name();
4196 if name.to_string_lossy().starts_with("tmp_") {
4197 tmp.push(file.path());
4198 }
4199 }
4200 }
4201 tmp
4202 }
4203
4204 #[test]
4205 fn loose_write_honours_level() {
4206 let payload: Vec<u8> = (0..4096).map(|i| (i % 251) as u8).collect();
4207
4208 let (_dir, git_dir, objects) = git_dir_with_config("");
4209 let odb = Odb::new(&objects).with_config_git_dir(git_dir.clone());
4210 let oid_default = odb.write(ObjectKind::Blob, &payload).unwrap();
4211 let bytes_default = fs::read(odb.object_path(&oid_default)).unwrap();
4212 assert!(zlib_flevel(&bytes_default) <= 1);
4213 assert_eq!(odb.read(&oid_default).unwrap().data, payload);
4214
4215 for level in -1i32..=9 {
4216 let config = format!("[core]\n\tloosecompression = {level}\n");
4217 let (_dir, git_dir, objects) = git_dir_with_config(&config);
4218 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
4219 let byte = u8::try_from(level.rem_euclid(256)).unwrap_or(0);
4220 let bytes: Vec<u8> = vec![byte; 512];
4221 let oid = odb.write(ObjectKind::Blob, &bytes).unwrap();
4222 let on_disk = fs::read(odb.object_path(&oid)).unwrap();
4223 assert!(on_disk.len() >= 2, "level {level}");
4224 assert_eq!(odb.read(&oid).unwrap().data, bytes.as_slice());
4225 }
4226
4227 let config = "[core]\n\tcompression = 9\n";
4228 let (_dir, git_dir, objects) = git_dir_with_config(config);
4229 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
4230 let oid = odb.write(ObjectKind::Blob, &payload).unwrap();
4231 assert_eq!(zlib_flevel(&fs::read(odb.object_path(&oid)).unwrap()), 3);
4232
4233 let config = "[core]\n\tcompression = 0\n\tloosecompression = 1\n";
4234 let (_dir, git_dir, objects) = git_dir_with_config(config);
4235 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
4236 let oid = odb.write(ObjectKind::Blob, &payload).unwrap();
4237 assert_eq!(zlib_flevel(&fs::read(odb.object_path(&oid)).unwrap()), 0);
4238
4239 let config = "[core]\n\tloosecompression = 99\n";
4240 let (_dir, git_dir, objects) = git_dir_with_config(config);
4241 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
4242 assert!(odb.write(ObjectKind::Blob, b"x").is_err());
4243 assert!(tmp_loose_object_paths(&objects).is_empty());
4244 }
4245
4246 fn git_run(dir: &Path, args: &[&str]) -> std::process::Output {
4247 std::process::Command::new("git") .current_dir(dir)
4249 .args(args)
4250 .env("GIT_CONFIG_GLOBAL", "/dev/null")
4251 .env("GIT_CONFIG_SYSTEM", "/dev/null")
4252 .env("GIT_AUTHOR_NAME", "T")
4253 .env("GIT_AUTHOR_EMAIL", "t@example.com")
4254 .env("GIT_COMMITTER_NAME", "T")
4255 .env("GIT_COMMITTER_EMAIL", "t@example.com")
4256 .output()
4257 .unwrap()
4258 }
4259
4260 fn git_in(dir: &Path, args: &[&str]) {
4261 let out = git_run(dir, args);
4262 assert!(
4263 out.status.success(),
4264 "git {:?}: {}",
4265 args,
4266 String::from_utf8_lossy(&out.stderr)
4267 );
4268 }
4269
4270 fn git_rev_parse_oid(dir: &Path, spec: &str) -> ObjectId {
4271 let out = git_run(dir, &["rev-parse", spec]);
4272 assert!(
4273 out.status.success(),
4274 "git rev-parse {spec}: {}",
4275 String::from_utf8_lossy(&out.stderr)
4276 );
4277 ObjectId::from_hex(std::str::from_utf8(&out.stdout).unwrap().trim()).unwrap()
4278 }
4279
4280 #[test]
4281 fn write_loose_zlib_prehashed_matches_write() {
4282 let dir = TempDir::new().unwrap();
4283 let objects = dir.path().join("objects");
4284 let odb = Odb::new(&objects);
4285 let data = b"parallel prehash path";
4286 let oid_normal = odb.write(ObjectKind::Blob, data).unwrap();
4287
4288 let dir2 = TempDir::new().unwrap();
4289 let objects2 = dir2.path().join("objects");
4290 let odb2 = Odb::new(&objects2);
4291 let oid = hash::hash_object(odb2.hash_algo(), ObjectKind::Blob, data).expect("hash");
4292 assert_eq!(oid, oid_normal);
4293 let zlib =
4294 super::zlib_compress_blob_loose(data, odb2.loose_compression().unwrap()).unwrap();
4295 let oid2 = odb2
4296 .write_loose_zlib_prehashed(
4297 &oid,
4298 &zlib,
4299 WriteOptions {
4300 assume_loose_only_existence: true,
4301 ..WriteOptions::default()
4302 },
4303 )
4304 .unwrap();
4305 assert_eq!(oid2, oid_normal);
4306 assert_eq!(
4307 fs::read(odb2.object_path(&oid2)).unwrap(),
4308 fs::read(odb.object_path(&oid_normal)).unwrap()
4309 );
4310 }
4311
4312 #[test]
4313 fn write_blob_prehashed_matches_write_and_skips_existing_objects() {
4314 let dir = TempDir::new().unwrap();
4315 let odb = Odb::new(&dir.path().join("objects"));
4316 let data = vec![7u8; 100_000];
4317 let oid_normal = odb.write(ObjectKind::Blob, &data).unwrap();
4318
4319 let dir2 = TempDir::new().unwrap();
4320 let odb2 = Odb::new(&dir2.path().join("objects"));
4321 let oid = hash::hash_object(odb2.hash_algo(), ObjectKind::Blob, &data).unwrap();
4322 let options = WriteOptions {
4323 assume_loose_only_existence: true,
4324 ..WriteOptions::default()
4325 };
4326 assert_eq!(
4327 odb2.write_blob_prehashed(&oid, &data, options).unwrap(),
4328 oid
4329 );
4330 assert_eq!(oid, oid_normal);
4331 assert!(odb2.object_path(&oid).is_file());
4332 let obj = odb2.read(&oid).unwrap();
4333 assert_eq!(
4334 (obj.kind, obj.data.as_slice()),
4335 (ObjectKind::Blob, data.as_slice())
4336 );
4337
4338 assert_eq!(
4340 odb2.write_blob_prehashed(&oid, b"other", options).unwrap(),
4341 oid
4342 );
4343 assert_eq!(odb2.read(&oid).unwrap().data, data);
4344 }
4345
4346 #[test]
4347 fn write_loose_zlib_prehashed_routes_to_custom_primary() {
4348 use crate::objects::HashAlgo;
4349 use std::sync::Arc;
4350
4351 let dir = TempDir::new().unwrap();
4352 let objects = dir.path().join("objects");
4353 fs::create_dir_all(&objects).unwrap();
4354 let store = Arc::new(MemoryStore::new(HashAlgo::Sha1));
4355 let odb = OdbBuilder::files(&objects)
4356 .primary(store.clone())
4357 .alternates(false)
4358 .build();
4359 let data = b"staged through a custom primary";
4360 let oid = hash::hash_object(HashAlgo::Sha1, ObjectKind::Blob, data).unwrap();
4361 let zlib = super::zlib_compress_blob_loose(data, Compression::fast()).unwrap();
4362
4363 odb.ensure_all_loose_prefix_dirs().unwrap();
4364 let written = odb
4365 .write_loose_zlib_prehashed(&oid, &zlib, WriteOptions::default())
4366 .unwrap();
4367 assert_eq!(written, oid);
4368 assert!(store.contains(&oid).unwrap());
4369 assert_eq!(odb.read(&oid).unwrap().data, data);
4370 assert_eq!(
4371 fs::read_dir(&objects).unwrap().count(),
4372 0,
4373 "no loose shard directories or files beside a custom primary"
4374 );
4375
4376 assert!(odb
4377 .write_loose_zlib_prehashed(&oid, b"not zlib", WriteOptions::default())
4378 .is_err());
4379 }
4380
4381 #[test]
4384 fn write_existing_loose_does_not_rewrite() {
4385 let dir = TempDir::new().unwrap();
4386 let objects = dir.path().join("objects");
4387 fs::create_dir_all(&objects).unwrap();
4388 let odb = Odb::new(&objects);
4389 let payload = b"already loose";
4390 let oid = odb.write(ObjectKind::Blob, payload).unwrap();
4391 let path = odb.object_path(&oid);
4392 let before_bytes = fs::read(&path).unwrap();
4393 let meta_before = fs::metadata(&path).unwrap();
4394 let before_mtime = filetime::FileTime::from_last_modification_time(&meta_before);
4395 #[cfg(unix)]
4396 let ino_before = {
4397 use std::os::unix::fs::MetadataExt;
4398 meta_before.ino()
4399 };
4400 std::thread::sleep(std::time::Duration::from_millis(20));
4401 let oid2 = odb.write(ObjectKind::Blob, payload).unwrap();
4402 assert_eq!(oid, oid2);
4403 assert_eq!(fs::read(&path).unwrap(), before_bytes);
4404 let meta_after = fs::metadata(&path).unwrap();
4405 let after_mtime = filetime::FileTime::from_last_modification_time(&meta_after);
4406 assert!(
4407 after_mtime >= before_mtime,
4408 "expected freshen to bump or preserve mtime"
4409 );
4410 #[cfg(unix)]
4411 {
4412 use std::os::unix::fs::MetadataExt;
4413 assert_eq!(
4414 meta_after.ino(),
4415 ino_before,
4416 "rewrite must not replace the loose object file"
4417 );
4418 }
4419 }
4420
4421 #[cfg_attr(
4423 not(feature = "full-tests"),
4424 ignore = "starts system git or is heavy; run with --features full-tests"
4425 )]
4426 #[test]
4427 fn write_skips_loose_when_object_in_pack_only() {
4428 let dir = TempDir::new().unwrap();
4429 git_in(dir.path(), &["init", "-q"]);
4430 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
4431 git_in(dir.path(), &["config", "user.name", "T"]);
4432 fs::write(dir.path().join("f"), b"pack-only payload").unwrap();
4433 git_in(dir.path(), &["add", "f"]);
4434 git_in(dir.path(), &["commit", "-m", "c"]);
4435 git_in(dir.path(), &["repack", "-ad"]);
4436 let objects = dir.path().join(".git").join("objects");
4437 let odb = Odb::new(&objects);
4438 let oid = odb.write(ObjectKind::Blob, b"pack-only payload").unwrap();
4439 assert!(!odb.object_path(&oid).exists());
4440 assert!(odb.exists(&oid));
4441 }
4442
4443 #[cfg_attr(
4445 not(feature = "full-tests"),
4446 ignore = "starts system git or is heavy; run with --features full-tests"
4447 )]
4448 #[test]
4449 fn write_freshens_loose_object_in_alternate() {
4450 use filetime::FileTime;
4451 use std::thread;
4452 use std::time::Duration;
4453
4454 let alt = TempDir::new().unwrap();
4455 git_in(alt.path(), &["init", "-q", "--bare"]);
4456 git_in(alt.path(), &["config", "user.email", "t@example.com"]);
4457 let alt_objects = alt.path().join("objects");
4458 let alt_odb = Odb::new(&alt_objects);
4459 let payload = b"alternate-loose";
4460 let oid = alt_odb.write(ObjectKind::Blob, payload).unwrap();
4461 let alt_loose = alt_odb.object_path(&oid);
4462
4463 let primary = TempDir::new().unwrap();
4464 let primary_objects = primary.path().join("objects");
4465 fs::create_dir_all(primary_objects.join("info")).unwrap();
4466 fs::write(
4467 primary_objects.join("info").join("alternates"),
4468 format!("{}\n", alt_objects.display()),
4469 )
4470 .unwrap();
4471 let odb = Odb::new(&primary_objects);
4472 thread::sleep(Duration::from_millis(50));
4473 let before = FileTime::from_last_modification_time(&fs::metadata(&alt_loose).unwrap());
4474 filetime::set_file_mtime(&alt_loose, before).unwrap();
4475
4476 let oid2 = odb.write(ObjectKind::Blob, payload).unwrap();
4477 assert_eq!(oid, oid2);
4478 assert!(!odb.object_path(&oid).exists());
4479 let after = FileTime::from_last_modification_time(&fs::metadata(&alt_loose).unwrap());
4480 assert!(after > before, "write must freshen alternate loose object");
4481 }
4482
4483 #[cfg_attr(
4484 not(feature = "full-tests"),
4485 ignore = "starts system git or is heavy; run with --features full-tests"
4486 )]
4487 #[test]
4488 fn write_freshens_packed_object_in_alternate() {
4489 use filetime::FileTime;
4490 use std::thread;
4491 use std::time::Duration;
4492
4493 let alt = TempDir::new().unwrap();
4494 git_in(alt.path(), &["init", "-q"]);
4495 git_in(alt.path(), &["config", "user.email", "t@example.com"]);
4496 git_in(alt.path(), &["config", "user.name", "T"]);
4497 let payload = b"alternate-pack";
4498 fs::write(alt.path().join("f"), payload).unwrap();
4499 git_in(alt.path(), &["add", "f"]);
4500 git_in(alt.path(), &["commit", "-m", "c"]);
4501 git_in(alt.path(), &["repack", "-ad"]);
4502 let alt_objects = alt.path().join(".git").join("objects");
4503 let alt_odb = Odb::new(&alt_objects);
4504 let oid = alt_odb.write(ObjectKind::Blob, payload).unwrap();
4505 let pack_path = fs::read_dir(alt_objects.join("pack"))
4506 .unwrap()
4507 .flatten()
4508 .find(|e| e.path().extension().is_some_and(|x| x == "pack"))
4509 .unwrap()
4510 .path();
4511 thread::sleep(Duration::from_millis(50));
4512 let before = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
4513
4514 let primary = TempDir::new().unwrap();
4515 let primary_objects = primary.path().join("objects");
4516 fs::create_dir_all(primary_objects.join("info")).unwrap();
4517 fs::write(
4518 primary_objects.join("info").join("alternates"),
4519 format!("{}\n", alt_objects.display()),
4520 )
4521 .unwrap();
4522 pack::clear_pack_cache();
4523 let odb = Odb::new(&primary_objects);
4524 assert!(odb.exists(&oid));
4525 let oid2 = odb.write(ObjectKind::Blob, payload).unwrap();
4526 assert_eq!(oid, oid2);
4527 assert!(!odb.object_path(&oid).exists());
4528 let after = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
4529 assert!(after >= before, "write must freshen alternate pack");
4530 }
4531
4532 #[cfg_attr(
4533 not(feature = "full-tests"),
4534 ignore = "starts system git or is heavy; run with --features full-tests"
4535 )]
4536 #[test]
4537 fn write_skips_loose_when_object_only_in_alternate() {
4538 let alt = TempDir::new().unwrap();
4539 git_in(alt.path(), &["init", "-q", "--bare"]);
4540 git_in(alt.path(), &["config", "user.email", "t@example.com"]);
4541 let alt_objects = alt.path().join("objects");
4542 let alt_odb = Odb::new(&alt_objects);
4543 let payload = b"alternate-only";
4544 let oid = alt_odb.write(ObjectKind::Blob, payload).unwrap();
4545
4546 let primary = TempDir::new().unwrap();
4547 let primary_objects = primary.path().join("objects");
4548 fs::create_dir_all(primary.path().join("objects").join("info")).unwrap();
4549 fs::write(
4550 primary
4551 .path()
4552 .join("objects")
4553 .join("info")
4554 .join("alternates"),
4555 format!("{}\n", alt_objects.display()),
4556 )
4557 .unwrap();
4558 let odb = Odb::new(&primary_objects);
4559 assert!(odb.exists(&oid));
4560 let oid2 = odb.write(ObjectKind::Blob, payload).unwrap();
4561 assert_eq!(oid, oid2);
4562 assert!(!odb.object_path(&oid).exists());
4563 }
4564
4565 #[cfg_attr(
4566 not(feature = "full-tests"),
4567 ignore = "starts system git or is heavy; run with --features full-tests"
4568 )]
4569 #[test]
4570 fn write_with_mem_overlay_stays_in_memory() {
4571 let dir = TempDir::new().unwrap();
4572 let objects = dir.path().join("objects");
4573 fs::create_dir_all(&objects).unwrap();
4574 let odb = Odb::new(&objects);
4575 odb.enable_mem_overlay();
4576 let payload = b"overlay blob";
4577 let oid = odb.write(ObjectKind::Blob, payload).unwrap();
4578 assert!(!odb.object_path(&oid).exists());
4579 assert_eq!(odb.read(&oid).unwrap().data, payload);
4580 odb.disable_mem_overlay();
4581 assert!(odb.read(&oid).is_err());
4582 }
4583
4584 #[test]
4585 fn read_alternates_relative_path_and_symlinked_objects_dir() {
4586 use crate::pack::read_alternates_recursive;
4587
4588 let layout = TempDir::new().unwrap();
4589 let alt_objects = layout.path().join("alt-store").join("objects");
4590 fs::create_dir_all(&alt_objects).unwrap();
4591 let primary_objects = layout.path().join("repo").join("objects");
4592 fs::create_dir_all(primary_objects.join("info")).unwrap();
4593 fs::write(
4594 primary_objects.join("info").join("alternates"),
4595 "../../alt-store/objects\n",
4596 )
4597 .unwrap();
4598 let resolved = read_alternates_recursive(&primary_objects).unwrap();
4599 let alt_canonical = alt_objects
4600 .canonicalize_simplified()
4601 .unwrap_or(alt_objects.clone());
4602 assert!(
4603 resolved.iter().any(|p| {
4604 p.canonicalize_simplified().unwrap_or_else(|_| p.clone()) == alt_canonical
4605 }),
4606 "relative alternate entry should resolve: {resolved:?}"
4607 );
4608
4609 #[cfg(unix)]
4610 {
4611 let link_parent = TempDir::new().unwrap();
4612 std::os::unix::fs::symlink(&primary_objects, link_parent.path().join("objects-link"))
4613 .unwrap();
4614 let via_link =
4615 read_alternates_recursive(&link_parent.path().join("objects-link")).unwrap();
4616 assert!(
4617 !via_link.is_empty(),
4618 "alternates through symlinked objects dir: {via_link:?}"
4619 );
4620 }
4621 }
4622 #[test]
4623 fn empty_tree_exists_without_hex() {
4624 let odb = Odb::new(std::path::Path::new("/nonexistent/objects"));
4625 let canon = ObjectId::from_hex("4b825dc642cb6eb9a060e54bf8d69288fbee4904").unwrap();
4626 let legacy = ObjectId::from_hex("4b825dc642cb6eb9a060e54bf899d69f7c6948d4").unwrap();
4627 assert!(odb.exists(&canon));
4628 assert!(odb.exists(&legacy));
4629 assert!(odb.exists_local(&canon));
4630 assert!(!odb.freshen_object(&canon));
4631 }
4632
4633 #[test]
4634 fn sha256_empty_tree_virtual_read_without_loose_file() {
4635 use crate::objects::HashAlgo;
4636 use std::sync::Arc;
4637
4638 let dir = TempDir::new().unwrap();
4639 let objects = dir.path().join("objects");
4640 fs::create_dir_all(&objects).unwrap();
4641 let store = Arc::new(MemoryStore::new(HashAlgo::Sha256));
4642 let odb = OdbBuilder::files(&objects)
4643 .primary(store)
4644 .alternates(false)
4645 .build();
4646 let empty = odb.empty_tree_oid().expect("hash empty tree");
4647 assert!(!odb.exists_materialized_local(&empty));
4648 assert!(odb.exists_local(&empty));
4649 assert!(odb.is_empty_tree(&empty));
4650 let obj = odb.read(&empty).expect("virtual read empty tree");
4651 assert_eq!(obj.kind, ObjectKind::Tree);
4652 assert!(obj.data.is_empty());
4653 }
4654
4655 #[test]
4656 fn empty_tree_oid_and_object_id_empty_tree_match_sha1() {
4657 let odb = Odb::new(std::path::Path::new("/nonexistent/objects"));
4658 let from_odb = odb.empty_tree_oid().expect("empty tree oid");
4659 assert_eq!(from_odb, ObjectId::empty_tree());
4660 assert!(odb.is_empty_tree(&ObjectId::empty_tree()));
4661 }
4662
4663 #[test]
4664 fn empty_tree_virtual_presence_skips_materialized_local() {
4665 let dir = TempDir::new().unwrap();
4666 let objects = dir.path().join("objects");
4667 fs::create_dir_all(&objects).unwrap();
4668 let odb = Odb::new(&objects);
4669 let canon = ObjectId::from_hex("4b825dc642cb6eb9a060e54bf8d69288fbee4904").unwrap();
4670 assert!(odb.exists_local(&canon));
4671 assert!(!odb.exists_materialized_local(&canon));
4672 }
4673
4674 #[test]
4675 fn write_materializes_well_known_empty_tree() {
4676 let dir = TempDir::new().unwrap();
4677 let objects = dir.path().join("objects");
4678 fs::create_dir_all(&objects).unwrap();
4679 let odb = Odb::new(&objects);
4680 let canon = ObjectId::from_hex("4b825dc642cb6eb9a060e54bf8d69288fbee4904").unwrap();
4681 assert!(!odb.exists_materialized_local(&canon));
4682 let oid = odb
4683 .write(crate::objects::ObjectKind::Tree, &[])
4684 .expect("write empty tree");
4685 assert_eq!(oid, canon);
4686 assert!(odb.exists_materialized_local(&canon));
4687 }
4688
4689 #[test]
4690 fn append_alternate_objects_line_errors_on_unreadable_existing_file() {
4691 let dir = TempDir::new().unwrap();
4692 let objects = dir.path().join("objects");
4693 fs::create_dir_all(objects.join("info")).unwrap();
4694 let alt_path = objects.join("info/alternates");
4695 fs::write(&alt_path, b"not valid utf-8 \xff\n").unwrap();
4696 let err = Odb::append_alternate_objects_line(&objects, Path::new("/tmp/other/objects"))
4697 .unwrap_err();
4698 assert!(matches!(err, Error::Io(_)));
4699 assert_eq!(fs::read(&alt_path).unwrap(), b"not valid utf-8 \xff\n");
4700 }
4701
4702 #[test]
4703 fn append_file_alternate_refreshes_cached_snapshot() {
4704 let primary = TempDir::new().unwrap();
4705 let alternate = TempDir::new().unwrap();
4706 let alt_objects = alternate.path().join("objects");
4707 std::fs::create_dir_all(&alt_objects).unwrap();
4708 let alt_odb = Odb::new(&alt_objects);
4709 let oid = alt_odb.write(ObjectKind::Blob, b"via-alternate").unwrap();
4710
4711 let primary_objects = primary.path().join("objects");
4712 std::fs::create_dir_all(primary_objects.join("info")).unwrap();
4713 let primary_odb = Odb::new(&primary_objects);
4714 assert!(!primary_odb.exists(&oid));
4715 let _ = primary_odb.file_alternate_dirs_snapshot();
4716 primary_odb.append_file_alternate(&alt_objects).unwrap();
4717 assert!(primary_odb.exists(&oid));
4718 }
4719
4720 #[test]
4721 fn exists_via_env_alternate_dirs_at_construction() {
4722 let primary = TempDir::new().unwrap();
4723 let alternate = TempDir::new().unwrap();
4724 let alt_objects = alternate.path().join("objects");
4725 std::fs::create_dir_all(&alt_objects).unwrap();
4726 let alt_odb = Odb::new(&alt_objects);
4727 let oid = alt_odb.write(ObjectKind::Blob, b"env-alt").unwrap();
4728
4729 let primary_objects = primary.path().join("objects");
4730 std::fs::create_dir_all(&primary_objects).unwrap();
4731 let primary_odb = Odb::new(&primary_objects).with_env_alternate_dirs(vec![alt_objects]);
4732 assert!(primary_odb.exists(&oid));
4733 }
4734
4735 fn flaky_odb(objects: &Path) -> (Odb, Arc<store::flaky::FlakyStore>) {
4736 let flaky = store::flaky::FlakyStore::new();
4737 let odb = OdbBuilder::files(objects)
4738 .primary(flaky.clone())
4739 .alternates(false)
4740 .build();
4741 (odb, flaky)
4742 }
4743
4744 #[test]
4745 fn contains_reports_primary_failure_instead_of_missing() {
4746 let dir = TempDir::new().unwrap();
4747 let (odb, flaky) = flaky_odb(dir.path());
4748 let oid = odb.write(ObjectKind::Blob, b"stored").unwrap();
4749 assert!(odb.contains(&oid).unwrap());
4750
4751 flaky.fail_all(true);
4752 assert!(store::flaky::is_injected(&odb.read(&oid).unwrap_err()));
4753 assert!(store::flaky::is_injected(&odb.contains(&oid).unwrap_err()));
4754 assert!(
4755 !odb.exists(&oid),
4756 "exists() keeps treating failures as absent"
4757 );
4758 assert!(odb.contains(&odb.empty_tree_oid().unwrap()).unwrap());
4759
4760 let try_local = odb.try_exists_local(&oid).unwrap_err();
4761 assert!(store::flaky::is_injected(&try_local));
4762 assert!(!odb.exists_local(&oid));
4763 let try_stored = odb.try_exists_stored(&oid).unwrap_err();
4764 assert!(store::flaky::is_injected(&try_stored));
4765 assert!(!odb.exists_stored(&oid));
4766 }
4767
4768 #[test]
4769 fn contains_reports_extra_read_source_failure() {
4770 let dir = TempDir::new().unwrap();
4771 let flaky = store::flaky::FlakyStore::new();
4772 let odb = OdbBuilder::files(dir.path())
4773 .push_read_source(flaky.clone())
4774 .alternates(false)
4775 .build();
4776 let local = odb.write(ObjectKind::Blob, b"in primary").unwrap();
4777 let absent = HashAlgo::Sha1
4778 .hash_object(ObjectKind::Blob, b"nowhere")
4779 .unwrap();
4780 assert!(!odb.contains(&absent).unwrap());
4781
4782 flaky.fail_contains(true);
4783 assert!(
4784 odb.contains(&local).unwrap(),
4785 "primary hit wins over a failing source"
4786 );
4787 assert!(store::flaky::is_injected(
4788 &odb.contains(&absent).unwrap_err()
4789 ));
4790 assert!(!odb.exists(&absent));
4791 }
4792
4793 #[test]
4794 fn contains_reports_primary_failure_even_when_later_source_has_object() {
4795 let dir = TempDir::new().unwrap();
4796 let extra = Arc::new(store::MemoryStore::new(HashAlgo::Sha1));
4797 let flaky = store::flaky::FlakyStore::new();
4798 let odb = OdbBuilder::files(dir.path())
4799 .primary(flaky.clone())
4800 .push_read_source(extra.clone())
4801 .alternates(false)
4802 .build();
4803 let oid = extra
4804 .write(ObjectKind::Blob, b"extra", WriteOptions::default())
4805 .unwrap();
4806 assert!(odb.contains(&oid).unwrap());
4807
4808 flaky.fail_all(true);
4809 assert!(store::flaky::is_injected(&odb.read(&oid).unwrap_err()));
4810 assert!(
4811 store::flaky::is_injected(&odb.contains(&oid).unwrap_err()),
4812 "contains must agree with read"
4813 );
4814 assert!(odb.exists(&oid), "exists() skips the failing primary");
4815 }
4816
4817 #[test]
4818 fn contains_reports_alternate_failure() {
4819 let primary = TempDir::new().unwrap();
4820 let alternate = TempDir::new().unwrap();
4821 let alt_objects = alternate.path().join("objects");
4822 std::fs::create_dir_all(&alt_objects).unwrap();
4823 let primary_objects = primary.path().join("objects");
4824 std::fs::create_dir_all(&primary_objects).unwrap();
4825 let odb = Odb::new(&primary_objects).with_env_alternate_dirs(vec![alt_objects]);
4826 let local = odb.write(ObjectKind::Blob, b"local").unwrap();
4827 let absent = HashAlgo::Sha1
4828 .hash_object(ObjectKind::Blob, b"nowhere")
4829 .unwrap();
4830
4831 let flaky = store::flaky::FlakyStore::new();
4834 odb.sources().unwrap();
4835 odb.alternate_sources_cache.write().unwrap().sources = Some(Arc::new(CompositeStore::new(
4836 vec![flaky.clone()],
4837 HashAlgo::Sha1,
4838 )));
4839 assert!(!odb.contains(&absent).unwrap());
4840
4841 flaky.fail_contains(true);
4842 assert!(odb.contains(&local).unwrap());
4843 assert!(store::flaky::is_injected(
4844 &odb.contains(&absent).unwrap_err()
4845 ));
4846 assert!(!odb.exists(&absent));
4847 }
4848
4849 #[test]
4850 fn contains_on_files_backend_covers_present_absent_empty_tree_and_alternate() {
4851 let primary = TempDir::new().unwrap();
4852 let alternate = TempDir::new().unwrap();
4853 let alt_objects = alternate.path().join("objects");
4854 std::fs::create_dir_all(&alt_objects).unwrap();
4855 let alt_oid = Odb::new(&alt_objects)
4856 .write(ObjectKind::Blob, b"alternate")
4857 .unwrap();
4858 let primary_objects = primary.path().join("objects");
4859 std::fs::create_dir_all(&primary_objects).unwrap();
4860 let odb = Odb::new(&primary_objects).with_env_alternate_dirs(vec![alt_objects]);
4861 let present = odb.write(ObjectKind::Blob, b"present").unwrap();
4862 let absent = HashAlgo::Sha1
4863 .hash_object(ObjectKind::Blob, b"absent")
4864 .unwrap();
4865
4866 assert!(odb.contains(&present).unwrap());
4867 assert!(odb.contains(&odb.empty_tree_oid().unwrap()).unwrap());
4868 assert!(odb.contains(&alt_oid).unwrap());
4869 assert!(!odb.contains(&absent).unwrap());
4870 let empty_tree = odb.empty_tree_oid().unwrap();
4871 assert_eq!(
4872 odb.contains_many(&[present, absent, alt_oid, empty_tree])
4873 .unwrap(),
4874 vec![true, false, true, true]
4875 );
4876 assert!(odb.contains_many(&[]).unwrap().is_empty());
4877 assert_eq!(
4878 odb.present_oids([present, absent, alt_oid]).unwrap(),
4879 vec![present, alt_oid]
4880 );
4881 assert_eq!(
4882 odb.missing_oids([present, absent, alt_oid]).unwrap(),
4883 vec![absent]
4884 );
4885 assert_eq!(
4886 odb.missing_stored_oids([present, absent, alt_oid]).unwrap(),
4887 vec![absent, alt_oid],
4888 "stored ignores alternates"
4889 );
4890 }
4891
4892 #[test]
4893 fn missing_stored_oids_sees_packs_written_since_the_last_probe() {
4894 let dir = TempDir::new().unwrap();
4895 let odb = Odb::new(dir.path());
4896 let other = TempDir::new().unwrap();
4897 let source = Odb::new(other.path());
4898 let blobs: Vec<ObjectId> = (0..3)
4899 .map(|i| {
4900 source
4901 .write(ObjectKind::Blob, format!("b{i}").as_bytes())
4902 .unwrap()
4903 })
4904 .collect();
4905 assert_eq!(odb.missing_stored_oids(blobs.clone()).unwrap(), blobs);
4906 let pack = crate::pack_objects::build_pack_from_send_list(
4908 &source,
4909 &blobs[..2],
4910 &std::collections::HashSet::new(),
4911 &crate::pack_objects::PackBuildOptions::default(),
4912 &std::collections::HashMap::new(),
4913 )
4914 .unwrap();
4915 crate::index_pack::install_pack_bytes(pack, &Odb::new(dir.path()), &Default::default())
4916 .unwrap();
4917 assert_eq!(
4918 odb.missing_stored_oids(blobs.clone()).unwrap(),
4919 vec![blobs[2]]
4920 );
4921 }
4922
4923 #[test]
4924 fn oid_filters_propagate_backend_failure() {
4925 let dir = TempDir::new().unwrap();
4926 let (odb, flaky) = flaky_odb(dir.path());
4927 let oid = odb.write(ObjectKind::Blob, b"x").unwrap();
4928 assert_eq!(odb.contains_many(&[oid]).unwrap(), vec![true]);
4929 flaky.fail_contains(true);
4930 assert!(store::flaky::is_injected(
4931 &odb.contains_many(&[oid]).unwrap_err()
4932 ));
4933 assert!(store::flaky::is_injected(
4934 &odb.present_oids([oid]).unwrap_err()
4935 ));
4936 assert!(store::flaky::is_injected(
4937 &odb.missing_oids([oid]).unwrap_err()
4938 ));
4939 assert!(store::flaky::is_injected(
4940 &odb.missing_stored_oids([oid]).unwrap_err()
4941 ));
4942 }
4943
4944 #[test]
4945 fn write_existing_loose_uses_path_stat_not_exists() {
4946 let dir = TempDir::new().unwrap();
4947 let odb = Odb::new(dir.path());
4948 let oid = odb.write(ObjectKind::Blob, b"once").unwrap();
4949 let path = odb.object_path(&oid);
4950 assert!(path.is_file());
4951 odb.reset_exists_probe();
4952 let oid2 = odb.write(ObjectKind::Blob, b"once").unwrap();
4953 assert_eq!(oid, oid2);
4954 assert_eq!(
4955 odb.exists_probe_count(),
4956 0,
4957 "loose fast path must not call exists()"
4958 );
4959 }
4960
4961 #[cfg_attr(
4962 not(feature = "full-tests"),
4963 ignore = "starts system git or is heavy; run with --features full-tests"
4964 )]
4965 #[test]
4966 fn exists_and_write_after_invalidate_packs_use_odb_store() {
4967 let dir = TempDir::new().unwrap();
4968 git_in(dir.path(), &["init", "-q"]);
4969 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
4970 git_in(dir.path(), &["config", "user.name", "T"]);
4971 let payload = b"invalidate-pack-exists";
4972 fs::write(dir.path().join("f"), payload).unwrap();
4973 git_in(dir.path(), &["add", "f"]);
4974 git_in(dir.path(), &["commit", "-m", "c"]);
4975 git_in(dir.path(), &["repack", "-ad"]);
4976 let objects = dir.path().join(".git").join("objects");
4977 let odb = Odb::new(&objects);
4978 let oid = odb.hash(ObjectKind::Blob, payload).expect("hash");
4979 assert!(odb.exists(&oid), "blob must live in pack after repack");
4980 let pack_dir = objects.join("pack");
4981 for entry in fs::read_dir(&pack_dir).unwrap().flatten() {
4982 let path = entry.path();
4983 if path.extension().is_some_and(|e| e == "pack" || e == "idx") {
4984 fs::remove_file(path).unwrap();
4985 }
4986 }
4987 odb.invalidate_packs();
4988 assert!(
4989 !odb.exists(&oid),
4990 "exists must not read stale legacy pack listing after invalidate_packs"
4991 );
4992 let oid2 = odb.write(ObjectKind::Blob, payload).unwrap();
4993 assert_eq!(oid, oid2);
4994 assert!(
4995 odb.object_path(&oid).is_file(),
4996 "write must materialize loose object when pack copy is gone"
4997 );
4998 odb.read(&oid).expect("read after write");
4999 }
5000
5001 #[cfg_attr(
5002 not(feature = "full-tests"),
5003 ignore = "starts system git or is heavy; run with --features full-tests"
5004 )]
5005 #[test]
5006 fn write_uses_single_exists_probe_when_not_loose() {
5007 let dir = TempDir::new().unwrap();
5008 git_in(dir.path(), &["init", "-q"]);
5009 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5010 git_in(dir.path(), &["config", "user.name", "T"]);
5011 fs::write(dir.path().join("f"), b"probe-once").unwrap();
5012 git_in(dir.path(), &["add", "f"]);
5013 git_in(dir.path(), &["commit", "-m", "c"]);
5014 git_in(dir.path(), &["repack", "-ad"]);
5015 let objects = dir.path().join(".git").join("objects");
5016 let odb = Odb::new(&objects);
5017 let oid = odb.write(ObjectKind::Blob, b"probe-once").unwrap();
5018 assert!(!odb.object_path(&oid).exists());
5019 odb.reset_exists_probe();
5020 let oid2 = odb.write(ObjectKind::Blob, b"probe-once").unwrap();
5021 assert_eq!(oid, oid2);
5022 assert_eq!(
5023 odb.exists_probe_count(),
5024 1,
5025 "pack/alternate write path must call exists() exactly once"
5026 );
5027 }
5028 #[test]
5029 fn write_existing_object_freshens() {
5030 use filetime::FileTime;
5031 use std::fs;
5032 use std::thread;
5033 use std::time::Duration;
5034
5035 let dir = TempDir::new().unwrap();
5036 let odb = Odb::new(dir.path());
5037 let oid = odb.write(ObjectKind::Blob, b"hello").unwrap();
5038 let path = odb.object_path(&oid);
5039 thread::sleep(Duration::from_millis(50));
5040 let before = FileTime::from_last_modification_time(&fs::metadata(&path).unwrap());
5041 filetime::set_file_mtime(&path, before).unwrap();
5042
5043 let _ = odb.write(ObjectKind::Blob, b"hello").unwrap();
5044 let after = FileTime::from_last_modification_time(&fs::metadata(&path).unwrap());
5045 assert!(
5046 after > before,
5047 "rewriting an existing loose object must freshen its mtime"
5048 );
5049 }
5050
5051 #[test]
5052 fn freshen_pack_once_per_odb() {
5053 use filetime::FileTime;
5054 use flate2::write::ZlibEncoder;
5055 use flate2::Compression;
5056 use std::fs;
5057 use std::io::Write;
5058 use std::thread;
5059 use std::time::Duration;
5060
5061 fn append_pack_object(buf: &mut Vec<u8>, type_code: u8, data: &[u8]) {
5062 let mut size = data.len();
5063 let first = ((type_code & 0x7) << 4) | (size & 0x0f) as u8;
5064 size >>= 4;
5065 if size > 0 {
5066 buf.push(first | 0x80);
5067 while size > 0 {
5068 let b = (size & 0x7f) as u8;
5069 size >>= 7;
5070 buf.push(if size > 0 { b | 0x80 } else { b });
5071 }
5072 } else {
5073 buf.push(first);
5074 }
5075 let mut enc = ZlibEncoder::new(Vec::new(), Compression::default());
5076 enc.write_all(data).unwrap();
5077 buf.extend_from_slice(&enc.finish().unwrap());
5078 }
5079
5080 fn write_v2_idx(
5081 idx_path: &Path,
5082 pack_path: &Path,
5083 entries: &[(ObjectId, u64)],
5084 ) -> Result<()> {
5085 let mut sorted = entries.to_vec();
5086 sorted.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
5087 let n = sorted.len();
5088 let mut fanout = [0u32; 256];
5089 for byte in 0u32..256 {
5090 let count = sorted
5091 .iter()
5092 .filter(|(oid, _)| u32::from(oid.as_bytes()[0]) <= byte)
5093 .count();
5094 fanout[byte as usize] = u32::try_from(count).unwrap_or(u32::MAX);
5095 }
5096 let mut buf = Vec::new();
5097 buf.extend_from_slice(b"\xfftOc");
5098 buf.extend_from_slice(&2u32.to_be_bytes());
5099 for f in fanout {
5100 buf.extend_from_slice(&f.to_be_bytes());
5101 }
5102 for (oid, _) in &sorted {
5103 buf.extend_from_slice(oid.as_bytes());
5104 }
5105 for _ in 0..n {
5106 buf.extend_from_slice(&0u32.to_be_bytes());
5107 }
5108 for (_, off) in &sorted {
5109 buf.extend_from_slice(&u32::try_from(*off).unwrap_or(0x8000_0000).to_be_bytes());
5110 }
5111 let pack_bytes = fs::read(pack_path)?;
5112 buf.extend_from_slice(&pack_bytes[pack_bytes.len() - 20..]);
5113 buf.extend_from_slice(&HashAlgo::Sha1.checksum_digest(&buf));
5114 fs::write(idx_path, buf)?;
5115 Ok(())
5116 }
5117
5118 let dir = TempDir::new().unwrap();
5119 let objects_dir = dir.path();
5120 let odb = Odb::new(objects_dir);
5121
5122 let oid_a = odb.write(ObjectKind::Blob, b"alpha").unwrap();
5123 let oid_b = odb.write(ObjectKind::Blob, b"beta").unwrap();
5124 fs::remove_file(odb.object_path(&oid_a)).unwrap();
5125 fs::remove_file(odb.object_path(&oid_b)).unwrap();
5126
5127 let mut pack = Vec::new();
5128 pack.extend_from_slice(b"PACK");
5129 pack.extend_from_slice(&2u32.to_be_bytes());
5130 pack.extend_from_slice(&2u32.to_be_bytes());
5131 let off_a = pack.len() as u64;
5132 append_pack_object(&mut pack, 3, b"alpha");
5133 let off_b = pack.len() as u64;
5134 append_pack_object(&mut pack, 3, b"beta");
5135 pack.extend_from_slice(&HashAlgo::Sha1.checksum_digest(&pack));
5136
5137 let pack_dir = objects_dir.join("pack");
5138 fs::create_dir_all(&pack_dir).unwrap();
5139 let pack_path = pack_dir.join("test-177.pack");
5140 let idx_path = pack_dir.join("test-177.idx");
5141 fs::write(&pack_path, &pack).unwrap();
5142 write_v2_idx(&idx_path, &pack_path, &[(oid_a, off_a), (oid_b, off_b)]).unwrap();
5143 odb.invalidate_packs();
5144
5145 thread::sleep(Duration::from_millis(50));
5146 let before = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
5147
5148 assert!(odb.freshen_object(&oid_a));
5149 let after_first = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
5150 assert!(after_first > before);
5151
5152 assert!(odb.freshen_object(&oid_b));
5153 let after_second =
5154 FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
5155 assert_eq!(
5156 after_second, after_first,
5157 "second freshen of another object in the same pack must not touch the pack again"
5158 );
5159 }
5160
5161 #[test]
5162 fn write_existing_packed_object_freshens_pack() {
5163 use filetime::FileTime;
5164 use flate2::write::ZlibEncoder;
5165 use flate2::Compression;
5166 use std::fs;
5167 use std::io::Write;
5168 use std::thread;
5169 use std::time::Duration;
5170
5171 let dir = TempDir::new().unwrap();
5172 let objects_dir = dir.path();
5173 let odb = Odb::new(objects_dir);
5174 let oid = odb.write(ObjectKind::Blob, b"packed-only").unwrap();
5175 fs::remove_file(odb.object_path(&oid)).unwrap();
5176
5177 let mut pack = Vec::new();
5178 pack.extend_from_slice(b"PACK");
5179 pack.extend_from_slice(&2u32.to_be_bytes());
5180 pack.extend_from_slice(&1u32.to_be_bytes());
5181 let off = pack.len() as u64;
5182 let mut enc = ZlibEncoder::new(Vec::new(), Compression::default());
5183 enc.write_all(b"packed-only").unwrap();
5184 let compressed = enc.finish().unwrap();
5185 pack.push(0x30 | 0x03); pack.extend_from_slice(&compressed);
5187 pack.extend_from_slice(&HashAlgo::Sha1.checksum_digest(&pack));
5188
5189 let pack_dir = objects_dir.join("pack");
5190 fs::create_dir_all(&pack_dir).unwrap();
5191 let pack_path = pack_dir.join("single-177.pack");
5192 fs::write(&pack_path, &pack).unwrap();
5193 let idx_path = pack_dir.join("single-177.idx");
5195 let mut sorted = [(oid, off)];
5196 sorted.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
5197 let mut fanout = [0u32; 256];
5198 for byte in 0u32..256 {
5199 fanout[byte as usize] = if u32::from(oid.as_bytes()[0]) <= byte {
5200 1
5201 } else {
5202 0
5203 };
5204 }
5205 let mut buf = Vec::new();
5206 buf.extend_from_slice(b"\xfftOc");
5207 buf.extend_from_slice(&2u32.to_be_bytes());
5208 for f in fanout {
5209 buf.extend_from_slice(&f.to_be_bytes());
5210 }
5211 buf.extend_from_slice(oid.as_bytes());
5212 buf.extend_from_slice(&0u32.to_be_bytes());
5213 buf.extend_from_slice(&(off as u32).to_be_bytes());
5214 buf.extend_from_slice(&pack[pack.len() - 20..]);
5215 buf.extend_from_slice(&HashAlgo::Sha1.checksum_digest(&buf));
5216 fs::write(&idx_path, buf).unwrap();
5217 odb.invalidate_packs();
5218
5219 thread::sleep(Duration::from_millis(50));
5220 let before = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
5221
5222 odb.write(ObjectKind::Blob, b"packed-only").unwrap();
5223
5224 let after = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
5225 assert!(
5226 after > before,
5227 "write of an existing packed object must freshen the pack mtime"
5228 );
5229 }
5230
5231 #[cfg_attr(
5235 not(feature = "full-tests"),
5236 ignore = "starts system git or is heavy; run with --features full-tests"
5237 )]
5238 #[test]
5239 fn write_materializes_loose_when_pack_freshen_fails() {
5240 let dir = TempDir::new().unwrap();
5241 git_in(dir.path(), &["init", "-q"]);
5242 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5243 git_in(dir.path(), &["config", "user.name", "T"]);
5244 let payload = b"packed-fallback";
5245 fs::write(dir.path().join("f"), payload).unwrap();
5246 git_in(dir.path(), &["add", "f"]);
5247 git_in(dir.path(), &["commit", "-m", "c"]);
5248 git_in(dir.path(), &["repack", "-ad"]);
5249 let objects = dir.path().join(".git").join("objects");
5250 let odb = Odb::new(&objects);
5251 let oid = odb.write(ObjectKind::Blob, payload).unwrap();
5252 assert!(!odb.object_path(&oid).exists());
5253 assert!(odb.exists(&oid));
5254
5255 let pack_path = fs::read_dir(objects.join("pack"))
5256 .unwrap()
5257 .flatten()
5258 .find(|e| e.path().extension().is_some_and(|x| x == "pack"))
5259 .unwrap()
5260 .path();
5261 fs::remove_file(&pack_path).unwrap();
5262 fs::create_dir(&pack_path).unwrap();
5263 odb.invalidate_packs();
5264 assert!(!odb.freshen_object(&oid));
5265
5266 odb.write(ObjectKind::Blob, payload).unwrap();
5267 assert!(odb.object_path(&oid).exists());
5268 }
5269 #[test]
5270 fn write_with_options_silent_skips_freshen_on_existing_loose_object() {
5271 let dir = TempDir::new().unwrap();
5272 let odb = Odb::new(dir.path());
5273 let oid = odb.write(ObjectKind::Blob, b"stable").unwrap();
5274 let path = odb.object_path(&oid);
5275 let before = std::fs::metadata(&path).unwrap().modified().unwrap();
5276 std::thread::sleep(std::time::Duration::from_millis(20));
5277 let again = odb
5278 .write_with_options(
5279 ObjectKind::Blob,
5280 b"stable",
5281 WriteOptions {
5282 silent: true,
5283 ..WriteOptions::default()
5284 },
5285 )
5286 .unwrap();
5287 assert_eq!(oid, again);
5288 let after = std::fs::metadata(&path).unwrap().modified().unwrap();
5289 assert_eq!(before, after);
5290
5291 let touched = odb
5292 .write_with_options(
5293 ObjectKind::Blob,
5294 b"stable",
5295 WriteOptions {
5296 silent: false,
5297 ..WriteOptions::default()
5298 },
5299 )
5300 .unwrap();
5301 assert_eq!(oid, touched);
5302 let freshened = std::fs::metadata(&path).unwrap().modified().unwrap();
5303 assert!(freshened >= before);
5304 }
5305
5306 #[cfg_attr(
5307 not(feature = "full-tests"),
5308 ignore = "starts system git or is heavy; run with --features full-tests"
5309 )]
5310 #[test]
5311 fn promisor_marker_checked_once_per_prepare() {
5312 use crate::pack::{
5313 clear_pack_cache, pack_cache_test_guard, test_pack_marker_stat_count,
5314 test_reset_pack_marker_stat_count,
5315 };
5316
5317 let _guard = pack_cache_test_guard();
5318 let dir = TempDir::new().unwrap();
5319 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5320 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5321 git_in(dir.path(), &["config", "user.name", "T"]);
5322 for i in 0..64 {
5323 fs::write(dir.path().join(format!("f{i}.txt")), format!("body {i}")).unwrap();
5324 git_in(dir.path(), &["add", "."]);
5325 git_in(dir.path(), &["commit", "-m", &format!("c{i}")]);
5326 }
5327 git_in(dir.path(), &["repack", "-a", "-d"]);
5328 let objects = dir.path().join(".git/objects");
5329 let oid = git_rev_parse_oid(dir.path(), "HEAD");
5330
5331 clear_pack_cache();
5332 test_reset_pack_marker_stat_count();
5333 let pack_count = fs::read_dir(objects.join("pack"))
5334 .unwrap()
5335 .filter_map(|e| e.ok())
5336 .filter(|e| e.path().extension().is_some_and(|x| x == "pack"))
5337 .count();
5338 let expected_prepare_stats = u64::try_from(pack_count * 2).unwrap();
5339 let odb = Odb::new(&objects);
5340 let before_prepare = test_pack_marker_stat_count();
5341 assert!(odb.exists_local(&oid));
5342 assert_eq!(
5343 test_pack_marker_stat_count().saturating_sub(before_prepare),
5344 expected_prepare_stats,
5345 "prepare should stat promisor and mtimes once per pack"
5346 );
5347
5348 test_reset_pack_marker_stat_count();
5349 for _ in 0..10_000 {
5350 let before_probe = test_pack_marker_stat_count();
5351 assert!(odb.exists_local(&oid));
5352 assert_eq!(
5353 test_pack_marker_stat_count(),
5354 before_probe,
5355 "cached sidecar flags must not stat markers on each exists_local probe"
5356 );
5357 }
5358 }
5359
5360 #[cfg_attr(
5361 not(feature = "full-tests"),
5362 ignore = "starts system git or is heavy; run with --features full-tests"
5363 )]
5364 #[test]
5365 fn promisor_marker_added_after_prepare_is_seen_after_reprepare() {
5366 use crate::pack::{
5367 clear_pack_cache, pack_cache_test_guard, reprepare_pack_directory_on_miss,
5368 };
5369
5370 let _guard = pack_cache_test_guard();
5371 let dir = TempDir::new().unwrap();
5372 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5373 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5374 git_in(dir.path(), &["config", "user.name", "T"]);
5375 fs::write(dir.path().join("one.txt"), b"x").unwrap();
5376 git_in(dir.path(), &["add", "one.txt"]);
5377 git_in(dir.path(), &["commit", "-m", "one"]);
5378 git_in(dir.path(), &["repack", "-a", "-d"]);
5379 let objects = dir.path().join(".git/objects");
5380 let oid = git_rev_parse_oid(dir.path(), "HEAD");
5381
5382 clear_pack_cache();
5383 let odb = Odb::new(&objects);
5384 assert!(odb.exists_local(&oid));
5385
5386 let pack_path = fs::read_dir(objects.join("pack"))
5387 .unwrap()
5388 .filter_map(|e| e.ok())
5389 .map(|e| e.path())
5390 .find(|p| p.extension().is_some_and(|x| x == "pack"))
5391 .unwrap();
5392 fs::write(pack_path.with_extension("promisor"), b"").unwrap();
5393 filetime::set_file_mtime(objects.join("pack"), filetime::FileTime::now()).unwrap();
5394
5395 assert!(
5396 odb.with_pack_store_for(&objects, || {
5397 reprepare_pack_directory_on_miss(&objects).unwrap()
5398 }),
5399 "pack directory change must require reprepare"
5400 );
5401 assert!(
5402 !odb.exists_local(&oid),
5403 "promisor marker must hide pack objects after reprepare refreshed sidecar flags"
5404 );
5405 }
5406
5407 #[cfg_attr(
5408 not(feature = "full-tests"),
5409 ignore = "starts system git or is heavy; run with --features full-tests"
5410 )]
5411 #[test]
5412 fn git_promisor_marker_pack_not_materialized_for_exists_local() {
5413 use crate::pack::clear_pack_cache;
5414
5415 let dir = TempDir::new().unwrap();
5416 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5417 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5418 git_in(dir.path(), &["config", "user.name", "T"]);
5419 fs::write(dir.path().join("blob.txt"), b"git promisor compat").unwrap();
5420 git_in(dir.path(), &["add", "blob.txt"]);
5421 git_in(dir.path(), &["commit", "-m", "seed"]);
5422 git_in(dir.path(), &["repack", "-a", "-d"]);
5423 let objects = dir.path().join(".git/objects");
5424 let oid = git_rev_parse_oid(dir.path(), "HEAD");
5425 let pack_path = fs::read_dir(objects.join("pack"))
5426 .unwrap()
5427 .filter_map(|e| e.ok())
5428 .map(|e| e.path())
5429 .find(|p| p.extension().is_some_and(|x| x == "pack"))
5430 .unwrap();
5431 fs::write(pack_path.with_extension("promisor"), b"").unwrap();
5432
5433 clear_pack_cache();
5434 let odb = Odb::new(&objects);
5435 assert!(
5436 !odb.exists_local(&oid),
5437 "git-written pack with .promisor must not count as local materialization"
5438 );
5439
5440 let cat = git_run(dir.path(), &["cat-file", "-e", &oid.to_hex()]);
5441 assert!(
5442 cat.status.success(),
5443 "git still resolves the object in a promisor-marked pack"
5444 );
5445 }
5446
5447 #[cfg_attr(
5448 not(feature = "full-tests"),
5449 ignore = "starts system git or is heavy; run with --features full-tests"
5450 )]
5451 #[test]
5452 fn packed_object_read_avoids_loose_path_open() {
5453 use crate::hot_path_test_metrics::HotPathMetricsScope;
5454 use crate::pack::clear_pack_cache;
5455
5456 let dir = TempDir::new().unwrap();
5457 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5458 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5459 git_in(dir.path(), &["config", "user.name", "T"]);
5460 fs::write(dir.path().join("packed.txt"), b"packed-only payload").unwrap();
5461 git_in(dir.path(), &["add", "packed.txt"]);
5462 git_in(dir.path(), &["commit", "-m", "pack"]);
5463 git_in(dir.path(), &["repack", "-a", "-d"]);
5464 let objects = dir.path().join(".git/objects");
5465 let oid = git_rev_parse_oid(dir.path(), "HEAD");
5466
5467 clear_pack_cache();
5468 let odb = Odb::new(&objects);
5469 odb.hot_path_metrics().set_loose_open_counting(true);
5470 odb.hot_path_metrics().reset_loose_path_open_attempts();
5471 let _scope = HotPathMetricsScope::install(odb.hot_path_metrics_arc());
5472
5473 let obj = odb.read(&oid).expect("packed read");
5474 assert_eq!(obj.kind, ObjectKind::Commit);
5475 assert_eq!(
5476 odb.hot_path_metrics().loose_path_open_attempts(),
5477 0,
5478 "packed objects must not attempt a loose-path open"
5479 );
5480 }
5481
5482 #[test]
5483 fn loose_only_read_records_loose_path_open() {
5484 use crate::hot_path_test_metrics::HotPathMetricsScope;
5485
5486 let dir = TempDir::new().unwrap();
5487 let objects = dir.path().join("objects");
5488 fs::create_dir_all(&objects).unwrap();
5489 let odb = Odb::new(&objects);
5490 let oid = odb
5491 .write(ObjectKind::Blob, b"loose-only")
5492 .expect("write loose");
5493 odb.hot_path_metrics().set_loose_open_counting(true);
5494 odb.hot_path_metrics().reset_loose_path_open_attempts();
5495 let _scope = HotPathMetricsScope::install(odb.hot_path_metrics_arc());
5496
5497 odb.read(&oid).expect("loose read");
5498 assert_eq!(
5499 odb.hot_path_metrics().loose_path_open_attempts(),
5500 1,
5501 "loose fallback must record a path open attempt"
5502 );
5503 }
5504
5505 #[cfg_attr(
5506 not(feature = "full-tests"),
5507 ignore = "starts system git or is heavy; run with --features full-tests"
5508 )]
5509 #[test]
5510 fn loose_and_packed_copies_return_identical_bytes() {
5511 use crate::pack::clear_pack_cache;
5512
5513 let dir = TempDir::new().unwrap();
5514 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5515 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5516 git_in(dir.path(), &["config", "user.name", "T"]);
5517 fs::write(dir.path().join("dup.txt"), b"duplicate loose and packed\n").unwrap();
5518 git_in(dir.path(), &["add", "dup.txt"]);
5519 git_in(dir.path(), &["commit", "-m", "seed"]);
5520 let objects = dir.path().join(".git/objects");
5521 let tree_oid = git_rev_parse_oid(dir.path(), "HEAD^{tree}");
5522 git_in(dir.path(), &["repack", "-a"]);
5523 clear_pack_cache();
5524 let odb = Odb::new(&objects);
5525 let loose_path = tree_oid.loose_path_in(&objects);
5526 assert!(
5527 loose_path.is_file(),
5528 "loose copy must remain after repack -a"
5529 );
5530 let loose = Odb::read_loose_verify_oid(&loose_path, &tree_oid).expect("loose tree");
5531 let packed = odb.read(&tree_oid).expect("packed tree");
5532 assert_eq!(packed.data, loose.data);
5533 assert_eq!(packed.kind, loose.kind);
5534 }
5535
5536 #[cfg_attr(
5537 not(feature = "full-tests"),
5538 ignore = "starts system git or is heavy; run with --features full-tests"
5539 )]
5540 #[test]
5541 fn freshly_written_loose_object_visible_before_repack() {
5542 let dir = TempDir::new().unwrap();
5543 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5544 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5545 git_in(dir.path(), &["config", "user.name", "T"]);
5546 fs::write(dir.path().join("seed.txt"), b"seed").unwrap();
5547 git_in(dir.path(), &["add", "seed.txt"]);
5548 git_in(dir.path(), &["commit", "-m", "seed"]);
5549 git_in(dir.path(), &["repack", "-a", "-d"]);
5550 let objects = dir.path().join(".git/objects");
5551 let odb = Odb::new(&objects);
5552 let fresh = b"brand-new loose blob\n";
5553 let oid = odb.write(ObjectKind::Blob, fresh).expect("loose write");
5554 let obj = odb.read(&oid).expect("immediate loose read");
5555 assert_eq!(obj.data.as_slice(), fresh);
5556 }
5557
5558 #[cfg_attr(
5559 not(feature = "full-tests"),
5560 ignore = "starts system git or is heavy; run with --features full-tests"
5561 )]
5562 #[test]
5563 fn pack_added_after_first_read_found_via_odb_reprepare() {
5564 use crate::pack::{clear_pack_cache, pack_cache_test_guard};
5565
5566 let _guard = pack_cache_test_guard();
5567 clear_pack_cache();
5568 let dir = TempDir::new().unwrap();
5569 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5570 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5571 git_in(dir.path(), &["config", "user.name", "T"]);
5572 fs::write(dir.path().join("a.txt"), b"pack-a").unwrap();
5573 git_in(dir.path(), &["add", "a.txt"]);
5574 git_in(dir.path(), &["commit", "-m", "a"]);
5575 git_in(dir.path(), &["repack", "-a", "-d"]);
5576 let objects = dir.path().join(".git/objects");
5577 let pack_dir = objects.join("pack");
5578 let git_dir = dir.path().join(".git");
5579
5580 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
5581 let warm_oid = git_rev_parse_oid(dir.path(), "HEAD");
5582 odb.read(&warm_oid).expect("warm read");
5583
5584 fs::write(dir.path().join("b.txt"), b"pack-b").unwrap();
5585 git_in(dir.path(), &["add", "b.txt"]);
5586 git_in(dir.path(), &["commit", "-m", "b"]);
5587 let new_oid = git_rev_parse_oid(dir.path(), "HEAD");
5588
5589 std::thread::sleep(std::time::Duration::from_secs(1));
5590 filetime::set_file_mtime(&pack_dir, filetime::FileTime::now()).unwrap();
5591
5592 let obj = odb.read(&new_oid).expect("new pack via reprepare");
5593 assert_eq!(obj.kind, ObjectKind::Commit);
5594 }
5595
5596 #[cfg_attr(
5597 not(feature = "full-tests"),
5598 ignore = "starts system git or is heavy; run with --features full-tests"
5599 )]
5600 #[test]
5601 fn late_midx_after_warm_read_resolves_new_commit() {
5602 use crate::pack::{clear_pack_cache, pack_cache_test_guard};
5603
5604 let _guard = pack_cache_test_guard();
5605 clear_pack_cache();
5606 let dir = TempDir::new().unwrap();
5607 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5608 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5609 git_in(dir.path(), &["config", "user.name", "T"]);
5610 fs::write(dir.path().join("a.txt"), b"pack-a").unwrap();
5611 git_in(dir.path(), &["add", "a.txt"]);
5612 git_in(dir.path(), &["commit", "-m", "a"]);
5613 git_in(dir.path(), &["repack", "-a", "-d"]);
5614 let objects = dir.path().join(".git/objects");
5615 let git_dir = dir.path().join(".git");
5616 assert!(
5617 crate::midx::cached_tip_midx_path(&objects.join("pack")).is_none(),
5618 "fixture starts without a MIDX"
5619 );
5620
5621 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
5622 let warm_oid = git_rev_parse_oid(dir.path(), "HEAD");
5623 odb.read(&warm_oid).expect("warm read before MIDX exists");
5624
5625 fs::write(dir.path().join("b.txt"), b"pack-b").unwrap();
5626 git_in(dir.path(), &["add", "b.txt"]);
5627 git_in(dir.path(), &["commit", "-m", "b"]);
5628 git_in(dir.path(), &["repack", "-a", "-d"]);
5629 git_in(dir.path(), &["multi-pack-index", "write"]);
5630 let new_oid = git_rev_parse_oid(dir.path(), "HEAD");
5631
5632 let cat = git_run(dir.path(), &["cat-file", "-e", &new_oid.to_hex()]);
5633 assert!(
5634 cat.status.success(),
5635 "system git must resolve the new commit"
5636 );
5637
5638 let obj = odb
5639 .read(&new_oid)
5640 .expect("MIDX-covered commit after late MIDX write");
5641 assert_eq!(obj.kind, ObjectKind::Commit);
5642 assert!(
5643 crate::midx::cached_tip_midx_path(&objects.join("pack")).is_some(),
5644 "successful read after late MIDX must refresh the tip cache"
5645 );
5646 }
5647}