1pub mod builder;
25mod commit;
26mod peel;
27mod read_store;
28pub mod store;
29mod stream;
30mod tree;
31
32pub(crate) use stream::deflate_stream_to_writer;
33
34#[doc(inline)]
35pub use builder::OdbBuilder;
36
37pub(crate) use store::loose::{
38 decompress_zlib_loose_bytes, for_each_loose_object_id, parse_object_bytes,
39 read_loose_object_info,
40};
41#[doc(inline)]
42pub use store::LooseStore;
43pub use store::{CompositeStore, FilesSource, ObjectStream};
44#[doc(inline)]
45pub use store::{MemoryStore, ObjectStore, WritableObjectStore};
46
47use crate::git_path::CanonicalizeExt;
48use std::collections::{HashMap, HashSet};
49use std::fs;
50use std::hash::{Hash, Hasher};
51use std::ops::ControlFlow;
52use std::path::{Path, PathBuf};
53#[cfg(test)]
54use std::sync::atomic::AtomicUsize;
55use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
56use std::sync::{Arc, Mutex, OnceLock, RwLock};
57
58use crate::config::ConfigSet;
59use crate::diagnostics::NullDiagnostics;
60use crate::error::{Error, Result};
61use crate::hash;
62use crate::midx::validate_midx_referenced_packs_with_diagnostics;
63use crate::objects::{
64 oid_hash_set, HashAlgo, Object, ObjectId, ObjectInfo, ObjectKind, OidHashMap, OidHashSet,
65};
66use crate::pack;
67use crate::pack_store::PackStore;
68use filetime::{set_file_mtime, FileTime};
69use flate2::Compression;
70
71pub(crate) use store::BoundedRead;
72use store::{LocalSizedRead, PackedObjects};
73
74#[derive(Debug, Clone, Copy, PartialEq, Eq)]
76pub(crate) enum UnreachablePackRetention {
77 KeepAll,
79 KeepGraceProtected(std::time::SystemTime),
81 DropAll,
83}
84
85pub(crate) struct RepackPackInventorySnapshot {
87 pub pack_paths: HashSet<PathBuf>,
88 pub kept_at_inventory: HashSet<PathBuf>,
89}
90
91type MemOdbOverlayStore = Arc<RwLock<Option<Arc<MemoryStore>>>>;
92
93struct AlternateSourcesCache {
95 file_generation: u64,
96 env_fingerprint: u64,
97 submodule_generation: u64,
98 sources: Option<Arc<CompositeStore>>,
99}
100
101struct AlternateSourcesKey {
102 file_generation: u64,
103 env_fingerprint: u64,
104 submodule_generation: u64,
105}
106
107struct FileAlternatesCache {
109 generation: u64,
110 snapshot: Option<Arc<Vec<PathBuf>>>,
111}
112
113#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
115pub struct WriteOptions {
116 pub silent: bool,
118 pub assume_loose_only_existence: bool,
124 pub trust_new_loose: bool,
127 pub loose_prefix_dirs_ready: bool,
130 pub freshen_existing: bool,
140}
141
142pub const BULK_NEW_LOOSE_WRITE: WriteOptions = WriteOptions {
147 assume_loose_only_existence: true,
148 trust_new_loose: true,
149 silent: false,
150 loose_prefix_dirs_ready: true,
151 freshen_existing: false,
152};
153
154pub const BULK_TRACKED_RESTAGE_LOOSE_WRITE: WriteOptions = WriteOptions {
156 assume_loose_only_existence: true,
157 trust_new_loose: false,
158 silent: false,
159 loose_prefix_dirs_ready: true,
160 freshen_existing: false,
161};
162
163fn exists_materialized_in_objects_dir(objects_dir: &Path, oid: &ObjectId) -> bool {
169 store::local_object_materialized(objects_dir, oid)
170}
171
172fn filter_oids(
174 oids: impl IntoIterator<Item = ObjectId>,
175 mut keep: impl FnMut(&ObjectId) -> Result<bool>,
176) -> Result<Vec<ObjectId>> {
177 oids.into_iter()
178 .filter_map(|oid| keep(&oid).map(|k| k.then_some(oid)).transpose())
179 .collect()
180}
181
182fn objects_dir_has_pack_index_files(objects_dir: &Path) -> bool {
183 let pack_dir = objects_dir.join("pack");
184 let Ok(entries) = fs::read_dir(pack_dir) else {
185 return false;
186 };
187 entries
188 .filter_map(|e| e.ok())
189 .any(|ent| ent.path().extension().is_some_and(|ext| ext == "idx"))
190}
191
192#[derive(Clone)]
212pub struct Odb {
213 objects_dir: PathBuf,
214 work_tree: Option<PathBuf>,
216 submodule_alternate_dirs: Arc<Mutex<Vec<PathBuf>>>,
218 config_git_dir: Option<PathBuf>,
220 core_multi_pack_index_cache: Arc<OnceLock<bool>>,
226 local_pack_indexes_seen: Arc<OnceLock<bool>>,
228 file_alternate_dirs_cache: Arc<RwLock<FileAlternatesCache>>,
230 #[cfg(test)]
231 exists_probe: Arc<AtomicUsize>,
232 #[cfg(test)]
233 pub(crate) hot_path_test_metrics: Arc<crate::hot_path_test_metrics::HotPathTestMetrics>,
234 env_alternate_dirs: Vec<PathBuf>,
236 env_alternate_lazy: Arc<OnceLock<Arc<Vec<PathBuf>>>>,
238 mem_overlay: MemOdbOverlayStore,
243 mem_overlay_active: Arc<AtomicBool>,
245 primary_source: Arc<OnceLock<Arc<FilesSource>>>,
247 alternate_sources_cache: Arc<RwLock<AlternateSourcesCache>>,
249 midx_packs_validated: Arc<OnceLock<()>>,
251 hash_algo_cache: Arc<OnceLock<HashAlgo>>,
255 loose_zlib_cache: Arc<OnceLock<Compression>>,
257 pack_zlib_cache: Arc<OnceLock<Compression>>,
259 big_file_threshold_cache: Arc<OnceLock<u64>>,
261 shared_config_state: Option<crate::repo::RepositoryConfigSnapshot>,
263 delta_base_cache_sync: Arc<OnceLock<()>>,
265 shared_environment: Option<Arc<crate::environment::Environment>>,
267 empty_config_env: Arc<OnceLock<crate::environment::Environment>>,
269 freshened_packs: Arc<Mutex<HashSet<PathBuf>>>,
272 pack_store: Arc<PackStore>,
274 alternate_pack_stores: Arc<Mutex<HashMap<PathBuf, Arc<PackStore>>>>,
276 submodule_sources_generation: Arc<AtomicU64>,
278 custom_primary: Option<Arc<dyn WritableObjectStore>>,
280 extra_read_sources: Vec<Arc<dyn ObjectStore>>,
282 alternates_enabled: bool,
284 attached_commit_graph: Option<Arc<crate::commit_graph_file::CommitGraphChain>>,
286 session_loose_reads: Arc<std::sync::Mutex<SessionLooseReads>>,
288 session_tree_entries: Arc<Mutex<SessionTreeEntries>>,
290}
291
292const SESSION_BOUNDED_LOOSE_CACHE_MAX: usize = 64 * 1024;
296
297const SESSION_READ_BLOB_CACHE_MAX: usize = 4 * 1024;
301
302const SESSION_LOOSE_CACHE_MAX_ENTRIES: usize = 256;
304
305const SESSION_LOOSE_CACHE_MAX_BYTES: usize = 16 * 1024 * 1024;
308
309#[derive(Default)]
312struct SessionLooseReads {
313 objects: OidHashMap<Object>,
314 bytes: usize,
315}
316
317impl SessionLooseReads {
318 fn get(&self, oid: &ObjectId) -> Option<&Object> {
319 self.objects.get(oid)
320 }
321
322 fn insert(&mut self, oid: ObjectId, obj: &Object) {
323 let len = obj.data.len();
324 if len > SESSION_LOOSE_CACHE_MAX_BYTES {
325 return;
326 }
327 if self.objects.len() > SESSION_LOOSE_CACHE_MAX_ENTRIES
328 || self.bytes.saturating_add(len) > SESSION_LOOSE_CACHE_MAX_BYTES
329 {
330 self.objects.clear();
331 self.bytes = 0;
332 }
333 if let Some(old) = self.objects.insert(oid, obj.clone()) {
334 self.bytes = self.bytes.saturating_sub(old.data.len());
335 }
336 self.bytes += len;
337 }
338
339 #[cfg(test)]
340 fn is_empty(&self) -> bool {
341 self.objects.is_empty()
342 }
343}
344
345#[derive(Clone)]
348pub(crate) enum SessionTree {
349 Parsed(Arc<Vec<crate::objects::TreeEntry>>),
351 Raw(Arc<Vec<u8>>),
353}
354
355impl SessionTree {
356 const PARSED_ENTRY_BYTES: usize = 80;
358
359 fn cost(&self) -> usize {
360 match self {
361 Self::Parsed(entries) => entries.len().saturating_mul(Self::PARSED_ENTRY_BYTES),
362 Self::Raw(data) => data.len(),
363 }
364 }
365}
366
367#[derive(Default)]
370struct SessionTreeEntries {
371 trees: OidHashMap<SessionTree>,
372 bytes: usize,
373}
374
375impl SessionTreeEntries {
376 const BYTE_BUDGET: usize = (1 << 16) * SessionTree::PARSED_ENTRY_BYTES;
380
381 fn insert(&mut self, oid: ObjectId, tree: SessionTree) {
382 let cost = tree.cost();
383 if cost > Self::BYTE_BUDGET {
384 return;
385 }
386 if let Some(old) = self.trees.remove(&oid) {
387 self.bytes -= old.cost();
388 }
389 if self.bytes + cost > Self::BYTE_BUDGET {
390 self.trees.clear();
391 self.bytes = 0;
392 }
393 self.bytes += cost;
394 self.trees.insert(oid, tree);
395 }
396}
397
398#[cfg(test)]
399mod session_tree_entries_tests {
400 use super::*;
401 use crate::objects::TreeEntry;
402
403 fn tree(entries: usize) -> SessionTree {
404 let entry = TreeEntry {
405 mode: 0o100644,
406 name: b"f".to_vec(),
407 oid: ObjectId::zero(),
408 };
409 SessionTree::Parsed(Arc::new(vec![entry; entries]))
410 }
411
412 fn raw(bytes: usize) -> SessionTree {
413 SessionTree::Raw(Arc::new(vec![0u8; bytes]))
414 }
415
416 fn oid(n: u8) -> ObjectId {
417 ObjectId::from_bytes(&[n; 20]).expect("oid")
418 }
419
420 const ENTRY_BUDGET: usize = SessionTreeEntries::BYTE_BUDGET / SessionTree::PARSED_ENTRY_BYTES;
421
422 #[test]
423 fn cache_starts_over_when_the_budget_is_exceeded() {
424 let mut cache = SessionTreeEntries::default();
425 let half = ENTRY_BUDGET / 2;
426 cache.insert(oid(1), tree(half));
427 cache.insert(oid(2), tree(half));
428 assert_eq!(cache.bytes, SessionTreeEntries::BYTE_BUDGET);
429 assert_eq!(cache.trees.len(), 2);
430
431 cache.insert(oid(3), tree(1));
432 assert_eq!(cache.trees.len(), 1);
433 assert!(cache.trees.contains_key(&oid(3)));
434 assert_eq!(cache.bytes, SessionTree::PARSED_ENTRY_BYTES);
435
436 cache.insert(oid(4), tree(ENTRY_BUDGET + 1));
437 assert!(
438 !cache.trees.contains_key(&oid(4)),
439 "an oversized tree is not cached"
440 );
441 assert!(cache.trees.contains_key(&oid(3)));
442 assert!(cache.bytes <= SessionTreeEntries::BYTE_BUDGET);
443 }
444
445 #[test]
446 fn raw_trees_count_their_payload_bytes() {
447 let mut cache = SessionTreeEntries::default();
448 cache.insert(oid(1), raw(1000));
449 cache.insert(oid(2), tree(10));
450 assert_eq!(cache.bytes, 1000 + 10 * SessionTree::PARSED_ENTRY_BYTES);
451
452 cache.insert(oid(3), raw(SessionTreeEntries::BYTE_BUDGET + 1));
453 assert!(
454 !cache.trees.contains_key(&oid(3)),
455 "an oversized payload is not cached"
456 );
457 assert_eq!(cache.trees.len(), 2);
458 }
459
460 #[test]
461 fn replacing_a_tree_releases_its_old_cost() {
462 let mut cache = SessionTreeEntries::default();
463 cache.insert(oid(1), raw(4000));
464 cache.insert(oid(1), tree(2));
465 assert_eq!(cache.trees.len(), 1);
466 assert_eq!(cache.bytes, 2 * SessionTree::PARSED_ENTRY_BYTES);
467 assert!(matches!(
468 cache.trees.get(&oid(1)),
469 Some(SessionTree::Parsed(_))
470 ));
471 }
472}
473
474const _: () = {
475 fn assert_send_sync<T: Send + Sync>() {}
476 let _ = assert_send_sync::<Odb>;
477};
478
479impl std::fmt::Debug for Odb {
480 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
481 f.debug_struct("Odb")
482 .field("objects_dir", &self.objects_dir)
483 .field("work_tree", &self.work_tree)
484 .field("submodule_alternate_dirs", &"<mutex>")
485 .field("config_git_dir", &self.config_git_dir)
486 .finish()
487 }
488}
489
490#[must_use]
495pub fn hash_algo_for_git_dir(env: &crate::environment::Environment, git_dir: &Path) -> HashAlgo {
496 ConfigSet::load(env, Some(git_dir), true)
497 .ok()
498 .and_then(|cfg| cfg.get("extensions.objectformat"))
499 .and_then(|v| HashAlgo::from_name(&v))
500 .unwrap_or(HashAlgo::Sha1)
501}
502
503#[must_use]
505pub fn hash_algo_for_objects_dir(
506 env: &crate::environment::Environment,
507 objects_dir: &Path,
508) -> HashAlgo {
509 objects_dir
510 .parent()
511 .map(|d| hash_algo_for_git_dir(env, d))
512 .unwrap_or(HashAlgo::Sha1)
513}
514
515impl Odb {
516 #[must_use]
523 pub fn new(objects_dir: &Path) -> Self {
524 Self {
525 objects_dir: objects_dir.to_path_buf(),
526 work_tree: None,
527 submodule_alternate_dirs: Arc::new(Mutex::new(Vec::new())),
528 config_git_dir: None,
529 core_multi_pack_index_cache: Arc::new(OnceLock::new()),
530 local_pack_indexes_seen: Arc::new(OnceLock::new()),
531 file_alternate_dirs_cache: Arc::new(RwLock::new(FileAlternatesCache {
532 generation: 0,
533 snapshot: None,
534 })),
535 env_alternate_dirs: Vec::new(),
536 env_alternate_lazy: Arc::new(OnceLock::new()),
537 mem_overlay: Arc::new(RwLock::new(None)),
538 mem_overlay_active: Arc::new(AtomicBool::new(false)),
539 primary_source: Arc::new(OnceLock::new()),
540 alternate_sources_cache: Arc::new(RwLock::new(AlternateSourcesCache {
541 file_generation: 0,
542 env_fingerprint: 0,
543 submodule_generation: 0,
544 sources: None,
545 })),
546 midx_packs_validated: Arc::new(OnceLock::new()),
547 hash_algo_cache: Arc::new(OnceLock::new()),
548 loose_zlib_cache: Arc::new(OnceLock::new()),
549 pack_zlib_cache: Arc::new(OnceLock::new()),
550 big_file_threshold_cache: Arc::new(OnceLock::new()),
551 shared_config_state: None,
552 delta_base_cache_sync: Arc::new(OnceLock::new()),
553 shared_environment: None,
554 empty_config_env: Arc::new(OnceLock::new()),
555 freshened_packs: Arc::new(Mutex::new(HashSet::new())),
556 pack_store: Arc::new(PackStore::new(objects_dir.to_path_buf())),
557 alternate_pack_stores: Arc::new(Mutex::new(HashMap::new())),
558 submodule_sources_generation: Arc::new(AtomicU64::new(0)),
559 custom_primary: None,
560 extra_read_sources: Vec::new(),
561 alternates_enabled: true,
562 attached_commit_graph: None,
563 session_loose_reads: Arc::new(Mutex::new(SessionLooseReads::default())),
564 session_tree_entries: Arc::new(Mutex::new(SessionTreeEntries::default())),
565 #[cfg(test)]
566 exists_probe: Arc::new(AtomicUsize::new(0)),
567 #[cfg(test)]
568 hot_path_test_metrics: crate::hot_path_test_metrics::HotPathTestMetrics::new(),
569 }
570 }
571
572 #[must_use]
577 pub fn with_work_tree(objects_dir: &Path, work_tree: &Path) -> Self {
578 Self {
579 objects_dir: objects_dir.to_path_buf(),
580 work_tree: Some(work_tree.to_path_buf()),
581 submodule_alternate_dirs: Arc::new(Mutex::new(Vec::new())),
582 config_git_dir: None,
583 core_multi_pack_index_cache: Arc::new(OnceLock::new()),
584 local_pack_indexes_seen: Arc::new(OnceLock::new()),
585 file_alternate_dirs_cache: Arc::new(RwLock::new(FileAlternatesCache {
586 generation: 0,
587 snapshot: None,
588 })),
589 env_alternate_dirs: Vec::new(),
590 env_alternate_lazy: Arc::new(OnceLock::new()),
591 mem_overlay: Arc::new(RwLock::new(None)),
592 mem_overlay_active: Arc::new(AtomicBool::new(false)),
593 primary_source: Arc::new(OnceLock::new()),
594 alternate_sources_cache: Arc::new(RwLock::new(AlternateSourcesCache {
595 file_generation: 0,
596 env_fingerprint: 0,
597 submodule_generation: 0,
598 sources: None,
599 })),
600 midx_packs_validated: Arc::new(OnceLock::new()),
601 hash_algo_cache: Arc::new(OnceLock::new()),
602 loose_zlib_cache: Arc::new(OnceLock::new()),
603 pack_zlib_cache: Arc::new(OnceLock::new()),
604 big_file_threshold_cache: Arc::new(OnceLock::new()),
605 shared_config_state: None,
606 delta_base_cache_sync: Arc::new(OnceLock::new()),
607 shared_environment: None,
608 empty_config_env: Arc::new(OnceLock::new()),
609 freshened_packs: Arc::new(Mutex::new(HashSet::new())),
610 pack_store: Arc::new(PackStore::new(objects_dir.to_path_buf())),
611 alternate_pack_stores: Arc::new(Mutex::new(HashMap::new())),
612 submodule_sources_generation: Arc::new(AtomicU64::new(0)),
613 custom_primary: None,
614 extra_read_sources: Vec::new(),
615 alternates_enabled: true,
616 attached_commit_graph: None,
617 session_loose_reads: Arc::new(Mutex::new(SessionLooseReads::default())),
618 session_tree_entries: Arc::new(Mutex::new(SessionTreeEntries::default())),
619 #[cfg(test)]
620 exists_probe: Arc::new(AtomicUsize::new(0)),
621 #[cfg(test)]
622 hot_path_test_metrics: crate::hot_path_test_metrics::HotPathTestMetrics::new(),
623 }
624 }
625
626 fn session_tree(&self, oid: &ObjectId) -> Option<SessionTree> {
627 self.session_tree_entries
628 .lock()
629 .ok()?
630 .trees
631 .get(oid)
632 .cloned()
633 }
634
635 fn remember_session_tree(&self, oid: &ObjectId, tree: SessionTree) {
636 if let Ok(mut guard) = self.session_tree_entries.lock() {
637 guard.insert(*oid, tree);
638 }
639 }
640
641 fn read_tree_payload(&self, oid: &ObjectId) -> Result<Vec<u8>> {
642 let obj = self.read_impl(oid, false)?;
643 if obj.kind != ObjectKind::Tree {
644 return Err(Error::CorruptObject(format!(
645 "expected tree, got {}",
646 obj.kind.as_str()
647 )));
648 }
649 Ok(obj.data)
650 }
651
652 pub(crate) fn cached_tree_entries(
654 &self,
655 oid: &ObjectId,
656 ) -> Result<Arc<Vec<crate::objects::TreeEntry>>> {
657 let parsed = match self.session_tree(oid) {
658 Some(SessionTree::Parsed(entries)) => return Ok(entries),
659 Some(SessionTree::Raw(data)) => {
660 crate::objects::parse_tree_with_oid_len(&data, self.hash_algo().len())?
661 }
662 None => crate::objects::parse_tree_with_oid_len(
663 &self.read_tree_payload(oid)?,
664 self.hash_algo().len(),
665 )?,
666 };
667 let arc = Arc::new(parsed);
668 self.remember_session_tree(oid, SessionTree::Parsed(Arc::clone(&arc)));
669 Ok(arc)
670 }
671
672 pub(crate) fn cached_tree_for_walk(&self, oid: &ObjectId) -> Result<SessionTree> {
677 if let Some(tree) = self.session_tree(oid) {
678 return Ok(tree);
679 }
680 let tree = SessionTree::Raw(Arc::new(self.read_tree_payload(oid)?));
681 self.remember_session_tree(oid, tree.clone());
682 Ok(tree)
683 }
684
685 fn session_cached_loose_read(&self, oid: &ObjectId) -> Option<Object> {
686 self.session_loose_reads.lock().ok()?.get(oid).cloned()
687 }
688
689 #[cfg(test)]
691 pub(crate) fn session_read_is_cached(&self, oid: &ObjectId) -> bool {
692 self.session_cached_loose_read(oid).is_some()
693 }
694
695 fn remember_session_loose_read(&self, oid: &ObjectId, obj: &Object) {
698 if let Ok(mut guard) = self.session_loose_reads.lock() {
699 guard.insert(*oid, obj);
700 }
701 }
702
703 fn finish_session_loose_read(
711 &self,
712 oid: &ObjectId,
713 obj: Object,
714 remember: bool,
715 ) -> Result<Object> {
716 if remember
717 && (obj.kind != ObjectKind::Blob || obj.data.len() <= SESSION_READ_BLOB_CACHE_MAX)
718 {
719 self.remember_session_loose_read(oid, &obj);
720 }
721 Ok(obj)
722 }
723
724 #[cfg(test)]
726 #[must_use]
727 pub(crate) fn hot_path_metrics(&self) -> &crate::hot_path_test_metrics::HotPathTestMetrics {
728 &self.hot_path_test_metrics
729 }
730
731 #[cfg(test)]
732 #[must_use]
733 pub(crate) fn hot_path_metrics_arc(
734 &self,
735 ) -> Arc<crate::hot_path_test_metrics::HotPathTestMetrics> {
736 Arc::clone(&self.hot_path_test_metrics)
737 }
738
739 #[must_use]
741 pub(crate) fn with_shared_config_state(
742 mut self,
743 state: crate::repo::RepositoryConfigSnapshot,
744 env: Arc<crate::environment::Environment>,
745 ) -> Self {
746 self.pack_store.set_repo_caches(Arc::clone(&state));
747 self.shared_config_state = Some(state);
748 self.shared_environment = Some(env);
749 self
750 }
751
752 fn diagnostics_handle(&self) -> crate::diagnostics::DiagnosticsHandle {
753 self.shared_config_state
754 .as_ref()
755 .map(|state| state.diagnostics_handle())
756 .unwrap_or_else(|| Arc::new(NullDiagnostics))
757 }
758
759 pub(crate) fn load_config_cascade(&self) -> Result<Arc<ConfigSet>> {
765 let git_dir = self
766 .config_git_dir
767 .as_deref()
768 .or_else(|| self.objects_dir.parent());
769 if let (Some(caches), Some(env)) = (&self.shared_config_state, &self.shared_environment) {
770 let opts = crate::config::LoadConfigOptions::for_repository(
771 env,
772 git_dir,
773 Some(Arc::clone(caches)),
774 );
775 return caches.load_config_with_options(env.as_ref(), git_dir, &opts);
776 }
777 match git_dir {
778 Some(git_dir) => {
779 ConfigSet::load(self.config_environment(), Some(git_dir), true).map(Arc::new)
780 }
781 None => Ok(Arc::new(ConfigSet::new())),
782 }
783 }
784
785 fn config_environment(&self) -> &crate::environment::Environment {
786 if let Some(env) = self.shared_environment.as_deref() {
787 return env;
788 }
789 self.empty_config_env
790 .get_or_init(crate::environment::Environment::empty)
791 }
792
793 fn env_alternate_dirs_snapshot(&self) -> Arc<Vec<PathBuf>> {
794 if !self.env_alternate_dirs.is_empty() {
795 return Arc::new(self.env_alternate_dirs.clone());
796 }
797 Arc::clone(self.env_alternate_lazy.get_or_init(|| {
798 Arc::new(Self::env_alternate_dirs_from_var(
799 self.config_environment(),
800 self.work_tree.as_deref(),
801 ))
802 }))
803 }
804
805 #[must_use]
809 pub fn env_alternate_dirs_from_var(
810 env: &crate::environment::Environment,
811 resolve_base: Option<&Path>,
812 ) -> Vec<PathBuf> {
813 match env.var("GIT_ALTERNATE_OBJECT_DIRECTORIES") {
814 Some(val) if !val.is_empty() => {
815 let mut dirs = parse_alternate_env(&val);
816 if let Some(base) = resolve_base {
817 for dir in &mut dirs {
818 if dir.is_relative() {
819 *dir = base.join(&dir);
820 }
821 }
822 }
823 dirs
824 }
825 _ => Vec::new(),
826 }
827 }
828
829 #[must_use]
831 pub fn with_env_alternate_dirs(mut self, dirs: Vec<PathBuf>) -> Self {
832 self.env_alternate_dirs = dirs;
833 self.clear_alternate_sources_cache();
834 self
835 }
836
837 fn clear_alternate_sources_cache(&self) {
838 if let Ok(mut guard) = self.alternate_sources_cache.write() {
839 guard.sources = None;
840 }
841 }
842
843 fn alternate_sources_key(&self) -> AlternateSourcesKey {
844 AlternateSourcesKey {
845 file_generation: self
846 .file_alternate_dirs_cache
847 .read()
848 .map(|g| g.generation)
849 .unwrap_or(0),
850 env_fingerprint: self.env_alternate_fingerprint(),
851 submodule_generation: self.submodule_sources_generation.load(Ordering::Relaxed),
852 }
853 }
854
855 fn env_alternate_fingerprint(&self) -> u64 {
856 let mut hasher = std::collections::hash_map::DefaultHasher::new();
857 for path in self.env_alternate_dirs_snapshot().iter() {
858 path.hash(&mut hasher);
859 }
860 hasher.finish()
861 }
862
863 pub fn refresh_file_alternates_from_disk(&self) -> Result<()> {
869 let snapshot = Arc::new(pack::read_alternates_recursive(&self.objects_dir)?);
870 if let Ok(mut guard) = self.file_alternate_dirs_cache.write() {
871 guard.generation = guard.generation.wrapping_add(1);
872 guard.snapshot = Some(snapshot);
873 }
874 self.clear_alternate_sources_cache();
875 Ok(())
876 }
877
878 pub fn append_file_alternate(&self, alternate_objects_dir: &Path) -> Result<()> {
884 Self::append_alternate_objects_line(&self.objects_dir, alternate_objects_dir)?;
885 self.refresh_file_alternates_from_disk()?;
886 Ok(())
887 }
888
889 pub fn append_alternate_objects_line(
894 objects_dir: &Path,
895 alternate_objects_dir: &Path,
896 ) -> Result<()> {
897 let info = objects_dir.join("info");
898 fs::create_dir_all(&info)?;
899 let alt_path = info.join("alternates");
900 let alt_abs = alternate_objects_dir
901 .canonicalize_simplified()
902 .unwrap_or_else(|_| alternate_objects_dir.to_path_buf());
903 let line = format!("{}\n", alt_abs.display());
904 let existing = match fs::read_to_string(&alt_path) {
905 Ok(s) => s,
906 Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(),
907 Err(e) => return Err(Error::Io(e)),
908 };
909 if existing
910 .lines()
911 .any(|l| l.trim() == alt_abs.to_string_lossy())
912 {
913 return Ok(());
914 }
915 let mut out = existing;
916 out.push_str(&line);
917 fs::write(alt_path, out).map_err(Error::Io)
918 }
919
920 fn file_alternate_dirs_snapshot(&self) -> Arc<Vec<PathBuf>> {
921 loop {
922 let observed_gen = self
923 .file_alternate_dirs_cache
924 .read()
925 .map(|g| g.generation)
926 .unwrap_or(0);
927 if let Ok(guard) = self.file_alternate_dirs_cache.read() {
928 if guard.generation == observed_gen {
929 if let Some(snapshot) = &guard.snapshot {
930 return Arc::clone(snapshot);
931 }
932 }
933 }
934 if let Ok(mut guard) = self.file_alternate_dirs_cache.write() {
935 if guard.generation != observed_gen {
936 continue;
937 }
938 if let Some(snapshot) = &guard.snapshot {
939 return Arc::clone(snapshot);
940 }
941 let snapshot = Arc::new(
942 pack::read_alternates_recursive(&self.objects_dir).unwrap_or_default(),
943 );
944 guard.snapshot = Some(Arc::clone(&snapshot));
945 return snapshot;
946 }
947 break;
948 }
949 Arc::new(Vec::new())
950 }
951
952 #[cfg(test)]
953 pub(crate) fn exists_probe_count(&self) -> usize {
954 self.exists_probe.load(Ordering::Relaxed)
955 }
956
957 #[cfg(test)]
958 pub(crate) fn reset_exists_probe(&self) {
959 self.exists_probe.store(0, Ordering::Relaxed);
960 }
961
962 pub fn enable_mem_overlay(&self) {
967 self.mem_overlay_active.store(true, Ordering::Release);
968 if let Ok(mut guard) = self.mem_overlay.write() {
969 if guard.is_none() {
970 *guard = Some(Arc::new(MemoryStore::new(self.hash_algo())));
971 }
972 }
973 }
974
975 pub fn disable_mem_overlay(&self) {
977 self.mem_overlay_active.store(false, Ordering::Release);
978 if let Ok(mut guard) = self.mem_overlay.write() {
979 *guard = None;
980 }
981 }
982
983 #[must_use]
985 pub(crate) fn is_mem_overlay_active(&self) -> bool {
986 self.mem_overlay_active.load(Ordering::Acquire)
987 }
988
989 fn overlay_active(&self) -> bool {
990 self.mem_overlay
991 .read()
992 .ok()
993 .and_then(|g| g.as_ref().map(|_| ()))
994 .is_some()
995 }
996
997 fn overlay_store(&self) -> Option<Arc<MemoryStore>> {
998 if !self.mem_overlay_active.load(Ordering::Acquire) {
999 return None;
1000 }
1001 self.mem_overlay.read().ok().and_then(|g| g.clone())
1002 }
1003
1004 #[cfg(test)]
1006 pub(crate) fn mem_overlay_len_for_tests(&self) -> Option<usize> {
1007 let store = self.overlay_store()?;
1008 let mut count = 0usize;
1009 store
1010 .for_each_object(&mut |_| {
1011 count += 1;
1012 ControlFlow::Continue(())
1013 })
1014 .ok()?;
1015 Some(count)
1016 }
1017
1018 fn overlay_store_object(&self, oid: ObjectId, kind: ObjectKind, data: &[u8]) -> bool {
1021 let Some(store) = self.overlay_store() else {
1022 return false;
1023 };
1024 store.insert_object(oid, kind, Arc::from(data.to_owned()));
1025 true
1026 }
1027
1028 fn overlay_read(&self, oid: &ObjectId) -> Option<Object> {
1030 let store = self.overlay_store()?;
1031 store.read(oid).ok().flatten()
1032 }
1033
1034 pub fn primary(&self) -> Result<Arc<FilesSource>> {
1046 self.require_files_primary("primary")?;
1047 if let Some(src) = self.primary_source.get() {
1048 return Ok(Arc::clone(src));
1049 }
1050 let src = self.build_files_source(
1051 &self.objects_dir,
1052 self.multi_pack_index_reads_enabled(),
1053 self.midx_tip_listing_enabled(),
1054 )?;
1055 match self.primary_source.set(Arc::clone(&src)) {
1056 Ok(()) => Ok(src),
1057 Err(existing) => Ok(existing),
1058 }
1059 }
1060
1061 #[must_use]
1066 pub(crate) fn has_secondary_object_stores(&self) -> bool {
1067 if !self.extra_read_sources.is_empty() {
1068 return true;
1069 }
1070 self.alternates_enabled && self.alternate_object_lookup_needed()
1071 }
1072
1073 #[must_use]
1075 fn alternate_object_lookup_needed(&self) -> bool {
1076 if !self.file_alternate_dirs_snapshot().is_empty() {
1077 return true;
1078 }
1079 if !self.env_alternate_dirs_snapshot().is_empty() {
1080 return true;
1081 }
1082 self.submodule_alternate_dirs
1083 .lock()
1084 .ok()
1085 .is_some_and(|guard| !guard.is_empty())
1086 }
1087
1088 pub fn sources(&self) -> Result<Arc<CompositeStore>> {
1098 if !self.alternates_enabled {
1099 return Ok(Arc::new(CompositeStore::new(Vec::new(), self.hash_algo())));
1100 }
1101 let key = self.alternate_sources_key();
1102 if let Ok(guard) = self.alternate_sources_cache.read() {
1103 if guard.file_generation == key.file_generation
1104 && guard.env_fingerprint == key.env_fingerprint
1105 && guard.submodule_generation == key.submodule_generation
1106 {
1107 if let Some(sources) = guard.sources.as_ref() {
1108 return Ok(Arc::clone(sources));
1109 }
1110 }
1111 }
1112 let use_midx = self.multi_pack_index_reads_enabled();
1113 let midx_tip = self.midx_tip_listing_enabled();
1114 let mut stores: Vec<Arc<dyn ObjectStore>> = Vec::new();
1115 for alt_dir in self.file_alternate_dirs_snapshot().iter() {
1116 stores.push(self.build_files_source(alt_dir, use_midx, midx_tip)?);
1117 }
1118 for alt_dir in self.env_alternate_dirs_snapshot().iter() {
1119 stores.push(self.build_files_source(alt_dir, use_midx, midx_tip)?);
1120 }
1121 if let Ok(guard) = self.submodule_alternate_dirs.lock() {
1122 for alt_dir in guard.iter() {
1123 stores.push(self.build_files_source(alt_dir, false, false)?);
1124 }
1125 }
1126 let composite = Arc::new(CompositeStore::new(stores, self.hash_algo()));
1127 if let Ok(mut guard) = self.alternate_sources_cache.write() {
1128 guard.file_generation = key.file_generation;
1129 guard.env_fingerprint = key.env_fingerprint;
1130 guard.submodule_generation = key.submodule_generation;
1131 guard.sources = Some(Arc::clone(&composite));
1132 }
1133 Ok(composite)
1134 }
1135
1136 fn midx_tip_listing_enabled(&self) -> bool {
1137 self.config_git_dir.is_some() && self.core_multi_pack_index_enabled()
1138 }
1139
1140 fn build_files_source(
1141 &self,
1142 objects_dir: &Path,
1143 use_midx: bool,
1144 midx_tip_listing_for_exists: bool,
1145 ) -> Result<Arc<FilesSource>> {
1146 let loose = self.loose_at(objects_dir)?;
1147 let pack_store = self.pack_store_for(objects_dir);
1148 let diagnostics = if objects_dir == self.objects_dir.as_path() {
1149 self.diagnostics_handle()
1150 } else {
1151 Arc::new(NullDiagnostics)
1152 };
1153 FilesSource::open(
1154 pack_store,
1155 loose,
1156 use_midx,
1157 midx_tip_listing_for_exists,
1158 diagnostics,
1159 )
1160 .map(Arc::new)
1161 }
1162
1163 pub fn register_submodule_object_directories_from_index(
1167 &self,
1168 work_tree: &Path,
1169 index: &crate::index::Index,
1170 ) {
1171 use crate::diff::submodule_embedded_git_dir;
1172
1173 let Ok(mut dirs) = self.submodule_alternate_dirs.lock() else {
1174 return;
1175 };
1176 dirs.clear();
1177 for e in &index.entries {
1178 if e.stage() != 0 || e.mode != crate::index::MODE_GITLINK {
1179 continue;
1180 }
1181 let path_str = String::from_utf8_lossy(&e.path);
1182 let abs = work_tree.join(path_str.as_ref());
1183 let Some(sub_git) = submodule_embedded_git_dir(&abs) else {
1184 continue;
1185 };
1186 let objects = sub_git.join("objects");
1187 if !objects.is_dir() {
1188 continue;
1189 }
1190 let canon = objects.canonicalize_simplified().unwrap_or(objects);
1191 if !dirs.iter().any(|p| p == &canon) {
1192 dirs.push(canon);
1193 }
1194 }
1195 self.submodule_sources_generation
1196 .fetch_add(1, Ordering::Relaxed);
1197 self.clear_alternate_sources_cache();
1198 }
1199
1200 #[must_use]
1202 pub fn with_config_git_dir(mut self, git_dir: PathBuf) -> Self {
1203 self.config_git_dir = Some(git_dir);
1204 self
1205 }
1206
1207 #[must_use]
1219 pub fn hash_algo(&self) -> HashAlgo {
1220 if let Some(primary) = &self.custom_primary {
1221 return primary.hash_algo();
1222 }
1223 *self.hash_algo_cache.get_or_init(|| {
1224 if self.config_git_dir.is_none() && self.objects_dir.parent().is_none() {
1225 return HashAlgo::Sha1;
1226 }
1227 let cfg = self.load_config_cascade().unwrap_or_default();
1228 cfg.get("extensions.objectformat")
1229 .and_then(|v| HashAlgo::from_name(&v))
1230 .unwrap_or(HashAlgo::Sha1)
1231 })
1232 }
1233
1234 fn resolve_loose_zlib_level(&self) -> Result<u32> {
1235 if self.config_git_dir.is_none() && self.objects_dir.parent().is_none() {
1236 return Ok(ConfigSet::LOOSE_OBJECTS_ZLIB_DEFAULT as u32);
1237 }
1238 let cfg = self.load_config_cascade()?;
1239 Ok(cfg.loose_objects_zlib_level()? as u32)
1240 }
1241
1242 pub(crate) fn loose_compression(&self) -> Result<Compression> {
1249 if let Some(cached) = self.loose_zlib_cache.get() {
1250 return Ok(*cached);
1251 }
1252 let level = self.resolve_loose_zlib_level()?;
1253 let compression = Compression::new(level);
1254 let _ = self.loose_zlib_cache.set(compression);
1255 Ok(compression)
1256 }
1257
1258 fn resolve_pack_zlib_level(&self) -> Result<i32> {
1259 if self.config_git_dir.is_none() && self.objects_dir.parent().is_none() {
1260 return Ok(6);
1261 }
1262 let cfg = self.load_config_cascade()?;
1263 cfg.pack_objects_zlib_level()
1264 }
1265
1266 pub(crate) fn pack_compression(&self) -> Result<Compression> {
1272 if let Some(cached) = self.pack_zlib_cache.get() {
1273 return Ok(*cached);
1274 }
1275 let level = self.resolve_pack_zlib_level()?;
1276 let compression = crate::pack_zlib::flate2_compression_for_pack_level(level);
1277 let _ = self.pack_zlib_cache.set(compression);
1278 Ok(compression)
1279 }
1280
1281 pub fn big_file_threshold(&self) -> Result<u64> {
1292 if let Some(cached) = self.big_file_threshold_cache.get() {
1293 return Ok(*cached);
1294 }
1295 let threshold = if self.config_git_dir.is_none() && self.objects_dir.parent().is_none() {
1296 ConfigSet::BIG_FILE_THRESHOLD_DEFAULT
1297 } else {
1298 self.load_config_cascade()?.big_file_threshold()?
1299 };
1300 let _ = self.big_file_threshold_cache.set(threshold);
1301 Ok(threshold)
1302 }
1303
1304 pub(crate) fn with_bounded_pack_cache(&self) -> Self {
1308 let mut odb = self.clone();
1309 odb.pack_store = Arc::new(PackStore::for_bounded_reads(self.objects_dir.clone()));
1310 odb.primary_source = Arc::new(OnceLock::new());
1311 odb.session_loose_reads = Arc::new(Mutex::new(SessionLooseReads::default()));
1312 odb.alternate_pack_stores = Arc::new(Mutex::new(HashMap::new()));
1313 odb.alternate_sources_cache = Arc::new(RwLock::new(AlternateSourcesCache {
1314 file_generation: 0,
1315 env_fingerprint: 0,
1316 submodule_generation: 0,
1317 sources: None,
1318 }));
1319 odb
1320 }
1321
1322 fn sync_delta_base_cache_limit(&self) {
1323 if self.pack_store.bounded_reads() {
1324 return;
1325 }
1326 self.delta_base_cache_sync.get_or_init(|| {
1327 if self.config_git_dir.is_some() {
1328 let cfg = self.load_config_cascade().unwrap_or_default();
1329 self.pack_store
1330 .configure_delta_base_from_config(Some(cfg.as_ref()));
1331 }
1332 });
1333 }
1334
1335 pub(crate) fn pack_store_for(&self, objects_dir: &Path) -> Arc<PackStore> {
1336 if objects_dir == self.objects_dir.as_path() {
1337 return Arc::clone(&self.pack_store);
1338 }
1339 let new_store = || {
1340 Arc::new(if self.pack_store.bounded_reads() {
1341 PackStore::for_bounded_reads(objects_dir.to_path_buf())
1342 } else {
1343 PackStore::new(objects_dir.to_path_buf())
1344 })
1345 };
1346 if let Ok(mut guard) = self.alternate_pack_stores.lock() {
1347 return Arc::clone(
1348 guard
1349 .entry(objects_dir.to_path_buf())
1350 .or_insert_with(new_store),
1351 );
1352 }
1353 new_store()
1354 }
1355
1356 pub fn pack_store(&self) -> Result<&Arc<PackStore>> {
1358 self.require_files_primary("pack_store")?;
1359 Ok(&self.pack_store)
1360 }
1361
1362 pub fn largest_pack_index_path(&self) -> Result<Option<PathBuf>> {
1371 self.require_files_primary("largest_pack_index_path")?;
1372 let mut indexes = pack::read_local_pack_indexes_cached(self.objects_dir())?;
1373 indexes.sort_by_key(|idx| std::cmp::Reverse(idx.len()));
1374 Ok(indexes.first().map(|idx| idx.idx_path.clone()))
1375 }
1376
1377 pub(crate) fn clear_delta_base_cache(&self) {
1381 self.pack_store.delta_bases().clear();
1382 }
1383
1384 pub fn invalidate_packs(&self) {
1391 self.pack_store.invalidate_all();
1392 PackStore::invalidate_legacy_for_objects_dir(&self.objects_dir);
1393 if let Ok(guard) = self.alternate_pack_stores.lock() {
1394 for store in guard.values() {
1395 store.invalidate_all();
1396 PackStore::invalidate_legacy_for_objects_dir(store.objects_dir());
1397 }
1398 }
1399 self.invalidate_change_id_header_probe();
1400 }
1401
1402 fn invalidate_change_id_header_probe(&self) {
1403 if let Some(caches) = self.shared_config_state.as_ref() {
1404 crate::change_id::invalidate_stored_change_id_header_probe(caches, self.objects_dir());
1405 }
1406 }
1407
1408 fn note_commit_object_written(&self, kind: ObjectKind) {
1409 if kind == ObjectKind::Commit {
1410 self.invalidate_change_id_header_probe();
1411 }
1412 }
1413
1414 pub(crate) fn loosen_unreachable_from_superseded_packs(
1428 &self,
1429 pack_dir: &Path,
1430 keep_pack: &Path,
1431 reachable: &crate::gc::LazyReachable<'_>,
1432 retention: UnreachablePackRetention,
1433 retained_packs: &HashSet<PathBuf>,
1434 inventory: &RepackPackInventorySnapshot,
1435 ) -> Result<usize> {
1436 if matches!(retention, UnreachablePackRetention::DropAll) {
1437 return Ok(0);
1438 }
1439 self.require_files_primary("repack")?;
1440 let keep_idx = keep_pack.with_extension("idx");
1441 let keep_pack_oids = if keep_idx.is_file() {
1442 PackedObjects::object_ids_from_index_path(&keep_idx)?
1443 } else {
1444 HashSet::new()
1445 };
1446 let kept_pack_oids =
1447 oids_in_inventory_kept_packs(pack_dir, keep_pack, &inventory.kept_at_inventory)?;
1448 let mut loosened = 0usize;
1449 for (idx_path, pack_mtime) in
1450 superseded_pack_indexes(pack_dir, keep_pack, retained_packs, inventory)?
1451 {
1452 let oids = PackedObjects::object_ids_from_index_path(&idx_path)?;
1453 for oid in oids {
1454 let keep = match retention {
1455 UnreachablePackRetention::KeepAll => true,
1456 UnreachablePackRetention::KeepGraceProtected(cutoff) => pack_mtime >= cutoff,
1457 UnreachablePackRetention::DropAll => false,
1458 };
1459 if !keep {
1460 continue;
1461 }
1462 if keep_pack_oids.contains(&oid) || kept_pack_oids.contains(&oid) {
1463 continue;
1464 }
1465 if reachable.get()?.contains(&oid) {
1466 continue;
1467 }
1468 let loose_path = oid.loose_path_in(&self.objects_dir);
1469 let pack_ft = system_time_to_file_time(pack_mtime);
1470 match retention {
1471 UnreachablePackRetention::KeepGraceProtected(cutoff) => {
1472 if loose_path.is_file() {
1473 let loose_mtime = fs::metadata(&loose_path)
1474 .and_then(|m| m.modified())
1475 .unwrap_or(std::time::SystemTime::UNIX_EPOCH);
1476 if loose_mtime >= cutoff {
1477 continue;
1478 }
1479 set_file_mtime(&loose_path, pack_ft).map_err(Error::Io)?;
1480 loosened += 1;
1481 continue;
1482 }
1483 }
1484 UnreachablePackRetention::KeepAll => {
1485 if loose_path.is_file() {
1486 continue;
1487 }
1488 }
1489 UnreachablePackRetention::DropAll => {}
1490 }
1491 let obj = self.read(&oid)?;
1492 self.write_loose_materialize(obj.kind, &obj.data)?;
1493 let primary = self.primary()?;
1494 let loose_path = primary.loose_store().object_path(&oid);
1495 set_file_mtime(&loose_path, pack_ft).map_err(Error::Io)?;
1496 loosened += 1;
1497 }
1498 }
1499 Ok(loosened)
1500 }
1501
1502 fn core_multi_pack_index_enabled(&self) -> bool {
1503 *self.core_multi_pack_index_cache.get_or_init(|| {
1507 if self.config_git_dir.is_none() {
1508 return false;
1509 }
1510 let cfg = self.load_config_cascade().unwrap_or_default();
1511 match cfg.get_bool("core.multiPackIndex") {
1512 Some(Ok(b)) => b,
1513 Some(Err(_)) => true,
1514 None => true,
1515 }
1516 })
1517 }
1518
1519 fn multi_pack_index_reads_enabled(&self) -> bool {
1520 if self.custom_primary.is_some() {
1521 return false;
1522 }
1523 if self.config_git_dir.is_none() {
1524 return false;
1525 }
1526 if !*self
1527 .local_pack_indexes_seen
1528 .get_or_init(|| objects_dir_has_pack_index_files(&self.objects_dir))
1529 {
1530 return false;
1531 }
1532 self.core_multi_pack_index_enabled()
1533 }
1534
1535 fn ensure_midx_prepared(&self) {
1536 if !self.multi_pack_index_reads_enabled() {
1537 return;
1538 }
1539 let _ = self.midx_packs_validated.get_or_init(|| {
1540 let diagnostics = self.diagnostics_handle();
1541 self.with_pack_store_for(&self.objects_dir, || {
1542 validate_midx_referenced_packs_with_diagnostics(
1543 &self.objects_dir,
1544 diagnostics.as_ref(),
1545 );
1546 });
1547 });
1548 }
1549
1550 fn try_read_packed_in_batch_context(&self, oid: &ObjectId) -> Result<Option<Object>> {
1555 if self.custom_primary.is_some()
1556 || !PackStore::read_context_matches_objects_dir(&self.objects_dir)
1557 || self.multi_pack_index_reads_enabled()
1558 {
1559 return Ok(None);
1560 }
1561 self.sync_delta_base_cache_limit();
1562 match pack::try_read_object_from_packs_with_options(
1563 &self.objects_dir,
1564 oid,
1565 pack::PackLookupOptions::ALL_LOCAL_PACKS,
1566 ) {
1567 Ok(obj) => Ok(Some(obj)),
1568 Err(Error::ObjectNotFound(_)) => Ok(None),
1569 Err(err) => Err(err),
1570 }
1571 }
1572
1573 fn try_read_info_packed_in_batch_context(&self, oid: &ObjectId) -> Result<Option<ObjectInfo>> {
1574 if self.custom_primary.is_some()
1575 || !PackStore::read_context_matches_objects_dir(&self.objects_dir)
1576 || self.multi_pack_index_reads_enabled()
1577 {
1578 return Ok(None);
1579 }
1580 match pack::try_read_object_info_from_packs_with_options(
1581 &self.objects_dir,
1582 oid,
1583 pack::PackLookupOptions::ALL_LOCAL_PACKS,
1584 ) {
1585 Ok(info) => Ok(Some(info)),
1586 Err(Error::ObjectNotFound(_)) => Ok(None),
1587 Err(err) => Err(err),
1588 }
1589 }
1590
1591 #[must_use]
1598 pub fn objects_dir(&self) -> &Path {
1599 &self.objects_dir
1600 }
1601
1602 pub(crate) fn commit_graph_chain(
1613 &self,
1614 ) -> Option<Arc<crate::commit_graph_file::CommitGraphChain>> {
1615 if let Some(chain) = &self.attached_commit_graph {
1616 return Some(Arc::clone(chain));
1617 }
1618 if self.custom_primary.is_some() {
1619 return None;
1620 }
1621 self.shared_config_state
1622 .as_ref()?
1623 .commit_graph_chain(self.objects_dir())
1624 }
1625
1626 fn try_read_extra_sources(&self, oid: &ObjectId) -> Result<Option<Object>> {
1627 for src in &self.extra_read_sources {
1628 if let Some(obj) = src.read(oid)? {
1629 return Ok(Some(obj));
1630 }
1631 }
1632 Ok(None)
1633 }
1634
1635 fn try_read_info_extra_sources(&self, oid: &ObjectId) -> Result<Option<ObjectInfo>> {
1636 for src in &self.extra_read_sources {
1637 if let Some(info) = src.read_info(oid)? {
1638 return Ok(Some(info));
1639 }
1640 }
1641 Ok(None)
1642 }
1643
1644 fn try_open_stream_extra_sources(
1645 &self,
1646 oid: &ObjectId,
1647 ) -> Result<Option<store::ObjectStream<'_>>> {
1648 for src in &self.extra_read_sources {
1649 if let Some(stream) = src.open_stream(oid)? {
1650 return Ok(Some(stream));
1651 }
1652 }
1653 Ok(None)
1654 }
1655
1656 #[must_use]
1662 pub fn config_git_dir(&self) -> Option<&Path> {
1663 self.config_git_dir.as_deref()
1664 }
1665
1666 #[must_use]
1668 pub fn object_path(&self, oid: &ObjectId) -> PathBuf {
1669 oid.loose_path_in(&self.objects_dir)
1670 }
1671
1672 pub(crate) fn local_pack_indexes(&self) -> Result<Vec<Arc<pack::PackIndex>>> {
1679 self.with_primary_read_context(|| pack::read_local_pack_indexes_cached(&self.objects_dir))
1682 }
1683
1684 pub(crate) fn contained_in_alternate_pack(&self, oid: &ObjectId) -> Result<bool> {
1692 if !self.alternates_enabled {
1693 return Ok(false);
1694 }
1695 for alt_dir in self.file_alternate_dirs_snapshot().iter() {
1696 if store::object_in_non_promisor_pack(alt_dir, oid) {
1697 return Ok(true);
1698 }
1699 }
1700 for alt_dir in self.env_alternate_dirs_snapshot().iter() {
1701 if store::object_in_non_promisor_pack(alt_dir, oid) {
1702 return Ok(true);
1703 }
1704 }
1705 if let Ok(guard) = self.submodule_alternate_dirs.lock() {
1706 for alt_dir in guard.iter() {
1707 if store::object_in_non_promisor_pack(alt_dir, oid) {
1708 return Ok(true);
1709 }
1710 }
1711 }
1712 Ok(false)
1713 }
1714
1715 pub(crate) fn remove_local_loose(&self, oid: &ObjectId) -> Result<bool> {
1724 match fs::remove_file(self.object_path(oid)) {
1725 Ok(()) => Ok(true),
1726 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(false),
1727 Err(err) => Err(Error::Io(err)),
1728 }
1729 }
1730
1731 pub(crate) fn local_loose_file_size(&self, oid: &ObjectId) -> Option<u64> {
1734 fs::metadata(self.object_path(oid))
1735 .ok()
1736 .filter(fs::Metadata::is_file)
1737 .map(|meta| meta.len())
1738 }
1739
1740 #[must_use]
1756 pub fn exists_local(&self, oid: &ObjectId) -> bool {
1757 self.try_exists_local(oid).unwrap_or(false)
1758 }
1759
1760 pub(crate) fn try_exists_local(&self, oid: &ObjectId) -> Result<bool> {
1762 if self.is_empty_tree(oid) {
1763 return Ok(true);
1764 }
1765 if let Some(primary) = &self.custom_primary {
1766 return primary.contains(oid);
1767 }
1768 Ok(self.with_pack_store_for(&self.objects_dir, || {
1769 exists_materialized_in_objects_dir(&self.objects_dir, oid)
1770 }))
1771 }
1772
1773 #[must_use]
1781 pub fn exists_stored(&self, oid: &ObjectId) -> bool {
1782 self.try_exists_stored(oid).unwrap_or(false)
1783 }
1784
1785 pub(crate) fn try_exists_stored(&self, oid: &ObjectId) -> Result<bool> {
1787 if oid.is_canonical_empty_tree() || self.overlay_read(oid).is_some() {
1788 return Ok(true);
1789 }
1790 if let Some(primary) = &self.custom_primary {
1791 return primary.contains(oid);
1792 }
1793 Ok(self.with_pack_store_for(&self.objects_dir, || {
1794 store::local_object_stored(&self.objects_dir, oid)
1795 }))
1796 }
1797
1798 #[must_use]
1808 pub fn exists_materialized_local(&self, oid: &ObjectId) -> bool {
1809 if let Some(primary) = &self.custom_primary {
1810 return primary.contains(oid).unwrap_or(false);
1811 }
1812 self.with_pack_store_for(&self.objects_dir, || {
1813 exists_materialized_in_objects_dir(&self.objects_dir, oid)
1814 })
1815 }
1816
1817 pub(crate) fn reprepare_local_packs(&self) {
1822 if self.custom_primary.is_some() {
1823 return;
1824 }
1825 self.with_pack_store_for(&self.objects_dir, || {
1826 let _ = crate::pack::reprepare_pack_directory_on_miss(&self.objects_dir);
1827 });
1828 }
1829
1830 pub(crate) fn exists_materialized_local_without_rescan(&self, oid: &ObjectId) -> bool {
1838 if self.custom_primary.is_some() {
1839 return self.exists_materialized_local(oid);
1840 }
1841 self.with_pack_store_for(&self.objects_dir, || {
1842 store::local_object_materialized_without_rescan(&self.objects_dir, oid)
1843 })
1844 }
1845
1846 pub fn contains(&self, oid: &ObjectId) -> Result<bool> {
1858 self.probe_sources(oid, Err)
1859 }
1860
1861 fn probe_sources(
1864 &self,
1865 oid: &ObjectId,
1866 mut on_err: impl FnMut(Error) -> Result<()>,
1867 ) -> Result<bool> {
1868 #[cfg(test)]
1869 self.exists_probe.fetch_add(1, Ordering::Relaxed);
1870 if self.is_empty_tree(oid) || self.overlay_read(oid).is_some() {
1871 return Ok(true);
1872 }
1873 let mut found = |probe: Result<bool>| match probe {
1874 Ok(present) => Ok(present),
1875 Err(err) => on_err(err).map(|()| false),
1876 };
1877 if found(self.primary_writable().and_then(|p| p.contains(oid)))? {
1878 return Ok(true);
1879 }
1880 for src in &self.extra_read_sources {
1881 if found(src.contains(oid))? {
1882 return Ok(true);
1883 }
1884 }
1885 if !self.alternate_object_lookup_needed() {
1886 return Ok(false);
1887 }
1888 found(self.sources().and_then(|s| s.contains(oid)))
1889 }
1890
1891 pub(crate) fn contains_many(&self, oids: &[ObjectId]) -> Result<Vec<bool>> {
1901 #[cfg(test)]
1902 self.exists_probe.fetch_add(oids.len(), Ordering::Relaxed);
1903 let mut present: Vec<bool> = oids
1904 .iter()
1905 .map(|oid| self.is_empty_tree(oid) || self.overlay_read(oid).is_some())
1906 .collect();
1907 let mut pending: Vec<usize> = (0..oids.len()).filter(|&i| !present[i]).collect();
1908 store::resolve_pending(oids, &mut present, &mut pending, |batch| {
1909 self.primary_writable()?.contains_many(batch)
1910 })?;
1911 for src in &self.extra_read_sources {
1912 store::resolve_pending(oids, &mut present, &mut pending, |batch| {
1913 src.contains_many(batch)
1914 })?;
1915 }
1916 if !pending.is_empty() && self.alternate_object_lookup_needed() {
1917 store::resolve_pending(oids, &mut present, &mut pending, |batch| {
1918 self.sources()?.contains_many(batch)
1919 })?;
1920 }
1921 Ok(present)
1922 }
1923
1924 pub(crate) fn present_oids(
1930 &self,
1931 oids: impl IntoIterator<Item = ObjectId>,
1932 ) -> Result<Vec<ObjectId>> {
1933 self.partition_oids(oids, true)
1934 }
1935
1936 pub(crate) fn missing_oids(
1942 &self,
1943 oids: impl IntoIterator<Item = ObjectId>,
1944 ) -> Result<Vec<ObjectId>> {
1945 self.partition_oids(oids, false)
1946 }
1947
1948 fn partition_oids(
1950 &self,
1951 oids: impl IntoIterator<Item = ObjectId>,
1952 want_present: bool,
1953 ) -> Result<Vec<ObjectId>> {
1954 let oids: Vec<ObjectId> = oids.into_iter().collect();
1955 let present = self.contains_many(&oids)?;
1956 Ok(oids
1957 .into_iter()
1958 .zip(present)
1959 .filter_map(|(oid, has)| (has == want_present).then_some(oid))
1960 .collect())
1961 }
1962
1963 pub(crate) fn missing_stored_oids(
1973 &self,
1974 oids: impl IntoIterator<Item = ObjectId>,
1975 ) -> Result<Vec<ObjectId>> {
1976 if self.custom_primary.is_some() {
1977 return filter_oids(oids, |oid| self.try_exists_stored(oid).map(|has| !has));
1978 }
1979 self.reprepare_local_packs();
1980 self.with_pack_store_for(&self.objects_dir, || {
1981 filter_oids(oids, |oid| {
1982 Ok(!(oid.is_canonical_empty_tree()
1983 || self.overlay_read(oid).is_some()
1984 || store::local_object_stored_without_rescan(&self.objects_dir, oid)))
1985 })
1986 })
1987 }
1988
1989 #[must_use]
1996 pub fn exists(&self, oid: &ObjectId) -> bool {
1997 self.probe_sources(oid, |_| Ok(())).unwrap_or(false)
1998 }
1999
2000 fn with_pack_store_for<R>(&self, objects_dir: &Path, f: impl FnOnce() -> R) -> R {
2002 let store = self.pack_store_for(objects_dir);
2003 PackStore::with_context(store, f)
2004 }
2005
2006 fn with_primary_read_context<R>(&self, f: impl FnOnce() -> R) -> R {
2012 if self.custom_primary.is_some()
2013 || PackStore::read_context_matches_objects_dir(&self.objects_dir)
2014 {
2015 f()
2016 } else {
2017 self.with_pack_store_for(&self.objects_dir, f)
2018 }
2019 }
2020
2021 pub fn with_pack_read_context<R>(&self, f: impl FnOnce() -> R) -> R {
2026 self.with_pack_store_for(&self.objects_dir, f)
2027 }
2028
2029 pub(crate) fn pack_read_context_guard(&self) -> crate::pack_store::PackReadContextGuard {
2032 crate::pack_store::PackReadContextGuard::install(self.pack_store_for(&self.objects_dir))
2033 }
2034
2035 pub(crate) fn open_session_pack_reader(&self) -> Result<Option<pack::SingleLocalPackReader>> {
2043 if self.custom_primary.is_some()
2044 || !PackStore::read_context_matches_objects_dir(&self.objects_dir)
2045 {
2046 return Ok(None);
2047 }
2048 self.sync_delta_base_cache_limit();
2049 pack::SingleLocalPackReader::open_allow_loose(&self.objects_dir)
2050 }
2051
2052 #[must_use]
2057 pub fn freshen_object(&self, oid: &ObjectId) -> bool {
2058 if self.is_empty_tree(oid) {
2059 return false;
2060 }
2061
2062 let loose = self.object_path(oid);
2063 if loose.is_file() {
2064 return self.touch_object_mtime(&loose).is_some();
2065 }
2066 self.freshen_packed_or_alternate(oid)
2067 }
2068
2069 fn freshen_packed_or_alternate(&self, oid: &ObjectId) -> bool {
2073 if self.is_empty_tree(oid) {
2074 return false;
2075 }
2076 if self.freshen_packed_in_objects_dir(&self.objects_dir, oid) {
2077 return true;
2078 }
2079
2080 let file_alts = self.file_alternate_dirs_snapshot();
2081 for alt_dir in file_alts.iter() {
2082 if self.freshen_object_in_objects_dir(alt_dir, oid) {
2083 return true;
2084 }
2085 }
2086
2087 for alt_dir in self.env_alternate_dirs_snapshot().iter() {
2088 if self.freshen_object_in_objects_dir(alt_dir, oid) {
2089 return true;
2090 }
2091 }
2092
2093 if let Ok(guard) = self.submodule_alternate_dirs.lock() {
2094 for alt_dir in guard.iter() {
2095 if self.freshen_object_in_objects_dir(alt_dir, oid) {
2096 return true;
2097 }
2098 }
2099 }
2100
2101 false
2102 }
2103
2104 fn freshen_object_in_objects_dir(&self, objects_dir: &Path, oid: &ObjectId) -> bool {
2105 let loose = objects_dir
2106 .join(oid.loose_prefix())
2107 .join(oid.loose_suffix());
2108 if loose.is_file() {
2109 return self.touch_object_mtime(&loose).is_some();
2110 }
2111 self.freshen_packed_in_objects_dir(objects_dir, oid)
2112 }
2113
2114 fn freshen_packed_in_objects_dir(&self, objects_dir: &Path, oid: &ObjectId) -> bool {
2115 self.with_pack_store_for(objects_dir, || {
2116 let Ok(indexes) = pack::read_local_pack_indexes_cached(objects_dir) else {
2117 return false;
2118 };
2119 for idx in &indexes {
2120 if idx.contains(oid).unwrap_or(false) {
2121 return self.freshen_pack_once(&idx.pack_path, idx.is_cruft);
2122 }
2123 }
2124 false
2125 })
2126 }
2127
2128 fn freshen_pack_once(&self, pack_path: &Path, is_cruft: bool) -> bool {
2133 let Ok(mut guard) = self.freshened_packs.lock() else {
2134 return false;
2135 };
2136 if guard.contains(pack_path) {
2137 return true;
2138 }
2139 if is_cruft {
2140 return false;
2141 }
2142 let Some(touched_at) = self.touch_object_mtime(pack_path) else {
2143 return false;
2144 };
2145 pack::refresh_pack_bytes_signature(pack_path, touched_at);
2146 guard.insert(pack_path.to_path_buf());
2147 true
2148 }
2149
2150 #[must_use]
2159 pub fn bulk_stage_write_options(&self) -> WriteOptions {
2160 let stored_beyond_loose = self.custom_primary.is_none()
2161 && (self.alternate_object_lookup_needed()
2162 || self.with_pack_read_context(|| {
2163 pack::read_local_pack_indexes_cached(&self.objects_dir)
2164 .is_ok_and(|indexes| !indexes.is_empty())
2165 }));
2166 WriteOptions {
2167 freshen_existing: stored_beyond_loose,
2168 ..BULK_NEW_LOOSE_WRITE
2169 }
2170 }
2171
2172 fn try_freshen_existing(&self, oid: &ObjectId) -> Option<ObjectId> {
2175 if let Ok(primary) = self.primary() {
2176 if let Ok(Some(existing)) =
2177 primary.try_finish_duplicate_write(oid, WriteOptions::default())
2178 {
2179 return Some(existing);
2180 }
2181 }
2182 if !self.exists(oid) {
2183 return None;
2184 }
2185 if self.freshen_object(oid) {
2186 return Some(*oid);
2187 }
2188 None
2189 }
2190
2191 fn try_freshen_existing_local(&self, oid: &ObjectId) -> Option<ObjectId> {
2192 let primary = self.primary().ok()?;
2193 primary
2194 .try_finish_duplicate_write(
2195 oid,
2196 WriteOptions {
2197 assume_loose_only_existence: true,
2198 ..WriteOptions::default()
2199 },
2200 )
2201 .ok()
2202 .flatten()
2203 }
2204
2205 fn object_exists_for_write_skip(&self, oid: &ObjectId) -> bool {
2210 if self.is_empty_tree(oid) {
2211 self.exists_materialized_local(oid)
2212 } else {
2213 self.exists(oid)
2214 }
2215 }
2216
2217 pub fn read_loose_verify_oid(path: &Path, expected_oid: &ObjectId) -> Result<Object> {
2228 LooseStore::read_verify_oid(path, expected_oid)
2229 }
2230
2231 fn local_loose(&self) -> Result<LooseStore> {
2232 self.require_files_primary("loose_write")?;
2233 Ok(LooseStore::new(
2234 self.objects_dir.clone(),
2235 self.hash_algo(),
2236 self.loose_compression()?,
2237 ))
2238 }
2239
2240 fn loose_at(&self, objects_dir: &Path) -> Result<LooseStore> {
2241 Ok(LooseStore::new(
2242 objects_dir.to_path_buf(),
2243 self.hash_algo(),
2244 self.loose_compression()?,
2245 ))
2246 }
2247
2248 pub fn read(&self, oid: &ObjectId) -> Result<Object> {
2256 self.read_impl(oid, true)
2257 }
2258
2259 fn read_impl(&self, oid: &ObjectId, remember: bool) -> Result<Object> {
2262 if self.is_empty_tree(oid) {
2263 return Ok(Object {
2264 kind: ObjectKind::Tree,
2265 data: Vec::new(),
2266 });
2267 }
2268
2269 if let Some(obj) = self.session_cached_loose_read(oid) {
2270 return Ok(obj);
2271 }
2272
2273 if let Some(obj) = self.overlay_read(oid) {
2274 return Ok(obj);
2275 }
2276 if let Some(obj) = self.try_read_packed_in_batch_context(oid)? {
2277 return Ok(obj);
2278 }
2279
2280 let use_midx = self.multi_pack_index_reads_enabled();
2281 if use_midx {
2282 self.ensure_midx_prepared();
2283 }
2284
2285 self.sync_delta_base_cache_limit();
2286
2287 let primary = self.primary_writable()?;
2288 let files_primary = self.primary().ok();
2289 let mut unreadable_local_loose = None;
2290 let read_primary = || primary.read(oid);
2291 match self.with_primary_read_context(read_primary) {
2292 Ok(Some(obj)) => return self.finish_session_loose_read(oid, obj, remember),
2293 Ok(None) => {}
2294 Err(err)
2295 if files_primary
2296 .as_ref()
2297 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2298 {
2299 unreadable_local_loose = Some(err);
2300 }
2301 Err(err) => return Err(err),
2302 }
2303
2304 if let Some(obj) = self.try_read_extra_sources(oid)? {
2305 return self.finish_session_loose_read(oid, obj, remember);
2306 }
2307
2308 if self.alternates_enabled {
2309 match self.sources()?.read(oid) {
2310 Ok(Some(obj)) => return self.finish_session_loose_read(oid, obj, remember),
2311 Ok(None) => {}
2312 Err(err) => return Err(err),
2313 }
2314 }
2315
2316 if let Some(err) = unreadable_local_loose {
2317 return Err(err);
2318 }
2319
2320 self.read_after_promisor_miss(oid)
2321 }
2322
2323 #[cold]
2324 #[inline(never)]
2325 fn read_after_promisor_miss(&self, oid: &ObjectId) -> Result<Object> {
2326 let mut lazy_retries = 0u32;
2327 loop {
2328 let Some(res) = self.resolve_promisor_miss(oid, lazy_retries)? else {
2329 return Err(Error::ObjectNotFound(oid.to_hex()));
2330 };
2331 match res {
2332 crate::promisor_fetch::PromisorMissResolution::RetryAfterFetch => {
2333 lazy_retries += 1;
2334 if let Some(obj) = self.try_read_once(oid)? {
2335 return Ok(obj);
2336 }
2337 }
2338 crate::promisor_fetch::PromisorMissResolution::NotPromisorRepo => {
2339 return Err(Error::ObjectNotFound(oid.to_hex()));
2340 }
2341 crate::promisor_fetch::PromisorMissResolution::PromisedMissing(err) => {
2342 return Err(err);
2343 }
2344 }
2345 }
2346 }
2347
2348 pub(crate) fn try_read_once(&self, oid: &ObjectId) -> Result<Option<Object>> {
2351 if self.is_empty_tree(oid) {
2352 return Ok(Some(Object {
2353 kind: ObjectKind::Tree,
2354 data: Vec::new(),
2355 }));
2356 }
2357
2358 if let Some(obj) = self.session_cached_loose_read(oid) {
2359 return Ok(Some(obj));
2360 }
2361
2362 if let Some(obj) = self.overlay_read(oid) {
2363 return Ok(Some(obj));
2364 }
2365 if let Some(obj) = self.try_read_packed_in_batch_context(oid)? {
2366 return Ok(Some(self.finish_session_loose_read(oid, obj, true)?));
2367 }
2368
2369 let use_midx = self.multi_pack_index_reads_enabled();
2370 if use_midx {
2371 self.ensure_midx_prepared();
2372 }
2373
2374 self.sync_delta_base_cache_limit();
2375
2376 let primary = self.primary_writable()?;
2377 let files_primary = self.primary().ok();
2378 let mut unreadable_local_loose = None;
2379 let read_primary = || primary.read(oid);
2380 match self.with_primary_read_context(read_primary) {
2381 Ok(Some(obj)) => return Ok(Some(self.finish_session_loose_read(oid, obj, true)?)),
2382 Ok(None) => {}
2383 Err(err)
2384 if files_primary
2385 .as_ref()
2386 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2387 {
2388 unreadable_local_loose = Some(err);
2389 }
2390 Err(err) => return Err(err),
2391 }
2392
2393 if let Some(obj) = self.try_read_extra_sources(oid)? {
2394 return Ok(Some(self.finish_session_loose_read(oid, obj, true)?));
2395 }
2396
2397 if self.alternates_enabled {
2398 match self.sources()?.read(oid) {
2399 Ok(Some(obj)) => return Ok(Some(self.finish_session_loose_read(oid, obj, true)?)),
2400 Ok(None) => {}
2401 Err(err) => return Err(err),
2402 }
2403 }
2404
2405 if let Some(err) = unreadable_local_loose {
2406 return Err(err);
2407 }
2408
2409 Ok(None)
2410 }
2411
2412 fn resolve_promisor_miss(
2413 &self,
2414 oid: &ObjectId,
2415 lazy_retries: u32,
2416 ) -> Result<Option<crate::promisor_fetch::PromisorMissResolution>> {
2417 let (Some(caches), Some(env)) = (&self.shared_config_state, &self.shared_environment)
2418 else {
2419 return Ok(None);
2420 };
2421 Ok(Some(crate::promisor_fetch::resolve_promisor_read_miss(
2422 caches,
2423 self,
2424 oid,
2425 env.as_ref(),
2426 lazy_retries,
2427 )?))
2428 }
2429
2430 pub(crate) fn prefetch_promisor_read_batch(
2438 &self,
2439 oids: impl IntoIterator<Item = ObjectId>,
2440 ) -> Result<()> {
2441 let (Some(caches), Some(env)) = (&self.shared_config_state, &self.shared_environment)
2442 else {
2443 return Ok(());
2444 };
2445 crate::promisor_fetch::prefetch_promisor_read_batch(caches, self, env.as_ref(), oids)
2446 }
2447
2448 pub fn read_into(&self, oid: &ObjectId, buf: &mut Vec<u8>) -> Result<ObjectKind> {
2457 if self.is_empty_tree(oid) {
2458 buf.clear();
2459 return Ok(ObjectKind::Tree);
2460 }
2461
2462 if let Some(obj) = self.session_cached_loose_read(oid) {
2463 buf.clear();
2464 buf.extend_from_slice(&obj.data);
2465 return Ok(obj.kind);
2466 }
2467
2468 if let Some(obj) = self.overlay_read(oid) {
2469 buf.clear();
2470 buf.extend_from_slice(&obj.data);
2471 return Ok(obj.kind);
2472 }
2473
2474 if let Some(kind) = self.try_read_packed_in_batch_context_into(oid, false, buf)? {
2475 return Ok(kind);
2476 }
2477
2478 let use_midx = self.multi_pack_index_reads_enabled();
2479 if use_midx {
2480 self.ensure_midx_prepared();
2481 }
2482
2483 self.sync_delta_base_cache_limit();
2484
2485 let primary = self.primary_writable()?;
2486 let files_primary = self.primary().ok();
2487 let mut unreadable_local_loose = None;
2488 let read_primary = || primary.read_into(oid, buf);
2489 match self.with_primary_read_context(read_primary) {
2490 Ok(Some(kind)) => return Ok(kind),
2491 Ok(None) => {}
2492 Err(err)
2493 if files_primary
2494 .as_ref()
2495 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2496 {
2497 unreadable_local_loose = Some(err);
2498 }
2499 Err(err) => return Err(err),
2500 }
2501
2502 if let Some(kind) = self.try_read_into_extra_sources(oid, buf)? {
2503 return Ok(kind);
2504 }
2505
2506 if self.alternates_enabled {
2507 match self.sources()?.read_into(oid, buf) {
2508 Ok(Some(kind)) => return Ok(kind),
2509 Ok(None) => {}
2510 Err(err) => return Err(err),
2511 }
2512 }
2513
2514 if let Some(err) = unreadable_local_loose {
2515 return Err(err);
2516 }
2517
2518 self.read_into_after_promisor_miss(oid, buf)
2519 }
2520
2521 pub(crate) fn read_into_caching_delta_bases(
2535 &self,
2536 oid: &ObjectId,
2537 buf: &mut Vec<u8>,
2538 ) -> Result<ObjectKind> {
2539 if let Some(kind) = self.try_read_packed_in_batch_context_into(oid, true, buf)? {
2540 return Ok(kind);
2541 }
2542 if self.custom_primary.is_none() && !self.is_empty_tree(oid) {
2543 if let Ok(files) = self.primary() {
2544 self.ensure_midx_prepared();
2545 self.sync_delta_base_cache_limit();
2546 let mut read = || files.read_into_with_delta_policy(oid, true, buf);
2547 let local = if PackStore::read_context_matches_objects_dir(&self.objects_dir) {
2548 read()
2549 } else {
2550 self.with_pack_store_for(&self.objects_dir, read)
2551 };
2552 if let Ok(Some(kind)) = local {
2553 return Ok(kind);
2554 }
2555 }
2556 }
2557 if self.custom_primary.is_some() {
2558 return self.read_into(oid, buf);
2559 }
2560 let object = self.read(oid)?;
2561 buf.clear();
2562 buf.extend_from_slice(&object.data);
2563 Ok(object.kind)
2564 }
2565
2566 pub(crate) fn read_packed_commit_header_block_into(
2579 &self,
2580 oid: &ObjectId,
2581 buf: &mut Vec<u8>,
2582 ) -> Result<bool> {
2583 if self.custom_primary.is_some() {
2584 return Ok(false);
2585 }
2586 let Ok(files) = self.primary() else {
2587 return Ok(false);
2588 };
2589 files.read_packed_commit_header_block_into(oid, buf)
2590 }
2591
2592 #[cold]
2593 #[inline(never)]
2594 fn read_into_after_promisor_miss(
2595 &self,
2596 oid: &ObjectId,
2597 buf: &mut Vec<u8>,
2598 ) -> Result<ObjectKind> {
2599 let mut lazy_retries = 0u32;
2600 loop {
2601 let Some(res) = self.resolve_promisor_miss(oid, lazy_retries)? else {
2602 return Err(Error::ObjectNotFound(oid.to_hex()));
2603 };
2604 match res {
2605 crate::promisor_fetch::PromisorMissResolution::RetryAfterFetch => {
2606 lazy_retries += 1;
2607 if let Some(kind) = self.try_read_into_once(oid, buf)? {
2608 return Ok(kind);
2609 }
2610 }
2611 crate::promisor_fetch::PromisorMissResolution::NotPromisorRepo => {
2612 return Err(Error::ObjectNotFound(oid.to_hex()));
2613 }
2614 crate::promisor_fetch::PromisorMissResolution::PromisedMissing(err) => {
2615 return Err(err);
2616 }
2617 }
2618 }
2619 }
2620
2621 fn try_read_into_once(&self, oid: &ObjectId, buf: &mut Vec<u8>) -> Result<Option<ObjectKind>> {
2622 if oid.is_well_known_empty_tree() {
2623 buf.clear();
2624 return Ok(Some(ObjectKind::Tree));
2625 }
2626
2627 if let Some(obj) = self.session_cached_loose_read(oid) {
2628 buf.clear();
2629 buf.extend_from_slice(&obj.data);
2630 return Ok(Some(obj.kind));
2631 }
2632
2633 if let Some(obj) = self.overlay_read(oid) {
2634 buf.clear();
2635 buf.extend_from_slice(&obj.data);
2636 return Ok(Some(obj.kind));
2637 }
2638
2639 if let Some(kind) = self.try_read_packed_in_batch_context_into(oid, false, buf)? {
2640 return Ok(Some(kind));
2641 }
2642
2643 let use_midx = self.multi_pack_index_reads_enabled();
2644 if use_midx {
2645 self.ensure_midx_prepared();
2646 }
2647
2648 self.sync_delta_base_cache_limit();
2649
2650 let primary = self.primary_writable()?;
2651 let files_primary = self.primary().ok();
2652 let mut unreadable_local_loose = None;
2653 let read_primary = || primary.read_into(oid, buf);
2654 match self.with_primary_read_context(read_primary) {
2655 Ok(Some(kind)) => return Ok(Some(kind)),
2656 Ok(None) => {}
2657 Err(err)
2658 if files_primary
2659 .as_ref()
2660 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2661 {
2662 unreadable_local_loose = Some(err);
2663 }
2664 Err(err) => return Err(err),
2665 }
2666
2667 if let Some(kind) = self.try_read_into_extra_sources(oid, buf)? {
2668 return Ok(Some(kind));
2669 }
2670
2671 if self.alternates_enabled {
2672 match self.sources()?.read_into(oid, buf) {
2673 Ok(Some(kind)) => return Ok(Some(kind)),
2674 Ok(None) => {}
2675 Err(err) => return Err(err),
2676 }
2677 }
2678
2679 if let Some(err) = unreadable_local_loose {
2680 return Err(err);
2681 }
2682
2683 Ok(None)
2684 }
2685
2686 fn try_read_into_extra_sources(
2687 &self,
2688 oid: &ObjectId,
2689 buf: &mut Vec<u8>,
2690 ) -> Result<Option<ObjectKind>> {
2691 for src in &self.extra_read_sources {
2692 if let Some(kind) = src.read_into(oid, buf)? {
2693 return Ok(Some(kind));
2694 }
2695 }
2696 Ok(None)
2697 }
2698
2699 fn try_read_packed_in_batch_context_into(
2700 &self,
2701 oid: &ObjectId,
2702 populate_delta_lru: bool,
2703 buf: &mut Vec<u8>,
2704 ) -> Result<Option<ObjectKind>> {
2705 if self.custom_primary.is_some()
2706 || !PackStore::read_context_matches_objects_dir(&self.objects_dir)
2707 || self.multi_pack_index_reads_enabled()
2708 {
2709 return Ok(None);
2710 }
2711 self.sync_delta_base_cache_limit();
2712 match pack::try_read_object_from_packs_with_options_into(
2713 &self.objects_dir,
2714 oid,
2715 pack::PackLookupOptions::ALL_LOCAL_PACKS,
2716 populate_delta_lru,
2717 buf,
2718 ) {
2719 Ok(kind) => Ok(Some(kind)),
2720 Err(Error::ObjectNotFound(_)) => Ok(None),
2721 Err(err) => Err(err),
2722 }
2723 }
2724
2725 pub fn read_info(&self, oid: &ObjectId) -> Result<ObjectInfo> {
2737 if self.is_empty_tree(oid) {
2738 return Ok(ObjectInfo {
2739 kind: ObjectKind::Tree,
2740 size: 0,
2741 });
2742 }
2743
2744 if let Some(obj) = self.overlay_read(oid) {
2745 return Ok(ObjectInfo {
2746 kind: obj.kind,
2747 size: u64::try_from(obj.data.len())
2748 .map_err(|_| Error::CorruptObject("object size overflow".to_owned()))?,
2749 });
2750 }
2751
2752 if let Some(info) = self.try_read_info_packed_in_batch_context(oid)? {
2753 return Ok(info);
2754 }
2755
2756 let use_midx = self.multi_pack_index_reads_enabled();
2757 if use_midx {
2758 self.ensure_midx_prepared();
2759 }
2760
2761 let primary = self.primary_writable()?;
2762 let files_primary = self.primary().ok();
2763 let mut unreadable_local_loose = None;
2764 match primary.read_info(oid) {
2765 Ok(Some(info)) => return Ok(info),
2766 Ok(None) => {}
2767 Err(err)
2768 if files_primary
2769 .as_ref()
2770 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2771 {
2772 unreadable_local_loose = Some(err);
2773 }
2774 Err(err) => return Err(err),
2775 }
2776
2777 if let Some(info) = self.try_read_info_extra_sources(oid)? {
2778 return Ok(info);
2779 }
2780
2781 if self.alternates_enabled {
2782 match self.sources()?.read_info(oid) {
2783 Ok(Some(info)) => return Ok(info),
2784 Ok(None) => {}
2785 Err(err) => return Err(err),
2786 }
2787 }
2788
2789 if let Some(err) = unreadable_local_loose {
2790 return Err(err);
2791 }
2792
2793 self.read_info_after_promisor_miss(oid)
2794 }
2795
2796 pub(crate) fn read_info_without_fetch(&self, oid: &ObjectId) -> Result<ObjectInfo> {
2801 self.try_read_info_once(oid)?
2802 .ok_or_else(|| Error::ObjectNotFound(oid.to_hex()))
2803 }
2804
2805 pub(crate) fn stored_object_kind(&self, oid: &ObjectId) -> Result<Option<ObjectKind>> {
2817 if oid.is_well_known_empty_tree() && !self.empty_tree_stored(oid) {
2818 return Ok(None);
2819 }
2820 match self.try_read_info_once(oid) {
2821 Ok(info) => Ok(info.map(|info| info.kind)),
2822 Err(Error::ObjectNotFound(_)) => Ok(None),
2823 Err(err) => Err(err),
2824 }
2825 }
2826
2827 fn empty_tree_stored(&self, oid: &ObjectId) -> bool {
2828 if self.overlay_read(oid).is_some() || self.exists_materialized_local(oid) {
2829 return true;
2830 }
2831 if !self.alternates_enabled {
2832 return false;
2833 }
2834 let file_dirs = self.file_alternate_dirs_snapshot();
2835 let env_dirs = self.env_alternate_dirs_snapshot();
2836 file_dirs
2837 .iter()
2838 .chain(env_dirs.iter())
2839 .any(|dir| self.with_pack_store_for(dir, || store::local_object_materialized(dir, oid)))
2840 }
2841
2842 #[cold]
2843 #[inline(never)]
2844 fn read_info_after_promisor_miss(&self, oid: &ObjectId) -> Result<ObjectInfo> {
2845 let mut lazy_retries = 0u32;
2846 loop {
2847 let Some(res) = self.resolve_promisor_miss(oid, lazy_retries)? else {
2848 return Err(Error::ObjectNotFound(oid.to_hex()));
2849 };
2850 match res {
2851 crate::promisor_fetch::PromisorMissResolution::PromisedMissing(err) => {
2852 return Err(err);
2853 }
2854 crate::promisor_fetch::PromisorMissResolution::NotPromisorRepo => {
2855 return Err(Error::ObjectNotFound(oid.to_hex()));
2856 }
2857 crate::promisor_fetch::PromisorMissResolution::RetryAfterFetch => {
2858 lazy_retries += 1;
2859 if let Some(info) = self.try_read_info_once(oid)? {
2860 return Ok(info);
2861 }
2862 }
2863 }
2864 }
2865 }
2866
2867 fn try_read_info_once(&self, oid: &ObjectId) -> Result<Option<ObjectInfo>> {
2868 if self.is_empty_tree(oid) {
2869 return Ok(Some(ObjectInfo {
2870 kind: ObjectKind::Tree,
2871 size: 0,
2872 }));
2873 }
2874
2875 if let Some(obj) = self.overlay_read(oid) {
2876 return Ok(Some(ObjectInfo {
2877 kind: obj.kind,
2878 size: u64::try_from(obj.data.len())
2879 .map_err(|_| Error::CorruptObject("object size overflow".to_owned()))?,
2880 }));
2881 }
2882
2883 if let Some(info) = self.try_read_info_packed_in_batch_context(oid)? {
2884 return Ok(Some(info));
2885 }
2886
2887 let use_midx = self.multi_pack_index_reads_enabled();
2888 if use_midx {
2889 self.ensure_midx_prepared();
2890 }
2891
2892 let primary = self.primary_writable()?;
2893 let files_primary = self.primary().ok();
2894 let mut unreadable_local_loose = None;
2895 match primary.read_info(oid) {
2896 Ok(Some(info)) => return Ok(Some(info)),
2897 Ok(None) => {}
2898 Err(err)
2899 if files_primary
2900 .as_ref()
2901 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
2902 {
2903 unreadable_local_loose = Some(err);
2904 }
2905 Err(err) => return Err(err),
2906 }
2907
2908 if let Some(info) = self.try_read_info_extra_sources(oid)? {
2909 return Ok(Some(info));
2910 }
2911
2912 if self.alternates_enabled {
2913 match self.sources()?.read_info(oid) {
2914 Ok(Some(info)) => return Ok(Some(info)),
2915 Ok(None) => {}
2916 Err(err) => return Err(err),
2917 }
2918 }
2919
2920 if let Some(err) = unreadable_local_loose {
2921 return Err(err);
2922 }
2923
2924 Ok(None)
2925 }
2926
2927 pub(crate) fn read_unless_larger(
2943 &self,
2944 oid: &ObjectId,
2945 limit: u64,
2946 cache_small: bool,
2947 ) -> Result<BoundedRead> {
2948 if cache_small {
2949 if let Some(read) = self.session_loose_reads.lock().ok().and_then(|cache| {
2950 cache.get(oid).map(|obj| {
2951 if obj.data.len() as u64 > limit {
2952 BoundedRead::Larger
2953 } else {
2954 BoundedRead::Object(obj.clone())
2955 }
2956 })
2957 }) {
2958 return Ok(read);
2959 }
2960 }
2961 if self.custom_primary.is_none() && !self.multi_pack_index_reads_enabled() {
2962 if let Some(read) =
2963 pack::pack_cache::try_read_object_via_single_pack_reader_unless_whole_larger(
2964 &self.objects_dir,
2965 oid,
2966 limit,
2967 )?
2968 {
2969 return Ok(read);
2970 }
2971 }
2972 if let Ok(files) = self.primary() {
2973 if self.multi_pack_index_reads_enabled() {
2974 self.ensure_midx_prepared();
2975 }
2976 match files.read_unless_larger(oid, limit)? {
2977 LocalSizedRead::Packed(Some(size)) if size > limit => {
2978 return Ok(BoundedRead::Larger);
2979 }
2980 LocalSizedRead::Packed(_) => return self.read(oid).map(BoundedRead::Object),
2981 LocalSizedRead::Loose(BoundedRead::Object(obj)) => {
2982 if cache_small && obj.data.len() <= SESSION_BOUNDED_LOOSE_CACHE_MAX {
2983 self.remember_session_loose_read(oid, &obj);
2984 }
2985 return Ok(BoundedRead::Object(obj));
2986 }
2987 LocalSizedRead::Loose(read) => return Ok(read),
2988 LocalSizedRead::Missing => {}
2989 }
2990 }
2991 let info = self.read_info(oid)?;
2992 if info.size > limit {
2993 return Ok(BoundedRead::Larger);
2994 }
2995 self.read(oid).map(BoundedRead::Object)
2996 }
2997
2998 pub fn open_stream(&self, oid: &ObjectId) -> Result<Option<ObjectStream<'static>>> {
3007 self.with_stream(oid, materialize_stream)
3008 }
3009
3010 pub fn with_stream<R>(
3027 &self,
3028 oid: &ObjectId,
3029 f: impl FnOnce(ObjectStream<'_>) -> Result<R>,
3030 ) -> Result<Option<R>> {
3031 use std::io::Cursor;
3032 if self.is_empty_tree(oid) {
3033 return f(ObjectStream {
3034 kind: ObjectKind::Tree,
3035 size: 0,
3036 reader: Box::new(std::io::empty()),
3037 })
3038 .map(Some);
3039 }
3040 if let Some(obj) = self.overlay_read(oid) {
3041 let size = u64::try_from(obj.data.len()).map_err(|_| {
3042 Error::CorruptObject(format!("object size overflow for {}", oid.to_hex()))
3043 })?;
3044 return f(ObjectStream {
3045 kind: obj.kind,
3046 size,
3047 reader: Box::new(Cursor::new(obj.data)),
3048 })
3049 .map(Some);
3050 }
3051 if self.multi_pack_index_reads_enabled() {
3052 self.ensure_midx_prepared();
3053 }
3054 self.sync_delta_base_cache_limit();
3055 let primary = self.primary_writable()?;
3056 let files_primary = self.primary().ok();
3057 let mut unreadable_local_loose = None;
3058 match primary.open_stream(oid) {
3059 Ok(Some(stream)) => return f(stream).map(Some),
3060 Ok(None) => {}
3061 Err(err)
3062 if files_primary
3063 .as_ref()
3064 .is_some_and(|p| p.local_loose_exists_but_unreadable(oid, &err)) =>
3065 {
3066 unreadable_local_loose = Some(err);
3067 }
3068 Err(err) => return Err(err),
3069 }
3070 if let Some(stream) = self.try_open_stream_extra_sources(oid)? {
3071 return f(stream).map(Some);
3072 }
3073 if self.alternates_enabled {
3074 let sources = self.sources()?;
3075 let stream = sources.open_stream(oid)?;
3076 if let Some(stream) = stream {
3077 return f(stream).map(Some);
3078 }
3079 }
3080 if let Some(err) = unreadable_local_loose {
3081 return Err(err);
3082 }
3083 Ok(None)
3084 }
3085
3086 pub fn for_each_object(&self, f: &mut dyn FnMut(&ObjectId) -> ControlFlow<()>) -> Result<()> {
3092 let mut seen = oid_hash_set();
3093 let mut visit = |store: &dyn ObjectStore| -> Result<bool> {
3094 store::for_each_propagate_break(store, &mut |oid| {
3095 if !seen.insert(*oid) {
3096 return ControlFlow::Continue(());
3097 }
3098 f(oid)
3099 })
3100 };
3101 if let Some(overlay) = self.overlay_store() {
3102 if visit(overlay.as_ref())? {
3103 return Ok(());
3104 }
3105 }
3106 if visit(self.primary_writable()?.as_ref())? {
3107 return Ok(());
3108 }
3109 for src in &self.extra_read_sources {
3110 if visit(src.as_ref())? {
3111 return Ok(());
3112 }
3113 }
3114 if self.alternates_enabled {
3115 let _ = visit(self.sources()?.as_ref())?;
3116 }
3117 Ok(())
3118 }
3119
3120 pub fn lookup_prefix(&self, prefix: &str, limit: usize, out: &mut Vec<ObjectId>) -> Result<()> {
3126 let mut seen: OidHashSet = out.iter().copied().collect();
3127 if let Some(overlay) = self.overlay_store() {
3128 append_lookup_prefix_layer(overlay.as_ref(), prefix, limit, out, &mut seen)?;
3129 if limit != 0 && out.len() >= limit {
3130 return Ok(());
3131 }
3132 }
3133 append_lookup_prefix_layer(
3134 self.primary_writable()?.as_ref(),
3135 prefix,
3136 limit,
3137 out,
3138 &mut seen,
3139 )?;
3140 if limit != 0 && out.len() >= limit {
3141 return Ok(());
3142 }
3143 for src in &self.extra_read_sources {
3144 append_lookup_prefix_layer(src.as_ref(), prefix, limit, out, &mut seen)?;
3145 if limit != 0 && out.len() >= limit {
3146 return Ok(());
3147 }
3148 }
3149 if self.alternates_enabled {
3150 append_lookup_prefix_layer(self.sources()?.as_ref(), prefix, limit, out, &mut seen)?;
3151 }
3152 Ok(())
3153 }
3154
3155 pub fn enumerate_local_loose_objects(&self) -> Result<Vec<(ObjectId, PathBuf)>> {
3163 let primary = self.primary()?;
3164 let loose = primary.loose_store();
3165 let mut out = Vec::new();
3166 loose.for_each_object(&mut |oid| {
3167 out.push((*oid, loose.object_path(oid)));
3168 ControlFlow::Continue(())
3169 })?;
3170 Ok(out)
3171 }
3172
3173 pub fn hash(&self, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
3181 hash_object_data_with(self.hash_algo(), kind, data)
3182 }
3183
3184 pub fn hash_stream(
3195 &self,
3196 kind: ObjectKind,
3197 size: u64,
3198 reader: &mut dyn std::io::Read,
3199 ) -> Result<ObjectId> {
3200 #[cfg(test)]
3201 self.hot_path_metrics().record_stream_hash();
3202 stream::hash_stream(self.hash_algo(), kind, size, reader)
3203 }
3204
3205 pub fn write_stream(
3227 &self,
3228 kind: ObjectKind,
3229 size: u64,
3230 reader: &mut dyn std::io::Read,
3231 options: WriteOptions,
3232 ) -> Result<ObjectId> {
3233 #[cfg(test)]
3234 self.hot_path_metrics().record_stream_write();
3235 if let Some(primary) = &self.custom_primary {
3236 if !self.overlay_active() {
3237 let oid = primary.write_stream(kind, size, reader, options)?;
3238 self.note_commit_object_written(kind);
3239 return Ok(oid);
3240 }
3241 }
3242 if self.overlay_active() || size < stream::STREAMED_WRITE_MIN_SIZE {
3243 let len = usize::try_from(size).map_err(|_| {
3244 Error::CorruptObject(format!("object size {size} exceeds address space"))
3245 })?;
3246 let mut data = Vec::with_capacity(len);
3247 let read =
3248 std::io::Read::read_to_end(&mut std::io::Read::take(reader, size), &mut data)?;
3249 if read != len {
3250 return Err(Error::ObjectStreamTruncated {
3251 expected: size,
3252 actual: read as u64,
3253 });
3254 }
3255 return self.write_with_options(kind, &data, options);
3256 }
3257 if size < self.big_file_threshold()? {
3258 let loose = self.local_loose()?;
3259 let oid = loose.write_payload_stream(kind, size, reader, options)?;
3260 self.note_commit_object_written(kind);
3261 return Ok(oid);
3262 }
3263 let staged = stream::StagedPack::write(
3264 &self.objects_dir.join("pack"),
3265 self.hash_algo(),
3266 kind,
3267 size,
3268 reader,
3269 self.pack_compression()?,
3270 )?;
3271 let oid = staged.oid();
3272 if self.object_exists_for_write_skip(&oid) {
3273 if !options.silent {
3274 let _ = self.freshen_object(&oid);
3275 }
3276 return Ok(oid);
3277 }
3278 staged.install()?;
3279 self.invalidate_packs();
3280 self.note_commit_object_written(kind);
3281 Ok(oid)
3282 }
3283
3284 pub fn empty_tree_oid(&self) -> Result<ObjectId> {
3292 self.hash(ObjectKind::Tree, b"")
3293 }
3294
3295 #[must_use]
3297 pub fn is_empty_tree(&self, oid: &ObjectId) -> bool {
3298 empty_tree_oid_matches(self.hash_algo(), oid)
3299 }
3300
3301 pub fn write(&self, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
3314 self.write_with_options(kind, data, WriteOptions::default())
3315 }
3316
3317 pub fn ensure_all_loose_prefix_dirs(&self) -> Result<()> {
3326 if self.custom_primary.is_some() {
3327 return Ok(());
3328 }
3329 self.local_loose()?.ensure_all_loose_prefix_dirs()
3330 }
3331
3332 #[cfg(test)]
3334 #[must_use]
3335 pub fn test_loose_object_path(&self, oid: &ObjectId) -> PathBuf {
3336 self.object_path(oid)
3337 }
3338
3339 #[cfg(test)]
3341 pub fn test_remove_loose_object(&self, oid: &ObjectId) {
3342 let _ = fs::remove_file(self.object_path(oid));
3343 }
3344
3345 pub fn count_loose_objects(&self) -> Result<usize> {
3351 let mut count = 0usize;
3352 for_each_loose_object_id(&self.objects_dir, self.hash_algo(), &mut |_| {
3353 count += 1;
3354 ControlFlow::Continue(())
3355 })?;
3356 Ok(count)
3357 }
3358
3359 pub fn approximate_object_count(&self) -> Result<usize> {
3369 let pack_objects: Result<usize> = self.with_pack_read_context(|| {
3370 let indexes = pack::read_local_pack_indexes_cached(self.objects_dir())?;
3371 Ok(indexes.iter().map(|idx| idx.len()).sum())
3372 });
3373 let mut count = pack_objects?;
3374 count += self.count_loose_objects()?;
3375 Ok(count)
3376 }
3377
3378 pub fn zlib_compress_loose_store(&self, store_bytes: &[u8]) -> Result<Vec<u8>> {
3384 self.local_loose()?.zlib_compress_store_bytes(store_bytes)
3385 }
3386
3387 pub fn write_loose_zlib_prehashed(
3401 &self,
3402 oid: &ObjectId,
3403 zlib_store: &[u8],
3404 options: WriteOptions,
3405 ) -> Result<ObjectId> {
3406 if self.custom_primary.is_some() {
3407 let raw = decompress_zlib_loose_bytes(zlib_store)?;
3408 let obj = parse_object_bytes(&raw)?;
3409 return self.write_with_options(obj.kind, &obj.data, options);
3410 }
3411 let primary = self.primary()?;
3412 if let Some(existing) = primary.try_finish_duplicate_write(oid, options)? {
3413 return Ok(existing);
3414 }
3415 if options.freshen_existing && self.freshen_packed_or_alternate(oid) {
3416 return Ok(*oid);
3417 }
3418
3419 let already_exists = !(options.trust_new_loose || options.assume_loose_only_existence)
3420 && self.object_exists_for_write_skip(oid);
3421
3422 if self.overlay_active() && !already_exists {
3423 if let Ok(raw) = decompress_zlib_loose_bytes(zlib_store) {
3424 if let Ok(obj) = parse_object_bytes(&raw) {
3425 if self.overlay_store_object(*oid, obj.kind, &obj.data) {
3426 return Ok(*oid);
3427 }
3428 }
3429 }
3430 }
3431
3432 if already_exists {
3433 if !options.silent {
3434 let _ = self.freshen_object(oid);
3435 }
3436 return Ok(*oid);
3437 }
3438
3439 primary.write_zlib_prehashed(oid, zlib_store, options)
3440 }
3441
3442 pub(crate) fn write_blob_prehashed(
3448 &self,
3449 oid: &ObjectId,
3450 data: &[u8],
3451 options: WriteOptions,
3452 ) -> Result<ObjectId> {
3453 if self.custom_primary.is_some() {
3454 return self.write_with_options(ObjectKind::Blob, data, options);
3455 }
3456 let primary = self.primary()?;
3457 if let Some(existing) = primary.try_finish_duplicate_write(oid, options)? {
3458 return Ok(existing);
3459 }
3460 if options.freshen_existing && self.freshen_packed_or_alternate(oid) {
3461 return Ok(*oid);
3462 }
3463 let already_exists = !(options.trust_new_loose || options.assume_loose_only_existence)
3464 && self.object_exists_for_write_skip(oid);
3465 if self.overlay_active()
3466 && !already_exists
3467 && self.overlay_store_object(*oid, ObjectKind::Blob, data)
3468 {
3469 return Ok(*oid);
3470 }
3471 if already_exists {
3472 if !options.silent {
3473 let _ = self.freshen_object(oid);
3474 }
3475 return Ok(*oid);
3476 }
3477 primary
3478 .loose_store()
3479 .write_prehashed(oid, ObjectKind::Blob, data, options)
3480 }
3481 pub fn write_with_options(
3483 &self,
3484 kind: ObjectKind,
3485 data: &[u8],
3486 options: WriteOptions,
3487 ) -> Result<ObjectId> {
3488 let oid = hash_object_data_with(self.hash_algo(), kind, data)?;
3489
3490 if let Some(primary) = &self.custom_primary {
3491 let already_exists = if options.trust_new_loose {
3492 false
3493 } else if options.assume_loose_only_existence {
3494 primary.contains(&oid)?
3495 } else {
3496 self.object_exists_for_write_skip(&oid)
3497 };
3498 if self.overlay_active()
3499 && !already_exists
3500 && self.overlay_store_object(oid, kind, data)
3501 {
3502 return Ok(oid);
3503 }
3504 if already_exists {
3505 return Ok(oid);
3506 }
3507 let oid = WritableObjectStore::write(primary.as_ref(), kind, data, options)?;
3508 self.note_commit_object_written(kind);
3509 return Ok(oid);
3510 }
3511
3512 let primary = self.primary()?;
3513 if let Some(existing) = primary.try_finish_duplicate_write(&oid, options)? {
3514 return Ok(existing);
3515 }
3516 if options.freshen_existing && self.freshen_packed_or_alternate(&oid) {
3517 return Ok(oid);
3518 }
3519
3520 let already_exists = !(options.trust_new_loose || options.assume_loose_only_existence)
3521 && self.object_exists_for_write_skip(&oid);
3522
3523 if self.overlay_active() && !already_exists && self.overlay_store_object(oid, kind, data) {
3526 return Ok(oid);
3527 }
3528
3529 if already_exists {
3530 if !options.silent {
3531 let _ = self.freshen_object(&oid);
3532 }
3533 return Ok(oid);
3534 }
3535
3536 let oid = WritableObjectStore::write(primary.as_ref(), kind, data, options)?;
3537 self.note_commit_object_written(kind);
3538 Ok(oid)
3539 }
3540
3541 pub fn write_local(&self, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
3557 if self.custom_primary.is_some() {
3558 return Err(Error::UnsupportedObjectStore {
3559 operation: "write_local",
3560 });
3561 }
3562 let oid = hash_object_data_with(self.hash_algo(), kind, data)?;
3563
3564 if let Some(existing) = self.try_freshen_existing_local(&oid) {
3565 return Ok(existing);
3566 }
3567
3568 let oid = self
3569 .local_loose()?
3570 .write(kind, data, WriteOptions::default())?;
3571 self.note_commit_object_written(kind);
3572 Ok(oid)
3573 }
3574
3575 pub(crate) fn write_local_stream_with(
3577 &self,
3578 kind: ObjectKind,
3579 size: u64,
3580 supply: impl FnMut(&mut dyn FnMut(&[u8]) -> Result<()>) -> Result<()>,
3581 ) -> Result<ObjectId> {
3582 if self.custom_primary.is_some() {
3583 return Err(Error::UnsupportedObjectStore {
3584 operation: "write_local_stream",
3585 });
3586 }
3587 let loose = self.local_loose()?;
3588 let oid = loose.write_payload_stream_with(
3589 kind,
3590 size,
3591 WriteOptions::default(),
3592 self.pack_compression()?,
3593 supply,
3594 )?;
3595 self.note_commit_object_written(kind);
3596 Ok(oid)
3597 }
3598
3599 pub fn write_loose_materialize(&self, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
3606 if self.custom_primary.is_some() {
3607 return Err(Error::UnsupportedObjectStore {
3608 operation: "write_loose_materialize",
3609 });
3610 }
3611 let oid = hash_object_data_with(self.hash_algo(), kind, data)?;
3612 if let Ok(primary) = self.primary() {
3613 if let Ok(Some(existing)) = primary
3614 .loose_store()
3615 .try_finish_if_loose_present(&oid, WriteOptions::default())
3616 {
3617 return Ok(existing);
3618 }
3619 }
3620
3621 self.local_loose()?
3622 .write(kind, data, WriteOptions::default())
3623 }
3624
3625 pub fn write_raw(&self, store_bytes: &[u8]) -> Result<ObjectId> {
3635 let obj = parse_object_bytes(store_bytes)?;
3636 let oid = hash_object_data_with(self.hash_algo(), obj.kind, &obj.data)?;
3637
3638 if let Some(primary) = &self.custom_primary {
3639 if self.exists(&oid) {
3640 return Ok(oid);
3641 }
3642 if self.overlay_active() && self.overlay_store_object(oid, obj.kind, &obj.data) {
3643 return Ok(oid);
3644 }
3645 return WritableObjectStore::write(
3646 primary.as_ref(),
3647 obj.kind,
3648 &obj.data,
3649 WriteOptions::default(),
3650 );
3651 }
3652
3653 if let Some(existing) = self.try_freshen_existing(&oid) {
3654 return Ok(existing);
3655 }
3656
3657 self.local_loose()?
3658 .write_store_prehashed(&oid, store_bytes, WriteOptions::default())
3659 }
3660
3661 pub fn write_raw_local(&self, store_bytes: &[u8]) -> Result<ObjectId> {
3669 if self.custom_primary.is_some() {
3670 return Err(Error::UnsupportedObjectStore {
3671 operation: "write_raw_local",
3672 });
3673 }
3674 let obj = parse_object_bytes(store_bytes)?;
3675 let oid = hash_object_data_with(self.hash_algo(), obj.kind, &obj.data)?;
3676 if let Some(existing) = self.try_freshen_existing_local(&oid) {
3677 return Ok(existing);
3678 }
3679
3680 self.local_loose()?
3681 .write_store_prehashed(&oid, store_bytes, WriteOptions::default())
3682 }
3683
3684 fn touch_object_mtime(&self, path: &Path) -> Option<std::time::SystemTime> {
3686 #[cfg(test)]
3687 self.hot_path_test_metrics.record_freshen();
3688 crate::fs_readonly::set_file_times_now(path)
3689 }
3690}
3691
3692fn hash_object_data_with(algo: HashAlgo, kind: ObjectKind, data: &[u8]) -> Result<ObjectId> {
3695 hash::hash_object(algo, kind, data)
3696}
3697
3698#[must_use]
3700pub(crate) fn empty_tree_oid_matches(algo: HashAlgo, oid: &ObjectId) -> bool {
3701 oid.is_empty_tree_for(algo)
3702}
3703
3704pub(crate) fn zlib_compress_blob_loose(
3707 data: &[u8],
3708 compression: flate2::Compression,
3709) -> Result<Vec<u8>> {
3710 use std::io::Write as _;
3711 let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), compression);
3712 let zlib_err = |e: std::io::Error| Error::Zlib(e.to_string());
3713 write!(encoder, "{} {}\0", ObjectKind::Blob, data.len()).map_err(zlib_err)?;
3714 encoder.write_all(data).map_err(zlib_err)?;
3715 encoder.finish().map_err(zlib_err)
3716}
3717
3718fn append_lookup_prefix_layer(
3719 store: &dyn ObjectStore,
3720 prefix: &str,
3721 limit: usize,
3722 out: &mut Vec<ObjectId>,
3723 seen: &mut OidHashSet,
3724) -> Result<()> {
3725 let mut scratch = Vec::new();
3726 store.lookup_prefix(prefix, 0, &mut scratch)?;
3727 for oid in scratch {
3728 if seen.insert(oid) {
3729 out.push(oid);
3730 if limit != 0 && out.len() >= limit {
3731 return Ok(());
3732 }
3733 }
3734 }
3735 Ok(())
3736}
3737
3738fn materialize_stream(stream: ObjectStream<'_>) -> Result<ObjectStream<'static>> {
3739 use std::io::{Cursor, Read};
3740 let ObjectStream {
3741 kind,
3742 size,
3743 mut reader,
3744 } = stream;
3745 let mut payload = Vec::new();
3746 reader.read_to_end(&mut payload).map_err(Error::Io)?;
3747 Ok(ObjectStream {
3748 kind,
3749 size,
3750 reader: Box::new(Cursor::new(payload)),
3751 })
3752}
3753
3754fn parse_alternate_env(val: &str) -> Vec<PathBuf> {
3757 let mut result = Vec::new();
3758 let mut chars = val.chars().peekable();
3759 while chars.peek().is_some() {
3760 if chars.peek() == Some(&':') {
3761 chars.next();
3762 continue;
3763 }
3764 if chars.peek() == Some(&'"') {
3765 chars.next(); let saved: Vec<char> = chars.clone().collect();
3769 let mut path = String::new();
3770 let mut properly_closed = false;
3771 loop {
3772 match chars.next() {
3773 None => break,
3774 Some('"') => {
3775 properly_closed = true;
3776 break;
3777 }
3778 Some('\\') => match chars.peek() {
3779 Some(c) if c.is_ascii_digit() => {
3780 let mut oct = String::new();
3781 for _ in 0..3 {
3782 if let Some(&c) = chars.peek() {
3783 if c.is_ascii_digit() {
3784 oct.push(c);
3785 chars.next();
3786 } else {
3787 break;
3788 }
3789 } else {
3790 break;
3791 }
3792 }
3793 if let Ok(byte) = u8::from_str_radix(&oct, 8) {
3794 path.push(byte as char);
3795 }
3796 }
3797 Some(_) => {
3798 if let Some(c) = chars.next() {
3799 match c {
3800 'n' => path.push('\n'),
3801 't' => path.push('\t'),
3802 'r' => path.push('\r'),
3803 _ => path.push(c),
3804 }
3805 }
3806 }
3807 None => {}
3808 },
3809 Some(c) => path.push(c),
3810 }
3811 }
3812 if !properly_closed {
3813 let raw: String = std::iter::once('"').chain(saved).collect();
3816 let raw_path = raw.split(':').next().unwrap_or(&raw);
3818 if !raw_path.is_empty() {
3819 result.push(PathBuf::from(raw_path));
3820 }
3821 let remainder = &raw[raw_path.len()..];
3825 if let Some(rest) = remainder.strip_prefix(':') {
3826 result.extend(parse_alternate_env(rest));
3828 }
3829 return result;
3830 } else if !path.is_empty() {
3831 result.push(PathBuf::from(path));
3832 }
3833 } else {
3834 let mut path = String::new();
3835 while let Some(&c) = chars.peek() {
3836 if c == ':' {
3837 break;
3838 }
3839 path.push(c);
3840 chars.next();
3841 }
3842 if !path.is_empty() {
3843 result.push(PathBuf::from(path));
3844 }
3845 }
3846 }
3847 result
3848}
3849
3850fn system_time_to_file_time(t: std::time::SystemTime) -> FileTime {
3851 let dur = t
3852 .duration_since(std::time::SystemTime::UNIX_EPOCH)
3853 .unwrap_or_default();
3854 FileTime::from_unix_time(
3855 i64::try_from(dur.as_secs()).unwrap_or(i64::MAX),
3856 dur.subsec_nanos(),
3857 )
3858}
3859
3860fn superseded_pack_indexes(
3861 pack_dir: &Path,
3862 keep_pack: &Path,
3863 retained_packs: &HashSet<PathBuf>,
3864 inventory: &RepackPackInventorySnapshot,
3865) -> Result<Vec<(PathBuf, std::time::SystemTime)>> {
3866 let keep_stem = keep_pack
3867 .file_stem()
3868 .and_then(|s| s.to_str())
3869 .ok_or_else(|| {
3870 Error::Io(std::io::Error::new(
3871 std::io::ErrorKind::InvalidInput,
3872 "bad pack path",
3873 ))
3874 })?;
3875 if !pack_dir.is_dir() {
3876 return Ok(Vec::new());
3877 }
3878 let mut out = Vec::new();
3879 for entry in fs::read_dir(pack_dir).map_err(Error::Io)? {
3880 let entry = entry.map_err(Error::Io)?;
3881 let name = entry.file_name();
3882 let name_str = name.to_string_lossy();
3883 if name_str == keep_stem || name_str.starts_with(&format!("{keep_stem}.")) {
3884 continue;
3885 }
3886 if !name_str.ends_with(".idx") {
3887 continue;
3888 }
3889 let idx_path = entry.path();
3890 let pack_path = idx_path.with_extension("pack");
3891 if !pack_path.is_file() {
3892 continue;
3893 }
3894 if inventory.kept_at_inventory.contains(&pack_path)
3895 || pack_path.with_extension("promisor").is_file()
3896 || retained_packs.contains(&pack_path)
3897 || !inventory.pack_paths.contains(&pack_path)
3898 {
3899 continue;
3900 }
3901 let pack_mtime = fs::metadata(&pack_path)
3902 .and_then(|m| m.modified())
3903 .unwrap_or(std::time::SystemTime::UNIX_EPOCH);
3904 out.push((idx_path, pack_mtime));
3905 }
3906 Ok(out)
3907}
3908
3909fn oids_in_inventory_kept_packs(
3910 pack_dir: &Path,
3911 keep_pack: &Path,
3912 inventory_kept_packs: &HashSet<PathBuf>,
3913) -> Result<OidHashSet> {
3914 let keep_stem = keep_pack.file_stem().and_then(|s| s.to_str()).unwrap_or("");
3915 let mut oids = oid_hash_set();
3916 for pack_path in inventory_kept_packs {
3917 let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str()) else {
3918 continue;
3919 };
3920 if stem == keep_stem {
3921 continue;
3922 }
3923 if pack_path.parent().is_some_and(|p| p != pack_dir) {
3924 continue;
3925 }
3926 let idx_path = pack_path.with_extension("idx");
3927 if idx_path.is_file() {
3928 oids.extend(PackedObjects::object_ids_from_index_path(&idx_path)?);
3929 }
3930 }
3931 Ok(oids)
3932}
3933
3934#[cfg(test)]
3935mod tests {
3936 #![allow(clippy::expect_used, clippy::unwrap_used)]
3937
3938 use super::*;
3939 use tempfile::TempDir;
3940
3941 #[test]
3942 fn round_trip_blob() {
3943 let dir = TempDir::new().unwrap();
3944 let odb = Odb::new(dir.path());
3945 let data = b"hello world";
3946 let oid = odb.write(ObjectKind::Blob, data).unwrap();
3947 let obj = odb.read(&oid).unwrap();
3948 assert_eq!(obj.kind, ObjectKind::Blob);
3949 assert_eq!(obj.data, data);
3950 }
3951
3952 #[test]
3953 fn known_blob_hash() {
3954 let oid = HashAlgo::Sha1
3957 .hash_object(ObjectKind::Blob, b"hello")
3958 .expect("hash");
3959 assert_eq!(oid.to_hex(), "b6fc4c620b67d95f953a5c1c1230aaab5db5a1b0");
3960 }
3961
3962 fn pack_objects(objects: &Path, oids: &[ObjectId]) {
3964 let odb = Odb::new(objects);
3965 let pack = crate::pack_objects::build_pack_from_send_list(
3966 &odb,
3967 oids,
3968 &HashSet::new(),
3969 &crate::pack_objects::PackBuildOptions::default(),
3970 &HashMap::new(),
3971 )
3972 .unwrap();
3973 for oid in oids {
3974 fs::remove_file(oid.loose_path_in(objects)).unwrap();
3975 }
3976 crate::index_pack::install_pack_bytes(
3977 pack,
3978 &odb,
3979 &crate::index_pack::IngestPackOptions::default(),
3980 )
3981 .unwrap();
3982 }
3983
3984 #[test]
3985 fn general_reads_cache_trees_and_tiny_blobs_but_not_larger_blobs() {
3986 let dir = TempDir::new().unwrap();
3987 let odb = Odb::new(dir.path());
3988 let blob = odb
3989 .write(
3990 ObjectKind::Blob,
3991 &vec![b'x'; SESSION_READ_BLOB_CACHE_MAX + 1],
3992 )
3993 .unwrap();
3994 let tiny = odb.write(ObjectKind::Blob, b"tiny\n").unwrap();
3995 let tree = odb
3996 .write(
3997 ObjectKind::Tree,
3998 &crate::objects::serialize_tree(&[crate::objects::TreeEntry {
3999 mode: 0o100644,
4000 name: b"f".to_vec(),
4001 oid: blob,
4002 }]),
4003 )
4004 .unwrap();
4005 assert_eq!(
4006 odb.read(&blob).unwrap().data.len(),
4007 SESSION_READ_BLOB_CACHE_MAX + 1
4008 );
4009 assert!(odb.session_loose_reads.lock().unwrap().is_empty());
4010 let tree_bytes = odb.read(&tree).unwrap().data;
4011 assert_eq!(odb.read(&tiny).unwrap().data, b"tiny\n");
4012 for oid in [blob, tree, tiny] {
4013 let hex = oid.to_hex();
4014 fs::remove_file(dir.path().join(&hex[..2]).join(&hex[2..])).unwrap();
4015 }
4016 assert_eq!(odb.read(&tree).unwrap().data, tree_bytes, "tree reused");
4017 assert_eq!(odb.read(&tiny).unwrap().data, b"tiny\n", "tiny blob reused");
4018 assert!(odb.read(&blob).is_err(), "blob was not retained");
4019 }
4020
4021 #[test]
4022 fn session_read_cache_is_bounded_by_bytes_and_entries() {
4023 let mut cache = SessionLooseReads::default();
4024 let chunk = SESSION_LOOSE_CACHE_MAX_BYTES / 4;
4025 let object = |len: usize| Object {
4026 kind: ObjectKind::Tree,
4027 data: vec![0; len],
4028 };
4029 let oid = |n: u32| {
4030 let mut raw = [0u8; 20];
4031 raw[..4].copy_from_slice(&n.to_be_bytes());
4032 ObjectId::from_bytes(&raw).unwrap()
4033 };
4034 for n in 0..4 {
4035 cache.insert(oid(n), &object(chunk));
4036 }
4037 assert_eq!(cache.bytes, SESSION_LOOSE_CACHE_MAX_BYTES);
4038 cache.insert(oid(4), &object(1));
4039 assert_eq!(cache.bytes, 1, "exceeding the byte budget starts over");
4040 assert!(cache.get(&oid(0)).is_none());
4041 cache.insert(oid(4), &object(10));
4042 assert_eq!(cache.bytes, 10, "replacing an entry drops its old size");
4043 cache.insert(oid(5), &object(SESSION_LOOSE_CACHE_MAX_BYTES + 1));
4044 assert!(
4045 cache.get(&oid(5)).is_none(),
4046 "oversized objects are never kept"
4047 );
4048 assert!(cache.get(&oid(4)).is_some());
4049 for n in 10..(10 + SESSION_LOOSE_CACHE_MAX_ENTRIES as u32 + 2) {
4050 cache.insert(oid(n), &object(1));
4051 }
4052 assert!(cache.objects.len() <= SESSION_LOOSE_CACHE_MAX_ENTRIES + 1);
4053 assert!(cache.bytes <= SESSION_LOOSE_CACHE_MAX_BYTES);
4054 }
4055
4056 fn bounded_read_fixture() -> (TempDir, PathBuf, ObjectId, ObjectId) {
4057 let dir = TempDir::new().unwrap();
4058 let objects = dir.path().to_path_buf();
4059 let odb = Odb::new(&objects);
4060 let small = odb.write(ObjectKind::Blob, b"small blob\n").unwrap();
4061 let big: Vec<u8> = (0..300_000u32).flat_map(|i| i.to_le_bytes()).collect();
4062 let big = odb.write(ObjectKind::Blob, &big).unwrap();
4063 (dir, objects, small, big)
4064 }
4065
4066 fn assert_bounded_reads(odb: &Odb, small: &ObjectId, big: &ObjectId) {
4067 match odb.read_unless_larger(small, 1000, false).unwrap() {
4068 BoundedRead::Object(obj) => assert_eq!(obj.data, b"small blob\n"),
4069 BoundedRead::Larger => panic!("small blob is within the limit"),
4070 }
4071 assert!(matches!(
4072 odb.read_unless_larger(big, 1000, false).unwrap(),
4073 BoundedRead::Larger
4074 ));
4075 match odb.read_unless_larger(big, 1_200_000, false).unwrap() {
4076 BoundedRead::Object(obj) => assert_eq!(obj.data.len(), 1_200_000),
4077 BoundedRead::Larger => panic!("limit equal to the size reads the object"),
4078 }
4079 let missing = ObjectId::from_hex("1234567890123456789012345678901234567890").unwrap();
4080 assert!(matches!(
4081 odb.read_unless_larger(&missing, 1000, false),
4082 Err(Error::ObjectNotFound(_))
4083 ));
4084 }
4085
4086 #[test]
4087 fn read_unless_larger_sizes_loose_objects_from_their_header() {
4088 let (_dir, objects, small, big) = bounded_read_fixture();
4089 assert_bounded_reads(&Odb::new(&objects), &small, &big);
4090 }
4091
4092 #[test]
4093 fn read_unless_larger_sizes_packed_objects_in_and_out_of_read_context() {
4094 let (_dir, objects, small, big) = bounded_read_fixture();
4095 pack_objects(&objects, &[small, big]);
4096 let odb = Odb::new(&objects);
4097 assert_bounded_reads(&odb, &small, &big);
4098 odb.with_pack_read_context(|| assert_bounded_reads(&odb, &small, &big));
4099 }
4100
4101 #[test]
4102 fn parsed_tree_reads_keep_no_raw_copy_in_the_session_cache() {
4103 let (_dir, objects, small, _big) = bounded_read_fixture();
4104 let odb = Odb::new(&objects);
4105 let entries = [crate::objects::TreeEntry {
4106 mode: 0o100644,
4107 name: b"small".to_vec(),
4108 oid: small,
4109 }];
4110 let tree = odb
4111 .write(ObjectKind::Tree, &crate::objects::serialize_tree(&entries))
4112 .unwrap();
4113 for packed in [false, true] {
4114 if packed {
4115 pack_objects(&objects, &[tree]);
4116 }
4117 let odb = Odb::new(&objects);
4118 assert_eq!(*odb.cached_tree_entries(&tree).unwrap(), entries);
4119 assert!(odb.session_cached_loose_read(&tree).is_none());
4120 assert!(Arc::ptr_eq(
4121 &odb.cached_tree_entries(&tree).unwrap(),
4122 &odb.cached_tree_entries(&tree).unwrap()
4123 ));
4124 assert_eq!(odb.read(&tree).unwrap().kind, ObjectKind::Tree);
4125 assert!(
4126 odb.session_cached_loose_read(&tree).is_some(),
4127 "plain reads still use the session cache (packed: {packed})"
4128 );
4129 }
4130 }
4131
4132 #[test]
4133 fn read_unless_larger_reuses_small_loose_reads_but_never_caches_big_ones() {
4134 let (_dir, objects, small, big) = bounded_read_fixture();
4135 let odb = Odb::new(&objects);
4136 assert_bounded_reads(&odb, &small, &big);
4137 assert!(odb.session_loose_reads.lock().unwrap().is_empty());
4138 assert!(matches!(
4139 odb.read_unless_larger(&small, 1000, true).unwrap(),
4140 BoundedRead::Object(_)
4141 ));
4142 assert!(matches!(
4143 odb.read_unless_larger(&big, 1_200_000, true).unwrap(),
4144 BoundedRead::Object(_)
4145 ));
4146 for oid in [small, big] {
4147 let hex = oid.to_hex();
4148 fs::remove_file(objects.join(&hex[..2]).join(&hex[2..])).unwrap();
4149 }
4150 match odb.read_unless_larger(&small, 1000, true).unwrap() {
4151 BoundedRead::Object(obj) => assert_eq!(obj.data, b"small blob\n"),
4152 BoundedRead::Larger => panic!("small blob is within the limit"),
4153 }
4154 assert!(matches!(
4155 odb.read_unless_larger(&small, 4, true),
4156 Ok(BoundedRead::Larger)
4157 ));
4158 assert!(matches!(
4159 odb.read_unless_larger(&big, 1_200_000, true),
4160 Err(Error::ObjectNotFound(_))
4161 ));
4162 }
4163
4164 #[test]
4165 fn read_unless_larger_finds_objects_across_several_packs() {
4166 let (_dir, objects, small, big) = bounded_read_fixture();
4167 pack_objects(&objects, &[small]);
4168 pack_objects(&objects, &[big]);
4169 let odb = Odb::new(&objects);
4170 odb.with_pack_read_context(|| assert_bounded_reads(&odb, &small, &big));
4171 }
4172
4173 #[test]
4174 fn parse_object_header_prefix_unknown_kind_is_typed_error() {
4175 let err = crate::odb::store::loose::parse_object_header_prefix(b"not-a-git-kind 0\0")
4176 .unwrap_err();
4177 assert!(matches!(err, Error::UnknownObjectType(_)));
4178 }
4179
4180 fn git_dir_with_config(config: &str) -> (TempDir, PathBuf, PathBuf) {
4181 let dir = TempDir::new().unwrap();
4182 let git_dir = dir.path().join(".git");
4183 let objects = git_dir.join("objects");
4184 fs::create_dir_all(&objects).unwrap();
4185 fs::write(git_dir.join("config"), config).unwrap();
4186 (dir, git_dir, objects)
4187 }
4188
4189 fn zlib_flevel(compressed: &[u8]) -> u8 {
4191 assert!(compressed.len() >= 2);
4192 (compressed[1] >> 6) & 3
4193 }
4194
4195 fn tmp_loose_object_paths(objects: &Path) -> Vec<PathBuf> {
4196 let mut tmp = Vec::new();
4197 let Ok(shards) = fs::read_dir(objects) else {
4198 return tmp;
4199 };
4200 for shard in shards.flatten() {
4201 if !shard.file_type().is_ok_and(|t| t.is_dir()) {
4202 continue;
4203 }
4204 let Ok(files) = fs::read_dir(shard.path()) else {
4205 continue;
4206 };
4207 for file in files.flatten() {
4208 let name = file.file_name();
4209 if name.to_string_lossy().starts_with("tmp_") {
4210 tmp.push(file.path());
4211 }
4212 }
4213 }
4214 tmp
4215 }
4216
4217 #[test]
4218 fn loose_write_honours_level() {
4219 let payload: Vec<u8> = (0..4096).map(|i| (i % 251) as u8).collect();
4220
4221 let (_dir, git_dir, objects) = git_dir_with_config("");
4222 let odb = Odb::new(&objects).with_config_git_dir(git_dir.clone());
4223 let oid_default = odb.write(ObjectKind::Blob, &payload).unwrap();
4224 let bytes_default = fs::read(odb.object_path(&oid_default)).unwrap();
4225 assert!(zlib_flevel(&bytes_default) <= 1);
4226 assert_eq!(odb.read(&oid_default).unwrap().data, payload);
4227
4228 for level in -1i32..=9 {
4229 let config = format!("[core]\n\tloosecompression = {level}\n");
4230 let (_dir, git_dir, objects) = git_dir_with_config(&config);
4231 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
4232 let byte = u8::try_from(level.rem_euclid(256)).unwrap_or(0);
4233 let bytes: Vec<u8> = vec![byte; 512];
4234 let oid = odb.write(ObjectKind::Blob, &bytes).unwrap();
4235 let on_disk = fs::read(odb.object_path(&oid)).unwrap();
4236 assert!(on_disk.len() >= 2, "level {level}");
4237 assert_eq!(odb.read(&oid).unwrap().data, bytes.as_slice());
4238 }
4239
4240 let config = "[core]\n\tcompression = 9\n";
4241 let (_dir, git_dir, objects) = git_dir_with_config(config);
4242 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
4243 let oid = odb.write(ObjectKind::Blob, &payload).unwrap();
4244 assert_eq!(zlib_flevel(&fs::read(odb.object_path(&oid)).unwrap()), 3);
4245
4246 let config = "[core]\n\tcompression = 0\n\tloosecompression = 1\n";
4247 let (_dir, git_dir, objects) = git_dir_with_config(config);
4248 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
4249 let oid = odb.write(ObjectKind::Blob, &payload).unwrap();
4250 assert_eq!(zlib_flevel(&fs::read(odb.object_path(&oid)).unwrap()), 0);
4251
4252 let config = "[core]\n\tloosecompression = 99\n";
4253 let (_dir, git_dir, objects) = git_dir_with_config(config);
4254 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
4255 assert!(odb.write(ObjectKind::Blob, b"x").is_err());
4256 assert!(tmp_loose_object_paths(&objects).is_empty());
4257 }
4258
4259 fn git_run(dir: &Path, args: &[&str]) -> std::process::Output {
4260 std::process::Command::new("git") .current_dir(dir)
4262 .args(args)
4263 .env("GIT_CONFIG_GLOBAL", "/dev/null")
4264 .env("GIT_CONFIG_SYSTEM", "/dev/null")
4265 .env("GIT_AUTHOR_NAME", "T")
4266 .env("GIT_AUTHOR_EMAIL", "t@example.com")
4267 .env("GIT_COMMITTER_NAME", "T")
4268 .env("GIT_COMMITTER_EMAIL", "t@example.com")
4269 .output()
4270 .unwrap()
4271 }
4272
4273 fn git_in(dir: &Path, args: &[&str]) {
4274 let out = git_run(dir, args);
4275 assert!(
4276 out.status.success(),
4277 "git {:?}: {}",
4278 args,
4279 String::from_utf8_lossy(&out.stderr)
4280 );
4281 }
4282
4283 fn git_rev_parse_oid(dir: &Path, spec: &str) -> ObjectId {
4284 let out = git_run(dir, &["rev-parse", spec]);
4285 assert!(
4286 out.status.success(),
4287 "git rev-parse {spec}: {}",
4288 String::from_utf8_lossy(&out.stderr)
4289 );
4290 ObjectId::from_hex(std::str::from_utf8(&out.stdout).unwrap().trim()).unwrap()
4291 }
4292
4293 #[test]
4294 fn write_loose_zlib_prehashed_matches_write() {
4295 let dir = TempDir::new().unwrap();
4296 let objects = dir.path().join("objects");
4297 let odb = Odb::new(&objects);
4298 let data = b"parallel prehash path";
4299 let oid_normal = odb.write(ObjectKind::Blob, data).unwrap();
4300
4301 let dir2 = TempDir::new().unwrap();
4302 let objects2 = dir2.path().join("objects");
4303 let odb2 = Odb::new(&objects2);
4304 let oid = hash::hash_object(odb2.hash_algo(), ObjectKind::Blob, data).expect("hash");
4305 assert_eq!(oid, oid_normal);
4306 let zlib =
4307 super::zlib_compress_blob_loose(data, odb2.loose_compression().unwrap()).unwrap();
4308 let oid2 = odb2
4309 .write_loose_zlib_prehashed(
4310 &oid,
4311 &zlib,
4312 WriteOptions {
4313 assume_loose_only_existence: true,
4314 ..WriteOptions::default()
4315 },
4316 )
4317 .unwrap();
4318 assert_eq!(oid2, oid_normal);
4319 assert_eq!(
4320 fs::read(odb2.object_path(&oid2)).unwrap(),
4321 fs::read(odb.object_path(&oid_normal)).unwrap()
4322 );
4323 }
4324
4325 #[test]
4326 fn write_blob_prehashed_matches_write_and_skips_existing_objects() {
4327 let dir = TempDir::new().unwrap();
4328 let odb = Odb::new(&dir.path().join("objects"));
4329 let data = vec![7u8; 100_000];
4330 let oid_normal = odb.write(ObjectKind::Blob, &data).unwrap();
4331
4332 let dir2 = TempDir::new().unwrap();
4333 let odb2 = Odb::new(&dir2.path().join("objects"));
4334 let oid = hash::hash_object(odb2.hash_algo(), ObjectKind::Blob, &data).unwrap();
4335 let options = WriteOptions {
4336 assume_loose_only_existence: true,
4337 ..WriteOptions::default()
4338 };
4339 assert_eq!(
4340 odb2.write_blob_prehashed(&oid, &data, options).unwrap(),
4341 oid
4342 );
4343 assert_eq!(oid, oid_normal);
4344 assert!(odb2.object_path(&oid).is_file());
4345 let obj = odb2.read(&oid).unwrap();
4346 assert_eq!(
4347 (obj.kind, obj.data.as_slice()),
4348 (ObjectKind::Blob, data.as_slice())
4349 );
4350
4351 assert_eq!(
4353 odb2.write_blob_prehashed(&oid, b"other", options).unwrap(),
4354 oid
4355 );
4356 assert_eq!(odb2.read(&oid).unwrap().data, data);
4357 }
4358
4359 #[test]
4360 fn write_loose_zlib_prehashed_routes_to_custom_primary() {
4361 use crate::objects::HashAlgo;
4362 use std::sync::Arc;
4363
4364 let dir = TempDir::new().unwrap();
4365 let objects = dir.path().join("objects");
4366 fs::create_dir_all(&objects).unwrap();
4367 let store = Arc::new(MemoryStore::new(HashAlgo::Sha1));
4368 let odb = OdbBuilder::files(&objects)
4369 .primary(store.clone())
4370 .alternates(false)
4371 .build();
4372 let data = b"staged through a custom primary";
4373 let oid = hash::hash_object(HashAlgo::Sha1, ObjectKind::Blob, data).unwrap();
4374 let zlib = super::zlib_compress_blob_loose(data, Compression::fast()).unwrap();
4375
4376 odb.ensure_all_loose_prefix_dirs().unwrap();
4377 let written = odb
4378 .write_loose_zlib_prehashed(&oid, &zlib, WriteOptions::default())
4379 .unwrap();
4380 assert_eq!(written, oid);
4381 assert!(store.contains(&oid).unwrap());
4382 assert_eq!(odb.read(&oid).unwrap().data, data);
4383 assert_eq!(
4384 fs::read_dir(&objects).unwrap().count(),
4385 0,
4386 "no loose shard directories or files beside a custom primary"
4387 );
4388
4389 assert!(odb
4390 .write_loose_zlib_prehashed(&oid, b"not zlib", WriteOptions::default())
4391 .is_err());
4392 }
4393
4394 #[test]
4397 fn write_existing_loose_does_not_rewrite() {
4398 let dir = TempDir::new().unwrap();
4399 let objects = dir.path().join("objects");
4400 fs::create_dir_all(&objects).unwrap();
4401 let odb = Odb::new(&objects);
4402 let payload = b"already loose";
4403 let oid = odb.write(ObjectKind::Blob, payload).unwrap();
4404 let path = odb.object_path(&oid);
4405 let before_bytes = fs::read(&path).unwrap();
4406 let meta_before = fs::metadata(&path).unwrap();
4407 let before_mtime = filetime::FileTime::from_last_modification_time(&meta_before);
4408 #[cfg(unix)]
4409 let ino_before = {
4410 use std::os::unix::fs::MetadataExt;
4411 meta_before.ino()
4412 };
4413 std::thread::sleep(std::time::Duration::from_millis(20));
4414 let oid2 = odb.write(ObjectKind::Blob, payload).unwrap();
4415 assert_eq!(oid, oid2);
4416 assert_eq!(fs::read(&path).unwrap(), before_bytes);
4417 let meta_after = fs::metadata(&path).unwrap();
4418 let after_mtime = filetime::FileTime::from_last_modification_time(&meta_after);
4419 assert!(
4420 after_mtime >= before_mtime,
4421 "expected freshen to bump or preserve mtime"
4422 );
4423 #[cfg(unix)]
4424 {
4425 use std::os::unix::fs::MetadataExt;
4426 assert_eq!(
4427 meta_after.ino(),
4428 ino_before,
4429 "rewrite must not replace the loose object file"
4430 );
4431 }
4432 }
4433
4434 #[cfg_attr(
4436 not(feature = "full-tests"),
4437 ignore = "starts system git or is heavy; run with --features full-tests"
4438 )]
4439 #[test]
4440 fn write_skips_loose_when_object_in_pack_only() {
4441 let dir = TempDir::new().unwrap();
4442 git_in(dir.path(), &["init", "-q"]);
4443 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
4444 git_in(dir.path(), &["config", "user.name", "T"]);
4445 fs::write(dir.path().join("f"), b"pack-only payload").unwrap();
4446 git_in(dir.path(), &["add", "f"]);
4447 git_in(dir.path(), &["commit", "-m", "c"]);
4448 git_in(dir.path(), &["repack", "-ad"]);
4449 let objects = dir.path().join(".git").join("objects");
4450 let odb = Odb::new(&objects);
4451 let oid = odb.write(ObjectKind::Blob, b"pack-only payload").unwrap();
4452 assert!(!odb.object_path(&oid).exists());
4453 assert!(odb.exists(&oid));
4454 }
4455
4456 #[cfg_attr(
4458 not(feature = "full-tests"),
4459 ignore = "starts system git or is heavy; run with --features full-tests"
4460 )]
4461 #[test]
4462 fn write_freshens_loose_object_in_alternate() {
4463 use filetime::FileTime;
4464 use std::thread;
4465 use std::time::Duration;
4466
4467 let alt = TempDir::new().unwrap();
4468 git_in(alt.path(), &["init", "-q", "--bare"]);
4469 git_in(alt.path(), &["config", "user.email", "t@example.com"]);
4470 let alt_objects = alt.path().join("objects");
4471 let alt_odb = Odb::new(&alt_objects);
4472 let payload = b"alternate-loose";
4473 let oid = alt_odb.write(ObjectKind::Blob, payload).unwrap();
4474 let alt_loose = alt_odb.object_path(&oid);
4475
4476 let primary = TempDir::new().unwrap();
4477 let primary_objects = primary.path().join("objects");
4478 fs::create_dir_all(primary_objects.join("info")).unwrap();
4479 fs::write(
4480 primary_objects.join("info").join("alternates"),
4481 format!("{}\n", alt_objects.display()),
4482 )
4483 .unwrap();
4484 let odb = Odb::new(&primary_objects);
4485 thread::sleep(Duration::from_millis(50));
4486 let before = FileTime::from_last_modification_time(&fs::metadata(&alt_loose).unwrap());
4487 filetime::set_file_mtime(&alt_loose, before).unwrap();
4488
4489 let oid2 = odb.write(ObjectKind::Blob, payload).unwrap();
4490 assert_eq!(oid, oid2);
4491 assert!(!odb.object_path(&oid).exists());
4492 let after = FileTime::from_last_modification_time(&fs::metadata(&alt_loose).unwrap());
4493 assert!(after > before, "write must freshen alternate loose object");
4494 }
4495
4496 #[cfg_attr(
4497 not(feature = "full-tests"),
4498 ignore = "starts system git or is heavy; run with --features full-tests"
4499 )]
4500 #[test]
4501 fn write_freshens_packed_object_in_alternate() {
4502 use filetime::FileTime;
4503 use std::thread;
4504 use std::time::Duration;
4505
4506 let alt = TempDir::new().unwrap();
4507 git_in(alt.path(), &["init", "-q"]);
4508 git_in(alt.path(), &["config", "user.email", "t@example.com"]);
4509 git_in(alt.path(), &["config", "user.name", "T"]);
4510 let payload = b"alternate-pack";
4511 fs::write(alt.path().join("f"), payload).unwrap();
4512 git_in(alt.path(), &["add", "f"]);
4513 git_in(alt.path(), &["commit", "-m", "c"]);
4514 git_in(alt.path(), &["repack", "-ad"]);
4515 let alt_objects = alt.path().join(".git").join("objects");
4516 let alt_odb = Odb::new(&alt_objects);
4517 let oid = alt_odb.write(ObjectKind::Blob, payload).unwrap();
4518 let pack_path = fs::read_dir(alt_objects.join("pack"))
4519 .unwrap()
4520 .flatten()
4521 .find(|e| e.path().extension().is_some_and(|x| x == "pack"))
4522 .unwrap()
4523 .path();
4524 thread::sleep(Duration::from_millis(50));
4525 let before = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
4526
4527 let primary = TempDir::new().unwrap();
4528 let primary_objects = primary.path().join("objects");
4529 fs::create_dir_all(primary_objects.join("info")).unwrap();
4530 fs::write(
4531 primary_objects.join("info").join("alternates"),
4532 format!("{}\n", alt_objects.display()),
4533 )
4534 .unwrap();
4535 pack::clear_pack_cache();
4536 let odb = Odb::new(&primary_objects);
4537 assert!(odb.exists(&oid));
4538 let oid2 = odb.write(ObjectKind::Blob, payload).unwrap();
4539 assert_eq!(oid, oid2);
4540 assert!(!odb.object_path(&oid).exists());
4541 let after = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
4542 assert!(after >= before, "write must freshen alternate pack");
4543 }
4544
4545 #[cfg_attr(
4546 not(feature = "full-tests"),
4547 ignore = "starts system git or is heavy; run with --features full-tests"
4548 )]
4549 #[test]
4550 fn write_skips_loose_when_object_only_in_alternate() {
4551 let alt = TempDir::new().unwrap();
4552 git_in(alt.path(), &["init", "-q", "--bare"]);
4553 git_in(alt.path(), &["config", "user.email", "t@example.com"]);
4554 let alt_objects = alt.path().join("objects");
4555 let alt_odb = Odb::new(&alt_objects);
4556 let payload = b"alternate-only";
4557 let oid = alt_odb.write(ObjectKind::Blob, payload).unwrap();
4558
4559 let primary = TempDir::new().unwrap();
4560 let primary_objects = primary.path().join("objects");
4561 fs::create_dir_all(primary.path().join("objects").join("info")).unwrap();
4562 fs::write(
4563 primary
4564 .path()
4565 .join("objects")
4566 .join("info")
4567 .join("alternates"),
4568 format!("{}\n", alt_objects.display()),
4569 )
4570 .unwrap();
4571 let odb = Odb::new(&primary_objects);
4572 assert!(odb.exists(&oid));
4573 let oid2 = odb.write(ObjectKind::Blob, payload).unwrap();
4574 assert_eq!(oid, oid2);
4575 assert!(!odb.object_path(&oid).exists());
4576 }
4577
4578 #[cfg_attr(
4579 not(feature = "full-tests"),
4580 ignore = "starts system git or is heavy; run with --features full-tests"
4581 )]
4582 #[test]
4583 fn write_with_mem_overlay_stays_in_memory() {
4584 let dir = TempDir::new().unwrap();
4585 let objects = dir.path().join("objects");
4586 fs::create_dir_all(&objects).unwrap();
4587 let odb = Odb::new(&objects);
4588 odb.enable_mem_overlay();
4589 let payload = b"overlay blob";
4590 let oid = odb.write(ObjectKind::Blob, payload).unwrap();
4591 assert!(!odb.object_path(&oid).exists());
4592 assert_eq!(odb.read(&oid).unwrap().data, payload);
4593 odb.disable_mem_overlay();
4594 assert!(odb.read(&oid).is_err());
4595 }
4596
4597 #[test]
4598 fn read_alternates_relative_path_and_symlinked_objects_dir() {
4599 use crate::pack::read_alternates_recursive;
4600
4601 let layout = TempDir::new().unwrap();
4602 let alt_objects = layout.path().join("alt-store").join("objects");
4603 fs::create_dir_all(&alt_objects).unwrap();
4604 let primary_objects = layout.path().join("repo").join("objects");
4605 fs::create_dir_all(primary_objects.join("info")).unwrap();
4606 fs::write(
4607 primary_objects.join("info").join("alternates"),
4608 "../../alt-store/objects\n",
4609 )
4610 .unwrap();
4611 let resolved = read_alternates_recursive(&primary_objects).unwrap();
4612 let alt_canonical = alt_objects
4613 .canonicalize_simplified()
4614 .unwrap_or(alt_objects.clone());
4615 assert!(
4616 resolved.iter().any(|p| {
4617 p.canonicalize_simplified().unwrap_or_else(|_| p.clone()) == alt_canonical
4618 }),
4619 "relative alternate entry should resolve: {resolved:?}"
4620 );
4621
4622 #[cfg(unix)]
4623 {
4624 let link_parent = TempDir::new().unwrap();
4625 std::os::unix::fs::symlink(&primary_objects, link_parent.path().join("objects-link"))
4626 .unwrap();
4627 let via_link =
4628 read_alternates_recursive(&link_parent.path().join("objects-link")).unwrap();
4629 assert!(
4630 !via_link.is_empty(),
4631 "alternates through symlinked objects dir: {via_link:?}"
4632 );
4633 }
4634 }
4635 #[test]
4636 fn empty_tree_exists_without_hex() {
4637 let odb = Odb::new(std::path::Path::new("/nonexistent/objects"));
4638 let canon = ObjectId::from_hex("4b825dc642cb6eb9a060e54bf8d69288fbee4904").unwrap();
4639 let legacy = ObjectId::from_hex("4b825dc642cb6eb9a060e54bf899d69f7c6948d4").unwrap();
4640 assert!(odb.exists(&canon));
4641 assert!(odb.exists(&legacy));
4642 assert!(odb.exists_local(&canon));
4643 assert!(!odb.freshen_object(&canon));
4644 }
4645
4646 #[test]
4647 fn sha256_empty_tree_virtual_read_without_loose_file() {
4648 use crate::objects::HashAlgo;
4649 use std::sync::Arc;
4650
4651 let dir = TempDir::new().unwrap();
4652 let objects = dir.path().join("objects");
4653 fs::create_dir_all(&objects).unwrap();
4654 let store = Arc::new(MemoryStore::new(HashAlgo::Sha256));
4655 let odb = OdbBuilder::files(&objects)
4656 .primary(store)
4657 .alternates(false)
4658 .build();
4659 let empty = odb.empty_tree_oid().expect("hash empty tree");
4660 assert!(!odb.exists_materialized_local(&empty));
4661 assert!(odb.exists_local(&empty));
4662 assert!(odb.is_empty_tree(&empty));
4663 let obj = odb.read(&empty).expect("virtual read empty tree");
4664 assert_eq!(obj.kind, ObjectKind::Tree);
4665 assert!(obj.data.is_empty());
4666 }
4667
4668 #[test]
4669 fn empty_tree_oid_and_object_id_empty_tree_match_sha1() {
4670 let odb = Odb::new(std::path::Path::new("/nonexistent/objects"));
4671 let from_odb = odb.empty_tree_oid().expect("empty tree oid");
4672 assert_eq!(from_odb, ObjectId::empty_tree());
4673 assert!(odb.is_empty_tree(&ObjectId::empty_tree()));
4674 }
4675
4676 #[test]
4677 fn empty_tree_virtual_presence_skips_materialized_local() {
4678 let dir = TempDir::new().unwrap();
4679 let objects = dir.path().join("objects");
4680 fs::create_dir_all(&objects).unwrap();
4681 let odb = Odb::new(&objects);
4682 let canon = ObjectId::from_hex("4b825dc642cb6eb9a060e54bf8d69288fbee4904").unwrap();
4683 assert!(odb.exists_local(&canon));
4684 assert!(!odb.exists_materialized_local(&canon));
4685 }
4686
4687 #[test]
4688 fn write_materializes_well_known_empty_tree() {
4689 let dir = TempDir::new().unwrap();
4690 let objects = dir.path().join("objects");
4691 fs::create_dir_all(&objects).unwrap();
4692 let odb = Odb::new(&objects);
4693 let canon = ObjectId::from_hex("4b825dc642cb6eb9a060e54bf8d69288fbee4904").unwrap();
4694 assert!(!odb.exists_materialized_local(&canon));
4695 let oid = odb
4696 .write(crate::objects::ObjectKind::Tree, &[])
4697 .expect("write empty tree");
4698 assert_eq!(oid, canon);
4699 assert!(odb.exists_materialized_local(&canon));
4700 }
4701
4702 #[test]
4703 fn append_alternate_objects_line_errors_on_unreadable_existing_file() {
4704 let dir = TempDir::new().unwrap();
4705 let objects = dir.path().join("objects");
4706 fs::create_dir_all(objects.join("info")).unwrap();
4707 let alt_path = objects.join("info/alternates");
4708 fs::write(&alt_path, b"not valid utf-8 \xff\n").unwrap();
4709 let err = Odb::append_alternate_objects_line(&objects, Path::new("/tmp/other/objects"))
4710 .unwrap_err();
4711 assert!(matches!(err, Error::Io(_)));
4712 assert_eq!(fs::read(&alt_path).unwrap(), b"not valid utf-8 \xff\n");
4713 }
4714
4715 #[test]
4716 fn append_file_alternate_refreshes_cached_snapshot() {
4717 let primary = TempDir::new().unwrap();
4718 let alternate = TempDir::new().unwrap();
4719 let alt_objects = alternate.path().join("objects");
4720 std::fs::create_dir_all(&alt_objects).unwrap();
4721 let alt_odb = Odb::new(&alt_objects);
4722 let oid = alt_odb.write(ObjectKind::Blob, b"via-alternate").unwrap();
4723
4724 let primary_objects = primary.path().join("objects");
4725 std::fs::create_dir_all(primary_objects.join("info")).unwrap();
4726 let primary_odb = Odb::new(&primary_objects);
4727 assert!(!primary_odb.exists(&oid));
4728 let _ = primary_odb.file_alternate_dirs_snapshot();
4729 primary_odb.append_file_alternate(&alt_objects).unwrap();
4730 assert!(primary_odb.exists(&oid));
4731 }
4732
4733 #[test]
4734 fn exists_via_env_alternate_dirs_at_construction() {
4735 let primary = TempDir::new().unwrap();
4736 let alternate = TempDir::new().unwrap();
4737 let alt_objects = alternate.path().join("objects");
4738 std::fs::create_dir_all(&alt_objects).unwrap();
4739 let alt_odb = Odb::new(&alt_objects);
4740 let oid = alt_odb.write(ObjectKind::Blob, b"env-alt").unwrap();
4741
4742 let primary_objects = primary.path().join("objects");
4743 std::fs::create_dir_all(&primary_objects).unwrap();
4744 let primary_odb = Odb::new(&primary_objects).with_env_alternate_dirs(vec![alt_objects]);
4745 assert!(primary_odb.exists(&oid));
4746 }
4747
4748 fn flaky_odb(objects: &Path) -> (Odb, Arc<store::flaky::FlakyStore>) {
4749 let flaky = store::flaky::FlakyStore::new();
4750 let odb = OdbBuilder::files(objects)
4751 .primary(flaky.clone())
4752 .alternates(false)
4753 .build();
4754 (odb, flaky)
4755 }
4756
4757 #[test]
4758 fn contains_reports_primary_failure_instead_of_missing() {
4759 let dir = TempDir::new().unwrap();
4760 let (odb, flaky) = flaky_odb(dir.path());
4761 let oid = odb.write(ObjectKind::Blob, b"stored").unwrap();
4762 assert!(odb.contains(&oid).unwrap());
4763
4764 flaky.fail_all(true);
4765 assert!(store::flaky::is_injected(&odb.read(&oid).unwrap_err()));
4766 assert!(store::flaky::is_injected(&odb.contains(&oid).unwrap_err()));
4767 assert!(
4768 !odb.exists(&oid),
4769 "exists() keeps treating failures as absent"
4770 );
4771 assert!(odb.contains(&odb.empty_tree_oid().unwrap()).unwrap());
4772
4773 let try_local = odb.try_exists_local(&oid).unwrap_err();
4774 assert!(store::flaky::is_injected(&try_local));
4775 assert!(!odb.exists_local(&oid));
4776 let try_stored = odb.try_exists_stored(&oid).unwrap_err();
4777 assert!(store::flaky::is_injected(&try_stored));
4778 assert!(!odb.exists_stored(&oid));
4779 }
4780
4781 #[test]
4782 fn contains_reports_extra_read_source_failure() {
4783 let dir = TempDir::new().unwrap();
4784 let flaky = store::flaky::FlakyStore::new();
4785 let odb = OdbBuilder::files(dir.path())
4786 .push_read_source(flaky.clone())
4787 .alternates(false)
4788 .build();
4789 let local = odb.write(ObjectKind::Blob, b"in primary").unwrap();
4790 let absent = HashAlgo::Sha1
4791 .hash_object(ObjectKind::Blob, b"nowhere")
4792 .unwrap();
4793 assert!(!odb.contains(&absent).unwrap());
4794
4795 flaky.fail_contains(true);
4796 assert!(
4797 odb.contains(&local).unwrap(),
4798 "primary hit wins over a failing source"
4799 );
4800 assert!(store::flaky::is_injected(
4801 &odb.contains(&absent).unwrap_err()
4802 ));
4803 assert!(!odb.exists(&absent));
4804 }
4805
4806 #[test]
4807 fn contains_reports_primary_failure_even_when_later_source_has_object() {
4808 let dir = TempDir::new().unwrap();
4809 let extra = Arc::new(store::MemoryStore::new(HashAlgo::Sha1));
4810 let flaky = store::flaky::FlakyStore::new();
4811 let odb = OdbBuilder::files(dir.path())
4812 .primary(flaky.clone())
4813 .push_read_source(extra.clone())
4814 .alternates(false)
4815 .build();
4816 let oid = extra
4817 .write(ObjectKind::Blob, b"extra", WriteOptions::default())
4818 .unwrap();
4819 assert!(odb.contains(&oid).unwrap());
4820
4821 flaky.fail_all(true);
4822 assert!(store::flaky::is_injected(&odb.read(&oid).unwrap_err()));
4823 assert!(
4824 store::flaky::is_injected(&odb.contains(&oid).unwrap_err()),
4825 "contains must agree with read"
4826 );
4827 assert!(odb.exists(&oid), "exists() skips the failing primary");
4828 }
4829
4830 #[test]
4831 fn contains_reports_alternate_failure() {
4832 let primary = TempDir::new().unwrap();
4833 let alternate = TempDir::new().unwrap();
4834 let alt_objects = alternate.path().join("objects");
4835 std::fs::create_dir_all(&alt_objects).unwrap();
4836 let primary_objects = primary.path().join("objects");
4837 std::fs::create_dir_all(&primary_objects).unwrap();
4838 let odb = Odb::new(&primary_objects).with_env_alternate_dirs(vec![alt_objects]);
4839 let local = odb.write(ObjectKind::Blob, b"local").unwrap();
4840 let absent = HashAlgo::Sha1
4841 .hash_object(ObjectKind::Blob, b"nowhere")
4842 .unwrap();
4843
4844 let flaky = store::flaky::FlakyStore::new();
4847 odb.sources().unwrap();
4848 odb.alternate_sources_cache.write().unwrap().sources = Some(Arc::new(CompositeStore::new(
4849 vec![flaky.clone()],
4850 HashAlgo::Sha1,
4851 )));
4852 assert!(!odb.contains(&absent).unwrap());
4853
4854 flaky.fail_contains(true);
4855 assert!(odb.contains(&local).unwrap());
4856 assert!(store::flaky::is_injected(
4857 &odb.contains(&absent).unwrap_err()
4858 ));
4859 assert!(!odb.exists(&absent));
4860 }
4861
4862 #[test]
4863 fn contains_on_files_backend_covers_present_absent_empty_tree_and_alternate() {
4864 let primary = TempDir::new().unwrap();
4865 let alternate = TempDir::new().unwrap();
4866 let alt_objects = alternate.path().join("objects");
4867 std::fs::create_dir_all(&alt_objects).unwrap();
4868 let alt_oid = Odb::new(&alt_objects)
4869 .write(ObjectKind::Blob, b"alternate")
4870 .unwrap();
4871 let primary_objects = primary.path().join("objects");
4872 std::fs::create_dir_all(&primary_objects).unwrap();
4873 let odb = Odb::new(&primary_objects).with_env_alternate_dirs(vec![alt_objects]);
4874 let present = odb.write(ObjectKind::Blob, b"present").unwrap();
4875 let absent = HashAlgo::Sha1
4876 .hash_object(ObjectKind::Blob, b"absent")
4877 .unwrap();
4878
4879 assert!(odb.contains(&present).unwrap());
4880 assert!(odb.contains(&odb.empty_tree_oid().unwrap()).unwrap());
4881 assert!(odb.contains(&alt_oid).unwrap());
4882 assert!(!odb.contains(&absent).unwrap());
4883 let empty_tree = odb.empty_tree_oid().unwrap();
4884 assert_eq!(
4885 odb.contains_many(&[present, absent, alt_oid, empty_tree])
4886 .unwrap(),
4887 vec![true, false, true, true]
4888 );
4889 assert!(odb.contains_many(&[]).unwrap().is_empty());
4890 assert_eq!(
4891 odb.present_oids([present, absent, alt_oid]).unwrap(),
4892 vec![present, alt_oid]
4893 );
4894 assert_eq!(
4895 odb.missing_oids([present, absent, alt_oid]).unwrap(),
4896 vec![absent]
4897 );
4898 assert_eq!(
4899 odb.missing_stored_oids([present, absent, alt_oid]).unwrap(),
4900 vec![absent, alt_oid],
4901 "stored ignores alternates"
4902 );
4903 }
4904
4905 #[test]
4906 fn missing_stored_oids_sees_packs_written_since_the_last_probe() {
4907 let dir = TempDir::new().unwrap();
4908 let odb = Odb::new(dir.path());
4909 let other = TempDir::new().unwrap();
4910 let source = Odb::new(other.path());
4911 let blobs: Vec<ObjectId> = (0..3)
4912 .map(|i| {
4913 source
4914 .write(ObjectKind::Blob, format!("b{i}").as_bytes())
4915 .unwrap()
4916 })
4917 .collect();
4918 assert_eq!(odb.missing_stored_oids(blobs.clone()).unwrap(), blobs);
4919 let pack = crate::pack_objects::build_pack_from_send_list(
4921 &source,
4922 &blobs[..2],
4923 &std::collections::HashSet::new(),
4924 &crate::pack_objects::PackBuildOptions::default(),
4925 &std::collections::HashMap::new(),
4926 )
4927 .unwrap();
4928 crate::index_pack::install_pack_bytes(pack, &Odb::new(dir.path()), &Default::default())
4929 .unwrap();
4930 assert_eq!(
4931 odb.missing_stored_oids(blobs.clone()).unwrap(),
4932 vec![blobs[2]]
4933 );
4934 }
4935
4936 #[test]
4937 fn oid_filters_propagate_backend_failure() {
4938 let dir = TempDir::new().unwrap();
4939 let (odb, flaky) = flaky_odb(dir.path());
4940 let oid = odb.write(ObjectKind::Blob, b"x").unwrap();
4941 assert_eq!(odb.contains_many(&[oid]).unwrap(), vec![true]);
4942 flaky.fail_contains(true);
4943 assert!(store::flaky::is_injected(
4944 &odb.contains_many(&[oid]).unwrap_err()
4945 ));
4946 assert!(store::flaky::is_injected(
4947 &odb.present_oids([oid]).unwrap_err()
4948 ));
4949 assert!(store::flaky::is_injected(
4950 &odb.missing_oids([oid]).unwrap_err()
4951 ));
4952 assert!(store::flaky::is_injected(
4953 &odb.missing_stored_oids([oid]).unwrap_err()
4954 ));
4955 }
4956
4957 #[test]
4958 fn write_existing_loose_uses_path_stat_not_exists() {
4959 let dir = TempDir::new().unwrap();
4960 let odb = Odb::new(dir.path());
4961 let oid = odb.write(ObjectKind::Blob, b"once").unwrap();
4962 let path = odb.object_path(&oid);
4963 assert!(path.is_file());
4964 odb.reset_exists_probe();
4965 let oid2 = odb.write(ObjectKind::Blob, b"once").unwrap();
4966 assert_eq!(oid, oid2);
4967 assert_eq!(
4968 odb.exists_probe_count(),
4969 0,
4970 "loose fast path must not call exists()"
4971 );
4972 }
4973
4974 #[cfg_attr(
4975 not(feature = "full-tests"),
4976 ignore = "starts system git or is heavy; run with --features full-tests"
4977 )]
4978 #[test]
4979 fn exists_and_write_after_invalidate_packs_use_odb_store() {
4980 let dir = TempDir::new().unwrap();
4981 git_in(dir.path(), &["init", "-q"]);
4982 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
4983 git_in(dir.path(), &["config", "user.name", "T"]);
4984 let payload = b"invalidate-pack-exists";
4985 fs::write(dir.path().join("f"), payload).unwrap();
4986 git_in(dir.path(), &["add", "f"]);
4987 git_in(dir.path(), &["commit", "-m", "c"]);
4988 git_in(dir.path(), &["repack", "-ad"]);
4989 let objects = dir.path().join(".git").join("objects");
4990 let odb = Odb::new(&objects);
4991 let oid = odb.hash(ObjectKind::Blob, payload).expect("hash");
4992 assert!(odb.exists(&oid), "blob must live in pack after repack");
4993 let pack_dir = objects.join("pack");
4994 for entry in fs::read_dir(&pack_dir).unwrap().flatten() {
4995 let path = entry.path();
4996 if path.extension().is_some_and(|e| e == "pack" || e == "idx") {
4997 fs::remove_file(path).unwrap();
4998 }
4999 }
5000 odb.invalidate_packs();
5001 assert!(
5002 !odb.exists(&oid),
5003 "exists must not read stale legacy pack listing after invalidate_packs"
5004 );
5005 let oid2 = odb.write(ObjectKind::Blob, payload).unwrap();
5006 assert_eq!(oid, oid2);
5007 assert!(
5008 odb.object_path(&oid).is_file(),
5009 "write must materialize loose object when pack copy is gone"
5010 );
5011 odb.read(&oid).expect("read after write");
5012 }
5013
5014 #[cfg_attr(
5015 not(feature = "full-tests"),
5016 ignore = "starts system git or is heavy; run with --features full-tests"
5017 )]
5018 #[test]
5019 fn write_uses_single_exists_probe_when_not_loose() {
5020 let dir = TempDir::new().unwrap();
5021 git_in(dir.path(), &["init", "-q"]);
5022 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5023 git_in(dir.path(), &["config", "user.name", "T"]);
5024 fs::write(dir.path().join("f"), b"probe-once").unwrap();
5025 git_in(dir.path(), &["add", "f"]);
5026 git_in(dir.path(), &["commit", "-m", "c"]);
5027 git_in(dir.path(), &["repack", "-ad"]);
5028 let objects = dir.path().join(".git").join("objects");
5029 let odb = Odb::new(&objects);
5030 let oid = odb.write(ObjectKind::Blob, b"probe-once").unwrap();
5031 assert!(!odb.object_path(&oid).exists());
5032 odb.reset_exists_probe();
5033 let oid2 = odb.write(ObjectKind::Blob, b"probe-once").unwrap();
5034 assert_eq!(oid, oid2);
5035 assert_eq!(
5036 odb.exists_probe_count(),
5037 1,
5038 "pack/alternate write path must call exists() exactly once"
5039 );
5040 }
5041 #[test]
5042 fn write_existing_object_freshens() {
5043 use filetime::FileTime;
5044 use std::fs;
5045 use std::thread;
5046 use std::time::Duration;
5047
5048 let dir = TempDir::new().unwrap();
5049 let odb = Odb::new(dir.path());
5050 let oid = odb.write(ObjectKind::Blob, b"hello").unwrap();
5051 let path = odb.object_path(&oid);
5052 thread::sleep(Duration::from_millis(50));
5053 let before = FileTime::from_last_modification_time(&fs::metadata(&path).unwrap());
5054 filetime::set_file_mtime(&path, before).unwrap();
5055
5056 let _ = odb.write(ObjectKind::Blob, b"hello").unwrap();
5057 let after = FileTime::from_last_modification_time(&fs::metadata(&path).unwrap());
5058 assert!(
5059 after > before,
5060 "rewriting an existing loose object must freshen its mtime"
5061 );
5062 }
5063
5064 #[test]
5065 fn freshen_pack_once_per_odb() {
5066 use filetime::FileTime;
5067 use flate2::write::ZlibEncoder;
5068 use flate2::Compression;
5069 use std::fs;
5070 use std::io::Write;
5071 use std::thread;
5072 use std::time::Duration;
5073
5074 fn append_pack_object(buf: &mut Vec<u8>, type_code: u8, data: &[u8]) {
5075 let mut size = data.len();
5076 let first = ((type_code & 0x7) << 4) | (size & 0x0f) as u8;
5077 size >>= 4;
5078 if size > 0 {
5079 buf.push(first | 0x80);
5080 while size > 0 {
5081 let b = (size & 0x7f) as u8;
5082 size >>= 7;
5083 buf.push(if size > 0 { b | 0x80 } else { b });
5084 }
5085 } else {
5086 buf.push(first);
5087 }
5088 let mut enc = ZlibEncoder::new(Vec::new(), Compression::default());
5089 enc.write_all(data).unwrap();
5090 buf.extend_from_slice(&enc.finish().unwrap());
5091 }
5092
5093 fn write_v2_idx(
5094 idx_path: &Path,
5095 pack_path: &Path,
5096 entries: &[(ObjectId, u64)],
5097 ) -> Result<()> {
5098 let mut sorted = entries.to_vec();
5099 sorted.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
5100 let n = sorted.len();
5101 let mut fanout = [0u32; 256];
5102 for byte in 0u32..256 {
5103 let count = sorted
5104 .iter()
5105 .filter(|(oid, _)| u32::from(oid.as_bytes()[0]) <= byte)
5106 .count();
5107 fanout[byte as usize] = u32::try_from(count).unwrap_or(u32::MAX);
5108 }
5109 let mut buf = Vec::new();
5110 buf.extend_from_slice(b"\xfftOc");
5111 buf.extend_from_slice(&2u32.to_be_bytes());
5112 for f in fanout {
5113 buf.extend_from_slice(&f.to_be_bytes());
5114 }
5115 for (oid, _) in &sorted {
5116 buf.extend_from_slice(oid.as_bytes());
5117 }
5118 for _ in 0..n {
5119 buf.extend_from_slice(&0u32.to_be_bytes());
5120 }
5121 for (_, off) in &sorted {
5122 buf.extend_from_slice(&u32::try_from(*off).unwrap_or(0x8000_0000).to_be_bytes());
5123 }
5124 let pack_bytes = fs::read(pack_path)?;
5125 buf.extend_from_slice(&pack_bytes[pack_bytes.len() - 20..]);
5126 buf.extend_from_slice(&HashAlgo::Sha1.checksum_digest(&buf));
5127 fs::write(idx_path, buf)?;
5128 Ok(())
5129 }
5130
5131 let dir = TempDir::new().unwrap();
5132 let objects_dir = dir.path();
5133 let odb = Odb::new(objects_dir);
5134
5135 let oid_a = odb.write(ObjectKind::Blob, b"alpha").unwrap();
5136 let oid_b = odb.write(ObjectKind::Blob, b"beta").unwrap();
5137 fs::remove_file(odb.object_path(&oid_a)).unwrap();
5138 fs::remove_file(odb.object_path(&oid_b)).unwrap();
5139
5140 let mut pack = Vec::new();
5141 pack.extend_from_slice(b"PACK");
5142 pack.extend_from_slice(&2u32.to_be_bytes());
5143 pack.extend_from_slice(&2u32.to_be_bytes());
5144 let off_a = pack.len() as u64;
5145 append_pack_object(&mut pack, 3, b"alpha");
5146 let off_b = pack.len() as u64;
5147 append_pack_object(&mut pack, 3, b"beta");
5148 pack.extend_from_slice(&HashAlgo::Sha1.checksum_digest(&pack));
5149
5150 let pack_dir = objects_dir.join("pack");
5151 fs::create_dir_all(&pack_dir).unwrap();
5152 let pack_path = pack_dir.join("test-177.pack");
5153 let idx_path = pack_dir.join("test-177.idx");
5154 fs::write(&pack_path, &pack).unwrap();
5155 write_v2_idx(&idx_path, &pack_path, &[(oid_a, off_a), (oid_b, off_b)]).unwrap();
5156 odb.invalidate_packs();
5157
5158 thread::sleep(Duration::from_millis(50));
5159 let before = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
5160
5161 assert!(odb.freshen_object(&oid_a));
5162 let after_first = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
5163 assert!(after_first > before);
5164
5165 assert!(odb.freshen_object(&oid_b));
5166 let after_second =
5167 FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
5168 assert_eq!(
5169 after_second, after_first,
5170 "second freshen of another object in the same pack must not touch the pack again"
5171 );
5172 }
5173
5174 #[test]
5175 fn write_existing_packed_object_freshens_pack() {
5176 use filetime::FileTime;
5177 use flate2::write::ZlibEncoder;
5178 use flate2::Compression;
5179 use std::fs;
5180 use std::io::Write;
5181 use std::thread;
5182 use std::time::Duration;
5183
5184 let dir = TempDir::new().unwrap();
5185 let objects_dir = dir.path();
5186 let odb = Odb::new(objects_dir);
5187 let oid = odb.write(ObjectKind::Blob, b"packed-only").unwrap();
5188 fs::remove_file(odb.object_path(&oid)).unwrap();
5189
5190 let mut pack = Vec::new();
5191 pack.extend_from_slice(b"PACK");
5192 pack.extend_from_slice(&2u32.to_be_bytes());
5193 pack.extend_from_slice(&1u32.to_be_bytes());
5194 let off = pack.len() as u64;
5195 let mut enc = ZlibEncoder::new(Vec::new(), Compression::default());
5196 enc.write_all(b"packed-only").unwrap();
5197 let compressed = enc.finish().unwrap();
5198 pack.push(0x30 | 0x03); pack.extend_from_slice(&compressed);
5200 pack.extend_from_slice(&HashAlgo::Sha1.checksum_digest(&pack));
5201
5202 let pack_dir = objects_dir.join("pack");
5203 fs::create_dir_all(&pack_dir).unwrap();
5204 let pack_path = pack_dir.join("single-177.pack");
5205 fs::write(&pack_path, &pack).unwrap();
5206 let idx_path = pack_dir.join("single-177.idx");
5208 let mut sorted = [(oid, off)];
5209 sorted.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
5210 let mut fanout = [0u32; 256];
5211 for byte in 0u32..256 {
5212 fanout[byte as usize] = if u32::from(oid.as_bytes()[0]) <= byte {
5213 1
5214 } else {
5215 0
5216 };
5217 }
5218 let mut buf = Vec::new();
5219 buf.extend_from_slice(b"\xfftOc");
5220 buf.extend_from_slice(&2u32.to_be_bytes());
5221 for f in fanout {
5222 buf.extend_from_slice(&f.to_be_bytes());
5223 }
5224 buf.extend_from_slice(oid.as_bytes());
5225 buf.extend_from_slice(&0u32.to_be_bytes());
5226 buf.extend_from_slice(&(off as u32).to_be_bytes());
5227 buf.extend_from_slice(&pack[pack.len() - 20..]);
5228 buf.extend_from_slice(&HashAlgo::Sha1.checksum_digest(&buf));
5229 fs::write(&idx_path, buf).unwrap();
5230 odb.invalidate_packs();
5231
5232 thread::sleep(Duration::from_millis(50));
5233 let before = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
5234
5235 odb.write(ObjectKind::Blob, b"packed-only").unwrap();
5236
5237 let after = FileTime::from_last_modification_time(&fs::metadata(&pack_path).unwrap());
5238 assert!(
5239 after > before,
5240 "write of an existing packed object must freshen the pack mtime"
5241 );
5242 }
5243
5244 #[cfg_attr(
5248 not(feature = "full-tests"),
5249 ignore = "starts system git or is heavy; run with --features full-tests"
5250 )]
5251 #[test]
5252 fn write_materializes_loose_when_pack_freshen_fails() {
5253 let dir = TempDir::new().unwrap();
5254 git_in(dir.path(), &["init", "-q"]);
5255 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5256 git_in(dir.path(), &["config", "user.name", "T"]);
5257 let payload = b"packed-fallback";
5258 fs::write(dir.path().join("f"), payload).unwrap();
5259 git_in(dir.path(), &["add", "f"]);
5260 git_in(dir.path(), &["commit", "-m", "c"]);
5261 git_in(dir.path(), &["repack", "-ad"]);
5262 let objects = dir.path().join(".git").join("objects");
5263 let odb = Odb::new(&objects);
5264 let oid = odb.write(ObjectKind::Blob, payload).unwrap();
5265 assert!(!odb.object_path(&oid).exists());
5266 assert!(odb.exists(&oid));
5267
5268 let pack_path = fs::read_dir(objects.join("pack"))
5269 .unwrap()
5270 .flatten()
5271 .find(|e| e.path().extension().is_some_and(|x| x == "pack"))
5272 .unwrap()
5273 .path();
5274 fs::remove_file(&pack_path).unwrap();
5275 fs::create_dir(&pack_path).unwrap();
5276 odb.invalidate_packs();
5277 assert!(!odb.freshen_object(&oid));
5278
5279 odb.write(ObjectKind::Blob, payload).unwrap();
5280 assert!(odb.object_path(&oid).exists());
5281 }
5282 #[test]
5283 fn write_with_options_silent_skips_freshen_on_existing_loose_object() {
5284 let dir = TempDir::new().unwrap();
5285 let odb = Odb::new(dir.path());
5286 let oid = odb.write(ObjectKind::Blob, b"stable").unwrap();
5287 let path = odb.object_path(&oid);
5288 let before = std::fs::metadata(&path).unwrap().modified().unwrap();
5289 std::thread::sleep(std::time::Duration::from_millis(20));
5290 let again = odb
5291 .write_with_options(
5292 ObjectKind::Blob,
5293 b"stable",
5294 WriteOptions {
5295 silent: true,
5296 ..WriteOptions::default()
5297 },
5298 )
5299 .unwrap();
5300 assert_eq!(oid, again);
5301 let after = std::fs::metadata(&path).unwrap().modified().unwrap();
5302 assert_eq!(before, after);
5303
5304 let touched = odb
5305 .write_with_options(
5306 ObjectKind::Blob,
5307 b"stable",
5308 WriteOptions {
5309 silent: false,
5310 ..WriteOptions::default()
5311 },
5312 )
5313 .unwrap();
5314 assert_eq!(oid, touched);
5315 let freshened = std::fs::metadata(&path).unwrap().modified().unwrap();
5316 assert!(freshened >= before);
5317 }
5318
5319 #[cfg_attr(
5320 not(feature = "full-tests"),
5321 ignore = "starts system git or is heavy; run with --features full-tests"
5322 )]
5323 #[test]
5324 fn promisor_marker_checked_once_per_prepare() {
5325 use crate::pack::{
5326 clear_pack_cache, pack_cache_test_guard, test_pack_marker_stat_count,
5327 test_reset_pack_marker_stat_count,
5328 };
5329
5330 let _guard = pack_cache_test_guard();
5331 let dir = TempDir::new().unwrap();
5332 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5333 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5334 git_in(dir.path(), &["config", "user.name", "T"]);
5335 for i in 0..64 {
5336 fs::write(dir.path().join(format!("f{i}.txt")), format!("body {i}")).unwrap();
5337 git_in(dir.path(), &["add", "."]);
5338 git_in(dir.path(), &["commit", "-m", &format!("c{i}")]);
5339 }
5340 git_in(dir.path(), &["repack", "-a", "-d"]);
5341 let objects = dir.path().join(".git/objects");
5342 let oid = git_rev_parse_oid(dir.path(), "HEAD");
5343
5344 clear_pack_cache();
5345 test_reset_pack_marker_stat_count();
5346 let pack_count = fs::read_dir(objects.join("pack"))
5347 .unwrap()
5348 .filter_map(|e| e.ok())
5349 .filter(|e| e.path().extension().is_some_and(|x| x == "pack"))
5350 .count();
5351 let expected_prepare_stats = u64::try_from(pack_count * 2).unwrap();
5352 let odb = Odb::new(&objects);
5353 let before_prepare = test_pack_marker_stat_count();
5354 assert!(odb.exists_local(&oid));
5355 assert_eq!(
5356 test_pack_marker_stat_count().saturating_sub(before_prepare),
5357 expected_prepare_stats,
5358 "prepare should stat promisor and mtimes once per pack"
5359 );
5360
5361 test_reset_pack_marker_stat_count();
5362 for _ in 0..10_000 {
5363 let before_probe = test_pack_marker_stat_count();
5364 assert!(odb.exists_local(&oid));
5365 assert_eq!(
5366 test_pack_marker_stat_count(),
5367 before_probe,
5368 "cached sidecar flags must not stat markers on each exists_local probe"
5369 );
5370 }
5371 }
5372
5373 #[cfg_attr(
5374 not(feature = "full-tests"),
5375 ignore = "starts system git or is heavy; run with --features full-tests"
5376 )]
5377 #[test]
5378 fn promisor_marker_added_after_prepare_is_seen_after_reprepare() {
5379 use crate::pack::{
5380 clear_pack_cache, pack_cache_test_guard, reprepare_pack_directory_on_miss,
5381 };
5382
5383 let _guard = pack_cache_test_guard();
5384 let dir = TempDir::new().unwrap();
5385 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5386 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5387 git_in(dir.path(), &["config", "user.name", "T"]);
5388 fs::write(dir.path().join("one.txt"), b"x").unwrap();
5389 git_in(dir.path(), &["add", "one.txt"]);
5390 git_in(dir.path(), &["commit", "-m", "one"]);
5391 git_in(dir.path(), &["repack", "-a", "-d"]);
5392 let objects = dir.path().join(".git/objects");
5393 let oid = git_rev_parse_oid(dir.path(), "HEAD");
5394
5395 clear_pack_cache();
5396 let odb = Odb::new(&objects);
5397 assert!(odb.exists_local(&oid));
5398
5399 let pack_path = fs::read_dir(objects.join("pack"))
5400 .unwrap()
5401 .filter_map(|e| e.ok())
5402 .map(|e| e.path())
5403 .find(|p| p.extension().is_some_and(|x| x == "pack"))
5404 .unwrap();
5405 fs::write(pack_path.with_extension("promisor"), b"").unwrap();
5406 filetime::set_file_mtime(objects.join("pack"), filetime::FileTime::now()).unwrap();
5407
5408 assert!(
5409 odb.with_pack_store_for(&objects, || {
5410 reprepare_pack_directory_on_miss(&objects).unwrap()
5411 }),
5412 "pack directory change must require reprepare"
5413 );
5414 assert!(
5415 !odb.exists_local(&oid),
5416 "promisor marker must hide pack objects after reprepare refreshed sidecar flags"
5417 );
5418 }
5419
5420 #[cfg_attr(
5421 not(feature = "full-tests"),
5422 ignore = "starts system git or is heavy; run with --features full-tests"
5423 )]
5424 #[test]
5425 fn git_promisor_marker_pack_not_materialized_for_exists_local() {
5426 use crate::pack::clear_pack_cache;
5427
5428 let dir = TempDir::new().unwrap();
5429 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5430 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5431 git_in(dir.path(), &["config", "user.name", "T"]);
5432 fs::write(dir.path().join("blob.txt"), b"git promisor compat").unwrap();
5433 git_in(dir.path(), &["add", "blob.txt"]);
5434 git_in(dir.path(), &["commit", "-m", "seed"]);
5435 git_in(dir.path(), &["repack", "-a", "-d"]);
5436 let objects = dir.path().join(".git/objects");
5437 let oid = git_rev_parse_oid(dir.path(), "HEAD");
5438 let pack_path = fs::read_dir(objects.join("pack"))
5439 .unwrap()
5440 .filter_map(|e| e.ok())
5441 .map(|e| e.path())
5442 .find(|p| p.extension().is_some_and(|x| x == "pack"))
5443 .unwrap();
5444 fs::write(pack_path.with_extension("promisor"), b"").unwrap();
5445
5446 clear_pack_cache();
5447 let odb = Odb::new(&objects);
5448 assert!(
5449 !odb.exists_local(&oid),
5450 "git-written pack with .promisor must not count as local materialization"
5451 );
5452
5453 let cat = git_run(dir.path(), &["cat-file", "-e", &oid.to_hex()]);
5454 assert!(
5455 cat.status.success(),
5456 "git still resolves the object in a promisor-marked pack"
5457 );
5458 }
5459
5460 #[cfg_attr(
5461 not(feature = "full-tests"),
5462 ignore = "starts system git or is heavy; run with --features full-tests"
5463 )]
5464 #[test]
5465 fn packed_object_read_avoids_loose_path_open() {
5466 use crate::hot_path_test_metrics::HotPathMetricsScope;
5467 use crate::pack::clear_pack_cache;
5468
5469 let dir = TempDir::new().unwrap();
5470 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5471 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5472 git_in(dir.path(), &["config", "user.name", "T"]);
5473 fs::write(dir.path().join("packed.txt"), b"packed-only payload").unwrap();
5474 git_in(dir.path(), &["add", "packed.txt"]);
5475 git_in(dir.path(), &["commit", "-m", "pack"]);
5476 git_in(dir.path(), &["repack", "-a", "-d"]);
5477 let objects = dir.path().join(".git/objects");
5478 let oid = git_rev_parse_oid(dir.path(), "HEAD");
5479
5480 clear_pack_cache();
5481 let odb = Odb::new(&objects);
5482 odb.hot_path_metrics().set_loose_open_counting(true);
5483 odb.hot_path_metrics().reset_loose_path_open_attempts();
5484 let _scope = HotPathMetricsScope::install(odb.hot_path_metrics_arc());
5485
5486 let obj = odb.read(&oid).expect("packed read");
5487 assert_eq!(obj.kind, ObjectKind::Commit);
5488 assert_eq!(
5489 odb.hot_path_metrics().loose_path_open_attempts(),
5490 0,
5491 "packed objects must not attempt a loose-path open"
5492 );
5493 }
5494
5495 #[test]
5496 fn loose_only_read_records_loose_path_open() {
5497 use crate::hot_path_test_metrics::HotPathMetricsScope;
5498
5499 let dir = TempDir::new().unwrap();
5500 let objects = dir.path().join("objects");
5501 fs::create_dir_all(&objects).unwrap();
5502 let odb = Odb::new(&objects);
5503 let oid = odb
5504 .write(ObjectKind::Blob, b"loose-only")
5505 .expect("write loose");
5506 odb.hot_path_metrics().set_loose_open_counting(true);
5507 odb.hot_path_metrics().reset_loose_path_open_attempts();
5508 let _scope = HotPathMetricsScope::install(odb.hot_path_metrics_arc());
5509
5510 odb.read(&oid).expect("loose read");
5511 assert_eq!(
5512 odb.hot_path_metrics().loose_path_open_attempts(),
5513 1,
5514 "loose fallback must record a path open attempt"
5515 );
5516 }
5517
5518 #[cfg_attr(
5519 not(feature = "full-tests"),
5520 ignore = "starts system git or is heavy; run with --features full-tests"
5521 )]
5522 #[test]
5523 fn loose_and_packed_copies_return_identical_bytes() {
5524 use crate::pack::clear_pack_cache;
5525
5526 let dir = TempDir::new().unwrap();
5527 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5528 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5529 git_in(dir.path(), &["config", "user.name", "T"]);
5530 fs::write(dir.path().join("dup.txt"), b"duplicate loose and packed\n").unwrap();
5531 git_in(dir.path(), &["add", "dup.txt"]);
5532 git_in(dir.path(), &["commit", "-m", "seed"]);
5533 let objects = dir.path().join(".git/objects");
5534 let tree_oid = git_rev_parse_oid(dir.path(), "HEAD^{tree}");
5535 git_in(dir.path(), &["repack", "-a"]);
5536 clear_pack_cache();
5537 let odb = Odb::new(&objects);
5538 let loose_path = tree_oid.loose_path_in(&objects);
5539 assert!(
5540 loose_path.is_file(),
5541 "loose copy must remain after repack -a"
5542 );
5543 let loose = Odb::read_loose_verify_oid(&loose_path, &tree_oid).expect("loose tree");
5544 let packed = odb.read(&tree_oid).expect("packed tree");
5545 assert_eq!(packed.data, loose.data);
5546 assert_eq!(packed.kind, loose.kind);
5547 }
5548
5549 #[cfg_attr(
5550 not(feature = "full-tests"),
5551 ignore = "starts system git or is heavy; run with --features full-tests"
5552 )]
5553 #[test]
5554 fn freshly_written_loose_object_visible_before_repack() {
5555 let dir = TempDir::new().unwrap();
5556 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5557 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5558 git_in(dir.path(), &["config", "user.name", "T"]);
5559 fs::write(dir.path().join("seed.txt"), b"seed").unwrap();
5560 git_in(dir.path(), &["add", "seed.txt"]);
5561 git_in(dir.path(), &["commit", "-m", "seed"]);
5562 git_in(dir.path(), &["repack", "-a", "-d"]);
5563 let objects = dir.path().join(".git/objects");
5564 let odb = Odb::new(&objects);
5565 let fresh = b"brand-new loose blob\n";
5566 let oid = odb.write(ObjectKind::Blob, fresh).expect("loose write");
5567 let obj = odb.read(&oid).expect("immediate loose read");
5568 assert_eq!(obj.data.as_slice(), fresh);
5569 }
5570
5571 #[cfg_attr(
5572 not(feature = "full-tests"),
5573 ignore = "starts system git or is heavy; run with --features full-tests"
5574 )]
5575 #[test]
5576 fn pack_added_after_first_read_found_via_odb_reprepare() {
5577 use crate::pack::{clear_pack_cache, pack_cache_test_guard};
5578
5579 let _guard = pack_cache_test_guard();
5580 clear_pack_cache();
5581 let dir = TempDir::new().unwrap();
5582 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5583 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5584 git_in(dir.path(), &["config", "user.name", "T"]);
5585 fs::write(dir.path().join("a.txt"), b"pack-a").unwrap();
5586 git_in(dir.path(), &["add", "a.txt"]);
5587 git_in(dir.path(), &["commit", "-m", "a"]);
5588 git_in(dir.path(), &["repack", "-a", "-d"]);
5589 let objects = dir.path().join(".git/objects");
5590 let pack_dir = objects.join("pack");
5591 let git_dir = dir.path().join(".git");
5592
5593 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
5594 let warm_oid = git_rev_parse_oid(dir.path(), "HEAD");
5595 odb.read(&warm_oid).expect("warm read");
5596
5597 fs::write(dir.path().join("b.txt"), b"pack-b").unwrap();
5598 git_in(dir.path(), &["add", "b.txt"]);
5599 git_in(dir.path(), &["commit", "-m", "b"]);
5600 let new_oid = git_rev_parse_oid(dir.path(), "HEAD");
5601
5602 std::thread::sleep(std::time::Duration::from_secs(1));
5603 filetime::set_file_mtime(&pack_dir, filetime::FileTime::now()).unwrap();
5604
5605 let obj = odb.read(&new_oid).expect("new pack via reprepare");
5606 assert_eq!(obj.kind, ObjectKind::Commit);
5607 }
5608
5609 #[cfg_attr(
5610 not(feature = "full-tests"),
5611 ignore = "starts system git or is heavy; run with --features full-tests"
5612 )]
5613 #[test]
5614 fn late_midx_after_warm_read_resolves_new_commit() {
5615 use crate::pack::{clear_pack_cache, pack_cache_test_guard};
5616
5617 let _guard = pack_cache_test_guard();
5618 clear_pack_cache();
5619 let dir = TempDir::new().unwrap();
5620 git_in(dir.path(), &["init", "-q", "-b", "main"]);
5621 git_in(dir.path(), &["config", "user.email", "t@example.com"]);
5622 git_in(dir.path(), &["config", "user.name", "T"]);
5623 fs::write(dir.path().join("a.txt"), b"pack-a").unwrap();
5624 git_in(dir.path(), &["add", "a.txt"]);
5625 git_in(dir.path(), &["commit", "-m", "a"]);
5626 git_in(dir.path(), &["repack", "-a", "-d"]);
5627 let objects = dir.path().join(".git/objects");
5628 let git_dir = dir.path().join(".git");
5629 assert!(
5630 crate::midx::cached_tip_midx_path(&objects.join("pack")).is_none(),
5631 "fixture starts without a MIDX"
5632 );
5633
5634 let odb = Odb::new(&objects).with_config_git_dir(git_dir);
5635 let warm_oid = git_rev_parse_oid(dir.path(), "HEAD");
5636 odb.read(&warm_oid).expect("warm read before MIDX exists");
5637
5638 fs::write(dir.path().join("b.txt"), b"pack-b").unwrap();
5639 git_in(dir.path(), &["add", "b.txt"]);
5640 git_in(dir.path(), &["commit", "-m", "b"]);
5641 git_in(dir.path(), &["repack", "-a", "-d"]);
5642 git_in(dir.path(), &["multi-pack-index", "write"]);
5643 let new_oid = git_rev_parse_oid(dir.path(), "HEAD");
5644
5645 let cat = git_run(dir.path(), &["cat-file", "-e", &new_oid.to_hex()]);
5646 assert!(
5647 cat.status.success(),
5648 "system git must resolve the new commit"
5649 );
5650
5651 let obj = odb
5652 .read(&new_oid)
5653 .expect("MIDX-covered commit after late MIDX write");
5654 assert_eq!(obj.kind, ObjectKind::Commit);
5655 assert!(
5656 crate::midx::cached_tip_midx_path(&objects.join("pack")).is_some(),
5657 "successful read after late MIDX must refresh the tip cache"
5658 );
5659 }
5660}