1use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque};
4use std::fmt;
5use std::sync::Arc;
6
7use crate::bitmap_walk::{BitmapWalkError, ReachabilityQuery, ReachableSet};
8use crate::environment::RepositoryOptions;
9use crate::error::Result;
10use crate::objects::{
11 oid_hash_map, oid_hash_set, parse_tag, CommitRef, Object, ObjectId, ObjectKind, OidHashMap,
12 OidHashSet, TreeRefIter,
13};
14use crate::odb::Odb;
15use crate::pack_bitmap::BitmapIndex;
16use crate::pack_name_hash::pack_name_hash;
17use crate::repo::Repository;
18use crate::rev_list::{MissingAction, ObjectFilter};
19
20#[derive(Clone)]
22#[allow(clippy::type_complexity)]
23pub struct PackObjectSelection {
24 pub send: Vec<ObjectId>,
26 pub have_closure: HashSet<ObjectId>,
28 pub peer_held: Option<Arc<ReachableSet>>,
30 pub name_hashes: HashMap<ObjectId, u32>,
34}
35
36impl fmt::Debug for PackObjectSelection {
37 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
38 f.debug_struct("PackObjectSelection")
39 .field("send", &self.send)
40 .field("have_closure", &self.have_closure)
41 .field(
42 "peer_held",
43 &self
44 .peer_held
45 .as_ref()
46 .map(|held| held.count())
47 .unwrap_or(0),
48 )
49 .field("name_hashes", &self.name_hashes)
50 .finish()
51 }
52}
53
54#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
56pub enum HaveBoundary {
57 #[default]
59 CommitClosure,
60 TipsOnly,
64}
65
66#[derive(Clone, Copy, Debug)]
68pub struct PackEnumerateOptions<'a> {
69 pub filter: Option<&'a ObjectFilter>,
71 pub shallow_grafts: &'a HashSet<ObjectId>,
75 pub have_shallow_grafts: Option<&'a HashSet<ObjectId>>,
78 pub use_bitmaps: bool,
80 pub exclude_objects: Option<&'a HashSet<ObjectId>>,
82 pub peer_odb: Option<&'a Odb>,
84 pub have_boundary: HaveBoundary,
86}
87
88pub fn enumerate_pack_objects(
94 odb: &Odb,
95 wants: &[ObjectId],
96 haves: &[ObjectId],
97 opts: &PackEnumerateOptions<'_>,
98) -> Result<PackObjectSelection> {
99 enumerate_pack_objects_naming(odb, wants, haves, opts, NameTrees::No)
100}
101
102#[derive(Clone, Copy, PartialEq, Eq)]
104pub(crate) enum NameTrees {
105 No,
107 Yes,
109}
110
111impl NameTrees {
112 pub(crate) fn for_delta_search(opts: &crate::pack_objects::PackBuildOptions) -> Self {
114 if opts.delta && opts.window > 0 && opts.max_depth > 0 {
115 Self::Yes
116 } else {
117 Self::No
118 }
119 }
120}
121
122pub(crate) fn enumerate_pack_objects_naming(
124 odb: &Odb,
125 wants: &[ObjectId],
126 haves: &[ObjectId],
127 opts: &PackEnumerateOptions<'_>,
128 name_trees: NameTrees,
129) -> Result<PackObjectSelection> {
130 if opts.use_bitmaps {
131 if let Some(sel) = try_bitmap_enumeration(odb, wants, haves, opts)? {
132 return Ok(sel);
133 }
134 }
135 walk_enumeration(odb, wants, haves, opts, name_trees)
136}
137
138pub fn try_bitmap_reachable_for_upload(
142 odb: &Odb,
143 wants: &[ObjectId],
144 haves: &[ObjectId],
145 filter: Option<&ObjectFilter>,
146 shallow_grafts: &HashSet<ObjectId>,
147) -> Result<Option<(ReachableSet, ReachableSet)>> {
148 if !shallow_grafts.is_empty() || filter.is_some_and(selects_object_type) {
149 return Ok(None);
150 }
151 let Some(git_dir) = odb.config_git_dir() else {
152 return Ok(None);
153 };
154 let repo = match Repository::open(git_dir, &RepositoryOptions::empty()) {
155 Ok(r) => r,
156 Err(_) => return Ok(None),
157 };
158 let Some(index) = BitmapIndex::open(&repo).ok().flatten() else {
159 return Ok(None);
160 };
161 let index = Arc::new(index);
162 let missing_wants = MissingAction::Error;
163
164 let Some(have_set) = try_bitmap_peer_held(odb, haves, shallow_grafts)? else {
165 return Ok(None);
166 };
167
168 let send_set = match index.reachability(
169 &repo,
170 ReachabilityQuery {
171 wants,
172 haves,
173 filter,
174 },
175 missing_wants,
176 ) {
177 Ok(set) => set,
178 Err(BitmapWalkError::Unsupported(_)) => return Ok(None),
179 Err(BitmapWalkError::Bitmap(_)) => return Ok(None),
180 Err(BitmapWalkError::Repository(e)) => return Err(e),
181 };
182
183 Ok(Some((send_set, have_set)))
184}
185
186fn selects_object_type(filter: &ObjectFilter) -> bool {
189 match filter {
190 ObjectFilter::ObjectType(_) => true,
191 ObjectFilter::Combine(parts) => parts.iter().any(selects_object_type),
192 _ => false,
193 }
194}
195
196fn try_bitmap_enumeration(
197 odb: &Odb,
198 wants: &[ObjectId],
199 haves: &[ObjectId],
200 opts: &PackEnumerateOptions<'_>,
201) -> Result<Option<PackObjectSelection>> {
202 let Some((send_set, peer_held)) =
203 try_bitmap_reachable_for_upload(odb, wants, haves, opts.filter, opts.shallow_grafts)?
204 else {
205 return Ok(None);
206 };
207 let index = send_set.bitmap_index();
208 let send: Vec<ObjectId> = send_set.iter_grouped_by_kind().collect();
209 let have_closure: HashSet<ObjectId> = haves.iter().copied().collect();
210 let name_hashes = name_hashes_from_bitmap(index, &send, &HashSet::new());
211 Ok(Some(PackObjectSelection {
212 send,
213 have_closure,
214 peer_held: Some(Arc::new(peer_held)),
215 name_hashes,
216 }))
217}
218
219pub(crate) fn name_hashes_for_send(
220 index: &BitmapIndex,
221 send: &[ObjectId],
222 have_closure: &HashSet<ObjectId>,
223) -> HashMap<ObjectId, u32> {
224 name_hashes_from_bitmap(index, send, have_closure)
225}
226
227fn name_hashes_from_bitmap(
228 index: &BitmapIndex,
229 send: &[ObjectId],
230 have_closure: &HashSet<ObjectId>,
231) -> HashMap<ObjectId, u32> {
232 let mut out = HashMap::new();
233 for oid in send.iter().chain(have_closure.iter()) {
234 let Some(pos) = index.position_of(oid) else {
235 continue;
236 };
237 if let Some(h) = index.name_hash(pos) {
238 out.insert(*oid, h);
239 }
240 }
241 out
242}
243
244fn walk_enumeration(
245 odb: &Odb,
246 wants: &[ObjectId],
247 haves: &[ObjectId],
248 opts: &PackEnumerateOptions<'_>,
249 name_trees: NameTrees,
250) -> Result<PackObjectSelection> {
251 let have_shallow = opts.have_shallow_grafts.unwrap_or(opts.shallow_grafts);
252 let have_commits = match opts.have_boundary {
253 HaveBoundary::CommitClosure => {
254 commits_reachable_from_haves(odb, haves, have_shallow, true)?
255 }
256 HaveBoundary::TipsOnly => send_pack_uninteresting_commits(odb, wants, haves, have_shallow)?,
257 };
258 let mut boundary: HashSet<ObjectId> = HashSet::new();
259 if let Some(extra) = opts.exclude_objects {
260 boundary.extend(extra.iter().copied());
261 }
262 let have_closure: HashSet<ObjectId> = haves.iter().copied().collect();
263 let peer_held = try_peer_held_bitmap(odb, haves, opts)?;
264 let (send, name_hashes) = match opts.filter {
265 Some(filter) if filter.uses_paths() => {
266 rev_list_filtered_objects(odb, wants, haves, filter, &have_commits)?
267 }
268 filter => collect_reachable_excluding_with_name_hashes(
269 odb,
270 wants,
271 &HashSet::new(),
272 &mut boundary,
273 &have_commits,
274 opts.peer_odb,
275 false,
276 opts.shallow_grafts,
277 filter,
278 name_trees,
279 )?,
280 };
281 Ok(PackObjectSelection {
282 send,
283 have_closure,
284 peer_held,
285 name_hashes,
286 })
287}
288
289const SEND_PACK_REACH_WANT: u8 = 1;
290const SEND_PACK_REACH_HAVE: u8 = 2;
291
292#[cfg(test)]
293pub(crate) mod send_pack_walk_test_hooks {
294 use std::cell::Cell;
295
296 thread_local! {
297 static COMMIT_READS: Cell<usize> = const { Cell::new(0) };
298 }
299
300 pub fn reset_commit_reads() {
301 COMMIT_READS.with(|c| c.set(0));
302 }
303
304 pub fn commit_reads() -> usize {
305 COMMIT_READS.with(|c| c.get())
306 }
307
308 pub(crate) fn record_commit_read() {
309 COMMIT_READS.with(|c| c.set(c.get().saturating_add(1)));
310 }
311}
312
313fn send_pack_reach_uninteresting(reach: u8) -> bool {
314 reach & SEND_PACK_REACH_HAVE != 0
315}
316
317fn send_pack_reach_interesting(reach: u8) -> bool {
318 reach & SEND_PACK_REACH_WANT != 0 && !send_pack_reach_uninteresting(reach)
319}
320
321fn send_pack_uninteresting_commits(
327 odb: &Odb,
328 wants: &[ObjectId],
329 haves: &[ObjectId],
330 shallow_grafts: &HashSet<ObjectId>,
331) -> Result<HashSet<ObjectId>> {
332 struct SendPackRevWalk<'a> {
333 odb: &'a Odb,
334 shallow_grafts: &'a HashSet<ObjectId>,
335 reach: HashMap<ObjectId, u8>,
336 heap: BinaryHeap<QueuedCommit>,
337 on_heap: HashSet<ObjectId>,
338 processed: HashSet<ObjectId>,
339 uninteresting: HashSet<ObjectId>,
340 next_seq: u64,
341 }
342
343 impl<'a> SendPackRevWalk<'a> {
344 fn note_reach(&mut self, oid: ObjectId, flag: u8) {
345 let entry = self.reach.entry(oid).or_insert(0);
346 *entry |= flag;
347 if send_pack_reach_uninteresting(*entry) {
348 self.uninteresting.insert(oid);
349 }
350 }
351
352 fn can_stop_after(&self) -> bool {
357 !self.heap.is_empty()
358 && self.heap.iter().all(|c| {
359 let flags = *self.reach.get(&c.oid).unwrap_or(&0);
360 !send_pack_reach_interesting(flags)
361 })
362 }
363
364 fn try_enqueue(&mut self, oid: ObjectId, flag: u8) -> Result<()> {
365 self.note_reach(oid, flag);
366 if self.processed.contains(&oid)
367 || self.on_heap.contains(&oid)
368 || self.shallow_grafts.contains(&oid)
369 {
370 return Ok(());
371 }
372 let obj = match self.odb.read(&oid) {
373 Ok(o) => o,
374 Err(_) => return Ok(()),
375 };
376 if obj.kind != ObjectKind::Commit {
377 return Ok(());
378 }
379 let parsed = CommitRef::parse(&obj.data)?;
380 #[cfg(test)]
381 send_pack_walk_test_hooks::record_commit_read();
382 self.on_heap.insert(oid);
383 self.heap.push(QueuedCommit {
384 time: parsed.committer_unix_seconds_for_ordering(),
385 seq: std::cmp::Reverse(self.next_seq),
386 oid,
387 tree: parsed.tree(),
388 parents: parsed.parents().to_vec(),
389 });
390 self.next_seq += 1;
391 Ok(())
392 }
393 }
394
395 let mut walk = SendPackRevWalk {
396 odb,
397 shallow_grafts,
398 reach: HashMap::new(),
399 heap: BinaryHeap::new(),
400 on_heap: HashSet::new(),
401 processed: HashSet::new(),
402 uninteresting: HashSet::new(),
403 next_seq: 0,
404 };
405
406 for &want in wants {
407 if let Ok(Some(commit)) = odb.peel_to_commit(want) {
408 walk.try_enqueue(commit, SEND_PACK_REACH_WANT)?;
409 }
410 }
411 for &have in haves {
412 if odb.read_info(&have).is_err() {
413 continue;
414 }
415 if let Ok(Some(commit)) = odb.peel_to_commit(have) {
416 walk.try_enqueue(commit, SEND_PACK_REACH_HAVE)?;
417 }
418 }
419
420 while let Some(commit) = walk.heap.pop() {
421 walk.on_heap.remove(&commit.oid);
422 if !walk.processed.insert(commit.oid) {
423 continue;
424 }
425 let flags = *walk.reach.get(&commit.oid).unwrap_or(&0);
426 if send_pack_reach_uninteresting(flags) {
427 walk.uninteresting.insert(commit.oid);
428 }
429 let parent_flag = if send_pack_reach_uninteresting(flags) {
430 SEND_PACK_REACH_HAVE
431 } else {
432 SEND_PACK_REACH_WANT
433 };
434 if !walk.shallow_grafts.contains(&commit.oid) {
435 for parent in commit.parents {
436 walk.try_enqueue(parent, parent_flag)?;
437 }
438 }
439 if walk.can_stop_after() {
440 break;
441 }
442 }
443 Ok(walk.uninteresting)
444}
445
446fn commits_reachable_from_haves(
448 odb: &Odb,
449 haves: &[ObjectId],
450 shallow_grafts: &HashSet<ObjectId>,
451 skip_missing: bool,
452) -> Result<HashSet<ObjectId>> {
453 let mut seen = HashSet::new();
454 let mut queue: VecDeque<ObjectId> = haves.iter().copied().collect();
455 while let Some(oid) = queue.pop_front() {
456 if !seen.insert(oid) {
457 continue;
458 }
459 if shallow_grafts.contains(&oid) {
460 continue;
461 }
462 let obj = match odb.read(&oid) {
463 Ok(o) => o,
464 Err(_) if skip_missing => continue,
465 Err(e) => return Err(e),
466 };
467 if obj.kind != ObjectKind::Commit {
468 continue;
469 }
470 let parsed = CommitRef::parse(&obj.data)?;
471 for parent in parsed.parents() {
472 if !seen.contains(parent) {
473 queue.push_back(*parent);
474 }
475 }
476 }
477 Ok(seen)
478}
479
480fn objects_in_commit_snapshot(odb: &Odb, commit_oid: ObjectId) -> Result<HashSet<ObjectId>> {
482 let mut out = HashSet::new();
483 out.insert(commit_oid);
484 let commit_obj = match odb.read(&commit_oid) {
485 Ok(o) => o,
486 Err(_) => return Ok(out),
487 };
488 if commit_obj.kind != ObjectKind::Commit {
489 return Ok(out);
490 }
491 let commit = CommitRef::parse(&commit_obj.data)?;
492 let oid_len = odb.hash_algo().len();
493 let mut queue = VecDeque::from([commit.tree()]);
494 while let Some(tree_oid) = queue.pop_front() {
495 if !out.insert(tree_oid) {
496 continue;
497 }
498 let tree_obj = match odb.read(&tree_oid) {
499 Ok(o) => o,
500 Err(_) => continue,
501 };
502 for entry in TreeRefIter::with_oid_len(&tree_obj.data, oid_len) {
503 let entry = entry?;
504 if entry.mode == 0o160000 {
505 continue;
506 }
507 if entry.mode & 0o170000 == 0o040000 {
508 queue.push_back(entry.oid);
509 } else {
510 out.insert(entry.oid);
511 }
512 }
513 }
514 Ok(out)
515}
516
517fn try_peer_held_bitmap(
518 odb: &Odb,
519 haves: &[ObjectId],
520 opts: &PackEnumerateOptions<'_>,
521) -> Result<Option<Arc<ReachableSet>>> {
522 if haves.is_empty() || !opts.use_bitmaps {
523 return Ok(None);
524 }
525 let Some(peer) = try_bitmap_peer_held(odb, haves, opts.shallow_grafts)? else {
526 return Ok(None);
527 };
528 Ok(Some(Arc::new(peer)))
529}
530
531fn try_bitmap_peer_held(
532 odb: &Odb,
533 haves: &[ObjectId],
534 shallow_grafts: &HashSet<ObjectId>,
535) -> Result<Option<ReachableSet>> {
536 if !shallow_grafts.is_empty() {
537 return Ok(None);
538 }
539 let Some(git_dir) = odb.config_git_dir() else {
540 return Ok(None);
541 };
542 let repo = match Repository::open(git_dir, &RepositoryOptions::empty()) {
543 Ok(r) => r,
544 Err(_) => return Ok(None),
545 };
546 let Some(index) = BitmapIndex::open(&repo).ok().flatten() else {
547 return Ok(None);
548 };
549 let index = Arc::new(index);
550 match index.reachability(
551 &repo,
552 ReachabilityQuery {
553 wants: haves,
554 haves: &[],
555 filter: None,
556 },
557 MissingAction::Allow,
558 ) {
559 Ok(set) => Ok(Some(set)),
560 Err(BitmapWalkError::Unsupported(_)) => Ok(None),
561 Err(BitmapWalkError::Bitmap(_)) => Ok(None),
562 Err(BitmapWalkError::Repository(e)) => Err(e),
563 }
564}
565
566fn rev_list_filtered_objects(
568 odb: &Odb,
569 wants: &[ObjectId],
570 haves: &[ObjectId],
571 filter: &ObjectFilter,
572 have_commits: &HashSet<ObjectId>,
573) -> Result<(Vec<ObjectId>, HashMap<ObjectId, u32>)> {
574 let _ = have_commits;
575 let Some(git_dir) = odb.config_git_dir() else {
576 return Ok((Vec::new(), HashMap::new()));
577 };
578 let repo = Repository::open(git_dir, &RepositoryOptions::empty())?;
579 let wants: Vec<String> = wants.iter().map(ObjectId::to_hex).collect();
580 let haves: Vec<String> = haves.iter().map(ObjectId::to_hex).collect();
581 let opts = crate::rev_list::RevListOptions {
582 objects: true,
583 no_object_names: true,
584 quiet: true,
585 filter: Some(filter.clone()),
586 ..Default::default()
587 };
588 let result = crate::rev_list::rev_list(&repo, &wants, &haves, &opts)?;
589 let mut ordered = Vec::new();
590 let mut seen = HashSet::new();
591 for oid in result
592 .commits
593 .iter()
594 .chain(result.objects.iter().map(|(o, _)| o))
595 {
596 if seen.insert(*oid) {
597 ordered.push(*oid);
598 }
599 }
600 Ok((ordered, HashMap::new()))
601}
602
603#[allow(clippy::type_complexity, clippy::too_many_arguments)]
621fn collect_reachable_excluding_with_name_hashes(
622 odb: &Odb,
623 roots: &[ObjectId],
624 exclude: &HashSet<ObjectId>,
625 boundary: &mut HashSet<ObjectId>,
626 have_commits: &HashSet<ObjectId>,
627 peer_odb: Option<&Odb>,
628 skip_missing: bool,
629 shallow_grafts: &HashSet<ObjectId>,
630 filter: Option<&ObjectFilter>,
631 name_trees: NameTrees,
632) -> Result<(Vec<ObjectId>, HashMap<ObjectId, u32>)> {
633 let oid_len = odb.hash_algo().len();
634 let mut walk = EnumerationWalk {
635 odb,
636 exclude,
637 boundary,
638 have_commits,
639 peer_odb,
640 skip_missing,
641 filter,
642 depth_sensitive: filter.is_some_and(ObjectFilter::is_depth_sensitive),
643 name_trees,
644 visited: oid_hash_set(),
645 tree_depths: oid_hash_map(),
646 unlisted: oid_hash_set(),
647 ordered: Vec::new(),
648 commits: BinaryHeap::new(),
649 next_seq: 0,
650 queue: VecDeque::new(),
651 name_hashes: HashMap::new(),
652 };
653 let mut dropped_root_trees: Option<OidHashSet> = filter
655 .filter(|f| !f.includes_tree(0))
656 .map(|_| oid_hash_set());
657
658 let mut path_less_roots = Vec::new();
659 for &root in roots {
660 match odb.read_info(&root) {
661 Ok(info) if matches!(info.kind, ObjectKind::Tree | ObjectKind::Blob) => {
662 path_less_roots.push(root);
663 }
664 _ => walk.push_root(root)?,
665 }
666 }
667 let mut path_less_roots = path_less_roots.into_iter();
668
669 loop {
670 if let Some(commit) = walk.commits.pop() {
671 if !shallow_grafts.contains(&commit.oid) {
672 for parent in commit.parents {
673 walk.push_parent(parent)?;
674 }
675 }
676 if let Some(dropped) = dropped_root_trees.as_mut() {
677 dropped.insert(commit.tree);
678 }
679 walk.push_tree(commit.tree, String::new(), 0);
680 continue;
681 }
682 let Some((oid, path, depth)) = walk.queue.pop_front() else {
683 match path_less_roots.next() {
684 Some(root) => {
685 walk.push_object(root, String::new());
686 continue;
687 }
688 None => break,
689 }
690 };
691 let obj = match odb.read(&oid) {
692 Ok(o) => o,
693 Err(_) if skip_missing => continue,
694 Err(e) => return Err(e),
695 };
696 match obj.kind {
697 ObjectKind::Commit => {
698 walk.queue_commit(oid, obj)?;
699 continue;
700 }
701 ObjectKind::Tree => {
702 for entry in TreeRefIter::with_oid_len(&obj.data, oid_len) {
703 let entry = entry?;
704 if entry.mode == 0o160000 {
705 continue;
706 }
707 match entry.mode & 0o170000 {
714 0o120000 => walk.push_blob(entry.oid, None, depth)?,
715 0o100000 => {
716 walk.push_blob(entry.oid, Some((&path, entry.name)), depth)?;
717 }
718 file_type => {
719 let child_path = child_path(&path, entry.name);
720 if file_type == 0o040000 {
721 walk.push_tree(entry.oid, child_path, depth + 1);
722 } else {
723 walk.push_object(entry.oid, child_path);
724 }
725 }
726 }
727 }
728 }
729 ObjectKind::Tag => {
730 let tag = parse_tag(&obj.data)?;
731 walk.push_object(tag.object, path);
732 }
733 ObjectKind::Blob => {}
734 }
735 }
736
737 if let Some(dropped) = dropped_root_trees.filter(|d| !d.is_empty()) {
738 walk.ordered.retain(|oid| !dropped.contains(oid));
739 }
740 Ok((walk.ordered, walk.name_hashes))
741}
742
743fn child_path(parent: &str, name: &[u8]) -> String {
744 let name = String::from_utf8_lossy(name);
745 if parent.is_empty() {
746 name.into_owned()
747 } else {
748 format!("{parent}/{name}")
749 }
750}
751
752struct EnumerationWalk<'a> {
754 odb: &'a Odb,
755 exclude: &'a HashSet<ObjectId>,
756 boundary: &'a mut HashSet<ObjectId>,
758 have_commits: &'a HashSet<ObjectId>,
760 peer_odb: Option<&'a Odb>,
762 skip_missing: bool,
763 filter: Option<&'a ObjectFilter>,
764 depth_sensitive: bool,
766 name_trees: NameTrees,
767 visited: OidHashSet,
771 tree_depths: OidHashMap<u64>,
773 unlisted: OidHashSet,
776 ordered: Vec<ObjectId>,
777 commits: BinaryHeap<QueuedCommit>,
779 next_seq: u64,
780 queue: VecDeque<(ObjectId, String, u64)>,
783 name_hashes: HashMap<ObjectId, u32>,
784}
785
786struct QueuedCommit {
789 time: i64,
790 seq: std::cmp::Reverse<u64>,
791 oid: ObjectId,
792 tree: ObjectId,
793 parents: Vec<ObjectId>,
794}
795
796impl PartialEq for QueuedCommit {
797 fn eq(&self, other: &Self) -> bool {
798 (self.time, self.seq) == (other.time, other.seq)
799 }
800}
801
802impl Eq for QueuedCommit {}
803
804impl PartialOrd for QueuedCommit {
805 fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
806 Some(self.cmp(other))
807 }
808}
809
810impl Ord for QueuedCommit {
811 fn cmp(&self, other: &Self) -> std::cmp::Ordering {
812 (self.time, self.seq).cmp(&(other.time, other.seq))
813 }
814}
815
816impl<'a> EnumerationWalk<'a> {
817 fn list(&mut self, oid: ObjectId) {
818 if !self.exclude.contains(&oid) {
819 self.ordered.push(oid);
820 }
821 }
822
823 fn note_boundary(&mut self, oid: ObjectId) {
824 self.visited.insert(oid);
825 }
826
827 fn at_object_boundary(&self, oid: ObjectId) -> bool {
828 self.boundary.contains(&oid)
829 || self
830 .peer_odb
831 .is_some_and(|peer| peer.exists_materialized_local_without_rescan(&oid))
832 }
833
834 fn seal_have_reachable_commit(&mut self, oid: ObjectId) -> Result<()> {
835 if !self.visited.insert(oid) {
836 return Ok(());
837 }
838 self.boundary
839 .extend(objects_in_commit_snapshot(self.odb, oid)?);
840 Ok(())
841 }
842
843 fn commit_on_have_side(&mut self, oid: ObjectId) -> Result<bool> {
844 Ok(self.have_commits.contains(&oid))
845 }
846
847 fn push_object(&mut self, oid: ObjectId, path: String) {
849 if let Ok(true) = self.commit_on_have_side(oid) {
850 let _ = self.seal_have_reachable_commit(oid);
851 return;
852 }
853 if self.at_object_boundary(oid) {
854 self.note_boundary(oid);
855 return;
856 }
857 if self.visited.insert(oid) {
858 self.list(oid);
859 self.queue.push_back((oid, path, 0));
860 } else if self.unlisted.remove(&oid) {
861 self.list(oid);
862 }
863 }
864
865 fn push_root(&mut self, oid: ObjectId) -> Result<()> {
868 if self.commit_on_have_side(oid)? {
869 return self.seal_have_reachable_commit(oid);
870 }
871 if self.at_object_boundary(oid) {
872 self.note_boundary(oid);
873 return Ok(());
874 }
875 if !self.visited.insert(oid) {
876 if self.unlisted.remove(&oid) {
877 self.list(oid);
878 }
879 return Ok(());
880 }
881 self.list(oid);
882 match self.read(&oid)? {
883 Some(obj) if obj.kind == ObjectKind::Commit => self.queue_commit(oid, obj),
884 Some(_) => {
885 self.queue.push_back((oid, String::new(), 0));
886 Ok(())
887 }
888 None => Ok(()),
889 }
890 }
891
892 fn push_parent(&mut self, oid: ObjectId) -> Result<()> {
895 if self.commit_on_have_side(oid)? {
896 return self.seal_have_reachable_commit(oid);
897 }
898 if self.at_object_boundary(oid) {
899 self.note_boundary(oid);
900 return Ok(());
901 }
902 let listed = self
903 .filter
904 .is_none_or(|f| f.includes_commit_or_tag_object(ObjectKind::Commit));
905 if !self.visited.insert(oid) {
906 if listed && self.unlisted.remove(&oid) {
907 self.list(oid);
908 }
909 return Ok(());
910 }
911 if listed {
912 self.list(oid);
913 } else {
914 self.unlisted.insert(oid);
915 }
916 match self.read(&oid)? {
917 Some(obj) if obj.kind == ObjectKind::Commit => self.queue_commit(oid, obj),
918 Some(_) => {
919 self.queue.push_back((oid, String::new(), 0));
920 Ok(())
921 }
922 None => Ok(()),
923 }
924 }
925
926 fn read(&self, oid: &ObjectId) -> Result<Option<Object>> {
927 match self.odb.read(oid) {
928 Ok(obj) => Ok(Some(obj)),
929 Err(_) if self.skip_missing => Ok(None),
930 Err(e) => Err(e),
931 }
932 }
933
934 fn queue_commit(&mut self, oid: ObjectId, obj: Object) -> Result<()> {
936 let parsed = CommitRef::parse(&obj.data)?;
937 let queued = QueuedCommit {
938 time: parsed.committer_unix_seconds_for_ordering(),
939 seq: std::cmp::Reverse(self.next_seq),
940 oid,
941 tree: parsed.tree(),
942 parents: parsed.parents().to_vec(),
943 };
944 self.next_seq += 1;
945 self.commits.push(queued);
946 Ok(())
947 }
948
949 fn push_tree(&mut self, oid: ObjectId, path: String, depth: u64) {
951 if self.name_trees == NameTrees::Yes && !path.is_empty() && !self.visited.contains(&oid) {
952 self.name_hashes
953 .entry(oid)
954 .or_insert_with(|| pack_name_hash(&path));
955 }
956 let Some(filter) = self.filter else {
957 self.push_object(oid, path);
958 return;
959 };
960 let first_walk = if self.depth_sensitive {
961 match self.tree_depths.get(&oid) {
962 Some(&walked_at) if walked_at <= depth => false,
963 _ => {
964 self.tree_depths.insert(oid, depth);
965 true
966 }
967 }
968 } else {
969 self.visited.insert(oid)
970 };
971 if !first_walk {
972 return;
973 }
974 if filter.includes_tree(depth) {
975 if !self.depth_sensitive || self.visited.insert(oid) {
976 self.list(oid);
977 }
978 } else if !self.depth_sensitive {
979 self.unlisted.insert(oid);
980 }
981 if !filter.prunes_tree_entries(depth) {
982 self.queue.push_back((oid, path, depth));
983 }
984 }
985
986 fn push_blob(
989 &mut self,
990 oid: ObjectId,
991 name: Option<(&str, &[u8])>,
992 tree_depth: u64,
993 ) -> Result<()> {
994 if self.at_object_boundary(oid) {
995 self.note_boundary(oid);
996 return Ok(());
997 }
998 if self.visited.contains(&oid) {
999 return Ok(());
1000 }
1001 let (passes, size_checked) = match self.filter {
1002 None => (true, false),
1003 Some(filter) => self.blob_passes(filter, &oid, tree_depth)?,
1004 };
1005 if passes {
1006 self.visited.insert(oid);
1007 self.list(oid);
1008 if let Some((parent, entry_name)) = name {
1009 self.name_hashes
1010 .insert(oid, pack_name_hash(&child_path(parent, entry_name)));
1011 }
1012 } else if size_checked && !self.depth_sensitive {
1013 self.visited.insert(oid);
1014 self.unlisted.insert(oid);
1015 }
1016 Ok(())
1017 }
1018
1019 fn blob_passes(
1021 &self,
1022 filter: &ObjectFilter,
1023 oid: &ObjectId,
1024 tree_depth: u64,
1025 ) -> Result<(bool, bool)> {
1026 if !filter.includes_blob_under_tree(0, tree_depth) {
1028 return Ok((false, false));
1029 }
1030 if filter.includes_blob_under_tree(u64::MAX, tree_depth) {
1031 return Ok((true, false));
1032 }
1033 match self.odb.read_info(oid) {
1034 Ok(info) => Ok((filter.includes_blob_under_tree(info.size, tree_depth), true)),
1035 Err(_) if self.skip_missing => Ok((false, true)),
1036 Err(e) => Err(e),
1037 }
1038 }
1039}
1040
1041#[cfg(test)]
1042mod tests {
1043 #![allow(clippy::unwrap_used, clippy::expect_used)]
1044
1045 use super::*;
1046
1047 use crate::repo::{InitOptions, Repository};
1048 use tempfile::TempDir;
1049
1050 fn walk(repo: &Repository, roots: &[ObjectId], name_trees: NameTrees) -> PackObjectSelection {
1051 let empty = HashSet::new();
1052 enumerate_pack_objects_naming(
1053 repo.odb(),
1054 roots,
1055 &[],
1056 &PackEnumerateOptions {
1057 filter: None,
1058 shallow_grafts: &empty,
1059 have_shallow_grafts: Some(&empty),
1060 use_bitmaps: false,
1061 exclude_objects: None,
1062 peer_odb: None,
1063 have_boundary: HaveBoundary::CommitClosure,
1064 },
1065 name_trees,
1066 )
1067 .expect("walk")
1068 }
1069
1070 fn write_tree(repo: &Repository, entries: &[(u32, &str, ObjectId)]) -> ObjectId {
1071 let entries: Vec<crate::objects::TreeEntry> = entries
1072 .iter()
1073 .map(|&(mode, name, oid)| crate::objects::TreeEntry {
1074 mode,
1075 name: name.as_bytes().to_vec(),
1076 oid,
1077 })
1078 .collect();
1079 repo.odb()
1080 .write(ObjectKind::Tree, &crate::objects::serialize_tree(&entries))
1081 .expect("tree")
1082 }
1083
1084 fn write_commit(
1085 repo: &Repository,
1086 tree: ObjectId,
1087 parents: &[ObjectId],
1088 time: i64,
1089 ) -> ObjectId {
1090 let mut body = format!("tree {}\n", tree.to_hex());
1091 for parent in parents {
1092 body.push_str(&format!("parent {}\n", parent.to_hex()));
1093 }
1094 body.push_str(&format!(
1095 "author A <a@example.com> {time} +0000\ncommitter A <a@example.com> {time} +0000\n\nc{time}\n"
1096 ));
1097 repo.odb()
1098 .write(ObjectKind::Commit, body.as_bytes())
1099 .expect("commit")
1100 }
1101
1102 #[test]
1103 fn tips_only_boundary_matches_closure_on_linear_incremental_push() {
1104 let dir = TempDir::new().unwrap();
1105 let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
1106 let blob = |s: &str| repo.odb().write(ObjectKind::Blob, s.as_bytes()).unwrap();
1107 let mut parent = write_commit(
1108 &repo,
1109 write_tree(&repo, &[(0o100644, "f", blob("0"))]),
1110 &[],
1111 0,
1112 );
1113 for i in 1..800 {
1114 let tree = write_tree(&repo, &[(0o100644, "f", blob(&i.to_string()))]);
1115 parent = write_commit(&repo, tree, &[parent], i);
1116 }
1117 let remote_tip = parent;
1118 let new_tree = write_tree(&repo, &[(0o100644, "f", blob("new"))]);
1119 let new_tip = write_commit(&repo, new_tree, &[remote_tip], 900);
1120 let empty = HashSet::new();
1121 let closure_opts = PackEnumerateOptions {
1122 filter: None,
1123 shallow_grafts: &empty,
1124 have_shallow_grafts: Some(&empty),
1125 use_bitmaps: false,
1126 exclude_objects: None,
1127 peer_odb: None,
1128 have_boundary: HaveBoundary::CommitClosure,
1129 };
1130 let tips_opts = PackEnumerateOptions {
1131 have_boundary: HaveBoundary::TipsOnly,
1132 ..closure_opts
1133 };
1134 let closure = enumerate_pack_objects(repo.odb(), &[new_tip], &[remote_tip], &closure_opts)
1135 .expect("closure");
1136 let tips = enumerate_pack_objects(repo.odb(), &[new_tip], &[remote_tip], &tips_opts)
1137 .expect("tips");
1138 let closure_set: HashSet<ObjectId> = closure.send.iter().copied().collect();
1139 let tips_set: HashSet<ObjectId> = tips.send.iter().copied().collect();
1140 assert_eq!(closure_set, tips_set);
1141 assert!(
1142 tips.send.len() <= 8,
1143 "one new commit should enumerate a handful of objects, got {}",
1144 tips.send.len()
1145 );
1146 }
1147
1148 #[test]
1149 fn tips_only_send_pack_does_not_prewalk_deep_history() {
1150 use crate::merge_base::intersect_walk_test_hooks::{
1151 commit_parent_lookups, reset_commit_parent_lookups,
1152 };
1153
1154 let dir = TempDir::new().unwrap();
1155 let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
1156 let blob = |s: &str| repo.odb().write(ObjectKind::Blob, s.as_bytes()).unwrap();
1157 let mut parent = write_commit(
1158 &repo,
1159 write_tree(&repo, &[(0o100644, "f", blob("0"))]),
1160 &[],
1161 0,
1162 );
1163 for i in 1..800 {
1164 let tree = write_tree(&repo, &[(0o100644, "f", blob(&i.to_string()))]);
1165 parent = write_commit(&repo, tree, &[parent], i);
1166 }
1167 let remote_tip = parent;
1168 let new_tree = write_tree(&repo, &[(0o100644, "f", blob("new"))]);
1169 let new_tip = write_commit(&repo, new_tree, &[remote_tip], 900);
1170 let haves = vec![remote_tip];
1171 let empty = HashSet::new();
1172 reset_commit_parent_lookups();
1173 let _selection = enumerate_pack_objects(
1174 repo.odb(),
1175 &[new_tip],
1176 &haves,
1177 &PackEnumerateOptions {
1178 filter: None,
1179 shallow_grafts: &empty,
1180 have_shallow_grafts: Some(&empty),
1181 use_bitmaps: false,
1182 exclude_objects: None,
1183 peer_odb: None,
1184 have_boundary: HaveBoundary::TipsOnly,
1185 },
1186 )
1187 .expect("enumerate");
1188 assert!(
1189 commit_parent_lookups() <= 128,
1190 "send-pack hide must stay coupled to the want walk ({} parent lookups)",
1191 commit_parent_lookups()
1192 );
1193 }
1194
1195 #[test]
1196 fn tips_only_many_advertised_haves_does_not_prewalk_deep_history() {
1197 use crate::merge_base::intersect_walk_test_hooks::{
1198 commit_parent_lookups, reset_commit_parent_lookups,
1199 };
1200
1201 let dir = TempDir::new().unwrap();
1202 let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
1203 let blob = |s: &str| repo.odb().write(ObjectKind::Blob, s.as_bytes()).unwrap();
1204 let mut chain = Vec::new();
1205 let mut parent = write_commit(
1206 &repo,
1207 write_tree(&repo, &[(0o100644, "f", blob("0"))]),
1208 &[],
1209 0,
1210 );
1211 chain.push(parent);
1212 for i in 1..800 {
1213 let tree = write_tree(&repo, &[(0o100644, "f", blob(&i.to_string()))]);
1214 parent = write_commit(&repo, tree, &[parent], i);
1215 chain.push(parent);
1216 }
1217 let remote_tip = parent;
1218 let new_tree = write_tree(&repo, &[(0o100644, "f", blob("new"))]);
1219 let new_tip = write_commit(&repo, new_tree, &[remote_tip], 900);
1220 let mut haves = vec![remote_tip];
1221 for n in (2..400).step_by(2) {
1222 let idx = chain.len().checked_sub(1 + n);
1223 if let Some(idx) = idx {
1224 haves.push(chain[idx]);
1225 }
1226 }
1227 let empty = HashSet::new();
1228 reset_commit_parent_lookups();
1229 let selection = enumerate_pack_objects(
1230 repo.odb(),
1231 &[new_tip],
1232 &haves,
1233 &PackEnumerateOptions {
1234 filter: None,
1235 shallow_grafts: &empty,
1236 have_shallow_grafts: Some(&empty),
1237 use_bitmaps: false,
1238 exclude_objects: None,
1239 peer_odb: None,
1240 have_boundary: HaveBoundary::TipsOnly,
1241 },
1242 )
1243 .expect("enumerate");
1244 assert!(
1245 selection.send.len() <= 8,
1246 "one new commit should stay small with many haves, got {}",
1247 selection.send.len()
1248 );
1249 assert!(
1250 commit_parent_lookups() <= 512,
1251 "many advertised haves must not pre-walk deep history ({} parent lookups)",
1252 commit_parent_lookups()
1253 );
1254 }
1255
1256 #[test]
1257 fn send_pack_have_walk_stops_without_scanning_deep_haves() {
1258 use super::send_pack_walk_test_hooks::{commit_reads, reset_commit_reads};
1259
1260 let dir = TempDir::new().unwrap();
1261 let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
1262 let blob = |s: &str| repo.odb().write(ObjectKind::Blob, s.as_bytes()).unwrap();
1263 let mut parent = write_commit(
1264 &repo,
1265 write_tree(&repo, &[(0o100644, "f", blob("0"))]),
1266 &[],
1267 0,
1268 );
1269 let near_root = parent;
1270 for i in 1..800 {
1271 let tree = write_tree(&repo, &[(0o100644, "f", blob(&i.to_string()))]);
1272 parent = write_commit(&repo, tree, &[parent], i);
1273 }
1274 let direct_parent = parent;
1275 let new_tree = write_tree(&repo, &[(0o100644, "f", blob("new"))]);
1276 let new_tip = write_commit(&repo, new_tree, &[direct_parent], 900);
1277 let empty = HashSet::new();
1278 reset_commit_reads();
1279 let have_commits = send_pack_uninteresting_commits(
1280 repo.odb(),
1281 &[new_tip],
1282 &[direct_parent, near_root],
1283 &empty,
1284 )
1285 .expect("send-pack have walk");
1286 assert!(have_commits.contains(&direct_parent));
1287 assert!(have_commits.contains(&near_root));
1288 let reads = commit_reads();
1289 assert!(
1290 reads < 20,
1291 "send-pack have walk should not scan deep history, read {reads} commits"
1292 );
1293 }
1294
1295 #[test]
1296 fn walk_lists_trees_of_newer_commits_first() {
1297 let dir = TempDir::new().unwrap();
1298 let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
1299 let blob = |s: &str| repo.odb().write(ObjectKind::Blob, s.as_bytes()).unwrap();
1300 let tree_a = write_tree(&repo, &[(0o100644, "f", blob("a"))]);
1301 let tree_b = write_tree(&repo, &[(0o100644, "f", blob("b"))]);
1302 let tree_c = write_tree(&repo, &[(0o100644, "f", blob("c"))]);
1303 let base = write_commit(&repo, tree_a, &[], 100);
1304 let old_tip = write_commit(&repo, tree_b, &[base], 200);
1306 let new_tip = write_commit(&repo, tree_c, &[base], 300);
1307
1308 let selection = walk(&repo, &[old_tip, new_tip], NameTrees::No);
1309 let pos = |oid: ObjectId| selection.send.iter().position(|o| *o == oid).unwrap();
1310 assert!(pos(old_tip) < pos(base) && pos(new_tip) < pos(base));
1311 assert!(pos(tree_c) < pos(tree_b) && pos(tree_b) < pos(tree_a));
1312 }
1313
1314 #[test]
1315 fn walk_names_trees_and_keeps_history_paths_for_path_less_roots() {
1316 let dir = TempDir::new().unwrap();
1317 let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
1318 let file = repo.odb().write(ObjectKind::Blob, b"x\n").unwrap();
1319 let link = repo.odb().write(ObjectKind::Blob, b"target").unwrap();
1320 let sub = write_tree(
1321 &repo,
1322 &[(0o100644, "file.rs", file), (0o120000, "link", link)],
1323 );
1324 let root = write_tree(&repo, &[(0o040000, "src", sub)]);
1325 let commit = write_commit(&repo, root, &[], 100);
1326
1327 let selection = walk(&repo, &[sub, file, commit], NameTrees::Yes);
1329 let hash = |oid: ObjectId| selection.name_hashes.get(&oid).copied();
1330 assert_eq!(hash(sub), Some(pack_name_hash("src")));
1331 assert_eq!(hash(file), Some(pack_name_hash("src/file.rs")));
1332 assert_eq!(hash(root), None, "root trees have no path");
1333 assert_eq!(selection.send.len(), 5);
1334
1335 let blobs_only = walk(&repo, &[sub, file, commit], NameTrees::No);
1336 assert_eq!(blobs_only.name_hashes.get(&sub), None);
1337 assert_eq!(
1338 blobs_only.name_hashes.get(&file),
1339 Some(&pack_name_hash("src/file.rs"))
1340 );
1341 }
1342
1343 fn tree_entry(mode: &str, name: &str, oid: ObjectId) -> Vec<u8> {
1344 let mut bytes = format!("{mode} {name}\0").into_bytes();
1345 bytes.extend_from_slice(oid.as_bytes());
1346 bytes
1347 }
1348
1349 #[test]
1353 fn walk_descends_into_trees_with_non_canonical_modes() {
1354 let tmp = tempfile::tempdir().unwrap();
1355 Repository::init(tmp.path(), &InitOptions::new().with_bare(true)).unwrap();
1356 let repo = Repository::open(tmp.path(), &RepositoryOptions::empty()).unwrap();
1357 let odb = repo.odb();
1358 let deep = odb
1359 .write_loose_materialize(ObjectKind::Blob, b"deep\n")
1360 .unwrap();
1361 let odd = odb
1362 .write_loose_materialize(ObjectKind::Blob, b"odd\n")
1363 .unwrap();
1364 let inner = odb
1365 .write_loose_materialize(ObjectKind::Tree, &tree_entry("100644", "deep.txt", deep))
1366 .unwrap();
1367 let mut root = tree_entry("100664", "odd.txt", odd);
1368 root.extend(tree_entry("40755", "sub", inner));
1369 let root = odb
1370 .write_loose_materialize(ObjectKind::Tree, &root)
1371 .unwrap();
1372 let commit = format!(
1373 "tree {root}\nauthor T <t@e.com> 100 +0000\ncommitter T <t@e.com> 100 +0000\n\nodd modes\n"
1374 );
1375 let commit = odb
1376 .write_loose_materialize(ObjectKind::Commit, commit.as_bytes())
1377 .unwrap();
1378
1379 let empty = HashSet::new();
1380 let selection = enumerate_pack_objects(
1381 odb,
1382 &[commit],
1383 &[],
1384 &PackEnumerateOptions {
1385 filter: None,
1386 shallow_grafts: &empty,
1387 have_shallow_grafts: Some(&empty),
1388 use_bitmaps: false,
1389 exclude_objects: None,
1390 peer_odb: None,
1391 have_boundary: HaveBoundary::CommitClosure,
1392 },
1393 )
1394 .unwrap();
1395
1396 let ours: HashSet<ObjectId> = selection.send.iter().copied().collect();
1397 let git: HashSet<ObjectId> =
1398 grit_test_support::git(tmp.path(), &["rev-list", "--objects", &commit.to_hex()])
1399 .lines()
1400 .map(|line| ObjectId::from_hex(&line[..40]).unwrap())
1401 .collect();
1402 assert_eq!(ours, git, "same objects as git rev-list --objects");
1403 assert!(ours.contains(&inner) && ours.contains(&deep));
1404 assert_eq!(selection.send.len(), 5);
1405 assert_eq!(
1406 selection.name_hashes.get(&deep),
1407 Some(&pack_name_hash("sub/deep.txt"))
1408 );
1409 assert_eq!(
1410 selection.name_hashes.get(&odd),
1411 Some(&pack_name_hash("odd.txt"))
1412 );
1413 }
1414}