use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque};
use std::fmt;
use std::sync::Arc;
use crate::bitmap_walk::{BitmapWalkError, ReachabilityQuery, ReachableSet};
use crate::environment::RepositoryOptions;
use crate::error::Result;
use crate::objects::{
oid_hash_map, oid_hash_set, parse_tag, CommitRef, Object, ObjectId, ObjectKind, OidHashMap,
OidHashSet, TreeRefIter,
};
use crate::odb::Odb;
use crate::pack_bitmap::BitmapIndex;
use crate::pack_name_hash::pack_name_hash;
use crate::repo::Repository;
use crate::rev_list::{MissingAction, ObjectFilter};
#[derive(Clone)]
#[allow(clippy::type_complexity)]
pub struct PackObjectSelection {
pub send: Vec<ObjectId>,
pub have_closure: HashSet<ObjectId>,
pub peer_held: Option<Arc<ReachableSet>>,
pub name_hashes: HashMap<ObjectId, u32>,
}
impl fmt::Debug for PackObjectSelection {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("PackObjectSelection")
.field("send", &self.send)
.field("have_closure", &self.have_closure)
.field(
"peer_held",
&self
.peer_held
.as_ref()
.map(|held| held.count())
.unwrap_or(0),
)
.field("name_hashes", &self.name_hashes)
.finish()
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum HaveBoundary {
#[default]
CommitClosure,
TipsOnly,
}
#[derive(Clone, Copy, Debug)]
pub struct PackEnumerateOptions<'a> {
pub filter: Option<&'a ObjectFilter>,
pub shallow_grafts: &'a HashSet<ObjectId>,
pub have_shallow_grafts: Option<&'a HashSet<ObjectId>>,
pub use_bitmaps: bool,
pub exclude_objects: Option<&'a HashSet<ObjectId>>,
pub peer_odb: Option<&'a Odb>,
pub have_boundary: HaveBoundary,
}
pub fn enumerate_pack_objects(
odb: &Odb,
wants: &[ObjectId],
haves: &[ObjectId],
opts: &PackEnumerateOptions<'_>,
) -> Result<PackObjectSelection> {
enumerate_pack_objects_naming(odb, wants, haves, opts, NameTrees::No)
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum NameTrees {
No,
Yes,
}
impl NameTrees {
pub(crate) fn for_delta_search(opts: &crate::pack_objects::PackBuildOptions) -> Self {
if opts.delta && opts.window > 0 && opts.max_depth > 0 {
Self::Yes
} else {
Self::No
}
}
}
pub(crate) fn enumerate_pack_objects_naming(
odb: &Odb,
wants: &[ObjectId],
haves: &[ObjectId],
opts: &PackEnumerateOptions<'_>,
name_trees: NameTrees,
) -> Result<PackObjectSelection> {
if opts.use_bitmaps {
if let Some(sel) = try_bitmap_enumeration(odb, wants, haves, opts)? {
return Ok(sel);
}
}
walk_enumeration(odb, wants, haves, opts, name_trees)
}
pub fn try_bitmap_reachable_for_upload(
odb: &Odb,
wants: &[ObjectId],
haves: &[ObjectId],
filter: Option<&ObjectFilter>,
shallow_grafts: &HashSet<ObjectId>,
) -> Result<Option<(ReachableSet, ReachableSet)>> {
if !shallow_grafts.is_empty() || filter.is_some_and(selects_object_type) {
return Ok(None);
}
let Some(git_dir) = odb.config_git_dir() else {
return Ok(None);
};
let repo = match Repository::open(git_dir, &RepositoryOptions::empty()) {
Ok(r) => r,
Err(_) => return Ok(None),
};
let Some(index) = BitmapIndex::open(&repo).ok().flatten() else {
return Ok(None);
};
let index = Arc::new(index);
let missing_wants = MissingAction::Error;
let Some(have_set) = try_bitmap_peer_held(odb, haves, shallow_grafts)? else {
return Ok(None);
};
let send_set = match index.reachability(
&repo,
ReachabilityQuery {
wants,
haves,
filter,
},
missing_wants,
) {
Ok(set) => set,
Err(BitmapWalkError::Unsupported(_)) => return Ok(None),
Err(BitmapWalkError::Bitmap(_)) => return Ok(None),
Err(BitmapWalkError::Repository(e)) => return Err(e),
};
Ok(Some((send_set, have_set)))
}
fn selects_object_type(filter: &ObjectFilter) -> bool {
match filter {
ObjectFilter::ObjectType(_) => true,
ObjectFilter::Combine(parts) => parts.iter().any(selects_object_type),
_ => false,
}
}
fn try_bitmap_enumeration(
odb: &Odb,
wants: &[ObjectId],
haves: &[ObjectId],
opts: &PackEnumerateOptions<'_>,
) -> Result<Option<PackObjectSelection>> {
let Some((send_set, peer_held)) =
try_bitmap_reachable_for_upload(odb, wants, haves, opts.filter, opts.shallow_grafts)?
else {
return Ok(None);
};
let index = send_set.bitmap_index();
let send: Vec<ObjectId> = send_set.iter_grouped_by_kind().collect();
let have_closure: HashSet<ObjectId> = haves.iter().copied().collect();
let name_hashes = name_hashes_from_bitmap(index, &send, &HashSet::new());
Ok(Some(PackObjectSelection {
send,
have_closure,
peer_held: Some(Arc::new(peer_held)),
name_hashes,
}))
}
pub(crate) fn name_hashes_for_send(
index: &BitmapIndex,
send: &[ObjectId],
have_closure: &HashSet<ObjectId>,
) -> HashMap<ObjectId, u32> {
name_hashes_from_bitmap(index, send, have_closure)
}
fn name_hashes_from_bitmap(
index: &BitmapIndex,
send: &[ObjectId],
have_closure: &HashSet<ObjectId>,
) -> HashMap<ObjectId, u32> {
let mut out = HashMap::new();
for oid in send.iter().chain(have_closure.iter()) {
let Some(pos) = index.position_of(oid) else {
continue;
};
if let Some(h) = index.name_hash(pos) {
out.insert(*oid, h);
}
}
out
}
fn walk_enumeration(
odb: &Odb,
wants: &[ObjectId],
haves: &[ObjectId],
opts: &PackEnumerateOptions<'_>,
name_trees: NameTrees,
) -> Result<PackObjectSelection> {
let have_shallow = opts.have_shallow_grafts.unwrap_or(opts.shallow_grafts);
let have_commits = match opts.have_boundary {
HaveBoundary::CommitClosure => {
commits_reachable_from_haves(odb, haves, have_shallow, true)?
}
HaveBoundary::TipsOnly => send_pack_uninteresting_commits(odb, wants, haves, have_shallow)?,
};
let mut boundary: HashSet<ObjectId> = HashSet::new();
if let Some(extra) = opts.exclude_objects {
boundary.extend(extra.iter().copied());
}
let have_closure: HashSet<ObjectId> = haves.iter().copied().collect();
let peer_held = try_peer_held_bitmap(odb, haves, opts)?;
let (send, name_hashes) = match opts.filter {
Some(filter) if filter.uses_paths() => {
rev_list_filtered_objects(odb, wants, haves, filter, &have_commits)?
}
filter => collect_reachable_excluding_with_name_hashes(
odb,
wants,
&HashSet::new(),
&mut boundary,
&have_commits,
opts.peer_odb,
false,
opts.shallow_grafts,
filter,
name_trees,
)?,
};
Ok(PackObjectSelection {
send,
have_closure,
peer_held,
name_hashes,
})
}
const SEND_PACK_REACH_WANT: u8 = 1;
const SEND_PACK_REACH_HAVE: u8 = 2;
#[cfg(test)]
pub(crate) mod send_pack_walk_test_hooks {
use std::cell::Cell;
thread_local! {
static COMMIT_READS: Cell<usize> = const { Cell::new(0) };
}
pub fn reset_commit_reads() {
COMMIT_READS.with(|c| c.set(0));
}
pub fn commit_reads() -> usize {
COMMIT_READS.with(|c| c.get())
}
pub(crate) fn record_commit_read() {
COMMIT_READS.with(|c| c.set(c.get().saturating_add(1)));
}
}
fn send_pack_reach_uninteresting(reach: u8) -> bool {
reach & SEND_PACK_REACH_HAVE != 0
}
fn send_pack_reach_interesting(reach: u8) -> bool {
reach & SEND_PACK_REACH_WANT != 0 && !send_pack_reach_uninteresting(reach)
}
fn send_pack_uninteresting_commits(
odb: &Odb,
wants: &[ObjectId],
haves: &[ObjectId],
shallow_grafts: &HashSet<ObjectId>,
) -> Result<HashSet<ObjectId>> {
struct SendPackRevWalk<'a> {
odb: &'a Odb,
shallow_grafts: &'a HashSet<ObjectId>,
reach: HashMap<ObjectId, u8>,
heap: BinaryHeap<QueuedCommit>,
on_heap: HashSet<ObjectId>,
processed: HashSet<ObjectId>,
uninteresting: HashSet<ObjectId>,
next_seq: u64,
}
impl<'a> SendPackRevWalk<'a> {
fn note_reach(&mut self, oid: ObjectId, flag: u8) {
let entry = self.reach.entry(oid).or_insert(0);
*entry |= flag;
if send_pack_reach_uninteresting(*entry) {
self.uninteresting.insert(oid);
}
}
fn can_stop_after(&self) -> bool {
!self.heap.is_empty()
&& self.heap.iter().all(|c| {
let flags = *self.reach.get(&c.oid).unwrap_or(&0);
!send_pack_reach_interesting(flags)
})
}
fn try_enqueue(&mut self, oid: ObjectId, flag: u8) -> Result<()> {
self.note_reach(oid, flag);
if self.processed.contains(&oid)
|| self.on_heap.contains(&oid)
|| self.shallow_grafts.contains(&oid)
{
return Ok(());
}
let obj = match self.odb.read(&oid) {
Ok(o) => o,
Err(_) => return Ok(()),
};
if obj.kind != ObjectKind::Commit {
return Ok(());
}
let parsed = CommitRef::parse(&obj.data)?;
#[cfg(test)]
send_pack_walk_test_hooks::record_commit_read();
self.on_heap.insert(oid);
self.heap.push(QueuedCommit {
time: parsed.committer_unix_seconds_for_ordering(),
seq: std::cmp::Reverse(self.next_seq),
oid,
tree: parsed.tree(),
parents: parsed.parents().to_vec(),
});
self.next_seq += 1;
Ok(())
}
}
let mut walk = SendPackRevWalk {
odb,
shallow_grafts,
reach: HashMap::new(),
heap: BinaryHeap::new(),
on_heap: HashSet::new(),
processed: HashSet::new(),
uninteresting: HashSet::new(),
next_seq: 0,
};
for &want in wants {
if let Ok(Some(commit)) = odb.peel_to_commit(want) {
walk.try_enqueue(commit, SEND_PACK_REACH_WANT)?;
}
}
for &have in haves {
if odb.read_info(&have).is_err() {
continue;
}
if let Ok(Some(commit)) = odb.peel_to_commit(have) {
walk.try_enqueue(commit, SEND_PACK_REACH_HAVE)?;
}
}
while let Some(commit) = walk.heap.pop() {
walk.on_heap.remove(&commit.oid);
if !walk.processed.insert(commit.oid) {
continue;
}
let flags = *walk.reach.get(&commit.oid).unwrap_or(&0);
if send_pack_reach_uninteresting(flags) {
walk.uninteresting.insert(commit.oid);
}
let parent_flag = if send_pack_reach_uninteresting(flags) {
SEND_PACK_REACH_HAVE
} else {
SEND_PACK_REACH_WANT
};
if !walk.shallow_grafts.contains(&commit.oid) {
for parent in commit.parents {
walk.try_enqueue(parent, parent_flag)?;
}
}
if walk.can_stop_after() {
break;
}
}
Ok(walk.uninteresting)
}
fn commits_reachable_from_haves(
odb: &Odb,
haves: &[ObjectId],
shallow_grafts: &HashSet<ObjectId>,
skip_missing: bool,
) -> Result<HashSet<ObjectId>> {
let mut seen = HashSet::new();
let mut queue: VecDeque<ObjectId> = haves.iter().copied().collect();
while let Some(oid) = queue.pop_front() {
if !seen.insert(oid) {
continue;
}
if shallow_grafts.contains(&oid) {
continue;
}
let obj = match odb.read(&oid) {
Ok(o) => o,
Err(_) if skip_missing => continue,
Err(e) => return Err(e),
};
if obj.kind != ObjectKind::Commit {
continue;
}
let parsed = CommitRef::parse(&obj.data)?;
for parent in parsed.parents() {
if !seen.contains(parent) {
queue.push_back(*parent);
}
}
}
Ok(seen)
}
fn objects_in_commit_snapshot(odb: &Odb, commit_oid: ObjectId) -> Result<HashSet<ObjectId>> {
let mut out = HashSet::new();
out.insert(commit_oid);
let commit_obj = match odb.read(&commit_oid) {
Ok(o) => o,
Err(_) => return Ok(out),
};
if commit_obj.kind != ObjectKind::Commit {
return Ok(out);
}
let commit = CommitRef::parse(&commit_obj.data)?;
let oid_len = odb.hash_algo().len();
let mut queue = VecDeque::from([commit.tree()]);
while let Some(tree_oid) = queue.pop_front() {
if !out.insert(tree_oid) {
continue;
}
let tree_obj = match odb.read(&tree_oid) {
Ok(o) => o,
Err(_) => continue,
};
for entry in TreeRefIter::with_oid_len(&tree_obj.data, oid_len) {
let entry = entry?;
if entry.mode == 0o160000 {
continue;
}
if entry.mode & 0o170000 == 0o040000 {
queue.push_back(entry.oid);
} else {
out.insert(entry.oid);
}
}
}
Ok(out)
}
fn try_peer_held_bitmap(
odb: &Odb,
haves: &[ObjectId],
opts: &PackEnumerateOptions<'_>,
) -> Result<Option<Arc<ReachableSet>>> {
if haves.is_empty() || !opts.use_bitmaps {
return Ok(None);
}
let Some(peer) = try_bitmap_peer_held(odb, haves, opts.shallow_grafts)? else {
return Ok(None);
};
Ok(Some(Arc::new(peer)))
}
fn try_bitmap_peer_held(
odb: &Odb,
haves: &[ObjectId],
shallow_grafts: &HashSet<ObjectId>,
) -> Result<Option<ReachableSet>> {
if !shallow_grafts.is_empty() {
return Ok(None);
}
let Some(git_dir) = odb.config_git_dir() else {
return Ok(None);
};
let repo = match Repository::open(git_dir, &RepositoryOptions::empty()) {
Ok(r) => r,
Err(_) => return Ok(None),
};
let Some(index) = BitmapIndex::open(&repo).ok().flatten() else {
return Ok(None);
};
let index = Arc::new(index);
match index.reachability(
&repo,
ReachabilityQuery {
wants: haves,
haves: &[],
filter: None,
},
MissingAction::Allow,
) {
Ok(set) => Ok(Some(set)),
Err(BitmapWalkError::Unsupported(_)) => Ok(None),
Err(BitmapWalkError::Bitmap(_)) => Ok(None),
Err(BitmapWalkError::Repository(e)) => Err(e),
}
}
fn rev_list_filtered_objects(
odb: &Odb,
wants: &[ObjectId],
haves: &[ObjectId],
filter: &ObjectFilter,
have_commits: &HashSet<ObjectId>,
) -> Result<(Vec<ObjectId>, HashMap<ObjectId, u32>)> {
let _ = have_commits;
let Some(git_dir) = odb.config_git_dir() else {
return Ok((Vec::new(), HashMap::new()));
};
let repo = Repository::open(git_dir, &RepositoryOptions::empty())?;
let wants: Vec<String> = wants.iter().map(ObjectId::to_hex).collect();
let haves: Vec<String> = haves.iter().map(ObjectId::to_hex).collect();
let opts = crate::rev_list::RevListOptions {
objects: true,
no_object_names: true,
quiet: true,
filter: Some(filter.clone()),
..Default::default()
};
let result = crate::rev_list::rev_list(&repo, &wants, &haves, &opts)?;
let mut ordered = Vec::new();
let mut seen = HashSet::new();
for oid in result
.commits
.iter()
.chain(result.objects.iter().map(|(o, _)| o))
{
if seen.insert(*oid) {
ordered.push(*oid);
}
}
Ok((ordered, HashMap::new()))
}
#[allow(clippy::type_complexity, clippy::too_many_arguments)]
fn collect_reachable_excluding_with_name_hashes(
odb: &Odb,
roots: &[ObjectId],
exclude: &HashSet<ObjectId>,
boundary: &mut HashSet<ObjectId>,
have_commits: &HashSet<ObjectId>,
peer_odb: Option<&Odb>,
skip_missing: bool,
shallow_grafts: &HashSet<ObjectId>,
filter: Option<&ObjectFilter>,
name_trees: NameTrees,
) -> Result<(Vec<ObjectId>, HashMap<ObjectId, u32>)> {
let oid_len = odb.hash_algo().len();
let mut walk = EnumerationWalk {
odb,
exclude,
boundary,
have_commits,
peer_odb,
skip_missing,
filter,
depth_sensitive: filter.is_some_and(ObjectFilter::is_depth_sensitive),
name_trees,
visited: oid_hash_set(),
tree_depths: oid_hash_map(),
unlisted: oid_hash_set(),
ordered: Vec::new(),
commits: BinaryHeap::new(),
next_seq: 0,
queue: VecDeque::new(),
name_hashes: HashMap::new(),
};
let mut dropped_root_trees: Option<OidHashSet> = filter
.filter(|f| !f.includes_tree(0))
.map(|_| oid_hash_set());
let mut path_less_roots = Vec::new();
for &root in roots {
match odb.read_info(&root) {
Ok(info) if matches!(info.kind, ObjectKind::Tree | ObjectKind::Blob) => {
path_less_roots.push(root);
}
_ => walk.push_root(root)?,
}
}
let mut path_less_roots = path_less_roots.into_iter();
loop {
if let Some(commit) = walk.commits.pop() {
if !shallow_grafts.contains(&commit.oid) {
for parent in commit.parents {
walk.push_parent(parent)?;
}
}
if let Some(dropped) = dropped_root_trees.as_mut() {
dropped.insert(commit.tree);
}
walk.push_tree(commit.tree, String::new(), 0);
continue;
}
let Some((oid, path, depth)) = walk.queue.pop_front() else {
match path_less_roots.next() {
Some(root) => {
walk.push_object(root, String::new());
continue;
}
None => break,
}
};
let obj = match odb.read(&oid) {
Ok(o) => o,
Err(_) if skip_missing => continue,
Err(e) => return Err(e),
};
match obj.kind {
ObjectKind::Commit => {
walk.queue_commit(oid, obj)?;
continue;
}
ObjectKind::Tree => {
for entry in TreeRefIter::with_oid_len(&obj.data, oid_len) {
let entry = entry?;
if entry.mode == 0o160000 {
continue;
}
match entry.mode & 0o170000 {
0o120000 => walk.push_blob(entry.oid, None, depth)?,
0o100000 => {
walk.push_blob(entry.oid, Some((&path, entry.name)), depth)?;
}
file_type => {
let child_path = child_path(&path, entry.name);
if file_type == 0o040000 {
walk.push_tree(entry.oid, child_path, depth + 1);
} else {
walk.push_object(entry.oid, child_path);
}
}
}
}
}
ObjectKind::Tag => {
let tag = parse_tag(&obj.data)?;
walk.push_object(tag.object, path);
}
ObjectKind::Blob => {}
}
}
if let Some(dropped) = dropped_root_trees.filter(|d| !d.is_empty()) {
walk.ordered.retain(|oid| !dropped.contains(oid));
}
Ok((walk.ordered, walk.name_hashes))
}
fn child_path(parent: &str, name: &[u8]) -> String {
let name = String::from_utf8_lossy(name);
if parent.is_empty() {
name.into_owned()
} else {
format!("{parent}/{name}")
}
}
struct EnumerationWalk<'a> {
odb: &'a Odb,
exclude: &'a HashSet<ObjectId>,
boundary: &'a mut HashSet<ObjectId>,
have_commits: &'a HashSet<ObjectId>,
peer_odb: Option<&'a Odb>,
skip_missing: bool,
filter: Option<&'a ObjectFilter>,
depth_sensitive: bool,
name_trees: NameTrees,
visited: OidHashSet,
tree_depths: OidHashMap<u64>,
unlisted: OidHashSet,
ordered: Vec<ObjectId>,
commits: BinaryHeap<QueuedCommit>,
next_seq: u64,
queue: VecDeque<(ObjectId, String, u64)>,
name_hashes: HashMap<ObjectId, u32>,
}
struct QueuedCommit {
time: i64,
seq: std::cmp::Reverse<u64>,
oid: ObjectId,
tree: ObjectId,
parents: Vec<ObjectId>,
}
impl PartialEq for QueuedCommit {
fn eq(&self, other: &Self) -> bool {
(self.time, self.seq) == (other.time, other.seq)
}
}
impl Eq for QueuedCommit {}
impl PartialOrd for QueuedCommit {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for QueuedCommit {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
(self.time, self.seq).cmp(&(other.time, other.seq))
}
}
impl<'a> EnumerationWalk<'a> {
fn list(&mut self, oid: ObjectId) {
if !self.exclude.contains(&oid) {
self.ordered.push(oid);
}
}
fn note_boundary(&mut self, oid: ObjectId) {
self.visited.insert(oid);
}
fn at_object_boundary(&self, oid: ObjectId) -> bool {
self.boundary.contains(&oid)
|| self
.peer_odb
.is_some_and(|peer| peer.exists_materialized_local_without_rescan(&oid))
}
fn seal_have_reachable_commit(&mut self, oid: ObjectId) -> Result<()> {
if !self.visited.insert(oid) {
return Ok(());
}
self.boundary
.extend(objects_in_commit_snapshot(self.odb, oid)?);
Ok(())
}
fn commit_on_have_side(&mut self, oid: ObjectId) -> Result<bool> {
Ok(self.have_commits.contains(&oid))
}
fn push_object(&mut self, oid: ObjectId, path: String) {
if let Ok(true) = self.commit_on_have_side(oid) {
let _ = self.seal_have_reachable_commit(oid);
return;
}
if self.at_object_boundary(oid) {
self.note_boundary(oid);
return;
}
if self.visited.insert(oid) {
self.list(oid);
self.queue.push_back((oid, path, 0));
} else if self.unlisted.remove(&oid) {
self.list(oid);
}
}
fn push_root(&mut self, oid: ObjectId) -> Result<()> {
if self.commit_on_have_side(oid)? {
return self.seal_have_reachable_commit(oid);
}
if self.at_object_boundary(oid) {
self.note_boundary(oid);
return Ok(());
}
if !self.visited.insert(oid) {
if self.unlisted.remove(&oid) {
self.list(oid);
}
return Ok(());
}
self.list(oid);
match self.read(&oid)? {
Some(obj) if obj.kind == ObjectKind::Commit => self.queue_commit(oid, obj),
Some(_) => {
self.queue.push_back((oid, String::new(), 0));
Ok(())
}
None => Ok(()),
}
}
fn push_parent(&mut self, oid: ObjectId) -> Result<()> {
if self.commit_on_have_side(oid)? {
return self.seal_have_reachable_commit(oid);
}
if self.at_object_boundary(oid) {
self.note_boundary(oid);
return Ok(());
}
let listed = self
.filter
.is_none_or(|f| f.includes_commit_or_tag_object(ObjectKind::Commit));
if !self.visited.insert(oid) {
if listed && self.unlisted.remove(&oid) {
self.list(oid);
}
return Ok(());
}
if listed {
self.list(oid);
} else {
self.unlisted.insert(oid);
}
match self.read(&oid)? {
Some(obj) if obj.kind == ObjectKind::Commit => self.queue_commit(oid, obj),
Some(_) => {
self.queue.push_back((oid, String::new(), 0));
Ok(())
}
None => Ok(()),
}
}
fn read(&self, oid: &ObjectId) -> Result<Option<Object>> {
match self.odb.read(oid) {
Ok(obj) => Ok(Some(obj)),
Err(_) if self.skip_missing => Ok(None),
Err(e) => Err(e),
}
}
fn queue_commit(&mut self, oid: ObjectId, obj: Object) -> Result<()> {
let parsed = CommitRef::parse(&obj.data)?;
let queued = QueuedCommit {
time: parsed.committer_unix_seconds_for_ordering(),
seq: std::cmp::Reverse(self.next_seq),
oid,
tree: parsed.tree(),
parents: parsed.parents().to_vec(),
};
self.next_seq += 1;
self.commits.push(queued);
Ok(())
}
fn push_tree(&mut self, oid: ObjectId, path: String, depth: u64) {
if self.name_trees == NameTrees::Yes && !path.is_empty() && !self.visited.contains(&oid) {
self.name_hashes
.entry(oid)
.or_insert_with(|| pack_name_hash(&path));
}
let Some(filter) = self.filter else {
self.push_object(oid, path);
return;
};
let first_walk = if self.depth_sensitive {
match self.tree_depths.get(&oid) {
Some(&walked_at) if walked_at <= depth => false,
_ => {
self.tree_depths.insert(oid, depth);
true
}
}
} else {
self.visited.insert(oid)
};
if !first_walk {
return;
}
if filter.includes_tree(depth) {
if !self.depth_sensitive || self.visited.insert(oid) {
self.list(oid);
}
} else if !self.depth_sensitive {
self.unlisted.insert(oid);
}
if !filter.prunes_tree_entries(depth) {
self.queue.push_back((oid, path, depth));
}
}
fn push_blob(
&mut self,
oid: ObjectId,
name: Option<(&str, &[u8])>,
tree_depth: u64,
) -> Result<()> {
if self.at_object_boundary(oid) {
self.note_boundary(oid);
return Ok(());
}
if self.visited.contains(&oid) {
return Ok(());
}
let (passes, size_checked) = match self.filter {
None => (true, false),
Some(filter) => self.blob_passes(filter, &oid, tree_depth)?,
};
if passes {
self.visited.insert(oid);
self.list(oid);
if let Some((parent, entry_name)) = name {
self.name_hashes
.insert(oid, pack_name_hash(&child_path(parent, entry_name)));
}
} else if size_checked && !self.depth_sensitive {
self.visited.insert(oid);
self.unlisted.insert(oid);
}
Ok(())
}
fn blob_passes(
&self,
filter: &ObjectFilter,
oid: &ObjectId,
tree_depth: u64,
) -> Result<(bool, bool)> {
if !filter.includes_blob_under_tree(0, tree_depth) {
return Ok((false, false));
}
if filter.includes_blob_under_tree(u64::MAX, tree_depth) {
return Ok((true, false));
}
match self.odb.read_info(oid) {
Ok(info) => Ok((filter.includes_blob_under_tree(info.size, tree_depth), true)),
Err(_) if self.skip_missing => Ok((false, true)),
Err(e) => Err(e),
}
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used)]
use super::*;
use crate::repo::{InitOptions, Repository};
use tempfile::TempDir;
fn walk(repo: &Repository, roots: &[ObjectId], name_trees: NameTrees) -> PackObjectSelection {
let empty = HashSet::new();
enumerate_pack_objects_naming(
repo.odb(),
roots,
&[],
&PackEnumerateOptions {
filter: None,
shallow_grafts: &empty,
have_shallow_grafts: Some(&empty),
use_bitmaps: false,
exclude_objects: None,
peer_odb: None,
have_boundary: HaveBoundary::CommitClosure,
},
name_trees,
)
.expect("walk")
}
fn write_tree(repo: &Repository, entries: &[(u32, &str, ObjectId)]) -> ObjectId {
let entries: Vec<crate::objects::TreeEntry> = entries
.iter()
.map(|&(mode, name, oid)| crate::objects::TreeEntry {
mode,
name: name.as_bytes().to_vec(),
oid,
})
.collect();
repo.odb()
.write(ObjectKind::Tree, &crate::objects::serialize_tree(&entries))
.expect("tree")
}
fn write_commit(
repo: &Repository,
tree: ObjectId,
parents: &[ObjectId],
time: i64,
) -> ObjectId {
let mut body = format!("tree {}\n", tree.to_hex());
for parent in parents {
body.push_str(&format!("parent {}\n", parent.to_hex()));
}
body.push_str(&format!(
"author A <a@example.com> {time} +0000\ncommitter A <a@example.com> {time} +0000\n\nc{time}\n"
));
repo.odb()
.write(ObjectKind::Commit, body.as_bytes())
.expect("commit")
}
#[test]
fn tips_only_boundary_matches_closure_on_linear_incremental_push() {
let dir = TempDir::new().unwrap();
let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
let blob = |s: &str| repo.odb().write(ObjectKind::Blob, s.as_bytes()).unwrap();
let mut parent = write_commit(
&repo,
write_tree(&repo, &[(0o100644, "f", blob("0"))]),
&[],
0,
);
for i in 1..800 {
let tree = write_tree(&repo, &[(0o100644, "f", blob(&i.to_string()))]);
parent = write_commit(&repo, tree, &[parent], i);
}
let remote_tip = parent;
let new_tree = write_tree(&repo, &[(0o100644, "f", blob("new"))]);
let new_tip = write_commit(&repo, new_tree, &[remote_tip], 900);
let empty = HashSet::new();
let closure_opts = PackEnumerateOptions {
filter: None,
shallow_grafts: &empty,
have_shallow_grafts: Some(&empty),
use_bitmaps: false,
exclude_objects: None,
peer_odb: None,
have_boundary: HaveBoundary::CommitClosure,
};
let tips_opts = PackEnumerateOptions {
have_boundary: HaveBoundary::TipsOnly,
..closure_opts
};
let closure = enumerate_pack_objects(repo.odb(), &[new_tip], &[remote_tip], &closure_opts)
.expect("closure");
let tips = enumerate_pack_objects(repo.odb(), &[new_tip], &[remote_tip], &tips_opts)
.expect("tips");
let closure_set: HashSet<ObjectId> = closure.send.iter().copied().collect();
let tips_set: HashSet<ObjectId> = tips.send.iter().copied().collect();
assert_eq!(closure_set, tips_set);
assert!(
tips.send.len() <= 8,
"one new commit should enumerate a handful of objects, got {}",
tips.send.len()
);
}
#[test]
fn tips_only_send_pack_does_not_prewalk_deep_history() {
use crate::merge_base::intersect_walk_test_hooks::{
commit_parent_lookups, reset_commit_parent_lookups,
};
let dir = TempDir::new().unwrap();
let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
let blob = |s: &str| repo.odb().write(ObjectKind::Blob, s.as_bytes()).unwrap();
let mut parent = write_commit(
&repo,
write_tree(&repo, &[(0o100644, "f", blob("0"))]),
&[],
0,
);
for i in 1..800 {
let tree = write_tree(&repo, &[(0o100644, "f", blob(&i.to_string()))]);
parent = write_commit(&repo, tree, &[parent], i);
}
let remote_tip = parent;
let new_tree = write_tree(&repo, &[(0o100644, "f", blob("new"))]);
let new_tip = write_commit(&repo, new_tree, &[remote_tip], 900);
let haves = vec![remote_tip];
let empty = HashSet::new();
reset_commit_parent_lookups();
let _selection = enumerate_pack_objects(
repo.odb(),
&[new_tip],
&haves,
&PackEnumerateOptions {
filter: None,
shallow_grafts: &empty,
have_shallow_grafts: Some(&empty),
use_bitmaps: false,
exclude_objects: None,
peer_odb: None,
have_boundary: HaveBoundary::TipsOnly,
},
)
.expect("enumerate");
assert!(
commit_parent_lookups() <= 128,
"send-pack hide must stay coupled to the want walk ({} parent lookups)",
commit_parent_lookups()
);
}
#[test]
fn tips_only_many_advertised_haves_does_not_prewalk_deep_history() {
use crate::merge_base::intersect_walk_test_hooks::{
commit_parent_lookups, reset_commit_parent_lookups,
};
let dir = TempDir::new().unwrap();
let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
let blob = |s: &str| repo.odb().write(ObjectKind::Blob, s.as_bytes()).unwrap();
let mut chain = Vec::new();
let mut parent = write_commit(
&repo,
write_tree(&repo, &[(0o100644, "f", blob("0"))]),
&[],
0,
);
chain.push(parent);
for i in 1..800 {
let tree = write_tree(&repo, &[(0o100644, "f", blob(&i.to_string()))]);
parent = write_commit(&repo, tree, &[parent], i);
chain.push(parent);
}
let remote_tip = parent;
let new_tree = write_tree(&repo, &[(0o100644, "f", blob("new"))]);
let new_tip = write_commit(&repo, new_tree, &[remote_tip], 900);
let mut haves = vec![remote_tip];
for n in (2..400).step_by(2) {
let idx = chain.len().checked_sub(1 + n);
if let Some(idx) = idx {
haves.push(chain[idx]);
}
}
let empty = HashSet::new();
reset_commit_parent_lookups();
let selection = enumerate_pack_objects(
repo.odb(),
&[new_tip],
&haves,
&PackEnumerateOptions {
filter: None,
shallow_grafts: &empty,
have_shallow_grafts: Some(&empty),
use_bitmaps: false,
exclude_objects: None,
peer_odb: None,
have_boundary: HaveBoundary::TipsOnly,
},
)
.expect("enumerate");
assert!(
selection.send.len() <= 8,
"one new commit should stay small with many haves, got {}",
selection.send.len()
);
assert!(
commit_parent_lookups() <= 512,
"many advertised haves must not pre-walk deep history ({} parent lookups)",
commit_parent_lookups()
);
}
#[test]
fn send_pack_have_walk_stops_without_scanning_deep_haves() {
use super::send_pack_walk_test_hooks::{commit_reads, reset_commit_reads};
let dir = TempDir::new().unwrap();
let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
let blob = |s: &str| repo.odb().write(ObjectKind::Blob, s.as_bytes()).unwrap();
let mut parent = write_commit(
&repo,
write_tree(&repo, &[(0o100644, "f", blob("0"))]),
&[],
0,
);
let near_root = parent;
for i in 1..800 {
let tree = write_tree(&repo, &[(0o100644, "f", blob(&i.to_string()))]);
parent = write_commit(&repo, tree, &[parent], i);
}
let direct_parent = parent;
let new_tree = write_tree(&repo, &[(0o100644, "f", blob("new"))]);
let new_tip = write_commit(&repo, new_tree, &[direct_parent], 900);
let empty = HashSet::new();
reset_commit_reads();
let have_commits = send_pack_uninteresting_commits(
repo.odb(),
&[new_tip],
&[direct_parent, near_root],
&empty,
)
.expect("send-pack have walk");
assert!(have_commits.contains(&direct_parent));
assert!(have_commits.contains(&near_root));
let reads = commit_reads();
assert!(
reads < 20,
"send-pack have walk should not scan deep history, read {reads} commits"
);
}
#[test]
fn walk_lists_trees_of_newer_commits_first() {
let dir = TempDir::new().unwrap();
let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
let blob = |s: &str| repo.odb().write(ObjectKind::Blob, s.as_bytes()).unwrap();
let tree_a = write_tree(&repo, &[(0o100644, "f", blob("a"))]);
let tree_b = write_tree(&repo, &[(0o100644, "f", blob("b"))]);
let tree_c = write_tree(&repo, &[(0o100644, "f", blob("c"))]);
let base = write_commit(&repo, tree_a, &[], 100);
let old_tip = write_commit(&repo, tree_b, &[base], 200);
let new_tip = write_commit(&repo, tree_c, &[base], 300);
let selection = walk(&repo, &[old_tip, new_tip], NameTrees::No);
let pos = |oid: ObjectId| selection.send.iter().position(|o| *o == oid).unwrap();
assert!(pos(old_tip) < pos(base) && pos(new_tip) < pos(base));
assert!(pos(tree_c) < pos(tree_b) && pos(tree_b) < pos(tree_a));
}
#[test]
fn walk_names_trees_and_keeps_history_paths_for_path_less_roots() {
let dir = TempDir::new().unwrap();
let repo = Repository::init(dir.path(), &InitOptions::default()).expect("init");
let file = repo.odb().write(ObjectKind::Blob, b"x\n").unwrap();
let link = repo.odb().write(ObjectKind::Blob, b"target").unwrap();
let sub = write_tree(
&repo,
&[(0o100644, "file.rs", file), (0o120000, "link", link)],
);
let root = write_tree(&repo, &[(0o040000, "src", sub)]);
let commit = write_commit(&repo, root, &[], 100);
let selection = walk(&repo, &[sub, file, commit], NameTrees::Yes);
let hash = |oid: ObjectId| selection.name_hashes.get(&oid).copied();
assert_eq!(hash(sub), Some(pack_name_hash("src")));
assert_eq!(hash(file), Some(pack_name_hash("src/file.rs")));
assert_eq!(hash(root), None, "root trees have no path");
assert_eq!(selection.send.len(), 5);
let blobs_only = walk(&repo, &[sub, file, commit], NameTrees::No);
assert_eq!(blobs_only.name_hashes.get(&sub), None);
assert_eq!(
blobs_only.name_hashes.get(&file),
Some(&pack_name_hash("src/file.rs"))
);
}
fn tree_entry(mode: &str, name: &str, oid: ObjectId) -> Vec<u8> {
let mut bytes = format!("{mode} {name}\0").into_bytes();
bytes.extend_from_slice(oid.as_bytes());
bytes
}
#[test]
fn walk_descends_into_trees_with_non_canonical_modes() {
let tmp = tempfile::tempdir().unwrap();
Repository::init(tmp.path(), &InitOptions::new().with_bare(true)).unwrap();
let repo = Repository::open(tmp.path(), &RepositoryOptions::empty()).unwrap();
let odb = repo.odb();
let deep = odb
.write_loose_materialize(ObjectKind::Blob, b"deep\n")
.unwrap();
let odd = odb
.write_loose_materialize(ObjectKind::Blob, b"odd\n")
.unwrap();
let inner = odb
.write_loose_materialize(ObjectKind::Tree, &tree_entry("100644", "deep.txt", deep))
.unwrap();
let mut root = tree_entry("100664", "odd.txt", odd);
root.extend(tree_entry("40755", "sub", inner));
let root = odb
.write_loose_materialize(ObjectKind::Tree, &root)
.unwrap();
let commit = format!(
"tree {root}\nauthor T <t@e.com> 100 +0000\ncommitter T <t@e.com> 100 +0000\n\nodd modes\n"
);
let commit = odb
.write_loose_materialize(ObjectKind::Commit, commit.as_bytes())
.unwrap();
let empty = HashSet::new();
let selection = enumerate_pack_objects(
odb,
&[commit],
&[],
&PackEnumerateOptions {
filter: None,
shallow_grafts: &empty,
have_shallow_grafts: Some(&empty),
use_bitmaps: false,
exclude_objects: None,
peer_odb: None,
have_boundary: HaveBoundary::CommitClosure,
},
)
.unwrap();
let ours: HashSet<ObjectId> = selection.send.iter().copied().collect();
let git: HashSet<ObjectId> =
grit_test_support::git(tmp.path(), &["rev-list", "--objects", &commit.to_hex()])
.lines()
.map(|line| ObjectId::from_hex(&line[..40]).unwrap())
.collect();
assert_eq!(ours, git, "same objects as git rev-list --objects");
assert!(ours.contains(&inner) && ours.contains(&deep));
assert_eq!(selection.send.len(), 5);
assert_eq!(
selection.name_hashes.get(&deep),
Some(&pack_name_hash("sub/deep.txt"))
);
assert_eq!(
selection.name_hashes.get(&odd),
Some(&pack_name_hash("odd.txt"))
);
}
}