use flate2::Compression;
use flate2::read::ZlibDecoder;
use flate2::write::ZlibEncoder;
use flate2::{Decompress, FlushDecompress};
use parking_lot::RwLock;
use std::sync::Mutex;
use sley_core::{GitError, MissingObjectContext, ObjectFormat, ObjectId, Result};
use sley_formats::{Bundle, BundleReference};
use sley_object::{
Commit, EncodedObject, ObjectType, Tag, TreeEntries, parse_framed_object,
tree_entry_object_type,
};
use sley_pack::{
MultiPackIndex, MultiPackIndexOidLookup, PackBitmapIndex, PackBitmapWriter, PackFile,
PackIndex, PackIndexByteSource, PackIndexEntry, PackIndexViewData, PackInput,
PackReverseIndex, PackStreamIndexBuild, PackWrite, PackWriteOptions, PackWriteSummary,
};
use std::collections::{HashMap, HashSet};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, OnceLock};
use std::{env, fs};
use crate::{
grafted_parents, implied_empty_tree_object, unique_temp_path, with_missing_object_context,
ObjectReader, ObjectWriter,
};
use crate::install::{write_pack_component, write_promisor_pack_sidecar, RawPackInstaller, ReachablePackFile, ReachablePackWriteSummary};
use crate::pack::FileObjectDatabase;
use crate::reachability::{
BitmapPseudoMergeGroup, ReachablePackObject, ReachablePackObjectMeta, ReachablePackObjectsForWrite,
build_midx_bitmap, build_pack_bitmap, collect_reachable_object_ids_excluding,
collect_reachable_object_ids_excluding_promised_missing,
collect_reachable_object_ids_tolerating_missing, collect_reachable_pack_objects_for_write,
existing_pack_files, loose_object_id_set, loose_object_ids, pack_inputs, prune_loose_objects,
prune_obsolete_pack_paths, prune_stale_multi_pack_index, remove_file_if_exists,
sort_reachable_pack_metadata,
};
use crate::registry::{alternate_object_dirs, collect_packed_object_ids, object_ids_in_objects_dir, repository_objects_dir};
use crate::loose::LooseObjectStore;
use crate::pack::promisor_pack_object_ids;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RepackResult {
pub pack: Vec<u8>,
pub idx: Vec<u8>,
pub object_count: usize,
pub obsolete_packs: Vec<PathBuf>,
pub packed_loose: Vec<ObjectId>,
retained_pack_stems: Vec<String>,
promisor: bool,
pack_checksum: ObjectId,
index_entries: Vec<PackIndexEntry>,
}
#[derive(Debug, Clone, Default)]
pub struct RepackOptions {
pub local: bool,
pub pack_kept_objects: bool,
pub keep_pack_stems: HashSet<String>,
}
pub fn repack_reachable_objects(
git_dir: &Path,
format: ObjectFormat,
roots: &[ObjectId],
) -> Result<Option<RepackResult>> {
repack_reachable_objects_with_options(git_dir, format, roots, &RepackOptions::default())
}
pub fn repack_reachable_objects_with_options(
git_dir: &Path,
format: ObjectFormat,
roots: &[ObjectId],
options: &RepackOptions,
) -> Result<Option<RepackResult>> {
let objects_dir = repository_objects_dir(git_dir);
let database = if options.local {
FileObjectDatabase::without_alternates(objects_dir.clone(), format)
} else {
FileObjectDatabase::new(objects_dir.clone(), format)
};
let retained_pack_stems = repack_retained_pack_stems(
&objects_dir.join("pack"),
&options.keep_pack_stems,
!options.pack_kept_objects,
)?;
let excluded_oids = if options.pack_kept_objects {
HashSet::new()
} else {
pack_oids_for_stems(&objects_dir.join("pack"), format, &retained_pack_stems)?
};
let promisor_oids = promisor_pack_object_ids(&objects_dir, format)?;
let mut seen: HashSet<ObjectId> = HashSet::new();
let mut objects: Vec<ReachablePackObject> = Vec::new();
let mut pending: Vec<ObjectId> = roots.to_vec();
while let Some(oid) = pending.pop() {
if !seen.insert(oid) {
continue;
}
if promisor_oids.contains(&oid) {
continue;
}
let object = match database.read_object(&oid) {
Ok(object) => object,
Err(GitError::NotFound(_)) => continue,
Err(err) => return Err(err),
};
match object.object_type {
ObjectType::Commit => {
let commit = Commit::parse_ref(format, &object.body)?;
pending.extend(grafted_parents(&database, &oid, commit.parents));
pending.push(commit.tree);
}
ObjectType::Tree => {
for entry in TreeEntries::new(format, &object.body) {
let entry = entry?;
if !entry.is_gitlink() {
pending.push(entry.oid);
}
}
}
ObjectType::Tag => {
let tag = Tag::parse_ref(format, &object.body)?;
pending.push(tag.object);
}
ObjectType::Blob => {}
}
if !excluded_oids.contains(&oid) {
objects.push(ReachablePackObject { oid, object });
}
}
if !options.local {
for (alternate, oid) in alternate_packed_object_ids(&objects_dir, format)? {
if excluded_oids.contains(&oid) || !seen.insert(oid) {
continue;
}
let alternate_db = FileObjectDatabase::without_alternates(alternate, format);
match alternate_db.read_object(&oid) {
Ok(object) => objects.push(ReachablePackObject { oid, object }),
Err(GitError::NotFound(_)) => {}
Err(err) => return Err(err),
}
}
}
if objects.is_empty() {
return Ok(None);
}
let inputs = pack_inputs(&objects);
let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
let object_count = written.entries.len();
let new_pack_file_name = format!("pack-{}.pack", written.checksum.to_hex());
let obsolete_packs = existing_pack_files(&objects_dir.join("pack"))?
.into_iter()
.filter(|path| path.file_name().and_then(|name| name.to_str()) != Some(&new_pack_file_name))
.collect();
let packed_oid_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
let mut packed_loose: Vec<ObjectId> = loose_object_ids(&objects_dir, format)?
.into_iter()
.filter(|oid| packed_oid_set.contains(oid))
.collect();
packed_loose.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
let pack_checksum = written.checksum;
let index_entries = written.entries.clone();
Ok(Some(RepackResult {
pack: written.pack,
idx: written.index,
object_count,
obsolete_packs,
packed_loose,
retained_pack_stems,
promisor: false,
pack_checksum,
index_entries,
}))
}
fn repack_retained_pack_stems(
pack_dir: &Path,
explicit: &HashSet<String>,
keep_dot_keep: bool,
) -> Result<Vec<String>> {
let mut stems = explicit.clone();
if keep_dot_keep {
for pack_path in existing_pack_files(pack_dir)? {
if pack_path.with_extension("keep").exists()
&& let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str())
{
stems.insert(stem.to_string());
}
}
}
let mut stems = stems.into_iter().collect::<Vec<_>>();
stems.sort();
Ok(stems)
}
fn pack_oids_for_stems(
pack_dir: &Path,
format: ObjectFormat,
stems: &[String],
) -> Result<HashSet<ObjectId>> {
let wanted: HashSet<&str> = stems.iter().map(String::as_str).collect();
if wanted.is_empty() {
return Ok(HashSet::new());
}
let mut oids = HashSet::new();
for pack_path in existing_pack_files(pack_dir)? {
let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str()) else {
continue;
};
if !wanted.contains(stem) {
continue;
}
let index_path = pack_path.with_extension("idx");
if !index_path.exists() {
continue;
}
let index = PackIndex::parse(&fs::read(index_path)?, format)?;
oids.extend(index.entries.into_iter().map(|entry| entry.oid));
}
Ok(oids)
}
fn alternate_packed_object_ids(
objects_dir: &Path,
format: ObjectFormat,
) -> Result<Vec<(PathBuf, ObjectId)>> {
let mut oids = Vec::new();
for alternate in alternate_object_dirs(objects_dir) {
let mut alternate_oids = HashSet::new();
collect_packed_object_ids(&alternate.join("pack"), format, &mut alternate_oids)?;
oids.extend(
alternate_oids
.into_iter()
.map(|oid| (alternate.clone(), oid)),
);
}
oids.sort_by(|left, right| {
left.0
.cmp(&right.0)
.then(left.1.as_bytes().cmp(right.1.as_bytes()))
});
Ok(oids)
}
pub fn repack_all_objects(git_dir: &Path, format: ObjectFormat) -> Result<Option<RepackResult>> {
let objects_dir = repository_objects_dir(git_dir);
let database = FileObjectDatabase::new(objects_dir.clone(), format);
let all_oids = object_ids_in_objects_dir(&objects_dir, format)?;
if all_oids.is_empty() {
return Ok(None);
}
let mut objects = Vec::with_capacity(all_oids.len());
for oid in &all_oids {
objects.push(ReachablePackObject {
oid: *oid,
object: database.read_object(oid)?,
});
}
let inputs = pack_inputs(&objects);
let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
let object_count = written.entries.len();
let new_pack_file_name = format!("pack-{}.pack", written.checksum.to_hex());
let obsolete_packs = existing_pack_files(&objects_dir.join("pack"))?
.into_iter()
.filter(|path| path.file_name().and_then(|name| name.to_str()) != Some(&new_pack_file_name))
.collect();
let packed_oid_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
let mut packed_loose: Vec<ObjectId> = loose_object_ids(&objects_dir, format)?
.into_iter()
.filter(|oid| packed_oid_set.contains(oid))
.collect();
packed_loose.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
Ok(Some(RepackResult {
pack: written.pack,
idx: written.index,
object_count,
obsolete_packs,
packed_loose,
retained_pack_stems: Vec::new(),
promisor: false,
pack_checksum: written.checksum,
index_entries: written.entries,
}))
}
pub fn repack_promisor_objects(
git_dir: &Path,
format: ObjectFormat,
) -> Result<Option<RepackResult>> {
let objects_dir = repository_objects_dir(git_dir);
let pack_dir = objects_dir.join("pack");
let promisor_packs = existing_pack_files(&pack_dir)?
.into_iter()
.filter(|path| path.with_extension("promisor").exists())
.collect::<Vec<_>>();
if promisor_packs.len() <= 1 {
return Ok(None);
}
let database = FileObjectDatabase::new(objects_dir.clone(), format);
let mut seen = HashSet::new();
let mut objects = Vec::new();
for pack_path in &promisor_packs {
let index_path = pack_path.with_extension("idx");
if !index_path.exists() {
continue;
}
let index = PackIndex::parse(&fs::read(index_path)?, format)?;
for entry in index.entries {
if !seen.insert(entry.oid) {
continue;
}
objects.push(ReachablePackObject {
oid: entry.oid,
object: database.read_object(&entry.oid)?,
});
}
}
if objects.is_empty() {
return Ok(None);
}
objects.sort_by(|left, right| left.oid.as_bytes().cmp(right.oid.as_bytes()));
let inputs = pack_inputs(&objects);
let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
let object_count = written.entries.len();
let packed_oid_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
let mut packed_loose: Vec<ObjectId> = loose_object_ids(&objects_dir, format)?
.into_iter()
.filter(|oid| packed_oid_set.contains(oid))
.collect();
packed_loose.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
let pack_checksum = written.checksum;
let index_entries = written.entries.clone();
Ok(Some(RepackResult {
pack: written.pack,
idx: written.index,
object_count,
obsolete_packs: promisor_packs,
packed_loose,
retained_pack_stems: Vec::new(),
promisor: true,
pack_checksum,
index_entries,
}))
}
pub fn repack_loose_objects(git_dir: &Path, format: ObjectFormat) -> Result<Option<RepackResult>> {
let objects_dir = repository_objects_dir(git_dir);
let database = FileObjectDatabase::new(objects_dir.clone(), format);
let loose_oids = loose_object_ids(&objects_dir, format)?;
if loose_oids.is_empty() {
return Ok(None);
}
let mut objects = Vec::with_capacity(loose_oids.len());
for oid in &loose_oids {
objects.push(ReachablePackObject {
oid: *oid,
object: database.read_object(oid)?,
});
}
let inputs = pack_inputs(&objects);
let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
let object_count = written.entries.len();
let packed_oid_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
let mut packed_loose: Vec<ObjectId> = loose_oids
.into_iter()
.filter(|oid| packed_oid_set.contains(oid))
.collect();
packed_loose.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
let pack_checksum = written.checksum;
let index_entries = written.entries.clone();
Ok(Some(RepackResult {
pack: written.pack,
idx: written.index,
object_count,
obsolete_packs: Vec::new(),
packed_loose,
retained_pack_stems: Vec::new(),
promisor: false,
pack_checksum,
index_entries,
}))
}
#[derive(Debug, Clone)]
struct GeometryPack {
pack_path: PathBuf,
oids: Vec<ObjectId>,
weight: u64,
is_promisor: bool,
}
#[derive(Debug, Clone)]
pub struct GeometricRepackResult {
pub result: Option<RepackResult>,
pub rolled_up_packs: Vec<PathBuf>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GeometricRepackPlan {
pub split: usize,
pub pack_count: usize,
}
fn collect_geometry_packs(
objects_dir: &Path,
format: ObjectFormat,
kept_pack_stems: &HashSet<String>,
) -> Result<Vec<GeometryPack>> {
let pack_dir = objects_dir.join("pack");
let mut packs = Vec::new();
for pack_path in existing_pack_files(&pack_dir)? {
if pack_path.with_extension("mtimes").exists() {
continue;
}
if pack_path.with_extension("keep").exists() {
continue;
}
let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str()) else {
continue;
};
if kept_pack_stems.contains(stem) {
continue;
}
let index_path = pack_path.with_extension("idx");
if !index_path.exists() {
continue;
}
let index = PackIndex::parse(&fs::read(&index_path)?, format)?;
let oids: Vec<ObjectId> = index.entries.iter().map(|entry| entry.oid).collect();
let weight = oids.len() as u64;
packs.push(GeometryPack {
is_promisor: pack_path.with_extension("promisor").exists(),
pack_path,
oids,
weight,
});
}
packs.sort_by(|a, b| a.weight.cmp(&b.weight).then(a.pack_path.cmp(&b.pack_path)));
Ok(packs)
}
fn compute_geometry_split(packs: &[GeometryPack], split_factor: u64) -> usize {
let pack_nr = packs.len();
if pack_nr == 0 {
return 0;
}
let mut i = pack_nr - 1;
while i > 0 {
let ours = packs[i].weight;
let prev = packs[i - 1].weight;
if ours < split_factor.saturating_mul(prev) {
break;
}
i -= 1;
}
let mut split = i;
if split != 0 {
split += 1;
}
let mut total_size: u64 = packs[..split].iter().map(|p| p.weight).sum();
for pack in &packs[split..] {
if pack.weight < split_factor.saturating_mul(total_size) {
split += 1;
total_size = total_size.saturating_add(pack.weight);
} else {
break;
}
}
split
}
pub fn geometric_repack_plan(
git_dir: &Path,
format: ObjectFormat,
split_factor: u64,
kept_pack_stems: &HashSet<String>,
) -> Result<GeometricRepackPlan> {
let objects_dir = repository_objects_dir(git_dir);
let packs: Vec<GeometryPack> = collect_geometry_packs(&objects_dir, format, kept_pack_stems)?
.into_iter()
.filter(|pack| !pack.is_promisor)
.collect();
Ok(GeometricRepackPlan {
split: compute_geometry_split(&packs, split_factor),
pack_count: packs.len(),
})
}
pub fn repack_geometric(
git_dir: &Path,
format: ObjectFormat,
split_factor: u64,
kept_pack_stems: &HashSet<String>,
) -> Result<GeometricRepackResult> {
let objects_dir = repository_objects_dir(git_dir);
let database = FileObjectDatabase::new(objects_dir.clone(), format);
let all_packs = collect_geometry_packs(&objects_dir, format, kept_pack_stems)?;
let packs: Vec<GeometryPack> = all_packs
.into_iter()
.filter(|pack| !pack.is_promisor)
.collect();
let split = compute_geometry_split(&packs, split_factor);
let loose_oids = loose_object_ids(&objects_dir, format)?;
let mut excluded_oids: HashSet<ObjectId> = HashSet::new();
for pack in &packs[split..] {
excluded_oids.extend(pack.oids.iter().copied());
}
let mut included: Vec<ObjectId> = Vec::new();
let mut seen: HashSet<ObjectId> = HashSet::new();
for pack in &packs[..split] {
for oid in &pack.oids {
if excluded_oids.contains(oid) {
continue;
}
if seen.insert(*oid) {
included.push(*oid);
}
}
}
for oid in &loose_oids {
if excluded_oids.contains(oid) {
continue;
}
if seen.insert(*oid) {
included.push(*oid);
}
}
if included.is_empty() {
return Ok(GeometricRepackResult {
result: None,
rolled_up_packs: Vec::new(),
});
}
included.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
let mut objects = Vec::with_capacity(included.len());
for oid in &included {
objects.push(ReachablePackObject {
oid: *oid,
object: database.read_object(oid)?,
});
}
let inputs = pack_inputs(&objects);
let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
let object_count = written.entries.len();
let packed_oid_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
let mut packed_loose: Vec<ObjectId> = loose_oids
.into_iter()
.filter(|oid| packed_oid_set.contains(oid))
.collect();
packed_loose.sort_by(|left, right| left.as_bytes().cmp(right.as_bytes()));
let rolled_up_packs: Vec<PathBuf> = packs[..split]
.iter()
.map(|pack| pack.pack_path.clone())
.collect();
let pack_checksum = written.checksum;
let index_entries = written.entries.clone();
Ok(GeometricRepackResult {
result: Some(RepackResult {
pack: written.pack,
idx: written.index,
object_count,
obsolete_packs: rolled_up_packs.clone(),
packed_loose,
retained_pack_stems: Vec::new(),
promisor: false,
pack_checksum,
index_entries,
}),
rolled_up_packs,
})
}
pub fn install_repack_result(
git_dir: &Path,
format: ObjectFormat,
result: &RepackResult,
prune: bool,
) -> Result<()> {
install_repack_result_with_bitmap(git_dir, format, result, prune, None, None)
}
pub fn install_repack_result_with_bitmap(
git_dir: &Path,
format: ObjectFormat,
result: &RepackResult,
prune: bool,
bitmap_tips: Option<&HashSet<ObjectId>>,
bitmap_pseudo_merge_groups: Option<&[BitmapPseudoMergeGroup]>,
) -> Result<()> {
let objects_dir = repository_objects_dir(git_dir);
let pack_dir = objects_dir.join("pack");
fs::create_dir_all(&pack_dir)?;
validate_pack_checksum(&result.pack, format, &result.pack_checksum, "repack")?;
let parsed_index = PackIndex::parse(&result.idx, format)?;
if parsed_index.pack_checksum != result.pack_checksum {
return Err(GitError::InvalidFormat(
"repack index checksum does not match the new pack".into(),
));
}
if !pack_index_entries_match_writer(&parsed_index.entries, &result.index_entries) {
return Err(GitError::InvalidFormat(
"repack index does not match the new pack contents".into(),
));
}
let pack_name = format!("pack-{}", result.pack_checksum.to_hex());
let new_pack_path = pack_dir.join(format!("{pack_name}.pack"));
let new_rev_path = pack_dir.join(format!("{pack_name}.rev"));
let new_index_path = pack_dir.join(format!("{pack_name}.idx"));
let reverse_index = sley_pack::PackReverseIndex::write(
format,
&sley_pack::pack_order_index_positions(&parsed_index.entries),
&result.pack_checksum,
)?;
write_pack_component(&new_pack_path, &result.pack)?;
write_pack_component(&new_rev_path, &reverse_index)?;
write_pack_component(&new_index_path, &result.idx)?;
let new_promisor_path = write_promisor_pack_sidecar(&pack_dir, &pack_name, result.promisor)?;
if let Some(tips) = bitmap_tips {
let database = FileObjectDatabase::new(objects_dir.clone(), format);
if let Some(bitmap) = build_pack_bitmap(
&database,
format,
&result.index_entries,
&result.pack_checksum,
tips,
bitmap_pseudo_merge_groups.unwrap_or(&[]),
)? {
let bitmap_path = pack_dir.join(format!("{pack_name}.bitmap"));
remove_file_if_exists(&bitmap_path)?;
write_pack_component(&bitmap_path, &bitmap)?;
}
}
if !prune {
return Ok(());
}
let present: HashSet<ObjectId> = parsed_index.entries.iter().map(|entry| entry.oid).collect();
prune_obsolete_pack_paths(
&objects_dir,
format,
&result.obsolete_packs,
&new_pack_path,
&result.retained_pack_stems,
result.promisor,
)?;
prune_loose_objects(&objects_dir, format, result.packed_loose.iter(), &present)?;
if result.promisor && new_promisor_path.is_none() {
return Err(GitError::InvalidFormat(
"promisor repack did not write sidecar".into(),
));
}
Ok(())
}
pub fn install_geometric_repack_result(
git_dir: &Path,
format: ObjectFormat,
geometric: &GeometricRepackResult,
prune: bool,
bitmap_tips: Option<&HashSet<ObjectId>>,
) -> Result<()> {
let Some(result) = geometric.result.as_ref() else {
return Ok(());
};
let objects_dir = repository_objects_dir(git_dir);
let pack_dir = objects_dir.join("pack");
fs::create_dir_all(&pack_dir)?;
validate_pack_checksum(&result.pack, format, &result.pack_checksum, "repack")?;
let parsed_index = PackIndex::parse(&result.idx, format)?;
if parsed_index.pack_checksum != result.pack_checksum {
return Err(GitError::InvalidFormat(
"repack index checksum does not match the new pack".into(),
));
}
if !pack_index_entries_match_writer(&parsed_index.entries, &result.index_entries) {
return Err(GitError::InvalidFormat(
"repack index does not match the new pack contents".into(),
));
}
let pack_name = format!("pack-{}", result.pack_checksum.to_hex());
let new_pack_path = pack_dir.join(format!("{pack_name}.pack"));
let new_rev_path = pack_dir.join(format!("{pack_name}.rev"));
let new_index_path = pack_dir.join(format!("{pack_name}.idx"));
let reverse_index = sley_pack::PackReverseIndex::write(
format,
&sley_pack::pack_order_index_positions(&parsed_index.entries),
&result.pack_checksum,
)?;
write_pack_component(&new_pack_path, &result.pack)?;
write_pack_component(&new_rev_path, &reverse_index)?;
write_pack_component(&new_index_path, &result.idx)?;
if let Some(tips) = bitmap_tips {
let database = FileObjectDatabase::new(objects_dir.clone(), format);
if let Some(bitmap) = build_pack_bitmap(
&database,
format,
&result.index_entries,
&result.pack_checksum,
tips,
&[],
)? {
let bitmap_path = pack_dir.join(format!("{pack_name}.bitmap"));
remove_file_if_exists(&bitmap_path)?;
write_pack_component(&bitmap_path, &bitmap)?;
}
}
if !prune {
return Ok(());
}
for pack_path in &geometric.rolled_up_packs {
if *pack_path == new_pack_path {
continue;
}
if pack_path.with_extension("keep").exists() {
continue;
}
remove_file_if_exists(pack_path)?;
remove_file_if_exists(&pack_path.with_extension("idx"))?;
for ext in ["rev", "mtimes", "bitmap", "promisor"] {
remove_file_if_exists(&pack_path.with_extension(ext))?;
}
}
let present: HashSet<ObjectId> = parsed_index.entries.iter().map(|entry| entry.oid).collect();
prune_loose_objects(&objects_dir, format, result.packed_loose.iter(), &present)?;
let removed_stems: HashSet<String> = geometric
.rolled_up_packs
.iter()
.filter_map(|p| p.file_stem().map(|s| s.to_string_lossy().into_owned()))
.collect();
prune_stale_multi_pack_index(&pack_dir, format, &removed_stems)?;
Ok(())
}
fn validate_pack_checksum(
pack: &[u8],
format: ObjectFormat,
expected: &ObjectId,
context: &str,
) -> Result<()> {
if expected.format() != format {
return Err(GitError::InvalidObjectId(format!(
"{context} checksum format does not match object format"
)));
}
let hash_len = format.raw_len();
if pack.len() < 12 + hash_len {
return Err(GitError::InvalidFormat(format!(
"{context} pack file too short"
)));
}
if &pack[..4] != b"PACK" {
return Err(GitError::InvalidFormat(format!(
"{context} pack file missing PACK signature"
)));
}
let trailer_offset = pack.len() - hash_len;
let actual = sley_core::digest_bytes(format, &pack[..trailer_offset])?;
let trailer = ObjectId::from_raw(format, &pack[trailer_offset..])?;
if &actual != expected || trailer != *expected {
return Err(GitError::InvalidFormat(format!(
"{context} pack checksum does not match generated pack"
)));
}
Ok(())
}
fn path_mtime_secs(path: &Path) -> u32 {
fs::metadata(path)
.and_then(|metadata| metadata.modified())
.ok()
.and_then(|time| time.duration_since(std::time::UNIX_EPOCH).ok())
.map(|dur| dur.as_secs() as u32)
.unwrap_or(0)
}
#[derive(Debug, Clone)]
pub struct CruftPack {
pub pack: Vec<u8>,
pub idx: Vec<u8>,
pub rev: Vec<u8>,
pub mtimes: Vec<u8>,
pub checksum: ObjectId,
pub oids: Vec<ObjectId>,
}
#[derive(Debug, Clone)]
pub struct CruftRepackResult {
pub reachable: Option<RepackResult>,
pub cruft: Option<CruftPack>,
pub obsolete_packs: Vec<PathBuf>,
pub obsolete_cruft_packs: Vec<PathBuf>,
retained_pack_stems: Vec<String>,
}
pub fn object_mtimes_on_disk_pub(
objects_dir: &Path,
format: ObjectFormat,
) -> Result<HashMap<ObjectId, u32>> {
object_mtimes_on_disk(objects_dir, format)
}
fn object_mtimes_on_disk(
objects_dir: &Path,
format: ObjectFormat,
) -> Result<HashMap<ObjectId, u32>> {
let mut mtimes: HashMap<ObjectId, u32> = HashMap::new();
let mut record = |oid: ObjectId, mtime: u32| {
mtimes
.entry(oid)
.and_modify(|existing| {
if mtime > *existing {
*existing = mtime;
}
})
.or_insert(mtime);
};
let pack_dir = objects_dir.join("pack");
if let Ok(entries) = fs::read_dir(&pack_dir) {
let mut idx_paths: Vec<PathBuf> = Vec::new();
for entry in entries {
let path = entry?.path();
if path.extension().and_then(|ext| ext.to_str()) == Some("idx") {
idx_paths.push(path);
}
}
idx_paths.sort();
for idx_path in idx_paths {
let pack_path = idx_path.with_extension("pack");
if !pack_path.exists() {
continue;
}
let index = PackIndex::parse(&fs::read(&idx_path)?, format)?;
let mtimes_path = idx_path.with_extension("mtimes");
let pack_object_mtimes: Option<Vec<u32>> =
fs::read(&mtimes_path).ok().and_then(|bytes| {
sley_pack::PackMtimes::parse(&bytes, format, index.entries.len())
.ok()
.map(|parsed| parsed.mtimes)
});
let pack_mtime = path_mtime_secs(&pack_path);
for (pos, entry) in index.entries.iter().enumerate() {
let mtime = pack_object_mtimes
.as_ref()
.and_then(|table| table.get(pos).copied())
.unwrap_or(pack_mtime);
record(entry.oid, mtime);
}
}
}
let store = LooseObjectStore::new(objects_dir.to_path_buf(), format);
for oid in loose_object_ids(objects_dir, format)? {
let path = store.object_path(&oid)?;
record(oid, path_mtime_secs(&path));
}
Ok(mtimes)
}
pub fn build_cruft_pack_pub(
database: &FileObjectDatabase,
format: ObjectFormat,
survivors: &HashMap<ObjectId, u32>,
) -> Result<Option<CruftPack>> {
build_cruft_pack(database, format, survivors)
}
fn build_cruft_pack(
database: &FileObjectDatabase,
format: ObjectFormat,
survivors: &HashMap<ObjectId, u32>,
) -> Result<Option<CruftPack>> {
if survivors.is_empty() {
return Ok(None);
}
let mut ordered: Vec<(ObjectId, u32)> = survivors.iter().map(|(o, m)| (*o, *m)).collect();
ordered.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
let mut oids: Vec<ObjectId> = Vec::with_capacity(ordered.len());
let mut objects: Vec<Arc<EncodedObject>> = Vec::with_capacity(ordered.len());
let mut mtime_by_oid: HashMap<ObjectId, u32> = HashMap::with_capacity(ordered.len());
for (oid, mtime) in ordered {
match database.read_object(&oid) {
Ok(object) => {
oids.push(oid);
objects.push(object);
mtime_by_oid.insert(oid, mtime);
}
Err(GitError::NotFound(_)) => {}
Err(err) => return Err(err),
}
}
if oids.is_empty() {
return Ok(None);
}
let inputs: Vec<PackInput<'_>> = oids
.iter()
.zip(&objects)
.map(|(oid, object)| PackInput {
oid,
object: object.as_ref(),
})
.collect();
let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
let mut sorted_entries: Vec<&sley_pack::PackIndexEntry> = written.entries.iter().collect();
sorted_entries.sort_by(|a, b| a.oid.as_bytes().cmp(b.oid.as_bytes()));
let mtimes_table: Vec<u32> = sorted_entries
.iter()
.map(|entry| mtime_by_oid.get(&entry.oid).copied().unwrap_or(0))
.collect();
let positions = sley_pack::pack_order_index_positions(&written.entries);
let rev = sley_pack::PackReverseIndex::write(format, &positions, &written.checksum)?;
let mtimes = sley_pack::PackMtimes::write(format, &mtimes_table, &written.checksum)?;
let mut cruft_oids: Vec<ObjectId> = sorted_entries.iter().map(|e| e.oid).collect();
cruft_oids.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
Ok(Some(CruftPack {
pack: written.pack,
idx: written.index,
rev,
mtimes,
checksum: written.checksum,
oids: cruft_oids,
}))
}
pub fn repack_cruft(
git_dir: &Path,
format: ObjectFormat,
roots: &[ObjectId],
cruft_expiration: Option<u32>,
) -> Result<CruftRepackResult> {
repack_cruft_with_options(
git_dir,
format,
roots,
cruft_expiration,
&RepackOptions::default(),
)
}
pub fn repack_cruft_with_options(
git_dir: &Path,
format: ObjectFormat,
roots: &[ObjectId],
cruft_expiration: Option<u32>,
options: &RepackOptions,
) -> Result<CruftRepackResult> {
let objects_dir = repository_objects_dir(git_dir);
let database = FileObjectDatabase::new(objects_dir.clone(), format);
let pack_dir = objects_dir.join("pack");
let retained_pack_stems = repack_retained_pack_stems(
&pack_dir,
&options.keep_pack_stems,
!options.pack_kept_objects,
)?;
let excluded_oids = if options.pack_kept_objects {
HashSet::new()
} else {
pack_oids_for_stems(&pack_dir, format, &retained_pack_stems)?
};
let promisor_oids = promisor_pack_object_ids(&objects_dir, format)?;
let database = if promisor_oids.is_empty() {
database
} else {
database.with_promisor_remote_present(true)
};
let mut reachable_ids = collect_reachable_object_ids_excluding_promised_missing(
&database,
format,
roots.iter().copied(),
&promisor_oids,
)?;
reachable_ids.retain(|oid| !excluded_oids.contains(oid));
let reachable_result = if reachable_ids.is_empty() {
None
} else {
let mut ids: Vec<ObjectId> = reachable_ids.iter().copied().collect();
ids.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
let mut objects = Vec::with_capacity(ids.len());
for oid in &ids {
match database.read_object(oid) {
Ok(object) => objects.push(ReachablePackObject { oid: *oid, object }),
Err(GitError::NotFound(_)) => {}
Err(err) => return Err(err),
}
}
if objects.is_empty() {
None
} else {
let inputs = pack_inputs(&objects);
let written = PackFile::write_packed_with_known_ids(&inputs, format)?;
let packed_set: HashSet<&ObjectId> = written.entries.iter().map(|e| &e.oid).collect();
let mut packed_loose: Vec<ObjectId> = loose_object_ids(&objects_dir, format)?
.into_iter()
.filter(|oid| packed_set.contains(oid))
.collect();
packed_loose.sort_by(|a, b| a.as_bytes().cmp(b.as_bytes()));
Some(RepackResult {
pack: written.pack,
idx: written.index,
object_count: written.entries.len(),
obsolete_packs: Vec::new(),
packed_loose,
retained_pack_stems: Vec::new(),
promisor: false,
pack_checksum: written.checksum,
index_entries: written.entries,
})
}
};
let mut survivors: HashMap<ObjectId, u32> = object_mtimes_on_disk(&objects_dir, format)?
.into_iter()
.filter(|(oid, _)| {
!reachable_ids.contains(oid)
&& !excluded_oids.contains(oid)
&& !promisor_oids.contains(oid)
})
.collect();
if let Some(expiration) = cruft_expiration {
rescue_and_expire_cruft_objects(&database, format, &mut survivors, expiration)?;
}
let cruft = build_cruft_pack(&database, format, &survivors)?;
let mut obsolete_packs = Vec::new();
let mut obsolete_cruft_packs = Vec::new();
for pack_path in existing_pack_files(&pack_dir)? {
if let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str())
&& retained_pack_stems.iter().any(|retained| retained == stem)
{
continue;
}
if pack_path.with_extension("keep").exists() {
continue;
}
if pack_path.with_extension("mtimes").exists() {
obsolete_cruft_packs.push(pack_path);
} else {
obsolete_packs.push(pack_path);
}
}
Ok(CruftRepackResult {
reachable: reachable_result,
cruft,
obsolete_packs,
obsolete_cruft_packs,
retained_pack_stems,
})
}
fn rescue_and_expire_cruft_objects(
database: &FileObjectDatabase,
format: ObjectFormat,
survivors: &mut HashMap<ObjectId, u32>,
expiration: u32,
) -> Result<()> {
let recent: Vec<ObjectId> = survivors
.iter()
.filter(|(_, mtime)| **mtime > expiration)
.map(|(oid, _)| *oid)
.collect();
let mut keep: HashSet<ObjectId> = HashSet::new();
let mut pending: Vec<ObjectId> = recent.clone();
while let Some(oid) = pending.pop() {
if !keep.insert(oid) {
continue;
}
let Ok(object) = database.read_object(&oid) else {
continue;
};
match object.object_type {
ObjectType::Commit => {
if let Ok(commit) = Commit::parse_ref(format, &object.body) {
pending.extend(commit.parents.iter().copied());
pending.push(commit.tree);
}
}
ObjectType::Tree => {
for entry in TreeEntries::new(format, &object.body).flatten() {
if !entry.is_gitlink() {
pending.push(entry.oid);
}
}
}
ObjectType::Tag => {
if let Ok(tag) = Tag::parse_ref(format, &object.body) {
pending.push(tag.object);
}
}
ObjectType::Blob => {}
}
}
survivors.retain(|oid, mtime| *mtime > expiration || keep.contains(oid));
Ok(())
}
pub fn install_cruft_repack_result(
git_dir: &Path,
format: ObjectFormat,
result: &CruftRepackResult,
prune: bool,
) -> Result<()> {
let objects_dir = repository_objects_dir(git_dir);
let pack_dir = objects_dir.join("pack");
fs::create_dir_all(&pack_dir)?;
let new_reachable_name = result
.reachable
.as_ref()
.map(|r| format!("pack-{}.pack", r.pack_checksum.to_hex()));
let new_cruft_name = result
.cruft
.as_ref()
.map(|c| format!("pack-{}.pack", c.checksum.to_hex()));
if let Some(reachable) = result.reachable.as_ref() {
let parsed_index = PackIndex::parse(&reachable.idx, format)?;
let pack_name = format!("pack-{}", reachable.pack_checksum.to_hex());
let reverse_index = sley_pack::PackReverseIndex::write(
format,
&sley_pack::pack_order_index_positions(&parsed_index.entries),
&reachable.pack_checksum,
)?;
write_pack_component(&pack_dir.join(format!("{pack_name}.pack")), &reachable.pack)?;
write_pack_component(&pack_dir.join(format!("{pack_name}.rev")), &reverse_index)?;
write_pack_component(&pack_dir.join(format!("{pack_name}.idx")), &reachable.idx)?;
}
if let Some(cruft) = result.cruft.as_ref() {
let pack_name = format!("pack-{}", cruft.checksum.to_hex());
write_pack_component(&pack_dir.join(format!("{pack_name}.pack")), &cruft.pack)?;
write_pack_component(&pack_dir.join(format!("{pack_name}.rev")), &cruft.rev)?;
write_pack_component(&pack_dir.join(format!("{pack_name}.mtimes")), &cruft.mtimes)?;
write_pack_component(&pack_dir.join(format!("{pack_name}.idx")), &cruft.idx)?;
}
if !prune {
return Ok(());
}
let mut present: HashSet<ObjectId> = HashSet::new();
if let Some(reachable) = result.reachable.as_ref() {
present.extend(reachable.index_entries.iter().map(|e| e.oid));
}
if let Some(cruft) = result.cruft.as_ref() {
present.extend(cruft.oids.iter().copied());
}
let mut removed_stems: HashSet<String> = HashSet::new();
for pack_path in result
.obsolete_packs
.iter()
.chain(result.obsolete_cruft_packs.iter())
{
let file_name = pack_path.file_name().and_then(|n| n.to_str());
if file_name == new_reachable_name.as_deref() || file_name == new_cruft_name.as_deref() {
continue;
}
if let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str())
&& result
.retained_pack_stems
.iter()
.any(|retained| retained == stem)
{
continue;
}
if pack_path.with_extension("keep").exists() {
continue;
}
if pack_path.with_extension("promisor").exists() {
continue;
}
if let Some(stem) = pack_path.file_stem().and_then(|s| s.to_str()) {
removed_stems.insert(stem.to_string());
}
remove_file_if_exists(pack_path)?;
remove_file_if_exists(&pack_path.with_extension("idx"))?;
for ext in ["rev", "mtimes", "bitmap", "promisor"] {
remove_file_if_exists(&pack_path.with_extension(ext))?;
}
}
let loose_now_packed: Vec<ObjectId> = loose_object_ids(&objects_dir, format)?
.into_iter()
.filter(|oid| present.contains(oid))
.collect();
prune_loose_objects(&objects_dir, format, loose_now_packed.iter(), &present)?;
prune_stale_multi_pack_index(&pack_dir, format, &removed_stems)?;
Ok(())
}
pub(crate) fn pack_index_entries_match_writer(
parsed: &[PackIndexEntry],
writer_entries: &[PackIndexEntry],
) -> bool {
if parsed.len() != writer_entries.len() {
return false;
}
let mut writer_entries = writer_entries.iter().collect::<Vec<_>>();
writer_entries.sort_by(|left, right| left.oid.as_bytes().cmp(right.oid.as_bytes()));
parsed.iter().zip(writer_entries).all(|(left, right)| {
left.oid == right.oid && left.crc32 == right.crc32 && left.offset == right.offset
})
}