use std::collections::{BTreeSet, HashSet};
use std::fs;
use std::path::{Path, PathBuf};
#[cfg(feature = "tokio")]
use std::sync::Arc;
use crate::error::CasError;
use crate::hash::Hash;
use crate::pack::{PackFile, PackIndex};
use crate::store::BlobStore;
pub const TRASH_DIR: &str = "trash";
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct LiveSet {
pub live: BTreeSet<Hash>,
pub missing: Vec<Hash>,
pub roots: usize,
pub scanned: usize,
}
pub fn mark(store: &BlobStore, roots: &HashSet<Hash>) -> Result<LiveSet, CasError> {
let mut present: HashSet<Hash> = store.list_blobs()?.into_iter().collect();
present.extend(store.list_blobs_packed()?);
let mut live = BTreeSet::new();
let mut missing = Vec::new();
for hash in roots {
if present.contains(hash) {
live.insert(*hash);
} else {
missing.push(*hash);
}
}
missing.sort();
Ok(LiveSet {
live,
missing,
roots: roots.len(),
scanned: present.len(),
})
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum SweepMode {
#[default]
DryRun,
Trash,
Delete,
}
#[derive(Clone, Copy, Debug)]
pub struct SweepOptions {
pub mode: SweepMode,
pub rewrite_partial_packs: bool,
}
impl Default for SweepOptions {
fn default() -> Self {
Self {
mode: SweepMode::DryRun,
rewrite_partial_packs: true,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PackRewrite {
pub pack: PathBuf,
pub keep: Vec<Hash>,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct SweepPlan {
pub scanned: usize,
pub loose_present: usize,
pub packed_present: usize,
pub live: usize,
pub garbage: usize,
pub loose_garbage: Vec<Hash>,
pub loose_garbage_bytes: u64,
pub packs_with_garbage: usize,
pub packed_garbage_objects: usize,
pub packs_to_remove: Vec<PathBuf>,
pub packs_to_rewrite: Vec<PackRewrite>,
pub orphan_files: Vec<PathBuf>,
pub unreadable_packs: Vec<PathBuf>,
pub bytes_reclaimable: u64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SweepReport {
pub plan: SweepPlan,
pub mode: SweepMode,
pub executed: bool,
pub loose_removed: usize,
pub packs_removed: usize,
pub packs_rewritten: usize,
pub bytes_reclaimed: u64,
pub trash: Option<PathBuf>,
}
struct PackState {
pack_path: PathBuf,
hashes: BTreeSet<Hash>,
garbage: BTreeSet<Hash>,
bytes: u64,
}
pub fn plan_sweep(store: &BlobStore, live: &HashSet<Hash>) -> Result<SweepPlan, CasError> {
let loose = store.list_blobs()?;
let loose_set: BTreeSet<Hash> = loose.iter().copied().collect();
let pack_dir = store.pack_dir();
let mut packs: Vec<PackState> = Vec::new();
let mut orphan_files: Vec<PathBuf> = Vec::new();
let mut unreadable_packs: Vec<PathBuf> = Vec::new();
for pack_path in PackFile::list_packs(&pack_dir)? {
let idx_path = pack_path.with_extension("idx");
if !idx_path.exists() {
orphan_files.push(pack_path);
continue;
}
match PackIndex::load(&idx_path) {
Ok(index) => {
let hashes: BTreeSet<Hash> = index.hashes().into_iter().collect();
let garbage: BTreeSet<Hash> = hashes
.iter()
.copied()
.filter(|h| !live.contains(h))
.collect();
let bytes = file_len(&pack_path)? + file_len(&idx_path)?;
packs.push(PackState {
pack_path,
hashes,
garbage,
bytes,
});
}
Err(_) => unreadable_packs.push(pack_path),
}
}
let pack_names: Vec<PathBuf> = packs.iter().map(|p| p.pack_path.clone()).collect();
if pack_dir.exists() {
for entry in fs::read_dir(&pack_dir)? {
let entry = entry?;
let path = entry.path();
let is_idx = entry.file_name().to_string_lossy().ends_with(".idx");
if is_idx {
let pack = path.with_extension("pack");
if !pack_names.contains(&pack) && !pack.exists() {
orphan_files.push(path);
}
}
}
}
orphan_files.sort();
let packed_hashes: HashSet<Hash> = packs
.iter()
.flat_map(|p| p.hashes.iter().copied())
.collect();
let packed_present = packed_hashes.len();
let scanned_set: HashSet<Hash> = loose_set.iter().copied().chain(packed_hashes).collect();
let garbage_set: BTreeSet<Hash> = scanned_set
.iter()
.copied()
.filter(|h| !live.contains(h))
.collect();
let loose_garbage: Vec<Hash> = loose_set
.iter()
.copied()
.filter(|h| garbage_set.contains(h))
.collect();
let mut loose_garbage_bytes = 0u64;
for hash in &loose_garbage {
loose_garbage_bytes += file_len(&loose_path(store, hash))?;
}
let packed_garbage_objects = packs.iter().map(|p| p.garbage.len()).sum::<usize>();
let mut covered: HashSet<Hash> = loose_set
.iter()
.copied()
.filter(|h| !garbage_set.contains(h))
.collect();
let mut packs_to_remove = Vec::new();
let mut packs_to_rewrite = Vec::new();
let mut remove_bytes = 0u64;
let mut packs_with_garbage = 0usize;
for pack in &packs {
if pack.garbage.is_empty() {
covered.extend(pack.hashes.iter().copied());
}
}
for pack in &packs {
if pack.garbage.is_empty() {
continue;
}
packs_with_garbage += 1;
let needed: Vec<Hash> = pack
.hashes
.iter()
.copied()
.filter(|h| !garbage_set.contains(h) && !covered.contains(h))
.collect();
if needed.is_empty() {
packs_to_remove.push(pack.pack_path.clone());
remove_bytes += pack.bytes;
} else {
covered.extend(needed.iter().copied());
packs_to_rewrite.push(PackRewrite {
pack: pack.pack_path.clone(),
keep: needed,
});
}
}
let live_present = scanned_set.iter().filter(|h| live.contains(h)).count();
Ok(SweepPlan {
scanned: scanned_set.len(),
loose_present: loose_set.len(),
packed_present,
live: live_present,
garbage: garbage_set.len(),
loose_garbage,
loose_garbage_bytes,
packs_with_garbage,
packed_garbage_objects,
packs_to_remove,
packs_to_rewrite,
orphan_files,
unreadable_packs,
bytes_reclaimable: loose_garbage_bytes + remove_bytes,
})
}
pub fn sweep(
store: &BlobStore,
live: &HashSet<Hash>,
options: SweepOptions,
) -> Result<SweepReport, CasError> {
let plan = plan_sweep(store, live)?;
if options.mode == SweepMode::DryRun {
return Ok(SweepReport {
plan,
mode: options.mode,
executed: false,
loose_removed: 0,
packs_removed: 0,
packs_rewritten: 0,
bytes_reclaimed: 0,
trash: None,
});
}
let trash_root = store.root().join(TRASH_DIR);
let mut loose_removed = 0usize;
let mut packs_removed = 0usize;
let mut packs_rewritten = 0usize;
let mut bytes_reclaimed = 0u64;
if options.rewrite_partial_packs {
for rewrite in &plan.packs_to_rewrite {
let idx_path = rewrite.pack.with_extension("idx");
let index = PackIndex::load(&idx_path)?;
let mut objects = Vec::with_capacity(rewrite.keep.len());
for hash in &rewrite.keep {
let data = PackFile::read_blob(&rewrite.pack, &index, hash)?;
objects.push((*hash, data));
}
let (new_pack, new_idx) = PackFile::create(&store.pack_dir(), &objects)?;
let old_bytes = file_len(&rewrite.pack)? + file_len(&idx_path)?;
let new_bytes = file_len(&new_pack)? + file_len(&new_idx)?;
bytes_reclaimed += old_bytes.saturating_sub(new_bytes);
packs_rewritten += 1;
}
}
for hash in &plan.loose_garbage {
bytes_reclaimed +=
remove_or_trash(store, options.mode, &trash_root, &loose_path(store, hash))?;
loose_removed += 1;
}
for path in &plan.orphan_files {
bytes_reclaimed += remove_or_trash(store, options.mode, &trash_root, path)?;
}
let mut removals: Vec<PathBuf> = plan.packs_to_remove.clone();
if options.rewrite_partial_packs {
removals.extend(plan.packs_to_rewrite.iter().map(|r| r.pack.clone()));
}
for pack_path in &removals {
let idx_path = pack_path.with_extension("idx");
for path in [idx_path, pack_path.clone()] {
if path.exists() {
bytes_reclaimed += remove_or_trash(store, options.mode, &trash_root, &path)?;
}
}
packs_removed += 1;
}
store.invalidate_pack_cache();
Ok(SweepReport {
plan,
mode: options.mode,
executed: true,
loose_removed,
packs_removed,
packs_rewritten,
bytes_reclaimed,
trash: match options.mode {
SweepMode::Trash => Some(trash_root),
_ => None,
},
})
}
#[cfg(feature = "tokio")]
pub async fn mark_async(store: Arc<BlobStore>, roots: HashSet<Hash>) -> Result<LiveSet, CasError> {
tokio::task::spawn_blocking(move || mark(&store, &roots))
.await
.map_err(|e| CasError::TaskJoin(e.to_string()))?
}
#[cfg(feature = "tokio")]
pub async fn sweep_async(
store: Arc<BlobStore>,
live: HashSet<Hash>,
options: SweepOptions,
) -> Result<SweepReport, CasError> {
tokio::task::spawn_blocking(move || sweep(&store, &live, options))
.await
.map_err(|e| CasError::TaskJoin(e.to_string()))?
}
fn loose_path(store: &BlobStore, hash: &Hash) -> PathBuf {
let hex = hash.to_hex();
store.objects_dir().join(&hex[..2]).join(&hex[2..])
}
fn file_len(path: &Path) -> Result<u64, CasError> {
Ok(fs::metadata(path)?.len())
}
fn remove_or_trash(
store: &BlobStore,
mode: SweepMode,
trash_root: &Path,
path: &Path,
) -> Result<u64, CasError> {
let len = file_len(path)?;
match mode {
SweepMode::Delete => fs::remove_file(path)?,
SweepMode::Trash => {
let relative = path
.strip_prefix(store.root())
.map_err(|_| CasError::GcPathEscape(path.display().to_string()))?;
let destination = trash_root.join(relative);
if let Some(parent) = destination.parent() {
fs::create_dir_all(parent)?;
}
if destination.exists() {
let _ = fs::remove_file(&destination);
}
fs::rename(path, &destination)?;
}
SweepMode::DryRun => unreachable!("dry-run sweeps never reach deletion"),
}
Ok(len)
}