use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use anyhow::{Context, Result, bail};
use gix::bstr::{BString, ByteSlice};
#[derive(Debug, Clone)]
pub struct BlobStat {
pub oid: gix::ObjectId,
pub size: u64,
pub example_path: String,
pub commit_count: usize,
}
#[derive(Debug, Clone)]
pub struct ScanReport {
pub total_blobs: usize,
pub total_bytes: u64,
pub largest: Vec<BlobStat>,
pub commits_scanned: usize,
pub refs_scanned: usize,
}
#[derive(Debug, Clone)]
pub struct PurgeOptions {
pub paths: Vec<String>,
pub apply: bool,
}
#[derive(Debug, Clone)]
pub struct PurgeReport {
pub applied: bool,
pub no_op: bool,
pub commits_total: usize,
pub commits_rewritten: usize,
pub blobs_dropped: usize,
pub bytes_reclaimed: u64,
pub refs_updated: Vec<(String, String, String)>,
pub annotated_tags_skipped: Vec<String>,
pub backup_ref_prefix: Option<String>,
pub backup_file: Option<PathBuf>,
}
struct RepointRef {
name: gix::refs::FullName,
commit: gix::ObjectId,
}
struct RefsResolved {
repointable: Vec<RepointRef>,
annotated_tags: Vec<String>,
tips: Vec<gix::ObjectId>,
}
fn resolve_refs(repo: &gix::Repository) -> Result<RefsResolved> {
let mut repointable = Vec::new();
let mut annotated_tags = Vec::new();
let mut tip_set: HashSet<gix::ObjectId> = HashSet::new();
let platform = repo.references().context("open ref store")?;
for r in platform.all().context("iterate refs")? {
let mut r = match r {
Ok(r) => r,
Err(_) => continue,
};
if matches!(r.target(), gix::refs::TargetRef::Symbolic(_)) {
continue;
}
if r.name().as_bstr().starts_with(b"refs/original/") {
continue;
}
let direct = r.id().detach();
let peeled = match r.peel_to_id() {
Ok(id) => id.detach(),
Err(_) => continue, };
if repo.find_commit(peeled).is_err() {
continue;
}
tip_set.insert(peeled);
let name = r.name().to_owned();
if direct == peeled {
repointable.push(RepointRef {
name,
commit: peeled,
});
} else {
annotated_tags.push(name.as_bstr().to_str_lossy().into_owned());
}
}
Ok(RefsResolved {
repointable,
annotated_tags,
tips: tip_set.into_iter().collect(),
})
}
fn topo_order_parents_first(
repo: &gix::Repository,
tips: &[gix::ObjectId],
) -> Result<Vec<gix::ObjectId>> {
let mut order: Vec<gix::ObjectId> = Vec::new();
let mut visited: HashSet<gix::ObjectId> = HashSet::new();
let mut stack: Vec<(gix::ObjectId, bool)> = Vec::new();
for t in tips {
stack.push((*t, false));
}
while let Some((oid, expanded)) = stack.pop() {
if expanded {
order.push(oid);
continue;
}
if !visited.insert(oid) {
continue;
}
stack.push((oid, true));
let commit = repo
.find_commit(oid)
.with_context(|| format!("find commit {oid}"))?;
for pid in commit.parent_ids() {
let pid = pid.detach();
if !visited.contains(&pid) {
stack.push((pid, false));
}
}
}
Ok(order)
}
fn subtree_blobs<'a>(
repo: &gix::Repository,
tree_oid: gix::ObjectId,
cache: &'a mut HashMap<gix::ObjectId, Vec<(BString, gix::ObjectId, u64)>>,
) -> Result<&'a Vec<(BString, gix::ObjectId, u64)>> {
if !cache.contains_key(&tree_oid) {
let mut out: Vec<(BString, gix::ObjectId, u64)> = Vec::new();
let tree = repo
.find_tree(tree_oid)
.with_context(|| format!("find tree {tree_oid}"))?;
let mut subdirs: Vec<(BString, gix::ObjectId)> = Vec::new();
for entry in tree.iter() {
let entry = entry.context("decode tree entry")?;
let name = entry.inner.filename.to_owned();
let oid = entry.inner.oid.to_owned();
let mode = entry.inner.mode;
if mode.is_tree() {
subdirs.push((name, oid));
} else if mode.is_blob() || mode.is_link() {
let size = repo.find_header(oid).map(|h| h.size()).unwrap_or(0);
out.push((name, oid, size));
}
}
for (dname, doid) in subdirs {
let child = subtree_blobs(repo, doid, cache)?.clone();
for (cpath, coid, csize) in child {
let mut full = dname.clone();
full.push(b'/');
full.extend_from_slice(&cpath);
out.push((full, coid, csize));
}
}
cache.insert(tree_oid, out);
}
Ok(cache.get(&tree_oid).expect("just inserted"))
}
pub fn scan_blobs(repo_root: &Path, path_filter: Option<&str>, top_n: usize) -> Result<ScanReport> {
let repo =
gix::open(repo_root).with_context(|| format!("gix::open {}", repo_root.display()))?;
let refs = resolve_refs(&repo)?;
let commits = topo_order_parents_first(&repo, &refs.tips)?;
let mut cache: HashMap<gix::ObjectId, Vec<(BString, gix::ObjectId, u64)>> = HashMap::new();
let mut agg: HashMap<gix::ObjectId, (u64, String, usize)> = HashMap::new();
for &c in &commits {
let commit = repo
.find_commit(c)
.with_context(|| format!("find commit {c}"))?;
let tree_oid = commit.tree_id().context("commit tree id")?.detach();
let blobs = subtree_blobs(&repo, tree_oid, &mut cache)?.clone();
let mut seen_here: HashSet<gix::ObjectId> = HashSet::new();
for (path, oid, size) in blobs {
let path_str = path.to_str_lossy();
if let Some(f) = path_filter {
if !glob_match(f, &path_str) {
continue;
}
}
let e = agg
.entry(oid)
.or_insert_with(|| (size, path_str.clone().into_owned(), 0));
if seen_here.insert(oid) {
e.2 += 1;
}
}
}
let total_blobs = agg.len();
let total_bytes: u64 = agg.values().map(|(s, _, _)| *s).sum();
let mut largest: Vec<BlobStat> = agg
.into_iter()
.map(|(oid, (size, example_path, commit_count))| BlobStat {
oid,
size,
example_path,
commit_count,
})
.collect();
largest.sort_by(|a, b| b.size.cmp(&a.size).then(a.oid.cmp(&b.oid)));
largest.truncate(top_n);
Ok(ScanReport {
total_blobs,
total_bytes,
largest,
commits_scanned: commits.len(),
refs_scanned: refs.repointable.len() + refs.annotated_tags.len(),
})
}
pub fn purge_paths(repo_root: &Path, opts: &PurgeOptions) -> Result<PurgeReport> {
if opts.paths.is_empty() {
bail!("purge: no --path given (nothing to remove)");
}
let targets: Vec<String> = opts
.paths
.iter()
.map(|p| {
p.trim()
.trim_start_matches("./")
.trim_start_matches('/')
.to_string()
})
.collect();
if targets.iter().any(|p| p.is_empty()) {
bail!("purge: empty path component in --path");
}
let target_set: HashSet<&str> = targets.iter().map(|s| s.as_str()).collect();
let mut repo =
gix::open(repo_root).with_context(|| format!("gix::open {}", repo_root.display()))?;
let _ = repo.committer_or_set_generic_fallback();
let refs = resolve_refs(&repo)?;
let commits = topo_order_parents_first(&repo, &refs.tips)?;
let mut cache: HashMap<gix::ObjectId, Vec<(BString, gix::ObjectId, u64)>> = HashMap::new();
let mut tree_hits: HashMap<gix::ObjectId, bool> = HashMap::new(); let mut target_blobs: HashMap<gix::ObjectId, u64> = HashMap::new(); let mut kept_blobs: HashSet<gix::ObjectId> = HashSet::new();
for &c in &commits {
let commit = repo
.find_commit(c)
.with_context(|| format!("find commit {c}"))?;
let tree_oid = commit.tree_id().context("commit tree id")?.detach();
let blobs = subtree_blobs(&repo, tree_oid, &mut cache)?.clone();
let mut hit = false;
for (path, oid, size) in blobs {
let path_str = path.to_str_lossy();
if target_set.contains(path_str.as_ref()) {
hit = true;
target_blobs.insert(oid, size);
} else {
kept_blobs.insert(oid);
}
}
tree_hits.insert(c, hit);
}
let reclaimed: Vec<(gix::ObjectId, u64)> = target_blobs
.iter()
.filter(|(oid, _)| !kept_blobs.contains(*oid))
.map(|(o, s)| (*o, *s))
.collect();
let bytes_reclaimed: u64 = reclaimed.iter().map(|(_, s)| *s).sum();
let mut changed: HashMap<gix::ObjectId, bool> = HashMap::new();
for &c in &commits {
let commit = repo.find_commit(c)?;
let tree_hit = *tree_hits.get(&c).unwrap_or(&false);
let parent_changed = commit
.parent_ids()
.any(|p| *changed.get(&p.detach()).unwrap_or(&false));
changed.insert(c, tree_hit || parent_changed);
}
let commits_rewritten = changed.values().filter(|v| **v).count();
let would_move: Vec<&RepointRef> = refs
.repointable
.iter()
.filter(|r| *changed.get(&r.commit).unwrap_or(&false))
.collect();
let no_op = commits_rewritten == 0;
if !opts.apply {
let refs_updated = would_move
.iter()
.map(|r| {
(
r.name.as_bstr().to_str_lossy().into_owned(),
r.commit.to_string(),
"(dry-run)".to_string(),
)
})
.collect();
return Ok(PurgeReport {
applied: false,
no_op,
commits_total: commits.len(),
commits_rewritten,
blobs_dropped: reclaimed.len(),
bytes_reclaimed,
refs_updated,
annotated_tags_skipped: refs.annotated_tags,
backup_ref_prefix: None,
backup_file: None,
});
}
if no_op {
return Ok(PurgeReport {
applied: false,
no_op: true,
commits_total: commits.len(),
commits_rewritten: 0,
blobs_dropped: 0,
bytes_reclaimed: 0,
refs_updated: Vec::new(),
annotated_tags_skipped: refs.annotated_tags,
backup_ref_prefix: None,
backup_file: None,
});
}
let (backup_file, backup_ref_prefix) = write_backup(&repo, &refs.repointable)?;
let mut map: HashMap<gix::ObjectId, gix::ObjectId> = HashMap::new();
for &c in &commits {
let is_changed = *changed.get(&c).unwrap_or(&false);
if !is_changed {
map.insert(c, c); continue;
}
let new_oid = rewrite_commit(
&repo,
c,
&targets,
&map,
*tree_hits.get(&c).unwrap_or(&false),
)?;
map.insert(c, new_oid);
}
let mut refs_updated = Vec::new();
let mut edits = Vec::new();
for r in &refs.repointable {
let new = *map.get(&r.commit).unwrap_or(&r.commit);
if new == r.commit {
continue;
}
edits.push(update_ref_edit(r.name.clone(), new));
refs_updated.push((
r.name.as_bstr().to_str_lossy().into_owned(),
r.commit.to_string(),
new.to_string(),
));
}
if !edits.is_empty() {
repo.edit_references(edits)
.context("repoint refs to rewritten tips")?;
}
reconcile_worktree(repo_root, &targets)?;
Ok(PurgeReport {
applied: true,
no_op: false,
commits_total: commits.len(),
commits_rewritten,
blobs_dropped: reclaimed.len(),
bytes_reclaimed,
refs_updated,
annotated_tags_skipped: refs.annotated_tags,
backup_ref_prefix: Some(backup_ref_prefix),
backup_file: Some(backup_file),
})
}
fn rewrite_commit(
repo: &gix::Repository,
c: gix::ObjectId,
targets: &[String],
map: &HashMap<gix::ObjectId, gix::ObjectId>,
tree_hit: bool,
) -> Result<gix::ObjectId> {
let commit = repo
.find_commit(c)
.with_context(|| format!("find commit {c}"))?;
let old_tree = commit.tree_id().context("commit tree id")?.detach();
let new_tree = if tree_hit {
let mut editor = repo.edit_tree(old_tree).context("open tree editor")?;
for p in targets {
editor
.remove(p.as_str())
.with_context(|| format!("tree remove {p}"))?;
}
editor.write().context("write rewritten tree")?.detach()
} else {
old_tree
};
let new_parents: Vec<gix::ObjectId> = commit
.parent_ids()
.map(|p| *map.get(&p.detach()).unwrap_or(&p.detach()))
.collect();
let decoded = commit.decode().context("decode commit")?;
let author: gix::actor::Signature = decoded.author().context("decode author")?.into();
let committer: gix::actor::Signature = decoded.committer().context("decode committer")?.into();
let message: BString = decoded.message.to_owned();
let encoding: Option<BString> = decoded.encoding.map(|e| e.to_owned());
let extra_headers: Vec<(BString, BString)> = decoded
.extra_headers
.iter()
.filter(|(k, _)| k.as_bytes() != b"gpgsig")
.map(|(k, v)| ((*k).to_owned(), v.as_ref().to_owned()))
.collect();
let new_commit = gix::objs::Commit {
tree: new_tree,
parents: new_parents.into_iter().collect(),
author,
committer,
encoding,
message,
extra_headers,
};
let id = repo
.write_object(&new_commit)
.context("write rewritten commit")?
.detach();
Ok(id)
}
fn update_ref_edit(
name: gix::refs::FullName,
new: gix::ObjectId,
) -> gix::refs::transaction::RefEdit {
use gix::refs::Target;
use gix::refs::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog};
RefEdit {
change: Change::Update {
log: LogChange {
mode: RefLog::AndReference,
force_create_reflog: false,
message: "nornir: deep-clean history rewrite".into(),
},
expected: PreviousValue::Any,
new: Target::Object(new),
},
name,
deref: false,
}
}
fn write_backup(repo: &gix::Repository, repointable: &[RepointRef]) -> Result<(PathBuf, String)> {
use gix::refs::Target;
use gix::refs::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog};
let git_dir = repo.git_dir();
let stamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
let file = git_dir.join(format!("nornir-deepclean-backup-{stamp}.txt"));
let mut body = String::from("# nornir deep-clean backup — pre-rewrite ref tips\n");
let mut edits = Vec::new();
for r in repointable {
let name = r.name.as_bstr().to_str_lossy();
body.push_str(&format!("{} {}\n", r.commit, name));
let backup_name = format!("refs/original/{name}");
if let Ok(full) = gix::refs::FullName::try_from(backup_name.as_str()) {
edits.push(RefEdit {
change: Change::Update {
log: LogChange {
mode: RefLog::AndReference,
force_create_reflog: false,
message: "nornir: deep-clean backup".into(),
},
expected: PreviousValue::MustNotExist,
new: Target::Object(r.commit),
},
name: full,
deref: false,
});
}
}
std::fs::write(&file, body).with_context(|| format!("write backup file {}", file.display()))?;
if !edits.is_empty() {
let _ = repo.edit_references(edits);
}
Ok((file, "refs/original/".to_string()))
}
fn reconcile_worktree(repo_root: &Path, targets: &[String]) -> Result<()> {
let repo = gix::open(repo_root).context("reopen for worktree reconcile")?;
let Some(work_dir) = repo.workdir().map(|p| p.to_path_buf()) else {
return Ok(()); };
for p in targets {
let f = work_dir.join(p);
if f.exists() {
let _ = std::fs::remove_file(&f);
}
}
if let Ok(head) = repo.head_commit() {
if let Ok(tree) = head.tree_id() {
let tree = tree.detach();
if let Ok(mut index) = repo.index_from_tree(&tree) {
index
.write(gix::index::write::Options::default())
.context("write reconciled index")?;
}
}
}
Ok(())
}
fn glob_match(pattern: &str, text: &str) -> bool {
fn m(p: &[u8], t: &[u8]) -> bool {
if p.is_empty() {
return t.is_empty();
}
match p[0] {
b'*' => {
if m(&p[1..], t) {
return true;
}
!t.is_empty() && m(p, &t[1..])
}
b'?' => !t.is_empty() && m(&p[1..], &t[1..]),
c => !t.is_empty() && t[0] == c && m(&p[1..], &t[1..]),
}
}
m(pattern.as_bytes(), text.as_bytes())
}
#[cfg(test)]
mod tests {
use super::*;
fn emit(check: &str, ok: bool, detail: &str) -> bool {
let _ = (check, detail);
ok
}
fn make_repo() -> (tempfile::TempDir, u64, String) {
let td = tempfile::tempdir().expect("tempdir");
let root = td.path();
crate::gitio::init(root).expect("init");
std::fs::write(root.join("README.md"), b"# keeper\nhello\n").unwrap();
crate::gitio::commit_all(root, "c1: readme").unwrap();
std::fs::create_dir_all(root.join("docs")).unwrap();
let big = vec![0x42u8; 200_000];
std::fs::write(root.join("docs/book.pdf"), &big).unwrap();
std::fs::create_dir_all(root.join("src")).unwrap();
std::fs::write(root.join("src/main.rs"), b"fn main() {}\n").unwrap();
crate::gitio::commit_all(root, "c2: add book.pdf + src").unwrap();
std::fs::write(root.join("README.md"), b"# keeper\nhello world\n").unwrap();
crate::gitio::commit_all(root, "c3: edit readme").unwrap();
std::fs::write(
root.join("src/main.rs"),
b"fn main() { println!(\"hi\"); }\n",
)
.unwrap();
crate::gitio::commit_all(root, "c4: edit src").unwrap();
let repo = gix::open(root).unwrap();
let oid = repo.write_blob(&big).unwrap().detach(); (td, big.len() as u64, oid.to_string())
}
fn all_entries(root: &Path) -> (HashSet<String>, HashSet<String>) {
let repo = gix::open(root).unwrap();
let refs = resolve_refs(&repo).unwrap();
let commits = topo_order_parents_first(&repo, &refs.tips).unwrap();
let mut cache = HashMap::new();
let mut paths = HashSet::new();
let mut blobs = HashSet::new();
for c in commits {
let commit = repo.find_commit(c).unwrap();
let tree = commit.tree_id().unwrap().detach();
for (p, oid, _) in super::subtree_blobs(&repo, tree, &mut cache)
.unwrap()
.clone()
{
paths.insert(p.to_str_lossy().into_owned());
blobs.insert(oid.to_string());
}
}
(paths, blobs)
}
fn snapshot(root: &Path) -> Vec<(String, Vec<String>)> {
let repo = gix::open(root).unwrap();
let refs = resolve_refs(&repo).unwrap();
let commits = topo_order_parents_first(&repo, &refs.tips).unwrap();
let mut cache = HashMap::new();
let mut out = Vec::new();
for c in commits {
let commit = repo.find_commit(c).unwrap();
let msg = commit
.message_raw()
.unwrap()
.to_str_lossy()
.trim()
.to_string();
let tree = commit.tree_id().unwrap().detach();
let mut paths: Vec<String> = super::subtree_blobs(&repo, tree, &mut cache)
.unwrap()
.iter()
.map(|(p, _, _)| p.to_str_lossy().into_owned())
.collect();
paths.sort();
out.push((msg, paths));
}
out
}
#[test]
fn scan_reports_big_blob_as_largest() {
let (td, size, oid) = make_repo();
let report = scan_blobs(td.path(), None, 5).unwrap();
assert!(!report.largest.is_empty(), "scan found no blobs");
let top = &report.largest[0];
let ok = top.oid.to_string() == oid && top.size == size;
assert!(
emit(
"scan_largest_is_book_pdf",
ok,
&format!(
"top oid={} size={} example={}",
top.oid, top.size, top.example_path
)
),
"expected big blob {oid} ({size}B) as largest, got {} ({}B)",
top.oid,
top.size
);
assert_eq!(top.commit_count, 3, "book.pdf should be in 3 commits");
assert!(top.example_path.ends_with("book.pdf"));
}
#[test]
fn scan_path_filter_narrows() {
let (td, _size, _oid) = make_repo();
let all = scan_blobs(td.path(), None, 100).unwrap();
let pdf = scan_blobs(td.path(), Some("docs/book.pdf"), 100).unwrap();
assert!(
all.total_blobs > pdf.total_blobs,
"filter should shrink the set"
);
assert_eq!(pdf.total_blobs, 1, "only book.pdf matches");
}
#[test]
fn dry_run_mutates_nothing_and_reports() {
let (td, size, _oid) = make_repo();
let before = all_entries(td.path());
let rep = purge_paths(
td.path(),
&PurgeOptions {
paths: vec!["docs/book.pdf".into()],
apply: false,
},
)
.unwrap();
let after = all_entries(td.path());
assert_eq!(before.0, after.0, "dry-run must not change history");
assert!(!rep.applied && !rep.no_op);
assert_eq!(
rep.bytes_reclaimed, size,
"dry-run bytes must equal blob size"
);
assert_eq!(rep.blobs_dropped, 1);
assert!(rep.commits_rewritten >= 3, "commits 2..4 hold the blob");
}
#[test]
fn purge_removes_blob_from_all_history() {
let (td, size, oid) = make_repo();
let before = snapshot(td.path());
let commit_count_before = before.len();
let rep = purge_paths(
td.path(),
&PurgeOptions {
paths: vec!["docs/book.pdf".into()],
apply: true,
},
)
.unwrap();
assert!(rep.applied, "apply should perform the rewrite");
let (paths, blobs) = all_entries(td.path());
let no_pdf_path = !paths.iter().any(|p| p.ends_with("book.pdf"));
let blob_unreachable = !blobs.contains(&oid);
assert!(
emit(
"blob_purged",
no_pdf_path && blob_unreachable,
&format!(
"paths_with_pdf={} blob_present={}",
!no_pdf_path, !blob_unreachable
)
),
"book.pdf still reachable: path_gone={no_pdf_path} blob_gone={blob_unreachable}"
);
let after = snapshot(td.path());
assert_eq!(
after.len(),
commit_count_before,
"commit count must be preserved"
);
let messages_before: Vec<&String> = before.iter().map(|(m, _)| m).collect();
let messages_after: Vec<&String> = after.iter().map(|(m, _)| m).collect();
assert_eq!(messages_before, messages_after, "messages/order preserved");
let keeper_ok = after
.iter()
.all(|(_, ps)| !ps.iter().any(|p| p.ends_with("book.pdf")))
&& after
.iter()
.any(|(_, ps)| ps.iter().any(|p| p == "README.md"))
&& after
.iter()
.any(|(_, ps)| ps.iter().any(|p| p == "src/main.rs"));
assert!(
emit(
"keeper_survived",
keeper_ok,
"README.md + src/main.rs preserved, book.pdf gone"
),
"keeper content missing"
);
let repo = gix::open(td.path()).unwrap();
let head_tree = repo.head_commit().unwrap().tree().unwrap();
let readme = head_tree
.lookup_entry_by_path("README.md")
.unwrap()
.unwrap();
let data = repo.find_object(readme.id()).unwrap().data.clone();
assert_eq!(data, b"# keeper\nhello world\n", "README bytes changed");
assert_eq!(rep.bytes_reclaimed, size);
assert_eq!(rep.blobs_dropped, 1);
assert!(rep.backup_ref_prefix.is_some(), "apply must write a backup");
assert!(
rep.backup_file.as_ref().unwrap().exists(),
"backup file must exist"
);
}
#[test]
fn purge_is_idempotent() {
let (td, _size, _oid) = make_repo();
purge_paths(
td.path(),
&PurgeOptions {
paths: vec!["docs/book.pdf".into()],
apply: true,
},
)
.unwrap();
let after_first = snapshot(td.path());
let rep2 = purge_paths(
td.path(),
&PurgeOptions {
paths: vec!["docs/book.pdf".into()],
apply: true,
},
)
.unwrap();
let after_second = snapshot(td.path());
let noop = rep2.no_op
&& !rep2.applied
&& rep2.commits_rewritten == 0
&& after_first == after_second;
assert!(
emit(
"idempotent",
noop,
&format!("no_op={} rewritten={}", rep2.no_op, rep2.commits_rewritten)
),
"second purge should be a no-op, got {rep2:?}"
);
}
#[test]
fn glob_match_basics() {
assert!(glob_match("docs/book.pdf", "docs/book.pdf"));
assert!(glob_match("*.pdf", "docs/book.pdf"));
assert!(glob_match("docs/*", "docs/book.pdf"));
assert!(!glob_match("docs/book.pdf", "src/main.rs"));
assert!(glob_match("src/?ain.rs", "src/main.rs"));
}
}