use ipfrs_core::{Cid, Ipld, Result};
use ipfrs_storage::BlockStoreTrait;
use std::collections::HashSet;
use std::sync::Arc;
use std::time::{Duration, Instant};
use crate::pin::PinManager;
#[derive(Debug, Clone)]
pub struct GcStats {
pub blocks_collected: u64,
pub bytes_freed: u64,
pub blocks_marked: u64,
pub blocks_scanned: u64,
pub duration: Duration,
pub cancelled: bool,
}
#[derive(Debug, Clone)]
pub struct GcConfig {
pub min_age_seconds: u64,
pub max_blocks_per_run: u64,
pub dry_run: bool,
}
impl Default for GcConfig {
fn default() -> Self {
Self {
min_age_seconds: 3600, max_blocks_per_run: 0, dry_run: false,
}
}
}
pub struct GarbageCollector<S: BlockStoreTrait> {
storage: Arc<S>,
pin_manager: Arc<PinManager>,
}
impl<S: BlockStoreTrait> GarbageCollector<S> {
pub fn new(storage: Arc<S>, pin_manager: Arc<PinManager>) -> Self {
Self {
storage,
pin_manager,
}
}
pub async fn collect(&self, config: GcConfig) -> Result<GcStats> {
let start_time = Instant::now();
let mut stats = GcStats {
blocks_collected: 0,
bytes_freed: 0,
blocks_marked: 0,
blocks_scanned: 0,
duration: Duration::ZERO,
cancelled: false,
};
let reachable = self.mark_reachable().await?;
stats.blocks_marked = reachable.len() as u64;
let all_cids = self.storage.list_cids()?;
stats.blocks_scanned = all_cids.len() as u64;
let mut collected_count = 0u64;
for cid in all_cids {
if config.max_blocks_per_run > 0 && collected_count >= config.max_blocks_per_run {
break;
}
if reachable.contains(&cid) {
continue;
}
if let Some(block) = self.storage.get(&cid).await? {
let size = block.data().len() as u64;
if !config.dry_run {
self.storage.delete(&cid).await?;
}
stats.bytes_freed += size;
collected_count += 1;
}
}
stats.blocks_collected = collected_count;
stats.duration = start_time.elapsed();
Ok(stats)
}
async fn mark_reachable(&self) -> Result<HashSet<Cid>> {
let mut reachable = HashSet::new();
let mut to_visit = Vec::new();
let pins = self.pin_manager.list();
for pin in pins {
reachable.insert(pin.cid);
to_visit.push(pin.cid);
}
while let Some(cid) = to_visit.pop() {
if let Some(block) = self.storage.get(&cid).await? {
if let Ok(ipld) = Ipld::from_dag_cbor(block.data()) {
for link_cid in ipld.links() {
if reachable.insert(link_cid) {
to_visit.push(link_cid);
}
}
}
}
}
Ok(reachable)
}
pub fn count_unpinned(&self) -> Result<usize> {
let all_cids = self.storage.list_cids()?;
let unpinned = all_cids
.iter()
.filter(|cid| !self.pin_manager.is_pinned(cid))
.count();
Ok(unpinned)
}
pub async fn estimate_freeable_space(&self) -> Result<u64> {
let all_cids = self.storage.list_cids()?;
let mut total = 0u64;
for cid in all_cids {
if !self.pin_manager.is_pinned(&cid) {
if let Some(block) = self.storage.get(&cid).await? {
total += block.data().len() as u64;
}
}
}
Ok(total)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::pin::{PinManager, PinType};
use bytes::Bytes;
use ipfrs_core::Block;
use ipfrs_storage::{BlockStoreConfig, SledBlockStore};
#[tokio::test]
async fn test_gc_basic() {
let path = std::env::temp_dir().join(format!("ipfrs_gc_test_{}", std::process::id()));
let _ = std::fs::remove_dir_all(&path);
let config = BlockStoreConfig {
path: path.clone(),
..Default::default()
};
let storage = Arc::new(
SledBlockStore::new(config).expect("test: block store creation should succeed"),
);
let pin_manager = Arc::new(PinManager::new());
let gc = GarbageCollector::new(storage.clone(), pin_manager.clone());
let block1 =
Block::new(Bytes::from("test data 1")).expect("test: block creation should succeed");
let block2 =
Block::new(Bytes::from("test data 2")).expect("test: block creation should succeed");
let cid1 = *block1.cid();
let cid2 = *block2.cid();
storage
.put(&block1)
.await
.expect("test: put block1 should succeed");
storage
.put(&block2)
.await
.expect("test: put block2 should succeed");
pin_manager
.pin(cid1, PinType::Direct, None)
.expect("test: pin should succeed");
let config = GcConfig {
dry_run: false,
..Default::default()
};
let stats = gc
.collect(config)
.await
.expect("test: GC collect should succeed");
assert_eq!(stats.blocks_collected, 1);
assert!(stats.bytes_freed > 0);
assert!(storage
.has(&cid1)
.await
.expect("test: storage has check should succeed"));
assert!(!storage
.has(&cid2)
.await
.expect("test: storage has check should succeed"));
let _ = std::fs::remove_dir_all(&path);
}
#[tokio::test]
async fn test_gc_dry_run() {
let path = std::env::temp_dir().join(format!("ipfrs_gc_test_dry_{}", std::process::id()));
let _ = std::fs::remove_dir_all(&path);
let config = BlockStoreConfig {
path: path.clone(),
..Default::default()
};
let storage = Arc::new(
SledBlockStore::new(config).expect("test: block store creation should succeed"),
);
let pin_manager = Arc::new(PinManager::new());
let gc = GarbageCollector::new(storage.clone(), pin_manager.clone());
let block =
Block::new(Bytes::from("test data")).expect("test: block creation should succeed");
let cid = *block.cid();
storage.put(&block).await.expect("test: put should succeed");
let config = GcConfig {
dry_run: true,
..Default::default()
};
let stats = gc
.collect(config)
.await
.expect("test: GC dry run should succeed");
assert_eq!(stats.blocks_collected, 1);
assert!(storage
.has(&cid)
.await
.expect("test: storage has check should succeed"));
let _ = std::fs::remove_dir_all(&path);
}
fn unique_path(tag: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!("ipfrs-gc-{}-{}", tag, std::process::id()))
}
#[tokio::test]
async fn test_gc_dry_run_no_delete() {
let path = unique_path("dry-run-no-del");
let _ = std::fs::remove_dir_all(&path);
let config = BlockStoreConfig {
path: path.clone(),
..Default::default()
};
let storage = Arc::new(
SledBlockStore::new(config).expect("test: block store creation should succeed"),
);
let pin_manager = Arc::new(PinManager::new());
let gc = GarbageCollector::new(storage.clone(), pin_manager.clone());
let block =
Block::new(Bytes::from("orphan block")).expect("test: block creation should succeed");
let orphan_cid = *block.cid();
storage.put(&block).await.expect("test: put should succeed");
let stats = gc
.collect(GcConfig {
dry_run: true,
..Default::default()
})
.await
.expect("test: GC dry run should succeed");
assert_eq!(
stats.blocks_collected, 1,
"dry-run must report 1 collectable block"
);
assert!(
storage
.has(&orphan_cid)
.await
.expect("test: storage has check should succeed"),
"dry-run must not delete the block"
);
let _ = std::fs::remove_dir_all(&path);
}
#[tokio::test]
async fn test_gc_collects_orphans() {
let path = unique_path("collects-orphans");
let _ = std::fs::remove_dir_all(&path);
let config = BlockStoreConfig {
path: path.clone(),
..Default::default()
};
let storage = Arc::new(
SledBlockStore::new(config).expect("test: block store creation should succeed"),
);
let pin_manager = Arc::new(PinManager::new());
let gc = GarbageCollector::new(storage.clone(), pin_manager.clone());
let block_a =
Block::new(Bytes::from("pinned data")).expect("test: block creation should succeed");
let block_b =
Block::new(Bytes::from("orphan data")).expect("test: block creation should succeed");
let cid_a = *block_a.cid();
let cid_b = *block_b.cid();
storage
.put(&block_a)
.await
.expect("test: put block_a should succeed");
storage
.put(&block_b)
.await
.expect("test: put block_b should succeed");
pin_manager
.pin(cid_a, PinType::Direct, None)
.expect("test: pin should succeed");
let stats = gc
.collect(GcConfig {
dry_run: false,
min_age_seconds: 0,
..Default::default()
})
.await
.expect("test: GC collect should succeed");
assert_eq!(stats.blocks_collected, 1, "one orphan should be collected");
assert!(
storage
.has(&cid_a)
.await
.expect("test: storage has check should succeed"),
"pinned block must survive"
);
assert!(
!storage
.has(&cid_b)
.await
.expect("test: storage has check should succeed"),
"orphan block must be deleted"
);
let _ = std::fs::remove_dir_all(&path);
}
#[tokio::test]
async fn test_gc_count_unpinned() {
let path = std::env::temp_dir().join(format!("ipfrs_gc_test_count_{}", std::process::id()));
let _ = std::fs::remove_dir_all(&path);
let config = BlockStoreConfig {
path: path.clone(),
..Default::default()
};
let storage = Arc::new(
SledBlockStore::new(config).expect("test: block store creation should succeed"),
);
let pin_manager = Arc::new(PinManager::new());
let gc = GarbageCollector::new(storage.clone(), pin_manager.clone());
for i in 0..3 {
let block = Block::new(Bytes::from(format!("data {}", i)))
.expect("test: block creation should succeed");
let cid = *block.cid();
storage.put(&block).await.expect("test: put should succeed");
if i == 0 {
pin_manager
.pin(cid, PinType::Direct, None)
.expect("test: pin should succeed");
}
}
let count = gc
.count_unpinned()
.expect("test: count_unpinned should succeed");
assert_eq!(count, 2);
let _ = std::fs::remove_dir_all(&path);
}
}