use std::path::Path;
use crate::compaction::SegmentMerger;
use crate::error::{MemdError, Result};
use crate::index::Bm25Index;
#[derive(Debug, Default)]
pub struct MaintenanceReport {
pub tenants_scanned: usize,
pub orphan_snapshots_removed: u64,
pub orphan_bytes_freed: u64,
pub orphan_snapshots_failed: u64,
pub legacy_mapping_converted: u64,
pub segments_merged: u64,
pub sparse_segments_before: u64,
pub sparse_segments_after: u64,
}
pub fn run(
data_dir: &Path,
tenant_filter: Option<&str>,
dry_run: bool,
aggressive: bool,
) -> Result<MaintenanceReport> {
let mut report = MaintenanceReport::default();
let tenants_root = data_dir.join("tenants");
if !tenants_root.exists() {
return Ok(report);
}
let entries = std::fs::read_dir(&tenants_root).map_err(|e| {
MemdError::StorageError(format!(
"read tenants dir {}: {}",
tenants_root.display(),
e
))
})?;
for entry in entries {
let entry = entry.map_err(|e| MemdError::StorageError(format!("tenant entry: {}", e)))?;
let file_type = entry
.file_type()
.map_err(|e| MemdError::StorageError(format!("file_type: {}", e)))?;
if !file_type.is_dir() {
continue;
}
let tenant = entry.file_name().to_string_lossy().to_string();
if let Some(filter) = tenant_filter {
if tenant != filter {
continue;
}
}
report.tenants_scanned += 1;
sweep_warm_index(&entry.path().join("warm_index"), dry_run, &mut report)?;
}
if aggressive {
let (before, after) = merge_sparse_segments(data_dir, dry_run)?;
report.sparse_segments_before = before;
report.sparse_segments_after = after;
report.segments_merged = before.saturating_sub(after);
}
Ok(report)
}
fn merge_sparse_segments(data_dir: &Path, dry_run: bool) -> Result<(u64, u64)> {
let sparse_path = data_dir.join("sparse_index");
if !sparse_path.exists() {
return Ok((0, 0));
}
if dry_run {
let Some(index) = Bm25Index::with_path_read_only(sparse_path)? else {
return Ok((0, 0));
};
let before = index.segment_count()? as u64;
let after = if index.total_docs()? == 0 {
0
} else {
before.min(1)
};
return Ok((before, after));
}
let index = Bm25Index::with_path(Some(sparse_path))?;
let result = SegmentMerger::new().merge(&index)?;
Ok((result.segments_before as u64, result.segments_after as u64))
}
fn sweep_warm_index(warm: &Path, dry_run: bool, report: &mut MaintenanceReport) -> Result<()> {
if !warm.exists() {
return Ok(());
}
let entries = std::fs::read_dir(warm).map_err(|e| {
MemdError::StorageError(format!("read warm_index {}: {}", warm.display(), e))
})?;
for entry in entries {
let entry =
entry.map_err(|e| MemdError::StorageError(format!("warm_index entry: {}", e)))?;
let file_type = match entry.file_type() {
Ok(ft) => ft,
Err(_) => continue,
};
if !file_type.is_file() {
continue;
}
let name = entry.file_name();
let name = name.to_string_lossy();
if !(name.starts_with("graph-")
&& (name.ends_with(".hnsw.graph") || name.ends_with(".hnsw.data")))
{
continue;
}
let bytes = entry.metadata().map(|m| m.len()).unwrap_or(0);
if dry_run {
report.orphan_snapshots_removed += 1;
report.orphan_bytes_freed += bytes;
} else {
match std::fs::remove_file(entry.path()) {
Ok(()) => {
report.orphan_snapshots_removed += 1;
report.orphan_bytes_freed += bytes;
}
Err(e) => {
report.orphan_snapshots_failed += 1;
tracing::warn!(
path = ?entry.path(),
error = %e,
"failed to remove orphan snapshot"
);
}
}
}
}
Ok(())
}
pub fn render_report(report: &MaintenanceReport, dry_run: bool, aggressive: bool) -> String {
let action = if dry_run { "would_remove" } else { "removed" };
let mut out = String::new();
out.push_str(&format!("tenants_scanned: {}\n", report.tenants_scanned));
out.push_str(&format!(
"{action}_orphan_snapshots: {}\n",
report.orphan_snapshots_removed
));
out.push_str(&format!(
"{action}_orphan_bytes: {}\n",
report.orphan_bytes_freed
));
if !dry_run {
out.push_str(&format!(
"orphan_snapshots_failed: {}\n",
report.orphan_snapshots_failed
));
}
if aggressive {
out.push_str(&format!(
"legacy_mapping_converted: {}\n",
report.legacy_mapping_converted
));
out.push_str(&format!(
"sparse_segments_before: {}\n",
report.sparse_segments_before
));
out.push_str(&format!(
"sparse_segments_after: {}\n",
report.sparse_segments_after
));
out.push_str(&format!("segments_merged: {}\n", report.segments_merged));
}
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::index::SparseIndex;
use crate::types::{ChunkId, TenantId};
use tempfile::tempdir;
#[test]
fn aggressive_maintenance_force_merges_sparse_segments() {
let dir = tempdir().unwrap();
std::fs::create_dir_all(dir.path().join("tenants/test")).unwrap();
let sparse_path = dir.path().join("sparse_index");
let tenant = TenantId::new("test").unwrap();
{
let index = Bm25Index::with_path(Some(sparse_path.clone())).unwrap();
for i in 0..12 {
index
.insert(
&tenant,
&ChunkId::new(),
&[format!("maintenance segment {i}")],
)
.unwrap();
index.commit().unwrap();
}
}
let preview = run(dir.path(), None, true, true).unwrap();
assert!(preview.segments_merged > 0);
assert_eq!(preview.sparse_segments_after, 1);
let applied = run(dir.path(), None, false, true).unwrap();
assert_eq!(applied.segments_merged, preview.segments_merged);
assert_eq!(
applied.sparse_segments_before,
preview.sparse_segments_before
);
assert_eq!(applied.sparse_segments_after, 1);
let index = Bm25Index::with_path_read_only(sparse_path)
.unwrap()
.unwrap();
assert_eq!(index.segment_count().unwrap(), 1);
}
}