use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::error::Result;
use crate::store::manifest::{FileRecord, Manifest, RetentionRecord};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Policy {
pub max_age_days: Option<u32>,
pub max_bytes: Option<u64>,
}
impl Policy {
pub fn is_unbounded(&self) -> bool {
self.max_age_days.is_none() && self.max_bytes.is_none()
}
pub fn describe(&self) -> String {
let days = |d: u32| {
if d == 1 {
"1 day".to_string()
} else {
format!("{d} days")
}
};
match (self.max_age_days, self.max_bytes) {
(None, None) => "unbounded".to_string(),
(Some(d), None) => format!("keep {}", days(d)),
(None, Some(b)) => format!("keep under {}", size(b)),
(Some(d), Some(b)) => format!("keep {} and under {}", days(d), size(b)),
}
}
}
fn size(bytes: u64) -> String {
const UNITS: [(&str, f64); 4] = [("GB", 1e9), ("MB", 1e6), ("kB", 1e3), ("B", 1.0)];
for (name, scale) in UNITS {
if bytes as f64 >= scale {
return format!("{:.1} {name}", bytes as f64 / scale);
}
}
format!("{bytes} B")
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Removed {
pub files: usize,
pub rows: u64,
pub bytes: u64,
pub oldest_kept_wall: Option<i64>,
}
impl std::fmt::Display for Removed {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if self.files == 0 {
return write!(f, "nothing to remove");
}
write!(
f,
"removed {} file(s), {} rows, {:.2} MB",
self.files,
self.rows,
self.bytes as f64 / 1e6
)
}
}
pub fn select(files: &[FileRecord], policy: Policy, now_wall: i64) -> Vec<FileRecord> {
if policy.is_unbounded() || files.is_empty() {
return Vec::new();
}
let mut ordered: Vec<FileRecord> = files.to_vec();
ordered.sort_by_key(|f| f.last_recv_wall);
let mut doomed: Vec<FileRecord> = Vec::new();
let mut keep: Vec<&FileRecord> = Vec::new();
if let Some(days) = policy.max_age_days {
let cutoff = now_wall - (days as i64) * 86_400 * 1_000_000_000;
for f in &ordered {
if f.last_recv_wall < cutoff {
doomed.push(f.clone());
} else {
keep.push(f);
}
}
} else {
keep.extend(ordered.iter());
}
if let Some(budget) = policy.max_bytes {
let mut total: u64 = keep.iter().map(|f| f.bytes).sum();
let mut i = 0;
while total > budget && i < keep.len() {
total -= keep[i].bytes;
doomed.push(keep[i].clone());
i += 1;
}
}
doomed
}
pub fn enforce(root: impl AsRef<Path>, policy: Policy, now_wall: i64) -> Result<Removed> {
let root = root.as_ref();
if policy.is_unbounded() {
return Ok(Removed::default());
}
let mut manifest = Manifest::open(root)?;
let doomed = select(manifest.files(), policy, now_wall);
if doomed.is_empty() {
return Ok(Removed {
oldest_kept_wall: manifest.files().iter().map(|f| f.first_recv_wall).min(),
..Removed::default()
});
}
let mut removed = Removed {
files: doomed.len(),
rows: doomed.iter().map(|f| f.rows).sum(),
bytes: doomed.iter().map(|f| f.bytes).sum(),
oldest_kept_wall: None,
};
let reason = if policy.max_age_days.is_some() {
"aged out of the retention window"
} else {
"over the size budget"
};
manifest.record_retention(RetentionRecord {
removed: doomed.iter().map(|f| f.path.clone()).collect(),
reason: reason.to_string(),
policy: policy.describe(),
rows_removed: removed.rows,
bytes_removed: removed.bytes,
at_wall: now_wall,
})?;
for f in &doomed {
let path = root.join(&f.path);
if let Err(e) = std::fs::remove_file(&path)
&& e.kind() != std::io::ErrorKind::NotFound
{
tracing::warn!(path = %path.display(), error = %e, "could not remove a retired file");
}
}
manifest.sync_dir()?;
removed.oldest_kept_wall = manifest.files().iter().map(|f| f.first_recv_wall).min();
Ok(removed)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::{BookLevel, Symbol, VenueId};
const DAY: i64 = 86_400 * 1_000_000_000;
fn file(name: &str, last: i64, bytes: u64) -> FileRecord {
FileRecord {
path: name.to_string(),
venue: VenueId::Kraken,
symbol: Symbol::new("BTC", "USD"),
date: "2026-08-25".to_string(),
book_level: BookLevel::L2,
feed_depth: Some(10),
rows: 100,
bytes,
first_recv_wall: last - 1_000_000_000,
last_recv_wall: last,
closed_wall: last,
}
}
#[test]
fn an_unbounded_policy_removes_nothing() {
let files = vec![file("a", 0, 1_000), file("b", 100 * DAY, 1_000)];
assert!(select(&files, Policy::default(), 1_000 * DAY).is_empty());
assert_eq!(Policy::default().describe(), "unbounded");
}
#[test]
fn files_older_than_the_window_go_and_the_rest_stay() {
let now = 30 * DAY;
let files = vec![
file("old", now - 20 * DAY, 1),
file("edge", now - 6 * DAY, 1),
file("fresh", now - DAY, 1),
];
let policy = Policy {
max_age_days: Some(7),
max_bytes: None,
};
let doomed: Vec<String> = select(&files, policy, now)
.into_iter()
.map(|f| f.path)
.collect();
assert_eq!(doomed, vec!["old"]);
}
#[test]
fn a_file_survives_until_everything_in_it_has_aged_out() {
let now = 30 * DAY;
let mut straddling = file("straddling", now - 6 * DAY, 1);
straddling.first_recv_wall = now - 20 * DAY;
let policy = Policy {
max_age_days: Some(7),
max_bytes: None,
};
assert!(select(&[straddling], policy, now).is_empty());
}
#[test]
fn the_byte_budget_drops_the_oldest_first() {
let now = 10 * DAY;
let files = vec![
file("oldest", now - 3 * DAY, 500),
file("middle", now - 2 * DAY, 500),
file("newest", now - DAY, 500),
];
let policy = Policy {
max_age_days: None,
max_bytes: Some(1_000),
};
let doomed: Vec<String> = select(&files, policy, now)
.into_iter()
.map(|f| f.path)
.collect();
assert_eq!(
doomed,
vec!["oldest"],
"the newest data is the likeliest wanted"
);
}
#[test]
fn both_bounds_apply_together() {
let now = 30 * DAY;
let files = vec![
file("ancient", now - 40 * DAY, 900),
file("old", now - 5 * DAY, 900),
file("new", now - DAY, 900),
];
let policy = Policy {
max_age_days: Some(7),
max_bytes: Some(1_000),
};
let doomed: Vec<String> = select(&files, policy, now)
.into_iter()
.map(|f| f.path)
.collect();
assert_eq!(doomed, vec!["ancient", "old"]);
}
#[test]
fn a_policy_describes_itself_for_the_record() {
let p = Policy {
max_age_days: Some(30),
max_bytes: Some(50_000_000_000),
};
assert_eq!(p.describe(), "keep 30 days and under 50.0 GB");
assert_eq!(
Policy {
max_age_days: None,
max_bytes: Some(3_000_000)
}
.describe(),
"keep under 3.0 MB"
);
assert_eq!(
Policy {
max_age_days: Some(1),
max_bytes: None
}
.describe(),
"keep 1 day"
);
}
#[test]
fn enforcing_records_what_went_and_why() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
let now = 30 * DAY;
let mut manifest = Manifest::open(root).unwrap();
for (name, last) in [("old.parquet", now - 20 * DAY), ("new.parquet", now - DAY)] {
std::fs::write(root.join(name), b"x").unwrap();
manifest.record_file(file(name, last, 1)).unwrap();
}
drop(manifest);
let policy = Policy {
max_age_days: Some(7),
max_bytes: None,
};
let removed = enforce(root, policy, now).unwrap();
assert_eq!(removed.files, 1);
assert!(!root.join("old.parquet").exists());
assert!(root.join("new.parquet").exists());
let reopened = Manifest::open(root).unwrap();
assert_eq!(reopened.files().len(), 1);
let record = &reopened.retentions()[0];
assert_eq!(record.removed, vec!["old.parquet"]);
assert_eq!(record.policy, "keep 7 days");
assert!(record.reason.contains("aged out"));
assert!(reopened.truncations().is_empty());
}
#[test]
fn enforcing_twice_is_not_an_error() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
let now = 30 * DAY;
let mut manifest = Manifest::open(root).unwrap();
std::fs::write(root.join("old.parquet"), b"x").unwrap();
manifest
.record_file(file("old.parquet", now - 20 * DAY, 1))
.unwrap();
drop(manifest);
let policy = Policy {
max_age_days: Some(7),
max_bytes: None,
};
assert_eq!(enforce(root, policy, now).unwrap().files, 1);
assert_eq!(enforce(root, policy, now).unwrap().files, 0);
}
}