use std::collections::{BTreeMap, HashSet, VecDeque};
use std::fs;
use std::io::{BufRead, BufReader, Read, Write};
use std::path::{Path, PathBuf};
use std::time::Duration;
use rusqlite::backup::{Backup, StepResult};
use rusqlite::{params, Connection, OpenFlags, OptionalExtension};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest as _, Sha256};
use uuid::Uuid;
pub const CLASSIFIER: &str = "poll-dispatch-legacy-v1";
pub const HORIZON_SECONDS: i64 = 604_800;
const HORIZON_US: i64 = HORIZON_SECONDS * 1_000_000;
const DAY_US: i64 = 86_400 * 1_000_000;
pub const EXPENDABLE_VERBS: [&str; 6] = [
"comm.inbox",
"comm.cleanup_expired_quarantine",
"whoami",
"comm.probe",
"comm.cursor_get",
"comm.heartbeat",
];
pub const DEFAULT_BATCH_ROWS: usize = 2_000;
pub const MAX_BATCH_ROWS: usize = 10_000;
const GENERIC_DECISIONS: [&str; 3] = ["allow", "deny", "gate_unavailable"];
const EVENTS_COLUMNS: [&str; 18] = [
"id",
"namespace",
"verb",
"substrate",
"actor",
"kind",
"outcome",
"payload",
"payload_schema_version",
"profile_state_version",
"duration_us",
"target_id",
"session_id",
"aggregate_kind",
"aggregate_id",
"created_at",
"op_index",
"ref_resolution",
];
const OBSERVATION_COLUMNS: [&str; 5] =
["event_id", "entity_id", "referent_kind", "role", "position"];
const MAX_BUSY_RETRIES: u32 = 20;
#[derive(Debug, thiserror::Error)]
pub enum PruneError {
#[error("refused-schema: {0}")]
Schema(String),
#[error("digest-mismatch: expected {expected}, recomputed {actual} ({detail})")]
DigestMismatch {
expected: String,
actual: String,
detail: String,
},
#[error("no-backup: {0}")]
NoBackup(String),
#[error("control-mismatch: {0}")]
ControlMismatch(String),
#[error("interrupted-resumable: {0}")]
Interrupted(String),
#[error("invalid: {0}")]
Invalid(String),
#[error("sqlite: {0}")]
Sqlite(#[from] rusqlite::Error),
#[error("io: {0}")]
Io(#[from] std::io::Error),
#[error("json: {0}")]
Json(#[from] serde_json::Error),
}
impl PruneError {
pub fn exit_code(&self) -> i32 {
match self {
Self::Schema(_) => 3,
Self::DigestMismatch { .. } => 4,
Self::NoBackup(_) => 5,
Self::ControlMismatch(_) => 6,
Self::Interrupted(_) => 7,
Self::Invalid(_) | Self::Sqlite(_) | Self::Io(_) | Self::Json(_) => 1,
}
}
}
#[derive(Debug, Clone)]
pub struct PruneTarget {
pub events_db: PathBuf,
pub edge_dbs: Vec<PathBuf>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RetainReason {
Younger,
Malformed,
Aggregate,
NonGenericEnvelope,
RetentionStampMismatch,
ObservationsPresent,
EdgeReferenced,
}
impl RetainReason {
fn as_str(self) -> &'static str {
match self {
Self::Younger => "younger",
Self::Malformed => "malformed",
Self::Aggregate => "aggregate",
Self::NonGenericEnvelope => "non_generic_envelope",
Self::RetentionStampMismatch => "retention_stamp_mismatch",
Self::ObservationsPresent => "observations_present",
Self::EdgeReferenced => "edge_referenced",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Candidate {
pub id: Uuid,
pub verb: u8,
pub namespace: u32,
pub created_at: i64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct KeyCount {
pub kind: String,
pub verb: String,
pub rows_before_cutoff: u64,
pub rows: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct PageInfo {
pub page_size: i64,
pub page_count: i64,
pub freelist_count: i64,
pub auto_vacuum: i64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct StoreSize {
pub db_bytes: u64,
pub wal_bytes: u64,
pub pages: PageInfo,
}
#[derive(Debug, Clone, Serialize)]
pub struct Plan {
pub classifier: &'static str,
pub run_started_us: i64,
pub cutoff_us: i64,
pub schema: BTreeMap<String, Vec<String>>,
pub candidates: u64,
pub candidates_by_verb_and_age: BTreeMap<String, BTreeMap<String, u64>>,
pub retained_by_reason: BTreeMap<String, u64>,
pub protected_backlog: u64,
pub non_candidate_keys: Vec<KeyCount>,
pub rows_by_kind: BTreeMap<String, u64>,
pub total_rows: u64,
pub size: StoreSize,
pub digest: String,
#[serde(skip)]
pub namespaces: Vec<String>,
#[serde(skip)]
pub selected: Vec<Candidate>,
}
fn open_read_only(path: &Path) -> Result<Connection, PruneError> {
if !path.is_file() {
return Err(PruneError::Invalid(format!(
"{} is not a file",
path.display()
)));
}
let conn = Connection::open_with_flags(
path,
OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)?;
conn.busy_timeout(Duration::from_secs(5))?;
conn.pragma_update(None, "query_only", true)?;
Ok(conn)
}
fn open_read_write(path: &Path, busy_timeout: Duration) -> Result<Connection, PruneError> {
if !path.is_file() {
return Err(PruneError::Invalid(format!(
"{} is not a file",
path.display()
)));
}
let conn = Connection::open_with_flags(
path,
OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_NO_MUTEX,
)?;
conn.busy_timeout(busy_timeout)?;
Ok(conn)
}
fn table_columns(conn: &Connection, table: &str) -> Result<Vec<String>, PruneError> {
let mut stmt = conn.prepare("SELECT name FROM pragma_table_info(?1) ORDER BY cid")?;
let names = stmt
.query_map([table], |row| row.get::<_, String>(0))?
.collect::<Result<Vec<_>, _>>()?;
Ok(names)
}
fn check_schema(conn: &Connection) -> Result<BTreeMap<String, Vec<String>>, PruneError> {
let events = table_columns(conn, "events")?;
let observations = table_columns(conn, "event_observations")?;
let mut read = BTreeMap::new();
read.insert("events".to_string(), events.clone());
read.insert("event_observations".to_string(), observations.clone());
if events != EVENTS_COLUMNS || observations != OBSERVATION_COLUMNS {
return Err(PruneError::Schema(format!(
"events columns {events:?}, event_observations columns {observations:?}; this build knows \
events {EVENTS_COLUMNS:?} and event_observations {OBSERVATION_COLUMNS:?}"
)));
}
Ok(read)
}
fn load_edge_endpoints(edge_dbs: &[PathBuf]) -> Result<HashSet<Uuid>, PruneError> {
if edge_dbs.is_empty() {
return Err(PruneError::Invalid(
"at least one edge store is required for the reference check".into(),
));
}
let mut endpoints = HashSet::new();
for path in edge_dbs {
let conn = open_read_only(path)?;
let has_edges: bool = conn.query_row(
"SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'graph_edges')",
[],
|row| row.get(0),
)?;
if !has_edges {
return Err(PruneError::Invalid(format!(
"{} has no graph_edges table, so its references cannot be checked",
path.display()
)));
}
let mut stmt = conn.prepare("SELECT source_id, target_id FROM graph_edges")?;
let mut rows = stmt.query([])?;
while let Some(row) = rows.next()? {
for column in 0..2 {
let value: Option<String> = row.get(column)?;
if let Some(id) = value.as_deref().and_then(|text| Uuid::parse_str(text).ok()) {
endpoints.insert(id);
}
}
}
}
Ok(endpoints)
}
struct AuditRow {
id: String,
verb: String,
created_at: i64,
payload: String,
aggregate_kind: Option<String>,
aggregate_id: Option<String>,
}
fn classify(
row: &AuditRow,
cutoff_us: i64,
has_observations: bool,
edges: &HashSet<Uuid>,
) -> Result<Uuid, RetainReason> {
if row.created_at >= cutoff_us {
return Err(RetainReason::Younger);
}
let id = match Uuid::parse_str(&row.id) {
Ok(id) if id.hyphenated().to_string() == row.id => id,
_ => return Err(RetainReason::Malformed),
};
if row.aggregate_kind.is_some() || row.aggregate_id.is_some() {
return Err(RetainReason::Aggregate);
}
let payload = match serde_json::from_str::<Value>(&row.payload) {
Ok(Value::Object(map)) => map,
_ => return Err(RetainReason::Malformed),
};
let generic = payload.get("gate_impl").is_some_and(Value::is_string)
&& payload
.get("decision")
.and_then(Value::as_str)
.is_some_and(|decision| GENERIC_DECISIONS.contains(&decision))
&& payload.get("verb").and_then(Value::as_str) == Some(row.verb.as_str());
if !generic {
return Err(RetainReason::NonGenericEnvelope);
}
if let Some(stamp) = payload.get("retention") {
let expected = serde_json::json!({
"class": "expendable",
"policy": "poll-dispatch-v1",
"horizon_seconds": HORIZON_SECONDS
});
if *stamp != expected {
return Err(RetainReason::RetentionStampMismatch);
}
}
if has_observations {
return Err(RetainReason::ObservationsPresent);
}
if edges.contains(&id) {
return Err(RetainReason::EdgeReferenced);
}
Ok(id)
}
fn verb_index(verb: &str) -> Option<u8> {
EXPENDABLE_VERBS
.iter()
.position(|known| *known == verb)
.and_then(|index| u8::try_from(index).ok())
}
fn age_bucket(run_started_us: i64, created_at: i64) -> &'static str {
let age = run_started_us.saturating_sub(created_at);
if age < 14 * DAY_US {
"7-14d"
} else if age < 30 * DAY_US {
"14-30d"
} else if age < 90 * DAY_US {
"30-90d"
} else {
">90d"
}
}
pub fn candidate_digest<I: IntoIterator<Item = Uuid>>(ids: I) -> String {
let mut sorted: Vec<Uuid> = ids.into_iter().collect();
sorted.sort_unstable();
sorted.dedup();
let mut hasher = blake3::Hasher::new();
hasher.update(b"[");
for (index, id) in sorted.iter().enumerate() {
if index > 0 {
hasher.update(b",");
}
hasher.update(b"\"");
hasher.update(id.hyphenated().to_string().as_bytes());
hasher.update(b"\"");
}
hasher.update(b"]");
hasher.finalize().to_hex().to_string()
}
fn sidecar_bytes(path: &Path, suffix: &str) -> Result<u64, PruneError> {
let mut name = path.as_os_str().to_os_string();
name.push(suffix);
match fs::metadata(PathBuf::from(name)) {
Ok(metadata) => Ok(metadata.len()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(0),
Err(error) => Err(error.into()),
}
}
fn store_size(conn: &Connection, path: &Path) -> Result<StoreSize, PruneError> {
let pragma = |name: &str| -> Result<i64, PruneError> {
Ok(conn.query_row(&format!("PRAGMA {name}"), [], |row| row.get(0))?)
};
Ok(StoreSize {
db_bytes: fs::metadata(path)?.len(),
wal_bytes: sidecar_bytes(path, "-wal")?,
pages: PageInfo {
page_size: pragma("page_size")?,
page_count: pragma("page_count")?,
freelist_count: pragma("freelist_count")?,
auto_vacuum: pragma("auto_vacuum")?,
},
})
}
fn is_candidate_key(kind: &str, verb: &str) -> bool {
kind == "audit" && verb_index(verb).is_some()
}
fn key_counts(conn: &Connection, cutoff_us: i64) -> Result<(Vec<KeyCount>, u64), PruneError> {
let mut stmt = conn.prepare(
"SELECT kind, verb, SUM(created_at < ?1), COUNT(*) FROM events GROUP BY kind, verb ORDER BY kind, verb",
)?;
let mut rows = stmt.query([cutoff_us])?;
let mut keys = Vec::new();
let mut candidate_key_old = 0_u64;
while let Some(row) = rows.next()? {
let kind: String = row.get(0)?;
let verb: String = row.get(1)?;
let old = u64::try_from(row.get::<_, i64>(2)?).unwrap_or(0);
let all = u64::try_from(row.get::<_, i64>(3)?).unwrap_or(0);
if is_candidate_key(&kind, &verb) {
candidate_key_old += old;
} else {
keys.push(KeyCount {
kind,
verb,
rows_before_cutoff: old,
rows: all,
});
}
}
Ok((keys, candidate_key_old))
}
pub fn plan(target: &PruneTarget) -> Result<Plan, PruneError> {
let now = chrono::Utc::now().timestamp_micros();
plan_at(target, now, now - HORIZON_US)
}
pub fn plan_at(
target: &PruneTarget,
run_started_us: i64,
cutoff_us: i64,
) -> Result<Plan, PruneError> {
let conn = open_read_only(&target.events_db)?;
plan_on(&conn, target, run_started_us, cutoff_us)
}
fn plan_on(
conn: &Connection,
target: &PruneTarget,
run_started_us: i64,
cutoff_us: i64,
) -> Result<Plan, PruneError> {
if cutoff_us > run_started_us - HORIZON_US {
return Err(PruneError::Invalid(format!(
"cutoff {cutoff_us} is later than the run start minus {HORIZON_SECONDS} s ({})",
run_started_us - HORIZON_US
)));
}
let schema = check_schema(conn)?;
let edges = load_edge_endpoints(&target.edge_dbs)?;
let mut namespaces: Vec<String> = Vec::new();
let mut namespace_index: BTreeMap<String, u32> = BTreeMap::new();
let mut selected = Vec::new();
let mut by_verb_and_age: BTreeMap<String, BTreeMap<String, u64>> = BTreeMap::new();
let mut retained: BTreeMap<String, u64> = BTreeMap::new();
let mut observed =
conn.prepare("SELECT EXISTS(SELECT 1 FROM event_observations WHERE event_id = ?1)")?;
let mut stmt = conn.prepare(
"SELECT id, namespace, verb, created_at, payload, aggregate_kind, aggregate_id FROM events \
WHERE kind = 'audit' AND verb IN (?1, ?2, ?3, ?4, ?5, ?6)",
)?;
let mut rows = stmt.query(params![
EXPENDABLE_VERBS[0],
EXPENDABLE_VERBS[1],
EXPENDABLE_VERBS[2],
EXPENDABLE_VERBS[3],
EXPENDABLE_VERBS[4],
EXPENDABLE_VERBS[5],
])?;
while let Some(row) = rows.next()? {
let namespace: String = row.get(1)?;
let audit = AuditRow {
id: row.get(0)?,
verb: row.get(2)?,
created_at: row.get(3)?,
payload: row.get(4)?,
aggregate_kind: row.get(5)?,
aggregate_id: row.get(6)?,
};
let has_observations = audit.created_at < cutoff_us
&& observed.query_row([&audit.id], |found| found.get::<_, bool>(0))?;
match classify(&audit, cutoff_us, has_observations, &edges) {
Ok(id) => {
let verb = verb_index(&audit.verb).ok_or_else(|| {
PruneError::Invalid(format!("selected verb {} is not in the class", audit.verb))
})?;
let ns = match namespace_index.get(&namespace) {
Some(index) => *index,
None => {
let index = u32::try_from(namespaces.len())
.map_err(|_| PruneError::Invalid("too many namespaces".into()))?;
namespaces.push(namespace.clone());
namespace_index.insert(namespace, index);
index
}
};
*by_verb_and_age
.entry(audit.verb.clone())
.or_default()
.entry(age_bucket(run_started_us, audit.created_at).to_string())
.or_default() += 1;
selected.push(Candidate {
id,
verb,
namespace: ns,
created_at: audit.created_at,
});
}
Err(reason) => *retained.entry(reason.as_str().to_string()).or_default() += 1,
}
}
drop(rows);
drop(stmt);
drop(observed);
selected.sort_unstable_by_key(|candidate| (candidate.created_at, candidate.id));
let digest = candidate_digest(selected.iter().map(|candidate| candidate.id));
let protected_backlog = retained
.iter()
.filter(|(reason, _)| reason.as_str() != RetainReason::Younger.as_str())
.map(|(_, count)| *count)
.sum();
let (non_candidate_keys, _) = key_counts(conn, cutoff_us)?;
let rows_by_kind = rows_by_kind(conn)?;
let total_rows = rows_by_kind.values().sum();
let size = store_size(conn, &target.events_db)?;
Ok(Plan {
classifier: CLASSIFIER,
run_started_us,
cutoff_us,
schema,
candidates: selected.len() as u64,
candidates_by_verb_and_age: by_verb_and_age,
retained_by_reason: retained,
protected_backlog,
non_candidate_keys,
rows_by_kind,
total_rows,
size,
digest,
namespaces,
selected,
})
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct BackupReceipt {
pub classifier: String,
pub digest: String,
pub cutoff_us: i64,
pub source: PathBuf,
pub backup_path: PathBuf,
pub bytes: u64,
pub sha256: String,
pub taken_at_us: i64,
pub integrity: Vec<String>,
pub rows_by_kind: BTreeMap<String, u64>,
}
fn rows_by_kind(conn: &Connection) -> Result<BTreeMap<String, u64>, PruneError> {
let mut stmt = conn.prepare("SELECT kind, COUNT(*) FROM events GROUP BY kind")?;
let mut rows = stmt.query([])?;
let mut counts = BTreeMap::new();
while let Some(row) = rows.next()? {
counts.insert(
row.get::<_, String>(0)?,
u64::try_from(row.get::<_, i64>(1)?).unwrap_or(0),
);
}
Ok(counts)
}
fn sha256_file(path: &Path) -> Result<String, PruneError> {
let mut file = fs::File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = vec![0_u8; 1 << 20];
loop {
let read = file.read(&mut buffer)?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
}
Ok(hasher
.finalize()
.iter()
.map(|byte| format!("{byte:02x}"))
.collect())
}
pub fn receipt_path(backup_path: &Path) -> PathBuf {
let mut name = backup_path.as_os_str().to_os_string();
name.push(".receipt.json");
PathBuf::from(name)
}
pub fn backup(target: &PruneTarget, plan: &Plan, dest: &Path) -> Result<BackupReceipt, PruneError> {
let receipt_file = receipt_path(dest);
for path in [dest, receipt_file.as_path()] {
if path.exists() {
return Err(PruneError::Invalid(format!(
"{} already exists",
path.display()
)));
}
}
let source = open_read_only(&target.events_db)?;
check_schema(&source)?;
source.execute_batch("BEGIN")?;
let counted = rows_by_kind(&source)?;
let taken_at_us = chrono::Utc::now().timestamp_micros();
{
let mut copy = Connection::open(dest)?;
{
let backup = Backup::new(&source, &mut copy)?;
let mut attempts = 0_u32;
loop {
match backup.step(-1)? {
StepResult::Done => break,
StepResult::More => {}
StepResult::Busy | StepResult::Locked => {
attempts += 1;
if attempts > MAX_BUSY_RETRIES {
return Err(PruneError::Interrupted(
"the backup could not lock the source; rerun it".into(),
));
}
std::thread::sleep(Duration::from_millis(250));
}
_ => {}
}
}
}
let mode: String = copy.query_row("PRAGMA journal_mode = DELETE", [], |row| row.get(0))?;
if mode != "delete" {
return Err(PruneError::Invalid(format!(
"the copy kept journal mode {mode}"
)));
}
}
source.execute_batch("COMMIT")?;
drop(source);
let copy = open_read_only(dest)?;
let mut stmt = copy.prepare(&format!("PRAGMA integrity_check({})", i32::MAX))?;
let integrity = stmt
.query_map([], |row| row.get::<_, String>(0))?
.collect::<Result<Vec<_>, _>>()?;
drop(stmt);
let copied = rows_by_kind(©)?;
drop(copy);
if integrity != ["ok"] {
return Err(PruneError::NoBackup(format!(
"integrity_check of {} reported {integrity:?}",
dest.display()
)));
}
if copied != counted {
return Err(PruneError::NoBackup(format!(
"rows by kind differ: source {counted:?}, copy {copied:?}"
)));
}
let receipt = BackupReceipt {
classifier: CLASSIFIER.to_string(),
digest: plan.digest.clone(),
cutoff_us: plan.cutoff_us,
source: target.events_db.clone(),
backup_path: dest.to_path_buf(),
bytes: fs::metadata(dest)?.len(),
sha256: sha256_file(dest)?,
taken_at_us,
integrity,
rows_by_kind: counted,
};
let mut file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&receipt_file)?;
file.write_all(&serde_json::to_vec_pretty(&receipt)?)?;
file.sync_all()?;
Ok(receipt)
}
#[derive(Debug, Clone)]
pub struct ApplyOptions {
pub digest: String,
pub cutoff_us: i64,
pub backup_receipt: PathBuf,
pub state_dir: PathBuf,
pub batch_rows: usize,
pub pause: Duration,
pub busy_timeout: Duration,
pub max_batches: Option<usize>,
}
#[derive(Debug, Clone, Serialize)]
pub struct ApplyReport {
pub classifier: &'static str,
pub digest: String,
pub cutoff_us: i64,
pub backup_path: PathBuf,
pub removed: u64,
pub removed_total: u64,
pub skipped: BTreeMap<String, u64>,
pub batches: u64,
pub reconciled: u64,
pub controls_before: Vec<KeyCount>,
pub controls_after: Vec<KeyCount>,
pub candidate_key_rows_before_cutoff: [u64; 2],
pub size_before: StoreSize,
pub size_after: StoreSize,
pub expended_ids_file: PathBuf,
pub expended_ids_note: &'static str,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
enum Ledger {
Intent {
digest: String,
batch: u64,
ids: Vec<LedgerId>,
},
Commit {
digest: String,
batch: u64,
removed: Vec<String>,
skipped: Vec<(String, String)>,
at_us: i64,
},
Reconcile {
digest: String,
batch: u64,
removed: Vec<String>,
at_us: i64,
},
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) struct LedgerId {
pub(crate) id: String,
pub(crate) verb: String,
pub(crate) namespace: String,
pub(crate) created_at: i64,
}
const EXPENDED_NOTE: &str = "interim stand-in for the permanent expended-ID records of ADR-212 §3; not stored in the \
events store and not an archive: the removed payloads are not recoverable from it";
fn append_line(file: &mut fs::File, entry: &Ledger) -> Result<(), PruneError> {
let mut line = serde_json::to_vec(entry)?;
line.push(b'\n');
file.write_all(&line)?;
file.sync_data()?;
Ok(())
}
fn read_receipt(path: &Path, digest: &str, cutoff_us: i64) -> Result<BackupReceipt, PruneError> {
let text = fs::read(path).map_err(|error| {
PruneError::NoBackup(format!(
"cannot read backup receipt {}: {error}",
path.display()
))
})?;
let receipt: BackupReceipt = serde_json::from_slice(&text)
.map_err(|error| PruneError::NoBackup(format!("backup receipt does not parse: {error}")))?;
if receipt.classifier != CLASSIFIER
|| receipt.digest != digest
|| receipt.cutoff_us != cutoff_us
{
return Err(PruneError::NoBackup(format!(
"receipt is for classifier {} digest {} cutoff {}, not this apply's {CLASSIFIER} {digest} {cutoff_us}",
receipt.classifier, receipt.digest, receipt.cutoff_us
)));
}
if receipt.integrity != ["ok"] {
return Err(PruneError::NoBackup(format!(
"receipt records integrity {:?}",
receipt.integrity
)));
}
let bytes = fs::metadata(&receipt.backup_path)
.map_err(|error| {
PruneError::NoBackup(format!(
"backup {} is not readable: {error}",
receipt.backup_path.display()
))
})?
.len();
if bytes != receipt.bytes || sha256_file(&receipt.backup_path)? != receipt.sha256 {
return Err(PruneError::NoBackup(format!(
"backup {} no longer matches its receipt",
receipt.backup_path.display()
)));
}
Ok(receipt)
}
pub fn apply(target: &PruneTarget, options: &ApplyOptions) -> Result<ApplyReport, PruneError> {
apply_at(target, options, chrono::Utc::now().timestamp_micros())
}
pub fn apply_at(
target: &PruneTarget,
options: &ApplyOptions,
run_started_us: i64,
) -> Result<ApplyReport, PruneError> {
if options.batch_rows == 0 || options.batch_rows > MAX_BATCH_ROWS {
return Err(PruneError::Invalid(format!(
"batch rows {} outside 1..={MAX_BATCH_ROWS}",
options.batch_rows
)));
}
let receipt = read_receipt(&options.backup_receipt, &options.digest, options.cutoff_us)?;
if receipt.taken_at_us > run_started_us {
return Err(PruneError::NoBackup(
"the backup receipt is dated after this run's start".into(),
));
}
let conn = open_read_write(&target.events_db, options.busy_timeout)?;
check_schema(&conn)?;
fs::create_dir_all(&options.state_dir)?;
let ledger_path = options.state_dir.join("expended-ids.ndjson");
let mut intents: BTreeMap<u64, Vec<LedgerId>> = BTreeMap::new();
let mut removed_ids: HashSet<Uuid> = HashSet::new();
let mut batched_ids: HashSet<Uuid> = HashSet::new();
let mut last_batch = 0_u64;
if ledger_path.exists() {
for line in BufReader::new(fs::File::open(&ledger_path)?).lines() {
let line = line?;
if line.trim().is_empty() {
continue;
}
let entry: Ledger = serde_json::from_str(&line)?;
let (digest, batch) = match &entry {
Ledger::Intent { digest, batch, .. }
| Ledger::Commit { digest, batch, .. }
| Ledger::Reconcile { digest, batch, .. } => (digest.clone(), *batch),
};
if digest != options.digest {
return Err(PruneError::Invalid(format!(
"{} belongs to a run for digest {digest}",
ledger_path.display()
)));
}
last_batch = last_batch.max(batch);
match entry {
Ledger::Intent { batch, ids, .. } => {
for entry in &ids {
batched_ids.insert(Uuid::parse_str(&entry.id).map_err(|_| {
PruneError::Invalid(format!("ledger id {} is not a UUID", entry.id))
})?);
}
intents.insert(batch, ids);
}
Ledger::Commit { batch, removed, .. }
| Ledger::Reconcile { batch, removed, .. } => {
intents.remove(&batch);
for id in removed {
removed_ids.insert(Uuid::parse_str(&id).map_err(|_| {
PruneError::Invalid(format!("ledger id {id} is not a UUID"))
})?);
}
}
}
}
}
let mut ledger = fs::OpenOptions::new()
.create(true)
.append(true)
.open(&ledger_path)?;
let mut reconciled = 0_u64;
{
let mut present = conn.prepare("SELECT EXISTS(SELECT 1 FROM events WHERE id = ?1)")?;
for (batch, ids) in std::mem::take(&mut intents) {
let mut absent = Vec::new();
for entry in ids {
if !present.query_row([&entry.id], |row| row.get::<_, bool>(0))? {
absent.push(entry.id);
}
}
for id in &absent {
removed_ids.insert(
Uuid::parse_str(id).map_err(|_| {
PruneError::Invalid(format!("ledger id {id} is not a UUID"))
})?,
);
}
reconciled += absent.len() as u64;
append_line(
&mut ledger,
&Ledger::Reconcile {
digest: options.digest.clone(),
batch,
removed: absent,
at_us: chrono::Utc::now().timestamp_micros(),
},
)?;
}
}
let planned = plan_on(&conn, target, run_started_us, options.cutoff_us)?;
let recomputed = candidate_digest(
planned
.selected
.iter()
.map(|candidate| candidate.id)
.chain(removed_ids.iter().copied())
.chain(batched_ids.iter().copied()),
);
if recomputed != options.digest {
return Err(PruneError::DigestMismatch {
expected: options.digest.clone(),
actual: recomputed,
detail: format!(
"{} selected now, {} removed and {} batched earlier by this run",
planned.selected.len(),
removed_ids.len(),
batched_ids.len()
),
});
}
let (controls_before, candidate_old_before) = key_counts(&conn, options.cutoff_us)?;
let size_before = store_size(&conn, &target.events_db)?;
let mut queue: VecDeque<(u64, Vec<Candidate>, u32)> = planned
.selected
.chunks(options.batch_rows)
.enumerate()
.map(|(index, chunk)| (last_batch + 1 + index as u64, chunk.to_vec(), 0))
.collect();
let mut removed = 0_u64;
let mut skipped: BTreeMap<String, u64> = BTreeMap::new();
let mut batches = 0_u64;
let mut stopped = None;
while let Some((batch, rows, retries)) = queue.pop_front() {
if options
.max_batches
.is_some_and(|limit| batches as usize >= limit)
{
stopped = Some(format!(
"stopped after {batches} batch(es) as asked; {} batch(es) remain",
queue.len() + 1
));
break;
}
let ids: Vec<LedgerId> = rows
.iter()
.map(|candidate| LedgerId {
id: candidate.id.hyphenated().to_string(),
verb: EXPENDABLE_VERBS[candidate.verb as usize].to_string(),
namespace: planned.namespaces[candidate.namespace as usize].clone(),
created_at: candidate.created_at,
})
.collect();
let edges = load_edge_endpoints(&target.edge_dbs)?;
let began = conn.execute_batch("BEGIN IMMEDIATE");
if let Err(error) = began {
if is_busy(&error) {
if retries + 1 > MAX_BUSY_RETRIES {
stopped = Some(format!(
"batch {batch} found the store busy {MAX_BUSY_RETRIES} times; rerun to resume"
));
queue.push_front((batch, rows, retries));
break;
}
queue.push_back((batch, rows, retries + 1));
continue;
}
return Err(error.into());
}
append_line(
&mut ledger,
&Ledger::Intent {
digest: options.digest.clone(),
batch,
ids: ids.clone(),
},
)
.inspect_err(|_| {
let _ = conn.execute_batch("ROLLBACK");
})?;
let outcome = delete_batch(&conn, &ids, options.cutoff_us, &edges);
let (batch_removed, batch_skipped) = match outcome {
Ok(result) => result,
Err(error) => {
let _ = conn.execute_batch("ROLLBACK");
return Err(error);
}
};
if let Err(error) = conn.execute_batch("COMMIT") {
let _ = conn.execute_batch("ROLLBACK");
return Err(error.into());
}
removed += batch_removed.len() as u64;
for id in &batch_removed {
if let Ok(id) = Uuid::parse_str(id) {
removed_ids.insert(id);
}
}
for (_, reason) in &batch_skipped {
*skipped.entry(reason.clone()).or_default() += 1;
}
append_line(
&mut ledger,
&Ledger::Commit {
digest: options.digest.clone(),
batch,
removed: batch_removed,
skipped: batch_skipped,
at_us: chrono::Utc::now().timestamp_micros(),
},
)?;
batches += 1;
let _: (i64, i64, i64) = conn.query_row("PRAGMA wal_checkpoint(PASSIVE)", [], |row| {
Ok((row.get(0)?, row.get(1)?, row.get(2)?))
})?;
if !queue.is_empty() && !options.pause.is_zero() {
std::thread::sleep(options.pause);
}
}
let (controls_after, candidate_old_after) = key_counts(&conn, options.cutoff_us)?;
let size_after = store_size(&conn, &target.events_db)?;
let report = ApplyReport {
classifier: CLASSIFIER,
digest: options.digest.clone(),
cutoff_us: options.cutoff_us,
backup_path: receipt.backup_path,
removed,
removed_total: removed_ids.len() as u64,
skipped,
batches,
reconciled,
controls_before,
controls_after,
candidate_key_rows_before_cutoff: [candidate_old_before, candidate_old_after],
size_before,
size_after,
expended_ids_file: ledger_path,
expended_ids_note: EXPENDED_NOTE,
};
check_controls(
&report.controls_before,
&report.controls_after,
report.candidate_key_rows_before_cutoff,
report.removed,
)?;
if let Some(reason) = stopped {
return Err(PruneError::Interrupted(reason));
}
Ok(report)
}
fn is_busy(error: &rusqlite::Error) -> bool {
matches!(
error.sqlite_error_code(),
Some(rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked)
)
}
pub(crate) fn check_controls(
controls_before: &[KeyCount],
controls_after: &[KeyCount],
candidate_key_rows_before_cutoff: [u64; 2],
removed: u64,
) -> Result<(), PruneError> {
let before: BTreeMap<(&str, &str), &KeyCount> = controls_before
.iter()
.map(|key| ((key.kind.as_str(), key.verb.as_str()), key))
.collect();
let after: BTreeMap<(&str, &str), &KeyCount> = controls_after
.iter()
.map(|key| ((key.kind.as_str(), key.verb.as_str()), key))
.collect();
let mut differences = Vec::new();
for (key, was) in &before {
match after.get(key) {
None => differences.push(format!(
"{}/{}: {} rows, now absent",
key.0, key.1, was.rows
)),
Some(now) => {
if now.rows_before_cutoff != was.rows_before_cutoff {
differences.push(format!(
"{}/{}: rows before the cutoff {} -> {}",
key.0, key.1, was.rows_before_cutoff, now.rows_before_cutoff
));
}
if now.rows < was.rows {
differences.push(format!(
"{}/{}: rows {} -> {}",
key.0, key.1, was.rows, now.rows
));
}
}
}
}
for (key, now) in &after {
if !before.contains_key(key) && now.rows_before_cutoff > 0 {
differences.push(format!(
"{}/{}: {} rows before the cutoff appeared",
key.0, key.1, now.rows_before_cutoff
));
}
}
let [old_before, old_after] = candidate_key_rows_before_cutoff;
if old_before.checked_sub(removed) != Some(old_after) {
differences.push(format!(
"candidate-key rows before the cutoff {old_before} -> {old_after}, but {removed} were removed"
));
}
if differences.is_empty() {
Ok(())
} else {
Err(PruneError::ControlMismatch(differences.join("; ")))
}
}
pub(crate) type BatchOutcome = (Vec<String>, Vec<(String, String)>);
pub(crate) fn delete_batch(
conn: &Connection,
ids: &[LedgerId],
cutoff_us: i64,
edges: &HashSet<Uuid>,
) -> Result<BatchOutcome, PruneError> {
let mut read = conn.prepare_cached(
"SELECT kind, verb, created_at, payload, aggregate_kind, aggregate_id, \
EXISTS(SELECT 1 FROM event_observations WHERE event_id = events.id) FROM events WHERE id = ?1",
)?;
let mut delete = conn.prepare_cached(
"DELETE FROM events WHERE id = ?1 AND kind = 'audit' AND verb = ?2 AND created_at = ?3 \
AND created_at < ?4 AND payload = ?5 AND aggregate_kind IS NULL AND aggregate_id IS NULL \
AND NOT EXISTS(SELECT 1 FROM event_observations WHERE event_id = ?1)",
)?;
let mut removed = Vec::new();
let mut skipped = Vec::new();
for entry in ids {
let row = read
.query_row([&entry.id], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, i64>(2)?,
row.get::<_, String>(3)?,
row.get::<_, Option<String>>(4)?,
row.get::<_, Option<String>>(5)?,
row.get::<_, bool>(6)?,
))
})
.optional()?;
let Some((kind, verb, created_at, payload, aggregate_kind, aggregate_id, observed)) = row
else {
skipped.push((entry.id.clone(), "absent".to_string()));
continue;
};
if kind != "audit" || verb != entry.verb || verb_index(&verb).is_none() {
skipped.push((entry.id.clone(), "changed".to_string()));
continue;
}
let audit = AuditRow {
id: entry.id.clone(),
verb,
created_at,
payload,
aggregate_kind,
aggregate_id,
};
if let Err(reason) = classify(&audit, cutoff_us, observed, edges) {
skipped.push((entry.id.clone(), reason.as_str().to_string()));
continue;
}
let changed = delete.execute(params![
entry.id,
audit.verb,
audit.created_at,
cutoff_us,
audit.payload
])?;
if changed == 1 {
removed.push(entry.id.clone());
} else {
skipped.push((entry.id.clone(), "changed".to_string()));
}
}
Ok((removed, skipped))
}