Skip to main content

session_activity/
maintenance.rs

1use std::collections::HashSet;
2use std::fs;
3use std::io::Read;
4use std::path::{Path, PathBuf};
5use std::sync::atomic::{AtomicU64, Ordering};
6use std::time::{SystemTime, UNIX_EPOCH};
7
8use anyhow::{Result, ensure};
9use serde::{Deserialize, Serialize};
10
11use crate::store::{MAX_BATCH_BYTES, sync_directory, write_batch};
12use crate::{FileEvent, SessionKey, Store};
13
14static NEXT: AtomicU64 = AtomicU64::new(0);
15const CHECKPOINT: &str = ".checkpoint";
16
17/// The checkpoint is the atomic commit point. A reader ignores covered source
18/// batches even if a crash interrupted their deletion. Unreferenced packs are
19/// invisible, including ones written before an interrupted commit.
20#[derive(Default, Serialize, Deserialize)]
21struct Checkpoint {
22    version: u32,
23    #[serde(default)]
24    retained_from: Option<i64>,
25    chunks: Vec<String>,
26    covered: Vec<String>,
27}
28
29#[derive(Debug, Default, Serialize)]
30pub struct MaintenanceReport {
31    pub dry_run: bool,
32    pub batches_before: usize,
33    pub batches_after: usize,
34    pub events_before: usize,
35    pub events_after: usize,
36    pub events_removed: usize,
37    pub retained_from: Option<i64>,
38    pub cleanup_warnings: Vec<String>,
39}
40
41fn checkpoint(dir: &Path) -> Result<Checkpoint> {
42    let path = dir.join(CHECKPOINT);
43    let metadata = match fs::symlink_metadata(&path) {
44        Ok(metadata) => metadata,
45        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Checkpoint::default()),
46        Err(e) => return Err(e.into()),
47    };
48    ensure!(metadata.file_type().is_file(), "checkpoint must be a regular file");
49    let mut bytes = Vec::new();
50    fs::File::open(path)?.take(MAX_BATCH_BYTES + 1).read_to_end(&mut bytes)?;
51    ensure!(bytes.len() as u64 <= MAX_BATCH_BYTES, "checkpoint exceeds size limit");
52    let checkpoint: Checkpoint = serde_json::from_slice(&bytes)?;
53    ensure!(checkpoint.version == 1, "unsupported history checkpoint version");
54    for name in &checkpoint.chunks {
55        ensure!(safe_name(name, ".pack"), "invalid checkpoint chunk name");
56    }
57    for name in &checkpoint.covered {
58        ensure!(safe_name(name, ".json"), "invalid checkpoint source name");
59    }
60    ensure!(
61        checkpoint.chunks.iter().collect::<HashSet<_>>().len() == checkpoint.chunks.len(),
62        "duplicate checkpoint chunk"
63    );
64    Ok(checkpoint)
65}
66
67fn safe_name(name: &str, suffix: &str) -> bool {
68    name.ends_with(suffix) && crate::valid_id(name)
69}
70
71pub(crate) fn retained_from(dir: &Path) -> Result<Option<i64>> {
72    Ok(checkpoint(dir)?.retained_from)
73}
74
75pub(crate) fn active_paths(dir: &Path) -> Result<Vec<PathBuf>> {
76    let checkpoint = checkpoint(dir)?;
77    let covered: HashSet<_> = checkpoint.covered.into_iter().collect();
78    let mut incoming = Vec::new();
79    for entry in fs::read_dir(dir)? {
80        let path = entry?.path();
81        if path.extension().is_some_and(|e| e == "json")
82            && !path.file_name().and_then(|n| n.to_str()).is_some_and(|n| covered.contains(n))
83        {
84            incoming.push(path);
85        }
86    }
87    incoming.sort();
88    let mut paths: Vec<_> = checkpoint.chunks.iter().map(|name| dir.join(name)).collect();
89    paths.extend(incoming);
90    Ok(paths)
91}
92
93impl Store {
94    /// Fixed lower timestamp bound chosen by explicit retention. This also
95    /// prevents a resumed session from reimporting expired transcript events.
96    pub fn retained_from(&self, session: &SessionKey) -> Result<Option<i64>> {
97        let dir = self.directory(session)?;
98        if !dir.try_exists()? {
99            return Ok(None);
100        }
101        let _lock = crate::lock::Lock::acquire(&dir, false)?;
102        retained_from(&dir)
103    }
104    /// Pack history into fewer files, optionally retaining only timestamps at
105    /// or after `before`, in Unix seconds. Preview is read-only apart from the
106    /// advisory lock. Refuses damaged history, unexpected files, and cutoffs in
107    /// the future. Never reconciles or discards raw observations unless an
108    /// explicit retention cutoff is supplied.
109    pub fn maintain(&self, session: &SessionKey, before: Option<i64>, apply: bool) -> Result<MaintenanceReport> {
110        // A cutoff can never be lowered, and appends older than it are dropped,
111        // so a future one (such as milliseconds passed as seconds) would
112        // silently stop recording this session for good.
113        let now = i64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs())?;
114        ensure!(before.is_none_or(|cutoff| cutoff <= now), "retention cutoff is in the future");
115        let dir = self.directory(session)?;
116        let mut report = MaintenanceReport { dry_run: !apply, ..Default::default() };
117        if !dir.try_exists()? {
118            return Ok(report);
119        }
120        let _lock = crate::lock::Lock::acquire(&dir, true)?;
121        let before = before.max(retained_from(&dir)?);
122        report.retained_from = before;
123        let paths = active_paths(&dir)?;
124        // Covered names must match exactly what readers treat as active, or a
125        // batch could be packed while its source stays visible.
126        for path in paths.iter().filter(|path| path.extension().is_some_and(|e| e == "json")) {
127            ensure!(
128                path.file_name().and_then(|n| n.to_str()).is_some_and(|name| safe_name(name, ".json")),
129                "refusing maintenance: unexpected file in history: {}",
130                path.display()
131            );
132        }
133        let history = self.read_unlocked(session)?;
134        ensure!(history.warnings.is_empty(), "refusing maintenance: history contains unreadable or invalid batches");
135        report.batches_before = paths.len();
136        report.events_before = history.events.len();
137        let events: Vec<_> =
138            history.events.into_iter().filter(|e| before.is_none_or(|cutoff| e.timestamp >= cutoff)).collect();
139        report.events_after = events.len();
140        report.events_removed = report.events_before - report.events_after;
141        let chunks = pack(&events)?;
142        report.batches_after = chunks.len();
143        if !apply {
144            return Ok(report);
145        }
146
147        // Include old covered sources left by an interrupted cleanup, but do
148        // not accumulate names for files that were already successfully removed.
149        let mut covered: Vec<_> = paths
150            .iter()
151            .filter(|path| path.extension().is_some_and(|e| e == "json"))
152            .filter_map(|path| path.file_name().and_then(|n| n.to_str()).map(str::to_owned))
153            .collect();
154        covered.extend(checkpoint(&dir)?.covered.into_iter().filter(|name| dir.join(name).exists()));
155        let mut old_packs = Vec::new();
156        for entry in fs::read_dir(&dir)? {
157            let path = entry?.path();
158            let name = path.file_name().and_then(|n| n.to_str()).unwrap_or_default();
159            if safe_name(name, ".pack") {
160                old_packs.push(path);
161            }
162        }
163        covered.sort();
164        covered.dedup();
165        let nonce = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos();
166        let prefix = format!("compact-{nonce}-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed));
167        let mut names = Vec::new();
168        for (index, bytes) in chunks.iter().enumerate() {
169            let name = format!("{prefix}-{index}.pack");
170            publish(&dir, &name, bytes)?;
171            names.push(name);
172        }
173        let checkpoint = Checkpoint { version: 1, retained_from: before, chunks: names, covered: covered.clone() };
174        let bytes = serde_json::to_vec(&checkpoint)?;
175        ensure!(bytes.len() as u64 <= MAX_BATCH_BYTES, "checkpoint exceeds size limit");
176        publish(&dir, CHECKPOINT, &bytes)?;
177
178        // The checkpoint is durable before cleanup. Failure here only leaves
179        // ignored storage behind; report it so a later run can reclaim it.
180        for path in covered.iter().map(|name| dir.join(name)).chain(old_packs) {
181            if let Err(error) = fs::remove_file(&path) {
182                report.cleanup_warnings.push(format!("{}: {error}", path.display()));
183            }
184        }
185        sync_directory(&dir)?;
186        Ok(report)
187    }
188}
189
190fn publish(dir: &Path, name: &str, bytes: &[u8]) -> Result<()> {
191    let temporary = dir.join(format!("{name}.{}-{}.tmp", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed)));
192    let result = (|| {
193        write_batch(&temporary, bytes)?;
194        fs::rename(&temporary, dir.join(name))?;
195        sync_directory(dir)
196    })();
197    if result.is_err() {
198        let _ = fs::remove_file(temporary);
199    }
200    result
201}
202
203fn pack(events: &[FileEvent]) -> Result<Vec<Vec<u8>>> {
204    let mut chunks = Vec::new();
205    let mut chunk = vec![b'['];
206    for event in events {
207        let bytes = serde_json::to_vec(event)?;
208        ensure!(bytes.len() as u64 + 2 <= MAX_BATCH_BYTES, "event exceeds batch size limit");
209        if chunk.len() > 1 && (chunk.len() + bytes.len() + 2) as u64 > MAX_BATCH_BYTES {
210            chunk.push(b']');
211            chunks.push(chunk);
212            chunk = vec![b'['];
213        }
214        if chunk.len() > 1 {
215            chunk.push(b',');
216        }
217        chunk.extend(bytes);
218    }
219    if chunk.len() > 1 {
220        chunk.push(b']');
221        chunks.push(chunk);
222    }
223    Ok(chunks)
224}