gravityfile_core/
history.rs

1//! Scan history: one summary line per full scan, appended to `history.jsonl`
2//! in the data directory, so a place's growth can be followed over time.
3//!
4//! The file is JSON lines, read back leniently: a line that cannot be parsed,
5//! perhaps written by a newer version, is skipped rather than failing the
6//! whole history. It is trimmed to its newest half when it passes 4 MiB.
7
8use std::io::Write;
9use std::path::{Path, PathBuf};
10use std::time::{Duration, SystemTime};
11
12use serde::{Deserialize, Serialize};
13
14use crate::FileTree;
15
16/// The size at which the history is trimmed to its newest half.
17const MAX_BYTES: u64 = 4 * 1024 * 1024;
18
19/// How many of the root's largest entries a record keeps.
20pub const TOP_ENTRIES: usize = 20;
21
22/// The summary of one scan.
23#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
24pub struct Record {
25    /// When the scan finished.
26    pub at: SystemTime,
27    #[serde(with = "crate::path_serde")]
28    pub root: PathBuf,
29    /// Sizes are file lengths rather than disk usage.
30    pub apparent: bool,
31    pub size: u64,
32    pub files: u64,
33    pub directories: u64,
34    /// The largest entries directly under the root, largest first.
35    pub top: Vec<TopEntry>,
36}
37
38/// One of the largest entries under the root.
39#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
40pub struct TopEntry {
41    #[serde(with = "crate::path_serde")]
42    pub name: PathBuf,
43    pub size: u64,
44}
45
46impl Record {
47    /// The summary of `tree`.
48    pub fn of(tree: &FileTree) -> Self {
49        let mut top: Vec<TopEntry> = tree
50            .root
51            .children
52            .iter()
53            .map(|child| TopEntry {
54                name: PathBuf::from(child.file_name()),
55                size: child.size,
56            })
57            .collect();
58        top.sort_by(|a, b| b.size.cmp(&a.size).then_with(|| a.name.cmp(&b.name)));
59        top.truncate(TOP_ENTRIES);
60        Self {
61            at: tree.scanned_at,
62            root: tree.root_path.clone(),
63            apparent: tree.config.apparent_size,
64            size: tree.root.size,
65            files: tree.stats.total_files,
66            directories: tree.stats.total_dirs.saturating_sub(1),
67            top,
68        }
69    }
70
71    /// Whether the two records measured the same place the same way, so
72    /// their sizes can be compared.
73    pub fn comparable(&self, other: &Self) -> bool {
74        self.root == other.root && self.apparent == other.apparent
75    }
76
77    /// The top entry whose size changed most since `earlier`, and by how
78    /// much; entries missing on one side count as zero there.
79    pub fn largest_change<'a>(&'a self, earlier: &'a Self) -> Option<(&'a Path, i128)> {
80        let size_in = |record: &Self, name: &Path| {
81            record
82                .top
83                .iter()
84                .find(|entry| entry.name == name)
85                .map_or(0, |entry| i128::from(entry.size))
86        };
87        self.top
88            .iter()
89            .map(|entry| entry.name.as_path())
90            .chain(earlier.top.iter().map(|entry| entry.name.as_path()))
91            .map(|name| (name, size_in(self, name) - size_in(earlier, name)))
92            .filter(|(_, change)| *change != 0)
93            .max_by_key(|(name, change)| (change.unsigned_abs(), std::cmp::Reverse(*name)))
94    }
95}
96
97/// Append `record` to the history at `file`, creating it and its directory.
98pub fn append(file: &Path, record: &Record) -> std::io::Result<()> {
99    if let Some(directory) = file.parent() {
100        std::fs::create_dir_all(directory)?;
101    }
102    let mut line = serde_json::to_vec(record).map_err(std::io::Error::other)?;
103    line.push(b'\n');
104    // One write per record keeps concurrent appends from interleaving.
105    std::fs::OpenOptions::new()
106        .create(true)
107        .append(true)
108        .open(file)?
109        .write_all(&line)?;
110    if std::fs::metadata(file)?.len() > MAX_BYTES {
111        trim(file)?;
112    }
113    Ok(())
114}
115
116/// Keep the newer half of the history.
117fn trim(file: &Path) -> std::io::Result<()> {
118    let text = std::fs::read_to_string(file)?;
119    let lines: Vec<&str> = text.lines().collect();
120    let kept = lines[lines.len() / 2..].join("\n") + "\n";
121    let staged = file.with_extension("jsonl.tmp");
122    std::fs::write(&staged, kept)?;
123    std::fs::rename(staged, file)
124}
125
126/// Every record for `root`, oldest first, or for every root when `None`. A
127/// missing history is empty.
128pub fn read(file: &Path, root: Option<&Path>) -> std::io::Result<Vec<Record>> {
129    let text = match std::fs::read_to_string(file) {
130        Ok(text) => text,
131        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
132        Err(error) => return Err(error),
133    };
134    let mut records: Vec<Record> = text
135        .lines()
136        .filter_map(|line| serde_json::from_str::<Record>(line).ok())
137        .filter(|record| root.is_none_or(|root| record.root == root))
138        .collect();
139    // Appends from several processes can land slightly out of order.
140    records.sort_by_key(|record| record.at);
141    Ok(records)
142}
143
144/// The latest record comparable with `record` from before it, at least
145/// `min_gap` earlier so a rescan moments later is not the baseline.
146pub fn previous<'a>(
147    records: &'a [Record],
148    record: &Record,
149    min_gap: Duration,
150) -> Option<&'a Record> {
151    records
152        .iter()
153        .rev()
154        .filter(|earlier| earlier.comparable(record))
155        .find(|earlier| {
156            earlier.at < record.at
157                && record
158                    .at
159                    .duration_since(earlier.at)
160                    .is_ok_and(|gap| gap >= min_gap)
161        })
162}
163
164#[cfg(test)]
165mod tests {
166    use super::*;
167
168    fn record(root: &str, days: u64, size: u64, top: &[(&str, u64)]) -> Record {
169        Record {
170            at: SystemTime::UNIX_EPOCH + Duration::from_secs(86_400 * days),
171            root: PathBuf::from(root),
172            apparent: false,
173            size,
174            files: 1,
175            directories: 0,
176            top: top
177                .iter()
178                .map(|(name, size)| TopEntry {
179                    name: PathBuf::from(name),
180                    size: *size,
181                })
182                .collect(),
183        }
184    }
185
186    #[test]
187    fn records_append_and_read_back_per_root_in_time_order() {
188        let directory = tempfile::tempdir().unwrap();
189        let file = directory.path().join("data/history.jsonl");
190        let later = record("/home", 9, 300, &[("a", 200)]);
191        let earlier = record("/home", 2, 100, &[("a", 50)]);
192        let other = record("/srv", 5, 7, &[]);
193        for record in [&later, &other, &earlier] {
194            append(&file, record).unwrap();
195        }
196        // A line from elsewhere is skipped, not fatal.
197        std::fs::OpenOptions::new()
198            .append(true)
199            .open(&file)
200            .unwrap()
201            .write_all(b"{\"unexpected\":true}\nnot json\n")
202            .unwrap();
203        assert_eq!(
204            read(&file, Some(Path::new("/home"))).unwrap(),
205            [earlier, later]
206        );
207        assert_eq!(read(&file, None).unwrap().len(), 3);
208        assert!(
209            read(&directory.path().join("missing"), None)
210                .unwrap()
211                .is_empty()
212        );
213    }
214
215    #[test]
216    fn the_history_is_trimmed_to_its_newer_half() {
217        let directory = tempfile::tempdir().unwrap();
218        let file = directory.path().join("history.jsonl");
219        // Large records, so the limit is reached in about a hundred appends.
220        let bulky: Vec<(String, u64)> = (0..TOP_ENTRIES)
221            .map(|n| (format!("{n:0>2000}"), 1))
222            .collect();
223        let top: Vec<(&str, u64)> = bulky
224            .iter()
225            .map(|(name, size)| (name.as_str(), *size))
226            .collect();
227        let length = || std::fs::metadata(&file).map_or(0, |metadata| metadata.len());
228        let mut day = 0;
229        loop {
230            let before = length();
231            day += 1;
232            append(&file, &record("/home", day, day, &top)).unwrap();
233            if length() < before {
234                break;
235            }
236            assert!(day < 1_000, "the history never shrank");
237        }
238        let kept = read(&file, None).unwrap();
239        assert!(length() <= MAX_BYTES);
240        assert!(kept.len() < day as usize / 2 + 2, "about half was dropped");
241        assert_eq!(kept.last().unwrap().size, day, "the newest record survives");
242    }
243
244    #[test]
245    fn the_previous_record_is_comparable_and_old_enough() {
246        let records = [
247            record("/home", 1, 100, &[]),
248            record("/srv", 2, 5, &[]),
249            Record {
250                apparent: true,
251                ..record("/home", 3, 90, &[])
252            },
253            record("/home", 4, 150, &[]),
254        ];
255        // Ten minutes after the latest scan.
256        let mut now = record("/home", 4, 160, &[]);
257        now.at += Duration::from_secs(600);
258        let previous =
259            |gap| super::previous(&records, &now, Duration::from_secs(gap)).map(|r| r.size);
260        assert_eq!(previous(0), Some(150));
261        assert_eq!(
262            previous(3600),
263            Some(100),
264            "skips a scan moments ago and other measures"
265        );
266        assert_eq!(previous(86_400 * 10), None);
267    }
268
269    #[test]
270    fn the_largest_change_is_found_among_top_entries() {
271        let earlier = record("/home", 1, 0, &[("Library", 100), ("old", 40), ("same", 5)]);
272        let later = record("/home", 2, 0, &[("Library", 130), ("new", 60), ("same", 5)]);
273        assert_eq!(later.largest_change(&earlier), Some((Path::new("new"), 60)));
274        assert_eq!(later.largest_change(&later), None);
275    }
276}
gravityfile_core/history.rs

gravityfile_core/
history.rs
