Skip to main content

shaul_core/
cache.rs

1//! Content-addressed cache for verified downloads.
2//!
3//! The cache is opt-in and only used when both a checksum and a size are
4//! known. Entries are keyed by `<algorithm>/<hex>` and can be hardlinked or
5//! copied into a task destination. A cache hit is verified again before it is
6//! used, so a corrupted cache entry cannot silently satisfy a download.
7
8use std::path::{Path, PathBuf};
9use std::time::{Duration, SystemTime};
10
11use serde::Serialize;
12use uuid::Uuid;
13
14use crate::checksum::{Checksum, compute_file_checksum};
15use crate::error::{Error, Result};
16
17/// A content-addressed cache rooted at a directory.
18#[derive(Clone, Debug)]
19pub struct ContentCache {
20    root: PathBuf,
21}
22
23/// Aggregate size information for a content cache.
24#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
25pub struct CacheStats {
26    /// Number of regular cache entries.
27    pub entries: u64,
28    /// Total size of those entries in bytes.
29    pub bytes: u64,
30    /// Number of incomplete temporary files left behind by an interrupted
31    /// insertion.
32    pub temp_files: u64,
33}
34
35/// Result of pruning a content cache by age and/or total size.
36#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
37pub struct CachePruneReport {
38    /// Number of entries removed.
39    pub entries_removed: u64,
40    /// Bytes removed.
41    pub bytes_removed: u64,
42    /// Entries left after pruning.
43    pub entries_remaining: u64,
44    /// Bytes left after pruning.
45    pub bytes_remaining: u64,
46}
47
48impl ContentCache {
49    /// Creates a cache rooted at `root`, creating the directory if needed.
50    pub fn new(root: impl Into<PathBuf>) -> Result<Self> {
51        let root = root.into();
52        std::fs::create_dir_all(&root)?;
53        Ok(Self { root })
54    }
55
56    /// Returns the cache root.
57    pub fn root(&self) -> &Path {
58        &self.root
59    }
60
61    /// Returns the path an entry with `checksum` would use.
62    pub fn entry_path(&self, checksum: &Checksum) -> Result<PathBuf> {
63        validate_checksum(checksum)?;
64        let prefix = &checksum.hex[..2];
65        Ok(self
66            .root
67            .join(checksum.algorithm.as_str())
68            .join(prefix)
69            .join(&checksum.hex))
70    }
71
72    /// Looks up an entry and verifies its digest and size. A mismatching entry
73    /// is removed and reported as a miss.
74    pub async fn lookup_verified(&self, checksum: &Checksum, size: u64) -> Result<Option<PathBuf>> {
75        let path = self.entry_path(checksum)?;
76        let metadata = match tokio::fs::metadata(&path).await {
77            Ok(metadata) => metadata,
78            Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
79            Err(error) => return Err(Error::Io(error)),
80        };
81        if !metadata.is_file() || metadata.len() != size {
82            let _ = tokio::fs::remove_file(&path).await;
83            return Ok(None);
84        }
85        let actual = compute_file_checksum(path.clone(), checksum.algorithm).await?;
86        if checksum.matches(&actual) {
87            Ok(Some(path))
88        } else {
89            let _ = tokio::fs::remove_file(&path).await;
90            Ok(None)
91        }
92    }
93
94    /// Inserts a verified file into the cache. If an entry already exists with
95    /// the expected size it is reused.
96    pub async fn insert(&self, checksum: &Checksum, source: &Path, size: u64) -> Result<PathBuf> {
97        let target = self.entry_path(checksum)?;
98        if source == target {
99            return Ok(target);
100        }
101        let source_metadata = tokio::fs::metadata(source).await?;
102        if !source_metadata.is_file() || source_metadata.len() != size {
103            return Err(Error::Config(format!(
104                "cache source {} has the wrong size",
105                source.display()
106            )));
107        }
108        match tokio::fs::metadata(&target).await {
109            Ok(metadata) if metadata.is_file() && metadata.len() == size => {
110                return Ok(target);
111            }
112            Ok(_) => {
113                let _ = tokio::fs::remove_file(&target).await;
114            }
115            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
116            Err(error) => return Err(Error::Io(error)),
117        }
118
119        if let Some(parent) = target.parent() {
120            tokio::fs::create_dir_all(parent).await?;
121        }
122        let temp = self.root.join(format!(".tmp-{}", Uuid::new_v4()));
123        if tokio::fs::hard_link(source, &temp).await.is_err() {
124            tokio::fs::copy(source, &temp).await?;
125        }
126        match tokio::fs::rename(&temp, &target).await {
127            Ok(()) => Ok(target),
128            Err(_) => {
129                let _ = tokio::fs::remove_file(&temp).await;
130                if let Ok(metadata) = tokio::fs::metadata(&target).await
131                    && metadata.is_file()
132                    && metadata.len() == size
133                {
134                    return Ok(target);
135                }
136                Err(Error::Io(std::io::Error::other(format!(
137                    "could not publish cache entry {}",
138                    target.display()
139                ))))
140            }
141        }
142    }
143
144    /// Hardlinks or copies a cached file to `destination`. The caller is
145    /// responsible for applying overwrite/rename conflict policy first.
146    pub async fn materialize(&self, cached: &Path, destination: &Path, size: u64) -> Result<()> {
147        if cached == destination {
148            return Ok(());
149        }
150        if let Some(parent) = destination.parent() {
151            tokio::fs::create_dir_all(parent).await?;
152        }
153        let _ = tokio::fs::remove_file(destination).await;
154        match tokio::fs::hard_link(cached, destination).await {
155            Ok(()) => Ok(()),
156            Err(_) => {
157                let temp = destination.with_file_name(format!(
158                    ".{}.tmp-{}",
159                    destination
160                        .file_name()
161                        .and_then(|name| name.to_str())
162                        .unwrap_or("download"),
163                    Uuid::new_v4()
164                ));
165                tokio::fs::copy(cached, &temp).await?;
166                match tokio::fs::rename(&temp, destination).await {
167                    Ok(()) => Ok(()),
168                    Err(error) => {
169                        let _ = tokio::fs::remove_file(&temp).await;
170                        Err(Error::Io(error))
171                    }
172                }
173            }
174        }
175        .and_then(|()| {
176            let metadata = std::fs::metadata(destination)?;
177            if metadata.len() == size {
178                Ok(())
179            } else {
180                Err(Error::Config(format!(
181                    "cached file {} has the wrong size",
182                    destination.display()
183                )))
184            }
185        })
186    }
187
188    /// Returns aggregate entry and byte counts.
189    pub async fn stats(&self) -> Result<CacheStats> {
190        let root = self.root.clone();
191        tokio::task::spawn_blocking(move || stats_blocking(&root))
192            .await
193            .map_err(|error| Error::Other(format!("cache stats task failed: {error}")))?
194    }
195
196    /// Removes all cache entries and temporary files, returning the number of
197    /// bytes removed.
198    pub async fn clear(&self) -> Result<u64> {
199        let root = self.root.clone();
200        tokio::task::spawn_blocking(move || clear_blocking(&root))
201            .await
202            .map_err(|error| Error::Other(format!("cache clean task failed: {error}")))?
203    }
204
205    /// Removes incomplete temporary files, returning the number removed.
206    pub async fn prune_temp(&self) -> Result<u64> {
207        let root = self.root.clone();
208        tokio::task::spawn_blocking(move || prune_temp_blocking(&root))
209            .await
210            .map_err(|error| Error::Other(format!("cache prune task failed: {error}")))?
211    }
212
213    /// Removes entries older than `older_than` and, when `max_bytes` is set,
214    /// the oldest remaining entries until the cache fits the limit. Passing
215    /// both as `None` only reports current totals.
216    pub async fn prune(
217        &self,
218        older_than: Option<Duration>,
219        max_bytes: Option<u64>,
220    ) -> Result<CachePruneReport> {
221        let root = self.root.clone();
222        tokio::task::spawn_blocking(move || prune_blocking(&root, older_than, max_bytes, true))
223            .await
224            .map_err(|error| Error::Other(format!("cache prune task failed: {error}")))?
225    }
226
227    /// Reports what [`ContentCache::prune`] would remove without deleting
228    /// anything.
229    pub async fn prune_preview(
230        &self,
231        older_than: Option<Duration>,
232        max_bytes: Option<u64>,
233    ) -> Result<CachePruneReport> {
234        let root = self.root.clone();
235        tokio::task::spawn_blocking(move || prune_blocking(&root, older_than, max_bytes, false))
236            .await
237            .map_err(|error| Error::Other(format!("cache prune preview task failed: {error}")))?
238    }
239}
240
241fn validate_checksum(checksum: &Checksum) -> Result<()> {
242    if checksum.hex.len() != checksum.algorithm.hex_len() || hex::decode(&checksum.hex).is_err() {
243        return Err(Error::Config(format!(
244            "invalid {} checksum: {}",
245            checksum.algorithm, checksum.hex
246        )));
247    }
248    Ok(())
249}
250
251fn stats_blocking(root: &Path) -> Result<CacheStats> {
252    let mut stats = CacheStats::default();
253    walk(root, &mut stats)?;
254    Ok(stats)
255}
256
257fn walk(path: &Path, stats: &mut CacheStats) -> Result<()> {
258    let entries = match std::fs::read_dir(path) {
259        Ok(entries) => entries,
260        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
261        Err(error) => return Err(Error::Io(error)),
262    };
263    for entry in entries {
264        let entry = entry?;
265        let path = entry.path();
266        let metadata = std::fs::symlink_metadata(&path)?;
267        if metadata.file_type().is_symlink() {
268            continue;
269        }
270        if metadata.is_dir() {
271            walk(&path, stats)?;
272        } else if metadata.is_file() {
273            let temp = path
274                .file_name()
275                .and_then(|name| name.to_str())
276                .is_some_and(|name| name.starts_with(".tmp-"));
277            if temp {
278                stats.temp_files += 1;
279            } else {
280                stats.entries += 1;
281                stats.bytes += metadata.len();
282            }
283        }
284    }
285    Ok(())
286}
287
288fn clear_blocking(root: &Path) -> Result<u64> {
289    let stats = stats_blocking(root)?;
290    let entries = match std::fs::read_dir(root) {
291        Ok(entries) => entries,
292        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
293        Err(error) => return Err(Error::Io(error)),
294    };
295    for entry in entries {
296        let entry = entry?;
297        let path = entry.path();
298        let metadata = std::fs::symlink_metadata(&path)?;
299        if metadata.is_dir() && !metadata.file_type().is_symlink() {
300            std::fs::remove_dir_all(&path)?;
301        } else {
302            std::fs::remove_file(&path)?;
303        }
304    }
305    Ok(stats.bytes)
306}
307
308fn prune_blocking(
309    root: &Path,
310    older_than: Option<Duration>,
311    max_bytes: Option<u64>,
312    apply: bool,
313) -> Result<CachePruneReport> {
314    let mut entries = Vec::new();
315    collect_entries(root, &mut entries)?;
316    let now = SystemTime::now();
317    let mut remove = vec![false; entries.len()];
318
319    if let Some(age) = older_than {
320        for (index, entry) in entries.iter().enumerate() {
321            if entry
322                .modified
323                .and_then(|modified| now.duration_since(modified).ok())
324                .is_some_and(|elapsed| elapsed >= age)
325            {
326                remove[index] = true;
327            }
328        }
329    }
330
331    let mut remaining: u64 = entries
332        .iter()
333        .enumerate()
334        .filter(|(index, _)| !remove[*index])
335        .map(|(_, entry)| entry.size)
336        .sum();
337    if let Some(max_bytes) = max_bytes {
338        let mut candidates: Vec<usize> = entries
339            .iter()
340            .enumerate()
341            .filter(|(index, _)| !remove[*index])
342            .map(|(index, _)| index)
343            .collect();
344        candidates.sort_by_key(|index| entries[*index].modified.unwrap_or(SystemTime::UNIX_EPOCH));
345        for index in candidates {
346            if remaining <= max_bytes {
347                break;
348            }
349            remove[index] = true;
350            remaining = remaining.saturating_sub(entries[index].size);
351        }
352    }
353
354    let mut report = CachePruneReport::default();
355    for (index, entry) in entries.iter().enumerate() {
356        if !remove[index] {
357            continue;
358        }
359        if apply
360            && let Err(error) = std::fs::remove_file(&entry.path)
361            && error.kind() != std::io::ErrorKind::NotFound
362        {
363            return Err(Error::Io(error));
364        }
365        report.entries_removed += 1;
366        report.bytes_removed += entry.size;
367    }
368
369    let total_entries = entries.len() as u64;
370    let total_bytes: u64 = entries.iter().map(|entry| entry.size).sum();
371    report.entries_remaining = total_entries.saturating_sub(report.entries_removed);
372    report.bytes_remaining = total_bytes.saturating_sub(report.bytes_removed);
373    Ok(report)
374}
375
376struct CacheEntryMeta {
377    path: PathBuf,
378    size: u64,
379    modified: Option<SystemTime>,
380}
381
382fn collect_entries(root: &Path, entries: &mut Vec<CacheEntryMeta>) -> Result<()> {
383    let read_dir = match std::fs::read_dir(root) {
384        Ok(read_dir) => read_dir,
385        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
386        Err(error) => return Err(Error::Io(error)),
387    };
388    for entry in read_dir {
389        let entry = entry?;
390        let path = entry.path();
391        let metadata = std::fs::symlink_metadata(&path)?;
392        if metadata.file_type().is_symlink() {
393            continue;
394        }
395        if metadata.is_dir() {
396            collect_entries(&path, entries)?;
397        } else if metadata.is_file()
398            && !path
399                .file_name()
400                .and_then(|name| name.to_str())
401                .is_some_and(|name| name.starts_with(".tmp-"))
402        {
403            entries.push(CacheEntryMeta {
404                modified: metadata.modified().ok(),
405                path,
406                size: metadata.len(),
407            });
408        }
409    }
410    Ok(())
411}
412
413fn prune_temp_blocking(root: &Path) -> Result<u64> {
414    let entries = match std::fs::read_dir(root) {
415        Ok(entries) => entries,
416        Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
417        Err(error) => return Err(Error::Io(error)),
418    };
419    let mut removed = 0;
420    for entry in entries {
421        let entry = entry?;
422        let path = entry.path();
423        let metadata = std::fs::symlink_metadata(&path)?;
424        if metadata.is_file()
425            && path
426                .file_name()
427                .and_then(|name| name.to_str())
428                .is_some_and(|name| name.starts_with(".tmp-"))
429        {
430            std::fs::remove_file(&path)?;
431            removed += 1;
432        }
433    }
434    Ok(removed)
435}
436
437#[cfg(test)]
438mod tests {
439    use super::*;
440    use crate::checksum::ChecksumAlgorithm;
441
442    fn checksum(data: &[u8]) -> Checksum {
443        Checksum {
444            algorithm: ChecksumAlgorithm::Sha256,
445            hex: {
446                use sha2::{Digest, Sha256};
447                hex::encode(Sha256::digest(data))
448            },
449        }
450    }
451
452    #[tokio::test]
453    async fn inserts_and_looks_up_verified_content() {
454        let dir = tempfile::tempdir().unwrap();
455        let source = dir.path().join("source.bin");
456        let data = b"cached content";
457        tokio::fs::write(&source, data).await.unwrap();
458        let checksum = checksum(data);
459        let cache = ContentCache::new(dir.path().join("cache")).unwrap();
460
461        let inserted = cache
462            .insert(&checksum, &source, data.len() as u64)
463            .await
464            .unwrap();
465        assert!(inserted.is_file());
466        assert_eq!(
467            cache
468                .lookup_verified(&checksum, data.len() as u64)
469                .await
470                .unwrap(),
471            Some(inserted)
472        );
473        assert!(
474            cache
475                .lookup_verified(&checksum, data.len() as u64 + 1)
476                .await
477                .unwrap()
478                .is_none()
479        );
480    }
481
482    #[tokio::test]
483    async fn rejected_entries_are_removed_and_stats_are_reported() {
484        let dir = tempfile::tempdir().unwrap();
485        let cache = ContentCache::new(dir.path()).unwrap();
486        let data = b"good";
487        let checksum = checksum(data);
488        let entry = cache.entry_path(&checksum).unwrap();
489        tokio::fs::create_dir_all(entry.parent().unwrap())
490            .await
491            .unwrap();
492        tokio::fs::write(&entry, b"bad!").await.unwrap();
493        assert!(cache.lookup_verified(&checksum, 4).await.unwrap().is_none());
494        assert!(!entry.exists());
495    }
496
497    #[tokio::test]
498    async fn prunes_oldest_entries_to_a_size_limit() {
499        let dir = tempfile::tempdir().unwrap();
500        let cache = ContentCache::new(dir.path()).unwrap();
501        let old_path = dir.path().join("old");
502        let new_path = dir.path().join("new");
503        std::fs::write(&old_path, b"0123456789").unwrap();
504        std::fs::write(&new_path, b"12345").unwrap();
505        let old_time = SystemTime::now() - Duration::from_secs(3600);
506        std::fs::File::options()
507            .write(true)
508            .open(&old_path)
509            .unwrap()
510            .set_modified(old_time)
511            .unwrap();
512
513        let preview = cache.prune_preview(None, Some(10)).await.unwrap();
514        assert_eq!(preview.entries_removed, 1);
515        assert!(old_path.exists(), "preview must not delete entries");
516        let report = cache.prune(None, Some(10)).await.unwrap();
517        assert_eq!(report.entries_removed, 1);
518        assert_eq!(report.bytes_removed, 10);
519        assert_eq!(report.entries_remaining, 1);
520        assert_eq!(report.bytes_remaining, 5);
521        assert!(!old_path.exists());
522        assert!(new_path.exists());
523    }
524
525    #[tokio::test]
526    async fn clear_removes_entries_and_temp_files() {
527        let dir = tempfile::tempdir().unwrap();
528        let cache_root = dir.path().join("cache");
529        let cache = ContentCache::new(&cache_root).unwrap();
530        let source = dir.path().join("source.bin");
531        let data = b"clear me";
532        tokio::fs::write(&source, data).await.unwrap();
533        cache
534            .insert(&checksum(data), &source, data.len() as u64)
535            .await
536            .unwrap();
537        tokio::fs::write(cache_root.join(".tmp-leftover"), b"junk")
538            .await
539            .unwrap();
540        let stats = cache.stats().await.unwrap();
541        assert_eq!(stats.entries, 1);
542        assert_eq!(stats.temp_files, 1);
543        assert_eq!(cache.clear().await.unwrap(), data.len() as u64);
544        assert_eq!(cache.stats().await.unwrap(), CacheStats::default());
545    }
546}