ic_testkit/artifacts/
cache_fs.rs

1use fs2::FileExt as _;
2use std::{
3    fs::{self, File, OpenOptions},
4    io,
5    path::{Path, PathBuf},
6    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
7};
8
9use super::digest::write_atomic;
10
11const CACHE_DIRECTORY_TAG: &str = "Signature: 8a477f597d28d172789f06886806bc55\n\
12# This file is a cache directory tag created by ic-testkit.\n\
13# For information about cache directory tags see https://bford.info/cachedir/\n";
14pub(super) const CACHE_DIRECTORY_TAG_SIGNATURE: &str =
15    "Signature: 8a477f597d28d172789f06886806bc55\n";
16pub(super) const LAST_USED_FILE: &str = ".ic-testkit-last-used";
17
18/// Caller-selected retention limits for content-addressed artifact entries.
19///
20/// Age pruning runs before size pruning. A policy without either limit scans
21/// the selected cache namespace and updates its cache metadata without
22/// removing entries.
23#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
24pub struct ArtifactCachePrunePolicy {
25    max_age: Option<Duration>,
26    max_size_bytes: Option<u64>,
27}
28
29/// Summary of one lock-coordinated artifact-cache pruning pass.
30#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
31pub struct ArtifactCachePruneReport {
32    entries_scanned: usize,
33    entries_removed: usize,
34    bytes_before: u64,
35    bytes_removed: u64,
36    uncommitted_directories_removed: usize,
37    uncommitted_bytes_removed: u64,
38}
39
40/// Nonfatal retention attempted as part of a successful cache acquisition.
41#[non_exhaustive]
42#[derive(Clone, Debug, Eq, PartialEq)]
43pub enum ArtifactCacheMaintenance {
44    /// Configured retention completed under the cache lock.
45    Pruned(ArtifactCachePruneReport),
46    /// Configured retention failed after the requested artifacts were ready.
47    PruneFailed {
48        /// Cache error rendered without invalidating the successful acquisition.
49        message: String,
50    },
51}
52
53impl ArtifactCachePrunePolicy {
54    /// Create a policy that records cache metadata without removing entries.
55    #[must_use]
56    pub const fn new() -> Self {
57        Self {
58            max_age: None,
59            max_size_bytes: None,
60        }
61    }
62
63    /// Remove entries older than `max_age` before applying the size limit.
64    #[must_use]
65    pub const fn with_max_age(mut self, max_age: Duration) -> Self {
66        self.max_age = Some(max_age);
67        self
68    }
69
70    /// Remove least-recently-used entries until retained logical size is at most `bytes`.
71    #[must_use]
72    pub const fn with_max_size_bytes(mut self, bytes: u64) -> Self {
73        self.max_size_bytes = Some(bytes);
74        self
75    }
76
77    /// Configured maximum entry age, if any.
78    #[must_use]
79    pub const fn max_age(self) -> Option<Duration> {
80        self.max_age
81    }
82
83    /// Configured maximum logical cache size in bytes, if any.
84    #[must_use]
85    pub const fn max_size_bytes(self) -> Option<u64> {
86        self.max_size_bytes
87    }
88}
89
90impl ArtifactCachePruneReport {
91    /// Number of content-addressed directories considered for pruning.
92    #[must_use]
93    pub const fn entries_scanned(self) -> usize {
94        self.entries_scanned
95    }
96
97    /// Number of content-addressed directories removed.
98    #[must_use]
99    pub const fn entries_removed(self) -> usize {
100        self.entries_removed
101    }
102
103    /// Number of content-addressed directories retained.
104    #[must_use]
105    pub const fn entries_retained(self) -> usize {
106        self.entries_scanned.saturating_sub(self.entries_removed)
107    }
108
109    /// Logical bytes occupied by scanned entries before pruning.
110    #[must_use]
111    pub const fn bytes_before(self) -> u64 {
112        self.bytes_before
113    }
114
115    /// Logical bytes removed by pruning.
116    #[must_use]
117    pub const fn bytes_removed(self) -> u64 {
118        self.bytes_removed
119    }
120
121    /// Logical bytes occupied by retained entries after pruning.
122    #[must_use]
123    pub const fn bytes_retained(self) -> u64 {
124        self.bytes_before.saturating_sub(self.bytes_removed)
125    }
126
127    /// Abandoned transaction directories removed outside the committed-entry totals.
128    #[must_use]
129    pub const fn uncommitted_directories_removed(self) -> usize {
130        self.uncommitted_directories_removed
131    }
132
133    /// Logical bytes removed from abandoned transaction directories.
134    #[must_use]
135    pub const fn uncommitted_bytes_removed(self) -> u64 {
136        self.uncommitted_bytes_removed
137    }
138
139    pub(super) const fn record_uncommitted_removal(&mut self, bytes: u64) {
140        self.uncommitted_directories_removed += 1;
141        self.uncommitted_bytes_removed = self.uncommitted_bytes_removed.saturating_add(bytes);
142    }
143}
144
145impl ArtifactCacheMaintenance {
146    /// Successful pruning report, or `None` when maintenance failed.
147    #[must_use]
148    pub const fn prune_report(&self) -> Option<ArtifactCachePruneReport> {
149        match self {
150            Self::Pruned(report) => Some(*report),
151            Self::PruneFailed { .. } => None,
152        }
153    }
154
155    /// Rendered maintenance failure, or `None` when pruning succeeded.
156    #[must_use]
157    pub fn failure_message(&self) -> Option<&str> {
158        match self {
159            Self::Pruned(_) => None,
160            Self::PruneFailed { message } => Some(message),
161        }
162    }
163}
164
165#[derive(Debug)]
166pub(super) struct CacheFsError {
167    pub(super) operation: &'static str,
168    pub(super) path: PathBuf,
169    pub(super) source: io::Error,
170}
171
172pub(super) fn ensure_cache_directory_tag(cache_root: &Path) -> Result<(), CacheFsError> {
173    let path = cache_root.join("CACHEDIR.TAG");
174    if fs::read_to_string(&path)
175        .is_ok_and(|contents| contents.starts_with(CACHE_DIRECTORY_TAG_SIGNATURE))
176    {
177        return Ok(());
178    }
179    write_atomic(&path, CACHE_DIRECTORY_TAG.as_bytes()).map_err(|source| CacheFsError {
180        operation: "write cache directory tag",
181        path,
182        source,
183    })
184}
185
186pub(super) fn lock_cache_file(path: &Path) -> Result<(File, Duration), CacheFsError> {
187    let file = open_cache_lock_file(path)?;
188    let started = Instant::now();
189    file.lock_exclusive().map_err(|source| CacheFsError {
190        operation: "lock cache",
191        path: path.to_owned(),
192        source,
193    })?;
194    Ok((file, started.elapsed()))
195}
196
197pub(super) fn try_lock_cache_file(path: &Path) -> Result<Option<File>, CacheFsError> {
198    let file = open_cache_lock_file(path)?;
199    match file.try_lock_exclusive() {
200        Ok(()) => Ok(Some(file)),
201        Err(error) if error.kind() == io::ErrorKind::WouldBlock => Ok(None),
202        Err(source) => Err(CacheFsError {
203            operation: "try lock cache",
204            path: path.to_owned(),
205            source,
206        }),
207    }
208}
209
210fn open_cache_lock_file(path: &Path) -> Result<File, CacheFsError> {
211    if let Some(parent) = path.parent() {
212        fs::create_dir_all(parent).map_err(|source| CacheFsError {
213            operation: "create cache lock directory",
214            path: parent.to_owned(),
215            source,
216        })?;
217    }
218    OpenOptions::new()
219        .create(true)
220        .read(true)
221        .write(true)
222        .truncate(false)
223        .open(path)
224        .map_err(|source| CacheFsError {
225            operation: "open cache lock",
226            path: path.to_owned(),
227            source,
228        })
229}
230
231pub(super) fn record_cache_entry_use(path: &Path) -> Result<(), CacheFsError> {
232    write_last_used(path, SystemTime::now())
233}
234
235pub(super) fn write_last_used(path: &Path, last_used: SystemTime) -> Result<(), CacheFsError> {
236    let marker = path.join(LAST_USED_FILE);
237    let elapsed = last_used
238        .duration_since(UNIX_EPOCH)
239        .map_err(|source| CacheFsError {
240            operation: "encode cache use time",
241            path: marker.clone(),
242            source: io::Error::new(io::ErrorKind::InvalidInput, source),
243        })?;
244    write_atomic(&marker, elapsed.as_nanos().to_string().as_bytes()).map_err(|source| {
245        CacheFsError {
246            operation: "record cache use time",
247            path: marker,
248            source,
249        }
250    })
251}
252
253pub(super) fn prune_direct_child_directories(
254    cache_root: &Path,
255    policy: ArtifactCachePrunePolicy,
256    protected_entry: Option<&Path>,
257    is_eligible: impl Fn(&Path) -> bool,
258) -> Result<ArtifactCachePruneReport, CacheFsError> {
259    let mut entries = cache_entries(cache_root, is_eligible)?;
260    let bytes_before = entries
261        .iter()
262        .fold(0_u64, |total, entry| total.saturating_add(entry.bytes));
263    let mut report = ArtifactCachePruneReport {
264        entries_scanned: entries.len(),
265        entries_removed: 0,
266        bytes_before,
267        bytes_removed: 0,
268        uncommitted_directories_removed: 0,
269        uncommitted_bytes_removed: 0,
270    };
271    let now = SystemTime::now();
272
273    if let Some(max_age) = policy.max_age() {
274        for entry in &mut entries {
275            let age = now.duration_since(entry.last_used).unwrap_or_default();
276            if protected_entry != Some(entry.path.as_path()) && age > max_age {
277                remove_cache_entry(entry, &mut report)?;
278            }
279        }
280    }
281
282    if let Some(max_size_bytes) = policy.max_size_bytes() {
283        entries.sort_by(|left, right| {
284            left.last_used
285                .cmp(&right.last_used)
286                .then_with(|| left.path.cmp(&right.path))
287        });
288        for entry in &mut entries {
289            if report.bytes_retained() <= max_size_bytes {
290                break;
291            }
292            if protected_entry == Some(entry.path.as_path()) {
293                continue;
294            }
295            remove_cache_entry(entry, &mut report)?;
296        }
297    }
298
299    Ok(report)
300}
301
302pub(super) fn directory_logical_size(path: &Path) -> io::Result<u64> {
303    let mut total = 0_u64;
304    let mut pending = vec![path.to_owned()];
305    while let Some(current) = pending.pop() {
306        let metadata = fs::symlink_metadata(&current)?;
307        if metadata.is_dir() {
308            for entry in fs::read_dir(&current)? {
309                pending.push(entry?.path());
310            }
311        } else {
312            total = total.saturating_add(metadata.len());
313        }
314    }
315    Ok(total)
316}
317
318pub(super) fn is_sha256_directory(path: &Path) -> bool {
319    path.file_name().is_some_and(|name| {
320        let bytes = name.as_encoded_bytes();
321        bytes.len() == 64 && bytes.iter().all(u8::is_ascii_hexdigit)
322    })
323}
324
325pub(super) fn remove_path_if_present(path: &Path) -> io::Result<()> {
326    let metadata = match fs::symlink_metadata(path) {
327        Ok(metadata) => metadata,
328        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
329        Err(error) => return Err(error),
330    };
331    if metadata.file_type().is_dir() {
332        fs::remove_dir_all(path)
333    } else {
334        fs::remove_file(path)
335    }
336}
337
338struct CacheEntry {
339    path: PathBuf,
340    bytes: u64,
341    last_used: SystemTime,
342    removed: bool,
343}
344
345fn cache_entries(
346    cache_root: &Path,
347    is_eligible: impl Fn(&Path) -> bool,
348) -> Result<Vec<CacheEntry>, CacheFsError> {
349    let read_dir = match fs::read_dir(cache_root) {
350        Ok(read_dir) => read_dir,
351        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
352        Err(source) => {
353            return Err(CacheFsError {
354                operation: "read cache directory",
355                path: cache_root.to_owned(),
356                source,
357            });
358        }
359    };
360    let mut entries = Vec::new();
361    for directory_entry in read_dir {
362        let directory_entry = directory_entry.map_err(|source| CacheFsError {
363            operation: "read cache entry",
364            path: cache_root.to_owned(),
365            source,
366        })?;
367        let path = directory_entry.path();
368        let file_type = directory_entry.file_type().map_err(|source| CacheFsError {
369            operation: "inspect cache entry",
370            path: path.clone(),
371            source,
372        })?;
373        if !file_type.is_dir() || !is_eligible(&path) {
374            continue;
375        }
376        let bytes = directory_logical_size(&path).map_err(|source| CacheFsError {
377            operation: "measure cache entry",
378            path: path.clone(),
379            source,
380        })?;
381        let last_used = cache_entry_last_used(&path).map_err(|source| CacheFsError {
382            operation: "read cache use time",
383            path: path.clone(),
384            source,
385        })?;
386        entries.push(CacheEntry {
387            path,
388            bytes,
389            last_used,
390            removed: false,
391        });
392    }
393    Ok(entries)
394}
395
396fn cache_entry_last_used(path: &Path) -> io::Result<SystemTime> {
397    let marker = path.join(LAST_USED_FILE);
398    if let Ok(contents) = fs::read_to_string(&marker)
399        && let Ok(nanoseconds) = contents.parse::<u128>()
400    {
401        let seconds = nanoseconds / 1_000_000_000;
402        let subsecond_nanos = (nanoseconds % 1_000_000_000) as u32;
403        if let Ok(seconds) = u64::try_from(seconds)
404            && let Some(timestamp) = UNIX_EPOCH.checked_add(Duration::new(seconds, subsecond_nanos))
405        {
406            return Ok(timestamp);
407        }
408    }
409    fs::metadata(path)?.modified()
410}
411
412fn remove_cache_entry(
413    entry: &mut CacheEntry,
414    report: &mut ArtifactCachePruneReport,
415) -> Result<(), CacheFsError> {
416    if entry.removed {
417        return Ok(());
418    }
419    remove_path_if_present(&entry.path).map_err(|source| CacheFsError {
420        operation: "prune cache entry",
421        path: entry.path.clone(),
422        source,
423    })?;
424    entry.removed = true;
425    report.entries_removed += 1;
426    report.bytes_removed = report.bytes_removed.saturating_add(entry.bytes);
427    Ok(())
428}
ic_testkit/artifacts/cache_fs.rs

ic_testkit/artifacts/
cache_fs.rs
