ic_testkit/artifacts/
cache_fs.rs

1use fs2::FileExt as _;
2use std::{
3    fs::{self, File, OpenOptions},
4    io,
5    path::{Path, PathBuf},
6    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
7};
8
9use super::digest::write_atomic;
10
11const CACHE_DIRECTORY_TAG: &str = "Signature: 8a477f597d28d172789f06886806bc55\n\
12# This file is a cache directory tag created by ic-testkit.\n\
13# For information about cache directory tags see https://bford.info/cachedir/\n";
14pub const CACHE_DIRECTORY_TAG_SIGNATURE: &str = "Signature: 8a477f597d28d172789f06886806bc55\n";
15const LAST_USED_FILE: &str = ".ic-testkit-last-used";
16
17/// Caller-selected retention limits for content-addressed artifact entries.
18///
19/// Age pruning runs before size pruning. A policy without either limit scans
20/// the selected cache namespace and updates its cache metadata without
21/// removing entries.
22#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
23pub struct ArtifactCachePrunePolicy {
24    max_age: Option<Duration>,
25    max_size_bytes: Option<u64>,
26}
27
28/// Summary of one lock-coordinated artifact-cache pruning pass.
29#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
30pub struct ArtifactCachePruneReport {
31    entries_scanned: usize,
32    entries_removed: usize,
33    bytes_before: u64,
34    bytes_removed: u64,
35}
36
37/// Nonfatal retention attempted as part of a successful cache acquisition.
38#[non_exhaustive]
39#[derive(Clone, Debug, Eq, PartialEq)]
40pub enum ArtifactCacheMaintenance {
41    /// Configured retention completed under the cache lock.
42    Pruned(ArtifactCachePruneReport),
43    /// Configured retention failed after the requested artifacts were ready.
44    PruneFailed {
45        /// Cache error rendered without invalidating the successful acquisition.
46        message: String,
47    },
48}
49
50impl ArtifactCachePrunePolicy {
51    /// Create a policy that records cache metadata without removing entries.
52    #[must_use]
53    pub const fn new() -> Self {
54        Self {
55            max_age: None,
56            max_size_bytes: None,
57        }
58    }
59
60    /// Remove entries older than `max_age` before applying the size limit.
61    #[must_use]
62    pub const fn with_max_age(mut self, max_age: Duration) -> Self {
63        self.max_age = Some(max_age);
64        self
65    }
66
67    /// Remove least-recently-used entries until retained logical size is at most `bytes`.
68    #[must_use]
69    pub const fn with_max_size_bytes(mut self, bytes: u64) -> Self {
70        self.max_size_bytes = Some(bytes);
71        self
72    }
73
74    /// Configured maximum entry age, if any.
75    #[must_use]
76    pub const fn max_age(self) -> Option<Duration> {
77        self.max_age
78    }
79
80    /// Configured maximum logical cache size in bytes, if any.
81    #[must_use]
82    pub const fn max_size_bytes(self) -> Option<u64> {
83        self.max_size_bytes
84    }
85}
86
87impl ArtifactCachePruneReport {
88    /// Number of content-addressed directories considered for pruning.
89    #[must_use]
90    pub const fn entries_scanned(self) -> usize {
91        self.entries_scanned
92    }
93
94    /// Number of content-addressed directories removed.
95    #[must_use]
96    pub const fn entries_removed(self) -> usize {
97        self.entries_removed
98    }
99
100    /// Number of content-addressed directories retained.
101    #[must_use]
102    pub const fn entries_retained(self) -> usize {
103        self.entries_scanned.saturating_sub(self.entries_removed)
104    }
105
106    /// Logical bytes occupied by scanned entries before pruning.
107    #[must_use]
108    pub const fn bytes_before(self) -> u64 {
109        self.bytes_before
110    }
111
112    /// Logical bytes removed by pruning.
113    #[must_use]
114    pub const fn bytes_removed(self) -> u64 {
115        self.bytes_removed
116    }
117
118    /// Logical bytes occupied by retained entries after pruning.
119    #[must_use]
120    pub const fn bytes_retained(self) -> u64 {
121        self.bytes_before.saturating_sub(self.bytes_removed)
122    }
123}
124
125impl ArtifactCacheMaintenance {
126    /// Successful pruning report, or `None` when maintenance failed.
127    #[must_use]
128    pub const fn prune_report(&self) -> Option<ArtifactCachePruneReport> {
129        match self {
130            Self::Pruned(report) => Some(*report),
131            Self::PruneFailed { .. } => None,
132        }
133    }
134
135    /// Rendered maintenance failure, or `None` when pruning succeeded.
136    #[must_use]
137    pub fn failure_message(&self) -> Option<&str> {
138        match self {
139            Self::Pruned(_) => None,
140            Self::PruneFailed { message } => Some(message),
141        }
142    }
143}
144
145#[derive(Debug)]
146pub struct CacheFsError {
147    pub operation: &'static str,
148    pub path: PathBuf,
149    pub source: io::Error,
150}
151
152pub fn ensure_cache_directory_tag(cache_root: &Path) -> Result<(), CacheFsError> {
153    let path = cache_root.join("CACHEDIR.TAG");
154    if fs::read_to_string(&path)
155        .is_ok_and(|contents| contents.starts_with(CACHE_DIRECTORY_TAG_SIGNATURE))
156    {
157        return Ok(());
158    }
159    write_atomic(&path, CACHE_DIRECTORY_TAG.as_bytes()).map_err(|source| CacheFsError {
160        operation: "write cache directory tag",
161        path,
162        source,
163    })
164}
165
166pub fn lock_cache_file(path: &Path) -> Result<(File, Duration), CacheFsError> {
167    if let Some(parent) = path.parent() {
168        fs::create_dir_all(parent).map_err(|source| CacheFsError {
169            operation: "create cache lock directory",
170            path: parent.to_owned(),
171            source,
172        })?;
173    }
174    let file = OpenOptions::new()
175        .create(true)
176        .read(true)
177        .write(true)
178        .truncate(false)
179        .open(path)
180        .map_err(|source| CacheFsError {
181            operation: "open cache lock",
182            path: path.to_owned(),
183            source,
184        })?;
185    let started = Instant::now();
186    file.lock_exclusive().map_err(|source| CacheFsError {
187        operation: "lock cache",
188        path: path.to_owned(),
189        source,
190    })?;
191    Ok((file, started.elapsed()))
192}
193
194pub fn record_cache_entry_use(path: &Path) -> Result<(), CacheFsError> {
195    write_last_used(path, SystemTime::now())
196}
197
198pub fn write_last_used(path: &Path, last_used: SystemTime) -> Result<(), CacheFsError> {
199    let marker = path.join(LAST_USED_FILE);
200    let elapsed = last_used
201        .duration_since(UNIX_EPOCH)
202        .map_err(|source| CacheFsError {
203            operation: "encode cache use time",
204            path: marker.clone(),
205            source: io::Error::new(io::ErrorKind::InvalidInput, source),
206        })?;
207    write_atomic(&marker, elapsed.as_nanos().to_string().as_bytes()).map_err(|source| {
208        CacheFsError {
209            operation: "record cache use time",
210            path: marker,
211            source,
212        }
213    })
214}
215
216pub fn prune_direct_child_directories(
217    cache_root: &Path,
218    policy: ArtifactCachePrunePolicy,
219    protected_entry: Option<&Path>,
220    is_eligible: impl Fn(&Path) -> bool,
221) -> Result<ArtifactCachePruneReport, CacheFsError> {
222    let mut entries = cache_entries(cache_root, is_eligible)?;
223    let bytes_before = entries
224        .iter()
225        .fold(0_u64, |total, entry| total.saturating_add(entry.bytes));
226    let mut report = ArtifactCachePruneReport {
227        entries_scanned: entries.len(),
228        entries_removed: 0,
229        bytes_before,
230        bytes_removed: 0,
231    };
232    let now = SystemTime::now();
233
234    if let Some(max_age) = policy.max_age() {
235        for entry in &mut entries {
236            let age = now.duration_since(entry.last_used).unwrap_or_default();
237            if protected_entry != Some(entry.path.as_path()) && age > max_age {
238                remove_cache_entry(entry, &mut report)?;
239            }
240        }
241    }
242
243    if let Some(max_size_bytes) = policy.max_size_bytes() {
244        entries.sort_by(|left, right| {
245            left.last_used
246                .cmp(&right.last_used)
247                .then_with(|| left.path.cmp(&right.path))
248        });
249        for entry in &mut entries {
250            if report.bytes_retained() <= max_size_bytes {
251                break;
252            }
253            if protected_entry == Some(entry.path.as_path()) {
254                continue;
255            }
256            remove_cache_entry(entry, &mut report)?;
257        }
258    }
259
260    Ok(report)
261}
262
263pub fn directory_logical_size(path: &Path) -> io::Result<u64> {
264    let mut total = 0_u64;
265    let mut pending = vec![path.to_owned()];
266    while let Some(current) = pending.pop() {
267        let metadata = fs::symlink_metadata(&current)?;
268        if metadata.is_dir() {
269            for entry in fs::read_dir(&current)? {
270                pending.push(entry?.path());
271            }
272        } else {
273            total = total.saturating_add(metadata.len());
274        }
275    }
276    Ok(total)
277}
278
279struct CacheEntry {
280    path: PathBuf,
281    bytes: u64,
282    last_used: SystemTime,
283    removed: bool,
284}
285
286fn cache_entries(
287    cache_root: &Path,
288    is_eligible: impl Fn(&Path) -> bool,
289) -> Result<Vec<CacheEntry>, CacheFsError> {
290    let read_dir = match fs::read_dir(cache_root) {
291        Ok(read_dir) => read_dir,
292        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
293        Err(source) => {
294            return Err(CacheFsError {
295                operation: "read cache directory",
296                path: cache_root.to_owned(),
297                source,
298            });
299        }
300    };
301    let mut entries = Vec::new();
302    for directory_entry in read_dir {
303        let directory_entry = directory_entry.map_err(|source| CacheFsError {
304            operation: "read cache entry",
305            path: cache_root.to_owned(),
306            source,
307        })?;
308        let path = directory_entry.path();
309        let file_type = directory_entry.file_type().map_err(|source| CacheFsError {
310            operation: "inspect cache entry",
311            path: path.clone(),
312            source,
313        })?;
314        if !file_type.is_dir() || !is_eligible(&path) {
315            continue;
316        }
317        let bytes = directory_logical_size(&path).map_err(|source| CacheFsError {
318            operation: "measure cache entry",
319            path: path.clone(),
320            source,
321        })?;
322        let last_used = cache_entry_last_used(&path).map_err(|source| CacheFsError {
323            operation: "read cache use time",
324            path: path.clone(),
325            source,
326        })?;
327        entries.push(CacheEntry {
328            path,
329            bytes,
330            last_used,
331            removed: false,
332        });
333    }
334    Ok(entries)
335}
336
337fn cache_entry_last_used(path: &Path) -> io::Result<SystemTime> {
338    let marker = path.join(LAST_USED_FILE);
339    if let Ok(contents) = fs::read_to_string(&marker)
340        && let Ok(nanoseconds) = contents.parse::<u128>()
341    {
342        let seconds = nanoseconds / 1_000_000_000;
343        let subsecond_nanos = (nanoseconds % 1_000_000_000) as u32;
344        if let Ok(seconds) = u64::try_from(seconds)
345            && let Some(timestamp) = UNIX_EPOCH.checked_add(Duration::new(seconds, subsecond_nanos))
346        {
347            return Ok(timestamp);
348        }
349    }
350    fs::metadata(path)?.modified()
351}
352
353fn remove_cache_entry(
354    entry: &mut CacheEntry,
355    report: &mut ArtifactCachePruneReport,
356) -> Result<(), CacheFsError> {
357    if entry.removed {
358        return Ok(());
359    }
360    match fs::remove_dir_all(&entry.path) {
361        Ok(()) => {}
362        Err(error) if error.kind() == io::ErrorKind::NotFound => {}
363        Err(source) => {
364            return Err(CacheFsError {
365                operation: "prune cache entry",
366                path: entry.path.clone(),
367                source,
368            });
369        }
370    }
371    entry.removed = true;
372    report.entries_removed += 1;
373    report.bytes_removed = report.bytes_removed.saturating_add(entry.bytes);
374    Ok(())
375}
ic_testkit/artifacts/cache_fs.rs

ic_testkit/artifacts/
cache_fs.rs
