ic_testkit/artifacts/
cache_fs.rs

1use fs2::FileExt as _;
2use std::{
3    fs::{self, File, OpenOptions},
4    io::{self, Read as _},
5    path::{Component, Path, PathBuf},
6    sync::Arc,
7    thread,
8    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
9};
10
11use super::digest::{read_stamp_with_limit, write_atomic};
12
13/// Resolve existing components through symlinks and normalize a missing suffix.
14/// Parent traversal can return from a missing suffix to existing components.
15pub(super) fn canonicalize_allow_missing(path: &Path) -> io::Result<PathBuf> {
16    let absolute = if path.is_absolute() {
17        path.to_owned()
18    } else {
19        std::env::current_dir()?.join(path)
20    };
21    let mut resolved = PathBuf::new();
22    let mut missing_depth = 0_usize;
23    for component in absolute.components() {
24        match component {
25            Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {
26                let candidate = resolved.join(component.as_os_str());
27                if missing_depth == 0 && matches!(component, Component::Normal(_)) {
28                    match candidate.canonicalize() {
29                        Ok(canonical) => resolved = canonical,
30                        Err(error) if error.kind() == io::ErrorKind::NotFound => {
31                            resolved = candidate;
32                            missing_depth = 1;
33                        }
34                        Err(error) => return Err(error),
35                    }
36                } else {
37                    resolved = candidate;
38                    if matches!(component, Component::Normal(_)) && missing_depth > 0 {
39                        missing_depth += 1;
40                    }
41                }
42            }
43            Component::CurDir => {}
44            Component::ParentDir => {
45                resolved.pop();
46                missing_depth = missing_depth.saturating_sub(1);
47            }
48        }
49    }
50    Ok(resolved)
51}
52
53const CACHE_DIRECTORY_TAG: &str = "Signature: 8a477f597d28d172789f06886806bc55\n\
54# This file is a cache directory tag created by ic-testkit.\n\
55# For information about cache directory tags see https://bford.info/cachedir/\n";
56pub(super) const CACHE_DIRECTORY_TAG_SIGNATURE: &str =
57    "Signature: 8a477f597d28d172789f06886806bc55";
58pub(super) const LAST_USED_FILE: &str = ".ic-testkit-last-used";
59const LAST_MAINTENANCE_FILE: &str = ".ic-testkit-last-maintenance";
60// Nanoseconds are written as decimal u128 values, which need at most 39 bytes.
61const MAX_TIMESTAMP_BYTES: usize = 39;
62pub(super) const RETENTION_LOCK_FILE: &str = ".ic-testkit-retention-v1";
63
64/// Acquired under the producer/namespace lock before handing an entry to a
65/// consumer. Clones share ownership; the OS releases locks on process exit.
66#[derive(Clone, Debug)]
67pub(super) struct RetainedCacheEntry {
68    path: PathBuf,
69    _lock: Arc<File>,
70}
71
72impl PartialEq for RetainedCacheEntry {
73    fn eq(&self, other: &Self) -> bool {
74        self.path == other.path
75    }
76}
77
78impl Eq for RetainedCacheEntry {}
79
80impl RetainedCacheEntry {
81    pub(super) fn path(&self) -> &Path {
82        &self.path
83    }
84
85    pub(super) fn acquire(path: &Path) -> Result<Self, CacheFsError> {
86        let file = open_cache_lock_file(&path.join(RETENTION_LOCK_FILE))?;
87        fs2::FileExt::lock_shared(&file).map_err(|source| CacheFsError {
88            operation: "retain cache entry",
89            path: path.to_owned(),
90            source,
91        })?;
92        Ok(Self {
93            path: path.to_owned(),
94            _lock: Arc::new(file),
95        })
96    }
97}
98
99/// The caller must hold the producer/namespace lock throughout this operation.
100pub(super) fn remove_unretained_entry(path: &Path) -> Result<(), CacheFsError> {
101    let metadata = match fs::symlink_metadata(path) {
102        Ok(metadata) => metadata,
103        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
104        Err(source) => {
105            return Err(CacheFsError {
106                operation: "inspect cache entry",
107                path: path.to_owned(),
108                source,
109            });
110        }
111    };
112    if !metadata.is_dir() {
113        return remove_path_if_present(path).map_err(|source| CacheFsError {
114            operation: "remove invalid cache entry",
115            path: path.to_owned(),
116            source,
117        });
118    }
119    let _lock =
120        try_lock_cache_file(&path.join(RETENTION_LOCK_FILE))?.ok_or_else(|| CacheFsError {
121            operation: "replace retained cache entry",
122            path: path.to_owned(),
123            source: io::Error::new(
124                io::ErrorKind::WouldBlock,
125                "cache entry is retained by a consumer",
126            ),
127        })?;
128    remove_path_if_present(path).map_err(|source| CacheFsError {
129        operation: "remove cache entry",
130        path: path.to_owned(),
131        source,
132    })
133}
134
135/// Caller-selected retention limits for content-addressed artifact entries.
136///
137/// Age pruning runs before size pruning. A policy without either limit scans
138/// the selected cache namespace and updates its cache metadata without
139/// removing entries. Entries retained by live acquisition records are skipped,
140/// even when this temporarily exceeds the limits. They become eligible for the
141/// next maintenance pass after their final owner drops or its process exits.
142#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
143pub struct ArtifactCachePrunePolicy {
144    max_age: Option<Duration>,
145    max_size_bytes: Option<u64>,
146}
147
148/// Summary of one lock-coordinated artifact-cache pruning pass.
149#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
150pub struct ArtifactCachePruneReport {
151    entries_scanned: usize,
152    entries_removed: usize,
153    bytes_before: u64,
154    bytes_removed: u64,
155    uncommitted_directories_removed: usize,
156    uncommitted_bytes_removed: u64,
157}
158
159/// Nonfatal retention attempted as part of a successful cache acquisition.
160#[non_exhaustive]
161#[derive(Clone, Debug, Eq, PartialEq)]
162pub enum ArtifactCacheMaintenance {
163    /// Configured retention completed under the cache lock.
164    Pruned(ArtifactCachePruneReport),
165    /// Configured retention failed after the requested artifacts were ready.
166    PruneFailed {
167        /// Cache error rendered without invalidating the successful acquisition.
168        message: String,
169    },
170}
171
172impl ArtifactCachePrunePolicy {
173    /// Create a policy that records cache metadata without removing entries.
174    #[must_use]
175    pub const fn new() -> Self {
176        Self {
177            max_age: None,
178            max_size_bytes: None,
179        }
180    }
181
182    /// Remove entries older than `max_age` before applying the size limit.
183    #[must_use]
184    pub const fn with_max_age(mut self, max_age: Duration) -> Self {
185        self.max_age = Some(max_age);
186        self
187    }
188
189    /// Remove least-recently-used entries until retained logical size is at most `bytes`.
190    #[must_use]
191    pub const fn with_max_size_bytes(mut self, bytes: u64) -> Self {
192        self.max_size_bytes = Some(bytes);
193        self
194    }
195
196    /// Configured maximum entry age, if any.
197    #[must_use]
198    pub const fn max_age(self) -> Option<Duration> {
199        self.max_age
200    }
201
202    /// Configured maximum logical cache size in bytes, if any.
203    #[must_use]
204    pub const fn max_size_bytes(self) -> Option<u64> {
205        self.max_size_bytes
206    }
207
208    pub(super) fn maintenance_identity(self) -> String {
209        format!(
210            "age={:?};size={:?}",
211            self.max_age.map(|duration| duration.as_nanos()),
212            self.max_size_bytes
213        )
214    }
215}
216
217impl ArtifactCachePruneReport {
218    /// Number of content-addressed directories considered for pruning.
219    #[must_use]
220    pub const fn entries_scanned(self) -> usize {
221        self.entries_scanned
222    }
223
224    /// Number of content-addressed directories removed.
225    #[must_use]
226    pub const fn entries_removed(self) -> usize {
227        self.entries_removed
228    }
229
230    /// Number of content-addressed directories retained.
231    #[must_use]
232    pub const fn entries_retained(self) -> usize {
233        self.entries_scanned.saturating_sub(self.entries_removed)
234    }
235
236    /// Logical bytes occupied by scanned entries before pruning.
237    #[must_use]
238    pub const fn bytes_before(self) -> u64 {
239        self.bytes_before
240    }
241
242    /// Logical bytes removed by pruning.
243    #[must_use]
244    pub const fn bytes_removed(self) -> u64 {
245        self.bytes_removed
246    }
247
248    /// Logical bytes occupied by retained entries after pruning.
249    #[must_use]
250    pub const fn bytes_retained(self) -> u64 {
251        self.bytes_before.saturating_sub(self.bytes_removed)
252    }
253
254    /// Abandoned transaction directories removed outside the committed-entry totals.
255    #[must_use]
256    pub const fn uncommitted_directories_removed(self) -> usize {
257        self.uncommitted_directories_removed
258    }
259
260    /// Logical bytes removed from abandoned transaction directories.
261    #[must_use]
262    pub const fn uncommitted_bytes_removed(self) -> u64 {
263        self.uncommitted_bytes_removed
264    }
265
266    pub(super) const fn record_uncommitted_removal(&mut self, bytes: u64) {
267        self.uncommitted_directories_removed += 1;
268        self.uncommitted_bytes_removed = self.uncommitted_bytes_removed.saturating_add(bytes);
269    }
270}
271
272impl ArtifactCacheMaintenance {
273    /// Successful pruning report, or `None` when maintenance failed.
274    #[must_use]
275    pub const fn prune_report(&self) -> Option<ArtifactCachePruneReport> {
276        match self {
277            Self::Pruned(report) => Some(*report),
278            Self::PruneFailed { .. } => None,
279        }
280    }
281
282    /// Rendered maintenance failure, or `None` when pruning succeeded.
283    #[must_use]
284    pub fn failure_message(&self) -> Option<&str> {
285        match self {
286            Self::Pruned(_) => None,
287            Self::PruneFailed { message } => Some(message),
288        }
289    }
290}
291
292#[derive(Debug)]
293pub(super) struct CacheFsError {
294    pub(super) operation: &'static str,
295    pub(super) path: PathBuf,
296    pub(super) source: io::Error,
297}
298
299pub(super) fn ensure_cache_directory_tag(cache_root: &Path) -> Result<(), CacheFsError> {
300    let path = cache_root.join("CACHEDIR.TAG");
301    // The standard recognizes the first 43 bytes, without requiring a newline
302    // or interpreting the remaining text. Symlinks are not valid tag files.
303    let mut signature = [0_u8; CACHE_DIRECTORY_TAG_SIGNATURE.len()];
304    if fs::symlink_metadata(&path).is_ok_and(|metadata| metadata.file_type().is_file())
305        && File::open(&path)
306            .and_then(|mut file| file.read_exact(&mut signature))
307            .is_ok()
308        && signature == CACHE_DIRECTORY_TAG_SIGNATURE.as_bytes()
309    {
310        return Ok(());
311    }
312    write_atomic(&path, CACHE_DIRECTORY_TAG.as_bytes()).map_err(|source| CacheFsError {
313        operation: "write cache directory tag",
314        path,
315        source,
316    })
317}
318
319pub(super) fn lock_cache_file(path: &Path) -> Result<(File, Duration), CacheFsError> {
320    let file = open_cache_lock_file(path)?;
321    let started = Instant::now();
322    file.lock_exclusive().map_err(|source| CacheFsError {
323        operation: "lock cache",
324        path: path.to_owned(),
325        source,
326    })?;
327    Ok((file, started.elapsed()))
328}
329
330pub(super) fn lock_cache_file_with_wait_observer(
331    path: &Path,
332    poll_interval: Duration,
333    mut observer: impl FnMut(Duration),
334) -> Result<(File, Duration), CacheFsError> {
335    let file = open_cache_lock_file(path)?;
336    let started = Instant::now();
337    loop {
338        match file.try_lock_exclusive() {
339            Ok(()) => return Ok((file, started.elapsed())),
340            Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
341                observer(started.elapsed());
342                thread::sleep(poll_interval.min(Duration::from_millis(25)));
343            }
344            Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
345            Err(source) => {
346                return Err(CacheFsError {
347                    operation: "try lock cache",
348                    path: path.to_owned(),
349                    source,
350                });
351            }
352        }
353    }
354}
355
356pub(super) fn try_lock_cache_file(path: &Path) -> Result<Option<File>, CacheFsError> {
357    let file = open_cache_lock_file(path)?;
358    match file.try_lock_exclusive() {
359        Ok(()) => Ok(Some(file)),
360        Err(error) if error.kind() == io::ErrorKind::WouldBlock => Ok(None),
361        Err(source) => Err(CacheFsError {
362            operation: "try lock cache",
363            path: path.to_owned(),
364            source,
365        }),
366    }
367}
368
369fn open_cache_lock_file(path: &Path) -> Result<File, CacheFsError> {
370    if let Some(parent) = path.parent() {
371        fs::create_dir_all(parent).map_err(|source| CacheFsError {
372            operation: "create cache lock directory",
373            path: parent.to_owned(),
374            source,
375        })?;
376    }
377    OpenOptions::new()
378        .create(true)
379        .read(true)
380        .write(true)
381        .truncate(false)
382        .open(path)
383        .map_err(|source| CacheFsError {
384            operation: "open cache lock",
385            path: path.to_owned(),
386            source,
387        })
388}
389
390pub(super) fn record_cache_entry_use(path: &Path) -> Result<(), CacheFsError> {
391    write_last_used(path, SystemTime::now())
392}
393
394pub(super) fn cache_maintenance_due(
395    path: &Path,
396    minimum_interval: Option<Duration>,
397    maintenance_identity: &str,
398) -> Result<bool, CacheFsError> {
399    let Some(minimum_interval) = minimum_interval else {
400        return Ok(true);
401    };
402    let marker = path.join(LAST_MAINTENANCE_FILE);
403    // Allow both LF and CRLF for the timestamp and policy-identity lines.
404    let maximum_len = MAX_TIMESTAMP_BYTES + maintenance_identity.len() + 4;
405    let contents = match read_stamp_with_limit(&marker, maximum_len) {
406        Ok(Some(contents)) => contents,
407        Ok(None) => return Ok(true),
408        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(true),
409        Err(source) => {
410            return Err(CacheFsError {
411                operation: "read cache maintenance time",
412                path: marker,
413                source,
414            });
415        }
416    };
417    let mut lines = contents.lines();
418    let Some(last_maintenance) = lines.next().and_then(decode_system_time) else {
419        return Ok(true);
420    };
421    if lines.next() != Some(maintenance_identity) {
422        return Ok(true);
423    }
424    Ok(match SystemTime::now().duration_since(last_maintenance) {
425        Ok(elapsed) => elapsed >= minimum_interval,
426        Err(_) => true,
427    })
428}
429
430pub(super) fn record_cache_maintenance(
431    path: &Path,
432    maintenance_identity: &str,
433) -> Result<(), CacheFsError> {
434    let marker = path.join(LAST_MAINTENANCE_FILE);
435    let elapsed = encode_system_time(&marker, SystemTime::now())?;
436    let contents = format!("{}\n{maintenance_identity}\n", elapsed.as_nanos());
437    write_atomic(&marker, contents.as_bytes()).map_err(|source| CacheFsError {
438        operation: "record cache maintenance time",
439        path: marker,
440        source,
441    })
442}
443
444pub(super) fn perform_scheduled_cache_maintenance(
445    path: &Path,
446    minimum_interval: Option<Duration>,
447    maintenance_identity: &str,
448    maintenance: impl FnOnce() -> Result<ArtifactCachePruneReport, String>,
449) -> (Option<ArtifactCacheMaintenance>, Option<Duration>) {
450    let started = Instant::now();
451    match cache_maintenance_due(path, minimum_interval, maintenance_identity) {
452        Ok(false) => return (None, Some(started.elapsed())),
453        Ok(true) => {}
454        Err(error) => {
455            return (
456                Some(ArtifactCacheMaintenance::PruneFailed {
457                    message: error.to_string(),
458                }),
459                Some(started.elapsed()),
460            );
461        }
462    }
463
464    let result = maintenance();
465    let marker = record_cache_maintenance(path, maintenance_identity);
466    let outcome = match (result, marker) {
467        (Ok(report), Ok(())) => ArtifactCacheMaintenance::Pruned(report),
468        (Err(message), Ok(())) => ArtifactCacheMaintenance::PruneFailed { message },
469        (Ok(_), Err(error)) => ArtifactCacheMaintenance::PruneFailed {
470            message: error.to_string(),
471        },
472        (Err(message), Err(marker)) => ArtifactCacheMaintenance::PruneFailed {
473            message: format!(
474                "{message}; additionally failed to record the maintenance attempt: {marker}"
475            ),
476        },
477    };
478    (Some(outcome), Some(started.elapsed()))
479}
480
481pub(super) fn write_last_used(path: &Path, last_used: SystemTime) -> Result<(), CacheFsError> {
482    let marker = path.join(LAST_USED_FILE);
483    write_system_time(&marker, last_used, "record cache use time")
484}
485
486fn write_system_time(
487    path: &Path,
488    timestamp: SystemTime,
489    operation: &'static str,
490) -> Result<(), CacheFsError> {
491    let elapsed = encode_system_time(path, timestamp)?;
492    write_atomic(path, elapsed.as_nanos().to_string().as_bytes()).map_err(|source| CacheFsError {
493        operation,
494        path: path.to_owned(),
495        source,
496    })
497}
498
499fn encode_system_time(path: &Path, timestamp: SystemTime) -> Result<Duration, CacheFsError> {
500    timestamp
501        .duration_since(UNIX_EPOCH)
502        .map_err(|source| CacheFsError {
503            operation: "encode cache time",
504            path: path.to_owned(),
505            source: io::Error::new(io::ErrorKind::InvalidInput, source),
506        })
507}
508
509fn decode_system_time(contents: &str) -> Option<SystemTime> {
510    let nanoseconds = contents.parse::<u128>().ok()?;
511    let seconds = u64::try_from(nanoseconds / 1_000_000_000).ok()?;
512    let subsecond_nanos = (nanoseconds % 1_000_000_000) as u32;
513    UNIX_EPOCH.checked_add(Duration::new(seconds, subsecond_nanos))
514}
515
516pub(super) fn prune_direct_child_directories(
517    cache_root: &Path,
518    policy: ArtifactCachePrunePolicy,
519    protected_entry: Option<&Path>,
520    is_eligible: impl Fn(&Path) -> bool,
521) -> Result<ArtifactCachePruneReport, CacheFsError> {
522    let mut entries = cache_entries(cache_root, is_eligible)?;
523    let bytes_before = entries
524        .iter()
525        .fold(0_u64, |total, entry| total.saturating_add(entry.bytes));
526    let mut report = ArtifactCachePruneReport {
527        entries_scanned: entries.len(),
528        entries_removed: 0,
529        bytes_before,
530        bytes_removed: 0,
531        uncommitted_directories_removed: 0,
532        uncommitted_bytes_removed: 0,
533    };
534    let now = SystemTime::now();
535
536    if let Some(max_age) = policy.max_age() {
537        for entry in &mut entries {
538            let age = now.duration_since(entry.last_used).unwrap_or_default();
539            if protected_entry != Some(entry.path.as_path()) && age > max_age {
540                remove_cache_entry(entry, &mut report)?;
541            }
542        }
543    }
544
545    if let Some(max_size_bytes) = policy.max_size_bytes()
546        && report.bytes_retained() > max_size_bytes
547    {
548        entries.sort_by(|left, right| {
549            left.last_used
550                .cmp(&right.last_used)
551                .then_with(|| left.path.cmp(&right.path))
552        });
553        for entry in &mut entries {
554            if report.bytes_retained() <= max_size_bytes {
555                break;
556            }
557            if protected_entry == Some(entry.path.as_path()) {
558                continue;
559            }
560            remove_cache_entry(entry, &mut report)?;
561        }
562    }
563
564    Ok(report)
565}
566
567pub(super) fn directory_logical_size(path: &Path) -> io::Result<u64> {
568    let mut total = 0_u64;
569    let mut pending = vec![path.to_owned()];
570    while let Some(current) = pending.pop() {
571        let metadata = fs::symlink_metadata(&current)?;
572        if metadata.is_dir() {
573            for entry in fs::read_dir(&current)? {
574                let path = entry?.path();
575                let metadata = fs::symlink_metadata(&path)?;
576                if metadata.is_dir() {
577                    pending.push(path);
578                } else {
579                    total = total.saturating_add(metadata.len());
580                }
581            }
582        } else {
583            total = total.saturating_add(metadata.len());
584        }
585    }
586    Ok(total)
587}
588
589pub(super) fn is_sha256_directory(path: &Path) -> bool {
590    path.file_name().is_some_and(|name| {
591        let bytes = name.as_encoded_bytes();
592        bytes.len() == 64 && bytes.iter().all(u8::is_ascii_hexdigit)
593    })
594}
595
596pub(super) fn remove_path_if_present(path: &Path) -> io::Result<()> {
597    let metadata = match fs::symlink_metadata(path) {
598        Ok(metadata) => metadata,
599        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
600        Err(error) => return Err(error),
601    };
602    if metadata.file_type().is_dir() {
603        fs::remove_dir_all(path)
604    } else {
605        fs::remove_file(path)
606    }
607}
608
609struct CacheEntry {
610    path: PathBuf,
611    bytes: u64,
612    last_used: SystemTime,
613    removed: bool,
614}
615
616impl std::fmt::Display for CacheFsError {
617    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
618        write!(
619            formatter,
620            "failed to {} at {}: {}",
621            self.operation,
622            self.path.display(),
623            self.source
624        )
625    }
626}
627
628impl std::error::Error for CacheFsError {
629    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
630        Some(&self.source)
631    }
632}
633
634fn cache_entries(
635    cache_root: &Path,
636    is_eligible: impl Fn(&Path) -> bool,
637) -> Result<Vec<CacheEntry>, CacheFsError> {
638    let read_dir = match fs::read_dir(cache_root) {
639        Ok(read_dir) => read_dir,
640        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
641        Err(source) => {
642            return Err(CacheFsError {
643                operation: "read cache directory",
644                path: cache_root.to_owned(),
645                source,
646            });
647        }
648    };
649    let mut entries = Vec::new();
650    for directory_entry in read_dir {
651        let directory_entry = directory_entry.map_err(|source| CacheFsError {
652            operation: "read cache entry",
653            path: cache_root.to_owned(),
654            source,
655        })?;
656        let path = directory_entry.path();
657        let file_type = directory_entry.file_type().map_err(|source| CacheFsError {
658            operation: "inspect cache entry",
659            path: path.clone(),
660            source,
661        })?;
662        if !file_type.is_dir() || !is_eligible(&path) {
663            continue;
664        }
665        let bytes = directory_logical_size(&path).map_err(|source| CacheFsError {
666            operation: "measure cache entry",
667            path: path.clone(),
668            source,
669        })?;
670        let last_used = cache_entry_last_used(&path).map_err(|source| CacheFsError {
671            operation: "read cache use time",
672            path: path.clone(),
673            source,
674        })?;
675        entries.push(CacheEntry {
676            path,
677            bytes,
678            last_used,
679            removed: false,
680        });
681    }
682    Ok(entries)
683}
684
685pub(super) fn cache_entry_last_used(path: &Path) -> io::Result<SystemTime> {
686    let marker = path.join(LAST_USED_FILE);
687    if let Ok(Some(contents)) = read_stamp_with_limit(&marker, MAX_TIMESTAMP_BYTES)
688        && let Some(timestamp) = decode_system_time(&contents)
689    {
690        return Ok(timestamp);
691    }
692    fs::metadata(path)?.modified()
693}
694
695fn remove_cache_entry(
696    entry: &mut CacheEntry,
697    report: &mut ArtifactCachePruneReport,
698) -> Result<(), CacheFsError> {
699    if entry.removed {
700        return Ok(());
701    }
702    let Some(_retention_lock) = try_lock_cache_file(&entry.path.join(RETENTION_LOCK_FILE))? else {
703        return Ok(());
704    };
705    remove_path_if_present(&entry.path).map_err(|source| CacheFsError {
706        operation: "prune cache entry",
707        path: entry.path.clone(),
708        source,
709    })?;
710    entry.removed = true;
711    report.entries_removed += 1;
712    report.bytes_removed = report.bytes_removed.saturating_add(entry.bytes);
713    Ok(())
714}
715
716#[cfg(test)]
717mod tests {
718    #[cfg(unix)]
719    use super::canonicalize_allow_missing;
720    use super::directory_logical_size;
721    use crate::artifacts::test_support::unique_temp_directory;
722    use std::{
723        fs,
724        time::{Duration, SystemTime, UNIX_EPOCH},
725    };
726
727    #[test]
728    fn last_use_markers_preserve_timestamps_and_bounded_fallbacks() {
729        let root = unique_temp_directory("bounded-last-use-marker");
730        let marker = root.join(super::LAST_USED_FILE);
731        let modified = || fs::metadata(&root).unwrap().modified().unwrap();
732        assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
733        for timestamp in [
734            UNIX_EPOCH + Duration::from_nanos(123),
735            SystemTime::now() + Duration::from_secs(3600),
736        ] {
737            super::write_last_used(&root, timestamp).unwrap();
738            assert_eq!(super::cache_entry_last_used(&root).unwrap(), timestamp);
739        }
740        let overflowing_timestamp = u128::MAX.to_string();
741        for invalid in [
742            b"invalid".as_slice(),
743            overflowing_timestamp.as_bytes(),
744            &[0xff],
745        ] {
746            fs::write(&marker, invalid).unwrap();
747            assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
748        }
749        fs::File::create(&marker)
750            .unwrap()
751            .set_len(1024 * 1024 * 1024)
752            .unwrap();
753        assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
754        fs::remove_file(&marker).unwrap();
755        fs::create_dir(&marker).unwrap();
756        assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
757        fs::remove_dir_all(root).unwrap();
758    }
759
760    #[test]
761    fn maintenance_markers_preserve_policy_intervals_and_read_errors() {
762        let root = unique_temp_directory("bounded-maintenance-marker");
763        let marker = root.join(super::LAST_MAINTENANCE_FILE);
764        let identity = super::ArtifactCachePrunePolicy::new().maintenance_identity();
765        let interval = Some(Duration::from_secs(3600));
766        let due = || super::cache_maintenance_due(&root, interval, &identity).unwrap();
767        assert!(due());
768        super::record_cache_maintenance(&root, &identity).unwrap();
769        assert!(!due());
770        assert!(super::cache_maintenance_due(&root, interval, "other-policy").unwrap());
771        assert!(super::cache_maintenance_due(&root, Some(Duration::ZERO), &identity).unwrap());
772
773        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
774        for suffix in ["", "\r\n"] {
775            fs::write(&marker, format!("{}\r\n{identity}{suffix}", now.as_nanos())).unwrap();
776            assert!(!due());
777        }
778        for timestamp in [
779            "invalid".to_owned(),
780            "0".to_owned(),
781            (now + Duration::from_secs(3600)).as_nanos().to_string(),
782        ] {
783            fs::write(&marker, format!("{timestamp}\n{identity}\n")).unwrap();
784            assert!(due());
785        }
786        super::record_cache_maintenance(&root, &identity).unwrap();
787        fs::OpenOptions::new()
788            .write(true)
789            .open(&marker)
790            .unwrap()
791            .set_len(1024 * 1024 * 1024)
792            .unwrap();
793        assert!(due());
794
795        fs::write(&marker, [0xff]).unwrap();
796        let error = super::cache_maintenance_due(&root, interval, &identity).unwrap_err();
797        assert_eq!(error.operation, "read cache maintenance time");
798        assert_eq!(error.path, marker);
799        assert_eq!(error.source.kind(), std::io::ErrorKind::InvalidData);
800        fs::remove_file(&marker).unwrap();
801        fs::create_dir(&marker).unwrap();
802        assert!(super::cache_maintenance_due(&root, interval, &identity).is_err());
803        assert!(super::cache_maintenance_due(&root, None, &identity).unwrap());
804        fs::remove_dir_all(root).unwrap();
805    }
806
807    #[test]
808    fn cache_directory_tags_preserve_valid_standard_signatures() {
809        let root = unique_temp_directory("cache-tag-signatures");
810        let tag = root.join("CACHEDIR.TAG");
811        let signature = "Signature: 8a477f597d28d172789f06886806bc55";
812        for contents in [
813            signature.to_owned(),
814            format!("{signature}\r\n# Created by another application\r\n"),
815            format!("{signature}\n# {}\n", "comment".repeat(100_000)),
816        ] {
817            fs::write(&tag, &contents).unwrap();
818            super::ensure_cache_directory_tag(&root).unwrap();
819            assert_eq!(fs::read_to_string(&tag).unwrap(), contents);
820        }
821        for invalid in ["", &signature[..42], "Signature: incorrect"] {
822            fs::write(&tag, invalid).unwrap();
823            super::ensure_cache_directory_tag(&root).unwrap();
824            assert_eq!(
825                fs::read_to_string(&tag).unwrap(),
826                super::CACHE_DIRECTORY_TAG
827            );
828        }
829        fs::remove_dir_all(root).unwrap();
830    }
831
832    #[test]
833    #[cfg(unix)]
834    fn cache_directory_tag_replaces_symlinks_without_changing_referents() {
835        let root = unique_temp_directory("cache-tag-symlink");
836        let referent = root.join("other-application-tag");
837        let contents = "Signature: 8a477f597d28d172789f06886806bc55\n# Preserve this file\n";
838        fs::write(&referent, contents).unwrap();
839        let tag = root.join("CACHEDIR.TAG");
840        std::os::unix::fs::symlink(&referent, &tag).unwrap();
841        super::ensure_cache_directory_tag(&root).unwrap();
842        assert!(fs::symlink_metadata(&tag).unwrap().file_type().is_file());
843        assert_eq!(fs::read_to_string(&referent).unwrap(), contents);
844        assert_eq!(
845            fs::read_to_string(&tag).unwrap(),
846            super::CACHE_DIRECTORY_TAG
847        );
848
849        fs::remove_file(&tag).unwrap();
850        fs::create_dir(&tag).unwrap();
851        let error = super::ensure_cache_directory_tag(&root).unwrap_err();
852        assert_eq!(error.operation, "write cache directory tag");
853        assert!(tag.is_dir());
854        fs::remove_dir_all(root).unwrap();
855    }
856
857    #[test]
858    fn directory_size_sums_wide_and_nested_files() {
859        let root = unique_temp_directory("directory-logical-size");
860        assert_eq!(directory_logical_size(&root).unwrap(), 0);
861        fs::create_dir_all(root.join("wide")).unwrap();
862        fs::create_dir_all(root.join("nested/deep/empty")).unwrap();
863        let mut expected = 0;
864        for index in 0..128 {
865            let bytes = vec![42; index % 13];
866            fs::write(root.join("wide").join(index.to_string()), &bytes).unwrap();
867            expected += bytes.len() as u64;
868        }
869        let sparse = root.join("nested/deep/sparse");
870        fs::File::create(&sparse)
871            .unwrap()
872            .set_len(1024 * 1024)
873            .unwrap();
874        assert_eq!(directory_logical_size(&sparse).unwrap(), 1024 * 1024);
875        assert_eq!(
876            directory_logical_size(&root).unwrap(),
877            expected + 1024 * 1024
878        );
879        assert_eq!(
880            directory_logical_size(&root.join("missing"))
881                .unwrap_err()
882                .kind(),
883            std::io::ErrorKind::NotFound,
884        );
885        fs::remove_dir_all(root).unwrap();
886    }
887
888    #[test]
889    #[cfg(unix)]
890    fn directory_size_counts_symlinks_without_following_them() {
891        let root = unique_temp_directory("directory-size-symlinks");
892        let walked = root.join("walked");
893        fs::create_dir_all(&walked).unwrap();
894        fs::create_dir_all(root.join("external")).unwrap();
895        fs::write(root.join("external/payload"), vec![42; 4096]).unwrap();
896        fs::write(root.join("outside-file"), vec![42; 4096]).unwrap();
897        fs::write(walked.join("payload"), b"abc").unwrap();
898        let targets = ["../external", "../outside-file", "missing", "."];
899        for (index, target) in targets.iter().enumerate() {
900            std::os::unix::fs::symlink(target, walked.join(index.to_string())).unwrap();
901        }
902        let expected = 3 + targets
903            .iter()
904            .map(|target| target.len() as u64)
905            .sum::<u64>();
906        assert_eq!(directory_logical_size(&walked).unwrap(), expected);
907        assert_eq!(
908            directory_logical_size(&walked.join("0")).unwrap(),
909            targets[0].len() as u64,
910        );
911        fs::remove_dir_all(root).unwrap();
912    }
913
914    #[test]
915    #[cfg(unix)]
916    fn missing_parent_traversal_resumes_existing_symlink_resolution() {
917        let root = unique_temp_directory("canonical-missing-parent");
918        let target = root.join("real");
919        fs::create_dir_all(&target).unwrap();
920        std::os::unix::fs::symlink(&target, root.join("alias")).unwrap();
921        let path = root.join("missing/../alias/generated/nested/../output");
922        assert_eq!(
923            canonicalize_allow_missing(&path).unwrap(),
924            target.canonicalize().unwrap().join("generated/output")
925        );
926        assert_eq!(
927            canonicalize_allow_missing(&root.join("alias/../other/output")).unwrap(),
928            root.canonicalize().unwrap().join("other/output")
929        );
930        fs::remove_dir_all(root).unwrap();
931    }
932}
ic_testkit/artifacts/cache_fs.rs

ic_testkit/artifacts/
cache_fs.rs
