ic_testkit/artifacts/
cache_fs.rs

1use fs2::FileExt as _;
2use std::{
3    fs::{self, File, OpenOptions},
4    io::{self, Read as _},
5    path::{Component, Path, PathBuf},
6    sync::Arc,
7    thread,
8    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
9};
10
11use super::digest::{read_stamp_with_limit, write_atomic};
12
13/// Resolve existing components through symlinks and normalize a missing suffix.
14/// Parent traversal can return from a missing suffix to existing components.
15pub(super) fn canonicalize_allow_missing(path: &Path) -> io::Result<PathBuf> {
16    let absolute = if path.is_absolute() {
17        path.to_owned()
18    } else {
19        std::env::current_dir()?.join(path)
20    };
21    let mut resolved = PathBuf::new();
22    let mut missing_depth = 0_usize;
23    for component in absolute.components() {
24        match component {
25            Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {
26                let candidate = resolved.join(component.as_os_str());
27                if missing_depth == 0 && matches!(component, Component::Normal(_)) {
28                    match candidate.canonicalize() {
29                        Ok(canonical) => resolved = canonical,
30                        Err(error) if error.kind() == io::ErrorKind::NotFound => {
31                            resolved = candidate;
32                            missing_depth = 1;
33                        }
34                        Err(error) => return Err(error),
35                    }
36                } else {
37                    resolved = candidate;
38                    if matches!(component, Component::Normal(_)) && missing_depth > 0 {
39                        missing_depth += 1;
40                    }
41                }
42            }
43            Component::CurDir => {}
44            Component::ParentDir => {
45                resolved.pop();
46                missing_depth = missing_depth.saturating_sub(1);
47            }
48        }
49    }
50    Ok(resolved)
51}
52
53const CACHE_DIRECTORY_TAG: &str = "Signature: 8a477f597d28d172789f06886806bc55\n\
54# This file is a cache directory tag created by ic-testkit.\n\
55# For information about cache directory tags see https://bford.info/cachedir/\n";
56pub(super) const CACHE_DIRECTORY_TAG_SIGNATURE: &str =
57    "Signature: 8a477f597d28d172789f06886806bc55";
58pub(super) const LAST_USED_FILE: &str = ".ic-testkit-last-used";
59const LAST_MAINTENANCE_FILE: &str = ".ic-testkit-last-maintenance";
60// Nanoseconds are written as decimal u128 values, which need at most 39 bytes.
61const MAX_TIMESTAMP_BYTES: usize = 39;
62pub(super) const RETENTION_LOCK_FILE: &str = ".ic-testkit-retention-v1";
63
64/// Acquired under the producer/namespace lock before handing an entry to a
65/// consumer. Clones share ownership; the OS releases locks on process exit.
66#[derive(Clone, Debug)]
67pub(super) struct RetainedCacheEntry {
68    path: PathBuf,
69    _lock: Arc<File>,
70}
71
72impl PartialEq for RetainedCacheEntry {
73    fn eq(&self, other: &Self) -> bool {
74        self.path == other.path
75    }
76}
77
78impl Eq for RetainedCacheEntry {}
79
80impl RetainedCacheEntry {
81    pub(super) fn path(&self) -> &Path {
82        &self.path
83    }
84
85    pub(super) fn acquire(path: &Path) -> Result<Self, CacheFsError> {
86        let file = open_cache_lock_file(&path.join(RETENTION_LOCK_FILE))?;
87        fs2::FileExt::lock_shared(&file).map_err(|source| CacheFsError {
88            operation: "retain cache entry",
89            path: path.to_owned(),
90            source,
91        })?;
92        Ok(Self {
93            path: path.to_owned(),
94            _lock: Arc::new(file),
95        })
96    }
97}
98
99/// The caller must hold the producer/namespace lock throughout this operation.
100pub(super) fn remove_unretained_entry(path: &Path) -> Result<(), CacheFsError> {
101    let metadata = match fs::symlink_metadata(path) {
102        Ok(metadata) => metadata,
103        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
104        Err(source) => {
105            return Err(CacheFsError {
106                operation: "inspect cache entry",
107                path: path.to_owned(),
108                source,
109            });
110        }
111    };
112    if !metadata.is_dir() {
113        return remove_path_if_present(path).map_err(|source| CacheFsError {
114            operation: "remove invalid cache entry",
115            path: path.to_owned(),
116            source,
117        });
118    }
119    let _lock =
120        try_lock_cache_file(&path.join(RETENTION_LOCK_FILE))?.ok_or_else(|| CacheFsError {
121            operation: "replace retained cache entry",
122            path: path.to_owned(),
123            source: io::Error::new(
124                io::ErrorKind::WouldBlock,
125                "cache entry is retained by a consumer",
126            ),
127        })?;
128    remove_path_if_present(path).map_err(|source| CacheFsError {
129        operation: "remove cache entry",
130        path: path.to_owned(),
131        source,
132    })
133}
134
135/// Caller-selected retention limits for content-addressed artifact entries.
136///
137/// Age pruning runs before size pruning. A policy without either limit scans
138/// the selected cache namespace and updates its cache metadata without
139/// removing entries. Entries retained by live acquisition records are skipped,
140/// even when this temporarily exceeds the limits. They become eligible for the
141/// next maintenance pass after their final owner drops or its process exits.
142#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
143pub struct ArtifactCachePrunePolicy {
144    max_age: Option<Duration>,
145    max_size_bytes: Option<u64>,
146}
147
148/// Summary of one lock-coordinated artifact-cache pruning pass.
149#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
150pub struct ArtifactCachePruneReport {
151    entries_scanned: usize,
152    entries_removed: usize,
153    bytes_before: u64,
154    bytes_removed: u64,
155    uncommitted_directories_removed: usize,
156    uncommitted_bytes_removed: u64,
157}
158
159/// Nonfatal retention attempted as part of a successful cache acquisition.
160#[non_exhaustive]
161#[derive(Clone, Debug, Eq, PartialEq)]
162pub enum ArtifactCacheMaintenance {
163    /// Configured retention completed under the cache lock.
164    Pruned(ArtifactCachePruneReport),
165    /// Configured retention failed after the requested artifacts were ready.
166    PruneFailed {
167        /// Cache error rendered without invalidating the successful acquisition.
168        message: String,
169    },
170}
171
172impl ArtifactCachePrunePolicy {
173    /// Create a policy that records cache metadata without removing entries.
174    #[must_use]
175    pub const fn new() -> Self {
176        Self {
177            max_age: None,
178            max_size_bytes: None,
179        }
180    }
181
182    /// Remove entries older than `max_age` before applying the size limit.
183    #[must_use]
184    pub const fn with_max_age(mut self, max_age: Duration) -> Self {
185        self.max_age = Some(max_age);
186        self
187    }
188
189    /// Remove least-recently-used entries until retained logical size is at most `bytes`.
190    #[must_use]
191    pub const fn with_max_size_bytes(mut self, bytes: u64) -> Self {
192        self.max_size_bytes = Some(bytes);
193        self
194    }
195
196    /// Configured maximum entry age, if any.
197    #[must_use]
198    pub const fn max_age(self) -> Option<Duration> {
199        self.max_age
200    }
201
202    /// Configured maximum logical cache size in bytes, if any.
203    #[must_use]
204    pub const fn max_size_bytes(self) -> Option<u64> {
205        self.max_size_bytes
206    }
207
208    pub(super) fn maintenance_identity(self) -> String {
209        format!(
210            "age={:?};size={:?}",
211            self.max_age.map(|duration| duration.as_nanos()),
212            self.max_size_bytes
213        )
214    }
215}
216
217impl ArtifactCachePruneReport {
218    /// Number of content-addressed directories considered for pruning.
219    #[must_use]
220    pub const fn entries_scanned(self) -> usize {
221        self.entries_scanned
222    }
223
224    /// Number of content-addressed directories removed.
225    #[must_use]
226    pub const fn entries_removed(self) -> usize {
227        self.entries_removed
228    }
229
230    /// Number of content-addressed directories retained.
231    #[must_use]
232    pub const fn entries_retained(self) -> usize {
233        self.entries_scanned.saturating_sub(self.entries_removed)
234    }
235
236    /// Logical bytes occupied by scanned entries before pruning.
237    #[must_use]
238    pub const fn bytes_before(self) -> u64 {
239        self.bytes_before
240    }
241
242    /// Logical bytes removed by pruning.
243    #[must_use]
244    pub const fn bytes_removed(self) -> u64 {
245        self.bytes_removed
246    }
247
248    /// Logical bytes occupied by retained entries after pruning.
249    #[must_use]
250    pub const fn bytes_retained(self) -> u64 {
251        self.bytes_before.saturating_sub(self.bytes_removed)
252    }
253
254    /// Abandoned transaction directories removed outside the committed-entry totals.
255    #[must_use]
256    pub const fn uncommitted_directories_removed(self) -> usize {
257        self.uncommitted_directories_removed
258    }
259
260    /// Logical bytes removed from abandoned transaction directories.
261    #[must_use]
262    pub const fn uncommitted_bytes_removed(self) -> u64 {
263        self.uncommitted_bytes_removed
264    }
265
266    pub(super) const fn record_uncommitted_removal(&mut self, bytes: u64) {
267        self.uncommitted_directories_removed += 1;
268        self.uncommitted_bytes_removed = self.uncommitted_bytes_removed.saturating_add(bytes);
269    }
270}
271
272impl ArtifactCacheMaintenance {
273    /// Successful pruning report, or `None` when maintenance failed.
274    #[must_use]
275    pub const fn prune_report(&self) -> Option<ArtifactCachePruneReport> {
276        match self {
277            Self::Pruned(report) => Some(*report),
278            Self::PruneFailed { .. } => None,
279        }
280    }
281
282    /// Rendered maintenance failure, or `None` when pruning succeeded.
283    #[must_use]
284    pub fn failure_message(&self) -> Option<&str> {
285        match self {
286            Self::Pruned(_) => None,
287            Self::PruneFailed { message } => Some(message),
288        }
289    }
290}
291
292#[derive(Debug)]
293pub(super) struct CacheFsError {
294    pub(super) operation: &'static str,
295    pub(super) path: PathBuf,
296    pub(super) source: io::Error,
297}
298
299pub(super) fn ensure_cache_directory_tag(cache_root: &Path) -> Result<(), CacheFsError> {
300    let path = cache_root.join("CACHEDIR.TAG");
301    // The standard recognizes the first 43 bytes, without requiring a newline
302    // or interpreting the remaining text. Symlinks are not valid tag files.
303    let mut signature = [0_u8; CACHE_DIRECTORY_TAG_SIGNATURE.len()];
304    if fs::symlink_metadata(&path).is_ok_and(|metadata| metadata.file_type().is_file())
305        && File::open(&path)
306            .and_then(|mut file| file.read_exact(&mut signature))
307            .is_ok()
308        && signature == CACHE_DIRECTORY_TAG_SIGNATURE.as_bytes()
309    {
310        return Ok(());
311    }
312    write_atomic(&path, CACHE_DIRECTORY_TAG.as_bytes()).map_err(|source| CacheFsError {
313        operation: "write cache directory tag",
314        path,
315        source,
316    })
317}
318
319pub(super) fn lock_cache_file(path: &Path) -> Result<(File, Duration), CacheFsError> {
320    let file = open_cache_lock_file(path)?;
321    let started = Instant::now();
322    file.lock_exclusive().map_err(|source| CacheFsError {
323        operation: "lock cache",
324        path: path.to_owned(),
325        source,
326    })?;
327    Ok((file, started.elapsed()))
328}
329
330pub(super) fn lock_cache_file_with_wait_observer(
331    path: &Path,
332    poll_interval: Duration,
333    mut observer: impl FnMut(Duration),
334) -> Result<(File, Duration), CacheFsError> {
335    let file = open_cache_lock_file(path)?;
336    let started = Instant::now();
337    loop {
338        match file.try_lock_exclusive() {
339            Ok(()) => return Ok((file, started.elapsed())),
340            Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
341                observer(started.elapsed());
342                thread::sleep(poll_interval.min(Duration::from_millis(25)));
343            }
344            Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
345            Err(source) => {
346                return Err(CacheFsError {
347                    operation: "try lock cache",
348                    path: path.to_owned(),
349                    source,
350                });
351            }
352        }
353    }
354}
355
356pub(super) fn try_lock_cache_file(path: &Path) -> Result<Option<File>, CacheFsError> {
357    let file = open_cache_lock_file(path)?;
358    match file.try_lock_exclusive() {
359        Ok(()) => Ok(Some(file)),
360        Err(error) if error.kind() == io::ErrorKind::WouldBlock => Ok(None),
361        Err(source) => Err(CacheFsError {
362            operation: "try lock cache",
363            path: path.to_owned(),
364            source,
365        }),
366    }
367}
368
369fn open_cache_lock_file(path: &Path) -> Result<File, CacheFsError> {
370    if let Some(parent) = path.parent() {
371        fs::create_dir_all(parent).map_err(|source| CacheFsError {
372            operation: "create cache lock directory",
373            path: parent.to_owned(),
374            source,
375        })?;
376    }
377    OpenOptions::new()
378        .create(true)
379        .read(true)
380        .write(true)
381        .truncate(false)
382        .open(path)
383        .map_err(|source| CacheFsError {
384            operation: "open cache lock",
385            path: path.to_owned(),
386            source,
387        })
388}
389
390pub(super) fn record_cache_entry_use(path: &Path) -> Result<(), CacheFsError> {
391    write_last_used(path, SystemTime::now())
392}
393
394pub(super) fn cache_maintenance_due(
395    path: &Path,
396    minimum_interval: Option<Duration>,
397    maintenance_identity: &str,
398) -> Result<bool, CacheFsError> {
399    let Some(minimum_interval) = minimum_interval else {
400        return Ok(true);
401    };
402    let marker = path.join(LAST_MAINTENANCE_FILE);
403    // Allow both LF and CRLF for the timestamp and policy-identity lines.
404    let maximum_len = MAX_TIMESTAMP_BYTES + maintenance_identity.len() + 4;
405    let contents = match read_stamp_with_limit(&marker, maximum_len) {
406        Ok(Some(contents)) => contents,
407        Ok(None) => return Ok(true),
408        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(true),
409        Err(source) => {
410            return Err(CacheFsError {
411                operation: "read cache maintenance time",
412                path: marker,
413                source,
414            });
415        }
416    };
417    let mut lines = contents.lines();
418    let Some(last_maintenance) = lines.next().and_then(decode_system_time) else {
419        return Ok(true);
420    };
421    if lines.next() != Some(maintenance_identity) {
422        return Ok(true);
423    }
424    Ok(match SystemTime::now().duration_since(last_maintenance) {
425        Ok(elapsed) => elapsed >= minimum_interval,
426        Err(_) => true,
427    })
428}
429
430pub(super) fn record_cache_maintenance(
431    path: &Path,
432    maintenance_identity: &str,
433) -> Result<(), CacheFsError> {
434    let marker = path.join(LAST_MAINTENANCE_FILE);
435    let elapsed = encode_system_time(&marker, SystemTime::now())?;
436    let contents = format!("{}\n{maintenance_identity}\n", elapsed.as_nanos());
437    write_atomic(&marker, contents.as_bytes()).map_err(|source| CacheFsError {
438        operation: "record cache maintenance time",
439        path: marker,
440        source,
441    })
442}
443
444pub(super) fn perform_scheduled_cache_maintenance(
445    path: &Path,
446    minimum_interval: Option<Duration>,
447    maintenance_identity: &str,
448    maintenance: impl FnOnce() -> Result<ArtifactCachePruneReport, String>,
449) -> (Option<ArtifactCacheMaintenance>, Option<Duration>) {
450    let started = Instant::now();
451    match cache_maintenance_due(path, minimum_interval, maintenance_identity) {
452        Ok(false) => return (None, Some(started.elapsed())),
453        Ok(true) => {}
454        Err(error) => {
455            return (
456                Some(ArtifactCacheMaintenance::PruneFailed {
457                    message: error.to_string(),
458                }),
459                Some(started.elapsed()),
460            );
461        }
462    }
463
464    let result = maintenance();
465    let marker = record_cache_maintenance(path, maintenance_identity);
466    let outcome = match (result, marker) {
467        (Ok(report), Ok(())) => ArtifactCacheMaintenance::Pruned(report),
468        (Err(message), Ok(())) => ArtifactCacheMaintenance::PruneFailed { message },
469        (Ok(_), Err(error)) => ArtifactCacheMaintenance::PruneFailed {
470            message: error.to_string(),
471        },
472        (Err(message), Err(marker)) => ArtifactCacheMaintenance::PruneFailed {
473            message: format!(
474                "{message}; additionally failed to record the maintenance attempt: {marker}"
475            ),
476        },
477    };
478    (Some(outcome), Some(started.elapsed()))
479}
480
481pub(super) fn write_last_used(path: &Path, last_used: SystemTime) -> Result<(), CacheFsError> {
482    let marker = path.join(LAST_USED_FILE);
483    write_system_time(&marker, last_used, "record cache use time")
484}
485
486fn write_system_time(
487    path: &Path,
488    timestamp: SystemTime,
489    operation: &'static str,
490) -> Result<(), CacheFsError> {
491    let elapsed = encode_system_time(path, timestamp)?;
492    write_atomic(path, elapsed.as_nanos().to_string().as_bytes()).map_err(|source| CacheFsError {
493        operation,
494        path: path.to_owned(),
495        source,
496    })
497}
498
499fn encode_system_time(path: &Path, timestamp: SystemTime) -> Result<Duration, CacheFsError> {
500    timestamp
501        .duration_since(UNIX_EPOCH)
502        .map_err(|source| CacheFsError {
503            operation: "encode cache time",
504            path: path.to_owned(),
505            source: io::Error::new(io::ErrorKind::InvalidInput, source),
506        })
507}
508
509fn decode_system_time(contents: &str) -> Option<SystemTime> {
510    let nanoseconds = contents.parse::<u128>().ok()?;
511    let seconds = u64::try_from(nanoseconds / 1_000_000_000).ok()?;
512    let subsecond_nanos = (nanoseconds % 1_000_000_000) as u32;
513    UNIX_EPOCH.checked_add(Duration::new(seconds, subsecond_nanos))
514}
515
516pub(super) fn prune_direct_child_directories(
517    cache_root: &Path,
518    policy: ArtifactCachePrunePolicy,
519    protected_entry: Option<&Path>,
520    is_eligible: impl Fn(&Path) -> bool,
521) -> Result<ArtifactCachePruneReport, CacheFsError> {
522    let mut entries = cache_entries(cache_root, is_eligible)?;
523    let bytes_before = entries
524        .iter()
525        .fold(0_u64, |total, entry| total.saturating_add(entry.bytes));
526    let mut report = ArtifactCachePruneReport {
527        entries_scanned: entries.len(),
528        entries_removed: 0,
529        bytes_before,
530        bytes_removed: 0,
531        uncommitted_directories_removed: 0,
532        uncommitted_bytes_removed: 0,
533    };
534    let now = SystemTime::now();
535
536    if let Some(max_age) = policy.max_age() {
537        for entry in &mut entries {
538            let age = now.duration_since(entry.last_used).unwrap_or_default();
539            if protected_entry != Some(entry.path.as_path()) && age > max_age {
540                remove_cache_entry(entry, &mut report)?;
541            }
542        }
543    }
544
545    if let Some(max_size_bytes) = policy.max_size_bytes()
546        && report.bytes_retained() > max_size_bytes
547    {
548        entries.sort_by(|left, right| {
549            left.last_used
550                .cmp(&right.last_used)
551                .then_with(|| left.path.cmp(&right.path))
552        });
553        for entry in &mut entries {
554            if report.bytes_retained() <= max_size_bytes {
555                break;
556            }
557            if protected_entry == Some(entry.path.as_path()) {
558                continue;
559            }
560            remove_cache_entry(entry, &mut report)?;
561        }
562    }
563
564    Ok(report)
565}
566
567pub(super) fn directory_logical_size(path: &Path) -> io::Result<u64> {
568    let mut total = 0_u64;
569    let mut pending = vec![path.to_owned()];
570    while let Some(current) = pending.pop() {
571        let metadata = fs::symlink_metadata(&current)?;
572        if metadata.is_dir() {
573            for entry in fs::read_dir(&current)? {
574                let path = entry?.path();
575                let metadata = fs::symlink_metadata(&path)?;
576                if metadata.is_dir() {
577                    pending.push(path);
578                } else {
579                    total = total.saturating_add(metadata.len());
580                }
581            }
582        } else {
583            total = total.saturating_add(metadata.len());
584        }
585    }
586    Ok(total)
587}
588
589pub(super) fn is_sha256_directory(path: &Path) -> bool {
590    path.file_name().is_some_and(|name| {
591        let bytes = name.as_encoded_bytes();
592        bytes.len() == 64 && bytes.iter().all(u8::is_ascii_hexdigit)
593    })
594}
595
596pub(super) fn remove_path_if_present(path: &Path) -> io::Result<()> {
597    let metadata = match fs::symlink_metadata(path) {
598        Ok(metadata) => metadata,
599        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
600        Err(error) => return Err(error),
601    };
602    if metadata.file_type().is_dir() {
603        fs::remove_dir_all(path)
604    } else {
605        fs::remove_file(path)
606    }
607}
608
609struct CacheEntry {
610    path: PathBuf,
611    bytes: u64,
612    last_used: SystemTime,
613    removed: bool,
614}
615
616impl std::fmt::Display for CacheFsError {
617    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
618        write!(
619            formatter,
620            "failed to {} at {}: {}",
621            self.operation,
622            self.path.display(),
623            self.source
624        )
625    }
626}
627
628impl std::error::Error for CacheFsError {
629    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
630        Some(&self.source)
631    }
632}
633
634fn cache_entries(
635    cache_root: &Path,
636    is_eligible: impl Fn(&Path) -> bool,
637) -> Result<Vec<CacheEntry>, CacheFsError> {
638    let read_dir = match fs::read_dir(cache_root) {
639        Ok(read_dir) => read_dir,
640        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
641        Err(source) => {
642            return Err(CacheFsError {
643                operation: "read cache directory",
644                path: cache_root.to_owned(),
645                source,
646            });
647        }
648    };
649    let mut entries = Vec::new();
650    for directory_entry in read_dir {
651        let directory_entry = directory_entry.map_err(|source| CacheFsError {
652            operation: "read cache entry",
653            path: cache_root.to_owned(),
654            source,
655        })?;
656        let path = directory_entry.path();
657        let file_type = directory_entry.file_type().map_err(|source| CacheFsError {
658            operation: "inspect cache entry",
659            path: path.clone(),
660            source,
661        })?;
662        if !file_type.is_dir() || !is_eligible(&path) {
663            continue;
664        }
665        let bytes = directory_logical_size(&path).map_err(|source| CacheFsError {
666            operation: "measure cache entry",
667            path: path.clone(),
668            source,
669        })?;
670        let last_used = cache_entry_last_used(&path).map_err(|source| CacheFsError {
671            operation: "read cache use time",
672            path: path.clone(),
673            source,
674        })?;
675        entries.push(CacheEntry {
676            path,
677            bytes,
678            last_used,
679            removed: false,
680        });
681    }
682    Ok(entries)
683}
684
685pub(super) fn cache_entry_last_used(path: &Path) -> io::Result<SystemTime> {
686    let marker = path.join(LAST_USED_FILE);
687    if let Ok(Some(contents)) = read_stamp_with_limit(&marker, MAX_TIMESTAMP_BYTES)
688        && let Some(timestamp) = decode_system_time(&contents)
689    {
690        return Ok(timestamp);
691    }
692    fs::metadata(path)?.modified()
693}
694
695fn remove_cache_entry(
696    entry: &mut CacheEntry,
697    report: &mut ArtifactCachePruneReport,
698) -> Result<(), CacheFsError> {
699    if entry.removed {
700        return Ok(());
701    }
702    let Some(_retention_lock) = try_lock_cache_file(&entry.path.join(RETENTION_LOCK_FILE))? else {
703        return Ok(());
704    };
705    remove_path_if_present(&entry.path).map_err(|source| CacheFsError {
706        operation: "prune cache entry",
707        path: entry.path.clone(),
708        source,
709    })?;
710    entry.removed = true;
711    report.entries_removed += 1;
712    report.bytes_removed = report.bytes_removed.saturating_add(entry.bytes);
713    Ok(())
714}
715
716#[cfg(test)]
717mod tests {
718    #[cfg(unix)]
719    use super::canonicalize_allow_missing;
720    use super::directory_logical_size;
721    use crate::artifacts::test_support::unique_temp_directory;
722    use std::fs;
723
724    #[test]
725    fn last_use_markers_preserve_timestamps_and_bounded_fallbacks() {
726        use std::time::{Duration, SystemTime, UNIX_EPOCH};
727
728        let root = unique_temp_directory("bounded-last-use-marker");
729        let marker = root.join(super::LAST_USED_FILE);
730        let modified = || fs::metadata(&root).unwrap().modified().unwrap();
731        assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
732        for timestamp in [
733            UNIX_EPOCH + Duration::from_nanos(123),
734            SystemTime::now() + Duration::from_secs(3600),
735        ] {
736            super::write_last_used(&root, timestamp).unwrap();
737            assert_eq!(super::cache_entry_last_used(&root).unwrap(), timestamp);
738        }
739        let overflowing_timestamp = u128::MAX.to_string();
740        for invalid in [
741            b"invalid".as_slice(),
742            overflowing_timestamp.as_bytes(),
743            &[0xff],
744        ] {
745            fs::write(&marker, invalid).unwrap();
746            assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
747        }
748        fs::File::create(&marker)
749            .unwrap()
750            .set_len(1024 * 1024 * 1024)
751            .unwrap();
752        assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
753        fs::remove_file(&marker).unwrap();
754        fs::create_dir(&marker).unwrap();
755        assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
756        fs::remove_dir_all(root).unwrap();
757    }
758
759    #[test]
760    fn maintenance_markers_preserve_policy_intervals_and_read_errors() {
761        use std::time::{Duration, SystemTime, UNIX_EPOCH};
762
763        let root = unique_temp_directory("bounded-maintenance-marker");
764        let marker = root.join(super::LAST_MAINTENANCE_FILE);
765        let identity = super::ArtifactCachePrunePolicy::new().maintenance_identity();
766        let interval = Some(Duration::from_secs(3600));
767        let due = || super::cache_maintenance_due(&root, interval, &identity).unwrap();
768        assert!(due());
769        super::record_cache_maintenance(&root, &identity).unwrap();
770        assert!(!due());
771        assert!(super::cache_maintenance_due(&root, interval, "other-policy").unwrap());
772        assert!(super::cache_maintenance_due(&root, Some(Duration::ZERO), &identity).unwrap());
773
774        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
775        for suffix in ["", "\r\n"] {
776            fs::write(&marker, format!("{}\r\n{identity}{suffix}", now.as_nanos())).unwrap();
777            assert!(!due());
778        }
779        for timestamp in [
780            "invalid".to_owned(),
781            "0".to_owned(),
782            (now + Duration::from_secs(3600)).as_nanos().to_string(),
783        ] {
784            fs::write(&marker, format!("{timestamp}\n{identity}\n")).unwrap();
785            assert!(due());
786        }
787        super::record_cache_maintenance(&root, &identity).unwrap();
788        fs::OpenOptions::new()
789            .write(true)
790            .open(&marker)
791            .unwrap()
792            .set_len(1024 * 1024 * 1024)
793            .unwrap();
794        assert!(due());
795
796        fs::write(&marker, [0xff]).unwrap();
797        let error = super::cache_maintenance_due(&root, interval, &identity).unwrap_err();
798        assert_eq!(error.operation, "read cache maintenance time");
799        assert_eq!(error.path, marker);
800        assert_eq!(error.source.kind(), std::io::ErrorKind::InvalidData);
801        fs::remove_file(&marker).unwrap();
802        fs::create_dir(&marker).unwrap();
803        assert!(super::cache_maintenance_due(&root, interval, &identity).is_err());
804        assert!(super::cache_maintenance_due(&root, None, &identity).unwrap());
805        fs::remove_dir_all(root).unwrap();
806    }
807
808    #[test]
809    fn cache_directory_tags_preserve_valid_standard_signatures() {
810        let root = unique_temp_directory("cache-tag-signatures");
811        let tag = root.join("CACHEDIR.TAG");
812        let signature = "Signature: 8a477f597d28d172789f06886806bc55";
813        for contents in [
814            signature.to_owned(),
815            format!("{signature}\r\n# Created by another application\r\n"),
816            format!("{signature}\n# {}\n", "comment".repeat(100_000)),
817        ] {
818            fs::write(&tag, &contents).unwrap();
819            super::ensure_cache_directory_tag(&root).unwrap();
820            assert_eq!(fs::read_to_string(&tag).unwrap(), contents);
821        }
822        for invalid in ["", &signature[..42], "Signature: incorrect"] {
823            fs::write(&tag, invalid).unwrap();
824            super::ensure_cache_directory_tag(&root).unwrap();
825            assert_eq!(
826                fs::read_to_string(&tag).unwrap(),
827                super::CACHE_DIRECTORY_TAG
828            );
829        }
830        fs::remove_dir_all(root).unwrap();
831    }
832
833    #[test]
834    #[cfg(unix)]
835    fn cache_directory_tag_replaces_symlinks_without_changing_referents() {
836        let root = unique_temp_directory("cache-tag-symlink");
837        let referent = root.join("other-application-tag");
838        let contents = "Signature: 8a477f597d28d172789f06886806bc55\n# Preserve this file\n";
839        fs::write(&referent, contents).unwrap();
840        let tag = root.join("CACHEDIR.TAG");
841        std::os::unix::fs::symlink(&referent, &tag).unwrap();
842        super::ensure_cache_directory_tag(&root).unwrap();
843        assert!(fs::symlink_metadata(&tag).unwrap().file_type().is_file());
844        assert_eq!(fs::read_to_string(&referent).unwrap(), contents);
845        assert_eq!(
846            fs::read_to_string(&tag).unwrap(),
847            super::CACHE_DIRECTORY_TAG
848        );
849
850        fs::remove_file(&tag).unwrap();
851        fs::create_dir(&tag).unwrap();
852        let error = super::ensure_cache_directory_tag(&root).unwrap_err();
853        assert_eq!(error.operation, "write cache directory tag");
854        assert!(tag.is_dir());
855        fs::remove_dir_all(root).unwrap();
856    }
857
858    #[test]
859    fn directory_size_sums_wide_and_nested_files() {
860        let root = unique_temp_directory("directory-logical-size");
861        assert_eq!(directory_logical_size(&root).unwrap(), 0);
862        fs::create_dir_all(root.join("wide")).unwrap();
863        fs::create_dir_all(root.join("nested/deep/empty")).unwrap();
864        let mut expected = 0;
865        for index in 0..128 {
866            let bytes = vec![42; index % 13];
867            fs::write(root.join("wide").join(index.to_string()), &bytes).unwrap();
868            expected += bytes.len() as u64;
869        }
870        let sparse = root.join("nested/deep/sparse");
871        fs::File::create(&sparse)
872            .unwrap()
873            .set_len(1024 * 1024)
874            .unwrap();
875        assert_eq!(directory_logical_size(&sparse).unwrap(), 1024 * 1024);
876        assert_eq!(
877            directory_logical_size(&root).unwrap(),
878            expected + 1024 * 1024
879        );
880        assert_eq!(
881            directory_logical_size(&root.join("missing"))
882                .unwrap_err()
883                .kind(),
884            std::io::ErrorKind::NotFound,
885        );
886        fs::remove_dir_all(root).unwrap();
887    }
888
889    #[test]
890    #[cfg(unix)]
891    fn directory_size_counts_symlinks_without_following_them() {
892        let root = unique_temp_directory("directory-size-symlinks");
893        let walked = root.join("walked");
894        fs::create_dir_all(&walked).unwrap();
895        fs::create_dir_all(root.join("external")).unwrap();
896        fs::write(root.join("external/payload"), vec![42; 4096]).unwrap();
897        fs::write(root.join("outside-file"), vec![42; 4096]).unwrap();
898        fs::write(walked.join("payload"), b"abc").unwrap();
899        let targets = ["../external", "../outside-file", "missing", "."];
900        for (index, target) in targets.iter().enumerate() {
901            std::os::unix::fs::symlink(target, walked.join(index.to_string())).unwrap();
902        }
903        let expected = 3 + targets
904            .iter()
905            .map(|target| target.len() as u64)
906            .sum::<u64>();
907        assert_eq!(directory_logical_size(&walked).unwrap(), expected);
908        assert_eq!(
909            directory_logical_size(&walked.join("0")).unwrap(),
910            targets[0].len() as u64,
911        );
912        fs::remove_dir_all(root).unwrap();
913    }
914
915    #[test]
916    #[cfg(unix)]
917    fn missing_parent_traversal_resumes_existing_symlink_resolution() {
918        let root = unique_temp_directory("canonical-missing-parent");
919        let target = root.join("real");
920        fs::create_dir_all(&target).unwrap();
921        std::os::unix::fs::symlink(&target, root.join("alias")).unwrap();
922        let path = root.join("missing/../alias/generated/nested/../output");
923        assert_eq!(
924            canonicalize_allow_missing(&path).unwrap(),
925            target.canonicalize().unwrap().join("generated/output")
926        );
927        assert_eq!(
928            canonicalize_allow_missing(&root.join("alias/../other/output")).unwrap(),
929            root.canonicalize().unwrap().join("other/output")
930        );
931        fs::remove_dir_all(root).unwrap();
932    }
933}
ic_testkit/artifacts/cache_fs.rs

ic_testkit/artifacts/
cache_fs.rs
