ic_testkit/artifacts/
cache_fs.rs

1use fs2::FileExt as _;
2use std::{
3    fs::{self, File, OpenOptions},
4    io::{self, Read as _},
5    path::{Component, Path, PathBuf},
6    sync::Arc,
7    thread,
8    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
9};
10
11use super::digest::{read_stamp_with_limit, write_atomic};
12
13/// Resolve existing components through symlinks and normalize a missing suffix.
14/// Parent traversal can return from a missing suffix to existing components.
15pub(super) fn canonicalize_allow_missing(path: &Path) -> io::Result<PathBuf> {
16    let absolute = if path.is_absolute() {
17        path.to_owned()
18    } else {
19        std::env::current_dir()?.join(path)
20    };
21    let mut resolved = PathBuf::new();
22    let mut missing_depth = 0_usize;
23    for component in absolute.components() {
24        match component {
25            Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {
26                let candidate = resolved.join(component.as_os_str());
27                if missing_depth == 0 && matches!(component, Component::Normal(_)) {
28                    match candidate.canonicalize() {
29                        Ok(canonical) => resolved = canonical,
30                        Err(error) if error.kind() == io::ErrorKind::NotFound => {
31                            resolved = candidate;
32                            missing_depth = 1;
33                        }
34                        Err(error) => return Err(error),
35                    }
36                } else {
37                    resolved = candidate;
38                    if matches!(component, Component::Normal(_)) && missing_depth > 0 {
39                        missing_depth += 1;
40                    }
41                }
42            }
43            Component::CurDir => {}
44            Component::ParentDir => {
45                resolved.pop();
46                missing_depth = missing_depth.saturating_sub(1);
47            }
48        }
49    }
50    Ok(resolved)
51}
52
53const CACHE_DIRECTORY_TAG: &str = "Signature: 8a477f597d28d172789f06886806bc55\n\
54# This file is a cache directory tag created by ic-testkit.\n\
55# For information about cache directory tags see https://bford.info/cachedir/\n";
56pub(super) const CACHE_DIRECTORY_TAG_SIGNATURE: &str =
57    "Signature: 8a477f597d28d172789f06886806bc55";
58pub(super) const LAST_USED_FILE: &str = ".ic-testkit-last-used";
59const LAST_MAINTENANCE_FILE: &str = ".ic-testkit-last-maintenance";
60// Nanoseconds are written as decimal u128 values, which need at most 39 bytes.
61const MAX_TIMESTAMP_BYTES: usize = 39;
62pub(super) const RETENTION_LOCK_FILE: &str = ".ic-testkit-retention-v1";
63
64/// Acquired under the producer/namespace lock before handing an entry to a
65/// consumer. Clones share ownership; the OS releases locks on process exit.
66#[derive(Clone, Debug)]
67pub(super) struct RetainedCacheEntry {
68    path: PathBuf,
69    _lock: Arc<File>,
70}
71
72impl PartialEq for RetainedCacheEntry {
73    fn eq(&self, other: &Self) -> bool {
74        self.path == other.path
75    }
76}
77
78impl Eq for RetainedCacheEntry {}
79
80impl RetainedCacheEntry {
81    pub(super) fn path(&self) -> &Path {
82        &self.path
83    }
84
85    pub(super) fn acquire(path: &Path) -> Result<Self, CacheFsError> {
86        let file = open_cache_lock_file(&path.join(RETENTION_LOCK_FILE))?;
87        fs2::FileExt::lock_shared(&file).map_err(|source| CacheFsError {
88            operation: "retain cache entry",
89            path: path.to_owned(),
90            source,
91        })?;
92        Ok(Self {
93            path: path.to_owned(),
94            _lock: Arc::new(file),
95        })
96    }
97}
98
99/// The caller must hold the producer/namespace lock throughout this operation.
100pub(super) fn remove_unretained_entry(path: &Path) -> Result<(), CacheFsError> {
101    let metadata = match fs::symlink_metadata(path) {
102        Ok(metadata) => metadata,
103        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
104        Err(source) => {
105            return Err(CacheFsError {
106                operation: "inspect cache entry",
107                path: path.to_owned(),
108                source,
109            });
110        }
111    };
112    if !metadata.is_dir() {
113        return remove_path_if_present(path).map_err(|source| CacheFsError {
114            operation: "remove invalid cache entry",
115            path: path.to_owned(),
116            source,
117        });
118    }
119    let _lock =
120        try_lock_cache_file(&path.join(RETENTION_LOCK_FILE))?.ok_or_else(|| CacheFsError {
121            operation: "replace retained cache entry",
122            path: path.to_owned(),
123            source: io::Error::new(
124                io::ErrorKind::WouldBlock,
125                "cache entry is retained by a consumer",
126            ),
127        })?;
128    remove_path_if_present(path).map_err(|source| CacheFsError {
129        operation: "remove cache entry",
130        path: path.to_owned(),
131        source,
132    })
133}
134
135/// Caller-selected retention limits for content-addressed artifact entries.
136///
137/// Age pruning runs before size pruning. A policy without either limit scans
138/// the selected cache namespace and updates its cache metadata without
139/// removing entries. Entries retained by live acquisition records are skipped,
140/// even when this temporarily exceeds the limits. They become eligible for the
141/// next maintenance pass after their final owner drops or its process exits.
142#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
143pub struct ArtifactCachePrunePolicy {
144    max_age: Option<Duration>,
145    max_size_bytes: Option<u64>,
146}
147
148/// Summary of one lock-coordinated artifact-cache pruning pass.
149#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
150pub struct ArtifactCachePruneReport {
151    entries_scanned: usize,
152    entries_removed: usize,
153    bytes_before: u64,
154    bytes_removed: u64,
155    uncommitted_directories_removed: usize,
156    uncommitted_bytes_removed: u64,
157}
158
159/// Nonfatal retention attempted as part of a successful cache acquisition.
160#[non_exhaustive]
161#[derive(Clone, Debug, Eq, PartialEq)]
162pub enum ArtifactCacheMaintenance {
163    /// Configured retention completed under the cache lock.
164    Pruned(ArtifactCachePruneReport),
165    /// Configured retention failed after the requested artifacts were ready.
166    PruneFailed {
167        /// Cache error rendered without invalidating the successful acquisition.
168        message: String,
169    },
170}
171
172impl ArtifactCachePrunePolicy {
173    /// Create a policy that records cache metadata without removing entries.
174    #[must_use]
175    pub const fn new() -> Self {
176        Self {
177            max_age: None,
178            max_size_bytes: None,
179        }
180    }
181
182    /// Remove entries older than `max_age` before applying the size limit.
183    #[must_use]
184    pub const fn with_max_age(mut self, max_age: Duration) -> Self {
185        self.max_age = Some(max_age);
186        self
187    }
188
189    /// Remove least-recently-used entries until retained logical size is at most `bytes`.
190    #[must_use]
191    pub const fn with_max_size_bytes(mut self, bytes: u64) -> Self {
192        self.max_size_bytes = Some(bytes);
193        self
194    }
195
196    /// Configured maximum entry age, if any.
197    #[must_use]
198    pub const fn max_age(self) -> Option<Duration> {
199        self.max_age
200    }
201
202    /// Configured maximum logical cache size in bytes, if any.
203    #[must_use]
204    pub const fn max_size_bytes(self) -> Option<u64> {
205        self.max_size_bytes
206    }
207
208    pub(super) fn maintenance_identity(self) -> String {
209        format!(
210            "age={:?};size={:?}",
211            self.max_age.map(|duration| duration.as_nanos()),
212            self.max_size_bytes
213        )
214    }
215}
216
217impl ArtifactCachePruneReport {
218    /// Number of content-addressed directories considered for pruning.
219    #[must_use]
220    pub const fn entries_scanned(self) -> usize {
221        self.entries_scanned
222    }
223
224    /// Number of content-addressed directories removed.
225    #[must_use]
226    pub const fn entries_removed(self) -> usize {
227        self.entries_removed
228    }
229
230    /// Number of content-addressed directories retained.
231    #[must_use]
232    pub const fn entries_retained(self) -> usize {
233        self.entries_scanned.saturating_sub(self.entries_removed)
234    }
235
236    /// Logical bytes occupied by scanned entries before pruning.
237    #[must_use]
238    pub const fn bytes_before(self) -> u64 {
239        self.bytes_before
240    }
241
242    /// Logical bytes removed by pruning.
243    #[must_use]
244    pub const fn bytes_removed(self) -> u64 {
245        self.bytes_removed
246    }
247
248    /// Logical bytes occupied by retained entries after pruning.
249    #[must_use]
250    pub const fn bytes_retained(self) -> u64 {
251        self.bytes_before.saturating_sub(self.bytes_removed)
252    }
253
254    /// Abandoned transaction directories removed outside the committed-entry totals.
255    #[must_use]
256    pub const fn uncommitted_directories_removed(self) -> usize {
257        self.uncommitted_directories_removed
258    }
259
260    /// Logical bytes removed from abandoned transaction directories.
261    #[must_use]
262    pub const fn uncommitted_bytes_removed(self) -> u64 {
263        self.uncommitted_bytes_removed
264    }
265
266    pub(super) const fn record_uncommitted_removal(&mut self, bytes: u64) {
267        self.uncommitted_directories_removed += 1;
268        self.uncommitted_bytes_removed = self.uncommitted_bytes_removed.saturating_add(bytes);
269    }
270}
271
272impl ArtifactCacheMaintenance {
273    /// Successful pruning report, or `None` when maintenance failed.
274    #[must_use]
275    pub const fn prune_report(&self) -> Option<ArtifactCachePruneReport> {
276        match self {
277            Self::Pruned(report) => Some(*report),
278            Self::PruneFailed { .. } => None,
279        }
280    }
281
282    /// Rendered maintenance failure, or `None` when pruning succeeded.
283    #[must_use]
284    pub fn failure_message(&self) -> Option<&str> {
285        match self {
286            Self::Pruned(_) => None,
287            Self::PruneFailed { message } => Some(message),
288        }
289    }
290}
291
292#[derive(Debug)]
293pub(super) struct CacheFsError {
294    pub(super) operation: &'static str,
295    pub(super) path: PathBuf,
296    pub(super) source: io::Error,
297}
298
299pub(super) fn ensure_cache_directory_tag(cache_root: &Path) -> Result<(), CacheFsError> {
300    let path = cache_root.join("CACHEDIR.TAG");
301    // The standard recognizes the first 43 bytes, without requiring a newline
302    // or interpreting the remaining text. Symlinks are not valid tag files.
303    let mut signature = [0_u8; CACHE_DIRECTORY_TAG_SIGNATURE.len()];
304    if fs::symlink_metadata(&path).is_ok_and(|metadata| metadata.file_type().is_file())
305        && File::open(&path)
306            .and_then(|mut file| file.read_exact(&mut signature))
307            .is_ok()
308        && signature == CACHE_DIRECTORY_TAG_SIGNATURE.as_bytes()
309    {
310        return Ok(());
311    }
312    write_atomic(&path, CACHE_DIRECTORY_TAG.as_bytes()).map_err(|source| CacheFsError {
313        operation: "write cache directory tag",
314        path,
315        source,
316    })
317}
318
319pub(super) fn lock_cache_file(path: &Path) -> Result<(File, Duration), CacheFsError> {
320    let file = open_cache_lock_file(path)?;
321    let started = Instant::now();
322    file.lock_exclusive().map_err(|source| CacheFsError {
323        operation: "lock cache",
324        path: path.to_owned(),
325        source,
326    })?;
327    Ok((file, started.elapsed()))
328}
329
330pub(super) fn lock_cache_file_with_wait_observer(
331    path: &Path,
332    poll_interval: Duration,
333    mut observer: impl FnMut(Duration),
334) -> Result<(File, Duration), CacheFsError> {
335    let file = open_cache_lock_file(path)?;
336    let started = Instant::now();
337    loop {
338        match file.try_lock_exclusive() {
339            Ok(()) => return Ok((file, started.elapsed())),
340            Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
341                observer(started.elapsed());
342                thread::sleep(poll_interval.min(Duration::from_millis(25)));
343            }
344            Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
345            Err(source) => {
346                return Err(CacheFsError {
347                    operation: "try lock cache",
348                    path: path.to_owned(),
349                    source,
350                });
351            }
352        }
353    }
354}
355
356pub(super) fn try_lock_cache_file(path: &Path) -> Result<Option<File>, CacheFsError> {
357    let file = open_cache_lock_file(path)?;
358    match file.try_lock_exclusive() {
359        Ok(()) => Ok(Some(file)),
360        Err(error) if error.kind() == io::ErrorKind::WouldBlock => Ok(None),
361        Err(source) => Err(CacheFsError {
362            operation: "try lock cache",
363            path: path.to_owned(),
364            source,
365        }),
366    }
367}
368
369fn open_cache_lock_file(path: &Path) -> Result<File, CacheFsError> {
370    if let Some(parent) = path.parent() {
371        fs::create_dir_all(parent).map_err(|source| CacheFsError {
372            operation: "create cache lock directory",
373            path: parent.to_owned(),
374            source,
375        })?;
376    }
377    OpenOptions::new()
378        .create(true)
379        .read(true)
380        .write(true)
381        .truncate(false)
382        .open(path)
383        .map_err(|source| CacheFsError {
384            operation: "open cache lock",
385            path: path.to_owned(),
386            source,
387        })
388}
389
390pub(super) fn record_cache_entry_use(path: &Path) -> Result<(), CacheFsError> {
391    write_last_used(path, SystemTime::now())
392}
393
394pub(super) fn cache_maintenance_due(
395    path: &Path,
396    minimum_interval: Option<Duration>,
397    maintenance_identity: &str,
398) -> Result<bool, CacheFsError> {
399    let Some(minimum_interval) = minimum_interval else {
400        return Ok(true);
401    };
402    let marker = path.join(LAST_MAINTENANCE_FILE);
403    // Allow both LF and CRLF for the timestamp and policy-identity lines.
404    let maximum_len = MAX_TIMESTAMP_BYTES + maintenance_identity.len() + 4;
405    let contents = match read_stamp_with_limit(&marker, maximum_len) {
406        Ok(Some(contents)) => contents,
407        Ok(None) => return Ok(true),
408        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(true),
409        Err(source) => {
410            return Err(CacheFsError {
411                operation: "read cache maintenance time",
412                path: marker,
413                source,
414            });
415        }
416    };
417    let mut lines = contents.lines();
418    let Some(last_maintenance) = lines.next().and_then(decode_system_time) else {
419        return Ok(true);
420    };
421    if lines.next() != Some(maintenance_identity) {
422        return Ok(true);
423    }
424    Ok(match SystemTime::now().duration_since(last_maintenance) {
425        Ok(elapsed) => elapsed >= minimum_interval,
426        Err(_) => true,
427    })
428}
429
430pub(super) fn record_cache_maintenance(
431    path: &Path,
432    maintenance_identity: &str,
433) -> Result<(), CacheFsError> {
434    let marker = path.join(LAST_MAINTENANCE_FILE);
435    let elapsed = encode_system_time(&marker, SystemTime::now())?;
436    let contents = format!("{}\n{maintenance_identity}\n", elapsed.as_nanos());
437    write_atomic(&marker, contents.as_bytes()).map_err(|source| CacheFsError {
438        operation: "record cache maintenance time",
439        path: marker,
440        source,
441    })
442}
443
444pub(super) fn perform_scheduled_cache_maintenance(
445    path: &Path,
446    minimum_interval: Option<Duration>,
447    maintenance_identity: &str,
448    maintenance: impl FnOnce() -> Result<ArtifactCachePruneReport, String>,
449) -> (Option<ArtifactCacheMaintenance>, Option<Duration>) {
450    let started = Instant::now();
451    match cache_maintenance_due(path, minimum_interval, maintenance_identity) {
452        Ok(false) => return (None, Some(started.elapsed())),
453        Ok(true) => {}
454        Err(error) => {
455            return (
456                Some(ArtifactCacheMaintenance::PruneFailed {
457                    message: error.to_string(),
458                }),
459                Some(started.elapsed()),
460            );
461        }
462    }
463
464    let result = maintenance();
465    let marker = record_cache_maintenance(path, maintenance_identity);
466    let outcome = match (result, marker) {
467        (Ok(report), Ok(())) => ArtifactCacheMaintenance::Pruned(report),
468        (Err(message), Ok(())) => ArtifactCacheMaintenance::PruneFailed { message },
469        (Ok(_), Err(error)) => ArtifactCacheMaintenance::PruneFailed {
470            message: error.to_string(),
471        },
472        (Err(message), Err(marker)) => ArtifactCacheMaintenance::PruneFailed {
473            message: format!(
474                "{message}; additionally failed to record the maintenance attempt: {marker}"
475            ),
476        },
477    };
478    (Some(outcome), Some(started.elapsed()))
479}
480
481pub(super) fn write_last_used(path: &Path, last_used: SystemTime) -> Result<(), CacheFsError> {
482    let marker = path.join(LAST_USED_FILE);
483    write_system_time(&marker, last_used, "record cache use time")
484}
485
486fn write_system_time(
487    path: &Path,
488    timestamp: SystemTime,
489    operation: &'static str,
490) -> Result<(), CacheFsError> {
491    let elapsed = encode_system_time(path, timestamp)?;
492    write_atomic(path, elapsed.as_nanos().to_string().as_bytes()).map_err(|source| CacheFsError {
493        operation,
494        path: path.to_owned(),
495        source,
496    })
497}
498
499fn encode_system_time(path: &Path, timestamp: SystemTime) -> Result<Duration, CacheFsError> {
500    timestamp
501        .duration_since(UNIX_EPOCH)
502        .map_err(|source| CacheFsError {
503            operation: "encode cache time",
504            path: path.to_owned(),
505            source: io::Error::new(io::ErrorKind::InvalidInput, source),
506        })
507}
508
509fn decode_system_time(contents: &str) -> Option<SystemTime> {
510    let nanoseconds = contents.parse::<u128>().ok()?;
511    let seconds = u64::try_from(nanoseconds / 1_000_000_000).ok()?;
512    let subsecond_nanos = (nanoseconds % 1_000_000_000) as u32;
513    UNIX_EPOCH.checked_add(Duration::new(seconds, subsecond_nanos))
514}
515
516pub(super) fn prune_direct_child_directories(
517    cache_root: &Path,
518    policy: ArtifactCachePrunePolicy,
519    protected_entry: Option<&Path>,
520    is_eligible: impl Fn(&Path) -> bool,
521) -> Result<ArtifactCachePruneReport, CacheFsError> {
522    let mut entries = cache_entries(cache_root, is_eligible)?;
523    let bytes_before = entries
524        .iter()
525        .fold(0_u64, |total, entry| total.saturating_add(entry.bytes));
526    let mut report = ArtifactCachePruneReport {
527        entries_scanned: entries.len(),
528        entries_removed: 0,
529        bytes_before,
530        bytes_removed: 0,
531        uncommitted_directories_removed: 0,
532        uncommitted_bytes_removed: 0,
533    };
534    let now = SystemTime::now();
535
536    if let Some(max_age) = policy.max_age() {
537        for entry in &mut entries {
538            let age = now.duration_since(entry.last_used).unwrap_or_default();
539            if protected_entry != Some(entry.path.as_path()) && age > max_age {
540                remove_cache_entry(entry, &mut report)?;
541            }
542        }
543    }
544
545    if let Some(max_size_bytes) = policy.max_size_bytes() {
546        entries.sort_by(|left, right| {
547            left.last_used
548                .cmp(&right.last_used)
549                .then_with(|| left.path.cmp(&right.path))
550        });
551        for entry in &mut entries {
552            if report.bytes_retained() <= max_size_bytes {
553                break;
554            }
555            if protected_entry == Some(entry.path.as_path()) {
556                continue;
557            }
558            remove_cache_entry(entry, &mut report)?;
559        }
560    }
561
562    Ok(report)
563}
564
565pub(super) fn directory_logical_size(path: &Path) -> io::Result<u64> {
566    let mut total = 0_u64;
567    let mut pending = vec![path.to_owned()];
568    while let Some(current) = pending.pop() {
569        let metadata = fs::symlink_metadata(&current)?;
570        if metadata.is_dir() {
571            for entry in fs::read_dir(&current)? {
572                let path = entry?.path();
573                let metadata = fs::symlink_metadata(&path)?;
574                if metadata.is_dir() {
575                    pending.push(path);
576                } else {
577                    total = total.saturating_add(metadata.len());
578                }
579            }
580        } else {
581            total = total.saturating_add(metadata.len());
582        }
583    }
584    Ok(total)
585}
586
587pub(super) fn is_sha256_directory(path: &Path) -> bool {
588    path.file_name().is_some_and(|name| {
589        let bytes = name.as_encoded_bytes();
590        bytes.len() == 64 && bytes.iter().all(u8::is_ascii_hexdigit)
591    })
592}
593
594pub(super) fn remove_path_if_present(path: &Path) -> io::Result<()> {
595    let metadata = match fs::symlink_metadata(path) {
596        Ok(metadata) => metadata,
597        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
598        Err(error) => return Err(error),
599    };
600    if metadata.file_type().is_dir() {
601        fs::remove_dir_all(path)
602    } else {
603        fs::remove_file(path)
604    }
605}
606
607struct CacheEntry {
608    path: PathBuf,
609    bytes: u64,
610    last_used: SystemTime,
611    removed: bool,
612}
613
614impl std::fmt::Display for CacheFsError {
615    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
616        write!(
617            formatter,
618            "failed to {} at {}: {}",
619            self.operation,
620            self.path.display(),
621            self.source
622        )
623    }
624}
625
626impl std::error::Error for CacheFsError {
627    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
628        Some(&self.source)
629    }
630}
631
632fn cache_entries(
633    cache_root: &Path,
634    is_eligible: impl Fn(&Path) -> bool,
635) -> Result<Vec<CacheEntry>, CacheFsError> {
636    let read_dir = match fs::read_dir(cache_root) {
637        Ok(read_dir) => read_dir,
638        Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
639        Err(source) => {
640            return Err(CacheFsError {
641                operation: "read cache directory",
642                path: cache_root.to_owned(),
643                source,
644            });
645        }
646    };
647    let mut entries = Vec::new();
648    for directory_entry in read_dir {
649        let directory_entry = directory_entry.map_err(|source| CacheFsError {
650            operation: "read cache entry",
651            path: cache_root.to_owned(),
652            source,
653        })?;
654        let path = directory_entry.path();
655        let file_type = directory_entry.file_type().map_err(|source| CacheFsError {
656            operation: "inspect cache entry",
657            path: path.clone(),
658            source,
659        })?;
660        if !file_type.is_dir() || !is_eligible(&path) {
661            continue;
662        }
663        let bytes = directory_logical_size(&path).map_err(|source| CacheFsError {
664            operation: "measure cache entry",
665            path: path.clone(),
666            source,
667        })?;
668        let last_used = cache_entry_last_used(&path).map_err(|source| CacheFsError {
669            operation: "read cache use time",
670            path: path.clone(),
671            source,
672        })?;
673        entries.push(CacheEntry {
674            path,
675            bytes,
676            last_used,
677            removed: false,
678        });
679    }
680    Ok(entries)
681}
682
683pub(super) fn cache_entry_last_used(path: &Path) -> io::Result<SystemTime> {
684    let marker = path.join(LAST_USED_FILE);
685    if let Ok(Some(contents)) = read_stamp_with_limit(&marker, MAX_TIMESTAMP_BYTES)
686        && let Some(timestamp) = decode_system_time(&contents)
687    {
688        return Ok(timestamp);
689    }
690    fs::metadata(path)?.modified()
691}
692
693fn remove_cache_entry(
694    entry: &mut CacheEntry,
695    report: &mut ArtifactCachePruneReport,
696) -> Result<(), CacheFsError> {
697    if entry.removed {
698        return Ok(());
699    }
700    let Some(_retention_lock) = try_lock_cache_file(&entry.path.join(RETENTION_LOCK_FILE))? else {
701        return Ok(());
702    };
703    remove_path_if_present(&entry.path).map_err(|source| CacheFsError {
704        operation: "prune cache entry",
705        path: entry.path.clone(),
706        source,
707    })?;
708    entry.removed = true;
709    report.entries_removed += 1;
710    report.bytes_removed = report.bytes_removed.saturating_add(entry.bytes);
711    Ok(())
712}
713
714#[cfg(test)]
715mod tests {
716    #[cfg(unix)]
717    use super::canonicalize_allow_missing;
718    use super::directory_logical_size;
719    use crate::artifacts::test_support::unique_temp_directory;
720    use std::fs;
721
722    #[test]
723    fn last_use_markers_preserve_timestamps_and_bounded_fallbacks() {
724        use std::time::{Duration, SystemTime, UNIX_EPOCH};
725
726        let root = unique_temp_directory("bounded-last-use-marker");
727        let marker = root.join(super::LAST_USED_FILE);
728        let modified = || fs::metadata(&root).unwrap().modified().unwrap();
729        assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
730        for timestamp in [
731            UNIX_EPOCH + Duration::from_nanos(123),
732            SystemTime::now() + Duration::from_secs(3600),
733        ] {
734            super::write_last_used(&root, timestamp).unwrap();
735            assert_eq!(super::cache_entry_last_used(&root).unwrap(), timestamp);
736        }
737        let overflowing_timestamp = u128::MAX.to_string();
738        for invalid in [
739            b"invalid".as_slice(),
740            overflowing_timestamp.as_bytes(),
741            &[0xff],
742        ] {
743            fs::write(&marker, invalid).unwrap();
744            assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
745        }
746        fs::File::create(&marker)
747            .unwrap()
748            .set_len(1024 * 1024 * 1024)
749            .unwrap();
750        assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
751        fs::remove_file(&marker).unwrap();
752        fs::create_dir(&marker).unwrap();
753        assert_eq!(super::cache_entry_last_used(&root).unwrap(), modified());
754        fs::remove_dir_all(root).unwrap();
755    }
756
757    #[test]
758    fn maintenance_markers_preserve_policy_intervals_and_read_errors() {
759        use std::time::{Duration, SystemTime, UNIX_EPOCH};
760
761        let root = unique_temp_directory("bounded-maintenance-marker");
762        let marker = root.join(super::LAST_MAINTENANCE_FILE);
763        let identity = super::ArtifactCachePrunePolicy::new().maintenance_identity();
764        let interval = Some(Duration::from_secs(3600));
765        let due = || super::cache_maintenance_due(&root, interval, &identity).unwrap();
766        assert!(due());
767        super::record_cache_maintenance(&root, &identity).unwrap();
768        assert!(!due());
769        assert!(super::cache_maintenance_due(&root, interval, "other-policy").unwrap());
770        assert!(super::cache_maintenance_due(&root, Some(Duration::ZERO), &identity).unwrap());
771
772        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
773        for suffix in ["", "\r\n"] {
774            fs::write(&marker, format!("{}\r\n{identity}{suffix}", now.as_nanos())).unwrap();
775            assert!(!due());
776        }
777        for timestamp in [
778            "invalid".to_owned(),
779            "0".to_owned(),
780            (now + Duration::from_secs(3600)).as_nanos().to_string(),
781        ] {
782            fs::write(&marker, format!("{timestamp}\n{identity}\n")).unwrap();
783            assert!(due());
784        }
785        super::record_cache_maintenance(&root, &identity).unwrap();
786        fs::OpenOptions::new()
787            .write(true)
788            .open(&marker)
789            .unwrap()
790            .set_len(1024 * 1024 * 1024)
791            .unwrap();
792        assert!(due());
793
794        fs::write(&marker, [0xff]).unwrap();
795        let error = super::cache_maintenance_due(&root, interval, &identity).unwrap_err();
796        assert_eq!(error.operation, "read cache maintenance time");
797        assert_eq!(error.path, marker);
798        assert_eq!(error.source.kind(), std::io::ErrorKind::InvalidData);
799        fs::remove_file(&marker).unwrap();
800        fs::create_dir(&marker).unwrap();
801        assert!(super::cache_maintenance_due(&root, interval, &identity).is_err());
802        assert!(super::cache_maintenance_due(&root, None, &identity).unwrap());
803        fs::remove_dir_all(root).unwrap();
804    }
805
806    #[test]
807    fn cache_directory_tags_preserve_valid_standard_signatures() {
808        let root = unique_temp_directory("cache-tag-signatures");
809        let tag = root.join("CACHEDIR.TAG");
810        let signature = "Signature: 8a477f597d28d172789f06886806bc55";
811        for contents in [
812            signature.to_owned(),
813            format!("{signature}\r\n# Created by another application\r\n"),
814            format!("{signature}\n# {}\n", "comment".repeat(100_000)),
815        ] {
816            fs::write(&tag, &contents).unwrap();
817            super::ensure_cache_directory_tag(&root).unwrap();
818            assert_eq!(fs::read_to_string(&tag).unwrap(), contents);
819        }
820        for invalid in ["", &signature[..42], "Signature: incorrect"] {
821            fs::write(&tag, invalid).unwrap();
822            super::ensure_cache_directory_tag(&root).unwrap();
823            assert_eq!(
824                fs::read_to_string(&tag).unwrap(),
825                super::CACHE_DIRECTORY_TAG
826            );
827        }
828        fs::remove_dir_all(root).unwrap();
829    }
830
831    #[test]
832    #[cfg(unix)]
833    fn cache_directory_tag_replaces_symlinks_without_changing_referents() {
834        let root = unique_temp_directory("cache-tag-symlink");
835        let referent = root.join("other-application-tag");
836        let contents = "Signature: 8a477f597d28d172789f06886806bc55\n# Preserve this file\n";
837        fs::write(&referent, contents).unwrap();
838        let tag = root.join("CACHEDIR.TAG");
839        std::os::unix::fs::symlink(&referent, &tag).unwrap();
840        super::ensure_cache_directory_tag(&root).unwrap();
841        assert!(fs::symlink_metadata(&tag).unwrap().file_type().is_file());
842        assert_eq!(fs::read_to_string(&referent).unwrap(), contents);
843        assert_eq!(
844            fs::read_to_string(&tag).unwrap(),
845            super::CACHE_DIRECTORY_TAG
846        );
847
848        fs::remove_file(&tag).unwrap();
849        fs::create_dir(&tag).unwrap();
850        let error = super::ensure_cache_directory_tag(&root).unwrap_err();
851        assert_eq!(error.operation, "write cache directory tag");
852        assert!(tag.is_dir());
853        fs::remove_dir_all(root).unwrap();
854    }
855
856    #[test]
857    fn directory_size_sums_wide_and_nested_files() {
858        let root = unique_temp_directory("directory-logical-size");
859        assert_eq!(directory_logical_size(&root).unwrap(), 0);
860        fs::create_dir_all(root.join("wide")).unwrap();
861        fs::create_dir_all(root.join("nested/deep/empty")).unwrap();
862        let mut expected = 0;
863        for index in 0..128 {
864            let bytes = vec![42; index % 13];
865            fs::write(root.join("wide").join(index.to_string()), &bytes).unwrap();
866            expected += bytes.len() as u64;
867        }
868        let sparse = root.join("nested/deep/sparse");
869        fs::File::create(&sparse)
870            .unwrap()
871            .set_len(1024 * 1024)
872            .unwrap();
873        assert_eq!(directory_logical_size(&sparse).unwrap(), 1024 * 1024);
874        assert_eq!(
875            directory_logical_size(&root).unwrap(),
876            expected + 1024 * 1024
877        );
878        assert_eq!(
879            directory_logical_size(&root.join("missing"))
880                .unwrap_err()
881                .kind(),
882            std::io::ErrorKind::NotFound,
883        );
884        fs::remove_dir_all(root).unwrap();
885    }
886
887    #[test]
888    #[cfg(unix)]
889    fn directory_size_counts_symlinks_without_following_them() {
890        let root = unique_temp_directory("directory-size-symlinks");
891        let walked = root.join("walked");
892        fs::create_dir_all(&walked).unwrap();
893        fs::create_dir_all(root.join("external")).unwrap();
894        fs::write(root.join("external/payload"), vec![42; 4096]).unwrap();
895        fs::write(root.join("outside-file"), vec![42; 4096]).unwrap();
896        fs::write(walked.join("payload"), b"abc").unwrap();
897        let targets = ["../external", "../outside-file", "missing", "."];
898        for (index, target) in targets.iter().enumerate() {
899            std::os::unix::fs::symlink(target, walked.join(index.to_string())).unwrap();
900        }
901        let expected = 3 + targets
902            .iter()
903            .map(|target| target.len() as u64)
904            .sum::<u64>();
905        assert_eq!(directory_logical_size(&walked).unwrap(), expected);
906        assert_eq!(
907            directory_logical_size(&walked.join("0")).unwrap(),
908            targets[0].len() as u64,
909        );
910        fs::remove_dir_all(root).unwrap();
911    }
912
913    #[test]
914    #[cfg(unix)]
915    fn missing_parent_traversal_resumes_existing_symlink_resolution() {
916        let root = unique_temp_directory("canonical-missing-parent");
917        let target = root.join("real");
918        fs::create_dir_all(&target).unwrap();
919        std::os::unix::fs::symlink(&target, root.join("alias")).unwrap();
920        let path = root.join("missing/../alias/generated/nested/../output");
921        assert_eq!(
922            canonicalize_allow_missing(&path).unwrap(),
923            target.canonicalize().unwrap().join("generated/output")
924        );
925        assert_eq!(
926            canonicalize_allow_missing(&root.join("alias/../other/output")).unwrap(),
927            root.canonicalize().unwrap().join("other/output")
928        );
929        fs::remove_dir_all(root).unwrap();
930    }
931}
ic_testkit/artifacts/cache_fs.rs

ic_testkit/artifacts/
cache_fs.rs
