uv_cache/lib.rs
1use std::fmt::{Display, Formatter};
2use std::io;
3use std::io::Write;
4use std::ops::Deref;
5use std::path::{Path, PathBuf};
6use std::str::FromStr;
7use std::sync::Arc;
8
9use rustc_hash::FxHashMap;
10use tracing::{debug, trace, warn};
11
12use uv_cache_info::Timestamp;
13use uv_fs::{LockedFile, LockedFileError, LockedFileMode, Simplified, cachedir, directories};
14use uv_normalize::PackageName;
15use uv_pypi_types::ResolutionMetadata;
16
17pub use crate::by_timestamp::CachedByTimestamp;
18#[cfg(feature = "clap")]
19pub use crate::cli::CacheArgs;
20use crate::removal::Remover;
21pub use crate::removal::{Removal, RemovalAccounting};
22pub use crate::wheel::WheelCache;
23use crate::wheel::WheelCacheKind;
24pub use archive::ArchiveId;
25
26mod archive;
27mod by_timestamp;
28#[cfg(feature = "clap")]
29mod cli;
30mod removal;
31mod wheel;
32
33/// The version of the archive bucket.
34///
35/// Must be kept in-sync with the version in [`CacheBucket::to_str`].
36pub const ARCHIVE_VERSION: u8 = 0;
37
38/// Error locking a cache entry or shard
39#[derive(Debug, thiserror::Error)]
40pub enum Error {
41 #[error(transparent)]
42 Io(#[from] io::Error),
43 #[error("Failed to initialize cache at `{}`", _0.user_display())]
44 Init(PathBuf, #[source] io::Error),
45 #[error("Could not make the path absolute")]
46 Absolute(#[source] io::Error),
47 #[error("Could not acquire lock")]
48 Acquire(#[from] LockedFileError),
49}
50
51/// A [`CacheEntry`] which may or may not exist yet.
52#[derive(Debug, Clone)]
53pub struct CacheEntry(PathBuf);
54
55impl CacheEntry {
56 /// Create a new [`CacheEntry`] from a directory and a file name.
57 pub fn new(dir: impl Into<PathBuf>, file: impl AsRef<Path>) -> Self {
58 Self(dir.into().join(file))
59 }
60
61 /// Create a new [`CacheEntry`] from a path.
62 pub fn from_path(path: impl Into<PathBuf>) -> Self {
63 Self(path.into())
64 }
65
66 /// Return the cache entry's parent directory.
67 pub fn shard(&self) -> CacheShard {
68 CacheShard(self.dir().to_path_buf())
69 }
70
71 /// Convert the [`CacheEntry`] into a [`PathBuf`].
72 #[inline]
73 pub fn into_path_buf(self) -> PathBuf {
74 self.0
75 }
76
77 /// Return the path to the [`CacheEntry`].
78 #[inline]
79 pub fn path(&self) -> &Path {
80 &self.0
81 }
82
83 /// Return the cache entry's parent directory.
84 #[inline]
85 pub fn dir(&self) -> &Path {
86 self.0.parent().expect("Cache entry has no parent")
87 }
88
89 /// Create a new [`CacheEntry`] with the given file name.
90 #[must_use]
91 pub fn with_file(&self, file: impl AsRef<Path>) -> Self {
92 Self(self.dir().join(file))
93 }
94
95 /// Acquire the [`CacheEntry`] as an exclusive lock.
96 pub async fn lock(&self) -> Result<LockedFile, Error> {
97 fs_err::create_dir_all(self.dir())?;
98 Ok(LockedFile::acquire(
99 self.path(),
100 LockedFileMode::Exclusive,
101 self.path().display(),
102 )
103 .await?)
104 }
105}
106
107impl AsRef<Path> for CacheEntry {
108 fn as_ref(&self) -> &Path {
109 &self.0
110 }
111}
112
113/// A subdirectory within the cache.
114#[derive(Debug, Clone)]
115pub struct CacheShard(PathBuf);
116
117impl CacheShard {
118 /// Return a [`CacheEntry`] within this shard.
119 pub fn entry(&self, file: impl AsRef<Path>) -> CacheEntry {
120 CacheEntry::new(&self.0, file)
121 }
122
123 /// Return a [`CacheShard`] within this shard.
124 #[must_use]
125 pub fn shard(&self, dir: impl AsRef<Path>) -> Self {
126 Self(self.0.join(dir.as_ref()))
127 }
128
129 /// Acquire the cache entry as an exclusive lock.
130 pub async fn lock(&self) -> Result<LockedFile, Error> {
131 fs_err::create_dir_all(self.as_ref())?;
132 Ok(LockedFile::acquire(
133 self.join(".lock"),
134 LockedFileMode::Exclusive,
135 self.display(),
136 )
137 .await?)
138 }
139
140 /// Return the [`CacheShard`] as a [`PathBuf`].
141 pub fn into_path_buf(self) -> PathBuf {
142 self.0
143 }
144}
145
146impl AsRef<Path> for CacheShard {
147 fn as_ref(&self) -> &Path {
148 &self.0
149 }
150}
151
152impl Deref for CacheShard {
153 type Target = Path;
154
155 fn deref(&self) -> &Self::Target {
156 &self.0
157 }
158}
159
160/// The main cache abstraction.
161///
162/// While the cache is active, it holds a read (shared) lock that prevents cache cleaning
163#[derive(Debug, Clone)]
164pub struct Cache {
165 /// The cache directory.
166 root: PathBuf,
167 /// The refresh strategy to use when reading from the cache.
168 refresh: Refresh,
169 /// A temporary cache directory, if the user requested `--no-cache`.
170 ///
171 /// Included to ensure that the temporary directory exists for the length of the operation, but
172 /// is dropped at the end as appropriate.
173 temp_dir: Option<Arc<tempfile::TempDir>>,
174 /// Ensure that `uv cache` operations don't remove items from the cache that are used by another
175 /// uv process.
176 lock_file: Option<Arc<LockedFile>>,
177 /// The storage accounting used when removing cache entries.
178 removal_accounting: RemovalAccounting,
179}
180
181impl Cache {
182 /// A persistent cache directory at `root`.
183 pub fn from_path(root: impl Into<PathBuf>) -> Self {
184 Self {
185 root: root.into(),
186 refresh: Refresh::None(Timestamp::now()),
187 temp_dir: None,
188 lock_file: None,
189 removal_accounting: RemovalAccounting::Coarse,
190 }
191 }
192
193 /// Create a temporary cache directory.
194 pub fn temp() -> Result<Self, io::Error> {
195 let temp_dir = tempfile::tempdir()?;
196 Ok(Self {
197 root: temp_dir.path().to_path_buf(),
198 refresh: Refresh::None(Timestamp::now()),
199 temp_dir: Some(Arc::new(temp_dir)),
200 lock_file: None,
201 removal_accounting: RemovalAccounting::Coarse,
202 })
203 }
204
205 /// Set the [`Refresh`] policy for the cache.
206 #[must_use]
207 pub fn with_refresh(self, refresh: Refresh) -> Self {
208 Self { refresh, ..self }
209 }
210
211 /// Set the storage accounting used when removing cache entries.
212 ///
213 /// Falls back to [`RemovalAccounting::Coarse`] when fine-grained accounting is unsupported.
214 #[must_use]
215 pub fn with_removal_accounting(self, removal_accounting: RemovalAccounting) -> Self {
216 let removal_accounting = match removal_accounting {
217 RemovalAccounting::Fine if !uv_fs::supports_fine_grained_accounting() => {
218 RemovalAccounting::Coarse
219 }
220 removal_accounting => removal_accounting,
221 };
222 Self {
223 removal_accounting,
224 ..self
225 }
226 }
227
228 /// Create an empty removal summary using the cache's configured accounting.
229 pub fn removal(&self) -> Removal {
230 Removal::new(self.removal_accounting)
231 }
232
233 /// Acquire a lock that allows removing entries from the cache.
234 pub async fn with_exclusive_lock(self) -> Result<Self, LockedFileError> {
235 let Self {
236 root,
237 refresh,
238 temp_dir,
239 lock_file,
240 removal_accounting,
241 } = self;
242
243 // Release the existing lock, avoid deadlocks from a cloned cache.
244 if let Some(lock_file) = lock_file {
245 drop(
246 Arc::try_unwrap(lock_file).expect(
247 "cloning the cache before acquiring an exclusive lock causes a deadlock",
248 ),
249 );
250 }
251 let lock_file = LockedFile::acquire(
252 root.join(".lock"),
253 LockedFileMode::Exclusive,
254 root.simplified_display(),
255 )
256 .await?;
257
258 Ok(Self {
259 root,
260 refresh,
261 temp_dir,
262 lock_file: Some(Arc::new(lock_file)),
263 removal_accounting,
264 })
265 }
266
267 /// Acquire a lock that allows removing entries from the cache, if available.
268 ///
269 /// If the lock is not immediately available, returns [`Err`] with self.
270 pub fn with_exclusive_lock_no_wait(self) -> Result<Self, Self> {
271 let Self {
272 root,
273 refresh,
274 temp_dir,
275 lock_file,
276 removal_accounting,
277 } = self;
278
279 match LockedFile::acquire_no_wait(
280 root.join(".lock"),
281 LockedFileMode::Exclusive,
282 root.simplified_display(),
283 ) {
284 Some(lock_file) => Ok(Self {
285 root,
286 refresh,
287 temp_dir,
288 lock_file: Some(Arc::new(lock_file)),
289 removal_accounting,
290 }),
291 None => Err(Self {
292 root,
293 refresh,
294 temp_dir,
295 lock_file,
296 removal_accounting,
297 }),
298 }
299 }
300
301 /// Return the root of the cache.
302 pub fn root(&self) -> &Path {
303 &self.root
304 }
305
306 /// The folder for a specific cache bucket
307 pub fn bucket(&self, cache_bucket: CacheBucket) -> PathBuf {
308 self.root.join(cache_bucket.to_str())
309 }
310
311 /// Compute an entry in the cache.
312 pub fn shard(&self, cache_bucket: CacheBucket, dir: impl AsRef<Path>) -> CacheShard {
313 CacheShard(self.bucket(cache_bucket).join(dir.as_ref()))
314 }
315
316 /// Compute an entry in the cache.
317 pub fn entry(
318 &self,
319 cache_bucket: CacheBucket,
320 dir: impl AsRef<Path>,
321 file: impl AsRef<Path>,
322 ) -> CacheEntry {
323 CacheEntry::new(self.bucket(cache_bucket).join(dir), file)
324 }
325
326 /// Return the path to an archive in the cache.
327 pub fn archive(&self, id: &ArchiveId) -> PathBuf {
328 self.bucket(CacheBucket::Archive).join(id)
329 }
330
331 /// Create a temporary directory to be used as a Python virtual environment.
332 pub fn venv_dir(&self) -> io::Result<tempfile::TempDir> {
333 fs_err::create_dir_all(self.bucket(CacheBucket::Builds))?;
334 tempfile::tempdir_in(self.bucket(CacheBucket::Builds))
335 }
336
337 /// Create a temporary directory to be used for executing PEP 517 source distribution builds.
338 pub fn build_dir(&self) -> io::Result<tempfile::TempDir> {
339 fs_err::create_dir_all(self.bucket(CacheBucket::Builds))?;
340 tempfile::tempdir_in(self.bucket(CacheBucket::Builds))
341 }
342
343 /// Returns `true` if a cache entry must be revalidated given the [`Refresh`] policy.
344 pub fn must_revalidate_package(&self, package: &PackageName) -> bool {
345 match &self.refresh {
346 Refresh::None(_) => false,
347 Refresh::All(_) => true,
348 Refresh::Packages(packages, _, _) => packages.contains(package),
349 }
350 }
351
352 /// Returns `true` if a cache entry must be revalidated given the [`Refresh`] policy.
353 pub fn must_revalidate_path(&self, path: &Path) -> bool {
354 match &self.refresh {
355 Refresh::None(_) => false,
356 Refresh::All(_) => true,
357 Refresh::Packages(_, paths, _) => paths
358 .iter()
359 .any(|target| same_file::is_same_file(path, target).unwrap_or(false)),
360 }
361 }
362
363 /// Returns the [`Freshness`] for a cache entry, validating it against the [`Refresh`] policy.
364 ///
365 /// A cache entry is considered fresh if it was created after the cache itself was
366 /// initialized, or if the [`Refresh`] policy does not require revalidation.
367 pub fn freshness(
368 &self,
369 entry: &CacheEntry,
370 package: Option<&PackageName>,
371 path: Option<&Path>,
372 ) -> io::Result<Freshness> {
373 // Grab the cutoff timestamp, if it's relevant.
374 let timestamp = match &self.refresh {
375 Refresh::None(_) => return Ok(Freshness::Fresh),
376 Refresh::All(timestamp) => timestamp,
377 Refresh::Packages(packages, paths, timestamp) => {
378 if package.is_none_or(|package| packages.contains(package))
379 || path.is_some_and(|path| {
380 paths
381 .iter()
382 .any(|target| same_file::is_same_file(path, target).unwrap_or(false))
383 })
384 {
385 timestamp
386 } else {
387 return Ok(Freshness::Fresh);
388 }
389 }
390 };
391
392 match fs_err::metadata(entry.path()) {
393 Ok(metadata) => {
394 if Timestamp::from_metadata(&metadata) >= *timestamp {
395 Ok(Freshness::Fresh)
396 } else {
397 Ok(Freshness::Stale)
398 }
399 }
400 Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(Freshness::Missing),
401 Err(err) => Err(err),
402 }
403 }
404
405 /// Persist a temporary directory to the artifact store, returning its unique ID.
406 pub async fn persist(
407 &self,
408 temp_dir: impl AsRef<Path>,
409 path: impl AsRef<Path>,
410 ) -> io::Result<ArchiveId> {
411 // Create a unique ID for the artifact.
412 let id = ArchiveId::new();
413
414 // Move the temporary directory into the directory store.
415 let archive_entry = self.entry(CacheBucket::Archive, "", &id);
416 fs_err::create_dir_all(archive_entry.dir())?;
417 uv_fs::rename_with_retry(temp_dir.as_ref(), archive_entry.path()).await?;
418
419 // Create a symlink to the directory store.
420 fs_err::create_dir_all(path.as_ref().parent().expect("Cache entry to have parent"))?;
421 self.create_link(&id, path.as_ref())?;
422
423 Ok(id)
424 }
425
426 /// Persist a temporary directory to the artifact store under a caller-selected ID.
427 ///
428 /// If another writer has already persisted the same ID, discard `temp_dir` and link `path` to
429 /// the existing archive entry. The ID must therefore uniquely identify the directory contents.
430 pub async fn persist_with_id(
431 &self,
432 temp_dir: tempfile::TempDir,
433 path: impl AsRef<Path>,
434 id: ArchiveId,
435 ) -> io::Result<ArchiveId> {
436 // Move the temporary directory into the directory store.
437 let archive_entry = self.entry(CacheBucket::Archive, "", &id);
438 fs_err::create_dir_all(archive_entry.dir())?;
439 if let Err(err) = uv_fs::rename_with_retry(temp_dir.path(), archive_entry.path()).await {
440 if !archive_entry.path().is_dir() {
441 return Err(err);
442 }
443 }
444
445 // Create a symlink to the directory store.
446 fs_err::create_dir_all(path.as_ref().parent().expect("Cache entry to have parent"))?;
447 self.create_link(&id, path.as_ref())?;
448
449 Ok(id)
450 }
451
452 /// Returns `true` if the [`Cache`] is temporary.
453 pub fn is_temporary(&self) -> bool {
454 self.temp_dir.is_some()
455 }
456
457 /// Populate the cache scaffold.
458 fn create_base_files(root: &PathBuf) -> io::Result<()> {
459 // Create the cache directory, if it doesn't exist.
460 fs_err::create_dir_all(root)?;
461
462 // Add the CACHEDIR.TAG.
463 cachedir::ensure_tag(root)?;
464
465 // Add the .gitignore.
466 match fs_err::OpenOptions::new()
467 .write(true)
468 .create_new(true)
469 .open(root.join(".gitignore"))
470 {
471 Ok(mut file) => file.write_all(b"*")?,
472 Err(err) if err.kind() == io::ErrorKind::AlreadyExists => (),
473 Err(err) => return Err(err),
474 }
475
476 // Add an empty .gitignore to the build bucket, to ensure that the cache's own .gitignore
477 // doesn't interfere with source distribution builds. Build backends (like hatchling) will
478 // traverse upwards to look for .gitignore files.
479 fs_err::create_dir_all(root.join(CacheBucket::SourceDistributions.to_str()))?;
480 match fs_err::OpenOptions::new()
481 .write(true)
482 .create_new(true)
483 .open(
484 root.join(CacheBucket::SourceDistributions.to_str())
485 .join(".gitignore"),
486 ) {
487 Ok(_) => {}
488 Err(err) if err.kind() == io::ErrorKind::AlreadyExists => (),
489 Err(err) => return Err(err),
490 }
491
492 // Add a phony .git, if it doesn't exist, to ensure that the cache isn't considered to be
493 // part of a Git repository. (Some packages will include Git metadata (like a hash) in the
494 // built version if they're in a Git repository, but the cache should be viewed as an
495 // isolated store.).
496 // We have to put this below the gitignore. Otherwise, if the build backend uses the rust
497 // ignore crate it will walk up to the top level .gitignore and ignore its python source
498 // files.
499 let phony_git = root
500 .join(CacheBucket::SourceDistributions.to_str())
501 .join(".git");
502 match fs_err::OpenOptions::new()
503 .create(true)
504 .write(true)
505 .open(&phony_git)
506 {
507 Ok(_) => {}
508 // Handle read-only caches including sandboxed environments.
509 Err(err) if err.kind() == io::ErrorKind::ReadOnlyFilesystem => {
510 if !phony_git.exists() {
511 return Err(err);
512 }
513 }
514 Err(err) => return Err(err),
515 }
516
517 Ok(())
518 }
519
520 /// Initialize the [`Cache`].
521 pub async fn init(self) -> Result<Self, Error> {
522 let root = &self.root;
523
524 Self::create_base_files(root).map_err(|err| Error::Init(root.clone(), err))?;
525
526 // Block cache removal operations from interfering.
527 let lock_file = match LockedFile::acquire(
528 root.join(".lock"),
529 LockedFileMode::Shared,
530 root.simplified_display(),
531 )
532 .await
533 {
534 Ok(lock_file) => Some(Arc::new(lock_file)),
535 Err(err)
536 if err
537 .as_io_error()
538 .is_some_and(|err| err.kind() == io::ErrorKind::Unsupported) =>
539 {
540 warn!(
541 "Shared locking is not supported by the current platform or filesystem, \
542 reduced parallel process safety with `uv cache clean` and `uv cache prune`."
543 );
544 None
545 }
546 Err(err) => return Err(err.into()),
547 };
548
549 Ok(Self {
550 root: std::path::absolute(root).map_err(Error::Absolute)?,
551 lock_file,
552 ..self
553 })
554 }
555
556 /// Initialize the [`Cache`], assuming that there are no other uv processes running.
557 pub fn init_no_wait(self) -> Result<Option<Self>, Error> {
558 let root = &self.root;
559
560 Self::create_base_files(root).map_err(|err| Error::Init(root.clone(), err))?;
561
562 // Block cache removal operations from interfering.
563 let Some(lock_file) = LockedFile::acquire_no_wait(
564 root.join(".lock"),
565 LockedFileMode::Shared,
566 root.simplified_display(),
567 ) else {
568 return Ok(None);
569 };
570 Ok(Some(Self {
571 root: std::path::absolute(root).map_err(Error::Absolute)?,
572 lock_file: Some(Arc::new(lock_file)),
573 ..self
574 }))
575 }
576
577 /// Clear the cache, removing all entries.
578 pub fn clear(self, reporter: Box<dyn CleanReporter>) -> Result<Removal, io::Error> {
579 // Remove everything but `.lock`, Windows does not allow removal of a locked file
580 let mut removal = Remover::new(reporter)
581 .with_removal_accounting(self.removal_accounting)
582 .rm_rf(&self.root, true)?;
583 let Self {
584 root, lock_file, ..
585 } = self;
586
587 // Remove the `.lock` file, unlocking it first
588 if let Some(lock) = lock_file {
589 drop(lock);
590 fs_err::remove_file(root.join(".lock"))?;
591 }
592 removal.num_files += 1;
593
594 // Remove the root directory
595 match fs_err::remove_dir(root) {
596 Ok(()) => {
597 removal.num_dirs += 1;
598 }
599 // On Windows, when `--force` is used, the `.lock` file can exist and be unremovable,
600 // so we make this non-fatal
601 Err(err) if err.kind() == io::ErrorKind::DirectoryNotEmpty => {
602 trace!("Failed to remove root cache directory: not empty");
603 }
604 Err(err) => return Err(err),
605 }
606
607 Ok(removal)
608 }
609
610 /// Remove a package from the cache.
611 ///
612 /// Returns the number of entries removed from the cache.
613 pub fn remove(&self, name: &PackageName) -> io::Result<Removal> {
614 // Collect the set of referenced archives.
615 let references = self.find_archive_references()?;
616
617 // Remove any entries for the package from the cache.
618 let mut summary = self.removal();
619 for bucket in CacheBucket::iter() {
620 summary += bucket.remove(self, name)?;
621 }
622
623 if references.is_empty() {
624 return Ok(summary);
625 }
626
627 // Only remove targets in the archive bucket. Cache entries may contain unexpected links
628 // to paths outside the cache.
629 let archive_root = fs_err::canonicalize(&self.root)?.join(CacheBucket::Archive.to_str());
630
631 // Remove any archives that are no longer referenced.
632 for (target, references) in references {
633 if target.starts_with(&archive_root) && references.iter().all(|path| !path.exists()) {
634 debug!("Removing dangling cache entry: {}", target.display());
635 summary += self.remove_path(target)?;
636 }
637 }
638
639 Ok(summary)
640 }
641
642 /// Prune dangling cache entries and cached environments.
643 pub fn prune(&self, ci: bool) -> Result<Removal, io::Error> {
644 let mut summary = self.removal();
645
646 // First, remove any top-level directories that are unused. These typically represent
647 // outdated cache buckets (e.g., `wheels-v0`, when latest is `wheels-v1`).
648 for entry in fs_err::read_dir(&self.root)? {
649 let entry = entry?;
650 let metadata = entry.metadata()?;
651
652 if entry.file_name() == "CACHEDIR.TAG"
653 || entry.file_name() == ".gitignore"
654 || entry.file_name() == ".git"
655 || entry.file_name() == ".lock"
656 {
657 continue;
658 }
659
660 if metadata.is_dir() {
661 // If the directory is not a cache bucket, remove it.
662 if CacheBucket::iter().all(|bucket| entry.file_name() != bucket.to_str()) {
663 let path = entry.path();
664 debug!("Removing dangling cache bucket: {}", path.display());
665 summary += self.remove_path(path)?;
666 }
667 } else {
668 // If the file is not a marker file, remove it.
669 let path = entry.path();
670 debug!("Removing dangling cache bucket: {}", path.display());
671 summary += self.remove_path(path)?;
672 }
673 }
674
675 // Second, remove all cached environments. Centralized project environments can be
676 // referenced by `.venv` links, but are recreated when next needed.
677 match fs_err::read_dir(self.bucket(CacheBucket::Environments)) {
678 Ok(entries) => {
679 for entry in entries {
680 let entry = entry?;
681 let path = entry.path();
682 debug!("Removing cached environment: {}", path.display());
683 summary += self.remove_path(path)?;
684 }
685 }
686 Err(err) if err.kind() == io::ErrorKind::NotFound => (),
687 Err(err) => return Err(err),
688 }
689
690 // Third, if enabled, remove all unzipped wheels, leaving only the wheel archives.
691 if ci {
692 // Remove the entire pre-built wheel cache, since every entry is an unzipped wheel.
693 match fs_err::read_dir(self.bucket(CacheBucket::Wheels)) {
694 Ok(entries) => {
695 for entry in entries {
696 let entry = entry?;
697 let path = entry.path();
698 if path.is_dir() {
699 debug!("Removing unzipped wheel entry: {}", path.display());
700 summary += self.remove_path(path)?;
701 }
702 }
703 }
704 Err(err) if err.kind() == io::ErrorKind::NotFound => (),
705 Err(err) => return Err(err),
706 }
707
708 let source_distributions = self.bucket(CacheBucket::SourceDistributions);
709 if source_distributions.try_exists()? {
710 for entry in walkdir::WalkDir::new(source_distributions) {
711 let entry = entry?;
712
713 // If the directory contains a `metadata.msgpack`, then it's a built wheel revision.
714 if !entry.file_type().is_dir() {
715 continue;
716 }
717
718 if !entry.path().join("metadata.msgpack").exists() {
719 continue;
720 }
721
722 // Remove everything except the built wheel archive and the metadata.
723 for entry in fs_err::read_dir(entry.path())? {
724 let entry = entry?;
725 let path = entry.path();
726
727 // Retain the resolved metadata (`metadata.msgpack`).
728 if path
729 .file_name()
730 .is_some_and(|file_name| file_name == "metadata.msgpack")
731 {
732 continue;
733 }
734
735 // Retain any built wheel archives.
736 if path
737 .extension()
738 .is_some_and(|ext| ext.eq_ignore_ascii_case("whl"))
739 {
740 continue;
741 }
742
743 debug!("Removing unzipped built wheel entry: {}", path.display());
744 summary += self.remove_path(path)?;
745 }
746 }
747 }
748 }
749
750 // Fourth, remove any unused archives (by searching for archives that are not symlinked).
751 let references = self.find_archive_references()?;
752
753 match fs_err::read_dir(self.bucket(CacheBucket::Archive)) {
754 Ok(entries) => {
755 for entry in entries {
756 let entry = entry?;
757 let path = entry.path();
758 let target = fs_err::canonicalize(&path)?;
759 if !references.contains_key(&target) {
760 debug!("Removing dangling cache archive: {}", path.display());
761 summary += self.remove_path(path)?;
762 }
763 }
764 }
765 Err(err) if err.kind() == io::ErrorKind::NotFound => (),
766 Err(err) => return Err(err),
767 }
768
769 Ok(summary)
770 }
771
772 /// Remove a cache path using the cache's configured storage accounting.
773 pub fn remove_path(&self, path: impl AsRef<Path>) -> io::Result<Removal> {
774 Remover::default()
775 .with_removal_accounting(self.removal_accounting)
776 .rm_rf(path, false)
777 }
778
779 /// Find all references to entries in the archive bucket.
780 ///
781 /// Archive entries are often referenced by symlinks in other cache buckets. This method
782 /// searches for all such references.
783 ///
784 /// Returns a map from archive path to paths that reference it.
785 fn find_archive_references(&self) -> Result<FxHashMap<PathBuf, Vec<PathBuf>>, io::Error> {
786 let mut references = FxHashMap::<PathBuf, Vec<PathBuf>>::default();
787 for bucket in [CacheBucket::SourceDistributions, CacheBucket::Wheels] {
788 let bucket_path = self.bucket(bucket);
789 if bucket_path.is_dir() {
790 let walker = walkdir::WalkDir::new(&bucket_path).into_iter();
791 for entry in walker.filter_entry(|entry| {
792 !(
793 // As an optimization, ignore any `.lock`, `.whl`, `.msgpack`, `.rev`, or
794 // `.http` files, along with the `src` directory, which represents the
795 // unpacked source distribution.
796 entry.file_name() == "src"
797 || entry.file_name() == ".lock"
798 || entry.file_name() == ".gitignore"
799 || entry.path().extension().is_some_and(|ext| {
800 ext.eq_ignore_ascii_case("lock")
801 || ext.eq_ignore_ascii_case("whl")
802 || ext.eq_ignore_ascii_case("http")
803 || ext.eq_ignore_ascii_case("rev")
804 || ext.eq_ignore_ascii_case("msgpack")
805 })
806 )
807 }) {
808 let entry = entry?;
809
810 // On Unix, archive references use symlinks.
811 if cfg!(unix) {
812 if !entry.file_type().is_symlink() {
813 continue;
814 }
815 }
816
817 // On Windows, archive references are files containing structured data.
818 if cfg!(windows) {
819 if !entry.file_type().is_file() {
820 continue;
821 }
822 }
823
824 if let Ok(target) = self.resolve_link(entry.path()) {
825 references
826 .entry(target)
827 .or_default()
828 .push(entry.path().to_path_buf());
829 }
830 }
831 }
832 }
833 Ok(references)
834 }
835
836 /// Create a link to a directory in the archive bucket.
837 ///
838 /// On Windows, we write structured data ([`Link`]) to a file containing the archive ID and
839 /// version. On Unix, we create a symlink to the target directory.
840 #[cfg(windows)]
841 #[expect(clippy::unused_self)]
842 fn create_link(&self, id: &ArchiveId, dst: impl AsRef<Path>) -> io::Result<()> {
843 // Serialize the link.
844 let link = Link::new(id.clone());
845 let contents = link.to_string();
846
847 // First, attempt to create a file at the location, but fail if it already exists.
848 match fs_err::OpenOptions::new()
849 .write(true)
850 .create_new(true)
851 .open(dst.as_ref())
852 {
853 Ok(mut file) => {
854 // Write the target path to the file.
855 file.write_all(contents.as_bytes())?;
856 Ok(())
857 }
858 Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {
859 // Write to a temporary file, then move it into place.
860 let temp_dir = tempfile::tempdir_in(dst.as_ref().parent().unwrap())?;
861 let temp_file = temp_dir.path().join("link");
862 fs_err::write(&temp_file, contents.as_bytes())?;
863
864 // Move the symlink into the target location.
865 fs_err::rename(&temp_file, dst.as_ref())?;
866
867 Ok(())
868 }
869 Err(err) => Err(err),
870 }
871 }
872
873 /// Resolve an archive link, returning the fully-resolved path.
874 ///
875 /// Returns an error if the link target does not exist.
876 #[cfg(windows)]
877 pub fn resolve_link(&self, path: impl AsRef<Path>) -> io::Result<PathBuf> {
878 // Deserialize the link.
879 let contents = fs_err::read_to_string(path.as_ref())?;
880 let link = Link::from_str(&contents)?;
881
882 // Ignore stale links.
883 if link.version != ARCHIVE_VERSION {
884 return Err(io::Error::new(
885 io::ErrorKind::NotFound,
886 "The link target does not exist.",
887 ));
888 }
889
890 // Reconstruct the path.
891 let path = self.archive(&link.id);
892 path.canonicalize()
893 }
894
895 /// Create a link to a directory in the archive bucket.
896 ///
897 /// On Windows, we write structured data ([`Link`]) to a file containing the archive ID and
898 /// version. On Unix, we create a symlink to the target directory.
899 #[cfg(unix)]
900 fn create_link(&self, id: &ArchiveId, dst: impl AsRef<Path>) -> io::Result<()> {
901 let dst = dst.as_ref();
902 let dst_parent = dst.parent().expect("Cache entry to have parent");
903 // Construct the relative link target.
904 let src = uv_fs::relative_to(self.archive(id), dst_parent)?;
905
906 // Attempt to create the symlink directly.
907 match fs_err::os::unix::fs::symlink(&src, dst) {
908 Ok(()) => Ok(()),
909 Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {
910 // Create a symlink, using a temporary file to ensure atomicity.
911 let temp_dir = tempfile::tempdir_in(dst_parent)?;
912 let temp_file = temp_dir.path().join("link");
913 fs_err::os::unix::fs::symlink(&src, &temp_file)?;
914
915 // Move the symlink into the target location.
916 fs_err::rename(&temp_file, dst)?;
917
918 Ok(())
919 }
920 Err(err) => Err(err),
921 }
922 }
923
924 /// Resolve an archive link, returning the fully-resolved path.
925 ///
926 /// Returns an error if the link target does not exist.
927 #[cfg(unix)]
928 pub fn resolve_link(&self, path: impl AsRef<Path>) -> io::Result<PathBuf> {
929 path.as_ref().canonicalize()
930 }
931}
932
933/// An archive (unzipped wheel) that exists in the local cache.
934#[derive(Debug, Clone)]
935#[allow(unused)]
936struct Link {
937 /// The unique ID of the entry in the archive bucket.
938 id: ArchiveId,
939 /// The version of the archive bucket.
940 version: u8,
941}
942
943#[allow(unused)]
944impl Link {
945 /// Create a new [`Archive`] with the given ID and hashes.
946 fn new(id: ArchiveId) -> Self {
947 Self {
948 id,
949 version: ARCHIVE_VERSION,
950 }
951 }
952}
953
954impl Display for Link {
955 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
956 write!(f, "archive-v{}/{}", self.version, self.id)
957 }
958}
959
960impl FromStr for Link {
961 type Err = io::Error;
962
963 fn from_str(s: &str) -> Result<Self, Self::Err> {
964 let mut parts = s.splitn(2, '/');
965 let version = parts
966 .next()
967 .filter(|s| !s.is_empty())
968 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing version"))?;
969 let id = parts
970 .next()
971 .filter(|s| !s.is_empty())
972 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing ID"))?;
973
974 // Parse the archive version from `archive-v{version}/{id}`.
975 let version = version
976 .strip_prefix("archive-v")
977 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing version prefix"))?;
978 let version = u8::from_str(version).map_err(|err| {
979 io::Error::new(
980 io::ErrorKind::InvalidData,
981 format!("failed to parse version: {err}"),
982 )
983 })?;
984
985 // Parse the ID from `archive-v{version}/{id}`.
986 let id = ArchiveId::from_str(id).map_err(|err| {
987 io::Error::new(
988 io::ErrorKind::InvalidData,
989 format!("failed to parse ID: {err}"),
990 )
991 })?;
992
993 Ok(Self { id, version })
994 }
995}
996
997pub trait CleanReporter: Send + Sync {
998 /// Called after one file or directory is removed.
999 fn on_clean(&self);
1000
1001 /// Called after all files and directories are removed.
1002 fn on_complete(&self);
1003}
1004
1005/// The different kinds of data in the cache are stored in different bucket, which in our case
1006/// are subdirectories of the cache root.
1007#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
1008pub enum CacheBucket {
1009 /// Wheels (excluding built wheels), alongside their metadata and cache policy.
1010 ///
1011 /// There are three kinds from cache entries: Wheel metadata and policy as `MsgPack` files, the
1012 /// wheels themselves, and the unzipped wheel archives. If a wheel file is over an in-memory
1013 /// size threshold, we first download the zip file into the cache, then unzip it into a
1014 /// directory with the same name (exclusive of the `.whl` extension).
1015 ///
1016 /// Cache structure:
1017 /// * `wheel-metadata-v0/pypi/foo/{foo-1.0.0-py3-none-any.msgpack, foo-1.0.0-py3-none-any.whl}`
1018 /// * `wheel-metadata-v0/<digest(index-url)>/foo/{foo-1.0.0-py3-none-any.msgpack, foo-1.0.0-py3-none-any.whl}`
1019 /// * `wheel-metadata-v0/url/<digest(url)>/foo/{foo-1.0.0-py3-none-any.msgpack, foo-1.0.0-py3-none-any.whl}`
1020 ///
1021 /// See `uv_client::RegistryClient::wheel_metadata` for information on how wheel metadata
1022 /// is fetched.
1023 ///
1024 /// # Example
1025 ///
1026 /// Consider the following `requirements.in`:
1027 /// ```text
1028 /// # pypi wheel
1029 /// pandas
1030 /// # url wheel
1031 /// flask @ https://files.pythonhosted.org/packages/36/42/015c23096649b908c809c69388a805a571a3bea44362fe87e33fc3afa01f/flask-3.0.0-py3-none-any.whl
1032 /// ```
1033 ///
1034 /// When we run `pip compile`, it will only fetch and cache the metadata (and cache policy), it
1035 /// doesn't need the actual wheels yet:
1036 /// ```text
1037 /// wheel-v0
1038 /// ├── pypi
1039 /// │ ...
1040 /// │ ├── pandas
1041 /// │ │ └── pandas-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.msgpack
1042 /// │ ...
1043 /// └── url
1044 /// └── 4b8be67c801a7ecb
1045 /// └── flask
1046 /// └── flask-3.0.0-py3-none-any.msgpack
1047 /// ```
1048 ///
1049 /// We get the following `requirement.txt` from `pip compile`:
1050 ///
1051 /// ```text
1052 /// [...]
1053 /// flask @ https://files.pythonhosted.org/packages/36/42/015c23096649b908c809c69388a805a571a3bea44362fe87e33fc3afa01f/flask-3.0.0-py3-none-any.whl
1054 /// [...]
1055 /// pandas==2.1.3
1056 /// [...]
1057 /// ```
1058 ///
1059 /// If we run `pip sync` on `requirements.txt` on a different machine, it also fetches the
1060 /// wheels:
1061 ///
1062 /// TODO(konstin): This is still wrong, we need to store the cache policy too!
1063 /// ```text
1064 /// wheel-v0
1065 /// ├── pypi
1066 /// │ ...
1067 /// │ ├── pandas
1068 /// │ │ ├── pandas-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
1069 /// │ │ ├── pandas-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64
1070 /// │ ...
1071 /// └── url
1072 /// └── 4b8be67c801a7ecb
1073 /// └── flask
1074 /// └── flask-3.0.0-py3-none-any.whl
1075 /// ├── flask
1076 /// │ └── ...
1077 /// └── flask-3.0.0.dist-info
1078 /// └── ...
1079 /// ```
1080 ///
1081 /// If we run first `pip compile` and then `pip sync` on the same machine, we get both:
1082 ///
1083 /// ```text
1084 /// wheels-v0
1085 /// ├── pypi
1086 /// │ ├── ...
1087 /// │ ├── pandas
1088 /// │ │ ├── pandas-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.msgpack
1089 /// │ │ ├── pandas-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
1090 /// │ │ └── pandas-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64
1091 /// │ │ ├── pandas
1092 /// │ │ │ ├── ...
1093 /// │ │ ├── pandas-2.1.3.dist-info
1094 /// │ │ │ ├── ...
1095 /// │ │ └── pandas.libs
1096 /// │ ├── ...
1097 /// └── url
1098 /// └── 4b8be67c801a7ecb
1099 /// └── flask
1100 /// ├── flask-3.0.0-py3-none-any.msgpack
1101 /// ├── flask-3.0.0-py3-none-any.msgpack
1102 /// └── flask-3.0.0-py3-none-any
1103 /// ├── flask
1104 /// │ └── ...
1105 /// └── flask-3.0.0.dist-info
1106 /// └── ...
1107 Wheels,
1108 /// Source distributions, wheels built from source distributions, their extracted metadata, and the
1109 /// cache policy of the source distribution.
1110 ///
1111 /// The structure is similar of that of the `Wheel` bucket, except we have an additional layer
1112 /// for the source distribution filename and the metadata is at the source distribution-level,
1113 /// not at the wheel level.
1114 ///
1115 /// TODO(konstin): The cache policy should be on the source distribution level, the metadata we
1116 /// can put next to the wheels as in the `Wheels` bucket.
1117 ///
1118 /// The unzipped source distribution is stored in a directory matching the source distribution
1119 /// archive name.
1120 ///
1121 /// Source distributions are built into zipped wheel files (as PEP 517 specifies) and unzipped
1122 /// lazily before installing. So when resolving, we only build the wheel and store the archive
1123 /// file in the cache, when installing, we unpack it under the same name (exclusive of the
1124 /// `.whl` extension). You may find a mix of wheel archive zip files and unzipped wheel
1125 /// directories in the cache.
1126 ///
1127 /// Cache structure:
1128 /// * `built-wheels-v0/pypi/foo/34a17436ed1e9669/{manifest.msgpack, metadata.msgpack, foo-1.0.0.zip, foo-1.0.0-py3-none-any.whl, ...other wheels}`
1129 /// * `built-wheels-v0/<digest(index-url)>/foo/foo-1.0.0.zip/{manifest.msgpack, metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}`
1130 /// * `built-wheels-v0/url/<digest(url)>/foo/foo-1.0.0.zip/{manifest.msgpack, metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}`
1131 /// * `built-wheels-v0/git/<digest(url)>/<git sha>/foo/foo-1.0.0.zip/{metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}`
1132 ///
1133 /// But the url filename does not need to be a valid source dist filename
1134 /// (<https://github.com/search?q=path%3A**%2Frequirements.txt+master.zip&type=code>),
1135 /// so it could also be the following and we have to take any string as filename:
1136 /// * `built-wheels-v0/url/<sha256(url)>/master.zip/metadata.msgpack`
1137 ///
1138 /// # Example
1139 ///
1140 /// The following requirements:
1141 /// ```text
1142 /// # git source dist
1143 /// pydantic-extra-types @ git+https://github.com/pydantic/pydantic-extra-types.git
1144 /// # pypi source dist
1145 /// django_allauth==0.51.0
1146 /// # url source dist
1147 /// werkzeug @ https://files.pythonhosted.org/packages/0d/cc/ff1904eb5eb4b455e442834dabf9427331ac0fa02853bf83db817a7dd53d/werkzeug-3.0.1.tar.gz
1148 /// ```
1149 ///
1150 /// ...may be cached as:
1151 /// ```text
1152 /// built-wheels-v4/
1153 /// ├── git
1154 /// │ └── 2122faf3e081fb7a
1155 /// │ └── 7a2d650a4a7b4d04
1156 /// │ ├── metadata.msgpack
1157 /// │ └── pydantic_extra_types-2.9.0-py3-none-any.whl
1158 /// ├── pypi
1159 /// │ └── django-allauth
1160 /// │ └── 0.51.0
1161 /// │ ├── 0gH-_fwv8tdJ7JwwjJsUc
1162 /// │ │ ├── django-allauth-0.51.0.tar.gz
1163 /// │ │ │ └── [UNZIPPED CONTENTS]
1164 /// │ │ ├── django_allauth-0.51.0-py3-none-any.whl
1165 /// │ │ └── metadata.msgpack
1166 /// │ └── revision.http
1167 /// └── url
1168 /// └── 6781bd6440ae72c2
1169 /// ├── APYY01rbIfpAo_ij9sCY6
1170 /// │ ├── metadata.msgpack
1171 /// │ ├── werkzeug-3.0.1-py3-none-any.whl
1172 /// │ └── werkzeug-3.0.1.tar.gz
1173 /// │ └── [UNZIPPED CONTENTS]
1174 /// └── revision.http
1175 /// ```
1176 ///
1177 /// Structurally, the `manifest.msgpack` is empty, and only contains the caching information
1178 /// needed to invalidate the cache. The `metadata.msgpack` contains the metadata of the source
1179 /// distribution.
1180 SourceDistributions,
1181 /// Flat index responses, a format very similar to the simple metadata API.
1182 ///
1183 /// Cache structure:
1184 /// * `flat-index-v0/index/<digest(flat_index_url)>.msgpack`
1185 ///
1186 /// The response is stored as `Vec<File>`.
1187 FlatIndex,
1188 /// Git repositories.
1189 Git,
1190 /// Information about an interpreter at a path.
1191 ///
1192 /// To avoid caching pyenv shims, bash scripts which may redirect to a new python version
1193 /// without the shim itself changing, we only cache when the path equals `sys.executable`, i.e.
1194 /// the path we're running is the python executable itself and not a shim.
1195 ///
1196 /// Cache structure: `interpreter-v0/<digest(path)>.msgpack`
1197 ///
1198 /// # Example
1199 ///
1200 /// The contents of each of the `MsgPack` files has a timestamp field in unix time, the [PEP 508]
1201 /// markers and some information from the `sys`/`sysconfig` modules.
1202 ///
1203 /// ```json
1204 /// {
1205 /// "timestamp": 1698047994491,
1206 /// "data": {
1207 /// "markers": {
1208 /// "implementation_name": "cpython",
1209 /// "implementation_version": "3.12.0",
1210 /// "os_name": "posix",
1211 /// "platform_machine": "x86_64",
1212 /// "platform_python_implementation": "CPython",
1213 /// "platform_release": "6.5.0-13-generic",
1214 /// "platform_system": "Linux",
1215 /// "platform_version": "#13-Ubuntu SMP PREEMPT_DYNAMIC Fri Nov 3 12:16:05 UTC 2023",
1216 /// "python_full_version": "3.12.0",
1217 /// "python_version": "3.12",
1218 /// "sys_platform": "linux"
1219 /// },
1220 /// "base_exec_prefix": "/home/ferris/.pyenv/versions/3.12.0",
1221 /// "base_prefix": "/home/ferris/.pyenv/versions/3.12.0",
1222 /// "sys_executable": "/home/ferris/projects/uv/.venv/bin/python"
1223 /// }
1224 /// }
1225 /// ```
1226 ///
1227 /// [PEP 508]: https://peps.python.org/pep-0508/#environment-markers
1228 Interpreter,
1229 /// Index responses through the simple metadata API.
1230 ///
1231 /// Cache structure:
1232 /// * `simple-v0/pypi/<package_name>.rkyv`
1233 /// * `simple-v0/<digest(index_url)>/<package_name>.rkyv`
1234 ///
1235 /// The response is parsed into `uv_client::SimpleDetailMetadata` before storage.
1236 Simple,
1237 /// A cache of unzipped wheels, stored as directories. This is used internally within the cache.
1238 /// When other buckets need to store directories, they should persist them to
1239 /// [`CacheBucket::Archive`], and then symlink them into the appropriate bucket. This ensures
1240 /// that cache entries can be atomically replaced and removed, as storing directories in the
1241 /// other buckets directly would make atomic operations impossible.
1242 Archive,
1243 /// Ephemeral virtual environments used to execute PEP 517 builds and other operations.
1244 Builds,
1245 /// Reusable virtual environments for Python tools and projects.
1246 Environments,
1247 /// Cached Python downloads
1248 Python,
1249 /// Downloaded tool binaries (e.g., Ruff).
1250 Binaries,
1251 /// Cached vulnerability data from [OSV](https://osv.dev/).
1252 ///
1253 /// Cache structure:
1254 /// * `osv-v0/vulnerability/<vuln_id>.msgpack` — cached full vulnerability records
1255 Osv,
1256}
1257
1258impl CacheBucket {
1259 fn to_str(self) -> &'static str {
1260 match self {
1261 // Note that when bumping this, you'll also need to bump it
1262 // in `crates/uv/tests/build/cache_prune.rs`.
1263 Self::SourceDistributions => "sdists-v9",
1264 // Note that when bumping this, you'll also need to bump it
1265 // in `crates/uv/tests/lock/lock.rs`.
1266 Self::FlatIndex => "flat-index-v4",
1267 Self::Git => "git-v0",
1268 Self::Interpreter => "interpreter-v4",
1269 // Note that when bumping this, you'll also need to bump it
1270 // in `crates/uv/tests/build/cache_clean.rs`.
1271 Self::Simple => "simple-v24",
1272 // Note that when bumping this, you'll also need to bump it
1273 // in `crates/uv/tests/build/cache_prune.rs`.
1274 Self::Wheels => "wheels-v6",
1275 // Note that when bumping this, you'll also need to bump
1276 // `ARCHIVE_VERSION` in `crates/uv-cache/src/lib.rs`.
1277 Self::Archive => "archive-v0",
1278 Self::Builds => "builds-v0",
1279 Self::Environments => "environments-v2",
1280 Self::Python => "python-v0",
1281 Self::Binaries => "binaries-v0",
1282 Self::Osv => "osv-v0",
1283 }
1284 }
1285
1286 /// Remove a package from the cache bucket.
1287 ///
1288 /// Returns the number of entries removed from the cache.
1289 fn remove(self, cache: &Cache, name: &PackageName) -> Result<Removal, io::Error> {
1290 /// Returns `true` if the [`Path`] represents a built wheel for the given package.
1291 fn is_match(path: &Path, name: &PackageName) -> bool {
1292 let Ok(metadata) = fs_err::read(path.join("metadata.msgpack")) else {
1293 return false;
1294 };
1295 let Ok(metadata) = rmp_serde::from_slice::<ResolutionMetadata>(&metadata) else {
1296 return false;
1297 };
1298 metadata.name == *name
1299 }
1300
1301 let mut summary = cache.removal();
1302 match self {
1303 Self::Wheels => {
1304 // For `pypi` wheels, we expect a directory per package (indexed by name).
1305 let root = cache.bucket(self).join(WheelCacheKind::Pypi);
1306 summary += cache.remove_path(root.join(name.to_string()))?;
1307
1308 // For alternate indices, we expect a directory for every index (under an `index`
1309 // subdirectory), followed by a directory per package (indexed by name).
1310 let root = cache.bucket(self).join(WheelCacheKind::Index);
1311 for directory in directories(root)? {
1312 summary += cache.remove_path(directory.join(name.to_string()))?;
1313 }
1314
1315 // For direct URLs, we expect a directory for every URL, followed by a
1316 // directory per package (indexed by name).
1317 let root = cache.bucket(self).join(WheelCacheKind::Url);
1318 for directory in directories(root)? {
1319 summary += cache.remove_path(directory.join(name.to_string()))?;
1320 }
1321 }
1322 Self::SourceDistributions => {
1323 // For `pypi` wheels, we expect a directory per package (indexed by name).
1324 let root = cache.bucket(self).join(WheelCacheKind::Pypi);
1325 summary += cache.remove_path(root.join(name.to_string()))?;
1326
1327 // For alternate indices, we expect a directory for every index (under an `index`
1328 // subdirectory), followed by a directory per package (indexed by name).
1329 let root = cache.bucket(self).join(WheelCacheKind::Index);
1330 for directory in directories(root)? {
1331 summary += cache.remove_path(directory.join(name.to_string()))?;
1332 }
1333
1334 // For direct URLs, we expect a directory for every URL, followed by a
1335 // directory per version. To determine whether the URL is relevant, we need to
1336 // search for a wheel matching the package name.
1337 let root = cache.bucket(self).join(WheelCacheKind::Url);
1338 for url in directories(root)? {
1339 if directories(&url)?.any(|version| is_match(&version, name)) {
1340 summary += cache.remove_path(url)?;
1341 }
1342 }
1343
1344 // For local dependencies, we expect a directory for every path, followed by a
1345 // directory per version. To determine whether the path is relevant, we need to
1346 // search for a wheel matching the package name.
1347 let root = cache.bucket(self).join(WheelCacheKind::Path);
1348 for path in directories(root)? {
1349 if directories(&path)?.any(|version| is_match(&version, name)) {
1350 summary += cache.remove_path(path)?;
1351 }
1352 }
1353
1354 // For Git dependencies, we expect a directory for every repository, followed by a
1355 // directory for every SHA. To determine whether the SHA is relevant, we need to
1356 // search for a wheel matching the package name.
1357 let root = cache.bucket(self).join(WheelCacheKind::Git);
1358 for repository in directories(root)? {
1359 for sha in directories(repository)? {
1360 if is_match(&sha, name) {
1361 summary += cache.remove_path(sha)?;
1362 }
1363 }
1364 }
1365 }
1366 Self::Simple => {
1367 // For `pypi` wheels, we expect a rkyv file per package, indexed by name.
1368 let root = cache.bucket(self).join(WheelCacheKind::Pypi);
1369 summary += cache.remove_path(root.join(format!("{name}.rkyv")))?;
1370
1371 // For alternate indices, we expect a directory for every index (under an `index`
1372 // subdirectory), followed by a directory per package (indexed by name).
1373 let root = cache.bucket(self).join(WheelCacheKind::Index);
1374 for directory in directories(root)? {
1375 summary += cache.remove_path(directory.join(format!("{name}.rkyv")))?;
1376 }
1377 }
1378 Self::FlatIndex => {
1379 // We can't know if the flat index includes a package, so we just remove the entire
1380 // cache entry.
1381 let root = cache.bucket(self);
1382 summary += cache.remove_path(root)?;
1383 }
1384 Self::Git
1385 | Self::Interpreter
1386 | Self::Archive
1387 | Self::Builds
1388 | Self::Environments
1389 | Self::Python
1390 | Self::Binaries
1391 | Self::Osv => {
1392 // Nothing to do.
1393 }
1394 }
1395 Ok(summary)
1396 }
1397
1398 /// Return an iterator over all cache buckets.
1399 fn iter() -> impl Iterator<Item = Self> {
1400 [
1401 Self::Wheels,
1402 Self::SourceDistributions,
1403 Self::FlatIndex,
1404 Self::Git,
1405 Self::Interpreter,
1406 Self::Simple,
1407 Self::Archive,
1408 Self::Builds,
1409 Self::Environments,
1410 Self::Python,
1411 Self::Binaries,
1412 Self::Osv,
1413 ]
1414 .iter()
1415 .copied()
1416 }
1417}
1418
1419impl Display for CacheBucket {
1420 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1421 f.write_str(self.to_str())
1422 }
1423}
1424
1425#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1426pub enum Freshness {
1427 /// The cache entry is fresh according to the [`Refresh`] policy.
1428 Fresh,
1429 /// The cache entry is stale according to the [`Refresh`] policy.
1430 Stale,
1431 /// The cache entry does not exist.
1432 Missing,
1433}
1434
1435impl Freshness {
1436 pub const fn is_fresh(self) -> bool {
1437 matches!(self, Self::Fresh)
1438 }
1439}
1440
1441/// A refresh policy for cache entries.
1442#[derive(Debug, Clone)]
1443pub enum Refresh {
1444 /// Don't refresh any entries.
1445 None(Timestamp),
1446 /// Refresh entries linked to the given packages, if created before the given timestamp.
1447 Packages(Vec<PackageName>, Vec<Box<Path>>, Timestamp),
1448 /// Refresh all entries created before the given timestamp.
1449 All(Timestamp),
1450}
1451
1452impl Refresh {
1453 /// Determine the refresh strategy to use based on the command-line arguments.
1454 pub fn from_args(refresh: Option<bool>, refresh_package: Vec<PackageName>) -> Self {
1455 let timestamp = Timestamp::now();
1456 match refresh {
1457 Some(true) => Self::All(timestamp),
1458 Some(false) => Self::None(timestamp),
1459 None => {
1460 if refresh_package.is_empty() {
1461 Self::None(timestamp)
1462 } else {
1463 Self::Packages(refresh_package, vec![], timestamp)
1464 }
1465 }
1466 }
1467 }
1468
1469 /// Combine two [`Refresh`] policies, taking the "max" of the two policies.
1470 #[must_use]
1471 pub fn combine(self, other: Self) -> Self {
1472 match (self, other) {
1473 // If the policy is `None`, return the existing refresh policy.
1474 // Take the `max` of the two timestamps.
1475 (Self::None(t1), Self::None(t2)) => Self::None(t1.max(t2)),
1476 (Self::None(t1), Self::All(t2)) => Self::All(t1.max(t2)),
1477 (Self::None(t1), Self::Packages(packages, paths, t2)) => {
1478 Self::Packages(packages, paths, t1.max(t2))
1479 }
1480
1481 // If the policy is `All`, refresh all packages.
1482 (Self::All(t1), Self::None(t2) | Self::All(t2) | Self::Packages(.., t2)) => {
1483 Self::All(t1.max(t2))
1484 }
1485
1486 // If the policy is `Packages`, take the "max" of the two policies.
1487 (Self::Packages(packages, paths, t1), Self::None(t2)) => {
1488 Self::Packages(packages, paths, t1.max(t2))
1489 }
1490 (Self::Packages(.., t1), Self::All(t2)) => Self::All(t1.max(t2)),
1491 (Self::Packages(packages1, paths1, t1), Self::Packages(packages2, paths2, t2)) => {
1492 Self::Packages(
1493 packages1.into_iter().chain(packages2).collect(),
1494 paths1.into_iter().chain(paths2).collect(),
1495 t1.max(t2),
1496 )
1497 }
1498 }
1499 }
1500}
1501
1502#[cfg(test)]
1503mod tests {
1504 use std::str::FromStr;
1505
1506 use crate::ArchiveId;
1507
1508 use super::Link;
1509
1510 #[test]
1511 fn test_link_round_trip() {
1512 let id = ArchiveId::new();
1513 let link = Link::new(id);
1514 let s = link.to_string();
1515 let parsed = Link::from_str(&s).unwrap();
1516 assert_eq!(link.id, parsed.id);
1517 assert_eq!(link.version, parsed.version);
1518 }
1519
1520 #[test]
1521 fn test_link_deserialize() {
1522 assert!(Link::from_str("archive-v0/foo").is_ok());
1523 assert!(Link::from_str("archive/foo").is_err());
1524 assert!(Link::from_str("v1/foo").is_err());
1525 assert!(Link::from_str("archive-v0/").is_err());
1526 }
1527
1528 #[test]
1529 #[cfg(unix)]
1530 fn prune_does_not_follow_environment_symlinks() {
1531 use super::{Cache, CacheBucket};
1532
1533 let cache_root = tempfile::tempdir().unwrap();
1534 let victim_root = tempfile::tempdir().unwrap();
1535 let environments = cache_root.path().join(CacheBucket::Environments.to_str());
1536 let victim_dir = victim_root.path().join("victim-dir");
1537
1538 fs_err::create_dir_all(&environments).unwrap();
1539 fs_err::create_dir_all(&victim_dir).unwrap();
1540 fs_err::write(victim_dir.join("payload.txt"), "payload").unwrap();
1541 fs_err::os::unix::fs::symlink(&victim_dir, environments.join("escape")).unwrap();
1542
1543 let summary = Cache::from_path(cache_root.path()).prune(false).unwrap();
1544
1545 assert_eq!(summary.num_files, 1);
1546 assert_eq!(summary.num_dirs, 0);
1547 assert!(victim_dir.is_dir());
1548 assert!(victim_dir.join("payload.txt").is_file());
1549 assert!(fs_err::symlink_metadata(environments.join("escape")).is_err());
1550 }
1551
1552 #[test]
1553 #[cfg(unix)]
1554 fn prune_ci_does_not_follow_wheel_symlinks() {
1555 use super::{Cache, CacheBucket};
1556
1557 let cache_root = tempfile::tempdir().unwrap();
1558 let victim_root = tempfile::tempdir().unwrap();
1559 let wheels = cache_root.path().join(CacheBucket::Wheels.to_str());
1560 let source_distributions = cache_root
1561 .path()
1562 .join(CacheBucket::SourceDistributions.to_str());
1563 let victim_dir = victim_root.path().join("victim-dir");
1564 let symlink = wheels.join("escape");
1565
1566 fs_err::create_dir_all(&wheels).unwrap();
1567 fs_err::create_dir_all(&source_distributions).unwrap();
1568 fs_err::create_dir_all(&victim_dir).unwrap();
1569 fs_err::write(victim_dir.join("payload.txt"), "payload").unwrap();
1570 fs_err::os::unix::fs::symlink(&victim_dir, &symlink).unwrap();
1571
1572 let summary = Cache::from_path(cache_root.path()).prune(true).unwrap();
1573
1574 assert_eq!(summary.num_files, 1);
1575 assert_eq!(summary.num_dirs, 0);
1576 assert!(victim_dir.is_dir());
1577 assert!(victim_dir.join("payload.txt").is_file());
1578 assert!(fs_err::symlink_metadata(symlink).is_err());
1579 }
1580
1581 #[test]
1582 #[cfg(unix)]
1583 fn prune_does_not_follow_archive_symlinks() {
1584 use super::{Cache, CacheBucket};
1585
1586 let cache_root = tempfile::tempdir().unwrap();
1587 let victim_root = tempfile::tempdir().unwrap();
1588 let archives = cache_root.path().join(CacheBucket::Archive.to_str());
1589 let victim_dir = victim_root.path().join("victim-dir");
1590 let symlink = archives.join("escape");
1591
1592 fs_err::create_dir_all(&archives).unwrap();
1593 fs_err::create_dir_all(&victim_dir).unwrap();
1594 fs_err::write(victim_dir.join("payload.txt"), "payload").unwrap();
1595 fs_err::os::unix::fs::symlink(&victim_dir, &symlink).unwrap();
1596
1597 let summary = Cache::from_path(cache_root.path()).prune(false).unwrap();
1598
1599 assert_eq!(summary.num_files, 1);
1600 assert_eq!(summary.num_dirs, 0);
1601 assert!(victim_dir.is_dir());
1602 assert!(victim_dir.join("payload.txt").is_file());
1603 assert!(fs_err::symlink_metadata(symlink).is_err());
1604 }
1605}