uv_cache/lib.rs
1use std::fmt::{Display, Formatter};
2use std::io;
3use std::io::Write;
4use std::ops::Deref;
5use std::path::{Path, PathBuf};
6use std::str::FromStr;
7use std::sync::Arc;
8
9use rustc_hash::FxHashMap;
10use tracing::{debug, trace, warn};
11
12use uv_cache_info::Timestamp;
13use uv_fs::{LockedFile, LockedFileError, LockedFileMode, Simplified, cachedir, directories};
14use uv_normalize::PackageName;
15use uv_pypi_types::ResolutionMetadata;
16
17pub use crate::by_timestamp::CachedByTimestamp;
18#[cfg(feature = "clap")]
19pub use crate::cli::CacheArgs;
20use crate::removal::Remover;
21pub use crate::removal::{Removal, RemovalAccounting};
22pub use crate::wheel::WheelCache;
23use crate::wheel::WheelCacheKind;
24pub use archive::{ArchiveFileId, ArchiveId};
25
26mod archive;
27mod by_timestamp;
28#[cfg(feature = "clap")]
29mod cli;
30mod removal;
31mod wheel;
32
33/// The version of the archive bucket.
34///
35/// Must be kept in-sync with the version in [`CacheBucket::to_str`].
36pub const ARCHIVE_VERSION: u8 = 0;
37
38/// Error locking a cache entry or shard
39#[derive(Debug, thiserror::Error)]
40pub enum Error {
41 #[error(transparent)]
42 Io(#[from] io::Error),
43 #[error("Failed to initialize cache at `{}`", _0.user_display())]
44 Init(PathBuf, #[source] io::Error),
45 #[error("Could not make the path absolute")]
46 Absolute(#[source] io::Error),
47 #[error("Could not acquire lock")]
48 Acquire(#[from] LockedFileError),
49}
50
51/// A [`CacheEntry`] which may or may not exist yet.
52#[derive(Debug, Clone)]
53pub struct CacheEntry(PathBuf);
54
55impl CacheEntry {
56 /// Create a new [`CacheEntry`] from a directory and a file name.
57 pub fn new(dir: impl Into<PathBuf>, file: impl AsRef<Path>) -> Self {
58 Self(dir.into().join(file))
59 }
60
61 /// Create a new [`CacheEntry`] from a path.
62 pub fn from_path(path: impl Into<PathBuf>) -> Self {
63 Self(path.into())
64 }
65
66 /// Return the cache entry's parent directory.
67 pub fn shard(&self) -> CacheShard {
68 CacheShard(self.dir().to_path_buf())
69 }
70
71 /// Convert the [`CacheEntry`] into a [`PathBuf`].
72 #[inline]
73 pub fn into_path_buf(self) -> PathBuf {
74 self.0
75 }
76
77 /// Return the path to the [`CacheEntry`].
78 #[inline]
79 pub fn path(&self) -> &Path {
80 &self.0
81 }
82
83 /// Return the cache entry's parent directory.
84 #[inline]
85 pub fn dir(&self) -> &Path {
86 self.0.parent().expect("Cache entry has no parent")
87 }
88
89 /// Create a new [`CacheEntry`] with the given file name.
90 #[must_use]
91 pub fn with_file(&self, file: impl AsRef<Path>) -> Self {
92 Self(self.dir().join(file))
93 }
94
95 /// Acquire the [`CacheEntry`] as an exclusive lock.
96 pub async fn lock(&self) -> Result<LockedFile, Error> {
97 fs_err::create_dir_all(self.dir())?;
98 Ok(LockedFile::acquire(
99 self.path(),
100 LockedFileMode::Exclusive,
101 self.path().display(),
102 )
103 .await?)
104 }
105}
106
107impl AsRef<Path> for CacheEntry {
108 fn as_ref(&self) -> &Path {
109 &self.0
110 }
111}
112
113/// A subdirectory within the cache.
114#[derive(Debug, Clone)]
115pub struct CacheShard(PathBuf);
116
117impl CacheShard {
118 /// Return a [`CacheEntry`] within this shard.
119 pub fn entry(&self, file: impl AsRef<Path>) -> CacheEntry {
120 CacheEntry::new(&self.0, file)
121 }
122
123 /// Return a [`CacheShard`] within this shard.
124 #[must_use]
125 pub fn shard(&self, dir: impl AsRef<Path>) -> Self {
126 Self(self.0.join(dir.as_ref()))
127 }
128
129 /// Acquire the cache entry as an exclusive lock.
130 pub async fn lock(&self) -> Result<LockedFile, Error> {
131 fs_err::create_dir_all(self.as_ref())?;
132 Ok(LockedFile::acquire(
133 self.join(".lock"),
134 LockedFileMode::Exclusive,
135 self.display(),
136 )
137 .await?)
138 }
139
140 /// Return the [`CacheShard`] as a [`PathBuf`].
141 pub fn into_path_buf(self) -> PathBuf {
142 self.0
143 }
144}
145
146impl AsRef<Path> for CacheShard {
147 fn as_ref(&self) -> &Path {
148 &self.0
149 }
150}
151
152impl Deref for CacheShard {
153 type Target = Path;
154
155 fn deref(&self) -> &Self::Target {
156 &self.0
157 }
158}
159
160/// The main cache abstraction.
161///
162/// While the cache is active, it holds a read (shared) lock that prevents cache cleaning
163#[derive(Debug, Clone)]
164pub struct Cache {
165 /// The cache directory.
166 root: PathBuf,
167 /// The refresh strategy to use when reading from the cache.
168 refresh: Refresh,
169 /// A temporary cache directory, if the user requested `--no-cache`.
170 ///
171 /// Included to ensure that the temporary directory exists for the length of the operation, but
172 /// is dropped at the end as appropriate.
173 temp_dir: Option<Arc<tempfile::TempDir>>,
174 /// Ensure that `uv cache` operations don't remove items from the cache that are used by another
175 /// uv process.
176 lock_file: Option<Arc<LockedFile>>,
177 /// The storage accounting used when removing cache entries.
178 removal_accounting: RemovalAccounting,
179}
180
181impl Cache {
182 /// A persistent cache directory at `root`.
183 pub fn from_path(root: impl Into<PathBuf>) -> Self {
184 Self {
185 root: root.into(),
186 refresh: Refresh::None(Timestamp::now()),
187 temp_dir: None,
188 lock_file: None,
189 removal_accounting: RemovalAccounting::Coarse,
190 }
191 }
192
193 /// Create a temporary cache directory.
194 pub fn temp() -> Result<Self, io::Error> {
195 let temp_dir = tempfile::tempdir()?;
196 Ok(Self {
197 root: temp_dir.path().to_path_buf(),
198 refresh: Refresh::None(Timestamp::now()),
199 temp_dir: Some(Arc::new(temp_dir)),
200 lock_file: None,
201 removal_accounting: RemovalAccounting::Coarse,
202 })
203 }
204
205 /// Set the [`Refresh`] policy for the cache.
206 #[must_use]
207 pub fn with_refresh(self, refresh: Refresh) -> Self {
208 Self { refresh, ..self }
209 }
210
211 /// Set the storage accounting used when removing cache entries.
212 ///
213 /// Falls back to [`RemovalAccounting::Coarse`] when fine-grained accounting is unsupported.
214 #[must_use]
215 pub fn with_removal_accounting(self, removal_accounting: RemovalAccounting) -> Self {
216 let removal_accounting = match removal_accounting {
217 RemovalAccounting::Fine if !uv_fs::supports_fine_grained_accounting() => {
218 RemovalAccounting::Coarse
219 }
220 removal_accounting => removal_accounting,
221 };
222 Self {
223 removal_accounting,
224 ..self
225 }
226 }
227
228 /// Create an empty removal summary using the cache's configured accounting.
229 pub fn removal(&self) -> Removal {
230 Removal::new(self.removal_accounting)
231 }
232
233 /// Acquire a lock that allows removing entries from the cache.
234 pub async fn with_exclusive_lock(self) -> Result<Self, LockedFileError> {
235 let Self {
236 root,
237 refresh,
238 temp_dir,
239 lock_file,
240 removal_accounting,
241 } = self;
242
243 // Release the existing lock, avoid deadlocks from a cloned cache.
244 if let Some(lock_file) = lock_file {
245 drop(
246 Arc::try_unwrap(lock_file).expect(
247 "cloning the cache before acquiring an exclusive lock causes a deadlock",
248 ),
249 );
250 }
251 let lock_file = LockedFile::acquire(
252 root.join(".lock"),
253 LockedFileMode::Exclusive,
254 root.simplified_display(),
255 )
256 .await?;
257
258 Ok(Self {
259 root,
260 refresh,
261 temp_dir,
262 lock_file: Some(Arc::new(lock_file)),
263 removal_accounting,
264 })
265 }
266
267 /// Acquire a lock that allows removing entries from the cache, if available.
268 ///
269 /// If the lock is not immediately available, returns [`Err`] with self.
270 pub fn with_exclusive_lock_no_wait(self) -> Result<Self, Self> {
271 let Self {
272 root,
273 refresh,
274 temp_dir,
275 lock_file,
276 removal_accounting,
277 } = self;
278
279 match LockedFile::acquire_no_wait(
280 root.join(".lock"),
281 LockedFileMode::Exclusive,
282 root.simplified_display(),
283 ) {
284 Some(lock_file) => Ok(Self {
285 root,
286 refresh,
287 temp_dir,
288 lock_file: Some(Arc::new(lock_file)),
289 removal_accounting,
290 }),
291 None => Err(Self {
292 root,
293 refresh,
294 temp_dir,
295 lock_file,
296 removal_accounting,
297 }),
298 }
299 }
300
301 /// Return the root of the cache.
302 pub fn root(&self) -> &Path {
303 &self.root
304 }
305
306 /// The folder for a specific cache bucket
307 pub fn bucket(&self, cache_bucket: CacheBucket) -> PathBuf {
308 self.root.join(cache_bucket.to_str())
309 }
310
311 /// Compute an entry in the cache.
312 pub fn shard(&self, cache_bucket: CacheBucket, dir: impl AsRef<Path>) -> CacheShard {
313 CacheShard(self.bucket(cache_bucket).join(dir.as_ref()))
314 }
315
316 /// Compute an entry in the cache.
317 pub fn entry(
318 &self,
319 cache_bucket: CacheBucket,
320 dir: impl AsRef<Path>,
321 file: impl AsRef<Path>,
322 ) -> CacheEntry {
323 CacheEntry::new(self.bucket(cache_bucket).join(dir), file)
324 }
325
326 /// Return the path to an archive in the cache.
327 pub fn archive(&self, id: &ArchiveId) -> PathBuf {
328 self.bucket(CacheBucket::Archive).join(id)
329 }
330
331 /// Return the path to an archive file in the cache.
332 pub fn archive_file(&self, id: &ArchiveFileId) -> PathBuf {
333 self.bucket(CacheBucket::Files).join(id)
334 }
335
336 /// Create a temporary directory to be used as a Python virtual environment.
337 pub fn venv_dir(&self) -> io::Result<tempfile::TempDir> {
338 fs_err::create_dir_all(self.bucket(CacheBucket::Builds))?;
339 tempfile::tempdir_in(self.bucket(CacheBucket::Builds))
340 }
341
342 /// Create a temporary directory to be used for executing PEP 517 source distribution builds.
343 pub fn build_dir(&self) -> io::Result<tempfile::TempDir> {
344 fs_err::create_dir_all(self.bucket(CacheBucket::Builds))?;
345 tempfile::tempdir_in(self.bucket(CacheBucket::Builds))
346 }
347
348 /// Returns `true` if a cache entry must be revalidated given the [`Refresh`] policy.
349 pub fn must_revalidate_package(&self, package: &PackageName) -> bool {
350 match &self.refresh {
351 Refresh::None(_) => false,
352 Refresh::All(_) => true,
353 Refresh::Packages(packages, _, _) => packages.contains(package),
354 }
355 }
356
357 /// Returns `true` if a cache entry must be revalidated given the [`Refresh`] policy.
358 pub fn must_revalidate_path(&self, path: &Path) -> bool {
359 match &self.refresh {
360 Refresh::None(_) => false,
361 Refresh::All(_) => true,
362 Refresh::Packages(_, paths, _) => paths
363 .iter()
364 .any(|target| same_file::is_same_file(path, target).unwrap_or(false)),
365 }
366 }
367
368 /// Returns the [`Freshness`] for a cache entry, validating it against the [`Refresh`] policy.
369 ///
370 /// A cache entry is considered fresh if it was created after the cache itself was
371 /// initialized, or if the [`Refresh`] policy does not require revalidation.
372 pub fn freshness(
373 &self,
374 entry: &CacheEntry,
375 package: Option<&PackageName>,
376 path: Option<&Path>,
377 ) -> io::Result<Freshness> {
378 // Grab the cutoff timestamp, if it's relevant.
379 let timestamp = match &self.refresh {
380 Refresh::None(_) => return Ok(Freshness::Fresh),
381 Refresh::All(timestamp) => timestamp,
382 Refresh::Packages(packages, paths, timestamp) => {
383 if package.is_none_or(|package| packages.contains(package))
384 || path.is_some_and(|path| {
385 paths
386 .iter()
387 .any(|target| same_file::is_same_file(path, target).unwrap_or(false))
388 })
389 {
390 timestamp
391 } else {
392 return Ok(Freshness::Fresh);
393 }
394 }
395 };
396
397 match fs_err::metadata(entry.path()) {
398 Ok(metadata) => {
399 if Timestamp::from_metadata(&metadata) >= *timestamp {
400 Ok(Freshness::Fresh)
401 } else {
402 Ok(Freshness::Stale)
403 }
404 }
405 Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(Freshness::Missing),
406 Err(err) => Err(err),
407 }
408 }
409
410 /// Persist a temporary directory to the artifact store, returning its unique ID.
411 pub async fn persist(
412 &self,
413 temp_dir: impl AsRef<Path>,
414 path: impl AsRef<Path>,
415 ) -> io::Result<ArchiveId> {
416 // Create a unique ID for the artifact.
417 let id = ArchiveId::new();
418
419 // Move the temporary directory into the directory store.
420 let archive_entry = self.entry(CacheBucket::Archive, "", &id);
421 fs_err::create_dir_all(archive_entry.dir())?;
422 uv_fs::rename_with_retry(temp_dir.as_ref(), archive_entry.path()).await?;
423
424 // Create a symlink to the directory store.
425 fs_err::create_dir_all(path.as_ref().parent().expect("Cache entry to have parent"))?;
426 self.create_link(&id, path.as_ref())?;
427
428 Ok(id)
429 }
430
431 /// Persist a temporary directory to the artifact store under a caller-selected ID.
432 ///
433 /// If another writer has already persisted the same ID, discard `temp_dir` and link `path` to
434 /// the existing archive entry. The ID must therefore uniquely identify the directory contents.
435 pub async fn persist_with_id(
436 &self,
437 temp_dir: tempfile::TempDir,
438 path: impl AsRef<Path>,
439 id: ArchiveId,
440 ) -> io::Result<ArchiveId> {
441 // Move the temporary directory into the directory store.
442 let archive_entry = self.entry(CacheBucket::Archive, "", &id);
443 fs_err::create_dir_all(archive_entry.dir())?;
444 if let Err(err) = uv_fs::rename_with_retry(temp_dir.path(), archive_entry.path()).await {
445 if !archive_entry.path().is_dir() {
446 return Err(err);
447 }
448 }
449
450 // Create a symlink to the directory store.
451 fs_err::create_dir_all(path.as_ref().parent().expect("Cache entry to have parent"))?;
452 self.create_link(&id, path.as_ref())?;
453
454 Ok(id)
455 }
456
457 /// Returns `true` if the [`Cache`] is temporary.
458 pub fn is_temporary(&self) -> bool {
459 self.temp_dir.is_some()
460 }
461
462 /// Populate the cache scaffold.
463 fn create_base_files(root: &PathBuf) -> io::Result<()> {
464 // Create the cache directory, if it doesn't exist.
465 fs_err::create_dir_all(root)?;
466
467 // Add the CACHEDIR.TAG.
468 cachedir::ensure_tag(root)?;
469
470 // Add the .gitignore.
471 match fs_err::OpenOptions::new()
472 .write(true)
473 .create_new(true)
474 .open(root.join(".gitignore"))
475 {
476 Ok(mut file) => file.write_all(b"*")?,
477 Err(err) if err.kind() == io::ErrorKind::AlreadyExists => (),
478 Err(err) => return Err(err),
479 }
480
481 // Add an empty .gitignore to the build bucket, to ensure that the cache's own .gitignore
482 // doesn't interfere with source distribution builds. Build backends (like hatchling) will
483 // traverse upwards to look for .gitignore files.
484 fs_err::create_dir_all(root.join(CacheBucket::SourceDistributions.to_str()))?;
485 match fs_err::OpenOptions::new()
486 .write(true)
487 .create_new(true)
488 .open(
489 root.join(CacheBucket::SourceDistributions.to_str())
490 .join(".gitignore"),
491 ) {
492 Ok(_) => {}
493 Err(err) if err.kind() == io::ErrorKind::AlreadyExists => (),
494 Err(err) => return Err(err),
495 }
496
497 // Add a phony .git, if it doesn't exist, to ensure that the cache isn't considered to be
498 // part of a Git repository. (Some packages will include Git metadata (like a hash) in the
499 // built version if they're in a Git repository, but the cache should be viewed as an
500 // isolated store.).
501 // We have to put this below the gitignore. Otherwise, if the build backend uses the rust
502 // ignore crate it will walk up to the top level .gitignore and ignore its python source
503 // files.
504 let phony_git = root
505 .join(CacheBucket::SourceDistributions.to_str())
506 .join(".git");
507 match fs_err::OpenOptions::new()
508 .create(true)
509 .write(true)
510 .open(&phony_git)
511 {
512 Ok(_) => {}
513 // Handle read-only caches including sandboxed environments.
514 Err(err) if err.kind() == io::ErrorKind::ReadOnlyFilesystem => {
515 if !phony_git.exists() {
516 return Err(err);
517 }
518 }
519 Err(err) => return Err(err),
520 }
521
522 Ok(())
523 }
524
525 /// Initialize the [`Cache`].
526 pub async fn init(self) -> Result<Self, Error> {
527 let root = &self.root;
528
529 Self::create_base_files(root).map_err(|err| Error::Init(root.clone(), err))?;
530
531 // Block cache removal operations from interfering.
532 let lock_file = match LockedFile::acquire(
533 root.join(".lock"),
534 LockedFileMode::Shared,
535 root.simplified_display(),
536 )
537 .await
538 {
539 Ok(lock_file) => Some(Arc::new(lock_file)),
540 Err(err)
541 if err
542 .as_io_error()
543 .is_some_and(|err| err.kind() == io::ErrorKind::Unsupported) =>
544 {
545 warn!(
546 "Shared locking is not supported by the current platform or filesystem, \
547 reduced parallel process safety with `uv cache clean` and `uv cache prune`."
548 );
549 None
550 }
551 Err(err) => return Err(err.into()),
552 };
553
554 Ok(Self {
555 root: std::path::absolute(root).map_err(Error::Absolute)?,
556 lock_file,
557 ..self
558 })
559 }
560
561 /// Initialize the [`Cache`], assuming that there are no other uv processes running.
562 pub fn init_no_wait(self) -> Result<Option<Self>, Error> {
563 let root = &self.root;
564
565 Self::create_base_files(root).map_err(|err| Error::Init(root.clone(), err))?;
566
567 // Block cache removal operations from interfering.
568 let Some(lock_file) = LockedFile::acquire_no_wait(
569 root.join(".lock"),
570 LockedFileMode::Shared,
571 root.simplified_display(),
572 ) else {
573 return Ok(None);
574 };
575 Ok(Some(Self {
576 root: std::path::absolute(root).map_err(Error::Absolute)?,
577 lock_file: Some(Arc::new(lock_file)),
578 ..self
579 }))
580 }
581
582 /// Clear the cache, removing all entries.
583 pub fn clear(self, reporter: Box<dyn CleanReporter>) -> Result<Removal, io::Error> {
584 // Remove everything but `.lock`, Windows does not allow removal of a locked file
585 let mut removal = Remover::new(reporter)
586 .with_removal_accounting(self.removal_accounting)
587 .rm_rf(&self.root, true)?;
588 let Self {
589 root, lock_file, ..
590 } = self;
591
592 // Remove the `.lock` file, unlocking it first
593 if let Some(lock) = lock_file {
594 drop(lock);
595 fs_err::remove_file(root.join(".lock"))?;
596 }
597 removal.num_files += 1;
598
599 // Remove the root directory
600 match fs_err::remove_dir(root) {
601 Ok(()) => {
602 removal.num_dirs += 1;
603 }
604 // On Windows, when `--force` is used, the `.lock` file can exist and be unremovable,
605 // so we make this non-fatal
606 Err(err) if err.kind() == io::ErrorKind::DirectoryNotEmpty => {
607 trace!("Failed to remove root cache directory: not empty");
608 }
609 Err(err) => return Err(err),
610 }
611
612 Ok(removal)
613 }
614
615 /// Remove a package from the cache.
616 ///
617 /// Unreferenced file objects are removed separately by [`Cache::prune_archive_files`].
618 ///
619 /// Returns the number of entries removed from the cache.
620 pub fn remove(&self, name: &PackageName) -> io::Result<Removal> {
621 // Collect the set of referenced archives.
622 let references = self.find_archive_references()?;
623
624 // Remove any entries for the package from the cache.
625 let mut summary = self.removal();
626 for bucket in CacheBucket::iter() {
627 summary += bucket.remove(self, name)?;
628 }
629
630 if references.is_empty() {
631 return Ok(summary);
632 }
633
634 // Only remove targets in the archive bucket. Cache entries may contain unexpected links
635 // to paths outside the cache.
636 let archive_root = fs_err::canonicalize(&self.root)?.join(CacheBucket::Archive.to_str());
637
638 // Remove any archives that are no longer referenced.
639 for (target, references) in references {
640 if target.starts_with(&archive_root) && references.iter().all(|path| !path.exists()) {
641 debug!("Removing dangling cache entry: {}", target.display());
642 summary += self.remove_path(target)?;
643 }
644 }
645
646 Ok(summary)
647 }
648
649 /// Remove file objects with no hardlinks outside the files bucket.
650 ///
651 /// Archives refer to these objects via hardlinks, independently of the installation link mode.
652 /// Installed copies and reflinks remain valid when the cached file is removed, so they do not
653 /// need to keep the file object alive.
654 pub fn prune_archive_files(&self) -> Result<Removal, io::Error> {
655 let root = self.bucket(CacheBucket::Files);
656 if !root.exists() {
657 return Ok(self.removal());
658 }
659
660 let mut summary = self.removal();
661 let mut directories = Vec::new();
662 let mut entries = walkdir::WalkDir::new(&root).min_depth(1).into_iter();
663 while let Some(entry) = entries.next() {
664 let entry = entry?;
665 if entry.file_type().is_file() {
666 match uv_fs::hardlink_count(entry.path()) {
667 Ok(1) => summary += self.remove_path(entry.path())?,
668 Ok(_) => {}
669 Err(err) if err.kind() == io::ErrorKind::NotFound => {}
670 Err(err) => return Err(err),
671 }
672 } else if entry.file_type().is_dir() {
673 if let Some(files) = uv_fs::files_with_one_hardlink(entry.path())? {
674 entries.skip_current_dir();
675 for file in files {
676 summary += self.remove_path(file)?;
677 }
678 }
679 directories.push(entry.into_path());
680 }
681 }
682 // The walk visits parents first so the bulk path can skip their contents.
683 // Remove directories in reverse order so children are removed before parents.
684 for directory in directories.into_iter().rev() {
685 match fs_err::remove_dir(directory) {
686 Ok(()) => {
687 summary.num_dirs += 1;
688 }
689 Err(err)
690 if matches!(
691 err.kind(),
692 io::ErrorKind::DirectoryNotEmpty | io::ErrorKind::NotFound
693 ) => {}
694 Err(err) => return Err(err),
695 }
696 }
697
698 Ok(summary)
699 }
700
701 /// Prune dangling cache entries and cached environments.
702 pub fn prune(&self, ci: bool) -> Result<Removal, io::Error> {
703 let mut summary = self.removal();
704
705 // First, remove any top-level directories that are unused. These typically represent
706 // outdated cache buckets (e.g., `wheels-v0`, when latest is `wheels-v1`).
707 for entry in fs_err::read_dir(&self.root)? {
708 let entry = entry?;
709 let metadata = entry.metadata()?;
710
711 if entry.file_name() == "CACHEDIR.TAG"
712 || entry.file_name() == ".gitignore"
713 || entry.file_name() == ".git"
714 || entry.file_name() == ".lock"
715 {
716 continue;
717 }
718
719 if metadata.is_dir() {
720 // If the directory is not a cache bucket, remove it.
721 if CacheBucket::iter().all(|bucket| entry.file_name() != bucket.to_str()) {
722 let path = entry.path();
723 debug!("Removing dangling cache bucket: {}", path.display());
724 summary += self.remove_path(path)?;
725 }
726 } else {
727 // If the file is not a marker file, remove it.
728 let path = entry.path();
729 debug!("Removing dangling cache bucket: {}", path.display());
730 summary += self.remove_path(path)?;
731 }
732 }
733
734 // Second, remove all cached environments. Centralized project environments can be
735 // referenced by `.venv` links, but are recreated when next needed.
736 match fs_err::read_dir(self.bucket(CacheBucket::Environments)) {
737 Ok(entries) => {
738 for entry in entries {
739 let entry = entry?;
740 let path = entry.path();
741 debug!("Removing cached environment: {}", path.display());
742 summary += self.remove_path(path)?;
743 }
744 }
745 Err(err) if err.kind() == io::ErrorKind::NotFound => (),
746 Err(err) => return Err(err),
747 }
748
749 // Third, if enabled, remove all unzipped wheels, leaving only the wheel archives.
750 if ci {
751 // Remove the entire pre-built wheel cache, since every entry is an unzipped wheel.
752 match fs_err::read_dir(self.bucket(CacheBucket::Wheels)) {
753 Ok(entries) => {
754 for entry in entries {
755 let entry = entry?;
756 let path = entry.path();
757 if path.is_dir() {
758 debug!("Removing unzipped wheel entry: {}", path.display());
759 summary += self.remove_path(path)?;
760 }
761 }
762 }
763 Err(err) if err.kind() == io::ErrorKind::NotFound => (),
764 Err(err) => return Err(err),
765 }
766
767 let source_distributions = self.bucket(CacheBucket::SourceDistributions);
768 if source_distributions.try_exists()? {
769 for entry in walkdir::WalkDir::new(source_distributions) {
770 let entry = entry?;
771
772 // If the directory contains a `metadata.msgpack`, then it's a built wheel revision.
773 if !entry.file_type().is_dir() {
774 continue;
775 }
776
777 if !entry.path().join("metadata.msgpack").exists() {
778 continue;
779 }
780
781 // Remove everything except the built wheel archive and the metadata.
782 for entry in fs_err::read_dir(entry.path())? {
783 let entry = entry?;
784 let path = entry.path();
785
786 // Retain the resolved metadata (`metadata.msgpack`).
787 if path
788 .file_name()
789 .is_some_and(|file_name| file_name == "metadata.msgpack")
790 {
791 continue;
792 }
793
794 // Retain any built wheel archives.
795 if path
796 .extension()
797 .is_some_and(|ext| ext.eq_ignore_ascii_case("whl"))
798 {
799 continue;
800 }
801
802 debug!("Removing unzipped built wheel entry: {}", path.display());
803 summary += self.remove_path(path)?;
804 }
805 }
806 }
807 }
808
809 // Fourth, remove any unused archives (by searching for archives that are not symlinked).
810 let references = self.find_archive_references()?;
811
812 match fs_err::read_dir(self.bucket(CacheBucket::Archive)) {
813 Ok(entries) => {
814 for entry in entries {
815 let entry = entry?;
816 let path = entry.path();
817 let target = fs_err::canonicalize(&path)?;
818 if !references.contains_key(&target) {
819 debug!("Removing dangling cache archive: {}", path.display());
820 summary += self.remove_path(path)?;
821 }
822 }
823 }
824 Err(err) if err.kind() == io::ErrorKind::NotFound => (),
825 Err(err) => return Err(err),
826 }
827
828 summary += self.prune_archive_files()?;
829
830 Ok(summary)
831 }
832
833 /// Remove a cache path using the cache's configured storage accounting.
834 pub fn remove_path(&self, path: impl AsRef<Path>) -> io::Result<Removal> {
835 Remover::default()
836 .with_removal_accounting(self.removal_accounting)
837 .rm_rf(path, false)
838 }
839
840 /// Find all references to entries in the archive bucket.
841 ///
842 /// Archive entries are often referenced by symlinks in other cache buckets. This method
843 /// searches for all such references.
844 ///
845 /// Returns a map from archive path to paths that reference it.
846 fn find_archive_references(&self) -> Result<FxHashMap<PathBuf, Vec<PathBuf>>, io::Error> {
847 let mut references = FxHashMap::<PathBuf, Vec<PathBuf>>::default();
848 for bucket in [CacheBucket::SourceDistributions, CacheBucket::Wheels] {
849 let bucket_path = self.bucket(bucket);
850 if bucket_path.is_dir() {
851 let walker = walkdir::WalkDir::new(&bucket_path).into_iter();
852 for entry in walker.filter_entry(|entry| {
853 !(
854 // As an optimization, ignore any `.lock`, `.whl`, `.msgpack`, `.rev`, or
855 // `.http` files, along with the `src` directory, which represents the
856 // unpacked source distribution.
857 entry.file_name() == "src"
858 || entry.file_name() == ".lock"
859 || entry.file_name() == ".gitignore"
860 || entry.path().extension().is_some_and(|ext| {
861 ext.eq_ignore_ascii_case("lock")
862 || ext.eq_ignore_ascii_case("whl")
863 || ext.eq_ignore_ascii_case("http")
864 || ext.eq_ignore_ascii_case("rev")
865 || ext.eq_ignore_ascii_case("msgpack")
866 })
867 )
868 }) {
869 let entry = entry?;
870
871 // On Unix, archive references use symlinks.
872 if cfg!(unix) {
873 if !entry.file_type().is_symlink() {
874 continue;
875 }
876 }
877
878 // On Windows, archive references are files containing structured data.
879 if cfg!(windows) {
880 if !entry.file_type().is_file() {
881 continue;
882 }
883 }
884
885 if let Ok(target) = self.resolve_link(entry.path()) {
886 references
887 .entry(target)
888 .or_default()
889 .push(entry.path().to_path_buf());
890 }
891 }
892 }
893 }
894 Ok(references)
895 }
896
897 /// Create a link to a directory in the archive bucket.
898 ///
899 /// On Windows, we write structured data ([`Link`]) to a file containing the archive ID and
900 /// version. On Unix, we create a symlink to the target directory.
901 #[cfg(windows)]
902 #[expect(clippy::unused_self)]
903 fn create_link(&self, id: &ArchiveId, dst: impl AsRef<Path>) -> io::Result<()> {
904 // Serialize the link.
905 let link = Link::new(id.clone());
906 let contents = link.to_string();
907
908 // First, attempt to create a file at the location, but fail if it already exists.
909 match fs_err::OpenOptions::new()
910 .write(true)
911 .create_new(true)
912 .open(dst.as_ref())
913 {
914 Ok(mut file) => {
915 // Write the target path to the file.
916 file.write_all(contents.as_bytes())?;
917 Ok(())
918 }
919 Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {
920 uv_fs::write_atomic_sync(dst, contents.as_bytes())
921 }
922 Err(err) => Err(err),
923 }
924 }
925
926 /// Resolve an archive link, returning the fully-resolved path.
927 ///
928 /// Returns an error if the link target does not exist.
929 #[cfg(windows)]
930 pub fn resolve_link(&self, path: impl AsRef<Path>) -> io::Result<PathBuf> {
931 // Deserialize the link.
932 let contents = fs_err::read_to_string(path.as_ref())?;
933 let link = Link::from_str(&contents)?;
934
935 // Ignore stale links.
936 if link.version != ARCHIVE_VERSION {
937 return Err(io::Error::new(
938 io::ErrorKind::NotFound,
939 "The link target does not exist.",
940 ));
941 }
942
943 // Reconstruct the path.
944 let path = self.archive(&link.id);
945 path.canonicalize()
946 }
947
948 /// Create a link to a directory in the archive bucket.
949 ///
950 /// On Windows, we write structured data ([`Link`]) to a file containing the archive ID and
951 /// version. On Unix, we create a symlink to the target directory.
952 #[cfg(unix)]
953 fn create_link(&self, id: &ArchiveId, dst: impl AsRef<Path>) -> io::Result<()> {
954 let dst = dst.as_ref();
955 let dst_parent = dst.parent().expect("Cache entry to have parent");
956 // Construct the relative link target.
957 let src = uv_fs::relative_to(self.archive(id), dst_parent)?;
958
959 uv_fs::replace_symlink(&src, dst)
960 }
961
962 /// Resolve an archive link, returning the fully-resolved path.
963 ///
964 /// Returns an error if the link target does not exist.
965 #[cfg(unix)]
966 pub fn resolve_link(&self, path: impl AsRef<Path>) -> io::Result<PathBuf> {
967 path.as_ref().canonicalize()
968 }
969}
970
971/// An archive (unzipped wheel) that exists in the local cache.
972#[derive(Debug, Clone)]
973#[allow(unused)]
974struct Link {
975 /// The unique ID of the entry in the archive bucket.
976 id: ArchiveId,
977 /// The version of the archive bucket.
978 version: u8,
979}
980
981#[allow(unused)]
982impl Link {
983 /// Create a new [`Archive`] with the given ID and hashes.
984 fn new(id: ArchiveId) -> Self {
985 Self {
986 id,
987 version: ARCHIVE_VERSION,
988 }
989 }
990}
991
992impl Display for Link {
993 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
994 write!(f, "archive-v{}/{}", self.version, self.id)
995 }
996}
997
998impl FromStr for Link {
999 type Err = io::Error;
1000
1001 fn from_str(s: &str) -> Result<Self, Self::Err> {
1002 let mut parts = s.splitn(2, '/');
1003 let version = parts
1004 .next()
1005 .filter(|s| !s.is_empty())
1006 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing version"))?;
1007 let id = parts
1008 .next()
1009 .filter(|s| !s.is_empty())
1010 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing ID"))?;
1011
1012 // Parse the archive version from `archive-v{version}/{id}`.
1013 let version = version
1014 .strip_prefix("archive-v")
1015 .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing version prefix"))?;
1016 let version = u8::from_str(version).map_err(|err| {
1017 io::Error::new(
1018 io::ErrorKind::InvalidData,
1019 format!("failed to parse version: {err}"),
1020 )
1021 })?;
1022
1023 // Parse the ID from `archive-v{version}/{id}`.
1024 let id = ArchiveId::from_str(id).map_err(|err| {
1025 io::Error::new(
1026 io::ErrorKind::InvalidData,
1027 format!("failed to parse ID: {err}"),
1028 )
1029 })?;
1030
1031 Ok(Self { id, version })
1032 }
1033}
1034
1035pub trait CleanReporter: Send + Sync {
1036 /// Called after one file or directory is removed.
1037 fn on_clean(&self);
1038
1039 /// Called after all files and directories are removed.
1040 fn on_complete(&self);
1041}
1042
1043/// The different kinds of data in the cache are stored in different bucket, which in our case
1044/// are subdirectories of the cache root.
1045#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
1046pub enum CacheBucket {
1047 /// Wheels (excluding built wheels), alongside their metadata and cache policy.
1048 ///
1049 /// There are three kinds from cache entries: Wheel metadata and policy as `MsgPack` files, the
1050 /// wheels themselves, and the unzipped wheel archives. If a wheel file is over an in-memory
1051 /// size threshold, we first download the zip file into the cache, then unzip it into a
1052 /// directory with the same name (exclusive of the `.whl` extension).
1053 ///
1054 /// Cache structure:
1055 /// * `wheel-metadata-v0/pypi/foo/{foo-1.0.0-py3-none-any.msgpack, foo-1.0.0-py3-none-any.whl}`
1056 /// * `wheel-metadata-v0/<digest(index-url)>/foo/{foo-1.0.0-py3-none-any.msgpack, foo-1.0.0-py3-none-any.whl}`
1057 /// * `wheel-metadata-v0/url/<digest(url)>/foo/{foo-1.0.0-py3-none-any.msgpack, foo-1.0.0-py3-none-any.whl}`
1058 ///
1059 /// See `uv_client::RegistryClient::wheel_metadata` for information on how wheel metadata
1060 /// is fetched.
1061 ///
1062 /// # Example
1063 ///
1064 /// Consider the following `requirements.in`:
1065 /// ```text
1066 /// # pypi wheel
1067 /// pandas
1068 /// # url wheel
1069 /// flask @ https://files.pythonhosted.org/packages/36/42/015c23096649b908c809c69388a805a571a3bea44362fe87e33fc3afa01f/flask-3.0.0-py3-none-any.whl
1070 /// ```
1071 ///
1072 /// When we run `pip compile`, it will only fetch and cache the metadata (and cache policy), it
1073 /// doesn't need the actual wheels yet:
1074 /// ```text
1075 /// wheel-v0
1076 /// ├── pypi
1077 /// │ ...
1078 /// │ ├── pandas
1079 /// │ │ └── pandas-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.msgpack
1080 /// │ ...
1081 /// └── url
1082 /// └── 4b8be67c801a7ecb
1083 /// └── flask
1084 /// └── flask-3.0.0-py3-none-any.msgpack
1085 /// ```
1086 ///
1087 /// We get the following `requirement.txt` from `pip compile`:
1088 ///
1089 /// ```text
1090 /// [...]
1091 /// flask @ https://files.pythonhosted.org/packages/36/42/015c23096649b908c809c69388a805a571a3bea44362fe87e33fc3afa01f/flask-3.0.0-py3-none-any.whl
1092 /// [...]
1093 /// pandas==2.1.3
1094 /// [...]
1095 /// ```
1096 ///
1097 /// If we run `pip sync` on `requirements.txt` on a different machine, it also fetches the
1098 /// wheels:
1099 ///
1100 /// TODO(konstin): This is still wrong, we need to store the cache policy too!
1101 /// ```text
1102 /// wheel-v0
1103 /// ├── pypi
1104 /// │ ...
1105 /// │ ├── pandas
1106 /// │ │ ├── pandas-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
1107 /// │ │ ├── pandas-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64
1108 /// │ ...
1109 /// └── url
1110 /// └── 4b8be67c801a7ecb
1111 /// └── flask
1112 /// └── flask-3.0.0-py3-none-any.whl
1113 /// ├── flask
1114 /// │ └── ...
1115 /// └── flask-3.0.0.dist-info
1116 /// └── ...
1117 /// ```
1118 ///
1119 /// If we run first `pip compile` and then `pip sync` on the same machine, we get both:
1120 ///
1121 /// ```text
1122 /// wheels-v0
1123 /// ├── pypi
1124 /// │ ├── ...
1125 /// │ ├── pandas
1126 /// │ │ ├── pandas-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.msgpack
1127 /// │ │ ├── pandas-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
1128 /// │ │ └── pandas-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64
1129 /// │ │ ├── pandas
1130 /// │ │ │ ├── ...
1131 /// │ │ ├── pandas-2.1.3.dist-info
1132 /// │ │ │ ├── ...
1133 /// │ │ └── pandas.libs
1134 /// │ ├── ...
1135 /// └── url
1136 /// └── 4b8be67c801a7ecb
1137 /// └── flask
1138 /// ├── flask-3.0.0-py3-none-any.msgpack
1139 /// ├── flask-3.0.0-py3-none-any.msgpack
1140 /// └── flask-3.0.0-py3-none-any
1141 /// ├── flask
1142 /// │ └── ...
1143 /// └── flask-3.0.0.dist-info
1144 /// └── ...
1145 Wheels,
1146 /// Source distributions, wheels built from source distributions, their extracted metadata, and the
1147 /// cache policy of the source distribution.
1148 ///
1149 /// The structure is similar of that of the `Wheel` bucket, except we have an additional layer
1150 /// for the source distribution filename and the metadata is at the source distribution-level,
1151 /// not at the wheel level.
1152 ///
1153 /// TODO(konstin): The cache policy should be on the source distribution level, the metadata we
1154 /// can put next to the wheels as in the `Wheels` bucket.
1155 ///
1156 /// The unzipped source distribution is stored in a directory matching the source distribution
1157 /// archive name.
1158 ///
1159 /// Source distributions are built into zipped wheel files (as PEP 517 specifies) and unzipped
1160 /// lazily before installing. So when resolving, we only build the wheel and store the archive
1161 /// file in the cache, when installing, we unpack it under the same name (exclusive of the
1162 /// `.whl` extension). You may find a mix of wheel archive zip files and unzipped wheel
1163 /// directories in the cache.
1164 ///
1165 /// Cache structure:
1166 /// * `built-wheels-v0/pypi/foo/34a17436ed1e9669/{manifest.msgpack, metadata.msgpack, foo-1.0.0.zip, foo-1.0.0-py3-none-any.whl, ...other wheels}`
1167 /// * `built-wheels-v0/<digest(index-url)>/foo/foo-1.0.0.zip/{manifest.msgpack, metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}`
1168 /// * `built-wheels-v0/url/<digest(url)>/foo/foo-1.0.0.zip/{manifest.msgpack, metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}`
1169 /// * `built-wheels-v0/git/<digest(url)>/<git sha>/foo/foo-1.0.0.zip/{metadata.msgpack, foo-1.0.0-py3-none-any.whl, ...other wheels}`
1170 ///
1171 /// But the url filename does not need to be a valid source dist filename
1172 /// (<https://github.com/search?q=path%3A**%2Frequirements.txt+master.zip&type=code>),
1173 /// so it could also be the following and we have to take any string as filename:
1174 /// * `built-wheels-v0/url/<sha256(url)>/master.zip/metadata.msgpack`
1175 ///
1176 /// # Example
1177 ///
1178 /// The following requirements:
1179 /// ```text
1180 /// # git source dist
1181 /// pydantic-extra-types @ git+https://github.com/pydantic/pydantic-extra-types.git
1182 /// # pypi source dist
1183 /// django_allauth==0.51.0
1184 /// # url source dist
1185 /// werkzeug @ https://files.pythonhosted.org/packages/0d/cc/ff1904eb5eb4b455e442834dabf9427331ac0fa02853bf83db817a7dd53d/werkzeug-3.0.1.tar.gz
1186 /// ```
1187 ///
1188 /// ...may be cached as:
1189 /// ```text
1190 /// built-wheels-v4/
1191 /// ├── git
1192 /// │ └── 2122faf3e081fb7a
1193 /// │ └── 7a2d650a4a7b4d04
1194 /// │ ├── metadata.msgpack
1195 /// │ └── pydantic_extra_types-2.9.0-py3-none-any.whl
1196 /// ├── pypi
1197 /// │ └── django-allauth
1198 /// │ └── 0.51.0
1199 /// │ ├── 0gH-_fwv8tdJ7JwwjJsUc
1200 /// │ │ ├── django-allauth-0.51.0.tar.gz
1201 /// │ │ │ └── [UNZIPPED CONTENTS]
1202 /// │ │ ├── django_allauth-0.51.0-py3-none-any.whl
1203 /// │ │ └── metadata.msgpack
1204 /// │ └── revision.http
1205 /// └── url
1206 /// └── 6781bd6440ae72c2
1207 /// ├── APYY01rbIfpAo_ij9sCY6
1208 /// │ ├── metadata.msgpack
1209 /// │ ├── werkzeug-3.0.1-py3-none-any.whl
1210 /// │ └── werkzeug-3.0.1.tar.gz
1211 /// │ └── [UNZIPPED CONTENTS]
1212 /// └── revision.http
1213 /// ```
1214 ///
1215 /// Structurally, the `manifest.msgpack` is empty, and only contains the caching information
1216 /// needed to invalidate the cache. The `metadata.msgpack` contains the metadata of the source
1217 /// distribution.
1218 SourceDistributions,
1219 /// Flat index responses, a format very similar to the simple metadata API.
1220 ///
1221 /// Cache structure:
1222 /// * `flat-index-v0/index/<digest(flat_index_url)>.msgpack`
1223 ///
1224 /// The response is stored as `Vec<File>`.
1225 FlatIndex,
1226 /// Git repositories.
1227 Git,
1228 /// Information about an interpreter at a path.
1229 ///
1230 /// To avoid caching pyenv shims, bash scripts which may redirect to a new python version
1231 /// without the shim itself changing, we only cache when the path equals `sys.executable`, i.e.
1232 /// the path we're running is the python executable itself and not a shim.
1233 ///
1234 /// Cache structure: `interpreter-v0/<digest(path)>.msgpack`
1235 ///
1236 /// # Example
1237 ///
1238 /// The contents of each of the `MsgPack` files has a timestamp field in unix time, the [PEP 508]
1239 /// markers and some information from the `sys`/`sysconfig` modules.
1240 ///
1241 /// ```json
1242 /// {
1243 /// "timestamp": 1698047994491,
1244 /// "data": {
1245 /// "markers": {
1246 /// "implementation_name": "cpython",
1247 /// "implementation_version": "3.12.0",
1248 /// "os_name": "posix",
1249 /// "platform_machine": "x86_64",
1250 /// "platform_python_implementation": "CPython",
1251 /// "platform_release": "6.5.0-13-generic",
1252 /// "platform_system": "Linux",
1253 /// "platform_version": "#13-Ubuntu SMP PREEMPT_DYNAMIC Fri Nov 3 12:16:05 UTC 2023",
1254 /// "python_full_version": "3.12.0",
1255 /// "python_version": "3.12",
1256 /// "sys_platform": "linux"
1257 /// },
1258 /// "base_exec_prefix": "/home/ferris/.pyenv/versions/3.12.0",
1259 /// "base_prefix": "/home/ferris/.pyenv/versions/3.12.0",
1260 /// "sys_executable": "/home/ferris/projects/uv/.venv/bin/python"
1261 /// }
1262 /// }
1263 /// ```
1264 ///
1265 /// [PEP 508]: https://peps.python.org/pep-0508/#environment-markers
1266 Interpreter,
1267 /// Index responses through the simple metadata API.
1268 ///
1269 /// Cache structure:
1270 /// * `simple-v0/pypi/<package_name>.rkyv`
1271 /// * `simple-v0/<digest(index_url)>/<package_name>.rkyv`
1272 ///
1273 /// The response is parsed into `uv_client::SimpleDetailMetadata` before storage.
1274 Simple,
1275 /// A cache of unzipped wheels, stored as directories. This is used internally within the cache.
1276 /// When other buckets need to store directories, they should persist them to
1277 /// [`CacheBucket::Archive`], and then symlink them into the appropriate bucket. This ensures
1278 /// that cache entries can be atomically replaced and removed, as storing directories in the
1279 /// other buckets directly would make atomic operations impossible.
1280 Archive,
1281 /// Content-addressed files that are hardlinked into cached archives.
1282 Files,
1283 /// Ephemeral virtual environments used to execute PEP 517 builds and other operations.
1284 Builds,
1285 /// Reusable virtual environments for Python tools and projects.
1286 Environments,
1287 /// Cached Python downloads
1288 Python,
1289 /// Downloaded tool binaries (e.g., Ruff).
1290 Binaries,
1291 /// Cached vulnerability data from [OSV](https://osv.dev/).
1292 ///
1293 /// Cache structure:
1294 /// * `osv-v0/vulnerability/<vuln_id>.msgpack` — cached full vulnerability records
1295 Osv,
1296}
1297
1298impl CacheBucket {
1299 fn to_str(self) -> &'static str {
1300 match self {
1301 // Note that when bumping this, you'll also need to bump it
1302 // in `crates/uv/tests/build/cache_prune.rs`.
1303 // TODO(ww): Remove `uv_pypi_types::HashDigestWire` on the next cache bump
1304 // or breaking release.
1305 Self::SourceDistributions => "sdists-v9",
1306 // Note that when bumping this, you'll also need to bump it
1307 // in `crates/uv/tests/lock/lock.rs`.
1308 Self::FlatIndex => "flat-index-v5",
1309 Self::Git => "git-v1",
1310 Self::Interpreter => "interpreter-v4",
1311 // Note that when bumping this, you'll also need to bump it
1312 // in `crates/uv/tests/build/cache_clean.rs`.
1313 Self::Simple => "simple-v25",
1314 // Note that when bumping this, you'll also need to bump it
1315 // in `crates/uv/tests/build/cache_prune.rs`.
1316 Self::Wheels => "wheels-v6",
1317 // Note that when bumping this, you'll also need to bump
1318 // `ARCHIVE_VERSION` in `crates/uv-cache/src/lib.rs`.
1319 Self::Archive => "archive-v0",
1320 Self::Files => "files-v0",
1321 Self::Builds => "builds-v0",
1322 Self::Environments => "environments-v2",
1323 Self::Python => "python-v0",
1324 Self::Binaries => "binaries-v0",
1325 Self::Osv => "osv-v0",
1326 }
1327 }
1328
1329 /// Remove a package from the cache bucket.
1330 ///
1331 /// Returns the number of entries removed from the cache.
1332 fn remove(self, cache: &Cache, name: &PackageName) -> Result<Removal, io::Error> {
1333 /// Returns `true` if the [`Path`] represents a built wheel for the given package.
1334 fn is_match(path: &Path, name: &PackageName) -> bool {
1335 let Ok(metadata) = fs_err::read(path.join("metadata.msgpack")) else {
1336 return false;
1337 };
1338 let Ok(metadata) = rmp_serde::from_slice::<ResolutionMetadata>(&metadata) else {
1339 return false;
1340 };
1341 metadata.name == *name
1342 }
1343
1344 let mut summary = cache.removal();
1345 match self {
1346 Self::Wheels => {
1347 // For `pypi` wheels, we expect a directory per package (indexed by name).
1348 let root = cache.bucket(self).join(WheelCacheKind::Pypi);
1349 summary += cache.remove_path(root.join(name.to_string()))?;
1350
1351 // For alternate indices, we expect a directory for every index (under an `index`
1352 // subdirectory), followed by a directory per package (indexed by name).
1353 let root = cache.bucket(self).join(WheelCacheKind::Index);
1354 for directory in directories(root)? {
1355 summary += cache.remove_path(directory.join(name.to_string()))?;
1356 }
1357
1358 // For direct URLs, we expect a directory for every URL, followed by a
1359 // directory per package (indexed by name).
1360 let root = cache.bucket(self).join(WheelCacheKind::Url);
1361 for directory in directories(root)? {
1362 summary += cache.remove_path(directory.join(name.to_string()))?;
1363 }
1364 }
1365 Self::SourceDistributions => {
1366 // For `pypi` wheels, we expect a directory per package (indexed by name).
1367 let root = cache.bucket(self).join(WheelCacheKind::Pypi);
1368 summary += cache.remove_path(root.join(name.to_string()))?;
1369
1370 // For alternate indices, we expect a directory for every index (under an `index`
1371 // subdirectory), followed by a directory per package (indexed by name).
1372 let root = cache.bucket(self).join(WheelCacheKind::Index);
1373 for directory in directories(root)? {
1374 summary += cache.remove_path(directory.join(name.to_string()))?;
1375 }
1376
1377 // For direct URLs, we expect a directory for every URL, followed by a
1378 // directory per version. To determine whether the URL is relevant, we need to
1379 // search for a wheel matching the package name.
1380 let root = cache.bucket(self).join(WheelCacheKind::Url);
1381 for url in directories(root)? {
1382 if directories(&url)?.any(|version| is_match(&version, name)) {
1383 summary += cache.remove_path(url)?;
1384 }
1385 }
1386
1387 // For local dependencies, we expect a directory for every path, followed by a
1388 // directory per version. To determine whether the path is relevant, we need to
1389 // search for a wheel matching the package name.
1390 let root = cache.bucket(self).join(WheelCacheKind::Path);
1391 for path in directories(root)? {
1392 if directories(&path)?.any(|version| is_match(&version, name)) {
1393 summary += cache.remove_path(path)?;
1394 }
1395 }
1396
1397 // For Git dependencies, we expect a directory for every repository, followed by a
1398 // directory for every SHA. To determine whether the SHA is relevant, we need to
1399 // search for a wheel matching the package name.
1400 let root = cache.bucket(self).join(WheelCacheKind::Git);
1401 for repository in directories(root)? {
1402 for sha in directories(repository)? {
1403 if is_match(&sha, name) {
1404 summary += cache.remove_path(sha)?;
1405 }
1406 }
1407 }
1408 }
1409 Self::Simple => {
1410 // For `pypi` wheels, we expect a rkyv file per package, indexed by name.
1411 let root = cache.bucket(self).join(WheelCacheKind::Pypi);
1412 summary += cache.remove_path(root.join(format!("{name}.rkyv")))?;
1413
1414 // For alternate indices, we expect a directory for every index (under an `index`
1415 // subdirectory), followed by a directory per package (indexed by name).
1416 let root = cache.bucket(self).join(WheelCacheKind::Index);
1417 for directory in directories(root)? {
1418 summary += cache.remove_path(directory.join(format!("{name}.rkyv")))?;
1419 }
1420 }
1421 Self::FlatIndex => {
1422 // We can't know if the flat index includes a package, so we just remove the entire
1423 // cache entry.
1424 let root = cache.bucket(self);
1425 summary += cache.remove_path(root)?;
1426 }
1427 Self::Git
1428 | Self::Interpreter
1429 | Self::Archive
1430 | Self::Files
1431 | Self::Builds
1432 | Self::Environments
1433 | Self::Python
1434 | Self::Binaries
1435 | Self::Osv => {
1436 // Nothing to do.
1437 }
1438 }
1439 Ok(summary)
1440 }
1441
1442 /// Return an iterator over all cache buckets.
1443 fn iter() -> impl Iterator<Item = Self> {
1444 [
1445 Self::Wheels,
1446 Self::SourceDistributions,
1447 Self::FlatIndex,
1448 Self::Git,
1449 Self::Interpreter,
1450 Self::Simple,
1451 Self::Archive,
1452 Self::Files,
1453 Self::Builds,
1454 Self::Environments,
1455 Self::Python,
1456 Self::Binaries,
1457 Self::Osv,
1458 ]
1459 .iter()
1460 .copied()
1461 }
1462}
1463
1464impl Display for CacheBucket {
1465 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
1466 f.write_str(self.to_str())
1467 }
1468}
1469
1470#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1471pub enum Freshness {
1472 /// The cache entry is fresh according to the [`Refresh`] policy.
1473 Fresh,
1474 /// The cache entry is stale according to the [`Refresh`] policy.
1475 Stale,
1476 /// The cache entry does not exist.
1477 Missing,
1478}
1479
1480impl Freshness {
1481 pub const fn is_fresh(self) -> bool {
1482 matches!(self, Self::Fresh)
1483 }
1484}
1485
1486/// A refresh policy for cache entries.
1487#[derive(Debug, Clone)]
1488pub enum Refresh {
1489 /// Don't refresh any entries.
1490 None(Timestamp),
1491 /// Refresh entries linked to the given packages, if created before the given timestamp.
1492 Packages(Vec<PackageName>, Vec<Box<Path>>, Timestamp),
1493 /// Refresh all entries created before the given timestamp.
1494 All(Timestamp),
1495}
1496
1497impl Refresh {
1498 /// Determine the refresh strategy to use based on the command-line arguments.
1499 pub fn from_args(refresh: Option<bool>, refresh_package: Vec<PackageName>) -> Self {
1500 let timestamp = Timestamp::now();
1501 match refresh {
1502 Some(true) => Self::All(timestamp),
1503 Some(false) => Self::None(timestamp),
1504 None => {
1505 if refresh_package.is_empty() {
1506 Self::None(timestamp)
1507 } else {
1508 Self::Packages(refresh_package, vec![], timestamp)
1509 }
1510 }
1511 }
1512 }
1513
1514 /// Combine two [`Refresh`] policies, taking the "max" of the two policies.
1515 #[must_use]
1516 pub fn combine(self, other: Self) -> Self {
1517 match (self, other) {
1518 // If the policy is `None`, return the existing refresh policy.
1519 // Take the `max` of the two timestamps.
1520 (Self::None(t1), Self::None(t2)) => Self::None(t1.max(t2)),
1521 (Self::None(t1), Self::All(t2)) => Self::All(t1.max(t2)),
1522 (Self::None(t1), Self::Packages(packages, paths, t2)) => {
1523 Self::Packages(packages, paths, t1.max(t2))
1524 }
1525
1526 // If the policy is `All`, refresh all packages.
1527 (Self::All(t1), Self::None(t2) | Self::All(t2) | Self::Packages(.., t2)) => {
1528 Self::All(t1.max(t2))
1529 }
1530
1531 // If the policy is `Packages`, take the "max" of the two policies.
1532 (Self::Packages(packages, paths, t1), Self::None(t2)) => {
1533 Self::Packages(packages, paths, t1.max(t2))
1534 }
1535 (Self::Packages(.., t1), Self::All(t2)) => Self::All(t1.max(t2)),
1536 (Self::Packages(packages1, paths1, t1), Self::Packages(packages2, paths2, t2)) => {
1537 Self::Packages(
1538 packages1.into_iter().chain(packages2).collect(),
1539 paths1.into_iter().chain(paths2).collect(),
1540 t1.max(t2),
1541 )
1542 }
1543 }
1544 }
1545}
1546
1547#[cfg(test)]
1548mod tests {
1549 use std::str::FromStr;
1550
1551 use crate::ArchiveId;
1552
1553 use super::{Cache, Link};
1554
1555 #[test]
1556 fn test_link_round_trip() {
1557 let id = ArchiveId::new();
1558 let link = Link::new(id);
1559 let s = link.to_string();
1560 let parsed = Link::from_str(&s).unwrap();
1561 assert_eq!(link.id, parsed.id);
1562 assert_eq!(link.version, parsed.version);
1563 }
1564
1565 #[test]
1566 fn test_replace_archive_link() {
1567 let cache = Cache::temp().unwrap();
1568 let link = cache.root().join("link");
1569 for id in [ArchiveId::new(), ArchiveId::new()] {
1570 let archive = cache.archive(&id);
1571 fs_err::create_dir_all(&archive).unwrap();
1572 cache.create_link(&id, &link).unwrap();
1573 assert_eq!(
1574 cache.resolve_link(&link).unwrap(),
1575 archive.canonicalize().unwrap()
1576 );
1577 }
1578 }
1579
1580 #[test]
1581 fn test_link_deserialize() {
1582 assert!(Link::from_str("archive-v0/foo").is_ok());
1583 assert!(Link::from_str("archive/foo").is_err());
1584 assert!(Link::from_str("v1/foo").is_err());
1585 assert!(Link::from_str("archive-v0/").is_err());
1586 }
1587
1588 #[test]
1589 #[cfg(unix)]
1590 fn prune_does_not_follow_environment_symlinks() {
1591 use super::{Cache, CacheBucket};
1592
1593 let cache_root = tempfile::tempdir().unwrap();
1594 let victim_root = tempfile::tempdir().unwrap();
1595 let environments = cache_root.path().join(CacheBucket::Environments.to_str());
1596 let victim_dir = victim_root.path().join("victim-dir");
1597
1598 fs_err::create_dir_all(&environments).unwrap();
1599 fs_err::create_dir_all(&victim_dir).unwrap();
1600 fs_err::write(victim_dir.join("payload.txt"), "payload").unwrap();
1601 fs_err::os::unix::fs::symlink(&victim_dir, environments.join("escape")).unwrap();
1602
1603 let summary = Cache::from_path(cache_root.path()).prune(false).unwrap();
1604
1605 assert_eq!(summary.num_files, 1);
1606 assert_eq!(summary.num_dirs, 0);
1607 assert!(victim_dir.is_dir());
1608 assert!(victim_dir.join("payload.txt").is_file());
1609 assert!(fs_err::symlink_metadata(environments.join("escape")).is_err());
1610 }
1611
1612 #[test]
1613 #[cfg(unix)]
1614 fn prune_ci_does_not_follow_wheel_symlinks() {
1615 use super::{Cache, CacheBucket};
1616
1617 let cache_root = tempfile::tempdir().unwrap();
1618 let victim_root = tempfile::tempdir().unwrap();
1619 let wheels = cache_root.path().join(CacheBucket::Wheels.to_str());
1620 let source_distributions = cache_root
1621 .path()
1622 .join(CacheBucket::SourceDistributions.to_str());
1623 let victim_dir = victim_root.path().join("victim-dir");
1624 let symlink = wheels.join("escape");
1625
1626 fs_err::create_dir_all(&wheels).unwrap();
1627 fs_err::create_dir_all(&source_distributions).unwrap();
1628 fs_err::create_dir_all(&victim_dir).unwrap();
1629 fs_err::write(victim_dir.join("payload.txt"), "payload").unwrap();
1630 fs_err::os::unix::fs::symlink(&victim_dir, &symlink).unwrap();
1631
1632 let summary = Cache::from_path(cache_root.path()).prune(true).unwrap();
1633
1634 assert_eq!(summary.num_files, 1);
1635 assert_eq!(summary.num_dirs, 0);
1636 assert!(victim_dir.is_dir());
1637 assert!(victim_dir.join("payload.txt").is_file());
1638 assert!(fs_err::symlink_metadata(symlink).is_err());
1639 }
1640
1641 #[test]
1642 #[cfg(unix)]
1643 fn prune_does_not_follow_archive_symlinks() {
1644 use super::{Cache, CacheBucket};
1645
1646 let cache_root = tempfile::tempdir().unwrap();
1647 let victim_root = tempfile::tempdir().unwrap();
1648 let archives = cache_root.path().join(CacheBucket::Archive.to_str());
1649 let victim_dir = victim_root.path().join("victim-dir");
1650 let symlink = archives.join("escape");
1651
1652 fs_err::create_dir_all(&archives).unwrap();
1653 fs_err::create_dir_all(&victim_dir).unwrap();
1654 fs_err::write(victim_dir.join("payload.txt"), "payload").unwrap();
1655 fs_err::os::unix::fs::symlink(&victim_dir, &symlink).unwrap();
1656
1657 let summary = Cache::from_path(cache_root.path()).prune(false).unwrap();
1658
1659 assert_eq!(summary.num_files, 1);
1660 assert_eq!(summary.num_dirs, 0);
1661 assert!(victim_dir.is_dir());
1662 assert!(victim_dir.join("payload.txt").is_file());
1663 assert!(fs_err::symlink_metadata(symlink).is_err());
1664 }
1665}