uv_metadata/
lib.rs

1//! Read metadata from wheels and source distributions.
2//!
3//! This module reads all fields exhaustively. The fields are defined in the [Core metadata
4//! specification](https://packaging.python.org/en/latest/specifications/core-metadata/).
5
6use std::io;
7use std::path::Path;
8
9use async_zip::base::read::seek::ZipFileReader;
10use futures::executor::block_on;
11use futures::io::{AllowStdIo, AsyncBufRead, AsyncSeek};
12use thiserror::Error;
13use tokio::io::AsyncReadExt;
14use tokio_util::compat::FuturesAsyncReadCompatExt;
15use uv_distribution_filename::{VariantLabel, WheelFilename};
16use uv_distribution_types::{VariantsJsonContent, VariantsJsonParseError};
17use uv_normalize::InvalidNameError;
18use uv_pypi_types::ResolutionMetadata;
19
20pub use dist_info_stem::DistInfoStem;
21
22mod dist_info_stem;
23
24/// The caller is responsible for attaching the path or url we failed to read.
25#[derive(Debug, Error)]
26pub enum Error {
27    #[error("Failed to read `dist-info` metadata from built wheel")]
28    DistInfo,
29    #[error("No .dist-info directory found")]
30    MissingDistInfo,
31    #[error("Multiple .dist-info directories found: {0}")]
32    MultipleDistInfo(String),
33    #[error(
34        "The .dist-info directory does not consist of the normalized package name and version: {0}"
35    )]
36    MissingDistInfoSegments(String),
37    #[error("The .dist-info directory `{0}` does not start with the normalized package name: {1}")]
38    MissingDistInfoPackageName(String, String),
39    #[error("The .dist-info directory name contains invalid characters")]
40    InvalidName(#[from] InvalidNameError),
41    #[error("The metadata at `{0}` is invalid")]
42    InvalidMetadata(String, Box<uv_pypi_types::MetadataError>),
43    #[error("Missing `variant.json` in the wheel's .dist-info directory")]
44    MissingVariantMetadata,
45    #[error(transparent)]
46    InvalidVariantMetadata(#[from] VariantsJsonParseError),
47    #[error("Bad CRC (got {computed:08x}, expected {expected:08x}) for file: {path}")]
48    BadCrc32 {
49        path: String,
50        computed: u32,
51        expected: u32,
52    },
53    #[error("Failed to read from zip file")]
54    AsyncZip(#[source] async_zip::error::ZipError),
55    #[error(
56        "Archive contains a file with an unsupported compression method; files must be compressed with 'stored', 'DEFLATE', or 'zstd'"
57    )]
58    UnsupportedCompression,
59    // No `#[from]` to enforce manual review of `io::Error` sources.
60    #[error(transparent)]
61    Io(io::Error),
62}
63
64impl From<async_zip::error::ZipError> for Error {
65    fn from(err: async_zip::error::ZipError) -> Self {
66        match err {
67            async_zip::error::ZipError::CompressionNotSupported(_) => Self::UnsupportedCompression,
68            o => Self::AsyncZip(o),
69        }
70    }
71}
72
73/// Find the `.dist-info` directory in a zipped wheel.
74///
75/// Returns the validated [`DistInfoStem`].
76///
77/// Reference implementation: <https://github.com/pypa/pip/blob/36823099a9cdd83261fdbc8c1d2a24fa2eea72ca/src/pip/_internal/utils/wheel.py#L38>
78pub fn find_archive_dist_info<'a, T: Copy>(
79    filename: &WheelFilename,
80    files: impl Iterator<Item = (T, &'a str)>,
81) -> Result<(T, DistInfoStem<'a>), Error> {
82    let metadatas: Vec<_> = files
83        .filter_map(|(payload, path)| {
84            let (dist_info_dir, file) = path.split_once('/')?;
85            if file != "METADATA" {
86                return None;
87            }
88            let dist_info_stem = dist_info_dir.strip_suffix(".dist-info")?;
89            Some((payload, dist_info_stem))
90        })
91        .collect();
92
93    // Like `pip`, assert that there is exactly one `.dist-info` directory.
94    let (payload, dist_info_stem) = match metadatas[..] {
95        [] => {
96            return Err(Error::MissingDistInfo);
97        }
98        [(payload, path)] => (payload, path),
99        _ => {
100            return Err(Error::MultipleDistInfo(
101                metadatas
102                    .into_iter()
103                    .map(|(_, dist_info_dir)| dist_info_dir.to_string())
104                    .collect::<Vec<_>>()
105                    .join(", "),
106            ));
107        }
108    };
109
110    Ok((payload, DistInfoStem::new(dist_info_stem, &filename.name)?))
111}
112
113/// Return the validated [`DistInfoStem`] if the path is the requested entry.
114fn dist_info_entry<'a>(
115    path: &'a str,
116    filename: &WheelFilename,
117    entry_name: &str,
118) -> Result<Option<DistInfoStem<'a>>, Error> {
119    let Some((dist_info_dir, file)) = path.split_once('/') else {
120        return Ok(None);
121    };
122    if file != entry_name {
123        return Ok(None);
124    }
125    let Some(dist_info_stem) = dist_info_dir.strip_suffix(".dist-info") else {
126        return Ok(None);
127    };
128
129    DistInfoStem::new(dist_info_stem, &filename.name).map(Some)
130}
131
132/// Given an archive, read the `METADATA` from the `.dist-info` directory.
133pub fn read_archive_metadata(
134    filename: &WheelFilename,
135    reader: impl std::io::BufRead + std::io::Seek + Unpin,
136) -> Result<Vec<u8>, Error> {
137    block_on(async {
138        let mut zip_reader = ZipFileReader::new(AllowStdIo::new(reader)).await?;
139
140        let (metadata_index, dist_info_stem) = find_archive_dist_info(
141            filename,
142            zip_reader
143                .file()
144                .entries()
145                .iter()
146                .enumerate()
147                .filter_map(|(index, entry)| Some((index, entry.filename().as_str().ok()?))),
148        )?;
149        if let Some(label) = filename.variant() {
150            let path = format!("{dist_info_stem}.dist-info/variant.json");
151            validate_archive_variant(&mut zip_reader, &path, label).await?;
152        }
153
154        let mut buffer = Vec::new();
155        zip_reader
156            .reader_with_entry(metadata_index)
157            .await?
158            .read_to_end_checked(&mut buffer)
159            .await?;
160
161        Ok(buffer)
162    })
163}
164
165/// Validate the `variant.json` next to the wheel's `METADATA` before using it for resolution.
166pub async fn validate_archive_variant<R: AsyncBufRead + AsyncSeek + Unpin>(
167    reader: &mut ZipFileReader<R>,
168    path: &str,
169    label: &VariantLabel,
170) -> Result<(), Error> {
171    let index = reader
172        .file()
173        .entries()
174        .iter()
175        .position(|entry| entry.filename().as_str().ok() == Some(path))
176        .ok_or(Error::MissingVariantMetadata)?;
177    let mut contents = Vec::new();
178    reader
179        .reader_with_entry(index)
180        .await?
181        .read_to_end_checked(&mut contents)
182        .await?;
183    VariantsJsonContent::from_wheel(&contents, label)?;
184    Ok(())
185}
186
187/// Find the `.dist-info` directory in an unzipped wheel.
188///
189/// See: <https://github.com/PyO3/python-pkginfo-rs>
190pub fn find_flat_dist_info(
191    filename: &WheelFilename,
192    path: impl AsRef<Path>,
193) -> Result<DistInfoStem<'static>, Error> {
194    // Iterate over `path` to find the `.dist-info` directory. It should be at the top-level.
195    let Some(dist_info_stem) = fs_err::read_dir(path.as_ref())
196        .map_err(Error::Io)?
197        .find_map(|entry| {
198            let entry = entry.ok()?;
199            let file_type = entry.file_type().ok()?;
200            if file_type.is_dir() {
201                let path = entry.path();
202
203                let extension = path.extension()?;
204                if extension != "dist-info" {
205                    return None;
206                }
207
208                let dist_info_stem = path.file_stem()?.to_str()?;
209                Some(dist_info_stem.to_string())
210            } else {
211                None
212            }
213        })
214    else {
215        return Err(Error::MissingDistInfo);
216    };
217
218    DistInfoStem::new(dist_info_stem, &filename.name)
219}
220
221/// Read the wheel `METADATA` metadata from a `.dist-info` directory.
222fn read_dist_info_metadata(
223    dist_info_stem: &DistInfoStem<'_>,
224    wheel: impl AsRef<Path>,
225) -> Result<Vec<u8>, Error> {
226    let metadata_file = wheel
227        .as_ref()
228        .join(format!("{dist_info_stem}.dist-info/METADATA"));
229    fs_err::read(metadata_file).map_err(Error::Io)
230}
231
232/// Read `METADATA` and, for a variant wheel, validate `variant.json` in the same pass.
233pub async fn read_metadata_async_stream<R: futures::AsyncRead + Unpin>(
234    filename: &WheelFilename,
235    debug_path: &str,
236    reader: R,
237) -> Result<ResolutionMetadata, Error> {
238    let reader = futures::io::BufReader::with_capacity(128 * 1024, reader);
239    let mut zip = async_zip::base::read::stream::ZipFileReader::new(reader);
240    let mut dist_info = None;
241    let mut metadata = None;
242    let mut pending_variant = filename.variant();
243
244    while let Some(mut entry) = zip.next_with_entry().await? {
245        let path = entry.reader().entry().filename().as_str()?.to_owned();
246        let file = if let Some(stem) = dist_info_entry(&path, filename, "METADATA")? {
247            Some((stem, None))
248        } else if let Some(label) = pending_variant
249            && let Some(stem) = dist_info_entry(&path, filename, "variant.json")?
250        {
251            Some((stem, Some(label)))
252        } else {
253            None
254        };
255
256        if let Some((stem, variant_label)) = file {
257            // A variant wheel needs both files from the same directory, in either archive order.
258            let dir = dist_info.get_or_insert_with(|| stem.to_string());
259            if dir != stem.as_str() {
260                return Err(Error::MultipleDistInfo(format!("{dir}, {stem}")));
261            }
262
263            let mut reader = entry.reader_mut().compat();
264            let mut contents = Vec::new();
265            reader.read_to_end(&mut contents).await.map_err(Error::Io)?;
266
267            // Validate the CRC of any file we unpack
268            // (It would be nice if async_zip made it harder to Not do this...)
269            let reader = reader.into_inner();
270            let computed = reader.compute_hash();
271            let expected = reader.entry().crc32();
272            if computed != expected {
273                let error = Error::BadCrc32 {
274                    path,
275                    computed,
276                    expected,
277                };
278                // There are some cases where we fail to get a proper CRC.
279                // This is probably connected to out-of-line data descriptors
280                // which are problematic to access in a streaming context.
281                // In those cases the CRC seems to reliably be stubbed inline as 0,
282                // so we downgrade this to a (hidden-by-default) warning.
283                if expected == 0 {
284                    tracing::warn!("presumed missing CRC: {error}");
285                } else {
286                    return Err(error);
287                }
288            }
289
290            if let Some(label) = variant_label {
291                VariantsJsonContent::from_wheel(&contents, label)?;
292                pending_variant = None;
293            } else {
294                metadata = Some(contents);
295            }
296            if pending_variant.is_none()
297                && let Some(metadata) = metadata.take()
298            {
299                return ResolutionMetadata::parse_metadata(&metadata)
300                    .map_err(|err| Error::InvalidMetadata(debug_path.to_string(), Box::new(err)));
301            }
302        }
303
304        // Close current file to get access to the next one. See docs:
305        // https://docs.rs/async_zip/0.0.16/async_zip/base/read/stream/
306        (.., zip) = entry.skip().await?;
307    }
308
309    if metadata.is_some() {
310        Err(Error::MissingVariantMetadata)
311    } else {
312        Err(Error::MissingDistInfo)
313    }
314}
315
316/// Read the [`ResolutionMetadata`] from an unzipped wheel.
317pub fn read_flat_wheel_metadata(
318    filename: &WheelFilename,
319    wheel: impl AsRef<Path>,
320) -> Result<ResolutionMetadata, Error> {
321    let wheel = wheel.as_ref();
322    let dist_info_stem = find_flat_dist_info(filename, wheel)?;
323    if let Some(label) = filename.variant() {
324        let path = wheel.join(format!("{dist_info_stem}.dist-info/variant.json"));
325        read_wheel_variant(&path, label)?;
326    }
327    let metadata = read_dist_info_metadata(&dist_info_stem, wheel)?;
328    ResolutionMetadata::parse_metadata(&metadata).map_err(|err| {
329        Error::InvalidMetadata(
330            format!("{dist_info_stem}.dist-info/METADATA"),
331            Box::new(err),
332        )
333    })
334}
335
336/// Read a wheel's `variant.json` and check that it describes the variant named in the filename.
337pub fn read_wheel_variant(path: &Path, label: &VariantLabel) -> Result<VariantsJsonContent, Error> {
338    let contents = fs_err::read(path).map_err(Error::Io)?;
339    Ok(VariantsJsonContent::from_wheel(&contents, label)?)
340}
341
342#[cfg(test)]
343mod test {
344    use super::{DistInfoStem, dist_info_entry, find_archive_dist_info};
345    use std::str::FromStr;
346    use uv_distribution_filename::WheelFilename;
347
348    #[test]
349    fn test_dot_in_name() {
350        let files = [
351            "mastodon/Mastodon.py",
352            "mastodon/__init__.py",
353            "mastodon/streaming.py",
354            "Mastodon.py-1.5.1.dist-info/DESCRIPTION.rst",
355            "Mastodon.py-1.5.1.dist-info/metadata.json",
356            "Mastodon.py-1.5.1.dist-info/top_level.txt",
357            "Mastodon.py-1.5.1.dist-info/WHEEL",
358            "Mastodon.py-1.5.1.dist-info/METADATA",
359            "Mastodon.py-1.5.1.dist-info/RECORD",
360        ];
361        let filename = WheelFilename::from_str("Mastodon.py-1.5.1-py2.py3-none-any.whl").unwrap();
362        let (_, dist_info_stem) =
363            find_archive_dist_info(&filename, files.into_iter().map(|file| (file, file))).unwrap();
364        assert_eq!(dist_info_stem.as_str(), "Mastodon.py-1.5.1");
365    }
366
367    #[test]
368    fn test_dist_info_stem_compatibility() {
369        let filename = WheelFilename::from_str("friendly_bard-1.0-py3-none-any.whl")
370            .expect("valid wheel filename");
371        for name in [
372            "friendly-bard-1.0",
373            "FrIeNdLy-._.-bArD-1.0",
374            "friendly_bard-1.0+local",
375            "friendly_bard",
376            "friendly_bard_extra-2.0",
377        ] {
378            let path = format!("{name}.dist-info/METADATA");
379            let ((), archive_name) =
380                find_archive_dist_info(&filename, [((), path.as_str())].into_iter())
381                    .expect("accepted archive directory name");
382            let stream_name = dist_info_entry(&path, &filename, "METADATA")
383                .expect("accepted streaming directory name")
384                .expect("metadata entry");
385            let owned_name = DistInfoStem::new(name.to_owned(), &filename.name)
386                .expect("accepted owned directory name");
387
388            assert_eq!(archive_name.as_str(), name);
389            assert_eq!(stream_name.as_str(), name);
390            assert_eq!(owned_name.as_str(), name);
391        }
392    }
393
394    #[test]
395    fn test_dist_info_stem_mismatch() {
396        let filename = WheelFilename::from_str("friendly_bard-1.0-py3-none-any.whl")
397            .expect("valid wheel filename");
398        let name = "other_package-1.0";
399        let path = format!("{name}.dist-info/METADATA");
400        let expected = "The .dist-info directory `other_package-1.0` does not start with the normalized package name: friendly-bard";
401
402        assert_eq!(
403            find_archive_dist_info(&filename, [((), path.as_str())].into_iter())
404                .expect_err("mismatched archive directory name")
405                .to_string(),
406            expected
407        );
408        assert_eq!(
409            dist_info_entry(&path, &filename, "METADATA")
410                .expect_err("mismatched streaming directory name")
411                .to_string(),
412            expected
413        );
414        assert_eq!(
415            DistInfoStem::new(name.to_owned(), &filename.name)
416                .expect_err("mismatched owned directory name")
417                .to_string(),
418            expected
419        );
420
421        assert!(
422            dist_info_entry("other_package-1.0.dist-info/WHEEL", &filename, "METADATA")
423                .expect("non-metadata entry")
424                .is_none()
425        );
426        assert_eq!(
427            find_archive_dist_info(
428                &filename,
429                [
430                    ((), path.as_str()),
431                    ((), "friendly_bard-1.0.dist-info/METADATA"),
432                ]
433                .into_iter(),
434            )
435            .expect_err("multiple metadata directories")
436            .to_string(),
437            "Multiple .dist-info directories found: other_package-1.0, friendly_bard-1.0"
438        );
439    }
440}
uv_metadata/lib.rs

uv_metadata/
lib.rs
