uv_metadata/
lib.rs

1//! Read metadata from wheels and source distributions.
2//!
3//! This module reads all fields exhaustively. The fields are defined in the [Core metadata
4//! specification](https://packaging.python.org/en/latest/specifications/core-metadata/).
5
6use futures::executor::block_on;
7use futures::io::AllowStdIo;
8use std::io;
9use std::path::Path;
10use thiserror::Error;
11use tokio::io::AsyncReadExt;
12use tokio_util::compat::FuturesAsyncReadCompatExt;
13use uv_distribution_filename::WheelFilename;
14use uv_normalize::{DistInfoName, InvalidNameError};
15use uv_pypi_types::ResolutionMetadata;
16
17/// The caller is responsible for attaching the path or url we failed to read.
18#[derive(Debug, Error)]
19pub enum Error {
20    #[error("Failed to read `dist-info` metadata from built wheel")]
21    DistInfo,
22    #[error("No .dist-info directory found")]
23    MissingDistInfo,
24    #[error("Multiple .dist-info directories found: {0}")]
25    MultipleDistInfo(String),
26    #[error(
27        "The .dist-info directory does not consist of the normalized package name and version: `{0}`"
28    )]
29    MissingDistInfoSegments(String),
30    #[error("The .dist-info directory {0} does not start with the normalized package name: {1}")]
31    MissingDistInfoPackageName(String, String),
32    #[error("The .dist-info directory name contains invalid characters")]
33    InvalidName(#[from] InvalidNameError),
34    #[error("The metadata at {0} is invalid")]
35    InvalidMetadata(String, Box<uv_pypi_types::MetadataError>),
36    #[error("Bad CRC (got {computed:08x}, expected {expected:08x}) for file: {path}")]
37    BadCrc32 {
38        path: String,
39        computed: u32,
40        expected: u32,
41    },
42    #[error("Failed to read from zip file")]
43    AsyncZip(#[source] async_zip::error::ZipError),
44    #[error(
45        "Archive contains a file with an unsupported compression method; files must be compressed with 'stored', 'DEFLATE', or 'zstd'"
46    )]
47    UnsupportedCompression,
48    // No `#[from]` to enforce manual review of `io::Error` sources.
49    #[error(transparent)]
50    Io(io::Error),
51}
52
53impl From<async_zip::error::ZipError> for Error {
54    fn from(err: async_zip::error::ZipError) -> Self {
55        match err {
56            async_zip::error::ZipError::CompressionNotSupported(_) => Self::UnsupportedCompression,
57            o => Self::AsyncZip(o),
58        }
59    }
60}
61
62/// Find the `.dist-info` directory in a zipped wheel.
63///
64/// Returns the dist info dir prefix without the `.dist-info` extension.
65///
66/// Reference implementation: <https://github.com/pypa/pip/blob/36823099a9cdd83261fdbc8c1d2a24fa2eea72ca/src/pip/_internal/utils/wheel.py#L38>
67pub fn find_archive_dist_info<'a, T: Copy>(
68    filename: &WheelFilename,
69    files: impl Iterator<Item = (T, &'a str)>,
70) -> Result<(T, &'a str), Error> {
71    let metadatas: Vec<_> = files
72        .filter_map(|(payload, path)| {
73            let (dist_info_dir, file) = path.split_once('/')?;
74            if file != "METADATA" {
75                return None;
76            }
77            let dist_info_prefix = dist_info_dir.strip_suffix(".dist-info")?;
78            Some((payload, dist_info_prefix))
79        })
80        .collect();
81
82    // Like `pip`, assert that there is exactly one `.dist-info` directory.
83    let (payload, dist_info_prefix) = match metadatas[..] {
84        [] => {
85            return Err(Error::MissingDistInfo);
86        }
87        [(payload, path)] => (payload, path),
88        _ => {
89            return Err(Error::MultipleDistInfo(
90                metadatas
91                    .into_iter()
92                    .map(|(_, dist_info_dir)| dist_info_dir.to_string())
93                    .collect::<Vec<_>>()
94                    .join(", "),
95            ));
96        }
97    };
98
99    // Like `pip`, validate that the `.dist-info` directory is prefixed with the canonical
100    // package name.
101    let normalized_prefix = DistInfoName::new(dist_info_prefix);
102    if !normalized_prefix
103        .as_ref()
104        .starts_with(filename.name.as_str())
105    {
106        return Err(Error::MissingDistInfoPackageName(
107            dist_info_prefix.to_string(),
108            filename.name.to_string(),
109        ));
110    }
111
112    Ok((payload, dist_info_prefix))
113}
114
115/// Returns `true` if the file is a `METADATA` file in a `.dist-info` directory that matches the
116/// wheel filename.
117fn is_metadata_entry(path: &str, filename: &WheelFilename) -> Result<bool, Error> {
118    let Some((dist_info_dir, file)) = path.split_once('/') else {
119        return Ok(false);
120    };
121    if file != "METADATA" {
122        return Ok(false);
123    }
124    let Some(dist_info_prefix) = dist_info_dir.strip_suffix(".dist-info") else {
125        return Ok(false);
126    };
127
128    // Like `pip`, validate that the `.dist-info` directory is prefixed with the canonical
129    // package name.
130    let normalized_prefix = DistInfoName::new(dist_info_prefix);
131    if !normalized_prefix
132        .as_ref()
133        .starts_with(filename.name.as_str())
134    {
135        return Err(Error::MissingDistInfoPackageName(
136            dist_info_prefix.to_string(),
137            filename.name.to_string(),
138        ));
139    }
140
141    Ok(true)
142}
143
144/// Given an archive, read the `METADATA` from the `.dist-info` directory.
145pub fn read_archive_metadata(
146    filename: &WheelFilename,
147    reader: impl std::io::BufRead + std::io::Seek + Unpin,
148) -> Result<Vec<u8>, Error> {
149    block_on(async {
150        let mut zip_reader =
151            async_zip::base::read::seek::ZipFileReader::new(AllowStdIo::new(reader)).await?;
152
153        let (metadata_index, _dist_info_prefix) = find_archive_dist_info(
154            filename,
155            zip_reader
156                .file()
157                .entries()
158                .iter()
159                .enumerate()
160                .filter_map(|(index, entry)| Some((index, entry.filename().as_str().ok()?))),
161        )?;
162
163        let mut buffer = Vec::new();
164        zip_reader
165            .reader_with_entry(metadata_index)
166            .await?
167            .read_to_end_checked(&mut buffer)
168            .await?;
169
170        Ok(buffer)
171    })
172}
173
174/// Find the `.dist-info` directory in an unzipped wheel.
175///
176/// See: <https://github.com/PyO3/python-pkginfo-rs>
177fn find_flat_dist_info(filename: &WheelFilename, path: impl AsRef<Path>) -> Result<String, Error> {
178    // Iterate over `path` to find the `.dist-info` directory. It should be at the top-level.
179    let Some(dist_info_prefix) = fs_err::read_dir(path.as_ref())
180        .map_err(Error::Io)?
181        .find_map(|entry| {
182            let entry = entry.ok()?;
183            let file_type = entry.file_type().ok()?;
184            if file_type.is_dir() {
185                let path = entry.path();
186
187                let extension = path.extension()?;
188                if extension != "dist-info" {
189                    return None;
190                }
191
192                let dist_info_prefix = path.file_stem()?.to_str()?;
193                Some(dist_info_prefix.to_string())
194            } else {
195                None
196            }
197        })
198    else {
199        return Err(Error::MissingDistInfo);
200    };
201
202    // Like `pip`, validate that the `.dist-info` directory is prefixed with the canonical
203    // package name.
204    let normalized_prefix = DistInfoName::new(&dist_info_prefix);
205    if !normalized_prefix
206        .as_ref()
207        .starts_with(filename.name.as_str())
208    {
209        return Err(Error::MissingDistInfoPackageName(
210            dist_info_prefix,
211            filename.name.to_string(),
212        ));
213    }
214
215    Ok(dist_info_prefix)
216}
217
218/// Read the wheel `METADATA` metadata from a `.dist-info` directory.
219fn read_dist_info_metadata(
220    dist_info_prefix: &str,
221    wheel: impl AsRef<Path>,
222) -> Result<Vec<u8>, Error> {
223    let metadata_file = wheel
224        .as_ref()
225        .join(format!("{dist_info_prefix}.dist-info/METADATA"));
226    fs_err::read(metadata_file).map_err(Error::Io)
227}
228
229/// Read and parse a wheel's `METADATA` file from a zip stream without seeking.
230pub async fn read_metadata_async_stream<R: futures::AsyncRead + Unpin>(
231    filename: &WheelFilename,
232    debug_path: &str,
233    reader: R,
234) -> Result<ResolutionMetadata, Error> {
235    let reader = futures::io::BufReader::with_capacity(128 * 1024, reader);
236    let mut zip = async_zip::base::read::stream::ZipFileReader::new(reader);
237
238    while let Some(mut entry) = zip.next_with_entry().await? {
239        // Find the `METADATA` entry.
240        let path = entry.reader().entry().filename().as_str()?.to_owned();
241
242        if is_metadata_entry(&path, filename)? {
243            let mut reader = entry.reader_mut().compat();
244            let mut contents = Vec::new();
245            reader.read_to_end(&mut contents).await.map_err(Error::Io)?;
246
247            // Validate the CRC of any file we unpack
248            // (It would be nice if async_zip made it harder to Not do this...)
249            let reader = reader.into_inner();
250            let computed = reader.compute_hash();
251            let expected = reader.entry().crc32();
252            if computed != expected {
253                let error = Error::BadCrc32 {
254                    path,
255                    computed,
256                    expected,
257                };
258                // There are some cases where we fail to get a proper CRC.
259                // This is probably connected to out-of-line data descriptors
260                // which are problematic to access in a streaming context.
261                // In those cases the CRC seems to reliably be stubbed inline as 0,
262                // so we downgrade this to a (hidden-by-default) warning.
263                if expected == 0 {
264                    tracing::warn!("presumed missing CRC: {error}");
265                } else {
266                    return Err(error);
267                }
268            }
269
270            let metadata = ResolutionMetadata::parse_metadata(&contents)
271                .map_err(|err| Error::InvalidMetadata(debug_path.to_string(), Box::new(err)))?;
272            return Ok(metadata);
273        }
274
275        // Close current file to get access to the next one. See docs:
276        // https://docs.rs/async_zip/0.0.16/async_zip/base/read/stream/
277        (.., zip) = entry.skip().await?;
278    }
279
280    Err(Error::MissingDistInfo)
281}
282
283/// Read the [`ResolutionMetadata`] from an unzipped wheel.
284pub fn read_flat_wheel_metadata(
285    filename: &WheelFilename,
286    wheel: impl AsRef<Path>,
287) -> Result<ResolutionMetadata, Error> {
288    let dist_info_prefix = find_flat_dist_info(filename, &wheel)?;
289    let metadata = read_dist_info_metadata(&dist_info_prefix, &wheel)?;
290    ResolutionMetadata::parse_metadata(&metadata).map_err(|err| {
291        Error::InvalidMetadata(
292            format!("{dist_info_prefix}.dist-info/METADATA"),
293            Box::new(err),
294        )
295    })
296}
297
298#[cfg(test)]
299mod test {
300    use super::find_archive_dist_info;
301    use std::str::FromStr;
302    use uv_distribution_filename::WheelFilename;
303
304    #[test]
305    fn test_dot_in_name() {
306        let files = [
307            "mastodon/Mastodon.py",
308            "mastodon/__init__.py",
309            "mastodon/streaming.py",
310            "Mastodon.py-1.5.1.dist-info/DESCRIPTION.rst",
311            "Mastodon.py-1.5.1.dist-info/metadata.json",
312            "Mastodon.py-1.5.1.dist-info/top_level.txt",
313            "Mastodon.py-1.5.1.dist-info/WHEEL",
314            "Mastodon.py-1.5.1.dist-info/METADATA",
315            "Mastodon.py-1.5.1.dist-info/RECORD",
316        ];
317        let filename = WheelFilename::from_str("Mastodon.py-1.5.1-py2.py3-none-any.whl").unwrap();
318        let (_, dist_info_prefix) =
319            find_archive_dist_info(&filename, files.into_iter().map(|file| (file, file))).unwrap();
320        assert_eq!(dist_info_prefix, "Mastodon.py-1.5.1");
321    }
322}
uv_metadata/lib.rs

uv_metadata/
lib.rs
