zip 9.0.3

//! Code related to stream reading

use crate::ZipReadOptions;
use crate::extra_fields::ExtraFields;
use crate::format::blocks::{FixedSizeBlock, Pod, ZipCentralEntryBlock, ZipLocalEntryBlock};
use crate::format::magic::Magic;
use crate::read::extract::{MAX_SYMLINK_TARGET_LEN, make_symlink_as_file};
use crate::read::extract::{make_symlink, make_symlink_as_file_utf8};
use crate::read::{
    SymlinkExtractAction, ZipFile, ZipFileData, ZipFileEntry, ZipResult,
    central_header_to_zip_file_inner,
};
use crate::result::{ZipError, invalid};

use indexmap::IndexMap;
use std::borrow::Cow;
use std::io::{self, Read};
use std::path::{Path, PathBuf};

/// Stream decoder for zip.
#[derive(Debug)]
pub struct ZipStreamReader<R>(R);

impl<R> ZipStreamReader<R> {
    /// Create a new `ZipStreamReader`
    pub const fn new(reader: R) -> Self {
        Self(reader)
    }
}

fn parse_central_directory<R: Read>(reader: &mut R) -> ZipResult<ZipFileEntry<'static>> {
    // Give archive_offset and central_header_start dummy value 0, since
    // they are not used in the output.
    let archive_offset = 0;
    let central_header_start = 0;

    // Parse central header
    let block = ZipCentralEntryBlock::parse(reader)?;
    let (file, file_name_raw) =
        central_header_to_zip_file_inner(reader, archive_offset, central_header_start, block)?;
    Ok(ZipFileEntry {
        file_name_raw: Cow::Owned(file_name_raw),
        data: Cow::Owned(file),
    })
}

impl<R: Read> ZipStreamReader<R> {
    /// Iterate over the stream and extract all file and their
    /// metadata.
    pub fn visit<V: ZipStreamVisitor>(mut self, visitor: &mut V) -> ZipResult<()> {
        while let Some(mut file) = read_zipfile_from_stream(&mut self.0)? {
            visitor.visit_file(&mut file)?;
        }

        // `read_zipfile_from_stream` returned `None` because it read the signature of the
        // first central directory header, so the stream is now 4 bytes into that header. Put
        // the signature back in front so every header, the first one included, parses whole.
        let signature = Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE.to_le_bytes();
        let mut central_directory = (&signature[..]).chain(&mut self.0);
        while let Ok(metadata) = parse_central_directory(&mut central_directory) {
            visitor.visit_additional_metadata(&metadata)?;
        }

        Ok(())
    }

    /// Extract a Zip archive into a directory, overwriting files if they
    /// already exist. Paths are sanitized with [`ZipFile::enclosed_name`].
    ///
    /// Extraction is not atomic; If an error is encountered, some of the files
    /// may be left on disk. The file(s) and dir(s) are first created, then the
    /// permissions are applied to them
    ///
    /// Extraction of symlink is not possible since we don't have access to the
    /// external attributes in the local headers of the entries
    pub fn extract<P: AsRef<Path>>(self, directory: P) -> ZipResult<()> {
        /// Destination, every entry written in this pass (raw local name -> the checked output
        /// path `visit_file` wrote it to), and the modes the central directory gives them.
        struct Extractor {
            outpath: PathBuf,
            resolved_paths: IndexMap<Box<[u8]>, PathBuf>,
            #[cfg(unix)]
            file_permissions: std::collections::BTreeMap<PathBuf, u32>,
        }
        impl ZipStreamVisitor for Extractor {
            fn visit_file<R: Read>(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> {
                let extract_options = crate::read::extract::ExtractOptions::default();
                let mut outpath = self.outpath.clone();
                file.safe_prepare_path(
                    &self.outpath,
                    &mut outpath,
                    None::<&(_, fn(&Path) -> bool)>,
                )?;
                self.resolved_paths
                    .insert(file.name_raw().into(), outpath.clone());

                if file.is_symlink() {
                    // Same bound as `ZipArchive::extract`: the declared
                    // uncompressed size is attacker-controlled, so reject
                    // oversized entries rather than pre-allocating them.
                    //
                    // `is_symlink()` is currently always false on this path
                    // (`from_local_block` has no external attributes to read),
                    // so this is defence in depth -- see
                    // `stream_does_not_expose_the_symlink_bit`.
                    let declared_len = file.size();
                    if declared_len > MAX_SYMLINK_TARGET_LEN {
                        return Err(invalid!(
                            "symlink target declares {} bytes, more than the {} byte limit",
                            declared_len,
                            MAX_SYMLINK_TARGET_LEN
                        ));
                    }

                    let mut target = Vec::with_capacity(declared_len as usize);
                    file.read_to_end(&mut target)?;
                    match extract_options.symlink_action {
                        SymlinkExtractAction::ExtractAsFile => {
                            make_symlink_as_file(&outpath, &target)?;
                        }
                        SymlinkExtractAction::ExtractAsFileUtf8 => {
                            make_symlink_as_file_utf8(&outpath, &target)?;
                        }
                        _ => {
                            make_symlink(
                                &self.outpath,
                                &outpath,
                                &target,
                                &self.resolved_paths,
                                extract_options.symlink_action,
                            )?;
                        }
                    }
                    return Ok(());
                }
                if file.is_dir() {
                    fs::create_dir_all(&outpath)?;
                } else {
                    let mut outfile = fs::File::create(&outpath)?;
                    io::copy(file, &mut outfile)?;
                }

                Ok(())
            }

            #[allow(unused)]
            fn visit_additional_metadata(&mut self, metadata: &ZipFileEntry<'_>) -> ZipResult<()> {
                // The central directory is not checked against the local headers, so its
                // names only select entries this pass already wrote: a name that wasn't
                // written gets no mode, and the mode goes to the path `visit_file` checked,
                // never to a path rebuilt from the central directory.
                #[cfg(unix)]
                if let (Some(outpath), Some(mode)) = (
                    self.resolved_paths.get(metadata.name_raw()),
                    metadata.unix_mode(),
                ) {
                    self.file_permissions.insert(outpath.clone(), mode);
                }
                Ok(())
            }
        }
        use std::fs;
        fs::create_dir_all(&directory)?;
        let directory = directory.as_ref().canonicalize()?;

        let mut extractor = Extractor {
            outpath: directory,
            resolved_paths: IndexMap::new(),
            #[cfg(unix)]
            file_permissions: Default::default(),
        };
        self.visit(&mut extractor)?;

        // As `ZipArchive::extract`: children before parents, so a read-only directory does not
        // block setting its contents. Never through a symlink: `set_permissions` follows them.
        #[cfg(unix)]
        {
            for (path, mode) in extractor.file_permissions.into_iter().rev() {
                use std::os::unix::fs::PermissionsExt;
                crate::read::extract::set_permissions_no_follow_symlink(
                    &path,
                    std::fs::Permissions::from_mode(mode),
                )?;
            }
        }
        Ok(())
    }
}

/// Visitor for `ZipStreamReader`
pub trait ZipStreamVisitor {
    ///  * `file` - contains the content of the file and most of the metadata,
    ///    except:
    ///     - `comment`: set to an empty string
    ///     - `data_start`: set to 0
    ///     - `external_attributes`: `unix_mode()`: will return None
    fn visit_file<R: Read>(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()>;

    /// This function is guaranteed to be called after all `visit_file`s.
    ///
    ///  * `metadata` - Provides missing metadata in `visit_file`.
    fn visit_additional_metadata(&mut self, metadata: &ZipFileEntry<'_>) -> ZipResult<()>;
}

/// Read `ZipFile` structures from a non-seekable reader.
///
/// This is an alternative method to read a zip file. If possible, use the `ZipArchive` functions
/// as some information will be missing when reading this manner.
///
/// Reads a file header from the start of the stream. Will return `Ok(Some(..))` if a file is
/// present at the start of the stream. Returns `Ok(None)` if the start of the central directory
/// is encountered. No more files should be read after this.
///
/// The Drop implementation of `ZipFile` ensures that the reader will be correctly positioned after
/// the structure is done.
///
/// Missing fields are:
/// * `comment`: set to an empty string
/// * `data_start`: set to 0
/// * `external_attributes`: `unix_mode()`: will return None
pub fn read_zipfile_from_stream<R: Read>(reader: &mut R) -> ZipResult<Option<ZipFile<'_, R>>> {
    read_zipfile_from_stream_with_options(reader, ZipReadOptions::default())
}

/// Read `ZipFile` from a non-seekable reader like [`read_zipfile_from_stream`] does, but assume the
/// given compressed size and don't read any further ahead than that.
pub fn read_zipfile_from_stream_with_compressed_size<R: Read>(
    reader: &mut R,
    compressed_size: u64,
) -> ZipResult<Option<ZipFile<'_, R>>> {
    let options = ZipReadOptions::default().override_compressed_size(compressed_size);
    read_zipfile_from_stream_with_options(reader, options)
}

/// Same as `read_zipfile_from_stream` but with `ZipReadOptions`
/// Since LZMA decoding requires the uncompressed length, you will need to override it
pub fn read_zipfile_from_stream_with_options<'a, R: Read>(
    reader: &'a mut R,
    mut options: ZipReadOptions<'a>,
) -> ZipResult<Option<ZipFile<'a, R>>> {
    // We can't use the typical [`ZipLocalEntryBlock::parse`] method, as we follow separate code paths depending on the
    // "magic" value (since the magic value will be from the central directory header if we've
    // finished iterating over all the actual files).
    let mut magic_buf = [0; size_of::<u32>()];
    reader.read_exact(&mut magic_buf)?;

    match Magic::from_le_bytes(magic_buf) {
        Magic::LOCAL_FILE_HEADER_SIGNATURE => (),
        Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE => return Ok(None),
        _ => return Err(ZipLocalEntryBlock::WRONG_MAGIC_ERROR),
    }

    let mut block = ZipLocalEntryBlock::zeroed();
    reader.read_exact(block.as_bytes_mut())?;

    let block = block.from_le();
    // parse file_name_raw
    let file_name_length: usize = block.file_name_length.into();
    let mut file_name_raw = vec![0u8; file_name_length];
    if let Err(e) = reader.read_exact(&mut file_name_raw) {
        if e.kind() == std::io::ErrorKind::UnexpectedEof {
            return Err(invalid!("File name extends beyond file boundary"));
        }
        return Err(e.into());
    }

    // parse extra fields raw
    let extra_field_length: usize = block.extra_field_length.into();
    let mut extra_fields_raw = vec![0u8; extra_field_length];
    if let Err(e) = reader.read_exact(&mut extra_fields_raw) {
        if e.kind() == std::io::ErrorKind::UnexpectedEof {
            return Err(invalid!("Extra field extends beyond file boundary"));
        }
        return Err(e.into());
    }
    // parse extra fields
    let extra_fields = ExtraFields::parse(&extra_fields_raw, &block, true)?;
    let mut data = ZipFileData::from_local_block(block, extra_fields)?;
    data.apply_extra_fields(&mut file_name_raw)?;
    if data.is_using_data_descriptor() {
        if let Some(comp_size) = options.force_compressed_size {
            data.compressed_size = comp_size;
        } else {
            return Err(ZipError::UnsupportedArchive(
                "The file length is not available in the local header",
            ));
        }
    }
    if let Some(uncomp_size) = options.force_uncompressed_size {
        data.uncompressed_size = uncomp_size;
    } else if data.is_using_data_descriptor() {
        // The local header doesn't carry the size; it follows the data.
        options.size_unknown = true;
    }
    if let Some(crc) = options.force_crc {
        data.crc32 = crc;
    }

    let limit_reader = reader.take(data.compressed_size);
    let reader = data.make_reader(limit_reader, options)?;
    Ok(Some(ZipFile {
        file_name_raw: Cow::Owned(file_name_raw),
        data: Cow::Owned(data),
        reader,
    }))
}

#[cfg(test)]
mod tests {
    use crate::read::ZipFile;
    use crate::read::ZipFileEntry;
    use crate::read::stream::{ZipStreamReader, ZipStreamVisitor};
    use crate::result::ZipResult;
    use std::collections::BTreeSet;
    use std::io::{Cursor, Read};

    #[test]
    fn zip_read_streaming_visitor() {
        let reader =
            ZipStreamReader::new(Cursor::new(include_bytes!("../../tests/data/mimetype.zip")));

        #[derive(Default)]
        struct V {
            filenames: BTreeSet<Box<str>>,
        }
        impl ZipStreamVisitor for V {
            fn visit_file<R: Read>(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> {
                if file.is_file() {
                    let file_name = file.name().unwrap();
                    self.filenames.insert(file_name.into());
                }

                Ok(())
            }
            fn visit_additional_metadata(
                &mut self,
                zip_file_metadata: &ZipFileEntry<'_>,
            ) -> ZipResult<()> {
                if zip_file_metadata.is_file() {
                    let file_name = zip_file_metadata.name().unwrap();
                    assert!(
                        self.filenames.contains::<str>(file_name.as_ref()),
                        "{} is missing its file content",
                        file_name
                    );
                }

                Ok(())
            }
        }

        reader.visit(&mut V::default()).unwrap();
    }

    #[test]
    fn file_and_dir_predicates() {
        let reader = ZipStreamReader::new(Cursor::new(include_bytes!(
            "../../tests/data/files_and_dirs.zip"
        )));

        #[derive(Default)]
        struct V {
            filenames: BTreeSet<Box<str>>,
        }
        impl ZipStreamVisitor for V {
            fn visit_file<R: Read>(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> {
                let full_name = file.enclosed_name().unwrap();
                let file_name = full_name.file_name().unwrap().to_str().unwrap();
                assert!(
                    (file_name.starts_with("dir") && file.is_dir())
                        || (file_name.starts_with("file") && file.is_file())
                );

                if file.is_file() {
                    let file_name = file.name().unwrap();
                    self.filenames.insert(file_name.into());
                }

                Ok(())
            }
            fn visit_additional_metadata(
                &mut self,
                zip_file_metadata: &ZipFileEntry<'_>,
            ) -> ZipResult<()> {
                if zip_file_metadata.is_file() {
                    let file_name = zip_file_metadata.name().unwrap();
                    assert!(
                        self.filenames.contains::<str>(file_name.as_ref()),
                        "{} is missing its file content",
                        file_name
                    );
                }

                Ok(())
            }
        }

        reader.visit(&mut V::default()).unwrap();
    }

    /// Symlinks being extracted shouldn't be followed out of the destination directory.
    /// Cannot use fs with miri CI
    #[cfg(not(miri))]
    #[test]
    fn test_cannot_symlink_outside_destination() -> ZipResult<()> {
        use crate::ZipWriter;
        use crate::write::SimpleFileOptions;
        use std::fs::create_dir;
        use tempfile::TempDir;

        let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
        writer.add_symlink("symlink/", "../dest-sibling/", SimpleFileOptions::default())?;
        writer.start_file("symlink/dest-file", SimpleFileOptions::default())?;
        let reader = ZipStreamReader::new(writer.finish()?);
        let dest_parent = TempDir::with_prefix("stream__cannot_symlink_outside_destination")?;
        let dest_sibling = dest_parent.path().join("dest-sibling");
        create_dir(&dest_sibling)?;
        let dest = dest_parent.path().join("dest");
        create_dir(&dest)?;
        assert!(reader.extract(dest).is_err());
        assert!(!dest_sibling.join("dest-file").exists());
        Ok(())
    }

    /// The stream reader builds each entry from the *local* file header, which
    /// has no external-attributes field (`ZipFileData::from_local_block` sets it
    /// to 0), so `is_symlink()` is always false on this path and the symlink
    /// branch in `extract` is currently unreachable.
    ///
    /// The `MAX_SYMLINK_TARGET_LEN` guard is still kept there, for consistency
    /// with `ZipArchive::extract` and so it is already in place if that ever
    /// changes. This test pins the current behaviour: if the local header starts
    /// carrying the attribute, the assertion fails and the guard needs a real
    /// regression test.
    #[test]
    fn stream_does_not_expose_the_symlink_bit() -> ZipResult<()> {
        use crate::ZipWriter;
        use crate::write::SimpleFileOptions;

        let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
        writer.add_symlink("link", "/target", SimpleFileOptions::default())?;
        let mut cur = writer.finish()?;
        cur.set_position(0);

        let file = crate::read::read_zipfile_from_stream(&mut cur)?.unwrap();
        assert!(
            !file.is_symlink(),
            "the stream path now sees the symlink bit -- the MAX_SYMLINK_TARGET_LEN \
             guard in `stream.rs` is reachable and needs a regression test"
        );

        Ok(())
    }

    /// Cannot use fs with miri CI
    #[cfg(not(miri))]
    #[test]
    fn test_can_create_destination() -> ZipResult<()> {
        use tempfile::TempDir;

        let v = include_bytes!("../../tests/data/mimetype.zip");
        let reader = ZipStreamReader::new(v.as_ref());
        let dest = TempDir::with_prefix("stream_test_can_create_destination").unwrap();
        reader.extract(&dest)?;
        assert!(dest.path().join("mimetype").exists());
        Ok(())
    }

    #[test]
    fn zip_read_streaming() {
        use super::read_zipfile_from_stream;

        let mut reader = Cursor::new(include_bytes!("../../tests/data/mimetype.zip"));
        loop {
            if read_zipfile_from_stream(&mut reader).unwrap().is_none() {
                break;
            }
        }
    }

    #[test]
    #[cfg(feature = "deflate")]
    fn zip_read_streaming_compressed() {
        use super::read_zipfile_from_stream_with_compressed_size;
        use crate::ZipWriter;
        use crate::write::SimpleFileOptions;
        use std::io::Write;

        let compression_method = crate::CompressionMethod::Deflated;
        let options = SimpleFileOptions::default()
            .compression_method(compression_method)
            .unix_permissions(0o755);

        let mut bytes = Vec::new();
        let mut writer = ZipWriter::new(std::io::Cursor::new(&mut bytes));
        writer.start_file("file.txt", options).unwrap();
        write!(&mut writer, "{}", "test-".repeat(100)).unwrap();
        writer.finish().unwrap();

        let compressed_size = u32::from_le_bytes(bytes[18..22].try_into().unwrap());
        let uncompressed_size = u32::from_le_bytes(bytes[22..26].try_into().unwrap());

        assert_eq!(compressed_size, 14);
        assert_eq!(uncompressed_size as usize, "test-".len() * 100);

        let mut reader = Cursor::new(bytes);
        loop {
            if read_zipfile_from_stream_with_compressed_size(
                &mut reader,
                u64::from(compressed_size),
            )
            .unwrap()
            .is_none()
            {
                break;
            }
        }
    }

    #[test]
    #[cfg(feature = "aes-crypto")]
    fn zip_read_streaming_compressed_and_aes() {
        use super::read_zipfile_from_stream_with_options;
        use crate::ZipReadOptions;

        let bytes = include_bytes!("../../tests/data/aes_archive.zip");
        let compressed_size = 46;

        let mut reader = Cursor::new(bytes);
        const PASSWORD: &[u8] = b"helloworld";
        let options = ZipReadOptions::new()
            .password(Some(PASSWORD))
            .override_compressed_size(compressed_size);

        // we simulate the fact that we need the compressed size like a streamed zip
        let result = read_zipfile_from_stream_with_options(&mut reader, options);
        let optional_file = result.unwrap();
        let mut file = optional_file.unwrap();

        let file_name = file.name().unwrap();
        assert_eq!(file_name, "secret_data_128");

        const SECRET_CONTENT: &str = "Lorem ipsum dolor sit amet";
        let mut decrypted_content = String::new();
        file.read_to_string(&mut decrypted_content)
            .expect("couldn't read encrypted file");
        assert_eq!(SECRET_CONTENT, decrypted_content);
    }

    #[test]
    #[cfg(feature = "aes-crypto")]
    fn zip_read_streaming_compressed_and_aes_without_size() {
        use super::read_zipfile_from_stream_with_options;
        use crate::ZipReadOptions;

        let bytes = include_bytes!("../../tests/data/aes_archive.zip");

        let mut reader = Cursor::new(bytes);
        const PASSWORD: &[u8] = b"helloworld";
        let options = ZipReadOptions::new().password(Some(PASSWORD));

        // the zip already has the compressed size (it's not a streamed zip)
        let result = read_zipfile_from_stream_with_options(&mut reader, options);
        let optional_file = result.unwrap();
        let mut file = optional_file.unwrap();

        let file_name = file.name().unwrap();
        assert_eq!(file_name, "secret_data_128");

        const SECRET_CONTENT: &str = "Lorem ipsum dolor sit amet";
        let mut decrypted_content = String::new();
        file.read_to_string(&mut decrypted_content)
            .expect("couldn't read encrypted file");
        assert_eq!(SECRET_CONTENT, decrypted_content);
    }

    #[test]
    fn zip_read_streaming_zipwriter() {
        use crate::CompressionMethod;
        use crate::ZipReadOptions;
        use crate::ZipWriter;
        use crate::read::read_zipfile_from_stream;
        use crate::read::read_zipfile_from_stream_with_compressed_size;
        use crate::read::read_zipfile_from_stream_with_options;
        use crate::write::SimpleFileOptions;
        use std::io::Write;

        let mut buffer = Vec::new();
        let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored);
        let mut archive = ZipWriter::new_stream(Cursor::new(&mut buffer));
        archive.start_file("name", options).unwrap();
        archive.write_all(b"test").unwrap();
        let compressed_size = b"test".len() as u64;
        archive.finish().unwrap();
        {
            // reading will fail because it's a streamed zipfile and we don't know the size
            let mut reader = Cursor::new(buffer.clone());
            let result = read_zipfile_from_stream(&mut reader);
            assert!(result.is_err());
        }
        {
            // reading the file will fail because of the invalid checksum
            let mut reader = Cursor::new(buffer.clone());
            let result =
                read_zipfile_from_stream_with_compressed_size(&mut reader, compressed_size);
            let optional_file = result.unwrap();
            let mut file = optional_file.unwrap();
            let file_name = file.name().unwrap();
            assert_eq!(file_name, "name");
            // test reading
            let mut content = Vec::new();
            let read_result = file.read_to_end(&mut content);
            assert!(read_result.is_err()); // invalid checksum

            let error_str = read_result.unwrap_err().to_string();
            // eprintln!("{error_str}");
            assert!(error_str.contains("Invalid checksum"));
        }
        {
            // reading the file will work because we override the checksum
            let mut reader = Cursor::new(buffer.clone());
            let read_options = ZipReadOptions::default()
                .override_compressed_size(compressed_size)
                .override_crc(3632233996);
            let result = read_zipfile_from_stream_with_options(&mut reader, read_options);
            let optional_file = result.unwrap();
            let mut file = optional_file.unwrap();
            let file_name = file.name().unwrap();
            assert_eq!(file_name, "name");
            // test reading
            let mut content = Vec::new();
            file.read_to_end(&mut content).unwrap();
            assert_eq!(content, b"test");
        }
        {
            // reading the file will work because we skip checksum
            let mut reader = Cursor::new(buffer.clone());
            let read_options = ZipReadOptions::default()
                .override_compressed_size(compressed_size)
                .ignore_crc32(true);
            let result = read_zipfile_from_stream_with_options(&mut reader, read_options);
            let optional_file = result.unwrap();
            let mut file = optional_file.unwrap();
            let file_name = file.name().unwrap();
            assert_eq!(file_name, "name");
            // test reading
            let mut content = Vec::new();
            let read_result = file.read_to_end(&mut content);
            assert!(read_result.is_ok());
            assert_eq!(content, b"test");
        }
    }
}