use crate::ZipReadOptions;
use crate::extra_fields::ExtraFields;
use crate::format::blocks::{FixedSizeBlock, Pod, ZipCentralEntryBlock, ZipLocalEntryBlock};
use crate::format::magic::Magic;
use crate::read::extract::{MAX_SYMLINK_TARGET_LEN, make_symlink_as_file};
use crate::read::extract::{make_symlink, make_symlink_as_file_utf8};
use crate::read::{
SymlinkExtractAction, ZipFile, ZipFileData, ZipFileEntry, ZipResult,
central_header_to_zip_file_inner,
};
use crate::result::{ZipError, invalid};
use crate::read::stream::ReadFromStreamResult::{
CentralDirectoryHeader, EndOfCentralDirectory, File,
};
use indexmap::IndexMap;
use std::borrow::Cow;
use std::io::{self, Read};
use std::path::{Path, PathBuf};
#[derive(Debug)]
pub struct ZipStreamReader<R>(R);
impl<R> ZipStreamReader<R> {
pub const fn new(reader: R) -> Self {
Self(reader)
}
}
fn parse_central_directory<R: Read>(reader: &mut R) -> ZipResult<ZipFileEntry<'static>> {
let archive_offset = 0;
let central_header_start = 0;
let block = ZipCentralEntryBlock::parse(reader)?;
let (file, file_name_raw) =
central_header_to_zip_file_inner(reader, archive_offset, central_header_start, block)?;
Ok(ZipFileEntry {
file_name_raw: Cow::Owned(file_name_raw),
data: Cow::Owned(file),
})
}
impl<R: Read> ZipStreamReader<R> {
pub fn visit<V: ZipStreamVisitor>(mut self, visitor: &mut V) -> ZipResult<()> {
let has_central_directory = loop {
let read_from_stream_result =
read_zipfile_from_stream_with_options_internal(&mut self.0, Default::default())?;
match read_from_stream_result {
File(mut file) => visitor.visit_file(&mut file)?,
CentralDirectoryHeader => break true,
EndOfCentralDirectory => break false,
}
};
if has_central_directory {
let signature = Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE.to_le_bytes();
let mut central_directory = (&signature[..]).chain(&mut self.0);
while let Ok(metadata) = parse_central_directory(&mut central_directory) {
visitor.visit_additional_metadata(&metadata)?;
}
}
Ok(())
}
pub fn extract<P: AsRef<Path>>(self, directory: P) -> ZipResult<()> {
struct Extractor {
outpath: PathBuf,
resolved_paths: IndexMap<Box<[u8]>, PathBuf>,
#[cfg(unix)]
file_permissions: std::collections::BTreeMap<PathBuf, u32>,
}
impl ZipStreamVisitor for Extractor {
fn visit_file<R: Read>(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> {
let extract_options = crate::read::extract::ExtractOptions::default();
let mut outpath = self.outpath.clone();
file.safe_prepare_path(
&self.outpath,
&mut outpath,
None::<&(_, fn(&Path) -> bool)>,
)?;
self.resolved_paths
.insert(file.name_raw().into(), outpath.clone());
if file.is_symlink() {
let declared_len = file.size();
if declared_len > MAX_SYMLINK_TARGET_LEN {
return Err(invalid!(
"symlink target declares {} bytes, more than the {} byte limit",
declared_len,
MAX_SYMLINK_TARGET_LEN
));
}
let mut target = Vec::with_capacity(declared_len as usize);
file.read_to_end(&mut target)?;
match extract_options.symlink_action {
SymlinkExtractAction::ExtractAsFile => {
make_symlink_as_file(&outpath, &target)?;
}
SymlinkExtractAction::ExtractAsFileUtf8 => {
make_symlink_as_file_utf8(&outpath, &target)?;
}
_ => {
make_symlink(
&self.outpath,
&outpath,
&target,
&self.resolved_paths,
extract_options.symlink_action,
)?;
}
}
return Ok(());
}
if file.is_dir() {
fs::create_dir_all(&outpath)?;
} else {
let mut outfile = fs::File::create(&outpath)?;
io::copy(file, &mut outfile)?;
}
Ok(())
}
#[allow(unused)]
fn visit_additional_metadata(&mut self, metadata: &ZipFileEntry<'_>) -> ZipResult<()> {
#[cfg(unix)]
if let (Some(outpath), Some(mode)) = (
self.resolved_paths.get(metadata.name_raw()),
metadata.unix_mode(),
) {
self.file_permissions.insert(outpath.clone(), mode);
}
Ok(())
}
}
use std::fs;
fs::create_dir_all(&directory)?;
let directory = directory.as_ref().canonicalize()?;
let mut extractor = Extractor {
outpath: directory,
resolved_paths: IndexMap::new(),
#[cfg(unix)]
file_permissions: Default::default(),
};
self.visit(&mut extractor)?;
#[cfg(unix)]
{
for (path, mode) in extractor.file_permissions.into_iter().rev() {
use std::os::unix::fs::PermissionsExt;
crate::read::extract::set_permissions_no_follow_symlink(
&path,
std::fs::Permissions::from_mode(mode),
)?;
}
}
Ok(())
}
}
pub trait ZipStreamVisitor {
fn visit_file<R: Read>(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()>;
fn visit_additional_metadata(&mut self, metadata: &ZipFileEntry<'_>) -> ZipResult<()>;
}
pub fn read_zipfile_from_stream<R: Read>(reader: &mut R) -> ZipResult<Option<ZipFile<'_, R>>> {
read_zipfile_from_stream_with_options(reader, ZipReadOptions::default())
}
pub fn read_zipfile_from_stream_with_compressed_size<R: Read>(
reader: &mut R,
compressed_size: u64,
) -> ZipResult<Option<ZipFile<'_, R>>> {
let options = ZipReadOptions::default().override_compressed_size(compressed_size);
read_zipfile_from_stream_with_options(reader, options)
}
pub fn read_zipfile_from_stream_with_options<'a, R: Read>(
reader: &'a mut R,
options: ZipReadOptions<'a>,
) -> ZipResult<Option<ZipFile<'a, R>>> {
let result = read_zipfile_from_stream_with_options_internal(reader, options)?;
match result {
ReadFromStreamResult::File(file) => Ok(Some(file)),
ReadFromStreamResult::CentralDirectoryHeader => Ok(None),
ReadFromStreamResult::EndOfCentralDirectory => Ok(None),
}
}
enum ReadFromStreamResult<'a, R: Read> {
File(ZipFile<'a, R>),
CentralDirectoryHeader,
EndOfCentralDirectory,
}
fn read_zipfile_from_stream_with_options_internal<'a, R: Read>(
reader: &'a mut R,
mut options: ZipReadOptions<'a>,
) -> ZipResult<ReadFromStreamResult<'a, R>> {
let mut magic_buf = [0; size_of::<u32>()];
reader.read_exact(&mut magic_buf)?;
match Magic::from_le_bytes(magic_buf) {
Magic::LOCAL_FILE_HEADER_SIGNATURE => (),
Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE => return Ok(CentralDirectoryHeader),
Magic::CENTRAL_DIRECTORY_END_SIGNATURE => return Ok(EndOfCentralDirectory), _ => return Err(ZipLocalEntryBlock::WRONG_MAGIC_ERROR),
}
let mut block = ZipLocalEntryBlock::zeroed();
reader.read_exact(block.as_bytes_mut())?;
let block = block.from_le();
let file_name_length: usize = block.file_name_length.into();
let mut file_name_raw = vec![0u8; file_name_length];
if let Err(e) = reader.read_exact(&mut file_name_raw) {
if e.kind() == std::io::ErrorKind::UnexpectedEof {
return Err(invalid!("File name extends beyond file boundary"));
}
return Err(e.into());
}
let extra_field_length: usize = block.extra_field_length.into();
let mut extra_fields_raw = vec![0u8; extra_field_length];
if let Err(e) = reader.read_exact(&mut extra_fields_raw) {
if e.kind() == std::io::ErrorKind::UnexpectedEof {
return Err(invalid!("Extra field extends beyond file boundary"));
}
return Err(e.into());
}
let extra_fields = ExtraFields::parse(&extra_fields_raw, &block, true)?;
let mut data = ZipFileData::from_local_block(block, extra_fields)?;
data.apply_extra_fields(&mut file_name_raw)?;
if data.is_using_data_descriptor() {
if let Some(comp_size) = options.force_compressed_size {
data.compressed_size = comp_size;
} else {
return Err(ZipError::UnsupportedArchive(
"The file length is not available in the local header",
));
}
}
if let Some(uncomp_size) = options.force_uncompressed_size {
data.uncompressed_size = uncomp_size;
} else if data.is_using_data_descriptor() {
options.size_unknown = true;
}
if let Some(crc) = options.force_crc {
data.crc32 = crc;
}
let limit_reader = reader.take(data.compressed_size);
let reader = data.make_reader(limit_reader, options)?;
Ok(File(ZipFile {
file_name_raw: Cow::Owned(file_name_raw),
data: Cow::Owned(data),
reader,
}))
}
#[cfg(test)]
mod tests {
use crate::ZipWriter;
use crate::read::ZipFile;
use crate::read::ZipFileEntry;
use crate::read::stream::{ZipStreamReader, ZipStreamVisitor};
use crate::result::ZipResult;
use std::collections::BTreeSet;
use std::io::{Cursor, Read};
#[test]
fn zip_read_streaming_visitor() {
let reader =
ZipStreamReader::new(Cursor::new(include_bytes!("../../tests/data/mimetype.zip")));
#[derive(Default)]
struct V {
filenames: BTreeSet<Box<str>>,
}
impl ZipStreamVisitor for V {
fn visit_file<R: Read>(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> {
if file.is_file() {
let file_name = file.name().unwrap();
self.filenames.insert(file_name.into());
}
Ok(())
}
fn visit_additional_metadata(
&mut self,
zip_file_metadata: &ZipFileEntry<'_>,
) -> ZipResult<()> {
if zip_file_metadata.is_file() {
let file_name = zip_file_metadata.name().unwrap();
assert!(
self.filenames.contains::<str>(file_name.as_ref()),
"{} is missing its file content",
file_name
);
}
Ok(())
}
}
reader.visit(&mut V::default()).unwrap();
}
#[test]
fn file_and_dir_predicates() {
let reader = ZipStreamReader::new(Cursor::new(include_bytes!(
"../../tests/data/files_and_dirs.zip"
)));
#[derive(Default)]
struct V {
filenames: BTreeSet<Box<str>>,
}
impl ZipStreamVisitor for V {
fn visit_file<R: Read>(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> {
let full_name = file.enclosed_name().unwrap();
let file_name = full_name.file_name().unwrap().to_str().unwrap();
assert!(
(file_name.starts_with("dir") && file.is_dir())
|| (file_name.starts_with("file") && file.is_file())
);
if file.is_file() {
let file_name = file.name().unwrap();
self.filenames.insert(file_name.into());
}
Ok(())
}
fn visit_additional_metadata(
&mut self,
zip_file_metadata: &ZipFileEntry<'_>,
) -> ZipResult<()> {
if zip_file_metadata.is_file() {
let file_name = zip_file_metadata.name().unwrap();
assert!(
self.filenames.contains::<str>(file_name.as_ref()),
"{} is missing its file content",
file_name
);
}
Ok(())
}
}
reader.visit(&mut V::default()).unwrap();
}
#[cfg(not(miri))]
#[test]
fn test_cannot_symlink_outside_destination() -> ZipResult<()> {
use crate::ZipWriter;
use crate::write::SimpleFileOptions;
use std::fs::create_dir;
use tempfile::TempDir;
let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
writer.add_symlink("symlink/", "../dest-sibling/", SimpleFileOptions::default())?;
writer.start_file("symlink/dest-file", SimpleFileOptions::default())?;
let reader = ZipStreamReader::new(writer.finish()?);
let dest_parent = TempDir::with_prefix("stream__cannot_symlink_outside_destination")?;
let dest_sibling = dest_parent.path().join("dest-sibling");
create_dir(&dest_sibling)?;
let dest = dest_parent.path().join("dest");
create_dir(&dest)?;
assert!(reader.extract(dest).is_err());
assert!(!dest_sibling.join("dest-file").exists());
Ok(())
}
#[test]
fn stream_does_not_expose_the_symlink_bit() -> ZipResult<()> {
use crate::ZipWriter;
use crate::write::SimpleFileOptions;
let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
writer.add_symlink("link", "/target", SimpleFileOptions::default())?;
let mut cur = writer.finish()?;
cur.set_position(0);
let file = crate::read::read_zipfile_from_stream(&mut cur)?.unwrap();
assert!(
!file.is_symlink(),
"the stream path now sees the symlink bit -- the MAX_SYMLINK_TARGET_LEN \
guard in `stream.rs` is reachable and needs a regression test"
);
Ok(())
}
#[cfg(not(miri))]
#[test]
fn test_can_create_destination() -> ZipResult<()> {
use tempfile::TempDir;
let v = include_bytes!("../../tests/data/mimetype.zip");
let reader = ZipStreamReader::new(v.as_ref());
let dest = TempDir::with_prefix("stream_test_can_create_destination").unwrap();
reader.extract(&dest)?;
assert!(dest.path().join("mimetype").exists());
Ok(())
}
#[test]
fn zip_read_streaming() {
use super::read_zipfile_from_stream;
let mut reader = Cursor::new(include_bytes!("../../tests/data/mimetype.zip"));
loop {
if read_zipfile_from_stream(&mut reader).unwrap().is_none() {
break;
}
}
}
#[test]
#[cfg(feature = "deflate")]
fn zip_read_streaming_compressed() {
use super::read_zipfile_from_stream_with_compressed_size;
use crate::ZipWriter;
use crate::write::SimpleFileOptions;
use std::io::Write;
let compression_method = crate::CompressionMethod::Deflated;
let options = SimpleFileOptions::default()
.compression_method(compression_method)
.unix_permissions(0o755);
let mut bytes = Vec::new();
let mut writer = ZipWriter::new(std::io::Cursor::new(&mut bytes));
writer.start_file("file.txt", options).unwrap();
write!(&mut writer, "{}", "test-".repeat(100)).unwrap();
writer.finish().unwrap();
let compressed_size = u32::from_le_bytes(bytes[18..22].try_into().unwrap());
let uncompressed_size = u32::from_le_bytes(bytes[22..26].try_into().unwrap());
assert_eq!(compressed_size, 14);
assert_eq!(uncompressed_size as usize, "test-".len() * 100);
let mut reader = Cursor::new(bytes);
loop {
if read_zipfile_from_stream_with_compressed_size(
&mut reader,
u64::from(compressed_size),
)
.unwrap()
.is_none()
{
break;
}
}
}
#[test]
#[cfg(feature = "aes-crypto")]
fn zip_read_streaming_compressed_and_aes() {
use super::read_zipfile_from_stream_with_options;
use crate::ZipReadOptions;
let bytes = include_bytes!("../../tests/data/aes_archive.zip");
let compressed_size = 46;
let mut reader = Cursor::new(bytes);
const PASSWORD: &[u8] = b"helloworld";
let options = ZipReadOptions::new()
.password(Some(PASSWORD))
.override_compressed_size(compressed_size);
let result = read_zipfile_from_stream_with_options(&mut reader, options);
let optional_file = result.unwrap();
let mut file = optional_file.unwrap();
let file_name = file.name().unwrap();
assert_eq!(file_name, "secret_data_128");
const SECRET_CONTENT: &str = "Lorem ipsum dolor sit amet";
let mut decrypted_content = String::new();
file.read_to_string(&mut decrypted_content)
.expect("couldn't read encrypted file");
assert_eq!(SECRET_CONTENT, decrypted_content);
}
#[test]
#[cfg(feature = "aes-crypto")]
fn zip_read_streaming_compressed_and_aes_without_size() {
use super::read_zipfile_from_stream_with_options;
use crate::ZipReadOptions;
let bytes = include_bytes!("../../tests/data/aes_archive.zip");
let mut reader = Cursor::new(bytes);
const PASSWORD: &[u8] = b"helloworld";
let options = ZipReadOptions::new().password(Some(PASSWORD));
let result = read_zipfile_from_stream_with_options(&mut reader, options);
let optional_file = result.unwrap();
let mut file = optional_file.unwrap();
let file_name = file.name().unwrap();
assert_eq!(file_name, "secret_data_128");
const SECRET_CONTENT: &str = "Lorem ipsum dolor sit amet";
let mut decrypted_content = String::new();
file.read_to_string(&mut decrypted_content)
.expect("couldn't read encrypted file");
assert_eq!(SECRET_CONTENT, decrypted_content);
}
#[test]
fn zip_read_streaming_zipwriter() {
use crate::CompressionMethod;
use crate::ZipReadOptions;
use crate::ZipWriter;
use crate::read::read_zipfile_from_stream;
use crate::read::read_zipfile_from_stream_with_compressed_size;
use crate::read::read_zipfile_from_stream_with_options;
use crate::write::SimpleFileOptions;
use std::io::Write;
let mut buffer = Vec::new();
let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored);
let mut archive = ZipWriter::new_stream(Cursor::new(&mut buffer));
archive.start_file("name", options).unwrap();
archive.write_all(b"test").unwrap();
let compressed_size = b"test".len() as u64;
archive.finish().unwrap();
{
let mut reader = Cursor::new(buffer.clone());
let result = read_zipfile_from_stream(&mut reader);
assert!(result.is_err());
}
{
let mut reader = Cursor::new(buffer.clone());
let result =
read_zipfile_from_stream_with_compressed_size(&mut reader, compressed_size);
let optional_file = result.unwrap();
let mut file = optional_file.unwrap();
let file_name = file.name().unwrap();
assert_eq!(file_name, "name");
let mut content = Vec::new();
let read_result = file.read_to_end(&mut content);
assert!(read_result.is_err());
let error_str = read_result.unwrap_err().to_string();
assert!(error_str.contains("Invalid checksum"));
}
{
let mut reader = Cursor::new(buffer.clone());
let read_options = ZipReadOptions::default()
.override_compressed_size(compressed_size)
.override_crc(3632233996);
let result = read_zipfile_from_stream_with_options(&mut reader, read_options);
let optional_file = result.unwrap();
let mut file = optional_file.unwrap();
let file_name = file.name().unwrap();
assert_eq!(file_name, "name");
let mut content = Vec::new();
file.read_to_end(&mut content).unwrap();
assert_eq!(content, b"test");
}
{
let mut reader = Cursor::new(buffer.clone());
let read_options = ZipReadOptions::default()
.override_compressed_size(compressed_size)
.ignore_crc32(true);
let result = read_zipfile_from_stream_with_options(&mut reader, read_options);
let optional_file = result.unwrap();
let mut file = optional_file.unwrap();
let file_name = file.name().unwrap();
assert_eq!(file_name, "name");
let mut content = Vec::new();
let read_result = file.read_to_end(&mut content);
assert!(read_result.is_ok());
assert_eq!(content, b"test");
}
}
#[test]
fn test_write_and_reopen_empty_archive() -> ZipResult<()> {
use crate::read::read_zipfile_from_stream;
use std::io::{Seek, SeekFrom};
let writer = ZipWriter::new(Cursor::new(Vec::new()));
let mut stream = writer.finish()?;
stream.seek(SeekFrom::Start(0))?;
assert!(read_zipfile_from_stream(&mut stream)?.is_none());
Ok(())
}
#[test]
fn test_write_and_reopen_empty_archive_with_comment() -> ZipResult<()> {
use crate::read::read_zipfile_from_stream;
use std::io::{Seek, SeekFrom};
let mut writer = ZipWriter::new(Cursor::new(Vec::new()));
writer.set_raw_comment(Box::new([255, 255]))?;
let mut writer = ZipWriter::new_append(writer.finish()?)?;
writer.set_raw_comment(Box::new([213, 255, 127, 255, 255, 255, 255, 0, 255, 7]))?;
writer.set_raw_comment(Box::new([0]))?;
let mut stream = writer.finish()?;
stream.seek(SeekFrom::Start(0))?;
assert!(read_zipfile_from_stream(&mut stream)?.is_none());
Ok(())
}
}