use std::collections::{BTreeMap, BTreeSet};
use std::io::{Read, Seek, SeekFrom};
use std::path::Path;
use super::READ_MAX_ENTRIES;
pub const LIST_FILE: &str = "read.tsv";
pub const ARCHIVE_FILE: &str = "read.zip";
pub const STAGING_FREE_MARGIN: u64 = 256 * 1024 * 1024;
const S_IFMT: u32 = 0o170_000;
const S_IFREG: u32 = 0o100_000;
const HOST_DOS: u8 = 0;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BundleFile {
pub collection: String,
pub rel: String,
pub size: u64,
}
impl BundleFile {
pub fn name(&self) -> String {
format!("{}/{}", self.collection, self.rel)
}
}
pub fn render_list(files: &[BundleFile]) -> Vec<u8> {
let mut out = Vec::new();
for file in files {
out.extend_from_slice(file.size.to_string().as_bytes());
out.push(b'\t');
out.extend_from_slice(file.name().as_bytes());
out.push(b'\n');
}
out
}
pub fn archive_limit(files: &[BundleFile]) -> u64 {
let total: u64 = files.iter().map(|f| f.size).sum();
let per_entry: u64 = files.iter().map(|f| 2 * f.name().len() as u64 + 192).sum();
total + total / 64 + per_entry + 4096
}
struct Record {
name: Vec<u8>,
host: u8,
external: u32,
local_offset: u64,
}
fn name_problem(raw: &[u8]) -> Option<&'static str> {
let Ok(name) = std::str::from_utf8(raw) else {
return Some("is not UTF-8");
};
if name.contains('\\') {
Some("holds a backslash")
} else if name.starts_with('/') {
Some("is an absolute path")
} else if name.ends_with('/') {
Some("is a directory entry")
} else if name.contains(':') {
Some("names a drive or a stream")
} else if name
.split('/')
.any(|c| c.is_empty() || c == "." || c == "..")
{
Some("has an empty, `.` or `..` component")
} else {
None
}
}
fn shown(raw: &[u8]) -> String {
format!("{:?}", String::from_utf8_lossy(raw))
}
fn central_records(
file: &mut (impl Read + Seek),
start: u64,
len: u64,
) -> Result<Vec<Record>, String> {
let unreadable = |e: std::io::Error| format!("the read archive is not a readable zip ({e})");
let cut_off =
|| "the read archive is not a readable zip (a central record is cut off)".to_string();
if start > len {
return Err(
"the read archive is not a readable zip (its central directory is outside it)"
.to_string(),
);
}
file.seek(SeekFrom::Start(start)).map_err(unreadable)?;
let mut reader = std::io::BufReader::with_capacity(64 * 1024, file.take(len - start));
let mut records = Vec::new();
let mut header = [0u8; 46];
let mut tail = Vec::new();
loop {
match reader.read_exact(&mut header[..4]) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
Err(e) => return Err(unreadable(e)),
}
if header[..4] != [0x50, 0x4b, 0x01, 0x02] {
break;
}
if records.len() > READ_MAX_ENTRIES {
return Err(format!(
"the read archive holds more than {READ_MAX_ENTRIES} entries"
));
}
reader.read_exact(&mut header[4..]).map_err(|e| {
if e.kind() == std::io::ErrorKind::UnexpectedEof {
cut_off()
} else {
unreadable(e)
}
})?;
let le16 = |at: usize| usize::from(u16::from_le_bytes([header[at], header[at + 1]]));
let le32 = |at: usize| {
u32::from_le_bytes([header[at], header[at + 1], header[at + 2], header[at + 3]])
};
let (name_len, extra_len) = (le16(28), le16(30));
tail.resize(name_len + extra_len + le16(32), 0);
reader.read_exact(&mut tail).map_err(|e| {
if e.kind() == std::io::ErrorKind::UnexpectedEof {
cut_off()
} else {
unreadable(e)
}
})?;
let mut local_offset = u64::from(le32(42));
if local_offset == u64::from(u32::MAX) {
local_offset = zip64_offset(&tail[name_len..name_len + extra_len], le32(24), le32(20))
.ok_or("the read archive is not a readable zip (a Zip64 record is malformed)")?;
}
records.push(Record {
name: tail[..name_len].to_vec(),
host: header[5],
external: le32(38),
local_offset,
});
}
Ok(records)
}
fn zip64_offset(extra: &[u8], uncompressed: u32, compressed: u32) -> Option<u64> {
let mut at = 0usize;
while at + 4 <= extra.len() {
let id = u16::from_le_bytes([extra[at], extra[at + 1]]);
let size = usize::from(u16::from_le_bytes([extra[at + 2], extra[at + 3]]));
let body = extra.get(at + 4..at + 4 + size)?;
if id == 0x0001 {
let mut skip = 0usize;
if uncompressed == u32::MAX {
skip += 8;
}
if compressed == u32::MAX {
skip += 8;
}
let value = body.get(skip..skip + 8)?;
return Some(u64::from_le_bytes(value.try_into().ok()?));
}
at += 4 + size;
}
None
}
pub fn extract(
archive: &Path,
files: &[BundleFile],
dest: &Path,
slack: u64,
) -> Result<u64, String> {
let opened = std::fs::File::open(archive)
.map_err(|e| format!("the read archive cannot be opened ({:?})", e.kind()))?;
let len = opened
.metadata()
.map_err(|e| format!("the read archive cannot be inspected ({:?})", e.kind()))?
.len();
let mut walk = opened
.try_clone()
.map_err(|e| format!("the read archive cannot be opened ({:?})", e.kind()))?;
let mut zip = zip::ZipArchive::new(opened)
.map_err(|e| format!("the read archive is not a readable zip ({e})"))?;
if zip.offset() != 0 {
return Err("the read archive has data before its first entry".to_string());
}
let records = central_records(&mut walk, zip.central_directory_start(), len)?;
let mut seen: BTreeSet<&[u8]> = BTreeSet::new();
for record in &records {
if !seen.insert(&record.name) {
return Err(format!(
"the read archive names {} twice",
shown(&record.name)
));
}
}
if records.len() != zip.len() {
return Err(format!(
"the read archive's central directory holds {} records for {} entries; an entry is named twice",
records.len(),
zip.len()
));
}
if records.iter().map(|r| r.local_offset).min().unwrap_or(0) != 0 {
return Err("the read archive has data before its first entry".to_string());
}
let by_name: BTreeMap<&[u8], &Record> =
records.iter().map(|r| (r.name.as_slice(), r)).collect();
let mut names: Vec<Vec<u8>> = Vec::with_capacity(zip.len());
for index in 0..zip.len() {
let entry = zip
.by_index_raw(index)
.map_err(|e| format!("the read archive is not a readable zip ({e})"))?;
let raw = entry.name_raw().to_vec();
if let Some(why) = name_problem(&raw) {
return Err(format!("the read archive entry {} {why}", shown(&raw)));
}
let record = by_name.get(raw.as_slice()).ok_or_else(|| {
format!(
"the read archive entry {} is not in its central directory",
shown(&raw)
)
})?;
let regular = match entry.unix_mode() {
Some(mode) => mode & S_IFMT == S_IFREG,
None => record.host == HOST_DOS && record.external == 0,
};
if !regular {
return Err(format!(
"the read archive entry {} is not a regular file",
shown(&raw)
));
}
if entry.encrypted() {
return Err(format!(
"the read archive entry {} is encrypted",
shown(&raw)
));
}
if !matches!(
entry.compression(),
zip::CompressionMethod::Stored | zip::CompressionMethod::Deflated
) {
return Err(format!(
"the read archive entry {} uses an unsupported compression method",
shown(&raw)
));
}
names.push(raw);
}
let listed: BTreeMap<String, &BundleFile> = files.iter().map(|f| (f.name(), f)).collect();
for raw in &names {
let name = std::str::from_utf8(raw).unwrap_or_default();
if !listed.contains_key(name) {
return Err(format!(
"the read archive holds {}, which was not listed",
shown(raw)
));
}
}
let present: BTreeSet<&str> = names
.iter()
.map(|raw| std::str::from_utf8(raw).unwrap_or_default())
.collect();
for file in files {
if !present.contains(file.name().as_str()) {
return Err(format!(
"config/resources/{} was listed but is not in the read archive",
file.name()
));
}
}
for (index, raw) in names.iter().enumerate() {
let name = std::str::from_utf8(raw).unwrap_or_default();
let file = listed[name];
let declared = zip
.by_index_raw(index)
.map_err(|e| format!("the read archive is not a readable zip ({e})"))?
.size();
if declared != file.size {
return Err(format!(
"config/resources/{name} is {declared} bytes in the read archive but was listed as {}; it changed while it was read",
file.size
));
}
}
let listed_total: u64 = files.iter().map(|f| f.size).sum();
let limit = listed_total.saturating_add(slack);
let mut written = 0u64;
for (index, raw) in names.iter().enumerate() {
let name = std::str::from_utf8(raw).unwrap_or_default();
let file = listed[name];
let target = file
.rel
.split('/')
.fold(dest.join(&file.collection), |p, c| p.join(c));
if let Some(parent) = target.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| format!("cannot create a download dir ({:?})", e.kind()))?;
}
let mut out = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&target)
.map_err(|e| format!("cannot write config/resources/{name} ({:?})", e.kind()))?;
let entry = zip
.by_index(index)
.map_err(|e| format!("config/resources/{name} cannot be extracted ({e})"))?;
let copied = copy_bounded(entry.take(file.size + 1), &mut out).map_err(|e| match e {
Copy::Read(e) => format!("config/resources/{name} cannot be extracted ({e})"),
Copy::Write(kind) => format!("cannot write config/resources/{name} ({kind:?})"),
})?;
if copied > file.size {
return Err(format!(
"config/resources/{name} inflates past its listed {} bytes",
file.size
));
}
if copied != file.size {
return Err(format!(
"config/resources/{name} is {copied} bytes but was listed as {}; it changed while it was read",
file.size
));
}
written += copied;
if written > limit {
return Err(format!(
"the read archive inflates past the {limit} bytes listed for it"
));
}
}
Ok(written)
}
enum Copy {
Read(std::io::Error),
Write(std::io::ErrorKind),
}
fn copy_bounded(mut from: impl Read, to: &mut impl std::io::Write) -> Result<u64, Copy> {
let mut buf = vec![0u8; 64 * 1024];
let mut total = 0u64;
loop {
let n = match from.read(&mut buf) {
Ok(0) => return Ok(total),
Ok(n) => n,
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
Err(e) => return Err(Copy::Read(e)),
};
to.write_all(&buf[..n]).map_err(|e| Copy::Write(e.kind()))?;
total += n as u64;
}
}
pub fn record_attributes(archive: &Path) -> Result<Vec<(String, u8, u32)>, String> {
let opened = std::fs::File::open(archive)
.map_err(|e| format!("the read archive cannot be opened ({:?})", e.kind()))?;
let len = opened
.metadata()
.map_err(|e| format!("the read archive cannot be inspected ({:?})", e.kind()))?
.len();
let mut walk = opened
.try_clone()
.map_err(|e| format!("the read archive cannot be opened ({:?})", e.kind()))?;
let zip = zip::ZipArchive::new(opened)
.map_err(|e| format!("the read archive is not a readable zip ({e})"))?;
Ok(
central_records(&mut walk, zip.central_directory_start(), len)?
.into_iter()
.map(|r| {
(
String::from_utf8_lossy(&r.name).into_owned(),
r.host,
r.external,
)
})
.collect(),
)
}
#[cfg(test)]
mod tests {
use super::*;
fn file(collection: &str, rel: &str, size: u64) -> BundleFile {
BundleFile {
collection: collection.to_string(),
rel: rel.to_string(),
size,
}
}
#[test]
fn render_list_is_size_tab_name_lf_in_order() {
let files = [
file("core", "ignition/a b,1/config.json", 10),
file("production", "ignition/caf\u{e9}.json", 0),
file("core", "ignition/b.json", 12_345_678_901),
];
assert_eq!(
render_list(&files),
"10\tcore/ignition/a b,1/config.json\n0\tproduction/ignition/caf\u{e9}.json\n12345678901\tcore/ignition/b.json\n"
.as_bytes()
);
assert!(render_list(&[]).is_empty());
}
#[test]
fn archive_limit_arithmetic() {
assert_eq!(archive_limit(&[]), 4096);
assert_eq!(archive_limit(&[file("core", "a/b.js", 6400)]), 10_810);
let two = [file("core", "a", 64), file("mode", "bc", 0)];
assert_eq!(archive_limit(&two), 64 + 1 + 204 + 206 + 4096);
}
#[test]
fn name_rules() {
for (name, want) in [
(&b"core/a\\b"[..], Some("holds a backslash")),
(b"/core/a", Some("is an absolute path")),
(b"core/a/", Some("is a directory entry")),
(b"C:/core/a", Some("names a drive or a stream")),
(b"core/a:s", Some("names a drive or a stream")),
(b"core//a", Some("has an empty, `.` or `..` component")),
(b"core/./a", Some("has an empty, `.` or `..` component")),
(b"core/../a", Some("has an empty, `.` or `..` component")),
(b"", Some("has an empty, `.` or `..` component")),
(b"core/\xff", Some("is not UTF-8")),
(b"core/caf\xc3\xa9/a b,1.json", None),
] {
assert_eq!(name_problem(name), want, "{}", shown(name));
}
}
#[test]
fn central_walk_counts_a_crafted_duplicate() {
use std::io::Write as _;
let mut writer = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
let options = zip::write::SimpleFileOptions::default()
.compression_method(zip::CompressionMethod::Deflated);
for name in ["core/a.json", "core/b.json"] {
writer.start_file(name, options).expect("entry");
writer.write_all(b"{}").expect("write");
}
let mut bytes = writer.finish().expect("finish").into_inner();
let from = b"core/b.json";
let mut at = 0;
while let Some(found) = bytes[at..].windows(from.len()).position(|w| w == from) {
bytes[at + found..at + found + from.len()].copy_from_slice(b"core/a.json");
at += found + from.len();
}
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("dup.zip");
std::fs::write(&path, &bytes).expect("write");
let mut opened = std::fs::File::open(&path).expect("open");
let zip =
zip::ZipArchive::new(opened.try_clone().expect("clone")).expect("the crate opens it");
let records = central_records(
&mut opened,
zip.central_directory_start(),
bytes.len() as u64,
)
.expect("walk");
assert_eq!(records.len(), 2, "both records are seen");
assert_eq!(zip.len(), 1, "the crate keeps one of them");
let why = extract(
&path,
&[file("core", "a.json", 2)],
&dir.path().join("out"),
1024,
)
.expect_err("refused");
assert!(why.contains("twice"), "{why}");
}
struct Counting<R> {
inner: R,
read: u64,
}
impl<R: Read> Read for Counting<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.inner.read(buf)?;
self.read += n as u64;
Ok(n)
}
}
impl<R: Seek> Seek for Counting<R> {
fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
self.inner.seek(pos)
}
}
fn crafted_records(count: usize) -> Vec<u8> {
let mut out = Vec::new();
for i in 0..count {
let name = format!("core/{i}.json");
let mut record = [0u8; 46];
record[..4].copy_from_slice(&[0x50, 0x4b, 0x01, 0x02]);
record[5] = 3;
record[28..30].copy_from_slice(&(name.len() as u16).to_le_bytes());
record[38..42].copy_from_slice(&(0o100_644u32 << 16).to_le_bytes());
out.extend_from_slice(&record);
out.extend_from_slice(name.as_bytes());
}
out
}
#[test]
fn central_walk_reads_only_the_records() {
let mut bytes = crafted_records(3);
let records_len = bytes.len() as u64;
bytes.extend(std::iter::repeat_n(0u8, 8 << 20));
let len = bytes.len() as u64;
let mut reader = Counting {
inner: std::io::Cursor::new(bytes),
read: 0,
};
let records = central_records(&mut reader, 0, len).expect("walk");
assert_eq!(records.len(), 3);
assert_eq!(records[2].name, b"core/2.json");
assert_eq!(records[2].host, 3);
assert!(
reader.read <= records_len + (256 << 10),
"the walk read {} bytes of a {len}-byte file",
reader.read
);
let mut bytes = crafted_records(READ_MAX_ENTRIES + 1);
let len = bytes.len() as u64;
let walked = central_records(&mut std::io::Cursor::new(&bytes), 0, len).expect("walk");
assert_eq!(walked.len(), READ_MAX_ENTRIES + 1);
bytes.extend(crafted_records(1));
let records_len = bytes.len() as u64;
bytes.extend(std::iter::repeat_n(0u8, 8 << 20));
let len = bytes.len() as u64;
let mut reader = Counting {
inner: std::io::Cursor::new(bytes),
read: 0,
};
let why = central_records(&mut reader, 0, len).err().expect("refused");
assert!(why.contains("more than 200000 entries"), "{why}");
assert!(
reader.read <= records_len + (256 << 10),
"the walk read {} bytes of a {len}-byte file",
reader.read
);
let bytes = crafted_records(2);
let cut = bytes.len() as u64 - 3;
let why = central_records(&mut std::io::Cursor::new(&bytes), 0, cut)
.err()
.expect("refused");
assert!(why.contains("a central record is cut off"), "{why}");
let why = central_records(&mut std::io::Cursor::new(&bytes), 0, 46 + 11 + 20)
.err()
.expect("refused");
assert!(why.contains("a central record is cut off"), "{why}");
}
#[test]
fn extract_round_trips_an_honest_archive() {
use std::io::Write as _;
let files = [
file("core", "ignition/a b,1/config.json", 5),
file("production", "ignition/caf\u{e9}/x.json", 0),
];
let mut writer = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
let options = zip::write::SimpleFileOptions::default()
.compression_method(zip::CompressionMethod::Deflated)
.system(zip::System::Unix);
for (f, body) in files.iter().zip([&b"hello"[..], b""]) {
writer.start_file(f.name(), options).expect("entry");
writer.write_all(body).expect("write");
}
let bytes = writer.finish().expect("finish").into_inner();
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("read.zip");
std::fs::write(&path, &bytes).expect("write");
assert!(bytes.len() as u64 <= archive_limit(&files));
let out = dir.path().join("out");
assert_eq!(extract(&path, &files, &out, 0).expect("extract"), 5);
assert_eq!(
std::fs::read(out.join("core/ignition/a b,1/config.json")).expect("read"),
b"hello"
);
assert_eq!(
std::fs::read(out.join("production/ignition/caf\u{e9}/x.json")).expect("read"),
b""
);
let attributes = record_attributes(&path).expect("attributes");
assert_eq!(attributes.len(), 2);
assert!(
attributes
.iter()
.all(|(_, host, ext)| *host == 3 && ext >> 16 == 0o100_644)
);
}
}