1use std::collections::{BTreeMap, BTreeSet};
41use std::io::{Read, Seek, SeekFrom};
42use std::path::Path;
43
44use super::READ_MAX_ENTRIES;
45
46pub const LIST_FILE: &str = "read.tsv";
48pub const ARCHIVE_FILE: &str = "read.zip";
50pub const STAGING_FREE_MARGIN: u64 = 256 * 1024 * 1024;
54
55const S_IFMT: u32 = 0o170_000;
57const S_IFREG: u32 = 0o100_000;
58const HOST_DOS: u8 = 0;
60
61#[derive(Debug, Clone, PartialEq, Eq)]
64pub struct BundleFile {
65 pub collection: String,
66 pub rel: String,
68 pub size: u64,
69}
70
71impl BundleFile {
72 pub fn name(&self) -> String {
74 format!("{}/{}", self.collection, self.rel)
75 }
76}
77
78pub fn render_list(files: &[BundleFile]) -> Vec<u8> {
82 let mut out = Vec::new();
83 for file in files {
84 out.extend_from_slice(file.size.to_string().as_bytes());
85 out.push(b'\t');
86 out.extend_from_slice(file.name().as_bytes());
87 out.push(b'\n');
88 }
89 out
90}
91
92pub fn archive_limit(files: &[BundleFile]) -> u64 {
103 let total: u64 = files.iter().map(|f| f.size).sum();
104 let per_entry: u64 = files.iter().map(|f| 2 * f.name().len() as u64 + 192).sum();
105 total + total / 64 + per_entry + 4096
106}
107
108struct Record {
110 name: Vec<u8>,
111 host: u8,
113 external: u32,
114 local_offset: u64,
115}
116
117fn name_problem(raw: &[u8]) -> Option<&'static str> {
119 let Ok(name) = std::str::from_utf8(raw) else {
120 return Some("is not UTF-8");
121 };
122 if name.contains('\\') {
123 Some("holds a backslash")
124 } else if name.starts_with('/') {
125 Some("is an absolute path")
126 } else if name.ends_with('/') {
127 Some("is a directory entry")
128 } else if name.contains(':') {
129 Some("names a drive or a stream")
130 } else if name
131 .split('/')
132 .any(|c| c.is_empty() || c == "." || c == "..")
133 {
134 Some("has an empty, `.` or `..` component")
135 } else {
136 None
137 }
138}
139
140fn shown(raw: &[u8]) -> String {
141 format!("{:?}", String::from_utf8_lossy(raw))
142}
143
144fn central_records(
151 file: &mut (impl Read + Seek),
152 start: u64,
153 len: u64,
154) -> Result<Vec<Record>, String> {
155 let unreadable = |e: std::io::Error| format!("the read archive is not a readable zip ({e})");
156 let cut_off =
157 || "the read archive is not a readable zip (a central record is cut off)".to_string();
158 if start > len {
159 return Err(
160 "the read archive is not a readable zip (its central directory is outside it)"
161 .to_string(),
162 );
163 }
164 file.seek(SeekFrom::Start(start)).map_err(unreadable)?;
165 let mut reader = std::io::BufReader::with_capacity(64 * 1024, file.take(len - start));
166 let mut records = Vec::new();
167 let mut header = [0u8; 46];
168 let mut tail = Vec::new();
169 loop {
170 match reader.read_exact(&mut header[..4]) {
173 Ok(()) => {}
174 Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
175 Err(e) => return Err(unreadable(e)),
176 }
177 if header[..4] != [0x50, 0x4b, 0x01, 0x02] {
178 break;
179 }
180 if records.len() > READ_MAX_ENTRIES {
181 return Err(format!(
182 "the read archive holds more than {READ_MAX_ENTRIES} entries"
183 ));
184 }
185 reader.read_exact(&mut header[4..]).map_err(|e| {
186 if e.kind() == std::io::ErrorKind::UnexpectedEof {
187 cut_off()
188 } else {
189 unreadable(e)
190 }
191 })?;
192 let le16 = |at: usize| usize::from(u16::from_le_bytes([header[at], header[at + 1]]));
193 let le32 = |at: usize| {
194 u32::from_le_bytes([header[at], header[at + 1], header[at + 2], header[at + 3]])
195 };
196 let (name_len, extra_len) = (le16(28), le16(30));
197 tail.resize(name_len + extra_len + le16(32), 0);
199 reader.read_exact(&mut tail).map_err(|e| {
200 if e.kind() == std::io::ErrorKind::UnexpectedEof {
201 cut_off()
202 } else {
203 unreadable(e)
204 }
205 })?;
206 let mut local_offset = u64::from(le32(42));
207 if local_offset == u64::from(u32::MAX) {
208 local_offset = zip64_offset(&tail[name_len..name_len + extra_len], le32(24), le32(20))
210 .ok_or("the read archive is not a readable zip (a Zip64 record is malformed)")?;
211 }
212 records.push(Record {
213 name: tail[..name_len].to_vec(),
214 host: header[5],
215 external: le32(38),
216 local_offset,
217 });
218 }
219 Ok(records)
220}
221
222fn zip64_offset(extra: &[u8], uncompressed: u32, compressed: u32) -> Option<u64> {
227 let mut at = 0usize;
228 while at + 4 <= extra.len() {
229 let id = u16::from_le_bytes([extra[at], extra[at + 1]]);
230 let size = usize::from(u16::from_le_bytes([extra[at + 2], extra[at + 3]]));
231 let body = extra.get(at + 4..at + 4 + size)?;
232 if id == 0x0001 {
233 let mut skip = 0usize;
234 if uncompressed == u32::MAX {
235 skip += 8;
236 }
237 if compressed == u32::MAX {
238 skip += 8;
239 }
240 let value = body.get(skip..skip + 8)?;
241 return Some(u64::from_le_bytes(value.try_into().ok()?));
242 }
243 at += 4 + size;
244 }
245 None
246}
247
248pub fn extract(
253 archive: &Path,
254 files: &[BundleFile],
255 dest: &Path,
256 slack: u64,
257) -> Result<u64, String> {
258 let opened = std::fs::File::open(archive)
259 .map_err(|e| format!("the read archive cannot be opened ({:?})", e.kind()))?;
260 let len = opened
261 .metadata()
262 .map_err(|e| format!("the read archive cannot be inspected ({:?})", e.kind()))?
263 .len();
264 let mut walk = opened
265 .try_clone()
266 .map_err(|e| format!("the read archive cannot be opened ({:?})", e.kind()))?;
267 let mut zip = zip::ZipArchive::new(opened)
268 .map_err(|e| format!("the read archive is not a readable zip ({e})"))?;
269
270 if zip.offset() != 0 {
272 return Err("the read archive has data before its first entry".to_string());
273 }
274 let records = central_records(&mut walk, zip.central_directory_start(), len)?;
275 let mut seen: BTreeSet<&[u8]> = BTreeSet::new();
276 for record in &records {
277 if !seen.insert(&record.name) {
278 return Err(format!(
279 "the read archive names {} twice",
280 shown(&record.name)
281 ));
282 }
283 }
284 if records.len() != zip.len() {
285 return Err(format!(
286 "the read archive's central directory holds {} records for {} entries; an entry is named twice",
287 records.len(),
288 zip.len()
289 ));
290 }
291 if records.iter().map(|r| r.local_offset).min().unwrap_or(0) != 0 {
292 return Err("the read archive has data before its first entry".to_string());
293 }
294 let by_name: BTreeMap<&[u8], &Record> =
295 records.iter().map(|r| (r.name.as_slice(), r)).collect();
296
297 let mut names: Vec<Vec<u8>> = Vec::with_capacity(zip.len());
299 for index in 0..zip.len() {
300 let entry = zip
301 .by_index_raw(index)
302 .map_err(|e| format!("the read archive is not a readable zip ({e})"))?;
303 let raw = entry.name_raw().to_vec();
304 if let Some(why) = name_problem(&raw) {
305 return Err(format!("the read archive entry {} {why}", shown(&raw)));
306 }
307 let record = by_name.get(raw.as_slice()).ok_or_else(|| {
308 format!(
309 "the read archive entry {} is not in its central directory",
310 shown(&raw)
311 )
312 })?;
313 let regular = match entry.unix_mode() {
314 Some(mode) => mode & S_IFMT == S_IFREG,
315 None => record.host == HOST_DOS && record.external == 0,
318 };
319 if !regular {
320 return Err(format!(
321 "the read archive entry {} is not a regular file",
322 shown(&raw)
323 ));
324 }
325 if entry.encrypted() {
326 return Err(format!(
327 "the read archive entry {} is encrypted",
328 shown(&raw)
329 ));
330 }
331 if !matches!(
332 entry.compression(),
333 zip::CompressionMethod::Stored | zip::CompressionMethod::Deflated
334 ) {
335 return Err(format!(
336 "the read archive entry {} uses an unsupported compression method",
337 shown(&raw)
338 ));
339 }
340 names.push(raw);
341 }
342
343 let listed: BTreeMap<String, &BundleFile> = files.iter().map(|f| (f.name(), f)).collect();
345 for raw in &names {
346 let name = std::str::from_utf8(raw).unwrap_or_default();
347 if !listed.contains_key(name) {
348 return Err(format!(
349 "the read archive holds {}, which was not listed",
350 shown(raw)
351 ));
352 }
353 }
354 let present: BTreeSet<&str> = names
355 .iter()
356 .map(|raw| std::str::from_utf8(raw).unwrap_or_default())
357 .collect();
358 for file in files {
359 if !present.contains(file.name().as_str()) {
360 return Err(format!(
361 "config/resources/{} was listed but is not in the read archive",
362 file.name()
363 ));
364 }
365 }
366
367 for (index, raw) in names.iter().enumerate() {
369 let name = std::str::from_utf8(raw).unwrap_or_default();
370 let file = listed[name];
371 let declared = zip
372 .by_index_raw(index)
373 .map_err(|e| format!("the read archive is not a readable zip ({e})"))?
374 .size();
375 if declared != file.size {
376 return Err(format!(
377 "config/resources/{name} is {declared} bytes in the read archive but was listed as {}; it changed while it was read",
378 file.size
379 ));
380 }
381 }
382
383 let listed_total: u64 = files.iter().map(|f| f.size).sum();
385 let limit = listed_total.saturating_add(slack);
386 let mut written = 0u64;
387 for (index, raw) in names.iter().enumerate() {
388 let name = std::str::from_utf8(raw).unwrap_or_default();
389 let file = listed[name];
390 let target = file
391 .rel
392 .split('/')
393 .fold(dest.join(&file.collection), |p, c| p.join(c));
394 if let Some(parent) = target.parent() {
395 std::fs::create_dir_all(parent)
396 .map_err(|e| format!("cannot create a download dir ({:?})", e.kind()))?;
397 }
398 let mut out = std::fs::OpenOptions::new()
399 .write(true)
400 .create_new(true)
401 .open(&target)
402 .map_err(|e| format!("cannot write config/resources/{name} ({:?})", e.kind()))?;
403 let entry = zip
404 .by_index(index)
405 .map_err(|e| format!("config/resources/{name} cannot be extracted ({e})"))?;
406 let copied = copy_bounded(entry.take(file.size + 1), &mut out).map_err(|e| match e {
407 Copy::Read(e) => format!("config/resources/{name} cannot be extracted ({e})"),
408 Copy::Write(kind) => format!("cannot write config/resources/{name} ({kind:?})"),
409 })?;
410 if copied > file.size {
411 return Err(format!(
412 "config/resources/{name} inflates past its listed {} bytes",
413 file.size
414 ));
415 }
416 if copied != file.size {
417 return Err(format!(
418 "config/resources/{name} is {copied} bytes but was listed as {}; it changed while it was read",
419 file.size
420 ));
421 }
422 written += copied;
423 if written > limit {
424 return Err(format!(
425 "the read archive inflates past the {limit} bytes listed for it"
426 ));
427 }
428 }
429 Ok(written)
430}
431
432enum Copy {
434 Read(std::io::Error),
435 Write(std::io::ErrorKind),
436}
437
438fn copy_bounded(mut from: impl Read, to: &mut impl std::io::Write) -> Result<u64, Copy> {
440 let mut buf = vec![0u8; 64 * 1024];
441 let mut total = 0u64;
442 loop {
443 let n = match from.read(&mut buf) {
444 Ok(0) => return Ok(total),
445 Ok(n) => n,
446 Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
447 Err(e) => return Err(Copy::Read(e)),
448 };
449 to.write_all(&buf[..n]).map_err(|e| Copy::Write(e.kind()))?;
450 total += n as u64;
451 }
452}
453
454pub fn record_attributes(archive: &Path) -> Result<Vec<(String, u8, u32)>, String> {
458 let opened = std::fs::File::open(archive)
459 .map_err(|e| format!("the read archive cannot be opened ({:?})", e.kind()))?;
460 let len = opened
461 .metadata()
462 .map_err(|e| format!("the read archive cannot be inspected ({:?})", e.kind()))?
463 .len();
464 let mut walk = opened
465 .try_clone()
466 .map_err(|e| format!("the read archive cannot be opened ({:?})", e.kind()))?;
467 let zip = zip::ZipArchive::new(opened)
468 .map_err(|e| format!("the read archive is not a readable zip ({e})"))?;
469 Ok(
470 central_records(&mut walk, zip.central_directory_start(), len)?
471 .into_iter()
472 .map(|r| {
473 (
474 String::from_utf8_lossy(&r.name).into_owned(),
475 r.host,
476 r.external,
477 )
478 })
479 .collect(),
480 )
481}
482
483#[cfg(test)]
484mod tests {
485 use super::*;
486
487 fn file(collection: &str, rel: &str, size: u64) -> BundleFile {
488 BundleFile {
489 collection: collection.to_string(),
490 rel: rel.to_string(),
491 size,
492 }
493 }
494
495 #[test]
496 fn render_list_is_size_tab_name_lf_in_order() {
497 let files = [
498 file("core", "ignition/a b,1/config.json", 10),
499 file("production", "ignition/caf\u{e9}.json", 0),
500 file("core", "ignition/b.json", 12_345_678_901),
501 ];
502 assert_eq!(
503 render_list(&files),
504 "10\tcore/ignition/a b,1/config.json\n0\tproduction/ignition/caf\u{e9}.json\n12345678901\tcore/ignition/b.json\n"
505 .as_bytes()
506 );
507 assert!(render_list(&[]).is_empty());
508 }
509
510 #[test]
511 fn archive_limit_arithmetic() {
512 assert_eq!(archive_limit(&[]), 4096);
513 assert_eq!(archive_limit(&[file("core", "a/b.js", 6400)]), 10_810);
515 let two = [file("core", "a", 64), file("mode", "bc", 0)];
516 assert_eq!(archive_limit(&two), 64 + 1 + 204 + 206 + 4096);
518 }
519
520 #[test]
521 fn name_rules() {
522 for (name, want) in [
523 (&b"core/a\\b"[..], Some("holds a backslash")),
524 (b"/core/a", Some("is an absolute path")),
525 (b"core/a/", Some("is a directory entry")),
526 (b"C:/core/a", Some("names a drive or a stream")),
527 (b"core/a:s", Some("names a drive or a stream")),
528 (b"core//a", Some("has an empty, `.` or `..` component")),
529 (b"core/./a", Some("has an empty, `.` or `..` component")),
530 (b"core/../a", Some("has an empty, `.` or `..` component")),
531 (b"", Some("has an empty, `.` or `..` component")),
532 (b"core/\xff", Some("is not UTF-8")),
533 (b"core/caf\xc3\xa9/a b,1.json", None),
534 ] {
535 assert_eq!(name_problem(name), want, "{}", shown(name));
536 }
537 }
538
539 #[test]
542 fn central_walk_counts_a_crafted_duplicate() {
543 use std::io::Write as _;
544 let mut writer = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
545 let options = zip::write::SimpleFileOptions::default()
546 .compression_method(zip::CompressionMethod::Deflated);
547 for name in ["core/a.json", "core/b.json"] {
548 writer.start_file(name, options).expect("entry");
549 writer.write_all(b"{}").expect("write");
550 }
551 let mut bytes = writer.finish().expect("finish").into_inner();
552 let from = b"core/b.json";
554 let mut at = 0;
555 while let Some(found) = bytes[at..].windows(from.len()).position(|w| w == from) {
556 bytes[at + found..at + found + from.len()].copy_from_slice(b"core/a.json");
557 at += found + from.len();
558 }
559 let dir = tempfile::tempdir().expect("tempdir");
560 let path = dir.path().join("dup.zip");
561 std::fs::write(&path, &bytes).expect("write");
562 let mut opened = std::fs::File::open(&path).expect("open");
563 let zip =
564 zip::ZipArchive::new(opened.try_clone().expect("clone")).expect("the crate opens it");
565 let records = central_records(
566 &mut opened,
567 zip.central_directory_start(),
568 bytes.len() as u64,
569 )
570 .expect("walk");
571 assert_eq!(records.len(), 2, "both records are seen");
572 assert_eq!(zip.len(), 1, "the crate keeps one of them");
573 let why = extract(
574 &path,
575 &[file("core", "a.json", 2)],
576 &dir.path().join("out"),
577 1024,
578 )
579 .expect_err("refused");
580 assert!(why.contains("twice"), "{why}");
581 }
582
583 struct Counting<R> {
585 inner: R,
586 read: u64,
587 }
588
589 impl<R: Read> Read for Counting<R> {
590 fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
591 let n = self.inner.read(buf)?;
592 self.read += n as u64;
593 Ok(n)
594 }
595 }
596
597 impl<R: Seek> Seek for Counting<R> {
598 fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
599 self.inner.seek(pos)
600 }
601 }
602
603 fn crafted_records(count: usize) -> Vec<u8> {
605 let mut out = Vec::new();
606 for i in 0..count {
607 let name = format!("core/{i}.json");
608 let mut record = [0u8; 46];
609 record[..4].copy_from_slice(&[0x50, 0x4b, 0x01, 0x02]);
610 record[5] = 3;
611 record[28..30].copy_from_slice(&(name.len() as u16).to_le_bytes());
612 record[38..42].copy_from_slice(&(0o100_644u32 << 16).to_le_bytes());
613 out.extend_from_slice(&record);
614 out.extend_from_slice(name.as_bytes());
615 }
616 out
617 }
618
619 #[test]
624 fn central_walk_reads_only_the_records() {
625 let mut bytes = crafted_records(3);
626 let records_len = bytes.len() as u64;
627 bytes.extend(std::iter::repeat_n(0u8, 8 << 20));
628 let len = bytes.len() as u64;
629 let mut reader = Counting {
630 inner: std::io::Cursor::new(bytes),
631 read: 0,
632 };
633 let records = central_records(&mut reader, 0, len).expect("walk");
634 assert_eq!(records.len(), 3);
635 assert_eq!(records[2].name, b"core/2.json");
636 assert_eq!(records[2].host, 3);
637 assert!(
638 reader.read <= records_len + (256 << 10),
639 "the walk read {} bytes of a {len}-byte file",
640 reader.read
641 );
642
643 let mut bytes = crafted_records(READ_MAX_ENTRIES + 1);
646 let len = bytes.len() as u64;
647 let walked = central_records(&mut std::io::Cursor::new(&bytes), 0, len).expect("walk");
648 assert_eq!(walked.len(), READ_MAX_ENTRIES + 1);
649 bytes.extend(crafted_records(1));
650 let records_len = bytes.len() as u64;
651 bytes.extend(std::iter::repeat_n(0u8, 8 << 20));
652 let len = bytes.len() as u64;
653 let mut reader = Counting {
654 inner: std::io::Cursor::new(bytes),
655 read: 0,
656 };
657 let why = central_records(&mut reader, 0, len).err().expect("refused");
658 assert!(why.contains("more than 200000 entries"), "{why}");
659 assert!(
660 reader.read <= records_len + (256 << 10),
661 "the walk read {} bytes of a {len}-byte file",
662 reader.read
663 );
664
665 let bytes = crafted_records(2);
667 let cut = bytes.len() as u64 - 3;
668 let why = central_records(&mut std::io::Cursor::new(&bytes), 0, cut)
669 .err()
670 .expect("refused");
671 assert!(why.contains("a central record is cut off"), "{why}");
672 let why = central_records(&mut std::io::Cursor::new(&bytes), 0, 46 + 11 + 20)
673 .err()
674 .expect("refused");
675 assert!(why.contains("a central record is cut off"), "{why}");
676 }
677
678 #[test]
683 fn extract_round_trips_an_honest_archive() {
684 use std::io::Write as _;
685 let files = [
686 file("core", "ignition/a b,1/config.json", 5),
687 file("production", "ignition/caf\u{e9}/x.json", 0),
688 ];
689 let mut writer = zip::ZipWriter::new(std::io::Cursor::new(Vec::new()));
690 let options = zip::write::SimpleFileOptions::default()
691 .compression_method(zip::CompressionMethod::Deflated)
692 .system(zip::System::Unix);
693 for (f, body) in files.iter().zip([&b"hello"[..], b""]) {
694 writer.start_file(f.name(), options).expect("entry");
695 writer.write_all(body).expect("write");
696 }
697 let bytes = writer.finish().expect("finish").into_inner();
698 let dir = tempfile::tempdir().expect("tempdir");
699 let path = dir.path().join("read.zip");
700 std::fs::write(&path, &bytes).expect("write");
701 assert!(bytes.len() as u64 <= archive_limit(&files));
702 let out = dir.path().join("out");
703 assert_eq!(extract(&path, &files, &out, 0).expect("extract"), 5);
704 assert_eq!(
705 std::fs::read(out.join("core/ignition/a b,1/config.json")).expect("read"),
706 b"hello"
707 );
708 assert_eq!(
709 std::fs::read(out.join("production/ignition/caf\u{e9}/x.json")).expect("read"),
710 b""
711 );
712 let attributes = record_attributes(&path).expect("attributes");
713 assert_eq!(attributes.len(), 2);
714 assert!(
715 attributes
716 .iter()
717 .all(|(_, host, ext)| *host == 3 && ext >> 16 == 0o100_644)
718 );
719 }
720}