Skip to main content

unityrs_core/
zip.rs

1// Ported from CPython 3.11 (PSF-2.0): Lib/zipfile.py. See THIRD_PARTY_LICENSES.
2//! ZIP indexing and stored/deflate reads, following `CPython`'s central-directory,
3//! ZIP64, local-name and CRC rules. All buffers and work are bounded.
4
5use crate::budget::DecodeBuf;
6use crate::error::{Error, LimitError, LimitKind};
7use crate::limits::{Limits, Meter, MeteredVec, heap_cost};
8use crate::raw::RawStr;
9use bytes::Bytes;
10use flate2::{Crc, Decompress, FlushDecompress, Status};
11use std::sync::Arc;
12
13mod index;
14use index::{directory, extra, item, sort, span, u16le, u32le};
15
16/// One member in central-directory order.
17#[derive(Debug)]
18pub struct Member {
19    /// Decoded member name (UTF-8 or CP437), truncated at NUL.
20    pub name: RawStr,
21    original: RawStr,
22    offset: i128,
23    end: i128,
24    compressed: u64,
25    size: u64,
26    crc: u32,
27    method: u16,
28    flags: u16,
29}
30
31/// Validated ZIP index. Its bytes, names and tables keep their charges.
32#[derive(Debug)]
33pub struct ZipArchive {
34    data: Bytes,
35    members: MeteredVec<Member>,
36    meter: Arc<Meter>,
37    names: MeteredVec<usize>,
38}
39
40const _: () = crate::send_sync::<Member>();
41const _: () = crate::send_sync::<ZipArchive>();
42
43pub(crate) fn bad(detail: &'static str) -> Error {
44    Error::Malformed {
45        what: "zip",
46        detail: detail.into(),
47    }
48}
49
50impl ZipArchive {
51    /// Index the central directory. Local headers are checked when read,
52    /// as ZipFile.open does (unused duplicate rows need not be readable).
53    pub fn open(data: Bytes, meter: &Arc<Meter>) -> Result<Self, Error> {
54        let (start, stop, concat) = directory(&data, meter)?;
55        let mut at = start;
56        let mut members = MeteredVec::new(meter);
57        while at < stop {
58            members.push(parse_member(&data, &mut at, stop, concat, meter)?)?;
59        }
60        let mut names = MeteredVec::new(meter);
61        for i in 0..members.len() {
62            names.push(i)?;
63        }
64        // CPython assigns each entry the next header's offset in reverse
65        // stable order, including equal offsets (only the first can be read).
66        sort(names.as_mut_slice(), |a, b| {
67            meter.count_decode(1)?;
68            Ok(item(members.as_slice(), *a)?
69                .offset
70                .cmp(&item(members.as_slice(), *b)?.offset)
71                .then(b.cmp(a)))
72        })?;
73        let mut end = start as i128;
74        for &i in names.as_slice().iter().rev() {
75            let m = members
76                .as_mut_slice()
77                .get_mut(i)
78                .ok_or_else(|| bad("member index"))?;
79            m.end = end;
80            end = m.offset;
81        }
82        sort(names.as_mut_slice(), |a, b| {
83            let (a_name, b_name) = (
84                &item(members.as_slice(), *a)?.name,
85                &item(members.as_slice(), *b)?.name,
86            );
87            meter.count_decode(a_name.len().min(b_name.len()) as u64 + 1)?;
88            Ok(a_name.cmp(b_name).then(a.cmp(b)))
89        })?;
90        Ok(Self {
91            data,
92            members,
93            meter: meter.clone(),
94            names,
95        })
96    }
97
98    /// Members in central-directory order, duplicates included.
99    #[must_use]
100    pub fn members(&self) -> &[Member] {
101        self.members.as_slice()
102    }
103
104    /// Last central-directory member of this name, as ZipFile.open(name).
105    /// Comparisons consume work and may refuse the search.
106    pub fn find(&self, name: &[u8]) -> Result<Option<usize>, Error> {
107        let (mut lo, mut hi) = (0, self.names.len());
108        while lo < hi {
109            let mid = lo + (hi - lo) / 2;
110            let m = item(self.members.as_slice(), *item(self.names.as_slice(), mid)?)?;
111            self.meter
112                .count_decode(m.name.len().min(name.len()) as u64 + 1)?;
113            if m.name.as_bytes() <= name {
114                lo = mid + 1;
115            } else {
116                hi = mid;
117            }
118        }
119        let Some(i) = lo
120            .checked_sub(1)
121            .and_then(|i| self.names.as_slice().get(i))
122            .copied()
123        else {
124            return Ok(None);
125        };
126        let m = item(self.members.as_slice(), i)?;
127        self.meter
128            .count_decode(m.name.len().min(name.len()) as u64 + 1)?;
129        Ok((m.name.as_bytes() == name).then_some(i))
130    }
131
132    fn input(&self, m: &Member) -> Result<&[u8], Error> {
133        let offset = usize::try_from(m.offset).map_err(|_| bad("bad local offset"))?;
134        let h = span(&self.data, offset, 30)?;
135        if h.get(..4) != Some(b"PK\x03\x04") {
136            return Err(bad("bad local signature"));
137        }
138        let n = usize::from(u16le(h, 26)?);
139        let x = usize::from(u16le(h, 28)?);
140        let raw = span(
141            &self.data,
142            offset + 30,
143            n.min(self.data.len() - offset - 30),
144        )?;
145        self.meter.count_decode((30 + n + x) as u64)?;
146        let local = member_name(raw, u16le(h, 6)?, &self.meter)?;
147        if local != m.original {
148            return Err(bad("local name mismatch"));
149        }
150        // Local extra bytes are skipped, never parsed. Local sizes, flags,
151        // method, CRC and any data descriptor do not determine the payload.
152        let begin = offset
153            .checked_add(30 + n + x)
154            .ok_or_else(|| bad("member overflow"))?
155            .min(self.data.len());
156        if begin as i128 + i128::from(m.compressed) > m.end {
157            return Err(bad("overlapping members"));
158        }
159        if m.flags & 0x61 != 0 {
160            return Err(bad("encrypted or patched ZIP is unsupported"));
161        }
162        if !matches!(m.method, 0 | 8) {
163            return Err(bad("unsupported compression method"));
164        }
165        let compressed = usize::try_from(m.compressed).unwrap_or(usize::MAX);
166        span(&self.data, begin, compressed.min(self.data.len() - begin))
167    }
168
169    /// Loading opens every member, but a zero-size stream's Length/seek does
170    /// not read payload or check CRC. Keep all ZipFile.open metadata checks.
171    pub(crate) fn read_for_load(&self, index: usize, limits: &Limits) -> Result<Bytes, Error> {
172        let m = self
173            .members
174            .as_slice()
175            .get(index)
176            .ok_or_else(|| bad("member index"))?;
177        if m.size == 0 {
178            self.input(m)?;
179            return Ok(Bytes::new());
180        }
181        self.read(index, limits)
182    }
183
184    /// Read the central compressed extent, truncating output to `file_size` as
185    /// `ZipExtFile` does. CRC at EOF decides validity, including short streams.
186    /// Trailing compressed bytes and a larger declared size are permitted.
187    pub fn read(&self, index: usize, limits: &Limits) -> Result<Bytes, Error> {
188        let m = self
189            .members
190            .as_slice()
191            .get(index)
192            .ok_or_else(|| bad("member index"))?;
193        let input = self.input(m)?;
194        // Never allocate from an untrusted declared count. DecodeBuf charges
195        // every later growth before allocation; empty payloads allocate nothing.
196        let mut out = DecodeBuf::new(
197            &self.meter,
198            0,
199            m.size.min(limits.max_inflate),
200            LimitKind::Inflate,
201        )?;
202        let short = (input.len() as u64) < m.compressed;
203        if input.is_empty() && short {
204            return Err(bad("unexpected EOF"));
205        }
206        if m.method == 0 {
207            self.meter.count_decode(input.len() as u64)?;
208            let bytes = input
209                .get(
210                    ..input
211                        .len()
212                        .min(usize::try_from(m.size).unwrap_or(usize::MAX)),
213                )
214                .unwrap_or_default();
215            self.meter.count_decode(bytes.len() as u64)?;
216            out.extend_from_slice(bytes)?;
217            if short && (out.len() as u64) < m.size {
218                return Err(bad("unexpected EOF"));
219            }
220        } else {
221            inflate(input, &mut out, m.size, short, limits, &self.meter)?;
222        }
223        let mut crc = Crc::new();
224        crc.update(out.as_slice());
225        if crc.sum() != m.crc {
226            return Err(bad("Bad CRC-32"));
227        }
228        Ok(out.into_bytes()?)
229    }
230}
231
232/// flate2 1.1.10's zlib-rs backend allocates one block containing decoder
233/// state, a 32 KiB window, SIMD padding and alignment. Reserve a conservative
234/// 64 KiB (plus allocator overhead) before construction; `zip_alloc` measures
235/// construction and decoding with the testkit's counting allocator.
236const DEFLATE_HEAP: u64 = heap_cost(64 * 1024);
237
238/// `UnityPy` asks its stream reader for Length first: `ZipExtFile.seek` reads
239/// through the declared size in `MAX_SEEK_READ` chunks. Each read uses
240/// `_read1`/`_read2`, with at least `MIN_READ_SIZE` compressed and output bytes.
241/// Finish that logical output window before applying `_left`, even when it
242/// discards output or encounters corruption beyond the retained prefix.
243fn inflate(
244    input: &[u8],
245    out: &mut DecodeBuf,
246    size: u64,
247    short: bool,
248    limits: &Limits,
249    meter: &Arc<Meter>,
250) -> Result<(), Error> {
251    const MIN_READ_SIZE: usize = 4096;
252    const MAX_SEEK_READ: u64 = 1 << 24;
253    // Declaration order matters: the decoder drops before its reservation.
254    let _backend = meter.charge(DEFLATE_HEAP)?;
255    let mut dec = Decompress::new(false);
256    let (mut pos, mut end) = (0_usize, 0_usize);
257    let mut scratch = [0; 8192];
258    let mut seek_end = size.min(MAX_SEEK_READ);
259    let mut read_left = seek_end;
260    loop {
261        let n = usize::try_from(read_left).map_err(|_| bad("inflate read size"))?;
262        let tail = end - pos;
263        let next = if n > tail {
264            input
265                .len()
266                .min(end.saturating_add((n - tail).max(MIN_READ_SIZE)))
267        } else {
268            end
269        };
270        // An initial zero-sized member still gets one minimum window to
271        // validate its stream and CRC, without reserving output storage.
272        let next = if end == 0 {
273            input.len().min(n.max(MIN_READ_SIZE))
274        } else {
275            next
276        };
277        let max_length = n.max(MIN_READ_SIZE);
278        meter.count_decode((next - end) as u64)?;
279        end = next;
280        let mut window = 0;
281        loop {
282            // When all compressed bytes have been consumed, CPython flushes
283            // buffered output, even past max_length, before slicing to _left.
284            let flushing = pos == end && end == input.len() && !short;
285            let room = if flushing {
286                scratch.len()
287            } else {
288                scratch.len().min(max_length - window)
289            };
290            let before = (dec.total_in(), dec.total_out());
291            let status = dec.decompress(
292                input.get(pos..end).unwrap_or_default(),
293                scratch
294                    .get_mut(..room)
295                    .ok_or_else(|| bad("inflate window"))?,
296                FlushDecompress::None,
297            );
298            let used = usize::try_from(dec.total_in() - before.0)
299                .map_err(|_| bad("inflate input overflow"))?;
300            let wrote = usize::try_from(dec.total_out() - before.1)
301                .map_err(|_| bad("inflate output overflow"))?;
302            pos += used;
303            window += wrote;
304            meter.count_decode(wrote as u64)?;
305            if dec.total_out() > limits.max_inflate {
306                return Err(Error::Limit(LimitError::refused(
307                    LimitKind::Inflate,
308                    dec.total_out(),
309                    limits.max_inflate,
310                )));
311            }
312            let status = status.map_err(|_| bad("invalid deflate stream"))?;
313            let keep = wrote.min(usize::try_from(size - out.len() as u64).unwrap_or(usize::MAX));
314            out.extend_from_slice(scratch.get(..keep).ok_or_else(|| bad("inflate overflow"))?)?;
315            if status == Status::StreamEnd {
316                return Ok(());
317            }
318            if used == 0 && wrote == 0 {
319                if pos == end {
320                    if end == input.len() {
321                        if short && (out.len() as u64) < size {
322                            return Err(bad("unexpected EOF"));
323                        }
324                        return Ok(());
325                    }
326                    break;
327                }
328                return Err(bad("invalid deflate stream"));
329            }
330            if window >= max_length && !(pos == end && end == input.len() && !short) {
331                break;
332            }
333        }
334        // _left is applied only after _read1 has completed its window.
335        if (out.len() as u64) >= size {
336            return Ok(());
337        }
338        if out.len() as u64 >= seek_end {
339            seek_end = seek_end.saturating_add(MAX_SEEK_READ).min(size);
340        }
341        // read() buffers _read1's excess beyond its requested length. It
342        // satisfies the start of the next seek chunk without decoding again.
343        // All decoded bytes already live in out; keep the seek boundary
344        // independent of that buffer's length so the excess carries forward.
345        read_left = seek_end - out.len() as u64;
346    }
347}
348
349fn parse_member(
350    data: &[u8],
351    at: &mut usize,
352    stop: usize,
353    concat: i128,
354    meter: &Arc<Meter>,
355) -> Result<Member, Error> {
356    let h = span(data, *at, 46)?;
357    if *at + 46 > stop || h.get(..4) != Some(b"PK\x01\x02") {
358        return Err(bad("bad central signature"));
359    }
360    if h.get(6).is_some_and(|&version| version > 63) {
361        return Err(bad("unsupported ZIP version"));
362    }
363    let flags = u16le(h, 8)?;
364    let n = usize::from(u16le(h, 28)?);
365    let x = usize::from(u16le(h, 30)?);
366    let c = usize::from(u16le(h, 32)?);
367    meter.count_decode((46 + n + x + c) as u64)?;
368    let raw = data
369        .get(*at + 46..(*at + 46 + n).min(stop))
370        .ok_or_else(|| bad("short name"))?;
371    let original = member_name(raw, flags, meter)?;
372    let name = RawStr::charged(
373        original
374            .as_bytes()
375            .split(|&b| b == 0)
376            .next()
377            .unwrap_or_default(),
378        meter,
379    )?;
380    let mut values = [
381        u64::from(u32le(h, 24)?),
382        u64::from(u32le(h, 20)?),
383        u64::from(u32le(h, 42)?),
384    ];
385    let extra_start = (*at + 46 + n).min(stop);
386    extra(
387        span(data, extra_start, x.min(stop - extra_start))?,
388        &mut values,
389    )?;
390    *at += 46 + n + x + c;
391    let offset = i128::from(values[2]) + concat;
392    Ok(Member {
393        name,
394        original,
395        offset,
396        end: stop as i128,
397        compressed: values[1],
398        size: values[0],
399        crc: u32le(h, 16)?,
400        method: u16le(h, 10)?,
401        flags,
402    })
403}
404
405fn member_name(raw: &[u8], flags: u16, meter: &Arc<Meter>) -> Result<RawStr, Error> {
406    if flags & 0x800 != 0 {
407        std::str::from_utf8(raw).map_err(|_| bad("invalid UTF-8 member name"))?;
408        return Ok(RawStr::charged(raw, meter)?);
409    }
410    let mut out = DecodeBuf::new(meter, 0, raw.len() as u64 * 3, LimitKind::InputMemory)?;
411    for &b in raw {
412        if b < 128 {
413            out.extend_from_slice(&[b])?;
414        } else {
415            let c = CP437
416                .chars()
417                .nth(usize::from(b - 128))
418                .ok_or_else(|| bad("CP437 name"))?;
419            out.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes())?;
420        }
421    }
422    Ok(RawStr::from_shared(out.into_bytes()?))
423}
424
425// IBM PC code page 437's upper half, in byte order.
426const CP437: &str = "ÇüéâäàåçêëèïîìÄÅÉæÆôöòûùÿÖÜ¢£¥₧ƒáíóúñѪº¿⌐¬½¼¡«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ ";