Skip to main content

unityrs_core/stream/
kind.rs

1// Ported from UnityPy 1.25.2 (MIT): UnityPy/streams/EndianBinaryReader.py. See THIRD_PARTY_LICENSES.
2//! Which of UnityPy's two readers a file is read with, and the stream
3//! reader's rules (`EndianBinaryReader_Streamable`,
4//! `streams/EndianBinaryReader.py:287-324`).
5//!
6//! `EndianBinaryReader(item)` picks its class by the input's type (`:40-63`):
7//! bytes get the memory reader, a file object the stream reader. UnityPy's
8//! `Environment` opens a path as a file (`environment.py:136-139`, the
9//! buffered file `fs.open(path, "rb")` gives; a folder's files through the
10//! builtin `open`, `environment.py:75-77`), so a file loaded from a path is read by the
11//! stream reader: its header, metadata and objects, the pure-Python typetree
12//! reader's reads, and the slice the Boost reader is given
13//! (`helpers/TypeTreeHelper.py:146-153`), and, for a reader kept as it is
14//! (a `.resS`), the resource reads. What UnityPy reads from memory stays on
15//! the memory reader whatever the source: a bundle's entries (its blocks are
16//! decoded into bytes, `files/BundleFile.py:173-186`), a blob's string
17//! buffer (`helpers/TypeTreeNode.py:118`) and a bundle's blocks-info.
18//! `EndianBinaryReader_Streamable_LocalFile` (`streams/EndianBinaryReader.py:327-332`), the reader of a
19//! `str`, is reached by no load: `Environment` opens a path before it makes
20//! a reader.
21//!
22//! The stream reader's `read` is the file's own (`:295`), its `Position` the
23//! file's `tell()` (`:298-299`), and its fixed-width reads
24//! `struct.unpack(stream.read(n))` (`:345-346`). Over a file of `len` bytes,
25//! recorded from UnityPy 1.25.2 on a file `fs.open` opened
26//! (`tests/fixtures/scaffold/reader.json`: the `stream_` cases on a file of
27//! the scratch folder, the `stream_tmpfs_` cases on a tmpfs file, whose
28//! largest is 2^63 - 1):
29//! - `Position = p` refuses a negative `p` (`ValueError` "Position cannot be
30//!   negative", `:302-305`) and one at or above 2^63, which the file's
31//!   `seek` cannot take on any machine (`ValueError` "cannot fit 'int' into
32//!   an offset-sized integer", `CPython`'s `PyNumber_AsOff_t`); any other moves
33//!   there, past the end included, where the file system takes it. Past the
34//!   largest file a file system takes (ext4's is 2^44 - 4096 with 4 KiB
35//!   blocks; tmpfs's 2^63 - 1, the `stream_tmpfs_` cases) `seek` raises
36//!   `OSError` (`EINVAL`), which depends on the machine: this module does not
37//!   model it, and moves there (`PARITY.md` section 19.3). A stream position
38//!   is never negative.
39//! - `read(n)`: `n` below -1 raises (`ValueError` "read length must be
40//!   non-negative or -1"); -1 reads to the end and 0 nothing; any other `n`
41//!   reads the bytes that exist, and the position moves by the bytes read,
42//!   so it stops at the end, and stays where it is past the end. A buffered
43//!   file allocates `n` bytes before it reads
44//!   (`_bufferedreader_read_generic`), so `n` at or above 2^63 - 33 raises
45//!   `OverflowError` ("byte string is too large", from `_PyBytes_FromSize`)
46//!   on any machine; an `n` below that but above what the machine can
47//!   allocate raises `MemoryError`, which depends on the machine: this
48//!   module does not model it, and reads what exists (`PARITY.md` section
49//!   19.3). The fixture's sizes are those that raise on every machine or
50//!   that every machine allocates. At a position where the buffered file's
51//!   fill would end past 2^63 - 1 a read raises `OSError` (`EINVAL`) on
52//!   Linux, whose `rw_verify_area` refuses a read whose end overflows (the
53//!   `stream_tmpfs_read` cases at 2^63 - 1); this module reads nothing
54//!   there, as past the end anywhere.
55//! - a fixed-width read is `read(n)`, which raises unless it got `n` bytes,
56//!   the position moved by what it got either way.
57//! - `read_string_to_null(max)` is the base class's loop (`:136-144`): it
58//!   reads a byte at a time until a NUL, `max` bytes, or the end, and drops
59//!   the last byte it read when that byte is no NUL. So with no NUL in reach
60//!   it returns one byte fewer than it read; at the end, or with `max` 0
61//!   anywhere, it returns `""`; and only past the end with `max` above 0,
62//!   where its first read is empty, it raises (`ValueError` "Unterminated
63//!   string: [b'']").
64//! - `align_stream` is the memory reader's arithmetic (`:155-156`), the
65//!   position never negative, assigned through the `Position` setter: so a
66//!   position it takes to 2^63 or more raises as that `seek` does
67//!   ([`ReaderKind::align`]).
68//!
69//! The memory reader's rules are the free functions of [`super`].
70
71use std::ops::Range;
72
73use super::{py_align, py_read, py_string_to_null_found, py_string_window, py_unpack_from};
74
75/// Which of UnityPy's readers a file is read with (module docs).
76#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
77pub enum ReaderKind {
78    /// `EndianBinaryReader_Memoryview`: bytes. What UnityPy is handed as
79    /// bytes ([`crate::env::Source::Bytes`] and [`crate::env::Source::Fetch`]),
80    /// and every entry of a container.
81    #[default]
82    Memory,
83    /// `EndianBinaryReader_Streamable` over the buffered file `Environment`
84    /// opens for a path: a top-level [`crate::env::Source::Path`].
85    Stream,
86    /// Stream reader backed by `io.BytesIO`: any negative read size means EOF.
87    BytesIo,
88}
89
90/// An exception the stream reader raises where the memory reader reads on
91/// (module docs). Its text is UnityPy's message.
92#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, thiserror::Error)]
93pub enum StreamRefusal {
94    /// `Position = p` with `p < 0` (`ValueError`).
95    #[error("Position cannot be negative")]
96    NegativePosition,
97    /// `Position = p` with `p >= 2^63`: the file's `seek` takes a C offset
98    /// (`ValueError`).
99    #[error("cannot fit 'int' into an offset-sized integer")]
100    PositionTooLarge,
101    /// `read(n)` with `n < -1` (`ValueError`).
102    #[error("read length must be non-negative or -1")]
103    NegativeRead,
104    /// `read(n)` with `2^63 - 33 <= n < 2^63`: the bytes object `n` needs
105    /// (`OverflowError`).
106    #[error("byte string is too large")]
107    ReadTooLarge,
108    /// `read(n)` with `n >= 2^63`: a C size (`OverflowError`).
109    #[error("cannot fit 'int' into an index-sized integer")]
110    ReadSizeOverflow,
111}
112
113impl StreamRefusal {
114    /// UnityPy's exception class.
115    #[must_use]
116    pub fn exception(self) -> &'static str {
117        match self {
118            Self::NegativePosition | Self::PositionTooLarge | Self::NegativeRead => "ValueError",
119            Self::ReadTooLarge | Self::ReadSizeOverflow => "OverflowError",
120        }
121    }
122
123    /// The message, as UnityPy's exception carries it.
124    #[must_use]
125    pub fn message(self) -> &'static str {
126        match self {
127            Self::NegativePosition => "Position cannot be negative",
128            Self::PositionTooLarge => "cannot fit 'int' into an offset-sized integer",
129            Self::NegativeRead => "read length must be non-negative or -1",
130            Self::ReadTooLarge => "byte string is too large",
131            Self::ReadSizeOverflow => "cannot fit 'int' into an index-sized integer",
132        }
133    }
134}
135
136impl From<StreamRefusal> for crate::error::Error {
137    fn from(e: StreamRefusal) -> Self {
138        Self::Malformed {
139            what: STREAM_READER,
140            detail: e.message().into(),
141        }
142    }
143}
144
145const _: () = {
146    crate::send_sync::<ReaderKind>();
147    crate::send_sync::<StreamRefusal>();
148};
149
150/// The `what` of an [`crate::Error::Malformed`] a [`StreamRefusal`] becomes.
151pub const STREAM_READER: &str = "stream reader";
152
153/// The smallest `read(n)` a buffered file refuses for its bytes object on
154/// every machine (`_PyBytes_FromSize` refuses a size above
155/// `PY_SSIZE_T_MAX - PyBytesObject_SIZE`, `PyBytesObject_SIZE` being 33:
156/// the fixture records `2^63 - 33` as "byte string is too large"). Below
157/// it the read allocates first, so its outcome depends on the machine
158/// (module docs).
159const READ_TOO_LARGE: i64 = i64::MAX - 32;
160
161impl ReaderKind {
162    /// `Position = pos`: the position it leaves, or the stream reader's
163    /// refusal of a negative one or one at or above 2^63.
164    pub fn seek(self, pos: i128) -> Result<i128, StreamRefusal> {
165        match self {
166            Self::Stream | Self::BytesIo if pos < 0 => Err(StreamRefusal::NegativePosition),
167            Self::Stream | Self::BytesIo if pos > i128::from(i64::MAX) => {
168                Err(StreamRefusal::PositionTooLarge)
169            }
170            Self::Memory | Self::Stream | Self::BytesIo => Ok(pos),
171        }
172    }
173
174    /// `align_stream(n)` at `pos`: `Position += (n - Position % n) % n`
175    /// ([`py_align`]), an assignment through the `Position` setter
176    /// (`streams/EndianBinaryReader.py:155-156`), so the stream reader
177    /// refuses a result its `seek` cannot take, as [`Self::seek`] does: at
178    /// 2^63 or more (from a position within `n - 1` of 2^63 - 1).
179    pub fn align(self, pos: i128, n: u32) -> Result<i128, StreamRefusal> {
180        self.seek(py_align(pos, n))
181    }
182
183    /// `read(size)` at `pos` over `len` bytes: the bytes' range and the new
184    /// position. The memory reader's is [`py_read`]; the stream reader's
185    /// reads what exists from `pos` (never negative on a stream, as
186    /// [`Self::seek`] refuses that) and moves by what it read.
187    pub fn read(
188        self,
189        len: usize,
190        pos: i128,
191        size: i64,
192    ) -> Result<(Range<usize>, i128), StreamRefusal> {
193        match self {
194            Self::Memory => Ok(py_read(len, pos, size)),
195            Self::Stream => stream_read(len, pos, size),
196            Self::BytesIo => Ok(stream_read_range(len, pos, size.max(-1))),
197        }
198    }
199
200    /// `read(size)` for a size a caller holds as a `u64` (a resource's): on
201    /// the memory reader a size past `i64::MAX` slices as `i64::MAX` does,
202    /// and the stream reader refuses it.
203    pub fn read_u64(
204        self,
205        len: usize,
206        pos: i128,
207        size: u64,
208    ) -> Result<(Range<usize>, i128), StreamRefusal> {
209        match (i64::try_from(size), self) {
210            (Ok(size), _) => self.read(len, pos, size),
211            (Err(_), Self::Memory) => Ok(py_read(len, pos, i64::MAX)),
212            (Err(_), Self::Stream | Self::BytesIo) => Err(StreamRefusal::ReadSizeOverflow),
213        }
214    }
215
216    /// The reader's own fixed-width read of `n` bytes (`reader.read_int()`
217    /// and the like) at `pos` over `len` bytes: the start of its `n` bytes,
218    /// or `None` where it raises, and the position after it. The memory
219    /// reader's is `struct.unpack_from(view, Position)` ([`py_unpack_from`]),
220    /// which moves the position by `n` on success only; the stream reader's
221    /// is `struct.unpack(stream.read(n))`, which moves it by what it read.
222    #[must_use]
223    pub fn unpack(self, len: usize, pos: i128, n: usize) -> (Option<usize>, i128) {
224        let step = i128::try_from(n).unwrap_or(i128::MAX);
225        match self {
226            Self::Memory => match py_unpack_from(len, pos, n) {
227                Some(at) => (Some(at), pos.saturating_add(step)),
228                None => (None, pos),
229            },
230            Self::Stream | Self::BytesIo => {
231                let size = i64::try_from(n).unwrap_or(i64::MAX);
232                match stream_read(len, pos, size) {
233                    Ok((range, next)) if range.len() == n => (Some(range.start), next),
234                    Ok((_, next)) => (None, next),
235                    Err(_) => (None, pos),
236                }
237            }
238        }
239    }
240
241    /// `read_string_to_null(max_length)` at `pos` over `data`: the string's
242    /// range and the new position, or `None` where it raises (module docs:
243    /// the stream reader raises only past the end with `max_length` above
244    /// 0).
245    #[must_use]
246    pub fn string_to_null(
247        self,
248        data: &[u8],
249        pos: i128,
250        max_length: usize,
251    ) -> Option<(Range<usize>, i128)> {
252        let window = py_string_window(data.len(), pos, max_length);
253        let nul = data
254            .get(window.clone())
255            .and_then(|w| w.iter().position(|b| *b == 0))
256            .map(|k| window.start + k);
257        self.string_to_null_found(data.len(), pos, max_length, nul)
258    }
259
260    /// [`Self::string_to_null`]'s answer once its search of
261    /// [`py_string_window`]'s window is done: `nul` is the window's first
262    /// NUL, as an index of the whole buffer. The stream reader's loop reads
263    /// the same window (from a position that is never negative), so one
264    /// search serves both readers.
265    #[must_use]
266    pub(crate) fn string_to_null_found(
267        self,
268        len: usize,
269        pos: i128,
270        max_length: usize,
271        nul: Option<usize>,
272    ) -> Option<(Range<usize>, i128)> {
273        match self {
274            Self::Memory => py_string_to_null_found(len, pos, max_length, nul),
275            Self::Stream | Self::BytesIo => stream_string_to_null(len, pos, max_length, nul),
276        }
277    }
278}
279
280/// The stream reader's `read(size)` (module docs).
281fn stream_read(len: usize, pos: i128, size: i64) -> Result<(Range<usize>, i128), StreamRefusal> {
282    if size < -1 {
283        return Err(StreamRefusal::NegativeRead);
284    }
285    if size >= READ_TOO_LARGE {
286        return Err(StreamRefusal::ReadTooLarge);
287    }
288    Ok(stream_read_range(len, pos, size))
289}
290
291fn stream_read_range(len: usize, pos: i128, size: i64) -> (Range<usize>, i128) {
292    let l = i128::try_from(len).unwrap_or(i128::MAX);
293    let start = pos.clamp(0, l);
294    if size == 0 || pos >= l {
295        // Nothing to read: the position stays, past the end included.
296        let at = usize::try_from(start).unwrap_or(len);
297        return (at..at, pos);
298    }
299    let end = if size == -1 {
300        l
301    } else {
302        start.saturating_add(i128::from(size)).min(l)
303    };
304    let range = usize::try_from(start).unwrap_or(len)..usize::try_from(end).unwrap_or(len);
305    (range, end)
306}
307
308/// The stream reader's `read_string_to_null` (`:136-144`, module docs):
309/// `nul` is the first NUL in the window `pos..min(pos + max_length, len)`.
310fn stream_string_to_null(
311    len: usize,
312    pos: i128,
313    max_length: usize,
314    nul: Option<usize>,
315) -> Option<(Range<usize>, i128)> {
316    let l = i128::try_from(len).ok()?;
317    let start = usize::try_from(pos.clamp(0, l)).ok()?;
318    if max_length == 0 || pos == l {
319        // The loop never runs (`len(ret) < max_length` or `Position !=
320        // Length` is false before its first read): `""`, nothing read,
321        // wherever the position is.
322        return Some((start..start, pos));
323    }
324    if pos > l {
325        // The loop's first `read(1)` is empty: "Unterminated string: [b'']".
326        return None;
327    }
328    if let Some(nul) = nul {
329        // The NUL is read and dropped.
330        return Some((start..nul, i128::try_from(nul).ok()? + 1));
331    }
332    // No NUL in reach: the bytes read, less the last.
333    let read = usize::try_from(l - pos).ok()?.min(max_length);
334    let end = start.checked_add(read)?;
335    Some((start..end - 1, i128::try_from(end).ok()?))
336}
337
338#[cfg(test)]
339mod tests {
340    use super::{ReaderKind, StreamRefusal};
341
342    #[test]
343    fn the_memory_kind_is_the_free_functions() {
344        // `ReaderKind::Memory` delegates: a negative position and a
345        // negative size read as Python slices, and nothing is refused.
346        let m = ReaderKind::Memory;
347        assert_eq!(m.seek(-5), Ok(-5));
348        assert_eq!(m.read(10, 3, -2), Ok((3..3, 1)));
349        assert_eq!(m.unpack(10, -4, 4), (Some(6), 0));
350        assert_eq!(
351            m.read_u64(10, 2, u64::MAX),
352            Ok((2..10, 2 + i128::from(i64::MAX)))
353        );
354    }
355
356    #[test]
357    fn the_stream_kind_refuses_what_a_buffered_file_refuses() {
358        // What CPython refuses on every machine, each edge recorded from
359        // UnityPy 1.25.2 (`tests/fixtures/scaffold/reader.json`'s `stream_`
360        // and `stream_tmpfs_` cases, `tests/fixtures/d/paths.json`'s
361        // `resource-reads`), which `tests/reader.rs` and `tests/env_paths.rs`
362        // replay in full.
363        let s = ReaderKind::Stream;
364        assert_eq!(s.seek(-1), Err(StreamRefusal::NegativePosition));
365        assert_eq!(
366            s.seek(i128::from(i64::MAX) + 1),
367            Err(StreamRefusal::PositionTooLarge)
368        );
369        assert_eq!(s.read(10, 3, -2), Err(StreamRefusal::NegativeRead));
370        assert_eq!(
371            s.read(10, 3, i64::MAX - 32),
372            Err(StreamRefusal::ReadTooLarge)
373        );
374        assert_eq!(s.read(10, 20, i64::MAX), Err(StreamRefusal::ReadTooLarge));
375        assert_eq!(
376            s.read_u64(10, 0, 1 << 63),
377            Err(StreamRefusal::ReadSizeOverflow)
378        );
379        // `align_stream` assigns `Position`, so an alignment to 2^63 is the
380        // seek's refusal; the memory reader's is plain arithmetic.
381        let near = i128::from(i64::MAX) - 2;
382        assert_eq!(s.align(near, 4), Err(StreamRefusal::PositionTooLarge));
383        assert_eq!(s.align(near - 1, 4), Ok(near - 1));
384        assert_eq!(ReaderKind::Memory.align(near, 4), Ok(near + 3));
385        // Past the end a read is empty and the position stays.
386        assert_eq!(s.read(10, 15, -1), Ok((10..10, 15)));
387        assert_eq!(s.unpack(10, 8, 4), (None, 10));
388        assert_eq!(s.unpack(10, 12, 0), (Some(10), 12));
389        // With `max_length` 0 the string loop never runs, past the end too.
390        assert_eq!(s.string_to_null(b"abc", 7, 0), Some((3..3, 7)));
391        assert_eq!(s.string_to_null(b"abc", 7, 1), None);
392    }
393
394    #[test]
395    fn the_stream_kind_reads_on_where_the_machine_decides() {
396        // Not UnityPy's outcome on every machine: unityrs's model of the
397        // edges `PARITY.md` section 19.3 admits ("a path's stream past the
398        // machine's limits"), where it reads what exists.
399        let s = ReaderKind::Stream;
400        // A seek past the file system's largest file raises `OSError`
401        // (`EINVAL`) in UnityPy; on tmpfs, whose largest is 2^63 - 1, it
402        // moves there (`reader.json`'s `stream_tmpfs_set_position`).
403        assert_eq!(s.seek(i128::from(i64::MAX)), Ok(i128::from(i64::MAX)));
404        // A read the machine cannot allocate raises `MemoryError` in UnityPy
405        // (not recorded: a buffered file allocates `n` bytes first).
406        assert_eq!(s.read(10, 3, i64::MAX - 33), Ok((3..10, 10)));
407        // A read of 4 GiB reads what exists where the machine allocates it,
408        // as the oracle's did (`paths.json`'s `resource-reads`).
409        assert_eq!(s.read_u64(10, 5, 1 << 32), Ok((5..10, 10)));
410    }
411}