unityrs_core/stream/kind.rs
1// Ported from UnityPy 1.25.2 (MIT): UnityPy/streams/EndianBinaryReader.py. See THIRD_PARTY_LICENSES.
2//! Which of UnityPy's two readers a file is read with, and the stream
3//! reader's rules (`EndianBinaryReader_Streamable`,
4//! `streams/EndianBinaryReader.py:287-324`).
5//!
6//! `EndianBinaryReader(item)` picks its class by the input's type (`:40-63`):
7//! bytes get the memory reader, a file object the stream reader. UnityPy's
8//! `Environment` opens a path as a file (`environment.py:136-139`, the
9//! buffered file `fs.open(path, "rb")` gives; a folder's files through the
10//! builtin `open`, `environment.py:75-77`), so a file loaded from a path is read by the
11//! stream reader: its header, metadata and objects, the pure-Python typetree
12//! reader's reads, and the slice the Boost reader is given
13//! (`helpers/TypeTreeHelper.py:146-153`), and, for a reader kept as it is
14//! (a `.resS`), the resource reads. What UnityPy reads from memory stays on
15//! the memory reader whatever the source: a bundle's entries (its blocks are
16//! decoded into bytes, `files/BundleFile.py:173-186`), a blob's string
17//! buffer (`helpers/TypeTreeNode.py:118`) and a bundle's blocks-info.
18//! `EndianBinaryReader_Streamable_LocalFile` (`streams/EndianBinaryReader.py:327-332`), the reader of a
19//! `str`, is reached by no load: `Environment` opens a path before it makes
20//! a reader.
21//!
22//! The stream reader's `read` is the file's own (`:295`), its `Position` the
23//! file's `tell()` (`:298-299`), and its fixed-width reads
24//! `struct.unpack(stream.read(n))` (`:345-346`). Over a file of `len` bytes,
25//! recorded from UnityPy 1.25.2 on a file `fs.open` opened
26//! (`tests/fixtures/scaffold/reader.json`: the `stream_` cases on a file of
27//! the scratch folder, the `stream_tmpfs_` cases on a tmpfs file, whose
28//! largest is 2^63 - 1):
29//! - `Position = p` refuses a negative `p` (`ValueError` "Position cannot be
30//! negative", `:302-305`) and one at or above 2^63, which the file's
31//! `seek` cannot take on any machine (`ValueError` "cannot fit 'int' into
32//! an offset-sized integer", `CPython`'s `PyNumber_AsOff_t`); any other moves
33//! there, past the end included, where the file system takes it. Past the
34//! largest file a file system takes (ext4's is 2^44 - 4096 with 4 KiB
35//! blocks; tmpfs's 2^63 - 1, the `stream_tmpfs_` cases) `seek` raises
36//! `OSError` (`EINVAL`), which depends on the machine: this module does not
37//! model it, and moves there (`PARITY.md` section 19.3). A stream position
38//! is never negative.
39//! - `read(n)`: `n` below -1 raises (`ValueError` "read length must be
40//! non-negative or -1"); -1 reads to the end and 0 nothing; any other `n`
41//! reads the bytes that exist, and the position moves by the bytes read,
42//! so it stops at the end, and stays where it is past the end. A buffered
43//! file allocates `n` bytes before it reads
44//! (`_bufferedreader_read_generic`), so `n` at or above 2^63 - 33 raises
45//! `OverflowError` ("byte string is too large", from `_PyBytes_FromSize`)
46//! on any machine; an `n` below that but above what the machine can
47//! allocate raises `MemoryError`, which depends on the machine: this
48//! module does not model it, and reads what exists (`PARITY.md` section
49//! 19.3). The fixture's sizes are those that raise on every machine or
50//! that every machine allocates. At a position where the buffered file's
51//! fill would end past 2^63 - 1 a read raises `OSError` (`EINVAL`) on
52//! Linux, whose `rw_verify_area` refuses a read whose end overflows (the
53//! `stream_tmpfs_read` cases at 2^63 - 1); this module reads nothing
54//! there, as past the end anywhere.
55//! - a fixed-width read is `read(n)`, which raises unless it got `n` bytes,
56//! the position moved by what it got either way.
57//! - `read_string_to_null(max)` is the base class's loop (`:136-144`): it
58//! reads a byte at a time until a NUL, `max` bytes, or the end, and drops
59//! the last byte it read when that byte is no NUL. So with no NUL in reach
60//! it returns one byte fewer than it read; at the end, or with `max` 0
61//! anywhere, it returns `""`; and only past the end with `max` above 0,
62//! where its first read is empty, it raises (`ValueError` "Unterminated
63//! string: [b'']").
64//! - `align_stream` is the memory reader's arithmetic (`:155-156`), the
65//! position never negative, assigned through the `Position` setter: so a
66//! position it takes to 2^63 or more raises as that `seek` does
67//! ([`ReaderKind::align`]).
68//!
69//! The memory reader's rules are the free functions of [`super`].
70
71use std::ops::Range;
72
73use super::{py_align, py_read, py_string_to_null_found, py_string_window, py_unpack_from};
74
75/// Which of UnityPy's readers a file is read with (module docs).
76#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
77pub enum ReaderKind {
78 /// `EndianBinaryReader_Memoryview`: bytes. What UnityPy is handed as
79 /// bytes ([`crate::env::Source::Bytes`] and [`crate::env::Source::Fetch`]),
80 /// and every entry of a container.
81 #[default]
82 Memory,
83 /// `EndianBinaryReader_Streamable` over the buffered file `Environment`
84 /// opens for a path: a top-level [`crate::env::Source::Path`].
85 Stream,
86 /// Stream reader backed by `io.BytesIO`: any negative read size means EOF.
87 BytesIo,
88}
89
90/// An exception the stream reader raises where the memory reader reads on
91/// (module docs). Its text is UnityPy's message.
92#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, thiserror::Error)]
93pub enum StreamRefusal {
94 /// `Position = p` with `p < 0` (`ValueError`).
95 #[error("Position cannot be negative")]
96 NegativePosition,
97 /// `Position = p` with `p >= 2^63`: the file's `seek` takes a C offset
98 /// (`ValueError`).
99 #[error("cannot fit 'int' into an offset-sized integer")]
100 PositionTooLarge,
101 /// `read(n)` with `n < -1` (`ValueError`).
102 #[error("read length must be non-negative or -1")]
103 NegativeRead,
104 /// `read(n)` with `2^63 - 33 <= n < 2^63`: the bytes object `n` needs
105 /// (`OverflowError`).
106 #[error("byte string is too large")]
107 ReadTooLarge,
108 /// `read(n)` with `n >= 2^63`: a C size (`OverflowError`).
109 #[error("cannot fit 'int' into an index-sized integer")]
110 ReadSizeOverflow,
111}
112
113impl StreamRefusal {
114 /// UnityPy's exception class.
115 #[must_use]
116 pub fn exception(self) -> &'static str {
117 match self {
118 Self::NegativePosition | Self::PositionTooLarge | Self::NegativeRead => "ValueError",
119 Self::ReadTooLarge | Self::ReadSizeOverflow => "OverflowError",
120 }
121 }
122
123 /// The message, as UnityPy's exception carries it.
124 #[must_use]
125 pub fn message(self) -> &'static str {
126 match self {
127 Self::NegativePosition => "Position cannot be negative",
128 Self::PositionTooLarge => "cannot fit 'int' into an offset-sized integer",
129 Self::NegativeRead => "read length must be non-negative or -1",
130 Self::ReadTooLarge => "byte string is too large",
131 Self::ReadSizeOverflow => "cannot fit 'int' into an index-sized integer",
132 }
133 }
134}
135
136impl From<StreamRefusal> for crate::error::Error {
137 fn from(e: StreamRefusal) -> Self {
138 Self::Malformed {
139 what: STREAM_READER,
140 detail: e.message().into(),
141 }
142 }
143}
144
145const _: () = {
146 crate::send_sync::<ReaderKind>();
147 crate::send_sync::<StreamRefusal>();
148};
149
150/// The `what` of an [`crate::Error::Malformed`] a [`StreamRefusal`] becomes.
151pub const STREAM_READER: &str = "stream reader";
152
153/// The smallest `read(n)` a buffered file refuses for its bytes object on
154/// every machine (`_PyBytes_FromSize` refuses a size above
155/// `PY_SSIZE_T_MAX - PyBytesObject_SIZE`, `PyBytesObject_SIZE` being 33:
156/// the fixture records `2^63 - 33` as "byte string is too large"). Below
157/// it the read allocates first, so its outcome depends on the machine
158/// (module docs).
159const READ_TOO_LARGE: i64 = i64::MAX - 32;
160
161impl ReaderKind {
162 /// `Position = pos`: the position it leaves, or the stream reader's
163 /// refusal of a negative one or one at or above 2^63.
164 pub fn seek(self, pos: i128) -> Result<i128, StreamRefusal> {
165 match self {
166 Self::Stream | Self::BytesIo if pos < 0 => Err(StreamRefusal::NegativePosition),
167 Self::Stream | Self::BytesIo if pos > i128::from(i64::MAX) => {
168 Err(StreamRefusal::PositionTooLarge)
169 }
170 Self::Memory | Self::Stream | Self::BytesIo => Ok(pos),
171 }
172 }
173
174 /// `align_stream(n)` at `pos`: `Position += (n - Position % n) % n`
175 /// ([`py_align`]), an assignment through the `Position` setter
176 /// (`streams/EndianBinaryReader.py:155-156`), so the stream reader
177 /// refuses a result its `seek` cannot take, as [`Self::seek`] does: at
178 /// 2^63 or more (from a position within `n - 1` of 2^63 - 1).
179 pub fn align(self, pos: i128, n: u32) -> Result<i128, StreamRefusal> {
180 self.seek(py_align(pos, n))
181 }
182
183 /// `read(size)` at `pos` over `len` bytes: the bytes' range and the new
184 /// position. The memory reader's is [`py_read`]; the stream reader's
185 /// reads what exists from `pos` (never negative on a stream, as
186 /// [`Self::seek`] refuses that) and moves by what it read.
187 pub fn read(
188 self,
189 len: usize,
190 pos: i128,
191 size: i64,
192 ) -> Result<(Range<usize>, i128), StreamRefusal> {
193 match self {
194 Self::Memory => Ok(py_read(len, pos, size)),
195 Self::Stream => stream_read(len, pos, size),
196 Self::BytesIo => Ok(stream_read_range(len, pos, size.max(-1))),
197 }
198 }
199
200 /// `read(size)` for a size a caller holds as a `u64` (a resource's): on
201 /// the memory reader a size past `i64::MAX` slices as `i64::MAX` does,
202 /// and the stream reader refuses it.
203 pub fn read_u64(
204 self,
205 len: usize,
206 pos: i128,
207 size: u64,
208 ) -> Result<(Range<usize>, i128), StreamRefusal> {
209 match (i64::try_from(size), self) {
210 (Ok(size), _) => self.read(len, pos, size),
211 (Err(_), Self::Memory) => Ok(py_read(len, pos, i64::MAX)),
212 (Err(_), Self::Stream | Self::BytesIo) => Err(StreamRefusal::ReadSizeOverflow),
213 }
214 }
215
216 /// The reader's own fixed-width read of `n` bytes (`reader.read_int()`
217 /// and the like) at `pos` over `len` bytes: the start of its `n` bytes,
218 /// or `None` where it raises, and the position after it. The memory
219 /// reader's is `struct.unpack_from(view, Position)` ([`py_unpack_from`]),
220 /// which moves the position by `n` on success only; the stream reader's
221 /// is `struct.unpack(stream.read(n))`, which moves it by what it read.
222 #[must_use]
223 pub fn unpack(self, len: usize, pos: i128, n: usize) -> (Option<usize>, i128) {
224 let step = i128::try_from(n).unwrap_or(i128::MAX);
225 match self {
226 Self::Memory => match py_unpack_from(len, pos, n) {
227 Some(at) => (Some(at), pos.saturating_add(step)),
228 None => (None, pos),
229 },
230 Self::Stream | Self::BytesIo => {
231 let size = i64::try_from(n).unwrap_or(i64::MAX);
232 match stream_read(len, pos, size) {
233 Ok((range, next)) if range.len() == n => (Some(range.start), next),
234 Ok((_, next)) => (None, next),
235 Err(_) => (None, pos),
236 }
237 }
238 }
239 }
240
241 /// `read_string_to_null(max_length)` at `pos` over `data`: the string's
242 /// range and the new position, or `None` where it raises (module docs:
243 /// the stream reader raises only past the end with `max_length` above
244 /// 0).
245 #[must_use]
246 pub fn string_to_null(
247 self,
248 data: &[u8],
249 pos: i128,
250 max_length: usize,
251 ) -> Option<(Range<usize>, i128)> {
252 let window = py_string_window(data.len(), pos, max_length);
253 let nul = data
254 .get(window.clone())
255 .and_then(|w| w.iter().position(|b| *b == 0))
256 .map(|k| window.start + k);
257 self.string_to_null_found(data.len(), pos, max_length, nul)
258 }
259
260 /// [`Self::string_to_null`]'s answer once its search of
261 /// [`py_string_window`]'s window is done: `nul` is the window's first
262 /// NUL, as an index of the whole buffer. The stream reader's loop reads
263 /// the same window (from a position that is never negative), so one
264 /// search serves both readers.
265 #[must_use]
266 pub(crate) fn string_to_null_found(
267 self,
268 len: usize,
269 pos: i128,
270 max_length: usize,
271 nul: Option<usize>,
272 ) -> Option<(Range<usize>, i128)> {
273 match self {
274 Self::Memory => py_string_to_null_found(len, pos, max_length, nul),
275 Self::Stream | Self::BytesIo => stream_string_to_null(len, pos, max_length, nul),
276 }
277 }
278}
279
280/// The stream reader's `read(size)` (module docs).
281fn stream_read(len: usize, pos: i128, size: i64) -> Result<(Range<usize>, i128), StreamRefusal> {
282 if size < -1 {
283 return Err(StreamRefusal::NegativeRead);
284 }
285 if size >= READ_TOO_LARGE {
286 return Err(StreamRefusal::ReadTooLarge);
287 }
288 Ok(stream_read_range(len, pos, size))
289}
290
291fn stream_read_range(len: usize, pos: i128, size: i64) -> (Range<usize>, i128) {
292 let l = i128::try_from(len).unwrap_or(i128::MAX);
293 let start = pos.clamp(0, l);
294 if size == 0 || pos >= l {
295 // Nothing to read: the position stays, past the end included.
296 let at = usize::try_from(start).unwrap_or(len);
297 return (at..at, pos);
298 }
299 let end = if size == -1 {
300 l
301 } else {
302 start.saturating_add(i128::from(size)).min(l)
303 };
304 let range = usize::try_from(start).unwrap_or(len)..usize::try_from(end).unwrap_or(len);
305 (range, end)
306}
307
308/// The stream reader's `read_string_to_null` (`:136-144`, module docs):
309/// `nul` is the first NUL in the window `pos..min(pos + max_length, len)`.
310fn stream_string_to_null(
311 len: usize,
312 pos: i128,
313 max_length: usize,
314 nul: Option<usize>,
315) -> Option<(Range<usize>, i128)> {
316 let l = i128::try_from(len).ok()?;
317 let start = usize::try_from(pos.clamp(0, l)).ok()?;
318 if max_length == 0 || pos == l {
319 // The loop never runs (`len(ret) < max_length` or `Position !=
320 // Length` is false before its first read): `""`, nothing read,
321 // wherever the position is.
322 return Some((start..start, pos));
323 }
324 if pos > l {
325 // The loop's first `read(1)` is empty: "Unterminated string: [b'']".
326 return None;
327 }
328 if let Some(nul) = nul {
329 // The NUL is read and dropped.
330 return Some((start..nul, i128::try_from(nul).ok()? + 1));
331 }
332 // No NUL in reach: the bytes read, less the last.
333 let read = usize::try_from(l - pos).ok()?.min(max_length);
334 let end = start.checked_add(read)?;
335 Some((start..end - 1, i128::try_from(end).ok()?))
336}
337
338#[cfg(test)]
339mod tests {
340 use super::{ReaderKind, StreamRefusal};
341
342 #[test]
343 fn the_memory_kind_is_the_free_functions() {
344 // `ReaderKind::Memory` delegates: a negative position and a
345 // negative size read as Python slices, and nothing is refused.
346 let m = ReaderKind::Memory;
347 assert_eq!(m.seek(-5), Ok(-5));
348 assert_eq!(m.read(10, 3, -2), Ok((3..3, 1)));
349 assert_eq!(m.unpack(10, -4, 4), (Some(6), 0));
350 assert_eq!(
351 m.read_u64(10, 2, u64::MAX),
352 Ok((2..10, 2 + i128::from(i64::MAX)))
353 );
354 }
355
356 #[test]
357 fn the_stream_kind_refuses_what_a_buffered_file_refuses() {
358 // What CPython refuses on every machine, each edge recorded from
359 // UnityPy 1.25.2 (`tests/fixtures/scaffold/reader.json`'s `stream_`
360 // and `stream_tmpfs_` cases, `tests/fixtures/d/paths.json`'s
361 // `resource-reads`), which `tests/reader.rs` and `tests/env_paths.rs`
362 // replay in full.
363 let s = ReaderKind::Stream;
364 assert_eq!(s.seek(-1), Err(StreamRefusal::NegativePosition));
365 assert_eq!(
366 s.seek(i128::from(i64::MAX) + 1),
367 Err(StreamRefusal::PositionTooLarge)
368 );
369 assert_eq!(s.read(10, 3, -2), Err(StreamRefusal::NegativeRead));
370 assert_eq!(
371 s.read(10, 3, i64::MAX - 32),
372 Err(StreamRefusal::ReadTooLarge)
373 );
374 assert_eq!(s.read(10, 20, i64::MAX), Err(StreamRefusal::ReadTooLarge));
375 assert_eq!(
376 s.read_u64(10, 0, 1 << 63),
377 Err(StreamRefusal::ReadSizeOverflow)
378 );
379 // `align_stream` assigns `Position`, so an alignment to 2^63 is the
380 // seek's refusal; the memory reader's is plain arithmetic.
381 let near = i128::from(i64::MAX) - 2;
382 assert_eq!(s.align(near, 4), Err(StreamRefusal::PositionTooLarge));
383 assert_eq!(s.align(near - 1, 4), Ok(near - 1));
384 assert_eq!(ReaderKind::Memory.align(near, 4), Ok(near + 3));
385 // Past the end a read is empty and the position stays.
386 assert_eq!(s.read(10, 15, -1), Ok((10..10, 15)));
387 assert_eq!(s.unpack(10, 8, 4), (None, 10));
388 assert_eq!(s.unpack(10, 12, 0), (Some(10), 12));
389 // With `max_length` 0 the string loop never runs, past the end too.
390 assert_eq!(s.string_to_null(b"abc", 7, 0), Some((3..3, 7)));
391 assert_eq!(s.string_to_null(b"abc", 7, 1), None);
392 }
393
394 #[test]
395 fn the_stream_kind_reads_on_where_the_machine_decides() {
396 // Not UnityPy's outcome on every machine: unityrs's model of the
397 // edges `PARITY.md` section 19.3 admits ("a path's stream past the
398 // machine's limits"), where it reads what exists.
399 let s = ReaderKind::Stream;
400 // A seek past the file system's largest file raises `OSError`
401 // (`EINVAL`) in UnityPy; on tmpfs, whose largest is 2^63 - 1, it
402 // moves there (`reader.json`'s `stream_tmpfs_set_position`).
403 assert_eq!(s.seek(i128::from(i64::MAX)), Ok(i128::from(i64::MAX)));
404 // A read the machine cannot allocate raises `MemoryError` in UnityPy
405 // (not recorded: a buffered file allocates `n` bytes first).
406 assert_eq!(s.read(10, 3, i64::MAX - 33), Ok((3..10, 10)));
407 // A read of 4 GiB reads what exists where the machine allocates it,
408 // as the oracle's did (`paths.json`'s `resource-reads`).
409 assert_eq!(s.read_u64(10, 5, 1 << 32), Ok((5..10, 10)));
410 }
411}