Skip to main content

unityrs_core/files/serialized/
parse.rs

1// Ported from UnityPy 1.25.2 (MIT): UnityPy/files/SerializedFile.py, UnityPy/files/ObjectReader.py. See THIRD_PARTY_LICENSES.
2// Ported from UnityPy git 417d595 (MIT): UnityPy/files/SerializedFile.py. See THIRD_PARTY_LICENSES.
3//! [`SerializedFile::parse`]: `SerializedFile.__init__`'s metadata read,
4//! by the rules the `serialized` module docs list, and the `objects` dict's
5//! order.
6
7use std::sync::Arc;
8
9use bytes::Bytes;
10
11use super::{External, ObjectInfo, ScriptRef, SerializedFile, SerializedHeader, SerializedType};
12use crate::error::Error;
13use crate::files::{Parent, ParentKind};
14use crate::limits::{Charge, Limits, Meter, MeteredVec, heap_cost};
15use crate::once::ResultCell;
16use crate::options::FileContext;
17use crate::pin::CopyCounts;
18use crate::raw::RawStr;
19use crate::stream::{Endian, Reader, ReaderKind};
20use crate::typetree::NodeList;
21use crate::typetree::commonstring::CommonStrings;
22use crate::typetree::parse::{is_blob_format, parse_blob, parse_old, string_to_null};
23use crate::version::resolve_serialized;
24
25impl SerializedFile {
26    /// `SerializedFile.__init__` over bytes (the memory reader's rules).
27    /// Everything it keeps is charged to `meter`, the input's.
28    pub fn parse(
29        name: RawStr,
30        data: Bytes,
31        parent: Option<Parent>,
32        meter: &Arc<Meter>,
33        ctx: &Arc<FileContext>,
34    ) -> Result<Arc<Self>, Error> {
35        Self::parse_with_reader(name, data, parent, meter, ctx, ReaderKind::Memory)
36    }
37
38    /// [`Self::parse`] by the rules of the reader UnityPy would read `data`
39    /// with ([`ReaderKind::Stream`] for a top-level file loaded from a path),
40    /// which its objects' reads follow too.
41    pub fn parse_with_reader(
42        name: RawStr,
43        data: Bytes,
44        parent: Option<Parent>,
45        meter: &Arc<Meter>,
46        ctx: &Arc<FileContext>,
47        reader: ReaderKind,
48    ) -> Result<Arc<Self>, Error> {
49        Self::parse_with_state(name, data, parent, meter, ctx, reader, &mut None)
50    }
51
52    /// Parse while retaining the caller reader's endian assignment on failure.
53    #[expect(
54        clippy::too_many_lines,
55        reason = "format-gated metadata and charged file construction"
56    )]
57    pub(crate) fn parse_with_state(
58        name: RawStr,
59        data: Bytes,
60        parent: Option<Parent>,
61        meter: &Arc<Meter>,
62        ctx: &Arc<FileContext>,
63        reader: ReaderKind,
64        endian: &mut Option<Endian>,
65    ) -> Result<Arc<Self>, Error> {
66        // The handle itself, in its `Arc`.
67        let mut charge = meter.charge(heap_cost(
68            u64::try_from(std::mem::size_of::<Self>() + 16).unwrap_or(u64::MAX),
69        ))?;
70        let mut p = Parser::new(&data, parent.as_ref(), meter, ctx, reader);
71        let header = p.header()?;
72        p.version = header.version;
73        p.r.set_endian(header.endian);
74        *endian = Some(header.endian);
75        let format = header.version;
76        // UnityPy files/SerializedFile.py:271-273, :351-360
77        let (unity_version, version) = if format >= 7 {
78            let own = string_to_null(&mut p.r, &data, meter)?;
79            let parent_engine = match parent.as_ref().map(|p| &p.kind) {
80                Some(ParentKind::Bundle { version_engine, .. }) => Some(version_engine),
81                _ => None,
82            };
83            let resolved =
84                resolve_serialized(&own, parent_engine, &ctx.version_policy(), &mut charge)?;
85            (Some(own), Some(resolved))
86        } else {
87            (None, None)
88        };
89        // :275-280
90        let target_platform = if format >= 8 {
91            Some(p.r.read_i32()?)
92        } else {
93            None
94        };
95        if format >= 13 {
96            p.enable_type_tree = p.r.read_bool()?;
97        }
98        // :282-284
99        let mut types = p.types(false)?;
100        // :286-288
101        let big_id_enabled = if (7..14).contains(&format) {
102            p.r.read_i32()?
103        } else {
104            0
105        };
106        // :290-295
107        let object_infos = p.objects(big_id_enabled, header.data_offset, &mut types)?;
108        // :297-300
109        let script_types = p.scripts()?;
110        // :302-304
111        let externals = p.externals()?;
112        // :306-308
113        let mut ref_types = if format >= 20 {
114            Some(p.types(true)?)
115        } else {
116            None
117        };
118        let head_roots = p.handlers.len();
119        if format >= 26 {
120            p.read_shared(&mut types, ref_types.as_mut())?;
121        }
122        let ref_types = ref_types.map(|t| keep(t, &mut charge)).transpose()?;
123        // files/SerializedFile.py:310-311: disable only after all trees were read.
124        if !ctx.parse_typetree_enabled() {
125            p.enable_type_tree = false;
126        }
127        // :313-314
128        let user_information = if format >= 5 {
129            Some(string_to_null(&mut p.r, &data, meter)?)
130        } else {
131            None
132        };
133        let enable_type_tree = p.enable_type_tree && ctx.serialized_file_parse_typetree;
134        charge.absorb(p.kept);
135        let (object_order, by_path) = dict_order(&object_infos, meter)?;
136        // Eager AssetBundle reading uses the same input reader and check_read
137        // requires consuming its complete object payload on success.
138        let container_end = ctx
139            .eager_asset_bundle
140            .then(|| {
141                object_order
142                    .as_slice()
143                    .iter()
144                    .filter_map(|index| object_infos.as_slice().get(*index as usize))
145                    .find(|object| object.class_id == ASSET_BUNDLE)
146                    .map(|object| object.byte_start + i128::from(object.byte_size))
147            })
148            .flatten();
149        let constructor_position = container_end.unwrap_or(p.r.position());
150        let types = keep(types, &mut charge)?;
151        let object_infos = keep(object_infos, &mut charge)?;
152        let script_types = keep(script_types, &mut charge)?;
153        let externals = keep(externals, &mut charge)?;
154        let object_order = keep(object_order, &mut charge)?;
155        let by_path = keep(by_path, &mut charge)?;
156        let copies = CopyCounts::new(p.trees.0, p.trees.1);
157        let head_handlers = p.handlers;
158        let file = Arc::new(Self {
159            changed: std::sync::atomic::AtomicBool::new(false),
160            edits: std::sync::Mutex::new(MeteredVec::new(meter)),
161            // Before `data` moves: the parser reads it.
162            copies,
163            name,
164            data,
165            header,
166            unity_version,
167            version,
168            target_platform,
169            enable_type_tree,
170            big_id_enabled,
171            types,
172            ref_types,
173            head_handlers,
174            head_roots,
175            script_types,
176            externals,
177            pending_externals: std::sync::Mutex::new(MeteredVec::new(meter)),
178            user_information,
179            object_infos,
180            object_order,
181            by_path,
182            container: ResultCell::new(),
183            parent,
184            dependency_path: crate::files::DependencyPath::default(),
185            reader,
186            constructor_position,
187            charge,
188            ctx: ctx.clone(),
189        });
190        // :316-324: the first AssetBundle in dict order is read at load; a
191        // failure fails the parse, whose grants the bundle counts.
192        if container_end.is_some() {
193            file.container_at_load()?;
194        }
195        Ok(file)
196    }
197}
198
199/// `ClassIDType.AssetBundle`.
200const ASSET_BUNDLE: i32 = 142;
201
202/// A table as the file keeps it, its charge moved into the file's.
203fn keep<T>(table: MeteredVec<T>, charge: &mut Charge) -> Result<Box<[T]>, Error> {
204    let (b, c) = table.into_boxed_slice()?;
205    charge.absorb(c);
206    Ok(b)
207}
208
209/// The metadata reader of one parse.
210pub(super) struct Parser<'a> {
211    pub(super) r: Reader<'a>,
212    pub(super) data: &'a Bytes,
213    /// The buffer `data` is a slice of ([`SerializedFile::pinned`]).
214    pinned: u64,
215    /// The typetrees whose strings were copied out of it, and their bytes
216    /// ([`crate::pin`]).
217    trees: (u64, u64),
218    pub(super) meter: &'a Arc<Meter>,
219    pub(super) limits: &'a Limits,
220    /// The caller's TPK's common strings, which blob trees read (`None`:
221    /// UnityPy's MIT table).
222    pub(super) common: Option<&'a CommonStrings>,
223    /// The format (`header.version`).
224    pub(super) version: u32,
225    /// `_enable_type_tree` as read so far.
226    enable_type_tree: bool,
227    /// Charges of what the type records keep outside their node lists.
228    kept: Charge,
229    pub(super) handlers: MeteredVec<super::head::Handler>,
230}
231
232impl<'a> Parser<'a> {
233    pub(super) fn new(
234        data: &'a Bytes,
235        parent: Option<&Parent>,
236        meter: &'a Arc<Meter>,
237        ctx: &'a FileContext,
238        reader: ReaderKind,
239    ) -> Self {
240        Self {
241            r: Reader::with_kind(data, Endian::Big, reader),
242            data,
243            pinned: super::pinned(parent, data),
244            trees: (0, 0),
245            meter,
246            limits: &ctx.limits,
247            common: ctx.providers.common_strings(),
248            version: 0,
249            enable_type_tree: true,
250            kept: Charge::zero(meter),
251            handlers: MeteredVec::new(meter),
252        }
253    }
254}
255
256impl Parser<'_> {
257    /// `SerializedFileHeader` and the header fields `__init__` reads after it
258    /// (`SerializedFile.py:248-269`): big-endian, the endian byte at 16 from
259    /// format 9, else at `file_size - metadata_size` (a Python position,
260    /// negative when the metadata is larger than the file, which the stream
261    /// reader refuses).
262    fn header(&mut self) -> Result<SerializedHeader, Error> {
263        let r = &mut self.r;
264        // files/SerializedFile.py:23-38 reads all four fields in one struct.
265        let [
266            meta0,
267            meta1,
268            meta2,
269            meta3,
270            file0,
271            file1,
272            file2,
273            file3,
274            ver0,
275            ver1,
276            ver2,
277            ver3,
278            off0,
279            off1,
280            off2,
281            off3,
282        ] = r.read_array::<16>()?;
283        let mut metadata_size = u32::from_be_bytes([meta0, meta1, meta2, meta3]);
284        let mut file_size = i64::from(u32::from_be_bytes([file0, file1, file2, file3]));
285        let version = u32::from_be_bytes([ver0, ver1, ver2, ver3]);
286        let mut data_offset = i64::from(u32::from_be_bytes([off0, off1, off2, off3]));
287        let (endian, reserved, unknown);
288        let order = |big: bool| if big { Endian::Big } else { Endian::Little };
289        if version >= 9 {
290            endian = order(r.read_bool()?);
291            // `read_bytes(3)`: short at the end without an error; every
292            // later read then fails.
293            let got = r.read_bytes_short(3);
294            let mut three = [0u8; 3];
295            for (d, s) in three.iter_mut().zip(got) {
296                *d = *s;
297            }
298            reserved = Some(three);
299            if version >= 22 {
300                metadata_size = r.read_u32()?;
301                file_size = r.read_i64()?;
302                data_offset = r.read_i64()?;
303                unknown = r.read_i64()?;
304            } else {
305                unknown = 0;
306            }
307        } else {
308            // A stream refuses a negative position (`reader.Position =`).
309            r.set_position(i128::from(file_size) - i128::from(metadata_size))?;
310            endian = order(r.read_bool()?);
311            reserved = None;
312            unknown = 0;
313        }
314        Ok(SerializedHeader {
315            metadata_size,
316            file_size,
317            version,
318            data_offset,
319            endian,
320            reserved,
321            unknown,
322        })
323    }
324
325    /// The script table (`LocalSerializedObjectIdentifier`), format 11 up.
326    fn scripts(&mut self) -> Result<MeteredVec<ScriptRef>, Error> {
327        let mut scripts = MeteredVec::new(self.meter);
328        if self.version < 11 {
329            return Ok(scripts);
330        }
331        for _ in 0..self.r.read_i32()?.max(0) {
332            let file_index = self.r.read_i32()?;
333            let identifier = if self.version < 14 {
334                i64::from(self.r.read_i32()?)
335            } else {
336                self.r.align(4)?;
337                self.r.read_i64()?
338            };
339            scripts.push(ScriptRef {
340                file_index,
341                identifier,
342            })?;
343        }
344        Ok(scripts)
345    }
346
347    /// The externals (`FileIdentifier`).
348    fn externals(&mut self) -> Result<MeteredVec<External>, Error> {
349        let mut externals = MeteredVec::new(self.meter);
350        for _ in 0..self.r.read_i32()?.max(0) {
351            let temp_empty = if self.version >= 6 {
352                Some(string_to_null(&mut self.r, self.data, self.meter)?)
353            } else {
354                None
355            };
356            let (guid, kind) = if self.version >= 5 {
357                // `read_bytes(16)` kept as it is: short (empty at a position
358                // in -16..0) without an error, as UnityPy keeps it.
359                (Some(self.r.read_short_array()), Some(self.r.read_i32()?))
360            } else {
361                (None, None)
362            };
363            let path = string_to_null(&mut self.r, self.data, self.meter)?;
364            externals.push(External {
365                temp_empty,
366                guid,
367                kind,
368                path,
369            })?;
370        }
371        Ok(externals)
372    }
373
374    /// `[SerializedType(reader, self, is_ref_type) for _ in range(count)]`.
375    fn types(&mut self, is_ref: bool) -> Result<MeteredVec<SerializedType>, Error> {
376        let mut types = MeteredVec::new(self.meter);
377        for _ in 0..self.r.read_i32()?.max(0) {
378            let t = self.serialized_type(is_ref)?;
379            types.push(t)?;
380        }
381        Ok(types)
382    }
383
384    /// `SerializedType.__init__` (`SerializedFile.py:112-149`).
385    fn serialized_type(&mut self, is_ref: bool) -> Result<SerializedType, Error> {
386        let v = self.version;
387        let class_id = self.r.read_i32()?;
388        let mut t = SerializedType::new(class_id, None);
389        if v >= 16 {
390            t.is_stripped_type = Some(self.r.read_bool()?);
391        }
392        if v >= 17 {
393            t.script_type_index = self.r.read_i16()?;
394        }
395        if v >= 13 {
396            if (is_ref && t.script_type_index >= 0)
397                || (v < 16 && class_id < 0)
398                || (v >= 16 && class_id == 114)
399            {
400                t.script_id = Some(self.r.read_short_array());
401            }
402            t.old_type_hash = Some(self.r.read_short_array());
403        }
404        if self.enable_type_tree {
405            if v >= 23 {
406                let handler = self.head_handler()?;
407                t.nodes = handler.nodes();
408                self.handlers.push(handler)?;
409            } else {
410                let mut rows = if is_blob_format(v) {
411                    parse_blob(
412                        &mut self.r,
413                        self.data,
414                        v,
415                        self.meter,
416                        self.limits,
417                        self.common,
418                    )?
419                } else {
420                    parse_old(&mut self.r, self.data, v, self.meter, self.limits)?
421                };
422                // A value keeps its tree's node list, whose names are slices of
423                // the file: copied out when they would keep its buffer alive
424                // (`crate::pin`; plan section 8, R3).
425                self.copy_tree_strings(&mut rows);
426                t.nodes = Some(NodeList::from_rows(rows)?.into_shared()?);
427            }
428            if v >= 21 {
429                if is_ref {
430                    t.class_name = Some(string_to_null(&mut self.r, self.data, self.meter)?);
431                    t.namespace = Some(string_to_null(&mut self.r, self.data, self.meter)?);
432                    t.assembly = Some(string_to_null(&mut self.r, self.data, self.meter)?);
433                } else {
434                    let deps = self.r.read_i32_array(self.meter)?;
435                    let (deps, c) = deps.into_boxed_slice()?;
436                    // The file keeps the list's charge.
437                    self.kept.absorb(c);
438                    t.type_dependencies = Some(deps);
439                }
440            }
441        }
442        Ok(t)
443    }
444
445    pub(super) fn copy_tree_strings(&mut self, rows: &mut MeteredVec<crate::typetree::NodeRow>) {
446        let n = crate::pin::copy_out(rows, self.data, self.pinned, self.meter);
447        if n > 0 {
448            self.trees = (
449                self.trees.0.saturating_add(1),
450                self.trees.1.saturating_add(n),
451            );
452        }
453    }
454
455    /// The object table (`ObjectReader.from_reader`, `ObjectReader.py:67-130`),
456    /// writing script indexes back into `types` at 11-16. Below 16 each
457    /// object's type is found through [`TypesByClass`].
458    fn objects(
459        &mut self,
460        big_id_enabled: i32,
461        data_offset: i64,
462        types: &mut MeteredVec<SerializedType>,
463    ) -> Result<MeteredVec<ObjectInfo>, Error> {
464        let v = self.version;
465        let mut objects = MeteredVec::new(self.meter);
466        let mut by_class: Option<TypesByClass> = None;
467        for _ in 0..self.r.read_i32()?.max(0) {
468            let r = &mut self.r;
469            let path_id = if big_id_enabled != 0 {
470                r.read_i64()?
471            } else if v < 14 {
472                i64::from(r.read_i32()?)
473            } else {
474                r.align(4)?;
475                r.read_i64()?
476            };
477            let stored = if v >= 22 {
478                i128::from(r.read_i64()?)
479            } else {
480                i128::from(r.read_u32()?)
481            };
482            let byte_start = stored + i128::from(data_offset);
483            let byte_size = r.read_u32()?;
484            let type_id = r.read_i32()?;
485            let (class_id, type_index) = if v < 16 {
486                let class_id = i32::from(r.read_u16()?);
487                // A script index written back below changes no class id, so
488                // the index built on the first object holds for every one.
489                let by_class = match &mut by_class {
490                    Some(t) => t,
491                    none => none.insert(TypesByClass::new(types.as_slice(), self.meter)?),
492                };
493                (class_id, by_class.first(type_id))
494            } else {
495                // `types[type_id]`: Python indexing.
496                let len = types.len();
497                let index = if type_id < 0 {
498                    len.checked_sub(usize::try_from(type_id.unsigned_abs()).unwrap_or(usize::MAX))
499                } else {
500                    usize::try_from(type_id).ok().filter(|i| *i < len)
501                };
502                let index = index.ok_or(Error::TypeIndex {
503                    index: type_id,
504                    len,
505                })?;
506                let class_id = types.as_slice().get(index).map_or(0, |t| t.class_id);
507                (class_id, Some(index))
508            };
509            let is_destroyed = if v < 11 { Some(r.read_u16()?) } else { None };
510            let script_type_index = if (11..17).contains(&v) {
511                let i = r.read_i16()?;
512                if let Some(t) = type_index.and_then(|t| types.as_mut_slice().get_mut(t)) {
513                    t.script_type_index = i;
514                }
515                Some(i)
516            } else {
517                None
518            };
519            let is_stripped = if v == 15 || v == 16 {
520                Some(r.read_i8()?)
521            } else {
522                None
523            };
524            objects.push(ObjectInfo {
525                path_id,
526                byte_start,
527                byte_size,
528                type_id,
529                class_id,
530                type_index,
531                is_destroyed,
532                is_stripped,
533                script_type_index,
534            })?;
535        }
536        Ok(objects)
537    }
538}
539
540/// The types by class id, below format 16, where an object's type is the
541/// first type whose class id is the object's stored type id
542/// (`ObjectReader.py:91-97`, a scan of `types` per object): `(class id,
543/// index)` pairs sorted, built once, on the first object, in a table charged
544/// to the input's meter whose bytes count as the open's work, as a decode's
545/// output does. A lookup is a binary search, so O objects over T types cost
546/// O log T steps, not the O x T of a scan each (round 4's sweep).
547struct TypesByClass {
548    pairs: MeteredVec<(i32, u32)>,
549}
550
551impl TypesByClass {
552    fn new(types: &[SerializedType], meter: &Arc<Meter>) -> Result<Self, Error> {
553        let bytes = types
554            .len()
555            .saturating_mul(std::mem::size_of::<(i32, u32)>());
556        meter.count_decode(u64::try_from(bytes).unwrap_or(u64::MAX))?;
557        let mut pairs = MeteredVec::with_capacity(meter, types.len())?;
558        for (i, t) in types.iter().enumerate() {
559            // The table is a `MeteredVec` of an `i32` count's rows.
560            pairs.push((t.class_id, u32::try_from(i).unwrap_or(u32::MAX)))?;
561        }
562        pairs.as_mut_slice().sort_unstable();
563        Ok(Self { pairs })
564    }
565
566    /// The index of the first type of class `class_id`.
567    fn first(&self, class_id: i32) -> Option<usize> {
568        let pairs = self.pairs.as_slice();
569        let at = pairs.partition_point(|(c, _)| *c < class_id);
570        pairs
571            .get(at)
572            .filter(|(c, _)| *c == class_id)
573            .and_then(|(_, i)| usize::try_from(*i).ok())
574    }
575}
576
577/// The `objects` dict (`self.objects[obj.path_id] = obj`): each path id at
578/// its first position, holding its last row; and the same entries sorted by
579/// path id. Both computed by sorting, in tables charged to `meter`.
580fn dict_order(
581    infos: &MeteredVec<ObjectInfo>,
582    meter: &Arc<Meter>,
583) -> Result<(MeteredVec<u32>, MeteredVec<u32>), Error> {
584    let rows = infos.as_slice();
585    let n = rows.len();
586    let index = |i: usize| u32::try_from(i).unwrap_or(u32::MAX);
587    let path = |i: &u32| {
588        usize::try_from(*i)
589            .ok()
590            .and_then(|i| rows.get(i))
591            .map_or(i64::MIN, |o| o.path_id)
592    };
593    let mut perm: MeteredVec<u32> = MeteredVec::with_capacity(meter, n)?;
594    for i in 0..n {
595        perm.push(index(i))?;
596    }
597    // Equal path ids stay in file order (the index breaks the tie), sorted
598    // in place: a stable sort's scratch would be heap no meter charged
599    // (plan 3.15).
600    perm.as_mut_slice().sort_unstable_by_key(|i| (path(i), *i));
601    let mut by_path: MeteredVec<u32> = MeteredVec::new(meter);
602    let mut groups: MeteredVec<(u32, u32)> = MeteredVec::new(meter);
603    let mut at = 0;
604    let all = perm.as_slice();
605    while let Some(first) = all.get(at) {
606        let key = path(first);
607        let mut last = *first;
608        let mut end = at + 1;
609        while let Some(next) = all.get(end).filter(|i| path(i) == key) {
610            last = *next;
611            end += 1;
612        }
613        by_path.push(last)?;
614        groups.push((*first, last))?;
615        at = end;
616    }
617    drop(perm);
618    groups.as_mut_slice().sort_unstable_by_key(|g| g.0);
619    let mut order: MeteredVec<u32> = MeteredVec::with_capacity(meter, groups.len())?;
620    for (_, last) in groups.as_slice() {
621        order.push(*last)?;
622    }
623    Ok((order, by_path))
624}