Skip to main content

unityrs_core/files/serialized/
mod.rs

1// Ported from UnityPy 1.25.2 (MIT): UnityPy/files/SerializedFile.py, UnityPy/files/ObjectReader.py. See THIRD_PARTY_LICENSES.
2//! `SerializedFile` metadata (`files/SerializedFile.py:206-360`,
3//! `files/ObjectReader.py:67-130`), ported from UnityPy, bytes-native.
4//!
5//! Rules ported exactly (plan section 3.7):
6//! - the header (`SerializedFile.py:255-269`): the endian byte at offset 16
7//!   from format 9, else at `file_size - metadata_size` (absolute); the
8//!   format-22 fields replace the legacy `u32`s;
9//! - gates (`SerializedFile.py:271-288`): the version string from format 7, the target platform
10//!   from 8, `enable_type_tree` from 13 (else true), `big_id_enabled` at 7-13;
11//! - type records (`SerializedFile.py:112-149`): `is_stripped_type` from 16, `script_type_index`
12//!   from 17 (else -1), `script_id` and `old_type_hash` from 13 under
13//!   UnityPy's condition, trees only when `enable_type_tree`, blobs at format
14//!   10 and 12 and up; at 21 ref types read three names and plain types a
15//!   dependency list;
16//! - objects (`ObjectReader.py:73-130`): path id as `i64` under bigID, `i32`
17//!   below 14, else aligned `i64`; `byte_start` `u32` (or `i64` from 22) plus
18//!   `data_offset`; below 16 the type is the first whose `class_id` equals the
19//!   object's `type_id`, from 16 `types[type_id]` with Python indexing (a
20//!   negative id counts from the end; out of range fails the file); at 11-16 the
21//!   object's script index is written back into its type, the last object
22//!   winning; the `objects` dict keeps a repeated path id at its first position
23//!   with its last entry;
24//! - `read_string_to_null`'s 32,767-byte window for every metadata string;
25//! - the reader UnityPy reads the file with ([`crate::stream::ReaderKind`]):
26//!   the memory reader for bytes and every entry of a container, the stream
27//!   reader for a file loaded from a path, whose rules the metadata's reads
28//!   and every object read follow ([`SerializedFile::reader_kind`]);
29//! - every table is built with `range(count)` (`SerializedFile.py:282-308`):
30//!   a negative count is zero records, and a table grows record by record, so
31//!   no count sizes an allocation;
32//! - with [`FileContext::eager_asset_bundle`], the first AssetBundle in dict
33//!   order is read at load with `check_read`, and a failure fails the file
34//!   (`SerializedFile.py:316-324`). UnityPy reads it in object mode
35//!   (`parse_as_object`); the core reads it in dict mode, and package D's
36//!   fixtures pin that the two agree (plan section 3.7).
37//!
38//! **Memory.** Every table (types, their dependency lists and trees, objects,
39//! the dict order, scripts, externals, ref types) is built in a
40//! [`crate::limits::MeteredVec`] on the input's meter, and the file keeps the
41//! charges: two directory rows over the same bytes parse twice and pay twice.
42//! Strings are zero-copy slices of the file.
43//!
44//! **Work.** The container read ([`SerializedFile::container`]) builds the
45//! whole AssetBundle value and keeps only its rows and preload table, so
46//! two directory rows over the same bytes would repeat a read whose memory
47//! is returned: what the meter granted the read on its own thread, beyond
48//! what its [`Container`] keeps, counts as the open's work
49//! ([`crate::error::LimitKind::DecodeWork`]), and so does all it was
50//! granted when it fails with an error its cell keeps. At load, a failure
51//! fails the parse, whose grants its bundle counts instead
52//! (`files::bundle`, "Memory and work"), so they count once (round 5).
53//!
54//! Owner: package C. The accessors and [`SerializedType`]'s constructors and
55//! [`SerializedType::tree`] are implemented by the scaffold, so package B can
56//! build ref types for its tests before C lands.
57
58use std::sync::{Arc, OnceLock};
59
60use bytes::Bytes;
61
62use crate::container::{Container, read_container};
63use crate::error::Error;
64use crate::files::Parent;
65use crate::limits::{Charge, ChargeError, GrantScope, Limits, Meter};
66use crate::object::ObjectRef;
67use crate::once::{CellError, ResultCell, get_or_publish_result};
68use crate::options::FileContext;
69use crate::pin::{Copies, CopyCounts};
70use crate::raw::RawStr;
71use crate::stream::{Endian, ReaderKind, ShortArray};
72use crate::typetree::{Tree, TreeError};
73use crate::value::Parsed;
74use crate::version::ResolvedVersion;
75use crate::walk::RefTypes;
76
77mod changed;
78mod head;
79mod parse;
80mod write;
81
82pub use write::SerializedSaveSnapshot;
83
84/// The header, as read.
85#[derive(Clone, Debug, PartialEq, Eq)]
86#[non_exhaustive]
87pub struct SerializedHeader {
88    /// `metadata_size`.
89    pub metadata_size: u32,
90    /// `file_size`.
91    pub file_size: i64,
92    /// The format number (UnityPy's `header.version`).
93    pub version: u32,
94    /// `data_offset`.
95    pub data_offset: i64,
96    /// The byte order.
97    pub endian: Endian,
98    /// The three reserved bytes (format 9 and up).
99    pub reserved: Option<[u8; 3]>,
100    /// Format 22's trailing `i64` (UnityPy's `unknown`), else 0.
101    pub unknown: i64,
102}
103
104/// A type record (`SerializedType`), plain or ref type. Not `Clone`: it
105/// lives in its file's table, which paid for it.
106#[derive(Debug)]
107#[non_exhaustive]
108pub struct SerializedType {
109    /// `class_id`.
110    pub class_id: i32,
111    /// Format 16 and up.
112    pub is_stripped_type: Option<bool>,
113    /// Read from format 17, else -1; at 11-16 the last object's write-back.
114    pub script_type_index: i16,
115    /// `script_id`, under UnityPy's condition (format 13 and up): the bytes
116    /// `read_bytes(16)` returned, short at a negative position or the end.
117    pub script_id: Option<ShortArray<16>>,
118    /// `old_type_hash` (format 13 and up), as `script_id`.
119    pub old_type_hash: Option<ShortArray<16>>,
120    /// The stored tree, when type trees are enabled.
121    pub nodes: Option<crate::typetree::SharedNodes>,
122    /// Plain types, format 21 and up, type trees enabled.
123    pub type_dependencies: Option<Box<[i32]>>,
124    /// Ref types, format 21 and up, type trees enabled: `m_ClassName`.
125    pub class_name: Option<RawStr>,
126    /// `m_NameSpace`.
127    pub namespace: Option<RawStr>,
128    /// `m_AssemblyName`.
129    pub assembly: Option<RawStr>,
130    tree: ResultCell<Arc<Tree>, TreeError>,
131}
132
133impl SerializedType {
134    /// A plain type record: `class_id` and its stored tree, every other
135    /// field as a format below 13 leaves it (`script_type_index` -1).
136    #[must_use]
137    pub fn new(class_id: i32, nodes: Option<crate::typetree::SharedNodes>) -> Self {
138        Self {
139            class_id,
140            is_stripped_type: None,
141            script_type_index: -1,
142            script_id: None,
143            old_type_hash: None,
144            nodes,
145            type_dependencies: None,
146            class_name: None,
147            namespace: None,
148            assembly: None,
149            tree: OnceLock::new(),
150        }
151    }
152
153    /// A ref type (format 21 and up): its class, namespace and assembly
154    /// names and its stored tree.
155    #[must_use]
156    pub fn ref_type(
157        class_id: i32,
158        class_name: RawStr,
159        namespace: RawStr,
160        assembly: RawStr,
161        nodes: Option<crate::typetree::SharedNodes>,
162    ) -> Self {
163        Self {
164            class_name: Some(class_name),
165            namespace: Some(namespace),
166            assembly: Some(assembly),
167            ..Self::new(class_id, nodes)
168        }
169    }
170
171    /// The compiled tree: `Tree::compile` over [`Self::nodes`] (package B),
172    /// computed on first use and published once, so every object and every
173    /// reference of the type reuses one compile. The compile runs outside
174    /// the cell ([`get_or_publish_result`]): two first callers may both
175    /// compile, and the loser's tree is dropped. Its rows are charged to the
176    /// meter the `NodeList` was built on. A compile the meter or the budget
177    /// refuses is returned and not kept, so the next call compiles again;
178    /// a failure the rows decide (depth) is kept. `None` when the record
179    /// stores no tree.
180    pub fn tree(&self, limits: &Limits) -> Option<Result<Arc<Tree>, TreeError>> {
181        self.tree_with(|nodes| Tree::compile(nodes, limits))
182    }
183
184    /// [`Self::tree`] once it is compiled and kept, without compiling or
185    /// waiting: `None` before that, or after a failed compile.
186    #[must_use]
187    pub fn compiled_tree(&self) -> Option<&Arc<Tree>> {
188        self.tree.get()?.as_ref().ok()
189    }
190
191    fn tree_with(
192        &self,
193        compile: impl FnOnce(crate::typetree::SharedNodes) -> Result<Arc<Tree>, TreeError>,
194    ) -> Option<Result<Arc<Tree>, TreeError>> {
195        let nodes = self.nodes.as_ref()?;
196        Some(get_or_publish_result(&self.tree, nodes.meter(), || compile(nodes.clone())).cloned())
197    }
198}
199
200/// One row of the object table.
201#[derive(Clone, Debug, PartialEq, Eq)]
202#[non_exhaustive]
203pub struct ObjectInfo {
204    /// `path_id`.
205    pub path_id: i64,
206    /// Absolute within the `SerializedFile`: the stored `u32` (or format 22's
207    /// `i64`) plus `data_offset`, added as Python adds them, so it can pass
208    /// `i64::MAX` (`2^63 - 1 + 144` loads in UnityPy) or go negative; a read
209    /// from it is by the file's reader's rules ([`SerializedFile::reader_kind`]):
210    /// a Python slice on the memory reader ([`crate::stream::py_read`]).
211    pub byte_start: i128,
212    /// `byte_size`.
213    pub byte_size: u32,
214    /// `type_id` as stored.
215    pub type_id: i32,
216    /// Below format 16 the stored `u16`; from 16 the type's `class_id`.
217    pub class_id: i32,
218    /// The index into `types` this object resolves to, if any.
219    pub type_index: Option<usize>,
220    /// Below format 11.
221    pub is_destroyed: Option<u16>,
222    /// Formats 15 and 16.
223    pub is_stripped: Option<i8>,
224    /// Formats 11-16.
225    pub script_type_index: Option<i16>,
226}
227
228/// One external file reference.
229#[derive(Clone, Debug, PartialEq, Eq)]
230#[non_exhaustive]
231pub struct External {
232    /// `temp_empty` (format 6 and up).
233    pub temp_empty: Option<RawStr>,
234    /// `guid` (format 5 and up): the bytes `read_bytes(16)` returned, short
235    /// at a negative position or the end.
236    pub guid: Option<ShortArray<16>>,
237    /// `type` (format 5 and up).
238    pub kind: Option<i32>,
239    /// `path`.
240    pub path: RawStr,
241}
242
243impl External {
244    /// A writable reference to `path`, with empty GUID and type zero.
245    #[must_use]
246    pub fn new(path: RawStr) -> Self {
247        Self {
248            temp_empty: Some(RawStr::from_static(b"")),
249            guid: Some([0; 16].into()),
250            kind: Some(0),
251            path,
252        }
253    }
254}
255
256/// One script type reference (`LocalSerializedObjectIdentifier`).
257#[derive(Clone, Copy, Debug, PartialEq, Eq)]
258#[non_exhaustive]
259pub struct ScriptRef {
260    /// `local_serialized_file_index`.
261    pub file_index: i32,
262    /// `local_identifier_in_file`.
263    pub identifier: i64,
264}
265
266/// A parsed `SerializedFile`: metadata and object table. Values are read on
267/// demand through [`ObjectRef`].
268pub struct SerializedFile {
269    changed: std::sync::atomic::AtomicBool,
270    edits: std::sync::Mutex<crate::limits::MeteredVec<(u32, Bytes)>>,
271    head_handlers: crate::limits::MeteredVec<head::Handler>,
272    head_roots: usize,
273    pub(super) pending_externals: std::sync::Mutex<crate::limits::MeteredVec<External>>,
274    name: RawStr,
275    data: Bytes,
276    header: SerializedHeader,
277    unity_version: Option<RawStr>,
278    version: Option<ResolvedVersion>,
279    target_platform: Option<i32>,
280    enable_type_tree: bool,
281    big_id_enabled: i32,
282    types: Box<[SerializedType]>,
283    ref_types: Option<Box<[SerializedType]>>,
284    script_types: Box<[ScriptRef]>,
285    externals: Box<[External]>,
286    user_information: Option<RawStr>,
287    object_infos: Box<[ObjectInfo]>,
288    object_order: Box<[u32]>,
289    /// The dict's entries (indexes into `object_infos`) sorted by path id,
290    /// for [`Self::object`].
291    by_path: Box<[u32]>,
292    /// Published by [`get_or_publish_result`]: a refused charge is not kept.
293    container: ResultCell<Container, Error>,
294    parent: Option<Parent>,
295    pub(super) dependency_path: super::DependencyPath,
296    /// The reader UnityPy reads it with, for its metadata and its objects.
297    reader: ReaderKind,
298    constructor_position: i128,
299    /// What its reads and its parse copied out of the buffer its bytes are
300    /// a slice of ([`Self::copies`]).
301    copies: CopyCounts,
302    /// The handle and every table above, charged to the input, with the
303    /// resolved version's heap ([`crate::version::UnityVersion::heap_bytes`], charged
304    /// by the chain before it was allocated;
305    /// its string is the file's own slice, or the parent's, never a copy).
306    charge: Charge,
307    ctx: Arc<FileContext>,
308}
309
310impl std::fmt::Debug for SerializedFile {
311    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
312        f.debug_struct("SerializedFile")
313            .field("name", &self.name)
314            .field("header", &self.header)
315            .field("unity_version", &self.unity_version)
316            .field("types", &self.types.len())
317            .field("objects", &self.object_order.len())
318            .finish_non_exhaustive()
319    }
320}
321
322impl SerializedFile {
323    /// The identity of this file occurrence while it is held. A dependency
324    /// keeps the same input address and directory path across cache reopens.
325    #[must_use]
326    pub fn occurrence_key(&self) -> (usize, &[usize]) {
327        self.dependency_path.0.get().map_or_else(
328            || (std::ptr::from_ref(self).addr(), &[][..]),
329            |path| (self.meter().input_address(), path.as_slice()),
330        )
331    }
332
333    /// The meter of the input it belongs to: what reading its objects
334    /// charges ([`crate::walk::ReadCtx::meter`]).
335    pub fn meter(&self) -> &Arc<Meter> {
336        self.charge.meter()
337    }
338
339    /// The name it was loaded under.
340    pub fn name(&self) -> &RawStr {
341        &self.name
342    }
343
344    /// The whole file's bytes.
345    pub fn data(&self) -> &Bytes {
346        &self.data
347    }
348
349    /// The length of the buffer its bytes are a slice of, which a slice of
350    /// them keeps alive: its container's decoded run, or the container's
351    /// own bytes' buffer ([`Parent::pinned`]); its own length for a
352    /// top-level file. A value read from it, and each typetree it parsed,
353    /// copy their slices out when they would keep much more than
354    /// themselves alive ([`crate::pin`]).
355    #[must_use]
356    pub fn pinned(&self) -> u64 {
357        pinned(self.parent.as_ref(), &self.data)
358    }
359
360    /// What its object reads and its parse copied out of the buffer its
361    /// bytes are a slice of so far ([`crate::pin`]; plan section 8, R3).
362    #[must_use]
363    pub fn copies(&self) -> Copies {
364        self.copies.get()
365    }
366
367    /// Copies `value`'s slices of this file out of the buffer they keep
368    /// alive, when they would keep much more than themselves alive
369    /// ([`crate::pin`]), and counts the copy: the bytes copied.
370    pub(crate) fn unpin(
371        &self,
372        value: &mut Parsed,
373        caller: Option<&Arc<Meter>>,
374    ) -> Result<(), ChargeError> {
375        let target = value.value_mut();
376        let n = if let Some(meter) = caller {
377            crate::pin::try_copy_out(target, &self.data, self.pinned(), meter, true)?
378        } else {
379            crate::pin::copy_out(target, &self.data, self.pinned(), self.meter())
380        };
381        self.copies.value(n);
382        Ok(())
383    }
384
385    /// Final metadata cursor of the constructor's input reader.
386    #[must_use]
387    pub fn constructor_position(&self) -> i128 {
388        self.constructor_position
389    }
390
391    /// The reader used for metadata and object reads: stream for paths,
392    /// otherwise memory ([`ObjectRef::read_typetree`], [`ObjectRef::raw_data`]).
393    pub fn reader_kind(&self) -> ReaderKind {
394        self.reader
395    }
396
397    /// The shared reader's position after loading metadata and the eager container.
398    pub fn reader_position(&self) -> i128 {
399        self.constructor_position
400    }
401
402    /// The header.
403    pub fn header(&self) -> &SerializedHeader {
404        &self.header
405    }
406
407    /// The version string as read (format 7 and up).
408    pub fn unity_version(&self) -> Option<&RawStr> {
409        self.unity_version.as_ref()
410    }
411
412    /// Chain 2's result (format 7 and up).
413    pub fn version(&self) -> Option<&ResolvedVersion> {
414        self.version.as_ref()
415    }
416
417    /// The raw target platform (format 8 and up).
418    pub fn target_platform(&self) -> Option<i32> {
419        self.target_platform
420    }
421
422    /// Whether type records carry trees.
423    pub fn enable_type_tree(&self) -> bool {
424        self.enable_type_tree
425    }
426
427    /// `big_id_enabled` (formats 7-13, else 0).
428    pub fn big_id_enabled(&self) -> i32 {
429        self.big_id_enabled
430    }
431
432    /// Type records, in file order, script indexes written back.
433    pub fn types(&self) -> &[SerializedType] {
434        &self.types
435    }
436
437    /// Ref types (format 20 and up).
438    pub fn ref_types(&self) -> Option<&[SerializedType]> {
439        self.ref_types.as_deref()
440    }
441
442    /// The reader's view of the ref types.
443    pub fn ref_types_view(&self) -> RefTypes<'_> {
444        match &self.ref_types {
445            Some(t) => RefTypes::List(t),
446            None => RefTypes::Absent,
447        }
448    }
449
450    /// Script type references.
451    pub fn script_types(&self) -> &[ScriptRef] {
452        &self.script_types
453    }
454
455    /// External file references.
456    pub fn externals(&self) -> &[External] {
457        &self.externals
458    }
459
460    /// `userInformation` (format 5 and up).
461    pub fn user_information(&self) -> Option<&RawStr> {
462        self.user_information.as_ref()
463    }
464
465    /// The object table in file order, repeated path ids kept.
466    pub fn object_infos(&self) -> &[ObjectInfo] {
467        &self.object_infos
468    }
469
470    /// UnityPy's `objects` dict order: indexes into [`Self::object_infos`] of
471    /// each path id's last entry, at its first position.
472    pub fn object_order(&self) -> &[u32] {
473        &self.object_order
474    }
475
476    /// The objects, in dict order.
477    pub fn objects(self: &Arc<Self>) -> impl Iterator<Item = ObjectRef> + '_ {
478        self.object_order
479            .iter()
480            .filter_map(|i| ObjectRef::new(self.clone(), *i))
481    }
482
483    /// The objects in ascending signed path id (UnityPy's
484    /// `sorted(objects.values(), key=lambda o: o.path_id)`): the table
485    /// [`Self::object`] searches, kept and charged with the file, so a
486    /// caller orders them without a copy or a sort.
487    pub fn objects_by_path(self: &Arc<Self>) -> impl Iterator<Item = ObjectRef> + '_ {
488        self.by_path
489            .iter()
490            .filter_map(|i| ObjectRef::new(self.clone(), *i))
491    }
492
493    /// The object `objects[path_id]` holds.
494    pub fn object(self: &Arc<Self>, path_id: i64) -> Option<ObjectRef> {
495        let at = self
496            .by_path
497            .binary_search_by_key(&path_id, |i| {
498                usize::try_from(*i)
499                    .ok()
500                    .and_then(|i| self.object_infos.get(i))
501                    .map_or(i64::MIN, |o| o.path_id)
502            })
503            .ok()?;
504        ObjectRef::new(self.clone(), *self.by_path.get(at)?)
505    }
506
507    /// The container rows of the first AssetBundle in dict order, read once
508    /// ([`crate::container::read_container`]) and published
509    /// ([`get_or_publish_result`]: the read runs outside the cell, and a
510    /// refused charge is returned, not kept). What the read was granted and
511    /// its container does not keep is the open's work (module docs, "Work").
512    pub fn container(self: &Arc<Self>) -> Result<&Container, Error> {
513        self.container_counted(true)
514    }
515
516    /// [`Self::container`] as the load reads it (`SerializedFile.py:316-324`):
517    /// a failure fails the parse, whose caller counts its grants.
518    fn container_at_load(self: &Arc<Self>) -> Result<&Container, Error> {
519        self.container_counted(false)
520    }
521
522    /// The container, its read's grants counted as work: on success what its
523    /// container does not keep, on a failure its cell keeps all of it when
524    /// `failure_counts`. A refusal the cell does not keep is
525    /// [`get_or_publish_result`]'s to count.
526    fn container_counted(self: &Arc<Self>, failure_counts: bool) -> Result<&Container, Error> {
527        let meter = self.meter();
528        get_or_publish_result(&self.container, meter, || {
529            let scope = GrantScope::enter(meter);
530            let got = read_container(self);
531            let granted = scope.finish(meter);
532            match &got {
533                Ok(c) => meter.add_work(granted.saturating_sub(c.charge().bytes())),
534                Err(e) if failure_counts && !e.is_transient() => meter.add_work(granted),
535                Err(_) => {}
536            }
537            got
538        })
539    }
540
541    /// The container this file is an entry of.
542    pub fn parent(&self) -> Option<&Parent> {
543        self.parent.as_ref()
544    }
545
546    /// The context it was loaded with.
547    pub fn context(&self) -> &Arc<FileContext> {
548        &self.ctx
549    }
550
551    /// A file of `rows` and nothing else, for the crate's tests before
552    /// package C's `parse` lands: dict order is file order.
553    #[cfg(test)]
554    pub(crate) fn for_tests(rows: Vec<ObjectInfo>) -> Arc<Self> {
555        let meter = Meter::unlimited();
556        let order = (0..u32::try_from(rows.len()).unwrap()).collect();
557        Arc::new(Self {
558            changed: std::sync::atomic::AtomicBool::new(false),
559            edits: std::sync::Mutex::new(crate::limits::MeteredVec::new(&meter)),
560            head_handlers: crate::limits::MeteredVec::new(&meter),
561            head_roots: 0,
562            pending_externals: std::sync::Mutex::new(crate::limits::MeteredVec::new(&meter)),
563            name: RawStr::from_static(b"CAB-test"),
564            data: Bytes::new(),
565            header: SerializedHeader {
566                metadata_size: 0,
567                file_size: 0,
568                version: 22,
569                data_offset: 0,
570                endian: Endian::Little,
571                reserved: None,
572                unknown: 0,
573            },
574            unity_version: None,
575            version: None,
576            target_platform: None,
577            enable_type_tree: true,
578            big_id_enabled: 0,
579            types: Box::new([]),
580            ref_types: None,
581            script_types: Box::new([]),
582            externals: Box::new([]),
583            user_information: None,
584            by_path: Box::new([]),
585            object_infos: rows.into_boxed_slice(),
586            object_order: order,
587            container: ResultCell::new(),
588            parent: None,
589            dependency_path: super::DependencyPath::default(),
590            reader: ReaderKind::Memory,
591            constructor_position: 0,
592            copies: CopyCounts::default(),
593            charge: Charge::zero(&meter),
594            ctx: Arc::new(FileContext::unitypy()),
595        })
596    }
597}
598
599/// [`SerializedFile::pinned`] for a file of `data` under `parent`: what its
600/// container says the buffer is, never less than its own bytes.
601pub(crate) fn pinned(parent: Option<&Parent>, data: &Bytes) -> u64 {
602    let own = u64::try_from(data.len()).unwrap_or(u64::MAX);
603    parent.map_or(own, |p| p.pinned.max(own))
604}
605
606#[cfg(test)]
607mod tests;