unityrs_core/files/serialized/mod.rs
1// Ported from UnityPy 1.25.2 (MIT): UnityPy/files/SerializedFile.py, UnityPy/files/ObjectReader.py. See THIRD_PARTY_LICENSES.
2//! `SerializedFile` metadata (`files/SerializedFile.py:206-360`,
3//! `files/ObjectReader.py:67-130`), ported from UnityPy, bytes-native.
4//!
5//! Rules ported exactly (plan section 3.7):
6//! - the header (`SerializedFile.py:255-269`): the endian byte at offset 16
7//! from format 9, else at `file_size - metadata_size` (absolute); the
8//! format-22 fields replace the legacy `u32`s;
9//! - gates (`SerializedFile.py:271-288`): the version string from format 7, the target platform
10//! from 8, `enable_type_tree` from 13 (else true), `big_id_enabled` at 7-13;
11//! - type records (`SerializedFile.py:112-149`): `is_stripped_type` from 16, `script_type_index`
12//! from 17 (else -1), `script_id` and `old_type_hash` from 13 under
13//! UnityPy's condition, trees only when `enable_type_tree`, blobs at format
14//! 10 and 12 and up; at 21 ref types read three names and plain types a
15//! dependency list;
16//! - objects (`ObjectReader.py:73-130`): path id as `i64` under bigID, `i32`
17//! below 14, else aligned `i64`; `byte_start` `u32` (or `i64` from 22) plus
18//! `data_offset`; below 16 the type is the first whose `class_id` equals the
19//! object's `type_id`, from 16 `types[type_id]` with Python indexing (a
20//! negative id counts from the end; out of range fails the file); at 11-16 the
21//! object's script index is written back into its type, the last object
22//! winning; the `objects` dict keeps a repeated path id at its first position
23//! with its last entry;
24//! - `read_string_to_null`'s 32,767-byte window for every metadata string;
25//! - the reader UnityPy reads the file with ([`crate::stream::ReaderKind`]):
26//! the memory reader for bytes and every entry of a container, the stream
27//! reader for a file loaded from a path, whose rules the metadata's reads
28//! and every object read follow ([`SerializedFile::reader_kind`]);
29//! - every table is built with `range(count)` (`SerializedFile.py:282-308`):
30//! a negative count is zero records, and a table grows record by record, so
31//! no count sizes an allocation;
32//! - with [`FileContext::eager_asset_bundle`], the first AssetBundle in dict
33//! order is read at load with `check_read`, and a failure fails the file
34//! (`SerializedFile.py:316-324`). UnityPy reads it in object mode
35//! (`parse_as_object`); the core reads it in dict mode, and package D's
36//! fixtures pin that the two agree (plan section 3.7).
37//!
38//! **Memory.** Every table (types, their dependency lists and trees, objects,
39//! the dict order, scripts, externals, ref types) is built in a
40//! [`crate::limits::MeteredVec`] on the input's meter, and the file keeps the
41//! charges: two directory rows over the same bytes parse twice and pay twice.
42//! Strings are zero-copy slices of the file.
43//!
44//! **Work.** The container read ([`SerializedFile::container`]) builds the
45//! whole AssetBundle value and keeps only its rows and preload table, so
46//! two directory rows over the same bytes would repeat a read whose memory
47//! is returned: what the meter granted the read on its own thread, beyond
48//! what its [`Container`] keeps, counts as the open's work
49//! ([`crate::error::LimitKind::DecodeWork`]), and so does all it was
50//! granted when it fails with an error its cell keeps. At load, a failure
51//! fails the parse, whose grants its bundle counts instead
52//! (`files::bundle`, "Memory and work"), so they count once (round 5).
53//!
54//! Owner: package C. The accessors and [`SerializedType`]'s constructors and
55//! [`SerializedType::tree`] are implemented by the scaffold, so package B can
56//! build ref types for its tests before C lands.
57
58use std::sync::{Arc, OnceLock};
59
60use bytes::Bytes;
61
62use crate::container::{Container, read_container};
63use crate::error::Error;
64use crate::files::Parent;
65use crate::limits::{Charge, ChargeError, GrantScope, Limits, Meter};
66use crate::object::ObjectRef;
67use crate::once::{CellError, ResultCell, get_or_publish_result};
68use crate::options::FileContext;
69use crate::pin::{Copies, CopyCounts};
70use crate::raw::RawStr;
71use crate::stream::{Endian, ReaderKind, ShortArray};
72use crate::typetree::{Tree, TreeError};
73use crate::value::Parsed;
74use crate::version::ResolvedVersion;
75use crate::walk::RefTypes;
76
77mod changed;
78mod head;
79mod parse;
80mod write;
81
82pub use write::SerializedSaveSnapshot;
83
84/// The header, as read.
85#[derive(Clone, Debug, PartialEq, Eq)]
86#[non_exhaustive]
87pub struct SerializedHeader {
88 /// `metadata_size`.
89 pub metadata_size: u32,
90 /// `file_size`.
91 pub file_size: i64,
92 /// The format number (UnityPy's `header.version`).
93 pub version: u32,
94 /// `data_offset`.
95 pub data_offset: i64,
96 /// The byte order.
97 pub endian: Endian,
98 /// The three reserved bytes (format 9 and up).
99 pub reserved: Option<[u8; 3]>,
100 /// Format 22's trailing `i64` (UnityPy's `unknown`), else 0.
101 pub unknown: i64,
102}
103
104/// A type record (`SerializedType`), plain or ref type. Not `Clone`: it
105/// lives in its file's table, which paid for it.
106#[derive(Debug)]
107#[non_exhaustive]
108pub struct SerializedType {
109 /// `class_id`.
110 pub class_id: i32,
111 /// Format 16 and up.
112 pub is_stripped_type: Option<bool>,
113 /// Read from format 17, else -1; at 11-16 the last object's write-back.
114 pub script_type_index: i16,
115 /// `script_id`, under UnityPy's condition (format 13 and up): the bytes
116 /// `read_bytes(16)` returned, short at a negative position or the end.
117 pub script_id: Option<ShortArray<16>>,
118 /// `old_type_hash` (format 13 and up), as `script_id`.
119 pub old_type_hash: Option<ShortArray<16>>,
120 /// The stored tree, when type trees are enabled.
121 pub nodes: Option<crate::typetree::SharedNodes>,
122 /// Plain types, format 21 and up, type trees enabled.
123 pub type_dependencies: Option<Box<[i32]>>,
124 /// Ref types, format 21 and up, type trees enabled: `m_ClassName`.
125 pub class_name: Option<RawStr>,
126 /// `m_NameSpace`.
127 pub namespace: Option<RawStr>,
128 /// `m_AssemblyName`.
129 pub assembly: Option<RawStr>,
130 tree: ResultCell<Arc<Tree>, TreeError>,
131}
132
133impl SerializedType {
134 /// A plain type record: `class_id` and its stored tree, every other
135 /// field as a format below 13 leaves it (`script_type_index` -1).
136 #[must_use]
137 pub fn new(class_id: i32, nodes: Option<crate::typetree::SharedNodes>) -> Self {
138 Self {
139 class_id,
140 is_stripped_type: None,
141 script_type_index: -1,
142 script_id: None,
143 old_type_hash: None,
144 nodes,
145 type_dependencies: None,
146 class_name: None,
147 namespace: None,
148 assembly: None,
149 tree: OnceLock::new(),
150 }
151 }
152
153 /// A ref type (format 21 and up): its class, namespace and assembly
154 /// names and its stored tree.
155 #[must_use]
156 pub fn ref_type(
157 class_id: i32,
158 class_name: RawStr,
159 namespace: RawStr,
160 assembly: RawStr,
161 nodes: Option<crate::typetree::SharedNodes>,
162 ) -> Self {
163 Self {
164 class_name: Some(class_name),
165 namespace: Some(namespace),
166 assembly: Some(assembly),
167 ..Self::new(class_id, nodes)
168 }
169 }
170
171 /// The compiled tree: `Tree::compile` over [`Self::nodes`] (package B),
172 /// computed on first use and published once, so every object and every
173 /// reference of the type reuses one compile. The compile runs outside
174 /// the cell ([`get_or_publish_result`]): two first callers may both
175 /// compile, and the loser's tree is dropped. Its rows are charged to the
176 /// meter the `NodeList` was built on. A compile the meter or the budget
177 /// refuses is returned and not kept, so the next call compiles again;
178 /// a failure the rows decide (depth) is kept. `None` when the record
179 /// stores no tree.
180 pub fn tree(&self, limits: &Limits) -> Option<Result<Arc<Tree>, TreeError>> {
181 self.tree_with(|nodes| Tree::compile(nodes, limits))
182 }
183
184 /// [`Self::tree`] once it is compiled and kept, without compiling or
185 /// waiting: `None` before that, or after a failed compile.
186 #[must_use]
187 pub fn compiled_tree(&self) -> Option<&Arc<Tree>> {
188 self.tree.get()?.as_ref().ok()
189 }
190
191 fn tree_with(
192 &self,
193 compile: impl FnOnce(crate::typetree::SharedNodes) -> Result<Arc<Tree>, TreeError>,
194 ) -> Option<Result<Arc<Tree>, TreeError>> {
195 let nodes = self.nodes.as_ref()?;
196 Some(get_or_publish_result(&self.tree, nodes.meter(), || compile(nodes.clone())).cloned())
197 }
198}
199
200/// One row of the object table.
201#[derive(Clone, Debug, PartialEq, Eq)]
202#[non_exhaustive]
203pub struct ObjectInfo {
204 /// `path_id`.
205 pub path_id: i64,
206 /// Absolute within the `SerializedFile`: the stored `u32` (or format 22's
207 /// `i64`) plus `data_offset`, added as Python adds them, so it can pass
208 /// `i64::MAX` (`2^63 - 1 + 144` loads in UnityPy) or go negative; a read
209 /// from it is by the file's reader's rules ([`SerializedFile::reader_kind`]):
210 /// a Python slice on the memory reader ([`crate::stream::py_read`]).
211 pub byte_start: i128,
212 /// `byte_size`.
213 pub byte_size: u32,
214 /// `type_id` as stored.
215 pub type_id: i32,
216 /// Below format 16 the stored `u16`; from 16 the type's `class_id`.
217 pub class_id: i32,
218 /// The index into `types` this object resolves to, if any.
219 pub type_index: Option<usize>,
220 /// Below format 11.
221 pub is_destroyed: Option<u16>,
222 /// Formats 15 and 16.
223 pub is_stripped: Option<i8>,
224 /// Formats 11-16.
225 pub script_type_index: Option<i16>,
226}
227
228/// One external file reference.
229#[derive(Clone, Debug, PartialEq, Eq)]
230#[non_exhaustive]
231pub struct External {
232 /// `temp_empty` (format 6 and up).
233 pub temp_empty: Option<RawStr>,
234 /// `guid` (format 5 and up): the bytes `read_bytes(16)` returned, short
235 /// at a negative position or the end.
236 pub guid: Option<ShortArray<16>>,
237 /// `type` (format 5 and up).
238 pub kind: Option<i32>,
239 /// `path`.
240 pub path: RawStr,
241}
242
243impl External {
244 /// A writable reference to `path`, with empty GUID and type zero.
245 #[must_use]
246 pub fn new(path: RawStr) -> Self {
247 Self {
248 temp_empty: Some(RawStr::from_static(b"")),
249 guid: Some([0; 16].into()),
250 kind: Some(0),
251 path,
252 }
253 }
254}
255
256/// One script type reference (`LocalSerializedObjectIdentifier`).
257#[derive(Clone, Copy, Debug, PartialEq, Eq)]
258#[non_exhaustive]
259pub struct ScriptRef {
260 /// `local_serialized_file_index`.
261 pub file_index: i32,
262 /// `local_identifier_in_file`.
263 pub identifier: i64,
264}
265
266/// A parsed `SerializedFile`: metadata and object table. Values are read on
267/// demand through [`ObjectRef`].
268pub struct SerializedFile {
269 changed: std::sync::atomic::AtomicBool,
270 edits: std::sync::Mutex<crate::limits::MeteredVec<(u32, Bytes)>>,
271 head_handlers: crate::limits::MeteredVec<head::Handler>,
272 head_roots: usize,
273 pub(super) pending_externals: std::sync::Mutex<crate::limits::MeteredVec<External>>,
274 name: RawStr,
275 data: Bytes,
276 header: SerializedHeader,
277 unity_version: Option<RawStr>,
278 version: Option<ResolvedVersion>,
279 target_platform: Option<i32>,
280 enable_type_tree: bool,
281 big_id_enabled: i32,
282 types: Box<[SerializedType]>,
283 ref_types: Option<Box<[SerializedType]>>,
284 script_types: Box<[ScriptRef]>,
285 externals: Box<[External]>,
286 user_information: Option<RawStr>,
287 object_infos: Box<[ObjectInfo]>,
288 object_order: Box<[u32]>,
289 /// The dict's entries (indexes into `object_infos`) sorted by path id,
290 /// for [`Self::object`].
291 by_path: Box<[u32]>,
292 /// Published by [`get_or_publish_result`]: a refused charge is not kept.
293 container: ResultCell<Container, Error>,
294 parent: Option<Parent>,
295 pub(super) dependency_path: super::DependencyPath,
296 /// The reader UnityPy reads it with, for its metadata and its objects.
297 reader: ReaderKind,
298 constructor_position: i128,
299 /// What its reads and its parse copied out of the buffer its bytes are
300 /// a slice of ([`Self::copies`]).
301 copies: CopyCounts,
302 /// The handle and every table above, charged to the input, with the
303 /// resolved version's heap ([`crate::version::UnityVersion::heap_bytes`], charged
304 /// by the chain before it was allocated;
305 /// its string is the file's own slice, or the parent's, never a copy).
306 charge: Charge,
307 ctx: Arc<FileContext>,
308}
309
310impl std::fmt::Debug for SerializedFile {
311 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
312 f.debug_struct("SerializedFile")
313 .field("name", &self.name)
314 .field("header", &self.header)
315 .field("unity_version", &self.unity_version)
316 .field("types", &self.types.len())
317 .field("objects", &self.object_order.len())
318 .finish_non_exhaustive()
319 }
320}
321
322impl SerializedFile {
323 /// The identity of this file occurrence while it is held. A dependency
324 /// keeps the same input address and directory path across cache reopens.
325 #[must_use]
326 pub fn occurrence_key(&self) -> (usize, &[usize]) {
327 self.dependency_path.0.get().map_or_else(
328 || (std::ptr::from_ref(self).addr(), &[][..]),
329 |path| (self.meter().input_address(), path.as_slice()),
330 )
331 }
332
333 /// The meter of the input it belongs to: what reading its objects
334 /// charges ([`crate::walk::ReadCtx::meter`]).
335 pub fn meter(&self) -> &Arc<Meter> {
336 self.charge.meter()
337 }
338
339 /// The name it was loaded under.
340 pub fn name(&self) -> &RawStr {
341 &self.name
342 }
343
344 /// The whole file's bytes.
345 pub fn data(&self) -> &Bytes {
346 &self.data
347 }
348
349 /// The length of the buffer its bytes are a slice of, which a slice of
350 /// them keeps alive: its container's decoded run, or the container's
351 /// own bytes' buffer ([`Parent::pinned`]); its own length for a
352 /// top-level file. A value read from it, and each typetree it parsed,
353 /// copy their slices out when they would keep much more than
354 /// themselves alive ([`crate::pin`]).
355 #[must_use]
356 pub fn pinned(&self) -> u64 {
357 pinned(self.parent.as_ref(), &self.data)
358 }
359
360 /// What its object reads and its parse copied out of the buffer its
361 /// bytes are a slice of so far ([`crate::pin`]; plan section 8, R3).
362 #[must_use]
363 pub fn copies(&self) -> Copies {
364 self.copies.get()
365 }
366
367 /// Copies `value`'s slices of this file out of the buffer they keep
368 /// alive, when they would keep much more than themselves alive
369 /// ([`crate::pin`]), and counts the copy: the bytes copied.
370 pub(crate) fn unpin(
371 &self,
372 value: &mut Parsed,
373 caller: Option<&Arc<Meter>>,
374 ) -> Result<(), ChargeError> {
375 let target = value.value_mut();
376 let n = if let Some(meter) = caller {
377 crate::pin::try_copy_out(target, &self.data, self.pinned(), meter, true)?
378 } else {
379 crate::pin::copy_out(target, &self.data, self.pinned(), self.meter())
380 };
381 self.copies.value(n);
382 Ok(())
383 }
384
385 /// Final metadata cursor of the constructor's input reader.
386 #[must_use]
387 pub fn constructor_position(&self) -> i128 {
388 self.constructor_position
389 }
390
391 /// The reader used for metadata and object reads: stream for paths,
392 /// otherwise memory ([`ObjectRef::read_typetree`], [`ObjectRef::raw_data`]).
393 pub fn reader_kind(&self) -> ReaderKind {
394 self.reader
395 }
396
397 /// The shared reader's position after loading metadata and the eager container.
398 pub fn reader_position(&self) -> i128 {
399 self.constructor_position
400 }
401
402 /// The header.
403 pub fn header(&self) -> &SerializedHeader {
404 &self.header
405 }
406
407 /// The version string as read (format 7 and up).
408 pub fn unity_version(&self) -> Option<&RawStr> {
409 self.unity_version.as_ref()
410 }
411
412 /// Chain 2's result (format 7 and up).
413 pub fn version(&self) -> Option<&ResolvedVersion> {
414 self.version.as_ref()
415 }
416
417 /// The raw target platform (format 8 and up).
418 pub fn target_platform(&self) -> Option<i32> {
419 self.target_platform
420 }
421
422 /// Whether type records carry trees.
423 pub fn enable_type_tree(&self) -> bool {
424 self.enable_type_tree
425 }
426
427 /// `big_id_enabled` (formats 7-13, else 0).
428 pub fn big_id_enabled(&self) -> i32 {
429 self.big_id_enabled
430 }
431
432 /// Type records, in file order, script indexes written back.
433 pub fn types(&self) -> &[SerializedType] {
434 &self.types
435 }
436
437 /// Ref types (format 20 and up).
438 pub fn ref_types(&self) -> Option<&[SerializedType]> {
439 self.ref_types.as_deref()
440 }
441
442 /// The reader's view of the ref types.
443 pub fn ref_types_view(&self) -> RefTypes<'_> {
444 match &self.ref_types {
445 Some(t) => RefTypes::List(t),
446 None => RefTypes::Absent,
447 }
448 }
449
450 /// Script type references.
451 pub fn script_types(&self) -> &[ScriptRef] {
452 &self.script_types
453 }
454
455 /// External file references.
456 pub fn externals(&self) -> &[External] {
457 &self.externals
458 }
459
460 /// `userInformation` (format 5 and up).
461 pub fn user_information(&self) -> Option<&RawStr> {
462 self.user_information.as_ref()
463 }
464
465 /// The object table in file order, repeated path ids kept.
466 pub fn object_infos(&self) -> &[ObjectInfo] {
467 &self.object_infos
468 }
469
470 /// UnityPy's `objects` dict order: indexes into [`Self::object_infos`] of
471 /// each path id's last entry, at its first position.
472 pub fn object_order(&self) -> &[u32] {
473 &self.object_order
474 }
475
476 /// The objects, in dict order.
477 pub fn objects(self: &Arc<Self>) -> impl Iterator<Item = ObjectRef> + '_ {
478 self.object_order
479 .iter()
480 .filter_map(|i| ObjectRef::new(self.clone(), *i))
481 }
482
483 /// The objects in ascending signed path id (UnityPy's
484 /// `sorted(objects.values(), key=lambda o: o.path_id)`): the table
485 /// [`Self::object`] searches, kept and charged with the file, so a
486 /// caller orders them without a copy or a sort.
487 pub fn objects_by_path(self: &Arc<Self>) -> impl Iterator<Item = ObjectRef> + '_ {
488 self.by_path
489 .iter()
490 .filter_map(|i| ObjectRef::new(self.clone(), *i))
491 }
492
493 /// The object `objects[path_id]` holds.
494 pub fn object(self: &Arc<Self>, path_id: i64) -> Option<ObjectRef> {
495 let at = self
496 .by_path
497 .binary_search_by_key(&path_id, |i| {
498 usize::try_from(*i)
499 .ok()
500 .and_then(|i| self.object_infos.get(i))
501 .map_or(i64::MIN, |o| o.path_id)
502 })
503 .ok()?;
504 ObjectRef::new(self.clone(), *self.by_path.get(at)?)
505 }
506
507 /// The container rows of the first AssetBundle in dict order, read once
508 /// ([`crate::container::read_container`]) and published
509 /// ([`get_or_publish_result`]: the read runs outside the cell, and a
510 /// refused charge is returned, not kept). What the read was granted and
511 /// its container does not keep is the open's work (module docs, "Work").
512 pub fn container(self: &Arc<Self>) -> Result<&Container, Error> {
513 self.container_counted(true)
514 }
515
516 /// [`Self::container`] as the load reads it (`SerializedFile.py:316-324`):
517 /// a failure fails the parse, whose caller counts its grants.
518 fn container_at_load(self: &Arc<Self>) -> Result<&Container, Error> {
519 self.container_counted(false)
520 }
521
522 /// The container, its read's grants counted as work: on success what its
523 /// container does not keep, on a failure its cell keeps all of it when
524 /// `failure_counts`. A refusal the cell does not keep is
525 /// [`get_or_publish_result`]'s to count.
526 fn container_counted(self: &Arc<Self>, failure_counts: bool) -> Result<&Container, Error> {
527 let meter = self.meter();
528 get_or_publish_result(&self.container, meter, || {
529 let scope = GrantScope::enter(meter);
530 let got = read_container(self);
531 let granted = scope.finish(meter);
532 match &got {
533 Ok(c) => meter.add_work(granted.saturating_sub(c.charge().bytes())),
534 Err(e) if failure_counts && !e.is_transient() => meter.add_work(granted),
535 Err(_) => {}
536 }
537 got
538 })
539 }
540
541 /// The container this file is an entry of.
542 pub fn parent(&self) -> Option<&Parent> {
543 self.parent.as_ref()
544 }
545
546 /// The context it was loaded with.
547 pub fn context(&self) -> &Arc<FileContext> {
548 &self.ctx
549 }
550
551 /// A file of `rows` and nothing else, for the crate's tests before
552 /// package C's `parse` lands: dict order is file order.
553 #[cfg(test)]
554 pub(crate) fn for_tests(rows: Vec<ObjectInfo>) -> Arc<Self> {
555 let meter = Meter::unlimited();
556 let order = (0..u32::try_from(rows.len()).unwrap()).collect();
557 Arc::new(Self {
558 changed: std::sync::atomic::AtomicBool::new(false),
559 edits: std::sync::Mutex::new(crate::limits::MeteredVec::new(&meter)),
560 head_handlers: crate::limits::MeteredVec::new(&meter),
561 head_roots: 0,
562 pending_externals: std::sync::Mutex::new(crate::limits::MeteredVec::new(&meter)),
563 name: RawStr::from_static(b"CAB-test"),
564 data: Bytes::new(),
565 header: SerializedHeader {
566 metadata_size: 0,
567 file_size: 0,
568 version: 22,
569 data_offset: 0,
570 endian: Endian::Little,
571 reserved: None,
572 unknown: 0,
573 },
574 unity_version: None,
575 version: None,
576 target_platform: None,
577 enable_type_tree: true,
578 big_id_enabled: 0,
579 types: Box::new([]),
580 ref_types: None,
581 script_types: Box::new([]),
582 externals: Box::new([]),
583 user_information: None,
584 by_path: Box::new([]),
585 object_infos: rows.into_boxed_slice(),
586 object_order: order,
587 container: ResultCell::new(),
588 parent: None,
589 dependency_path: super::DependencyPath::default(),
590 reader: ReaderKind::Memory,
591 constructor_position: 0,
592 copies: CopyCounts::default(),
593 charge: Charge::zero(&meter),
594 ctx: Arc::new(FileContext::unitypy()),
595 })
596 }
597}
598
599/// [`SerializedFile::pinned`] for a file of `data` under `parent`: what its
600/// container says the buffer is, never less than its own bytes.
601pub(crate) fn pinned(parent: Option<&Parent>, data: &Bytes) -> u64 {
602 let own = u64::try_from(data.len()).unwrap_or(u64::MAX);
603 parent.map_or(own, |p| p.pinned.max(own))
604}
605
606#[cfg(test)]
607mod tests;