Skip to main content

unityrs_testkit/
serialized.rs

1// Ported from UnityPy 1.25.2 (MIT): UnityPy/files/SerializedFile.py, UnityPy/files/ObjectReader.py. See THIRD_PARTY_LICENSES.
2//! `SerializedFile`s, every format from 1 to 22 (below 9 the header's endian
3//! byte sits at `file_size - metadata_size`), built from a spec, with
4//! overrides for the fields a parser test needs to get wrong.
5//!
6//! The layout is the one UnityPy 1.25.2 reads (`files/SerializedFile.py:235-324`,
7//! `files/ObjectReader.py:67-130`):
8//! - the header's four `u32`s, big-endian whatever the file's byte order;
9//!   from format 9 the endian byte and three reserved bytes follow at 16, and
10//!   at 22 the 64-bit fields (still big-endian: UnityPy switches the byte
11//!   order only after them, `SerializedFile.py:260-269`);
12//! - the metadata, in the file's byte order, right after the header from
13//!   format 9, and at the end of the file below 9, where its first byte is
14//!   the endian byte (`SerializedFile.py:266`);
15//! - the objects' data after the metadata (from format 9; below 9 after the
16//!   header), at `data_offset`, each object's start aligned to 8 within it.
17//!
18//! Owner: package C.
19
20use crate::blob::{NodeSpec, Strings, blob_tree, no_nul_blob, old_tree};
21use crate::writer::ByteWriter;
22
23/// One type record.
24#[derive(Clone, Debug, Default, PartialEq, Eq)]
25pub struct TypeSpec {
26    /// `class_id`.
27    pub class_id: i32,
28    /// `is_stripped_type` (format 16 and up).
29    pub stripped: bool,
30    /// `script_type_index` (format 17 and up).
31    pub script_type_index: i16,
32    /// `script_id`, written under UnityPy's condition (plan section 3.7).
33    pub script_id: [u8; 16],
34    /// `old_type_hash` (13 and up).
35    pub old_type_hash: [u8; 16],
36    /// The tree, written when type trees are enabled.
37    pub nodes: Vec<NodeSpec>,
38    /// Ref types at 21 and up: class, namespace, assembly.
39    pub ref_names: Option<(Vec<u8>, Vec<u8>, Vec<u8>)>,
40    /// Plain types at 21 and up: `type_dependencies`.
41    pub dependencies: Vec<i32>,
42}
43
44/// One object.
45#[derive(Clone, Debug, Default, PartialEq, Eq)]
46pub struct ObjectSpec {
47    /// `path_id`.
48    pub path_id: i64,
49    /// From 16 the index into `types`; below 16 the class id.
50    pub type_id: i32,
51    /// Below 16, the stored `class_id`.
52    pub class_id: u16,
53    /// The object's data, placed after the metadata in object order,
54    /// each start aligned to 8.
55    pub data: Vec<u8>,
56    /// Overrides the stored `byte_size` (for mismatch tests).
57    pub byte_size: Option<u32>,
58    /// The object row's `script_type_index` (formats 11-16), which loading
59    /// writes back into the object's type; `None` writes -1.
60    pub script_type_index: Option<i16>,
61    /// `is_destroyed` (below format 11); `None` writes 0.
62    pub is_destroyed: Option<u16>,
63    /// `is_stripped` (formats 15 and 16); `None` writes 0.
64    pub is_stripped: Option<i8>,
65}
66
67/// One external.
68#[derive(Clone, Debug, Default, PartialEq, Eq)]
69pub struct ExternalSpec {
70    /// `temp_empty` (6 and up).
71    pub temp_empty: Vec<u8>,
72    /// `guid` (5 and up).
73    pub guid: [u8; 16],
74    /// `type` (5 and up).
75    pub kind: i32,
76    /// `path`.
77    pub path: Vec<u8>,
78}
79
80/// A whole file.
81#[derive(Clone, Debug, PartialEq, Eq)]
82pub struct SerializedSpec {
83    /// The format number.
84    pub format: u32,
85    /// Big-endian metadata (the endian byte).
86    pub big_endian: bool,
87    /// The version string (7 and up).
88    pub unity_version: Vec<u8>,
89    /// The target platform (8 and up).
90    pub platform: i32,
91    /// `enable_type_tree` (13 and up).
92    pub type_trees: bool,
93    /// Type records.
94    pub types: Vec<TypeSpec>,
95    /// Objects, in table order.
96    pub objects: Vec<ObjectSpec>,
97    /// Script type references: `(file_index, identifier)`.
98    pub scripts: Vec<(i32, i64)>,
99    /// Externals.
100    pub externals: Vec<ExternalSpec>,
101    /// Ref types (20 and up).
102    pub ref_types: Vec<TypeSpec>,
103    /// `userInformation` (5 and up).
104    pub user_information: Vec<u8>,
105    /// `big_id_enabled` (formats 7-13): path ids as `i64` when non-zero.
106    pub big_id_enabled: i32,
107    /// Overrides the header's `metadata_size` (below 9 it also places the
108    /// endian byte, at `file_size - metadata_size`).
109    pub metadata_size: Option<u32>,
110    /// Overrides the header's `file_size`.
111    pub file_size: Option<i64>,
112    /// Overrides the header's `data_offset`.
113    pub data_offset: Option<i64>,
114}
115
116impl Default for SerializedSpec {
117    fn default() -> Self {
118        Self {
119            format: 22,
120            big_endian: false,
121            unity_version: b"2022.3.0f1".to_vec(),
122            platform: 13,
123            type_trees: true,
124            types: Vec::new(),
125            objects: Vec::new(),
126            scripts: Vec::new(),
127            externals: Vec::new(),
128            ref_types: Vec::new(),
129            user_information: Vec::new(),
130            big_id_enabled: 0,
131            metadata_size: None,
132            file_size: None,
133            data_offset: None,
134        }
135    }
136}
137
138/// The header's length: 16 bytes below 9, 20 from 9, 48 at 22 and up.
139fn header_len(format: u32) -> usize {
140    match format {
141        0..=8 => 16,
142        9..=21 => 20,
143        _ => 48,
144    }
145}
146
147/// The file's bytes.
148#[must_use]
149pub fn build(spec: &SerializedSpec) -> Vec<u8> {
150    let v = spec.format;
151    let header = header_len(v);
152    // The objects' data, each start aligned to 8, and each object's offset
153    // in it.
154    let mut data = ByteWriter::le();
155    let mut starts = Vec::with_capacity(spec.objects.len());
156    for o in &spec.objects {
157        data.align(8);
158        starts.push(data.len());
159        data.bytes(&o.data);
160    }
161    let data = data.into_inner();
162
163    // The metadata. Below 9 it starts with the endian byte and sits at the
164    // end of the file; from 9 it follows the header, and alignments inside
165    // it (`align_stream`) count from the file's start, so it is written at
166    // its absolute offset.
167    let meta_start = if v >= 9 { header } else { header + data.len() };
168    let mut w = if spec.big_endian {
169        ByteWriter::be()
170    } else {
171        ByteWriter::le()
172    };
173    w.bytes(&vec![0; meta_start]);
174    if v < 9 {
175        w.bool(spec.big_endian);
176    }
177    metadata(spec, &starts, &mut w);
178    let mut file = w.into_inner();
179    let meta_end = file.len();
180    let metadata_size = meta_end - meta_start;
181
182    let data_offset = if v >= 9 {
183        let at = meta_end.div_ceil(16) * 16;
184        file.resize(at, 0);
185        file.extend_from_slice(&data);
186        at
187    } else {
188        // Below 9 the data sits between the header and the metadata.
189        if let Some(region) = file.get_mut(header..header + data.len()) {
190            region.copy_from_slice(&data);
191        }
192        header
193    };
194    let file_size = spec
195        .file_size
196        .unwrap_or_else(|| i64::try_from(file.len()).unwrap_or(i64::MAX));
197    let metadata_size = spec
198        .metadata_size
199        .unwrap_or_else(|| u32::try_from(metadata_size).unwrap_or(u32::MAX));
200    let data_offset = spec
201        .data_offset
202        .unwrap_or_else(|| i64::try_from(data_offset).unwrap_or(i64::MAX));
203
204    // The header, always big-endian.
205    let mut h = ByteWriter::be();
206    if v >= 22 {
207        h.u32(0).u32(0).u32(v).u32(0);
208    } else {
209        h.u32(metadata_size)
210            .u32(truncate_u32(file_size))
211            .u32(v)
212            .u32(truncate_u32(data_offset));
213    }
214    if v >= 9 {
215        h.bool(spec.big_endian).bytes(&[0; 3]);
216    }
217    if v >= 22 {
218        h.u32(metadata_size).i64(file_size).i64(data_offset).i64(0);
219    }
220    let h = h.into_inner();
221    if let Some(region) = file.get_mut(..h.len()) {
222        region.copy_from_slice(&h);
223    }
224    file
225}
226
227fn truncate_u32(v: i64) -> u32 {
228    let b = v.to_le_bytes();
229    u32::from_le_bytes([b[0], b[1], b[2], b[3]])
230}
231
232/// The metadata after the endian byte, in UnityPy's reading order.
233fn metadata(spec: &SerializedSpec, starts: &[usize], w: &mut ByteWriter) {
234    let v = spec.format;
235    if v >= 7 {
236        w.cstr(&spec.unity_version);
237    }
238    if v >= 8 {
239        w.i32(spec.platform);
240    }
241    let trees = v < 13 || spec.type_trees;
242    if v >= 13 {
243        w.bool(spec.type_trees);
244    }
245    w.i32(len_i32(spec.types.len()));
246    for t in &spec.types {
247        type_record(t, v, spec.big_endian, trees, false, w);
248    }
249    // UnityPy reads `big_id_enabled` at 7-13 only and uses it as stored.
250    let big_id = (7..14).contains(&v) && spec.big_id_enabled != 0;
251    if (7..14).contains(&v) {
252        w.i32(spec.big_id_enabled);
253    }
254    w.i32(len_i32(spec.objects.len()));
255    for (o, start) in spec.objects.iter().zip(starts) {
256        if big_id {
257            w.i64(o.path_id);
258        } else if v < 14 {
259            w.i32(truncate_i32(o.path_id));
260        } else {
261            w.align(4);
262            w.i64(o.path_id);
263        }
264        let start = u32::try_from(*start).unwrap_or(u32::MAX);
265        if v >= 22 {
266            w.i64(i64::from(start));
267        } else {
268            w.u32(start);
269        }
270        let size = u32::try_from(o.data.len()).unwrap_or(u32::MAX);
271        w.u32(o.byte_size.unwrap_or(size));
272        w.i32(o.type_id);
273        if v < 16 {
274            w.u16(o.class_id);
275        }
276        if v < 11 {
277            w.u16(o.is_destroyed.unwrap_or(0));
278        }
279        if (11..17).contains(&v) {
280            w.i16(o.script_type_index.unwrap_or(-1));
281        }
282        if v == 15 || v == 16 {
283            w.i8(o.is_stripped.unwrap_or(0));
284        }
285    }
286    if v >= 11 {
287        w.i32(len_i32(spec.scripts.len()));
288        for (file_index, identifier) in &spec.scripts {
289            w.i32(*file_index);
290            if v < 14 {
291                w.i32(truncate_i32(*identifier));
292            } else {
293                w.align(4);
294                w.i64(*identifier);
295            }
296        }
297    }
298    w.i32(len_i32(spec.externals.len()));
299    for e in &spec.externals {
300        if v >= 6 {
301            w.cstr(&e.temp_empty);
302        }
303        if v >= 5 {
304            w.bytes(&e.guid);
305            w.i32(e.kind);
306        }
307        w.cstr(&e.path);
308    }
309    if v >= 20 {
310        w.i32(len_i32(spec.ref_types.len()));
311        for t in &spec.ref_types {
312            type_record(t, v, spec.big_endian, trees, true, w);
313        }
314    }
315    if v >= 5 {
316        w.cstr(&spec.user_information);
317    }
318}
319
320/// One `SerializedType` (`SerializedFile.py:112-149`).
321fn type_record(
322    t: &TypeSpec,
323    v: u32,
324    big_endian: bool,
325    trees: bool,
326    is_ref_type: bool,
327    w: &mut ByteWriter,
328) {
329    w.i32(t.class_id);
330    if v >= 16 {
331        w.bool(t.stripped);
332    }
333    // Below 17 the record holds -1 (`SerializedType.script_type_index`).
334    let script_type_index = if v >= 17 { t.script_type_index } else { -1 };
335    if v >= 17 {
336        w.i16(t.script_type_index);
337    }
338    if v >= 13 {
339        if (is_ref_type && script_type_index >= 0)
340            || (v < 16 && t.class_id < 0)
341            || (v >= 16 && t.class_id == 114)
342        {
343            w.bytes(&t.script_id);
344        }
345        w.bytes(&t.old_type_hash);
346    }
347    if trees {
348        if v >= 12 || v == 10 {
349            w.bytes(&blob_tree(&t.nodes, v, big_endian, Strings::Local, &[]));
350        } else {
351            w.bytes(&old_tree(&t.nodes, v, big_endian));
352        }
353        if v >= 21 {
354            if is_ref_type {
355                let empty = (Vec::new(), Vec::new(), Vec::new());
356                let (class, ns, asm) = t.ref_names.as_ref().unwrap_or(&empty);
357                w.cstr(class).cstr(ns).cstr(asm);
358            } else {
359                w.i32(len_i32(t.dependencies.len()));
360                for d in &t.dependencies {
361                    w.i32(*d);
362                }
363            }
364        }
365    }
366}
367
368/// Round 4's entry: a little-endian format-22 file (engine 2018.4.0f1) of
369/// one type, class 1, whose blob has `rows` rows naming offset 0 of a
370/// 65,536-byte string buffer with no NUL ([`no_nul_blob`]). `complete` ends
371/// it as a file of no objects ends (no dependencies, objects, scripts,
372/// externals or ref types, an empty user information); otherwise it ends
373/// right after the string buffer, and the type's dependency read raises.
374#[must_use]
375pub fn no_nul_strings(rows: usize, complete: bool) -> Vec<u8> {
376    const HEADER: usize = 48;
377    let mut w = ByteWriter::le();
378    w.bytes(&[0; HEADER]);
379    w.cstr(b"2018.4.0f1").i32(13).bool(true).i32(1);
380    // Class 1, not stripped, no script, the old type hash.
381    w.i32(1).bool(false).i16(-1).bytes(&[0; 16]);
382    w.bytes(&no_nul_blob(rows, 65_536));
383    if complete {
384        w.i32(0).i32(0).i32(0).i32(0).i32(0).cstr(b"");
385    }
386    let mut file = w.into_inner();
387    let metadata = u32::try_from(file.len() - HEADER).unwrap_or(u32::MAX);
388    let end = i64::try_from(file.len()).unwrap_or(i64::MAX);
389    let mut h = ByteWriter::be();
390    h.u32(0).u32(0).u32(22).u32(0).bool(false).bytes(&[0; 3]);
391    h.u32(metadata).i64(end).i64(end).i64(0);
392    file[..HEADER].copy_from_slice(h.as_slice());
393    file
394}
395
396fn len_i32(n: usize) -> i32 {
397    i32::try_from(n).unwrap_or(i32::MAX)
398}
399
400fn truncate_i32(v: i64) -> i32 {
401    let b = v.to_le_bytes();
402    i32::from_le_bytes([b[0], b[1], b[2], b[3]])
403}
404
405/// A minimal file of one object whose class has the tree `nodes`: `data` is
406/// the object's bytes, `class_id` its class. Every other field as
407/// [`SerializedSpec::default`] has it (format 22, little-endian, type trees
408/// on).
409#[must_use]
410pub fn one_object(class_id: i32, nodes: Vec<NodeSpec>, data: Vec<u8>) -> SerializedSpec {
411    SerializedSpec {
412        types: vec![TypeSpec {
413            class_id,
414            nodes,
415            ..TypeSpec::default()
416        }],
417        objects: vec![ObjectSpec {
418            path_id: 1,
419            type_id: 0,
420            class_id: u16::try_from(class_id).unwrap_or(0),
421            data,
422            ..ObjectSpec::default()
423        }],
424        ..SerializedSpec::default()
425    }
426}
427
428#[cfg(test)]
429mod tests {
430    use super::{ExternalSpec, ObjectSpec, SerializedSpec, TypeSpec, build, one_object};
431    use crate::blob::NodeSpec;
432
433    #[test]
434    fn overrides_default_to_what_unity_writes() {
435        let o = ObjectSpec::default();
436        assert_eq!(
437            (o.script_type_index, o.is_destroyed, o.is_stripped),
438            (None, None, None)
439        );
440        let s = SerializedSpec::default();
441        assert_eq!(s.big_id_enabled, 0);
442        assert_eq!(
443            (s.metadata_size, s.file_size, s.data_offset),
444            (None, None, None)
445        );
446        assert_eq!(NodeSpec::new(0, "Base", "Base").index, None);
447    }
448
449    fn be_u32(b: &[u8], at: usize) -> u32 {
450        u32::from_be_bytes(b[at..at + 4].try_into().unwrap())
451    }
452
453    #[test]
454    fn a_format_22_header_is_big_endian_with_its_64_bit_fields() {
455        let spec = one_object(1, vec![NodeSpec::new(0, "Base", "Base")], vec![7; 4]);
456        let f = build(&spec);
457        assert_eq!(be_u32(&f, 8), 22);
458        assert_eq!(f[16], 0, "little-endian metadata");
459        let file_size = i64::from_be_bytes(f[24..32].try_into().unwrap());
460        let data_offset = i64::from_be_bytes(f[32..40].try_into().unwrap());
461        assert_eq!(usize::try_from(file_size).unwrap(), f.len());
462        assert_eq!(data_offset % 16, 0);
463        assert_eq!(&f[usize::try_from(data_offset).unwrap()..], &[7; 4]);
464        // The metadata starts with the version string.
465        assert_eq!(&f[48..59], b"2022.3.0f1\0");
466    }
467
468    #[test]
469    fn below_9_the_metadata_is_at_the_end_after_the_endian_byte() {
470        let spec = SerializedSpec {
471            format: 8,
472            big_endian: true,
473            externals: vec![ExternalSpec {
474                path: b"x".to_vec(),
475                ..ExternalSpec::default()
476            }],
477            objects: vec![ObjectSpec {
478                path_id: 3,
479                data: vec![1, 2, 3],
480                ..ObjectSpec::default()
481            }],
482            types: vec![TypeSpec {
483                class_id: 1,
484                nodes: vec![NodeSpec::new(0, "Base", "Base")],
485                ..TypeSpec::default()
486            }],
487            ..SerializedSpec::default()
488        };
489        let f = build(&spec);
490        let metadata_size = usize::try_from(be_u32(&f, 0)).unwrap();
491        let file_size = usize::try_from(be_u32(&f, 4)).unwrap();
492        assert_eq!(file_size, f.len());
493        assert_eq!(f[file_size - metadata_size], 1, "the endian byte");
494        assert_eq!(be_u32(&f, 12), 16, "data right after the header");
495        assert_eq!(&f[16..19], &[1, 2, 3]);
496        assert_eq!(&f[file_size - metadata_size + 1..][..11], b"2022.3.0f1\0");
497    }
498
499    #[test]
500    fn the_overrides_replace_the_header_fields() {
501        let spec = SerializedSpec {
502            format: 17,
503            metadata_size: Some(5),
504            file_size: Some(-1),
505            data_offset: Some(0x1_0000_0002),
506            ..SerializedSpec::default()
507        };
508        let f = build(&spec);
509        assert_eq!(be_u32(&f, 0), 5);
510        assert_eq!(be_u32(&f, 4), u32::MAX);
511        assert_eq!(be_u32(&f, 12), 2);
512    }
513}