Skip to main content

frostbite_formats/ebx/
mod.rs

1//! Frostbite EBX documents: RIFF `EBX\0`/`EBXS` files holding the EBXD payload, EFIX fixups, EBXX array and
2//! boxed-value records, and a REFL or RFL2 reflection table. [`read`] turns one into typed values, [`write()`] lays a
3//! document out again, [`carry`] moves values (and what they point at) from one document into another, and [`remap`]
4//! gives a document's identities new GUIDs (a clone).
5//!
6//! This is a port of ReSkate's EBX reader, writer and merge (`Engine/Resource/ebx_document.cpp`,
7//! `ebx_writer.cpp` and `ebx_merge.cpp`; github.com/Dingo-Shenanigans/ReSkate, GPL-3.0-or-later), which is the
8//! reference for the choices the format leaves open: the donor reflection table is carried as stored, boxed values
9//! travel as their bytes plus what those bytes point at, and a new array takes the hash of an array of its field. The
10//! port adds bounds checks and strict validation where the C++ trusts its input. A byte comparison with that reference
11//! writer therefore measures agreement with the port's source; independent evidence comes from stock documents, whose
12//! values must survive a rewrite.
13//!
14//! Strings are bytes as stored (Frostbite strings are not guaranteed UTF-8). Offsets named `*_offset` are relative to
15//! the start of the EBXD payload (`data_start`, 16-aligned) unless they say otherwise.
16//!
17//! A document read and written again keeps its values; the bytes come back exactly when the document already has the
18//! writer's layout (a document the writer produced, or one laid out the same way).
19
20pub mod carry;
21mod decode;
22mod encode;
23pub mod remap;
24
25pub use decode::{read, read_file_guid};
26pub use encode::write;
27
28/// `FieldType`: bits 4..9 of an encoded type.
29#[derive(Clone, Copy, Debug, PartialEq, Eq)]
30#[repr(u8)]
31pub enum FieldType {
32    /// Not a field of its own: the type's base class, whose fields come first; `class_ref` names it.
33    Inherited = 0x00,
34    /// A database object (read as nothing, written as zero bytes).
35    DbObject = 0x01,
36    /// A nested structure stored inline, of the type `class_ref` names.
37    Structure = 0x02,
38    /// A reference to another instance of the document or to an import; see [`Pointer`].
39    Pointer = 0x03,
40    /// An array: a 4-byte displacement to the elements, with the count stored just before them.
41    Array = 0x04,
42    /// A fixed-length array (read as nothing, written as zero bytes).
43    FixedArray = 0x05,
44    /// A 32-byte inline string, zero padded.
45    String = 0x06,
46    /// A string stored elsewhere in the data, reached by a 4-byte displacement; -1 is the empty string.
47    CString = 0x07,
48    /// An enumeration, a signed 32-bit value.
49    Enumeration = 0x08,
50    /// A file reference: a string by displacement, then 4 more bytes.
51    FileRef = 0x09,
52    /// A boolean, one byte.
53    Boolean = 0x0A,
54    /// A signed 8-bit integer.
55    Int8 = 0x0B,
56    /// An unsigned 8-bit integer.
57    UInt8 = 0x0C,
58    /// A signed 16-bit integer.
59    Int16 = 0x0D,
60    /// An unsigned 16-bit integer.
61    UInt16 = 0x0E,
62    /// A signed 32-bit integer.
63    Int32 = 0x0F,
64    /// An unsigned 32-bit integer.
65    UInt32 = 0x10,
66    /// A signed 64-bit integer.
67    Int64 = 0x11,
68    /// An unsigned 64-bit integer.
69    UInt64 = 0x12,
70    /// A 32-bit float.
71    Float32 = 0x13,
72    /// A 64-bit float.
73    Float64 = 0x14,
74    /// A 16-byte GUID.
75    Guid = 0x15,
76    /// A 20-byte SHA-1.
77    Sha1 = 0x16,
78    /// A 64-bit resource reference.
79    ResourceRef = 0x17,
80    /// A function reference (read as nothing, written as zero bytes).
81    Function = 0x18,
82    /// A type reference: 4 bytes naming a primitive type or a reflected type, then 4 more bytes.
83    TypeRef = 0x19,
84    /// A reference to a boxed value in the boxed-value section; see [`BoxedRef`].
85    BoxedValueRef = 0x1A,
86    /// An interface reference (read as nothing, written as zero bytes).
87    Interface = 0x1B,
88    /// A delegate, read like a type reference.
89    Delegate = 0x1C,
90    /// 0x1D..0x1F: not a type ReSkate knows; read as nothing, written as zero bytes.
91    Unknown = 0x1F,
92}
93
94impl FieldType {
95    /// The type for the five-bit code `v`; a code that is not one of the known types is [`FieldType::Unknown`].
96    pub fn from_bits(v: u16) -> FieldType {
97        use FieldType::*;
98        const ALL: [FieldType; 29] = [
99            Inherited,
100            DbObject,
101            Structure,
102            Pointer,
103            Array,
104            FixedArray,
105            String,
106            CString,
107            Enumeration,
108            FileRef,
109            Boolean,
110            Int8,
111            UInt8,
112            Int16,
113            UInt16,
114            Int32,
115            UInt32,
116            Int64,
117            UInt64,
118            Float32,
119            Float64,
120            Guid,
121            Sha1,
122            ResourceRef,
123            Function,
124            TypeRef,
125            BoxedValueRef,
126            Interface,
127            Delegate,
128        ];
129        ALL.get(usize::from(v)).copied().unwrap_or(Unknown)
130    }
131}
132
133/// The category nibble of an encoded type (`FieldCategory`); only `Array` (4) changes how a field is read.
134const CATEGORY_ARRAY: u16 = 4;
135
136/// A field of a reflected type: its name, encoded type and where it lies in an instance.
137#[derive(Clone, Debug, PartialEq, Eq)]
138pub struct FieldDescriptor {
139    /// The field's name, looked up from the reflection table's strings.
140    pub name: String,
141    /// The name hash as stored: in a document with hashed names (RFL2) the key of the name table, otherwise the name's offset in the reflection strings.
142    pub name_hash: u32,
143    /// The encoded type as ReSkate keeps it: the stored u16 shifted right by one.
144    pub encoded_type: u16,
145    /// The type slot of a structure, enumeration, inherited or array-of-structure field (index into [`Document::types`]); not meaningful for other types.
146    pub class_ref: u16,
147    /// Byte offset of the field from the start of its instance.
148    pub data_offset: u32,
149}
150
151impl FieldDescriptor {
152    /// The field's [`FieldType`].
153    pub fn ty(&self) -> FieldType {
154        FieldType::from_bits((self.encoded_type >> 4) & 0x1F)
155    }
156    /// Whether the field is an array.
157    pub fn is_array(&self) -> bool {
158        self.encoded_type & 0x0F == CATEGORY_ARRAY
159    }
160}
161
162/// A reflected type: its name, size, alignment, GUID, signature and the run of [`Document::fields`] it owns.
163#[derive(Clone, Debug, PartialEq, Eq)]
164pub struct TypeDescriptor {
165    /// The type's name, looked up from the reflection table's strings.
166    pub name: String,
167    /// The name hash as stored; see [`FieldDescriptor::name_hash`].
168    pub name_hash: u32,
169    /// Index into [`Document::fields`] of the type's first field; the type owns `field_count` consecutive fields.
170    pub field_index: i32,
171    /// Number of fields the type owns.
172    pub field_count: u16,
173    /// The encoded type as ReSkate keeps it: the stored u16 shifted right by one.
174    pub encoded_type: u16,
175    /// Size in bytes of an instance of the type.
176    pub size: u16,
177    /// Alignment in bytes; the reader refuses a value over 255.
178    pub alignment: u16,
179    /// The type's GUID, taken from the reflection table's identifier list in step with the types.
180    pub guid: [u8; 16],
181    /// The type's signature, taken from the same list; with the GUID it ties an instance to its type.
182    pub signature: u32,
183}
184
185/// An import: the file GUID of another document and the GUID of the instance in it.
186#[derive(Clone, Copy, Debug, PartialEq, Eq)]
187pub struct ImportReference {
188    /// GUID of the imported document (the target's file GUID).
189    pub file_guid: [u8; 16],
190    /// GUID of the instance within that document.
191    pub class_guid: [u8; 16],
192}
193
194/// An EBXX array record: where an array's data starts, its length, its hash and element type.
195#[derive(Clone, Copy, Debug, PartialEq, Eq)]
196pub struct ArrayRecord {
197    /// Offset of the array's elements from the start of the EBXD payload.
198    pub offset: u32,
199    /// Number of elements.
200    pub count: u32,
201    /// The array hash, kept so that an array read and written again keeps it.
202    pub hash: u32,
203    /// As stored (not shifted).
204    pub encoded_type: u16,
205    /// For struct elements, the EFIX type slot; else 0xFFFF.
206    pub class_ref: u16,
207}
208
209/// One RFL2 group record: an array type the document uses.
210#[derive(Clone, Copy, Debug, PartialEq, Eq)]
211pub struct ReflectionGroup {
212    /// Hash that names the array type.
213    pub hash: u32,
214    /// Number of array types in the group.
215    pub count: u32,
216    /// Index of the group's first entry.
217    pub first: u32,
218}
219
220/// An EBXX boxed-value record: where a boxed value starts, its hash and type, and its raw bytes.
221#[derive(Clone, Debug, PartialEq, Eq)]
222pub struct BoxedValueRecord {
223    /// Offset of the boxed value from the start of the EBXD payload.
224    pub offset: u32,
225    /// Count as stored in the record.
226    pub count: u32,
227    /// The record's hash.
228    pub hash: u32,
229    /// The encoded type as stored (not shifted).
230    pub encoded_type: u16,
231    /// For struct values, the EFIX type slot; else 0xFFFF.
232    pub class_ref: u16,
233    /// The record's bytes, up to the next record or the string section.
234    pub raw: Vec<u8>,
235}
236
237/// An object stored in the document, with its type, its GUID if it is exported, and its decoded values.
238#[derive(Clone, Debug, PartialEq)]
239pub struct Instance {
240    /// Index of the instance's type in the EFIX type list ([`Document::fixup_type_guids`]).
241    pub fixup_type: u16,
242    /// Index into [`Document::types`]; `None` when its EFIX type has no reflection entry (then it is not read).
243    pub descriptor: Option<usize>,
244    /// Offset of the instance from the start of the EBXD payload.
245    pub data_offset: u32,
246    /// Whether the instance is one of the document's exported instances (the first ones the EFIX table lists), which carry a GUID in the 16 bytes before them.
247    pub exported: bool,
248    /// The instance's GUID if it is exported; zero otherwise.
249    pub guid: [u8; 16],
250    /// The instance's values; `None` when its type is unknown ([`Instance::descriptor`] is `None`).
251    pub object: Option<Object>,
252    /// The instance's bytes as read (its type's size): padding and unread fields are written back from here.
253    pub raw_image: Vec<u8>,
254}
255
256/// What a pointer field holds.
257#[derive(Clone, Copy, Debug, PartialEq, Eq)]
258pub enum Pointer {
259    /// A null pointer.
260    Null,
261    /// An instance of this document, by index.
262    Internal(usize),
263    /// An import, by index.
264    External(usize),
265}
266
267/// A type reference value: a primitive type or a type of the document's type list.
268///
269/// A reference to a type of the document is `slot << 2 | low bits` in [`TypeRef::encoded`], where `slot` indexes the
270/// EFIX type list ([`Document::fixup_type_guids`]), the list the instances' types are named from too. It is not an index
271/// into the reflection table: [`Document::type_ref_target`] finds the reflected type a slot names.
272#[derive(Clone, Copy, Debug, PartialEq, Eq)]
273pub struct TypeRef {
274    /// Whether the reference names a primitive type rather than a type of the document.
275    pub primitive: bool,
276    /// The primitive type, when `primitive` is set.
277    pub primitive_type: Option<FieldType>,
278    /// Index into [`Document::fixup_type_guids`], when the reference names a type of the document (`encoded >> 2`).
279    pub fixup_type: Option<usize>,
280    /// The reference as stored (u32): the writer emits this, so a changed [`TypeRef::fixup_type`] takes effect through
281    /// [`TypeRef::with_fixup_type`]. The low two bits are kept as stored; their meaning is not known.
282    pub encoded: u32,
283}
284
285impl TypeRef {
286    /// This reference with its EFIX type list slot replaced and the low two bits of `encoded` kept; `None` when the
287    /// slot does not fit the encoding or the reference is empty or primitive.
288    pub fn with_fixup_type(self, slot: usize) -> Option<TypeRef> {
289        if self.primitive || self.fixup_type.is_none() {
290            return None;
291        }
292        let shifted = u32::try_from(slot).ok().filter(|&s| s < 1 << 29)? << 2;
293        Some(TypeRef {
294            fixup_type: Some(slot),
295            encoded: shifted | (self.encoded & 3),
296            ..self
297        })
298    }
299}
300
301/// A reference to a boxed value in the document's boxed-value section.
302#[derive(Clone, Copy, Debug, PartialEq, Eq)]
303pub struct BoxedRef {
304    /// The encoded type of the boxed value as stored (u32).
305    pub encoded_type: u32,
306    /// Payload offset of the boxed value, or -1.
307    pub data_offset: i64,
308    /// The displacement as stored: from 8 bytes past the field to the boxed value.
309    pub relative_offset: i64,
310}
311
312/// A field's value.
313#[derive(Clone, Debug, PartialEq)]
314pub enum Value {
315    /// A field type that is read as nothing (database objects, functions and so on).
316    Nothing,
317    /// A boolean.
318    Bool(bool),
319    /// A signed integer of 8, 16, 32 or 64 bits, or an enumeration.
320    Int(i64),
321    /// An unsigned integer of 8, 16, 32 or 64 bits.
322    UInt(u64),
323    /// A 32-bit or 64-bit float, widened to `f64`. A NaN keeps the bits it was stored with (a 32-bit NaN widens by
324    /// shifting its payload, quiet or signalling) and is written back with them.
325    Float(f64),
326    /// String, CString and FileRef values, as stored.
327    Str(Vec<u8>),
328    /// A 16-byte GUID.
329    Guid([u8; 16]),
330    /// A 20-byte SHA-1.
331    Sha1([u8; 20]),
332    /// A 64-bit resource reference.
333    Resource(u64),
334    /// A pointer to an instance or an import.
335    Pointer(Pointer),
336    /// A type reference.
337    TypeRef(TypeRef),
338    /// A boxed-value reference.
339    Boxed(BoxedRef),
340    /// A nested structure.
341    Object(Box<Object>),
342    /// An array's elements, and the hash of the record it was read from (a new array has none).
343    Array(Vec<Value>, Option<u32>),
344}
345
346/// A field of an [`Object`] with its value.
347#[derive(Clone, Debug, PartialEq)]
348pub struct FieldValue {
349    /// Index into [`Document::fields`].
350    pub descriptor: usize,
351    /// The field's name.
352    pub name: String,
353    /// The field's value.
354    pub value: Value,
355}
356
357/// A structure: its type and its fields' values, in stored order.
358#[derive(Clone, Debug, PartialEq)]
359pub struct Object {
360    /// Index into [`Document::types`] of the object's type.
361    pub descriptor: usize,
362    /// The values of the type's fields, those of its base types first.
363    pub fields: Vec<FieldValue>,
364}
365
366impl Object {
367    /// The field named `name` (ASCII case-insensitive, as ReSkate looks fields up).
368    pub fn field(&self, name: &str) -> Option<&Value> {
369        self.fields
370            .iter()
371            .find(|f| f.name.eq_ignore_ascii_case(name))
372            .map(|f| &f.value)
373    }
374    /// The field named `name`, mutably.
375    pub fn field_mut(&mut self, name: &str) -> Option<&mut Value> {
376        self.fields
377            .iter_mut()
378            .find(|f| f.name.eq_ignore_ascii_case(name))
379            .map(|f| &mut f.value)
380    }
381}
382
383/// An EBX document read by [`read`]: its instances with typed values, together with the reflection data, fixup
384/// tables and payload that [`write()`] needs to lay it out again.
385#[derive(Clone, Debug, PartialEq)]
386pub struct Document {
387    /// The document's own GUID (the first 16 bytes of the EFIX chunk); other documents import it by this.
388    pub file_guid: [u8; 16],
389    /// Name of the first instance's type.
390    pub root_type: String,
391    /// Absolute offsets of the EBXD payload in the bytes read.
392    pub data_start: usize,
393    /// Absolute offset of the end of the EBXD payload in the bytes read.
394    pub data_end: usize,
395    /// Offset of the EBXX array section, from the start of the payload (EFIX).
396    pub arrays_offset: u32,
397    /// Offset of the boxed-value section, from the start of the payload (EFIX).
398    pub boxed_values_offset: u32,
399    /// Offset of the string section, from the start of the payload (EFIX).
400    pub strings_offset: u32,
401    /// Whether the RIFF form is `EBXS` rather than `EBX`.
402    pub serialized_layout: bool,
403    /// Whether the reflection chunk is RFL2 (names looked up by hash) rather than REFL.
404    pub hashed_reflection_names: bool,
405    /// The reflection chunk (REFL or RFL2) as stored.
406    pub reflection_data: Vec<u8>,
407    /// The GUIDs of the types the instances use (EFIX), indexed by [`Instance::fixup_type`].
408    pub fixup_type_guids: Vec<[u8; 16]>,
409    /// The signatures that go with `fixup_type_guids`; there may be fewer than GUIDs.
410    pub fixup_type_signatures: Vec<u32>,
411    /// The offsets of boxed-value references (EFIX), if the chunk carries them.
412    pub boxed_reference_offsets: Vec<u32>,
413    /// The reflected types.
414    pub types: Vec<TypeDescriptor>,
415    /// The reflected fields; each type owns a run of them.
416    pub fields: Vec<FieldDescriptor>,
417    /// The instances in EFIX order, the exported ones first.
418    pub instances: Vec<Instance>,
419    /// The documents this one imports from.
420    pub imports: Vec<ImportReference>,
421    /// The EBXX array records.
422    pub arrays: Vec<ArrayRecord>,
423    /// The EBXX boxed-value records, sorted by offset.
424    pub boxed_values: Vec<BoxedValueRecord>,
425    /// Payload offsets of the pointer fields (EFIX).
426    pub pointer_offsets: Vec<u32>,
427    /// Payload offsets of the resource-reference fields (EFIX).
428    pub resource_ref_offsets: Vec<u32>,
429    /// Payload offsets of the import pointers (EFIX).
430    pub import_offsets: Vec<u32>,
431    /// Payload offsets of the type-reference fields (EFIX).
432    pub type_info_offsets: Vec<u32>,
433    /// Payload offsets of the boxed-value fields found while reading values.
434    pub boxed_value_slots: Vec<u32>,
435    /// The EBXD bytes as read (from `data_start`): what boxed values point at is carried from here.
436    pub payload: Vec<u8>,
437    /// The RFL2 array-type groups.
438    pub reflection_groups: Vec<ReflectionGroup>,
439}
440
441impl Document {
442    /// The reflected type ([`Document::types`] index) the EFIX type list's `slot` names, by GUID and signature (the
443    /// GUID alone when the list has no signature for the slot); `None` when the slot is past the list or the document
444    /// reflects no such type (the list names GUIDs the reflection table lacks).
445    pub fn fixup_type_target(&self, slot: usize) -> Option<usize> {
446        let guid = self.fixup_type_guids.get(slot)?;
447        let signature = self.fixup_type_signatures.get(slot);
448        self.types
449            .iter()
450            .position(|t| t.guid == *guid && signature.is_none_or(|s| t.signature == *s))
451    }
452
453    /// The reflected type ([`Document::types`] index) a type reference names; `None` for an empty or primitive
454    /// reference and for one whose EFIX slot reflects no type.
455    pub fn type_ref_target(&self, t: &TypeRef) -> Option<usize> {
456        self.fixup_type_target(t.fixup_type?)
457    }
458
459    /// The members of the enumeration `field` (an index into [`Document::fields`], [`FieldValue::descriptor`]) declares: each
460    /// member's name and value. The reflection table lists an enumeration as a type whose fields are its members, the value
461    /// of a member being the field's `data_offset` read as a signed 32-bit number. Empty for a field that is not an
462    /// enumeration and for an enumeration with no members.
463    pub fn enum_members(&self, field: usize) -> impl Iterator<Item = (&str, i64)> + '_ {
464        let members = self
465            .fields
466            .get(field)
467            .filter(|f| f.ty() == FieldType::Enumeration)
468            .and_then(|f| self.types.get(usize::from(f.class_ref)))
469            .and_then(|t| {
470                let first = usize::try_from(t.field_index).ok()?;
471                self.fields
472                    .get(first..first.checked_add(usize::from(t.field_count))?)
473            })
474            .unwrap_or_default();
475        members
476            .iter()
477            .map(|m| (m.name.as_str(), i64::from(m.data_offset.cast_signed())))
478    }
479
480    /// The name of the member of the enumeration `field` ([`Document::enum_members`]) that has `value`; `None` when the
481    /// field is not an enumeration or no member has the value (one stock value, of 3,368,510, names no member).
482    pub fn enum_member(&self, field: usize, value: i64) -> Option<&str> {
483        self.enum_members(field)
484            .find(|&(_, v)| v == value)
485            .map(|(n, _)| n)
486    }
487
488    /// Where the elements of the array whose 4-byte slot lies at payload offset `slot` start (payload offsets, as
489    /// [`Instance::data_offset`]): the slot holds a displacement from itself to the first element, the count lying just
490    /// before it. `None` when the slot or the target lies outside the payload.
491    pub fn array_data_offset(&self, slot: u32) -> Option<u32> {
492        let at = usize::try_from(slot).ok()?;
493        let disp = i32::from_le_bytes(*self.payload.get(at..)?.first_chunk::<4>()?);
494        let target = i64::from(slot).checked_add(i64::from(disp))?;
495        let target = u32::try_from(target).ok()?;
496        (usize::try_from(target).ok()? <= self.payload.len()).then_some(target)
497    }
498
499    /// The type name of instance `i`.
500    pub fn type_name(&self, i: usize) -> Option<&str> {
501        let d = self.instances.get(i)?.descriptor?;
502        self.types.get(d).map(|t| t.name.as_str())
503    }
504}