Skip to main content

frostbite_formats/
lookup.rs

1//! The shader lookup tables of `dingolevel_root`: material key -> program indices (program lookup, type 0xE2E6955B,
2//! after the program records) and material key -> streamable texture set (type 0x2D254A89). Rows are a map: the first
3//! row of a repeated key wins, as the game's reader appends.
4
5use std::collections::BTreeMap;
6
7use crate::{Rd, Result, bad, capacity};
8
9/// The resource type id of the program lookup (ExpressionShaderDatabase).
10pub const PROGRAM_LOOKUP: u32 = 0xE2E6_955B;
11/// The resource type id of the streamable texture set lookup table.
12pub const TEXTURE_LOOKUP: u32 = 0x2D25_4A89;
13
14/// A program lookup resource: everything before the rows (the program records), and the rows.
15#[derive(Clone, Debug, PartialEq, Eq)]
16pub struct ProgramLookup {
17    /// Everything in the payload before the rows: the program records ([`read_records`]) and the relocation data, kept as stored.
18    pub head: Vec<u8>,
19    /// Material key to the program indices that material runs. Each row lists 1 to 4096 indices.
20    pub rows: BTreeMap<u64, Vec<u32>>,
21    /// Keys in stored order (equal to sorted order in every table seen; kept to reproduce the bytes).
22    pub order: Vec<u64>,
23}
24
25/// Read a program lookup from its payload and metadata. Refused: metadata that does not match the payload, a row of
26/// no programs or of more than 4096, a repeated key, data after the last row.
27pub fn read_programs(d: &[u8], meta: &[u8; 16]) -> Result<ProgramLookup> {
28    let m = Rd::new(meta, "program lookup meta");
29    let (program, reloc, lookup) = (m.size32le(0)?, m.size32le(4)?, m.size32le(8)?);
30    let start = program.saturating_add(reloc);
31    let (head, rows) = d.split_at_checked(start).ok_or_else(|| {
32        bad(
33            "program lookup",
34            0,
35            "metadata does not match the payload: the program and relocation sizes pass the end",
36        )
37    })?;
38    if lookup < 4 || lookup != rows.len() {
39        return Err(bad(
40            "program lookup",
41            0,
42            format!(
43                "metadata does not match the payload: expected {lookup} bytes of rows (at least 4), found {}",
44                rows.len()
45            ),
46        ));
47    }
48    let r = Rd::new(d, "program lookup");
49    let count = r.size32le(start)?;
50    let mut pos = start.saturating_add(4);
51    let mut out = ProgramLookup {
52        head: head.to_vec(),
53        rows: BTreeMap::new(),
54        order: Vec::with_capacity(capacity(
55            "program lookup",
56            count,
57            12,
58            rows.len().saturating_sub(4),
59        )?),
60    };
61    for _ in 0..count {
62        let (key, n) = (r.u64le(pos)?, r.size32le(pos.saturating_add(8))?);
63        pos = pos.saturating_add(12);
64        if n == 0 || n > 4096 {
65            return Err(bad("program lookup", pos, format!("row of {n} programs")));
66        }
67        let row = (0..n)
68            .map(|i| r.u32le(crate::at(pos, i, 4)))
69            .collect::<Result<Vec<_>>>()?;
70        pos = crate::at(pos, n, 4);
71        if out.rows.contains_key(&key) {
72            return Err(bad("program lookup", pos, format!("repeated key {key:016x}")));
73        }
74        out.rows.insert(key, row);
75        out.order.push(key);
76    }
77    if pos != d.len() {
78        return Err(bad("program lookup", pos, "trailing data"));
79    }
80    Ok(out)
81}
82
83/// The payload and metadata (only the rows' length changes) of a program lookup; rows in `order`, then any new keys
84/// sorted.
85pub fn write_programs(t: &ProgramLookup, meta: &[u8; 16]) -> Result<(Vec<u8>, [u8; 16])> {
86    const W: &str = "program lookup";
87    let mut keys = t.order.clone();
88    let mut extra: Vec<u64> = t.rows.keys().copied().filter(|k| !t.order.contains(k)).collect();
89    keys.retain(|k| t.rows.contains_key(k));
90    keys.append(&mut extra);
91    let mut out = t.head.clone();
92    out.extend(crate::u32_len(W, keys.len())?.to_le_bytes());
93    for k in keys {
94        let row = t.rows.get(&k).map(Vec::as_slice).unwrap_or_default();
95        out.extend(k.to_le_bytes());
96        out.extend(crate::u32_len(W, row.len())?.to_le_bytes());
97        for p in row {
98            out.extend(p.to_le_bytes());
99        }
100    }
101    let mut m = *meta;
102    let rows_len = crate::u32_len(W, out.len().saturating_sub(t.head.len()))?;
103    if let Some(slot) = m.get_mut(8..12) {
104        slot.copy_from_slice(&rows_len.to_le_bytes());
105    }
106    Ok((out, m))
107}
108
109/// Streamable texture set rows (u64 key, u64 value), in stored order.
110pub fn read_texture_sets(d: &[u8]) -> Result<Vec<(u64, u64)>> {
111    if !d.len().is_multiple_of(16) {
112        return Err(bad("texture set lookup", d.len(), "not a whole number of rows"));
113    }
114    let r = Rd::new(d, "texture set lookup");
115    (0..d.len() / 16)
116        .map(|i| Ok((r.u64le(crate::at(0, i, 16))?, r.u64le(crate::at(8, i, 16))?)))
117        .collect()
118}
119
120/// The bytes of texture set rows, the counterpart of [`read_texture_sets`].
121pub fn write_texture_sets(rows: &[(u64, u64)]) -> Vec<u8> {
122    rows.iter()
123        .flat_map(|(k, v)| k.to_le_bytes().into_iter().chain(v.to_le_bytes()))
124        .collect()
125}
126
127/// The size of a program record.
128pub const RECORD_LEN: usize = 168;
129
130/// One program record of a program lookup's head, as stored (168 bytes), with the fields read from it: its key (the
131/// first eight bytes) and the ids of the permutations it runs, an array of `u64` (the count at 0x60, a pointer to the
132/// array at 0x64). The rest of the record (arrays of state the permutations are drawn with) is kept as stored.
133#[derive(Clone, Debug, PartialEq, Eq)]
134pub struct ProgramRecord {
135    /// The material key (u64 at +0 of the record).
136    pub key: u64,
137    /// The ids of the permutations the program runs (a `u64` array: count at +0x60, pointer to it at +0x64).
138    pub permutations: Vec<u64>,
139    /// The whole 168-byte record as stored, including the parts not decoded into the fields above.
140    pub raw: [u8; RECORD_LEN],
141}
142
143/// The program records of a program lookup: the head's first word is their count, the eight bytes after it the
144/// offset of the first, and the records follow one another. Refused: records or a permutation array outside the
145/// program section the metadata states.
146pub fn read_records(d: &[u8], meta: &[u8; 16]) -> Result<Vec<ProgramRecord>> {
147    let m = Rd::new(meta, "program lookup meta");
148    let program = m.size32le(0)?;
149    let r = Rd::new(
150        d.get(..program)
151            .ok_or_else(|| bad("program records", 0, "metadata does not match the payload"))?,
152        "program records",
153    );
154    let (count, first) = (r.size32le(0)?, r.offset64le(4)?);
155    count
156        .checked_mul(RECORD_LEN)
157        .and_then(|n| n.checked_add(first))
158        .filter(|&end| end <= program)
159        .ok_or_else(|| {
160            bad(
161                "program records",
162                first,
163                format!("{count} records past the program section"),
164            )
165        })?;
166    (0..count)
167        .map(|i| {
168            let at = crate::at(first, i, RECORD_LEN);
169            let (n, ptr) = (
170                r.size32le(at.saturating_add(0x60))?,
171                r.offset64le(at.saturating_add(0x64))?,
172            );
173            // the whole array lies in the section before any id is read
174            r.bytes(
175                ptr,
176                n.checked_mul(8)
177                    .ok_or_else(|| bad("program records", at, "permutation count"))?,
178            )?;
179            Ok(ProgramRecord {
180                key: r.u64le(at)?,
181                permutations: (0..n)
182                    .map(|j| r.u64le(crate::at(ptr, j, 8)))
183                    .collect::<Result<_>>()?,
184                raw: r.arr::<RECORD_LEN>(at)?,
185            })
186        })
187        .collect()
188}
189
190/// ExpressionShaderStreamableTextureSet: what a texture lookup row's value is the id of. It lists the texture
191/// parameters of a material, not texture assets: neither the id of a texture's resource nor the GUID of its asset
192/// occurs in the set of any of the 64 materials of a sample of 15 bundles (0 of 987 references; the link from a
193/// material to its textures is the depot block's, [`crate::depot`]). Layout, as read from the first 192 bytes of
194/// three sets (5, 2 and 14 entries; not read by this crate):
195///
196/// ```text
197/// 0x00 u32 count    0x04 u32 end of the entries (16 + 32 x count in all three)    0x08 8 zero bytes
198/// 0x10 count x 32 bytes: u32 1, u32 0x21, f32 1.0, u32 0, u64 parameter name hash (0x37e6de277c4250cc, the
199///      first of one set, is a name in depot blocks), u32 type (0x14c5fc30, a texture reference), u16 1, u16 a
200///      word that is not a function of the name hash (0x3487 occurs under two different hashes)
201/// then a tail (88, 44 and 236 bytes in the three), not decoded
202/// ```
203pub const STREAMABLE_TEXTURE_SET: u32 = 0x7A63_0735;