Skip to main content

frostbite_formats/
folder.rs

1//! A data folder (a Patch folder Studio writes, or the game's `Data`): its TOCs, its CAS archives and the install
2//! chunks of its `layout.toc` that name the archives' directories; reading the files a bundle region points at.
3#![allow(
4    clippy::disallowed_methods,
5    clippy::disallowed_types,
6    reason = "reader I/O: opens the container files of a data folder the caller names"
7)]
8
9use std::collections::BTreeMap;
10use std::fs;
11use std::io::{Read, Seek, SeekFrom};
12use std::path::{Path, PathBuf};
13
14use crate::cas::Cas;
15use primase_foundations::compress::Decompressor;
16use primase_foundations::compress::oodle::Oodle;
17
18use crate::bundle::{self, FileRef, Manifest};
19use crate::layout::{self, Layout};
20use crate::toc::{self, Loc, Toc};
21use crate::{Error, Result};
22
23fn io(path: &Path) -> impl FnOnce(std::io::Error) -> Error + '_ {
24    move |source| Error::Io {
25        path: path.to_path_buf(),
26        source,
27    }
28}
29
30/// The file of manifest asset `index` among the assets that follow the `before` assets listed first, counting from
31/// `first` in a bundle's file table; an error when the table has no such file.
32fn file_entry(files: &[FileRef], first: usize, before: usize, index: usize) -> Result<FileRef> {
33    let at = first.saturating_add(before).saturating_add(index);
34    files.get(at).copied().ok_or_else(|| {
35        Error::Missing(format!(
36            "the bundle's file table has {} files, none at {at}",
37            files.len()
38        ))
39    })
40}
41
42/// A data folder opened for reading: its TOCs, its CAS archives and the install chunks that name them.
43#[derive(Debug)]
44pub struct Folder {
45    /// The folder, as given to [`Folder::open`].
46    pub root: PathBuf,
47    /// TOC paths relative to the folder, `/`-separated, sorted.
48    pub tocs: Vec<String>,
49    /// (lowercase directory relative to `Win32`, archive number) -> path.
50    archives: BTreeMap<(String, u32), PathBuf>,
51    /// install chunk id -> lowercase directory relative to `Win32`.
52    chunk_dirs: BTreeMap<u32, String>,
53    /// The folder's `layout.toc`, read.
54    pub layout: Layout,
55}
56
57fn walk(dir: &Path, out: &mut Vec<PathBuf>) -> Result<()> {
58    let mut entries: Vec<_> = fs::read_dir(dir)
59        .map_err(io(dir))?
60        .collect::<Result<_, _>>()
61        .map_err(io(dir))?;
62    entries.sort_by_key(|e| e.path());
63    for e in entries {
64        let p = e.path();
65        if p.is_dir() { walk(&p, out)? } else { out.push(p) }
66    }
67    Ok(())
68}
69
70impl Folder {
71    /// Open the folder `root` (the one that holds `layout.toc` and `Win32`): finds its TOCs and CAS archives and
72    /// reads the install chunks of its `layout.toc`.
73    pub fn open(root: impl Into<PathBuf>) -> Result<Self> {
74        let root = root.into();
75        let win = root.join("Win32");
76        let mut files = Vec::new();
77        walk(&win, &mut files)?;
78        let mut tocs = Vec::new();
79        let mut archives = BTreeMap::new();
80        for p in files {
81            let rel = |p: &Path, base: &Path| {
82                p.strip_prefix(base)
83                    .map(|r| r.to_string_lossy().replace('\\', "/"))
84                    .unwrap_or_default()
85            };
86            let name = p
87                .file_name()
88                .map(|n| n.to_string_lossy().to_lowercase())
89                .unwrap_or_default();
90            if name.ends_with(".toc") {
91                tocs.push(rel(&p, &root));
92            } else if let Some(stem) = name.strip_suffix(".cas") {
93                let digits: String = stem
94                    .chars()
95                    .rev()
96                    .take_while(char::is_ascii_digit)
97                    .collect::<Vec<_>>()
98                    .into_iter()
99                    .rev()
100                    .collect();
101                if let Ok(num) = digits.parse::<u32>() {
102                    let dir = p
103                        .parent()
104                        .map(|d| rel(d, &win))
105                        .unwrap_or_default()
106                        .to_lowercase();
107                    archives.insert((dir, num), p.clone());
108                }
109            }
110        }
111        tocs.sort();
112        let layout_path = root.join("layout.toc");
113        let layout = layout::read(&fs::read(&layout_path).map_err(io(&layout_path))?)?;
114        let chunk_dirs = layout
115            .install_chunks
116            .iter()
117            .filter_map(|c| Some((c.persistent_index?, c.directory().to_lowercase())))
118            .collect();
119        Ok(Self {
120            root,
121            tocs,
122            archives,
123            chunk_dirs,
124            layout,
125        })
126    }
127
128    /// The path of the TOC `rel` (relative to the folder, as in [`Folder::tocs`]).
129    pub fn toc_path(&self, rel: &str) -> PathBuf {
130        self.root.join(rel)
131    }
132
133    /// The TOC at `rel` (relative to the folder, as in [`Folder::tocs`]), read and parsed.
134    pub fn toc(&self, rel: &str) -> Result<Toc> {
135        let path = self.toc_path(rel);
136        toc::read(&fs::read(&path).map_err(io(&path))?)
137    }
138
139    /// The CAS archive a [`Loc`] names, if the folder has it.
140    pub fn archive(&self, loc: Loc) -> Option<&Path> {
141        let dir = self.chunk_dirs.get(&loc.chunk)?;
142        self.archives
143            .get(&(dir.clone(), u32::from(loc.archive)))
144            .map(PathBuf::as_path)
145    }
146
147    /// The next free archive number in `loc`'s directory and its path (for a writer appending new data).
148    pub fn new_archive(&self, loc: Loc) -> Result<(u16, PathBuf)> {
149        let dir = self
150            .chunk_dirs
151            .get(&loc.chunk)
152            .ok_or_else(|| Error::Missing(format!("install chunk {}", loc.chunk)))?;
153        let next = self
154            .archives
155            .keys()
156            .filter(|(d, _)| d == dir)
157            .map(|(_, n)| n.saturating_add(1))
158            .max()
159            .unwrap_or(1);
160        let mut p = self.root.join("Win32");
161        for part in dir.split('/') {
162            p.push(part);
163        }
164        p.push(format!("cas_{next:02}.cas"));
165        let next = u16::try_from(next).map_err(|_| Error::Missing("archive number overflow".into()))?;
166        Ok((next, p))
167    }
168
169    /// The `size` stored bytes at `off` of the archive `loc` names.
170    pub fn read(&self, (loc, off, size): FileRef) -> Result<Vec<u8>> {
171        let path = self
172            .archive(loc)
173            .ok_or_else(|| Error::Missing(format!("{}: no archive for {loc:?}", self.root.display())))?;
174        let mut f = fs::File::open(path).map_err(io(path))?;
175        let len = f.metadata().map_err(io(path))?.len();
176        if u64::from(off).saturating_add(u64::from(size)) > len {
177            return Err(Error::Missing(format!(
178                "{}: {size} bytes at {off} lie past the end ({len} bytes)",
179                path.display()
180            )));
181        }
182        f.seek(SeekFrom::Start(u64::from(off))).map_err(io(path))?;
183        let mut buf = vec![
184            0;
185            usize::try_from(size)
186                .map_err(|_| Error::Missing(format!("{size} bytes do not fit memory")))?
187        ];
188        f.read_exact(&mut buf).map_err(io(path))?;
189        Ok(buf)
190    }
191}
192
193/// A Patch folder with the game's data behind it: files a Patch TOC places in the base game are read there, and CAS
194/// payloads are decoded with the game's own Oodle when they use it.
195#[derive(Debug)]
196pub struct Source<'a> {
197    /// The folder to read, which holds the TOCs asked about.
198    pub folder: &'a Folder,
199    /// The game's own `Data` folder, where `folder` is a Patch folder.
200    pub base: Option<&'a Folder>,
201    /// The game's Oodle library, for payloads that use it.
202    pub oodle: Option<&'a Oodle>,
203}
204
205impl Source<'_> {
206    /// The stored bytes of a file a bundle region points at, from the folder or the base game.
207    pub fn raw(&self, f: FileRef) -> Result<Vec<u8>> {
208        if f.0.patch {
209            self.folder.read(f)
210        } else {
211            self.base
212                .ok_or_else(|| Error::Missing(format!("base game data is needed for {:?}", f.0)))?
213                .read(f)
214        }
215    }
216
217    /// The superbundle TOC at `rel` (relative to the folder, as in [`Folder::tocs`]) as the game sees it: the folder's
218    /// TOC merged with the base game's TOC of the same path ([`toc::merge`]) when its flags say the base holds bundles or
219    /// chunks it lacks, else the folder's TOC as it is. A flagged TOC the base game has no TOC for (a superbundle the
220    /// folder adds: 30 of the 199 flagged TOCs of the installed mods) stands alone. An error when the TOC is flagged
221    /// and the source has no base game folder.
222    pub fn merged_toc(&self, rel: &str) -> Result<Toc> {
223        let patch = self.folder.toc(rel)?;
224        if !patch.has_base_bundles() && !patch.has_base_chunks() {
225            return Ok(patch);
226        }
227        let base = self.base.ok_or_else(|| {
228            Error::Missing(format!(
229                "{rel}: its flags name a base TOC and the source has no base game folder"
230            ))
231        })?;
232        if base.tocs.binary_search_by(|t| t.as_str().cmp(rel)).is_err() {
233            return toc::merge(&Toc::default(), &patch);
234        }
235        toc::merge(&base.toc(rel)?, &patch)
236    }
237
238    /// A CAS-framed file decoded to exactly `expected` bytes.
239    pub fn payload(&self, f: FileRef, expected: usize) -> Result<Vec<u8>> {
240        let raw = self.raw(f)?;
241        Ok(Cas { oodle: self.oodle }.decompress(&raw, expected)?)
242    }
243
244    /// The bundle's manifest and the index of its first asset file, read the way ReSkate's merge reads it: inline
245    /// in the region (its assets then no more than the region's files), else the region file that is a
246    /// manifest (raw or CAS-framed) whose asset count matches.
247    pub fn manifest(&self, files: &[FileRef], inline: &[u8]) -> Result<Option<(Manifest, usize, Vec<u8>)>> {
248        if !inline.is_empty() {
249            let m = bundle::read_manifest(inline)?;
250            fits(&m, files.len())?;
251            return Ok(Some((m, 0, inline.to_vec())));
252        }
253        let expected = files.len().saturating_sub(1);
254        for &f in files {
255            if f.2 > 32 << 20 || (!f.0.patch && self.base.is_none()) {
256                continue;
257            }
258            let raw = self.raw(f)?;
259            let m = if bundle::is_manifest(&raw) {
260                raw
261            } else {
262                let Ok(blocks) = crate::cas::blocks(&raw) else {
263                    continue;
264                };
265                let n: usize = blocks.iter().map(|b| b.decoded_size).sum();
266                match (Cas { oodle: self.oodle }).decompress(&raw, n) {
267                    Ok(d) if bundle::is_manifest(&d) => d,
268                    _ => continue,
269                }
270            };
271            let b = bundle::read_manifest(&m)?;
272            if b.count() == expected {
273                return Ok(Some((b, 1, m)));
274            }
275        }
276        Ok(None)
277    }
278
279    /// The EBX document named `name` (ASCII case-insensitive, as bundle manifests name them: `characters/.../x`) in
280    /// the first bundle of the TOC at `toc` (relative to the folder, as in [`Folder::tocs`]) that holds it, read; with
281    /// the bundle's name. An error when no bundle of that TOC holds it.
282    pub fn ebx(&self, toc: &str, name: &str) -> Result<(String, crate::ebx::Document)> {
283        let t = self.folder.toc(toc)?;
284        for (bundle_name, region) in &t.bundles {
285            let (files, inline) = bundle::read_region(region)?;
286            let Some((m, first, _)) = self.manifest(&files, &inline)? else {
287                continue;
288            };
289            if let Some((i, e)) = m
290                .ebx
291                .iter()
292                .enumerate()
293                .find(|(_, e)| e.name.eq_ignore_ascii_case(name))
294            {
295                let file = file_entry(&files, first, 0, i)?;
296                let bytes = self.payload(file, e.size as usize)?;
297                return Ok((bundle_name.clone(), crate::ebx::read(&bytes)?));
298            }
299        }
300        Err(Error::Missing(format!("no bundle of {toc} holds the EBX {name}")))
301    }
302
303    /// Every EBX document of the bundle named `bundle_name` in the TOC at `toc`, by its file GUID: its name and where
304    /// it lies (read it with [`Source::read_ebx`]). Each document's payload is read for its GUID, not parsed.
305    pub fn ebx_index(&self, toc: &str, bundle_name: &str) -> Result<BTreeMap<[u8; 16], EbxEntry>> {
306        let t = self.folder.toc(toc)?;
307        let (_, region) = t
308            .bundles
309            .iter()
310            .find(|(n, _)| n.eq_ignore_ascii_case(bundle_name))
311            .ok_or_else(|| Error::Missing(format!("{toc} has no bundle {bundle_name}")))?;
312        let (files, inline) = bundle::read_region(region)?;
313        let (m, first, _) = self
314            .manifest(&files, &inline)?
315            .ok_or_else(|| Error::Missing(format!("{bundle_name} has no manifest")))?;
316        let mut out = BTreeMap::new();
317        for (i, e) in m.ebx.iter().enumerate() {
318            let entry = EbxEntry {
319                name: e.name.clone(),
320                file: file_entry(&files, first, 0, i)?,
321                size: e.size as usize,
322            };
323            let guid = crate::ebx::read_file_guid(&self.payload(entry.file, entry.size)?)?;
324            out.insert(guid, entry);
325        }
326        Ok(out)
327    }
328
329    /// The resource with id `rid` of the bundle named `bundle_name` in the TOC at `toc`.
330    pub fn resource(&self, toc: &str, bundle_name: &str, rid: u64) -> Result<Resource> {
331        let t = self.folder.toc(toc)?;
332        let (_, region) = t
333            .bundles
334            .iter()
335            .find(|(n, _)| n.eq_ignore_ascii_case(bundle_name))
336            .ok_or_else(|| Error::Missing(format!("{toc} has no bundle {bundle_name}")))?;
337        let (files, inline) = bundle::read_region(region)?;
338        let (m, first, _) = self
339            .manifest(&files, &inline)?
340            .ok_or_else(|| Error::Missing(format!("{bundle_name} has no manifest")))?;
341        let (i, a) = m
342            .res
343            .iter()
344            .enumerate()
345            .find(|(_, a)| a.rid == rid)
346            .ok_or_else(|| Error::Missing(format!("{bundle_name} has no resource {rid:#x}")))?;
347        Ok(Resource {
348            name: a.name.clone(),
349            rtype: a.rtype,
350            meta: a.rmeta,
351            bytes: self.payload(file_entry(&files, first, m.ebx.len(), i)?, a.size as usize)?,
352        })
353    }
354
355    /// The EBX document at `entry`, read.
356    pub fn read_ebx(&self, entry: &EbxEntry) -> Result<crate::ebx::Document> {
357        crate::ebx::read(&self.payload(entry.file, entry.size)?)
358    }
359}
360
361/// A resource of a bundle ([`Source::resource`]): its name, type and metadata as its manifest states them, and its
362/// bytes.
363#[derive(Clone, Debug, PartialEq, Eq)]
364pub struct Resource {
365    /// The resource's name as its manifest states it.
366    pub name: String,
367    /// The resource type id from the manifest.
368    pub rtype: u32,
369    /// The 16 metadata bytes from the manifest, kept as stored.
370    pub meta: [u8; 16],
371    /// The decoded payload.
372    pub bytes: Vec<u8>,
373}
374
375/// Where an EBX document of a bundle lies ([`Source::ebx_index`]).
376#[derive(Clone, Debug, PartialEq, Eq)]
377pub struct EbxEntry {
378    /// Its name in the bundle's manifest.
379    pub name: String,
380    /// Archive, offset and stored size of the document's bytes (see [`FileRef`]).
381    pub file: FileRef,
382    /// Its payload's size.
383    pub size: usize,
384}
385
386/// An inline manifest's assets are the region's files from the first: an error, naming both counts, when it lists more
387/// than the region holds (the entries' file references would lie past the table).
388fn fits(m: &Manifest, files: usize) -> Result<()> {
389    if m.count() > files {
390        return Err(crate::bad(
391            "bundle manifest",
392            0,
393            format!("{} assets for a region of {files} files", m.count()),
394        ));
395    }
396    Ok(())
397}
398
399#[cfg(test)]
400mod manifest_tests {
401    use super::*;
402
403    #[test]
404    fn an_inline_manifest_with_more_assets_than_files_is_refused() {
405        let asset = |kind| bundle::Asset {
406            kind,
407            name: String::new(),
408            sha1: [0; 20],
409            size: 0,
410            rtype: 0,
411            rmeta: [0; 16],
412            rid: 0,
413            guid: [0; 16],
414            loff: 0,
415            lsize: 0,
416        };
417        let m = Manifest {
418            ebx: vec![asset(bundle::Kind::Ebx), asset(bundle::Kind::Ebx)],
419            res: vec![asset(bundle::Kind::Res)],
420            chunks: Vec::new(),
421            meta: Vec::new(),
422            big_endian: false,
423        };
424        assert!(fits(&m, 3).is_ok());
425        let e = fits(&m, 2).unwrap_err().to_string();
426        assert!(e.contains("3 assets for a region of 2 files"), "{e}");
427    }
428}