Skip to main content

unityrs_core/env/
load.rs

1// Ported from UnityPy 1.25.2 (MIT): UnityPy/environment.py, UnityPy/files/BundleFile.py, UnityPy/files/File.py. See THIRD_PARTY_LICENSES.
2//! Loads: [`Environment::load`], [`Environment::load_dependency`] and the
3//! files the resolver returns, and the registration of their CABs.
4
5use std::sync::atomic::{AtomicUsize, Ordering};
6use std::sync::{Arc, Mutex};
7
8use super::listing::Listing;
9#[cfg(test)]
10use super::registry::Hook;
11use super::registry::{DepCache, Dependency, Open, Stage, open_source, with_room};
12use super::{Cab, Environment, LoadMode, RawCab, Source, source_name};
13use crate::budget::Reservation;
14use crate::counters::LoadId;
15use crate::error::Error;
16use crate::files::{BundleFile, File};
17use crate::limits::{Charge, Meter};
18use crate::raw::RawStr;
19
20/// Reopens a registered dependency without pinning its files in the cache.
21pub struct DependencyFile(Arc<Dependency>);
22
23const _: () = crate::send_sync::<DependencyFile>();
24
25impl DependencyFile {
26    /// Read through the registered open, retrying with room if the pool refuses.
27    pub fn read<T>(&self, mut read: impl FnMut(File) -> Result<T, Error>) -> Result<T, Error> {
28        self.0.with_room(|| {
29            let file = self.0.handle()?;
30            if self.0.unreachable()
31                && let Some(cache) = self.0.cache.upgrade()
32            {
33                cache.forget(&self.0);
34            }
35            read(file)
36        })
37    }
38
39    /// Whether a file belongs to this dependency, including across reopens.
40    #[must_use]
41    pub fn owns(&self, file: &crate::files::SerializedFile) -> bool {
42        self.0.meter().same_input(file.meter())
43    }
44}
45
46/// Whose CABs a registration makes: a caller's file (slots that may hold
47/// it), or a dependency's (slots that reopen it), registered one at a time
48/// or staged, every one charged before any is published (the registry's
49/// module docs, "A failed dependency load").
50pub(super) enum Target<'a> {
51    Caller,
52    Dependency(&'a Arc<Dependency>),
53    Staged(&'a Arc<Dependency>, &'a mut Stage),
54}
55
56/// The internal error of a dependency's registration under `Lazy` made one
57/// key at a time ([`Target::Dependency`]), which no load makes.
58fn unstaged() -> Error {
59    Error::Malformed {
60        what: "registry",
61        detail: "a dependency's lazy load registered unstaged".into(),
62    }
63}
64
65/// Who asked the resolver for the file [`Environment::load_resolved`]
66/// loads, which decides how it is loaded and keyed.
67#[derive(Clone, Copy, Debug)]
68pub(super) enum Asked<'a> {
69    /// `find_file(name, is_dependency)`: keyed by the source's name (UnityPy
70    /// keys the path it found, `load_file` keying a path by itself).
71    Find {
72        /// UnityPy's flag (a dereference's `find_file` takes its default,
73        /// `True`).
74        is_dependency: bool,
75    },
76    /// A resource read's fallback, `load_file(name, True)`: keyed by the
77    /// name it asked for.
78    Fallback(&'a RawStr),
79}
80
81impl Environment {
82    /// `load_file`: detect, parse, register the CABs, add to `files` under the
83    /// source's name (a repeated name replaces the value in place, as a dict).
84    // The source is handed over (the frozen signature); a caller's load
85    // keeps nothing of it but its name.
86    #[allow(clippy::needless_pass_by_value)]
87    pub fn load(&self, source: Source) -> Result<File, Error> {
88        let name = source_name(&source);
89        let load_id = LoadId::draw(&name);
90        let file = self.load_listed(&name, &source, false, load_id)?;
91        self.inner.loaded.fetch_add(1, Ordering::Relaxed);
92        Ok(file)
93    }
94
95    pub(super) fn load_data(
96        &self,
97        name: RawStr,
98        data: bytes::Bytes,
99        reader: crate::stream::ReaderKind,
100        meter: &Arc<Meter>,
101    ) -> Result<Option<File>, Error> {
102        let ty = crate::detect::check_file_type_with_reader(&data, &self.ctx().limits, reader)?;
103        if ty == crate::detect::FileType::Zip {
104            self.load_zip_data(data, meter)?;
105            self.list_zip_none(name, meter)?;
106            return Ok(None);
107        }
108        self.load_detected(name, data, reader, ty, meter, 0)
109    }
110
111    pub(super) fn load_detected(
112        &self,
113        name: RawStr,
114        data: bytes::Bytes,
115        reader: crate::stream::ReaderKind,
116        ty: crate::detect::FileType,
117        meter: &Arc<Meter>,
118        depth: u32,
119    ) -> Result<Option<File>, Error> {
120        let file = crate::files::parse_file_at_depth(
121            name.clone(),
122            data,
123            ty,
124            None,
125            meter,
126            self.ctx(),
127            reader,
128            depth,
129        )?;
130        let load = self.register_file(&name, &file, &mut Target::Caller)?;
131        self.list(name, file.clone(), false, load, meter)?;
132        self.inner.loaded.fetch_add(1, Ordering::Relaxed);
133        Ok(Some(file))
134    }
135
136    /// A load UnityPy's `load_file` makes (the caller's, or a resource
137    /// fallback's under `keep_resolver_loads`): its own input meter, its
138    /// CABs registered as a caller's, and a `files` entry under `name` at
139    /// the load's number, as its registrations are ranked, charged to the
140    /// file's meter (a resolver's name copied into its key first; a
141    /// caller's own name is shared, not copied). Its bundle's reads are
142    /// keyed by `load_id`, which the load drew before its first attempt
143    /// (CR8's test feature, `crate::testing`).
144    fn load_listed(
145        &self,
146        name: &RawStr,
147        source: &Source,
148        resolved: bool,
149        load_id: LoadId,
150    ) -> Result<File, Error> {
151        let ctx = self.ctx();
152        let meter = Meter::for_input(&ctx.limits, ctx.budget.clone(), Reservation::Input);
153        // UnityPy environment.py:139-160
154        let file = open_source(name, source, &meter, ctx, load_id)?;
155        let load = self.register_file(name, &file, &mut Target::Caller)?;
156        let key = if resolved {
157            RawStr::charged(name.as_bytes(), &meter)?
158        } else {
159            name.clone()
160        };
161        self.list(key, file.clone(), false, load, &meter)?;
162        Ok(file)
163    }
164
165    /// Registers the CABs of a top-level `file` loaded under `name`: the
166    /// file itself, or a bundle's entries (`Eager`: as UnityPy parses them;
167    /// `Lazy`: every directory entry's name now, parsed on first use), or
168    /// stages them ([`Target::Staged`]: room for a lazy bundle's directory
169    /// reserved first). Every registration is one load's, ranked by its
170    /// directory path ([`registry`]); returns the load's number, which its
171    /// `files` entry takes ([`Self::list`]).
172    ///
173    /// [`registry`]: super::registry
174    pub(super) fn register_file(
175        &self,
176        name: &RawStr,
177        file: &File,
178        target: &mut Target<'_>,
179    ) -> Result<u64, Error> {
180        // Never: a dependency's load under `Lazy` is staged, so that it
181        // publishes nothing when it fails (`try_load_dep`); one that came
182        // this way would publish one key at a time (CR1's review, round 2:
183        // only a lazy bundle's entries were checked).
184        if self.inner.options.load == LoadMode::Lazy && matches!(target, Target::Dependency(_)) {
185            return Err(unstaged());
186        }
187        let reg = &self.inner.registry;
188        let load = reg.next_load();
189        let registered = match file {
190            File::Serialized(sf) => self.register_cab(
191                name,
192                Cab::Serialized(sf.clone()),
193                file.meter(),
194                target,
195                load,
196                &[],
197            ),
198            File::Raw(raw) => self.register_cab(
199                name,
200                Cab::Raw(RawCab::File(raw.clone())),
201                file.meter(),
202                target,
203                load,
204                &[],
205            ),
206            File::Bundle(b) => match self.inner.options.load {
207                LoadMode::Eager => self.register_eager(b, target, load, &mut Vec::new()),
208                LoadMode::Lazy => {
209                    if let Target::Staged(_, stage) = target {
210                        stage.reserve(b.directory().len())?;
211                    }
212                    for (i, d) in b.directory().iter().enumerate() {
213                        match target {
214                            Target::Caller => {
215                                reg.register_entry(&d.path, b.clone(), i, load, &[i], b.meter())?;
216                            }
217                            // Never: refused at the top.
218                            Target::Dependency(_) => return Err(unstaged()),
219                            Target::Staged(dep, stage) => {
220                                reg.stage(stage, &d.path, dep, load, &[i], b.meter(), true)?;
221                            }
222                        }
223                    }
224                    Ok(())
225                }
226            },
227            File::Web(w) => self.register_web(w, target, load, &mut Vec::new()),
228        };
229        registered.map(|()| load)
230    }
231
232    fn register_cab(
233        &self,
234        name: &RawStr,
235        cab: Cab,
236        meter: &Arc<Meter>,
237        target: &mut Target<'_>,
238        load: u64,
239        path: &[usize],
240    ) -> Result<(), Error> {
241        let reg = &self.inner.registry;
242        match target {
243            Target::Caller => reg.register_ready(name, cab, load, path, meter),
244            Target::Dependency(dep) => reg.register_dependency(name, dep, load, path, meter),
245            Target::Staged(dep, stage) => reg.stage(stage, name, dep, load, path, meter, false),
246        }
247    }
248
249    /// UnityPy's eager bundle: every block decoded, then every entry parsed
250    /// in directory order and each `SerializedFile` and raw entry registered
251    /// as it is parsed (`files/BundleFile.py:168-186`, `files/File.py:76-90`),
252    /// a nested bundle's entries in place. A failing entry fails the bundle
253    /// with the CABs before it registered; a block that fails registers
254    /// nothing (plan section 3.13). [`BundleFile::load_all`] decodes and
255    /// parses first, so this walk only reads its cells: when it failed with
256    /// an entry's error, the blocks all decoded and the entries before the
257    /// failing one are registered; any other failure is the decode's.
258    fn register_eager(
259        &self,
260        b: &Arc<BundleFile>,
261        target: &mut Target<'_>,
262        load: u64,
263        path: &mut Vec<usize>,
264    ) -> Result<(), Error> {
265        let all = b.load_all();
266        if let Err(e) = &all
267            && !matches!(e, Error::Entry { .. })
268        {
269            return all;
270        }
271        for (i, d) in b.directory().iter().enumerate() {
272            path.push(i);
273            let step = b.entry(i).and_then(|f| {
274                self.inner.eager_parsed.fetch_add(1, Ordering::Relaxed);
275                match f {
276                    File::Bundle(nested) => self.register_eager(&nested, target, load, path),
277                    File::Serialized(sf) => self.register_cab(
278                        &d.path,
279                        Cab::Serialized(sf),
280                        b.meter(),
281                        target,
282                        load,
283                        path,
284                    ),
285                    File::Raw(_) => {
286                        let cab = Cab::Raw(RawCab::InBundle {
287                            bundle: b.clone(),
288                            dir_index: i,
289                        });
290                        self.register_cab(&d.path, cab, b.meter(), target, load, path)
291                    }
292                    File::Web(w) => self.register_web(&w, target, load, path),
293                }
294            });
295            path.pop();
296            if let Err(e) = step {
297                return Err(all.err().unwrap_or(e));
298            }
299        }
300        all
301    }
302
303    // files/File.py:76-90: WebFile members register just like bundle members.
304    fn register_web(
305        &self,
306        w: &Arc<crate::files::WebFile>,
307        target: &mut Target<'_>,
308        load: u64,
309        path: &mut Vec<usize>,
310    ) -> Result<(), Error> {
311        for (i, d) in w.directory().iter().enumerate() {
312            path.push(i);
313            let result = w.entry(i).and_then(|f| match f {
314                File::Bundle(b) => self.register_eager(&b, target, load, path),
315                File::Web(w) => self.register_web(&w, target, load, path),
316                File::Serialized(sf) => {
317                    self.register_cab(&d.path, Cab::Serialized(sf), w.meter(), target, load, path)
318                }
319                File::Raw(raw) => self.register_cab(
320                    &d.path,
321                    Cab::Raw(RawCab::File(raw)),
322                    w.meter(),
323                    target,
324                    load,
325                    path,
326                ),
327            });
328            path.pop();
329            result?;
330        }
331        Ok(())
332    }
333
334    /// Loads `source` as a dependency, as `find_file` loads one through the
335    /// resolver: its own meter under [`crate::budget::Reservation::Dependency`]
336    /// (so a refusal names the dependency pool), its CABs registered (lazily
337    /// under [`LoadMode::Lazy`]) without a strong handle, kept in the LRU and
338    /// evictable, and not added to [`Self::files`] (UnityPy has no such call;
339    /// a batch engine registers its inputs this way). The returned handle is
340    /// the caller's: an eviction drops only the LRU's, so while the caller
341    /// holds it the slots reach the same open file through their weak handle.
342    ///
343    /// Under [`LoadMode::Lazy`] a load that fails publishes nothing: every
344    /// registration is charged, with the LRU's entry, before the first is
345    /// published, so a refusal at any of them leaves the registry as it was
346    /// (the registry's module docs, "A failed dependency load"). Under
347    /// [`LoadMode::Eager`] the CABs are registered as they are parsed, as
348    /// UnityPy's `load_file` registers them, so a load that fails keeps
349    /// those registered before its failure (the partial registration of
350    /// the module docs).
351    // The source is handed over (the frozen signature) and kept by a clone.
352    pub fn load_dependency(&self, source: Source) -> Result<File, Error> {
353        self.load_dependency_with_handle(source)
354            .map(|(file, _)| file)
355    }
356
357    /// Load a dependency and give the caller an evictable handle to its exact
358    /// occurrence, even if another dependency registers the same CAB name.
359    /// Uses the same dependency pool and eviction retry as
360    /// [`Self::load_dependency`].
361    #[allow(clippy::needless_pass_by_value)]
362    pub fn load_dependency_with_handle(
363        &self,
364        source: Source,
365    ) -> Result<(File, DependencyFile), Error> {
366        let name = source_name(&source);
367        // Drawn once, so the retry keeps it (CR8's test feature).
368        let load_id = LoadId::draw(&name);
369        // A pool refusal evicts the others and retries once (plan 3.13); a
370        // second refusal fails naming the pool, and nothing of the
371        // dependency is kept (its first attempt's charges returned with its
372        // handle; under `Lazy` neither attempt published a key).
373        with_room(Some(&self.inner.cache), None, || {
374            self.try_load_dep(&name, &source, false, load_id)
375                .map(|(file, _, _, dep)| (file, DependencyFile(dep)))
376        })
377    }
378
379    /// One attempt of [`Self::load_dependency_with_handle`], on a meter of its own: the file,
380    /// the load's number and, with `listed` under `Lazy`, its `files`
381    /// entry under `name` charged and not yet published. Under `Lazy` its
382    /// registrations are staged, every one charged, and the LRU's entry and
383    /// the listing charged, before the first is published
384    /// ([`Self::register_staged`]); under `Eager` each is published as it
385    /// is registered, and a listing is the caller's to charge. Its
386    /// bundle's reads, and every reopen's, are keyed by `load_id` (CR8's
387    /// test feature).
388    fn try_load_dep(
389        &self,
390        name: &RawStr,
391        source: &Source,
392        listed: bool,
393        load_id: LoadId,
394    ) -> Result<(File, u64, Option<Listing>, Arc<Dependency>), Error> {
395        let ctx = self.ctx();
396        let meter = Meter::for_input(&ctx.limits, ctx.budget.clone(), Reservation::Dependency);
397        let file = open_source(name, source, &meter, ctx, load_id)?;
398        let dep = Arc::new(Dependency {
399            name: name.clone(),
400            source: source.clone(),
401            open: Mutex::new(Open::of(&file, meter)),
402            ctx: ctx.clone(),
403            cache: Arc::downgrade(&self.inner.cache),
404            slots: AtomicUsize::new(0),
405            #[cfg(feature = "test-counters")]
406            load_id,
407        });
408        let (load, room, listing) = match self.inner.options.load {
409            LoadMode::Lazy => {
410                let listed = listed.then_some(name);
411                let (load, room, listing) = self.register_staged(&dep, &file, listed)?;
412                (load, Some(room), listing)
413            }
414            LoadMode::Eager => {
415                let load = self.register_file(&dep.name, &file, &mut Target::Dependency(&dep))?;
416                (load, None, None)
417            }
418        };
419        #[cfg(test)]
420        self.inner.cache.hook(Hook::LoadRegistered);
421        self.inner.cache.loaded.fetch_add(1, Ordering::Relaxed);
422        match room {
423            Some(room) => self.inner.cache.touch_paid(&dep, &file, room),
424            None => self.inner.cache.touch(&dep, &file)?,
425        }
426        // Another load registered every key this one did meanwhile: no slot
427        // reaches it, and the LRU lets go of it ([`registry`]). The last
428        // slot to go before the touch told an LRU that did not hold it yet;
429        // one that goes after it tells the LRU itself (`DepSlot`).
430        if dep.unreachable() {
431            self.inner.cache.forget(&dep);
432        }
433        #[cfg(test)]
434        self.inner.cache.hook(Hook::LoadChecked);
435        Ok((file, load, listing, dep))
436    }
437
438    /// A dependency's registrations under `Lazy`, staged ([`Stage`]: the
439    /// list that holds them, room for a bundle's whole directory charged
440    /// first, then each key, slot, new key's share and copy, and position's
441    /// path, charged to the dependency's meter), then the LRU's entry
442    /// charged ([`DepCache::room`]) and, with `listed`, the `files` entry
443    /// under that name (its key's copy, its share and its lowered key,
444    /// [`Self::charge_listing`]), then every registration published in the
445    /// order it was staged ([`super::registry::Registry::commit`]). A
446    /// refusal before the commit drops the stage, which returns every
447    /// charge and was never in the registry, so no lookup, on this thread
448    /// or another, can have found any of it. Nothing after the commit's
449    /// start can fail the load: the `settle` each registration runs keeps a
450    /// refusal to itself and leaves the slot it could not reach listed for
451    /// a later call, as every `settle` does. The load's number, the LRU
452    /// entry's charge for [`DepCache::touch_paid`], and the listing.
453    fn register_staged(
454        &self,
455        dep: &Arc<Dependency>,
456        file: &File,
457        listed: Option<&RawStr>,
458    ) -> Result<(u64, Charge, Option<Listing>), Error> {
459        let mut stage = Stage::new(file.meter());
460        let load = self.register_file(&dep.name, file, &mut Target::Staged(dep, &mut stage))?;
461        let room = DepCache::room(file)?;
462        let listing = match listed {
463            Some(name) => {
464                let key = RawStr::charged(name.as_bytes(), file.meter())?;
465                Some(Self::charge_listing(key, file.meter())?)
466            }
467            None => None,
468        };
469        self.inner.registry.commit(stage)?;
470        Ok((load, room, listing))
471    }
472
473    /// A file the resolver returned (module docs, "Resolver loads in
474    /// `files`"). With `keep_resolver_loads`: `find_file`'s
475    /// ([`Asked::Find`]) with `is_dependency` is a dependency, listed in
476    /// `files` under the source's name (flagged a dependency unless it is a
477    /// raw reader, which UnityPy gives no flag); without `is_dependency` a caller's
478    /// load under that name (UnityPy's `load_file(fp, name=name,
479    /// is_dependency=False)`, `environment.py:332`, `:354`); a resource
480    /// fallback's ([`Asked::Fallback`]) is a caller's load under the name it
481    /// asked for. Without `keep_resolver_loads` every one is a dependency in
482    /// the LRU only (`PARITY.md` section 19.3's row "resolver loads under
483    /// `keep_resolver_loads` false", and section 19.2's
484    /// `SerializedFile.load_dependencies` row).
485    ///
486    /// Every one of these loads, whatever its listing, is one run of
487    /// [`with_room`]: a pool refusal anywhere in it (the source's fetcher's
488    /// own included, counted as it returns: `registry::from_callers`)
489    /// evicts the other dependencies and loads once more, as
490    /// [`Self::load_dep`] does (review round 12: a load listed as no
491    /// dependency, `find_file`'s with `is_dependency` false or a resource
492    /// fallback's under `keep_resolver_loads`, returned its fetcher's
493    /// refusal untried, unless the fallback's candidate step around it
494    /// retried it). A listing is charged to the loaded file's meter inside
495    /// that run, so a second run loads and lists again (review round 8: a
496    /// refused listing failed the load, whose file then stayed registered
497    /// and unlisted).
498    pub(super) fn load_resolved(&self, source: &Source, asked: Asked<'_>) -> Result<File, Error> {
499        let keep = self.inner.options.keep_resolver_loads;
500        let resolved;
501        let name = match asked {
502            Asked::Fallback(name) => name,
503            Asked::Find { .. } => {
504                resolved = source_name(source);
505                &resolved
506            }
507        };
508        // Drawn once, so the retry keeps it (CR8's test feature).
509        let load_id = LoadId::draw(name);
510        with_room(Some(&self.inner.cache), None, || match asked {
511            _ if !keep => self
512                .try_load_dep(name, source, false, load_id)
513                .map(|(file, ..)| file),
514            Asked::Fallback(_)
515            | Asked::Find {
516                is_dependency: false,
517            } => self.load_listed(name, source, true, load_id),
518            Asked::Find {
519                is_dependency: true,
520            } => {
521                // Under `Lazy` the listing is charged before the keys are
522                // published, so a refused one publishes none of them
523                // (`register_staged`); under `Eager`, after.
524                let (file, load, listing, _) = self.try_load_dep(name, source, true, load_id)?;
525                let listing = if let Some(listing) = listing {
526                    listing
527                } else {
528                    let meter = file.meter();
529                    let key = RawStr::charged(name.as_bytes(), meter)?;
530                    Self::charge_listing(key, meter)?
531                };
532                // A reader has no `is_dependency`: `parse_file` returns the
533                // reader itself (`helpers/ImportHelper.py:136-156`), which
534                // `getattr(f, "is_dependency", False)` reads false.
535                let flag = !matches!(file, File::Raw(_));
536                self.publish_listing(listing, file.clone(), flag, load);
537                Ok(file)
538            }
539        })
540    }
541}