Skip to main content

unityrs_core/env/
container_index.rs

1// Ported from UnityPy 1.25.2 (MIT): UnityPy/environment.py, UnityPy/files/ObjectReader.py, UnityPy/helpers/ContainerHelper.py. See THIRD_PARTY_LICENSES.
2//! The container: [`Environment::container`], [`Environment::container_path`]
3//! and the pass that builds their index (UnityPy's
4//! `_build_container_index`), the walk of each file's preload table.
5
6use std::collections::HashMap;
7use std::fmt;
8use std::mem::size_of;
9use std::sync::Arc;
10use std::sync::atomic::Ordering;
11
12use super::deref::Reached;
13use super::registry::{Rank, SlotKind, load_keys_here};
14use super::revisit::Pending;
15use super::watch::{FILES, Flag, REGISTRY, Walk};
16use super::{Cab, DerefError, Environment, Found, LoadMode, PPtr, never_evicts, pptr};
17use crate::budget::Reservation;
18use crate::container::{Container, ContainerPath, ContainerRow};
19use crate::error::{Error, Unsupported};
20use crate::files::{BundleFile, File, SerializedFile};
21use crate::limits::{Charge, GrantScope, HASH_TABLE_SHARES, Meter, MeteredVec};
22use crate::object::ObjectRef;
23use crate::once::CellError;
24use crate::pystr::{clean_external_path, py_lower, simplify_name};
25use crate::raw::RawStr;
26use crate::walk::STEP_WORK;
27
28/// The iterator [`Environment::container`] returns, CABs in registry order.
29pub struct ContainerItems<'a> {
30    env: &'a Environment,
31    after: Option<Rank>,
32    current: Option<(Arc<SerializedFile>, usize)>,
33}
34
35const _: () = crate::send_sync::<ContainerItems<'static>>();
36
37impl Iterator for ContainerItems<'_> {
38    type Item = Result<ContainerItem, Error>;
39
40    fn next(&mut self) -> Option<Self::Item> {
41        loop {
42            if let Some((sf, k)) = self.current.take() {
43                match sf.container() {
44                    Ok(c) => {
45                        if let Some(row) = c.rows().get(k) {
46                            let row = row.clone();
47                            self.current = Some((sf.clone(), k + 1));
48                            return Some(Ok(ContainerItem { row, file: sf }));
49                        }
50                    }
51                    Err(e) => return Some(Err(e)),
52                }
53            }
54            // UnityPy environment.py:221-224: every non-dependency CAB.
55            let (rank, slot) = self.env.registry().slot_after(self.after.as_ref())?;
56            self.after = Some(rank);
57            if slot.is_dependency() {
58                continue;
59            }
60            match self.env.registry().resolve(&slot) {
61                Ok(Some(Cab::Serialized(sf))) => self.current = Some((sf, 0)),
62                Ok(_) => {}
63                Err(e) => return Some(Err(e)),
64            }
65        }
66    }
67}
68
69impl fmt::Debug for ContainerItems<'_> {
70    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
71        f.debug_struct("ContainerItems").finish_non_exhaustive()
72    }
73}
74
75/// A container row with the object it points at, as `env.container` yields.
76#[derive(Clone, Debug)]
77pub struct ContainerItem {
78    /// The row.
79    pub row: ContainerRow,
80    /// The file whose container it is.
81    pub file: Arc<SerializedFile>,
82}
83
84const _: () = crate::send_sync::<ContainerItem>();
85
86/// A container pass's index of each bundle's entry names, lower-cased, to
87/// the entry the first of them names (rule 4's scan of a bundle's `files`
88/// as one lookup): built once for each bundle the pass meets, charged to the
89/// bundle's meter, and dropped with the pass.
90#[derive(Default)]
91pub(super) struct Siblings {
92    bundles: HashMap<usize, SiblingNames>,
93}
94
95struct SiblingNames {
96    /// Keeps the bundle, and so the address that keys it, for the pass.
97    _bundle: Arc<BundleFile>,
98    names: HashMap<RawStr, usize>,
99    _charge: Charge,
100}
101
102impl Siblings {
103    /// The directory index of `bundle`'s first `files` entry whose name
104    /// lowers to `clean`.
105    pub(super) fn find(
106        &mut self,
107        bundle: &Arc<BundleFile>,
108        clean: &RawStr,
109    ) -> Result<Option<usize>, Error> {
110        let key = Arc::as_ptr(bundle).addr();
111        let entry = match self.bundles.entry(key) {
112            std::collections::hash_map::Entry::Occupied(o) => o.into_mut(),
113            std::collections::hash_map::Entry::Vacant(v) => {
114                let meter = bundle.meter();
115                let slots = bundle.files();
116                let n = u64::try_from(slots.len()).unwrap_or(u64::MAX);
117                let charge = meter.charge(
118                    HASH_TABLE_SHARES
119                        .saturating_mul((size_of::<RawStr>() + size_of::<usize>() + 1) as u64)
120                        .saturating_mul(n.saturating_add(1)),
121                )?;
122                let mut names = HashMap::with_capacity(slots.len());
123                for s in slots {
124                    names
125                        .entry(py_lower(&s.name, meter)?)
126                        .or_insert(s.dir_index);
127                }
128                v.insert(SiblingNames {
129                    _bundle: bundle.clone(),
130                    names,
131                    _charge: charge,
132                })
133            }
134        };
135        Ok(entry.names.get(clean.as_bytes()).copied())
136    }
137}
138
139/// What one external costs a preload walk's map of answers:
140/// [`HASH_TABLE_SHARES`] buckets.
141const TARGET_SHARE: u64 = HASH_TABLE_SHARES * (size_of::<i32>() + size_of::<Known>() + 1) as u64;
142
143/// No table index: the end of a list in [`Targets::next`].
144const NONE: usize = usize::MAX;
145
146/// A preload walk's answers for its file's externals, each kept only while
147/// UnityPy's next dereference would give the same one: a bundle sibling, or
148/// a file id past the externals' end, for the walk; an answer of `files` or
149/// the registry (found or not) until a change to a key its lookup read
150/// ([`super::watch`]: a load's registration, any thread's, the resolver's
151/// included, and under [`LoadMode::Lazy`] the entries a reach registers),
152/// so the answer UnityPy's next lookup could find changed is asked again
153/// and every other is kept; and the resolver's load not at all (the next
154/// dereference looks again, and finds it if the load registered the name
155/// asked for, or loads again if not, as UnityPy's `find_file` does when its
156/// walk matched a longer file name by suffix). An external is looked up
157/// once, and once more per change to one of its keys (one the resolver
158/// loads under another name, once per dereference), so a walk costs its
159/// table and its externals' names (plan decision 11), plus those lookups
160/// again, each lookup past an external's first counted as work on the
161/// walking file's meter (`STEP_WORK` and the external's path's bytes;
162/// review round 4: every change, any thread's, expired every answer, so a
163/// table of pointers to long names cost the table times the names).
164///
165/// Each answer with a flag (one of `files`, the registry or the resolver's
166/// load) lists the table indexes dereferenced under it (`answers`, linked
167/// through `next`), so when a load sets its flag, the walk's own or another
168/// thread's ([`Walk::fired`]), those indexes are dereferenced again at the
169/// next row whose span reaches them, or later in the row the walk is in
170/// (review round 13: a walk that dereferenced each index once never gave a
171/// later row's path to the file a load in between put under the key the
172/// index's external resolves through; review round 14: only a load of the
173/// walk's own put them back, and not after another thread's had set the
174/// flag first).
175///
176/// [`LoadMode::Lazy`]: super::LoadMode::Lazy
177///
178/// Each charge is released after the memory it pays for (plan section 3.15):
179/// fields drop in order, so `known` and `answers` go first, then the walk's
180/// watches with their flags and their own charge, then `known`'s charge
181/// (review round 5: the watches' charge was released before their entries
182/// left the map).
183struct Targets<'a> {
184    known: HashMap<i32, Known>,
185    /// Every flag the walk set up, by its number, with the head of the list
186    /// of table indexes dereferenced under its answer.
187    answers: MeteredVec<Answer>,
188    /// Each table index's next in its answer's list, made at the first
189    /// link: an index is in one list at most, the one of the answer it last
190    /// got, and leaves it when the list is taken.
191    next: Option<MeteredVec<usize>>,
192    /// After `known`, whose flags drop before the walk's watches end.
193    walk: Walk<'a>,
194    /// `known`'s table, after it.
195    charge: Charge,
196}
197
198/// One external's answer in [`Targets`].
199struct Known {
200    found: Option<Found>,
201    /// Whether the walk keeps it: not the resolver's load, which the next
202    /// dereference asks again.
203    kept: bool,
204    /// Set when a key its lookup read changes; `None`: kept for the walk (a
205    /// sibling, a file id past the end).
206    flag: Option<Arc<Flag>>,
207}
208
209/// The table indexes dereferenced under one flag's answer: the head of a
210/// list linked through [`Targets::next`] (the walk holds the flag itself,
211/// [`Walk`]).
212struct Answer {
213    head: usize,
214}
215
216/// The work a preload walk counts on its file's meter for each index it
217/// puts back to be dereferenced again ([`Targets::reopen`]): a step for the
218/// dereference, and one per level of the [`Pending`] set for putting the
219/// index back (the find and the removal that dereference makes read no
220/// more words than a first dereference's).
221fn reopen_work(pending: &Pending) -> u64 {
222    STEP_WORK.saturating_mul(1 + u64::try_from(pending.levels()).unwrap_or(u64::MAX))
223}
224
225impl Targets<'_> {
226    /// The answer kept for `file_id`, if it holds.
227    fn kept(&self, file_id: i32) -> Option<&Known> {
228        self.known
229            .get(&file_id)
230            .filter(|k| k.kept && k.flag.as_ref().is_none_or(|f| !f.changed()))
231    }
232
233    /// Records `found` for `file_id`, kept (`kept`) while `flag` is unset
234    /// (`None`: for the walk).
235    fn keep(
236        &mut self,
237        file_id: i32,
238        found: Option<Found>,
239        kept: bool,
240        flag: Option<Arc<Flag>>,
241    ) -> Result<(), Error> {
242        if !self.known.contains_key(&file_id) {
243            self.charge.grow(TARGET_SHARE)?;
244        }
245        self.known.insert(file_id, Known { found, kept, flag });
246        Ok(())
247    }
248
249    /// Lists table index `i` (of a table of `len`) under the answer of flag
250    /// `id`.
251    fn link(&mut self, i: usize, id: usize, len: usize, meter: &Arc<Meter>) -> Result<(), Error> {
252        if self.next.is_none() {
253            let mut n = MeteredVec::with_capacity(meter, len)?;
254            for _ in 0..len {
255                n.push(NONE)?;
256            }
257            self.next = Some(n);
258        }
259        if let (Some(answer), Some(slot)) = (
260            self.answers.as_mut_slice().get_mut(id),
261            self.next.as_mut().and_then(|n| n.as_mut_slice().get_mut(i)),
262        ) {
263            *slot = answer.head;
264            answer.head = i;
265        }
266        Ok(())
267    }
268
269    /// Puts back into `pending` every table index listed under an answer
270    /// whose flag a load set since the last call, any thread's, each
271    /// counted on `meter` first ([`reopen_work`]); the lists are taken, so
272    /// an index is put back once per dereference.
273    fn reopen(&mut self, pending: &mut Pending, meter: &Meter) -> Result<(), Error> {
274        let Self {
275            walk,
276            answers,
277            next,
278            ..
279        } = self;
280        let links = next.as_ref().map(MeteredVec::as_slice);
281        for id in walk.fired() {
282            let Some(answer) = answers.as_mut_slice().get_mut(id) else {
283                continue;
284            };
285            let mut i = std::mem::replace(&mut answer.head, NONE);
286            while i != NONE {
287                meter.count_work(reopen_work(pending))?;
288                pending.insert(i);
289                i = links.and_then(|l| l.get(i)).copied().unwrap_or(NONE);
290            }
291        }
292        Ok(())
293    }
294}
295
296/// A dereference failure as a file-level error (the container preload's
297/// raise): a load failure as it is, any other as [`Error::Deref`].
298fn deref_error(e: DerefError) -> Error {
299    match e {
300        DerefError::Load(e) => e,
301        other => Error::Deref(Arc::new(other)),
302    }
303}
304
305impl Environment {
306    /// `container`: every non-dependency CAB's container rows, CABs in
307    /// registry order, after the preload index is built (`environment.py:208-226`),
308    /// as an iterator. It walks the registry, as UnityPy walks `env.cabs`, so
309    /// a CAB a dependency's replaced under its key is not listed.
310    pub fn container(&self) -> Result<ContainerItems<'_>, Error> {
311        self.build_container_index()?;
312        Ok(ContainerItems {
313            env: self,
314            after: None,
315            current: None,
316        })
317    }
318
319    /// `obj.container` (`files/ObjectReader.py:199-204`): the container index
320    /// built first ([`Self::container`]'s pass, whose failure is this call's),
321    /// then the path the object's file's `path_dict` holds for its path id
322    /// ([`crate::container`]): a row of the file's own container whose asset
323    /// is the object, the last such row winning, else the first row whose
324    /// preload span reached it. The paths live in the file the object was
325    /// read from, as UnityPy's live in its `assets_file`, so a second handle
326    /// a caller opened on the same bytes holds only its own rows'.
327    ///
328    /// Where a dependency may be evicted (every option but
329    /// [`EnvOptions::unitypy`]'s, [`EnvOptions::dep_cache_bytes`]), an
330    /// object of a dependency's file is refused with
331    /// [`Unsupported::EvictableContainerPath`]: an eviction drops the paths
332    /// the preload set in that file and its reopen does not walk again, so
333    /// the answer would depend on when the LRU evicted (`PARITY.md` section
334    /// 19.3). The preload sets no path in such a file.
335    ///
336    /// [`EnvOptions::unitypy`]: super::EnvOptions::unitypy
337    /// [`EnvOptions::dep_cache_bytes`]: super::EnvOptions::dep_cache_bytes
338    pub fn container_path(&self, obj: &ObjectRef) -> Result<Option<ContainerPath>, Error> {
339        let sf = obj.file();
340        if self.may_reopen(sf) {
341            return Err(Error::Unsupported(Unsupported::EvictableContainerPath));
342        }
343        self.build_container_index()?;
344        sf.container()?.path_of(obj.path_id(), sf.meter())
345    }
346
347    /// Whether `sf` is a dependency's file the LRU may evict and the
348    /// Environment open again.
349    fn may_reopen(&self, sf: &SerializedFile) -> bool {
350        !never_evicts(&self.inner.options) && sf.meter().reservation() == Reservation::Dependency
351    }
352
353    /// `_build_container_index` (`environment.py:208-215`): once until the
354    /// next load's registration, every registered `SerializedFile`'s preload
355    /// table, dependencies' included, in registry order. UnityPy sets its
356    /// flag before the loop and each registration clears it, so a pass that
357    /// fails leaves the index built unless the pass itself registered
358    /// something; here a pass marks built the registry generation it started
359    /// at when it ends or fails with an error the input decides, and any
360    /// load's registration since starts a new generation. A refusal (a
361    /// budget, meter or I/O failure, [`crate::once::CellError::is_transient`])
362    /// is never kept (plan 4.1): the pass is not marked built, and the next
363    /// ask walks again (review round 5: a pool refused once left the preload
364    /// unwalked for the generation). What a walk the file's own meter refused
365    /// was granted counts as that open's work, so a refusal the next ask
366    /// would repeat stops at [`crate::LimitKind::DecodeWork`], which is kept
367    /// ([`Self::walk_counted`]). A slot's table is marked walked only when its
368    /// walk ends (UnityPy clears `_preload_table` then). A pass whose own
369    /// dereferences make a new key fails as UnityPy's loop over
370    /// `cabs.values()` does (`RuntimeError: dictionary changed size during
371    /// iteration`), at the step after; a key a nested container's entries
372    /// make under [`LoadMode::Lazy`] is not one (an eager load made it at
373    /// load), nor is one another thread's load makes. Under `Lazy` the pass
374    /// first reaches every registered entry and every dropped one, an
375    /// evicted dependency's opened again (the registry's module docs; plan
376    /// section 9, Q13), so it walks what an eager load registered; a slot
377    /// dropped after a pass started is dropped by a later load, whose
378    /// registration starts a new generation in the step that lists the
379    /// slot whether or not it changes a key (review round 16: one ranked
380    /// between the key's position and its value changed none, started
381    /// none, and the entries its slot's reach registered were never
382    /// walked), or by the reach of such a load's slot, so a pass finished
383    /// at the current generation leaves none unreached but for a kept
384    /// refusal's. Two passes may run at once: each walks every table not yet marked, and each `path_dict`
385    /// entry keeps the least stamp ([`crate::container`]), so both leave
386    /// what one after the other would.
387    ///
388    /// [`LoadMode::Lazy`]: super::LoadMode::Lazy
389    fn build_container_index(&self) -> Result<(), Error> {
390        let reg = &self.inner.registry;
391        // Every slot dropped before this call is reached first, as by every
392        // other reader (plan section 3.16, R3); a refusal is the walk's to
393        // meet, or the next call's.
394        reg.settle().ok();
395        let generation = reg.generation();
396        if reg.built(generation) {
397            return Ok(());
398        }
399        let result = self.container_pass(generation);
400        if result.as_ref().err().is_none_or(|e| !e.is_transient()) {
401            reg.publish_built(generation);
402        }
403        result
404    }
405
406    /// [`Self::preload`] of `sf`'s table, as a lazy computation of `sf`'s
407    /// open (`crate::once`'s rule on work): when `sf`'s own meter refuses
408    /// it, what the meter granted it on this thread counts as the open's
409    /// work, and once that work has passed the allowance the refusal is
410    /// [`crate::LimitKind::DecodeWork`], which the input decides, so the pass
411    /// keeps it.
412    fn walk_counted(
413        &self,
414        sf: &Arc<SerializedFile>,
415        container: &Container,
416        at: (u64, &Rank),
417        siblings: &mut Siblings,
418    ) -> Result<(), Error> {
419        let meter = sf.meter();
420        let scope = GrantScope::enter(meter);
421        let walked = self.preload(sf, container, at, siblings);
422        let own = scope.finish(meter);
423        match walked {
424            Err(e) if e.is_meter_refusal() => {
425                meter.add_work(own);
426                Err(meter.work_refused().map_or(e, Error::Limit))
427            }
428            other => other,
429        }
430    }
431
432    fn container_pass(&self, generation: u64) -> Result<(), Error> {
433        let reg = &self.inner.registry;
434        let changed = || Error::Malformed {
435            what: "container",
436            detail: "dictionary changed size during iteration".into(),
437        };
438        // Under `Lazy`, every registered entry is reached before the walk
439        // (the registry's module docs), so a nested container's entries are
440        // walked at the positions and with the values an eager load gives
441        // them, whichever container the walk would meet first (review round
442        // 4: a later container's entry replaced the value of a key the walk
443        // had passed, and the walk had read the earlier one).
444        if self.inner.options.load == LoadMode::Lazy {
445            let mut after: Option<Rank> = None;
446            while let Some((rank, slot)) = reg.slot_after(after.as_ref()) {
447                reg.reach(&slot, None)?;
448                after = Some(rank);
449            }
450            // Every slot a registration dropped, those these reaches dropped
451            // included, and those of an evicted dependency, opened again
452            // (plan section 9, Q13); a refusal there is the pass's.
453            reg.settle_for_pass()?;
454        }
455        let mut siblings = Siblings::default();
456        let mut after: Option<Rank> = None;
457        let mut grew = false;
458        while let Some((rank, slot)) = reg.slot_after(after.as_ref()) {
459            if grew {
460                return Err(changed());
461            }
462            // The dependency's file is held while its table is walked, so
463            // an eviction the walk makes cannot drop the CAB's bundle.
464            let Some(held) = reg.resolve_held(&slot)? else {
465                after = Some(rank);
466                continue;
467            };
468            let Cab::Serialized(sf) = &held.cab else {
469                after = Some(rank);
470                continue;
471            };
472            if !slot.walked() {
473                let before = load_keys_here();
474                let mut walk = || {
475                    let container = sf.container()?;
476                    self.walk_counted(sf, container, (generation, &rank), &mut siblings)
477                };
478                // A dependency's table is walked on its meter: a walk the
479                // pool refuses evicts the others and walks again
480                // (`registry::with_room`; review round 7: under either mode
481                // a pool with no room for the walk failed every pass). A
482                // walk is idempotent: each `path_dict` entry keeps the
483                // least stamp.
484                match &slot.kind {
485                    SlotKind::Dependency { dep } => dep.with_room(walk)?,
486                    SlotKind::Entry { .. } | SlotKind::Ready(_) => walk()?,
487                }
488                slot.set_walked();
489                grew = load_keys_here() != before;
490            }
491            after = Some(rank);
492            drop(held);
493        }
494        if grew {
495            return Err(changed());
496        }
497        Ok(())
498    }
499
500    /// `ContainerHelper.parse_preload_table`
501    /// (`helpers/ContainerHelper.py:31-49`): every row's preload pointers
502    /// dereferenced, and each target's path id given the row's path in its
503    /// file's `path_dict` unless it has one (`setdefault`);
504    /// `FileNotFoundError` and `KeyError` skipped, any other failure raised.
505    ///
506    /// UnityPy's loop is quadratic (every row walks its whole span). Here a
507    /// table index is dereferenced at the first row whose span reaches it,
508    /// and again, from the walk's next search for an index on, once a load
509    /// has changed a key its external's lookup read since its last
510    /// dereference, the walk's own or another thread's ([`Pending`], and the
511    /// lists of [`Targets`]): UnityPy's dereference that gives the file the
512    /// last one gave sets nothing (`setdefault`), so each file a dereference
513    /// reaches gets the path of the first row that reaches it there, as in
514    /// UnityPy (review round 13: each index was dereferenced once, so the
515    /// file a load in between put under its external's key got no later
516    /// row's path). The walk takes what changed before each search, so a
517    /// load another thread makes gives what UnityPy's walk gives with each
518    /// key the load changes changed at the point of the walk at which it
519    /// was published, a point between two searches or dereferences (a load
520    /// publishes its keys one at a time; `super::watch`'s module docs;
521    /// review round 14: another thread's change put back no index, and set
522    /// a flag the walk's own later change of the key then found gone), but
523    /// for a key the load adds, which does not fail the pass (below).
524    /// Each external's answer is kept while it holds ([`Targets`]), so an
525    /// external is looked up once, and once more per change to a key its
526    /// lookup read, under every option set and whatever other keys other
527    /// threads change meanwhile (`EnvStats::preload_lookups` counts them);
528    /// each bundle's names indexed once per pass ([`Siblings`]), and each
529    /// row's path copied once, charged to this file's meter. So a walk no
530    /// load changes is linear in the rows, the table and the externals
531    /// (plan decision 11, counting no work), and what changes make it do
532    /// again (an index put back, an external looked up again) is counted as
533    /// work on this file's meter, whose allowance bounds a table whose
534    /// answers loads change over and over ([`crate::LimitKind::DecodeWork`]).
535    ///
536    /// A dereference of an external answered before answers as UnityPy's
537    /// would: `find_file` registers what it loads under the path it found,
538    /// whose basename is the name asked when the walk found that name, and a
539    /// longer one when it matched a file name by suffix (`ab.assets` finds
540    /// `xab.assets`, `environment.py:345-354`), so every dereference of that
541    /// external loads it again and the last copy keeps its key. The index
542    /// that loaded such a copy is dereferenced again only once a change to a
543    /// key its lookup read makes the name it asked findable; until then,
544    /// where rows' spans overlap, the external is loaded once per index, not
545    /// once per row and index (`PARITY.md` section 19.3; plan section 9,
546    /// Q11).
547    fn preload(
548        &self,
549        sf: &Arc<SerializedFile>,
550        container: &Container,
551        at: (u64, &Rank),
552        siblings: &mut Siblings,
553    ) -> Result<(), Error> {
554        if matches!(container.preload_table(), Ok(None)) {
555            return Ok(());
556        }
557        let malformed = |detail: &'static str| Error::Malformed {
558            what: "container",
559            detail: detail.into(),
560        };
561        let meter = sf.meter();
562        let mut pending: Option<Pending> = None;
563        let mut targets = Targets {
564            known: HashMap::new(),
565            answers: MeteredVec::new(meter),
566            next: None,
567            walk: self
568                .inner
569                .registry
570                .watches()
571                .start(meter, [&self.inner.registry, &self.inner.files]),
572            charge: Charge::zero(meter),
573        };
574        for (r, row) in container.rows().iter().enumerate() {
575            // `start = info.preloadIndex`, `size = info.preloadSize`: a
576            // member the AssetInfo lacks raises; one that is no `int` raises
577            // here too (PARITY.md section 19.3).
578            let (Some(start), Some(size)) = (row.preload_index, row.preload_size) else {
579                return Err(malformed("an AssetInfo index is not an int"));
580            };
581            if start < 0 || size <= 0 {
582                continue;
583            }
584            // `len(self._preload_table)`: a table that is not a list raises.
585            let table = container.preload_table()?.unwrap_or_default();
586            if start + size > i128::try_from(table.len()).unwrap_or(i128::MAX) {
587                continue;
588            }
589            let (Ok(s), Ok(e)) = (usize::try_from(start), usize::try_from(start + size)) else {
590                continue;
591            };
592            let pending = match &mut pending {
593                Some(u) => u,
594                None => pending.insert(Pending::full(table.len(), meter)?),
595            };
596            let mut path: Option<Option<RawStr>> = None;
597            let mut from = s;
598            loop {
599                #[cfg(test)]
600                self.inner.registry.hook(super::registry::Hook::Search {
601                    row: r,
602                    index: from,
603                });
604                // What a load changed since the last search, any thread's
605                // (the last dereference's own included, this index's
606                // answer too), is dereferenced again from here on in this
607                // row, or from the next row on: UnityPy's walk, which
608                // dereferences every element of every span, would find the
609                // change at each of those.
610                targets.reopen(pending, meter)?;
611                let Some(j) = pending.first_from(from).filter(|&j| j < e) else {
612                    break;
613                };
614                pending.remove(j);
615                from = j.saturating_add(1);
616                #[cfg(test)]
617                self.inner
618                    .registry
619                    .hook(super::registry::Hook::Step { row: r, index: j });
620                // An element that is no pointer raises (PARITY.md 19.3).
621                let Some(p) = table.get(j).copied().flatten() else {
622                    return Err(malformed("a preload table element is not a pointer"));
623                };
624                if !p.is_set() {
625                    continue;
626                }
627                self.inner.preload_derefs.fetch_add(1, Ordering::Relaxed);
628                let (target, answer) = self.preload_target(sf, p, &mut targets, siblings)?;
629                if let Some(id) = answer {
630                    targets.link(j, id, table.len(), meter)?;
631                }
632                if let Some(target) = target
633                    && !self.may_reopen(target.file())
634                {
635                    let tf = target.file();
636                    let path = match &path {
637                        Some(p) => p.clone(),
638                        None => path
639                            .insert(
640                                row.path
641                                    .as_ref()
642                                    .map(|p| RawStr::charged(p.as_bytes(), meter))
643                                    .transpose()?,
644                            )
645                            .clone(),
646                    };
647                    let stamp = (
648                        at.0,
649                        at.1.clone(),
650                        u32::try_from(r).unwrap_or(u32::MAX),
651                        u32::try_from(j).unwrap_or(u32::MAX),
652                    );
653                    tf.container()?
654                        .set_default(p.path_id, path.as_ref(), &stamp, tf.meter())?;
655                }
656            }
657        }
658        Ok(())
659    }
660
661    /// `pptr.deref()` for a preload element: the object, or `None` where the
662    /// preload skips (`FileNotFoundError`, `KeyError`); and the number of
663    /// the flag whose setting changes the answer (`None`: an answer for the
664    /// walk, the own file's, a sibling's or a file id past the end). An
665    /// external's answer is kept in `targets` while it holds (charged to its
666    /// charge); a lookup of an external the walk looked up before is
667    /// counted as work on `sf`'s meter first (`STEP_WORK` and the bytes of
668    /// the external's path, which the lookup cleans, simplifies and hashes).
669    fn preload_target(
670        &self,
671        sf: &Arc<SerializedFile>,
672        p: PPtr,
673        targets: &mut Targets,
674        siblings: &mut Siblings,
675    ) -> Result<(Option<ObjectRef>, Option<usize>), Error> {
676        let (found, answer) = if p.file_id == 0 {
677            (Some(Found::File(File::Serialized(sf.clone()))), None)
678        } else if let Some(k) = targets.kept(p.file_id) {
679            (k.found.clone(), k.flag.as_ref().map(|f| f.id()))
680        } else {
681            if targets.known.contains_key(&p.file_id) {
682                let bytes = Self::external_path(sf, p.file_id).map_or(0, <[u8]>::len);
683                sf.meter().count_work(
684                    STEP_WORK.saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX)),
685                )?;
686            }
687            self.inner.preload_lookups.fetch_add(1, Ordering::Relaxed);
688            // Watched before the lookup reads, so no change it misses is
689            // missed by the flag; its answer's list made first, so the flag's
690            // number is its place there.
691            let keys = Self::lookup_keys(sf, p.file_id, &targets.walk)?;
692            targets.answers.reserve(1)?;
693            let flag = targets.walk.watch(&keys)?;
694            let id = flag.id();
695            targets.answers.push(Answer { head: NONE })?;
696            #[cfg(test)]
697            self.inner.registry.hook(super::registry::Hook::Watched);
698            // What to record: kept until a change, kept for the walk, or
699            // not kept (the resolver's load).
700            let (found, kept, flag) = match self.external(sf, p.file_id, Some(siblings), None) {
701                Ok((t, Reached::Sibling)) => (Some(t), true, None),
702                Ok((t, Reached::Lookup)) => (Some(t), true, Some(flag)),
703                Ok((t, Reached::Load)) => (Some(t), false, Some(flag)),
704                Err(DerefError::FileIdPastEnd { .. }) => (None, true, None),
705                Err(DerefError::NotFound { .. }) => (None, true, Some(flag)),
706                Err(e) => return Err(deref_error(e)),
707            };
708            let answer = flag.is_some().then_some(id);
709            targets.keep(p.file_id, found.clone(), kept, flag)?;
710            (found, answer)
711        };
712        let Some(found) = found else {
713            return Ok((None, answer));
714        };
715        match Self::object_of(&found, p.path_id) {
716            Ok(o) => Ok((Some(o), answer)),
717            Err(DerefError::MissingObject { .. }) => Ok((None, answer)),
718            Err(e) => Err(deref_error(e)),
719        }
720    }
721
722    /// External `file_id`'s stored path, if `sf` has that external.
723    fn external_path(sf: &SerializedFile, file_id: i32) -> Option<&[u8]> {
724        let externals = sf.externals();
725        pptr::external_index(file_id, externals.len())
726            .ok()
727            .and_then(|i| externals.get(i))
728            .map(|e| e.path.as_bytes())
729    }
730
731    /// The keys external `file_id`'s lookup read past `sf`'s bundle names
732    /// ([`Self::external`]), each with the lock it is published under: the
733    /// registry key `find_file` asks (`simplify_name` twice), and for a
734    /// top-level file the lowered `files` key its sibling scan matches.
735    fn lookup_keys(
736        sf: &SerializedFile,
737        file_id: i32,
738        walk: &Walk,
739    ) -> Result<Vec<(usize, u64)>, Error> {
740        let externals = sf.externals();
741        let Some(external) = pptr::external_index(file_id, externals.len())
742            .ok()
743            .and_then(|i| externals.get(i))
744        else {
745            return Ok(Vec::new());
746        };
747        let meter = sf.meter();
748        let clean = clean_external_path(&external.path, meter)?;
749        let simple = simplify_name(&simplify_name(&clean, meter)?, meter)?;
750        let mut keys = vec![(REGISTRY, walk.hash(simple.as_bytes()))];
751        if sf.parent().is_none() {
752            keys.push((FILES, walk.hash(clean.as_bytes())));
753        }
754        Ok(keys)
755    }
756}
757
758#[cfg(test)]
759mod drop_tests;