unityrs_core/env/container_index.rs
1// Ported from UnityPy 1.25.2 (MIT): UnityPy/environment.py, UnityPy/files/ObjectReader.py, UnityPy/helpers/ContainerHelper.py. See THIRD_PARTY_LICENSES.
2//! The container: [`Environment::container`], [`Environment::container_path`]
3//! and the pass that builds their index (UnityPy's
4//! `_build_container_index`), the walk of each file's preload table.
5
6use std::collections::HashMap;
7use std::fmt;
8use std::mem::size_of;
9use std::sync::Arc;
10use std::sync::atomic::Ordering;
11
12use super::deref::Reached;
13use super::registry::{Rank, SlotKind, load_keys_here};
14use super::revisit::Pending;
15use super::watch::{FILES, Flag, REGISTRY, Walk};
16use super::{Cab, DerefError, Environment, Found, LoadMode, PPtr, never_evicts, pptr};
17use crate::budget::Reservation;
18use crate::container::{Container, ContainerPath, ContainerRow};
19use crate::error::{Error, Unsupported};
20use crate::files::{BundleFile, File, SerializedFile};
21use crate::limits::{Charge, GrantScope, HASH_TABLE_SHARES, Meter, MeteredVec};
22use crate::object::ObjectRef;
23use crate::once::CellError;
24use crate::pystr::{clean_external_path, py_lower, simplify_name};
25use crate::raw::RawStr;
26use crate::walk::STEP_WORK;
27
28/// The iterator [`Environment::container`] returns, CABs in registry order.
29pub struct ContainerItems<'a> {
30 env: &'a Environment,
31 after: Option<Rank>,
32 current: Option<(Arc<SerializedFile>, usize)>,
33}
34
35const _: () = crate::send_sync::<ContainerItems<'static>>();
36
37impl Iterator for ContainerItems<'_> {
38 type Item = Result<ContainerItem, Error>;
39
40 fn next(&mut self) -> Option<Self::Item> {
41 loop {
42 if let Some((sf, k)) = self.current.take() {
43 match sf.container() {
44 Ok(c) => {
45 if let Some(row) = c.rows().get(k) {
46 let row = row.clone();
47 self.current = Some((sf.clone(), k + 1));
48 return Some(Ok(ContainerItem { row, file: sf }));
49 }
50 }
51 Err(e) => return Some(Err(e)),
52 }
53 }
54 // UnityPy environment.py:221-224: every non-dependency CAB.
55 let (rank, slot) = self.env.registry().slot_after(self.after.as_ref())?;
56 self.after = Some(rank);
57 if slot.is_dependency() {
58 continue;
59 }
60 match self.env.registry().resolve(&slot) {
61 Ok(Some(Cab::Serialized(sf))) => self.current = Some((sf, 0)),
62 Ok(_) => {}
63 Err(e) => return Some(Err(e)),
64 }
65 }
66 }
67}
68
69impl fmt::Debug for ContainerItems<'_> {
70 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
71 f.debug_struct("ContainerItems").finish_non_exhaustive()
72 }
73}
74
75/// A container row with the object it points at, as `env.container` yields.
76#[derive(Clone, Debug)]
77pub struct ContainerItem {
78 /// The row.
79 pub row: ContainerRow,
80 /// The file whose container it is.
81 pub file: Arc<SerializedFile>,
82}
83
84const _: () = crate::send_sync::<ContainerItem>();
85
86/// A container pass's index of each bundle's entry names, lower-cased, to
87/// the entry the first of them names (rule 4's scan of a bundle's `files`
88/// as one lookup): built once for each bundle the pass meets, charged to the
89/// bundle's meter, and dropped with the pass.
90#[derive(Default)]
91pub(super) struct Siblings {
92 bundles: HashMap<usize, SiblingNames>,
93}
94
95struct SiblingNames {
96 /// Keeps the bundle, and so the address that keys it, for the pass.
97 _bundle: Arc<BundleFile>,
98 names: HashMap<RawStr, usize>,
99 _charge: Charge,
100}
101
102impl Siblings {
103 /// The directory index of `bundle`'s first `files` entry whose name
104 /// lowers to `clean`.
105 pub(super) fn find(
106 &mut self,
107 bundle: &Arc<BundleFile>,
108 clean: &RawStr,
109 ) -> Result<Option<usize>, Error> {
110 let key = Arc::as_ptr(bundle).addr();
111 let entry = match self.bundles.entry(key) {
112 std::collections::hash_map::Entry::Occupied(o) => o.into_mut(),
113 std::collections::hash_map::Entry::Vacant(v) => {
114 let meter = bundle.meter();
115 let slots = bundle.files();
116 let n = u64::try_from(slots.len()).unwrap_or(u64::MAX);
117 let charge = meter.charge(
118 HASH_TABLE_SHARES
119 .saturating_mul((size_of::<RawStr>() + size_of::<usize>() + 1) as u64)
120 .saturating_mul(n.saturating_add(1)),
121 )?;
122 let mut names = HashMap::with_capacity(slots.len());
123 for s in slots {
124 names
125 .entry(py_lower(&s.name, meter)?)
126 .or_insert(s.dir_index);
127 }
128 v.insert(SiblingNames {
129 _bundle: bundle.clone(),
130 names,
131 _charge: charge,
132 })
133 }
134 };
135 Ok(entry.names.get(clean.as_bytes()).copied())
136 }
137}
138
139/// What one external costs a preload walk's map of answers:
140/// [`HASH_TABLE_SHARES`] buckets.
141const TARGET_SHARE: u64 = HASH_TABLE_SHARES * (size_of::<i32>() + size_of::<Known>() + 1) as u64;
142
143/// No table index: the end of a list in [`Targets::next`].
144const NONE: usize = usize::MAX;
145
146/// A preload walk's answers for its file's externals, each kept only while
147/// UnityPy's next dereference would give the same one: a bundle sibling, or
148/// a file id past the externals' end, for the walk; an answer of `files` or
149/// the registry (found or not) until a change to a key its lookup read
150/// ([`super::watch`]: a load's registration, any thread's, the resolver's
151/// included, and under [`LoadMode::Lazy`] the entries a reach registers),
152/// so the answer UnityPy's next lookup could find changed is asked again
153/// and every other is kept; and the resolver's load not at all (the next
154/// dereference looks again, and finds it if the load registered the name
155/// asked for, or loads again if not, as UnityPy's `find_file` does when its
156/// walk matched a longer file name by suffix). An external is looked up
157/// once, and once more per change to one of its keys (one the resolver
158/// loads under another name, once per dereference), so a walk costs its
159/// table and its externals' names (plan decision 11), plus those lookups
160/// again, each lookup past an external's first counted as work on the
161/// walking file's meter (`STEP_WORK` and the external's path's bytes;
162/// review round 4: every change, any thread's, expired every answer, so a
163/// table of pointers to long names cost the table times the names).
164///
165/// Each answer with a flag (one of `files`, the registry or the resolver's
166/// load) lists the table indexes dereferenced under it (`answers`, linked
167/// through `next`), so when a load sets its flag, the walk's own or another
168/// thread's ([`Walk::fired`]), those indexes are dereferenced again at the
169/// next row whose span reaches them, or later in the row the walk is in
170/// (review round 13: a walk that dereferenced each index once never gave a
171/// later row's path to the file a load in between put under the key the
172/// index's external resolves through; review round 14: only a load of the
173/// walk's own put them back, and not after another thread's had set the
174/// flag first).
175///
176/// [`LoadMode::Lazy`]: super::LoadMode::Lazy
177///
178/// Each charge is released after the memory it pays for (plan section 3.15):
179/// fields drop in order, so `known` and `answers` go first, then the walk's
180/// watches with their flags and their own charge, then `known`'s charge
181/// (review round 5: the watches' charge was released before their entries
182/// left the map).
183struct Targets<'a> {
184 known: HashMap<i32, Known>,
185 /// Every flag the walk set up, by its number, with the head of the list
186 /// of table indexes dereferenced under its answer.
187 answers: MeteredVec<Answer>,
188 /// Each table index's next in its answer's list, made at the first
189 /// link: an index is in one list at most, the one of the answer it last
190 /// got, and leaves it when the list is taken.
191 next: Option<MeteredVec<usize>>,
192 /// After `known`, whose flags drop before the walk's watches end.
193 walk: Walk<'a>,
194 /// `known`'s table, after it.
195 charge: Charge,
196}
197
198/// One external's answer in [`Targets`].
199struct Known {
200 found: Option<Found>,
201 /// Whether the walk keeps it: not the resolver's load, which the next
202 /// dereference asks again.
203 kept: bool,
204 /// Set when a key its lookup read changes; `None`: kept for the walk (a
205 /// sibling, a file id past the end).
206 flag: Option<Arc<Flag>>,
207}
208
209/// The table indexes dereferenced under one flag's answer: the head of a
210/// list linked through [`Targets::next`] (the walk holds the flag itself,
211/// [`Walk`]).
212struct Answer {
213 head: usize,
214}
215
216/// The work a preload walk counts on its file's meter for each index it
217/// puts back to be dereferenced again ([`Targets::reopen`]): a step for the
218/// dereference, and one per level of the [`Pending`] set for putting the
219/// index back (the find and the removal that dereference makes read no
220/// more words than a first dereference's).
221fn reopen_work(pending: &Pending) -> u64 {
222 STEP_WORK.saturating_mul(1 + u64::try_from(pending.levels()).unwrap_or(u64::MAX))
223}
224
225impl Targets<'_> {
226 /// The answer kept for `file_id`, if it holds.
227 fn kept(&self, file_id: i32) -> Option<&Known> {
228 self.known
229 .get(&file_id)
230 .filter(|k| k.kept && k.flag.as_ref().is_none_or(|f| !f.changed()))
231 }
232
233 /// Records `found` for `file_id`, kept (`kept`) while `flag` is unset
234 /// (`None`: for the walk).
235 fn keep(
236 &mut self,
237 file_id: i32,
238 found: Option<Found>,
239 kept: bool,
240 flag: Option<Arc<Flag>>,
241 ) -> Result<(), Error> {
242 if !self.known.contains_key(&file_id) {
243 self.charge.grow(TARGET_SHARE)?;
244 }
245 self.known.insert(file_id, Known { found, kept, flag });
246 Ok(())
247 }
248
249 /// Lists table index `i` (of a table of `len`) under the answer of flag
250 /// `id`.
251 fn link(&mut self, i: usize, id: usize, len: usize, meter: &Arc<Meter>) -> Result<(), Error> {
252 if self.next.is_none() {
253 let mut n = MeteredVec::with_capacity(meter, len)?;
254 for _ in 0..len {
255 n.push(NONE)?;
256 }
257 self.next = Some(n);
258 }
259 if let (Some(answer), Some(slot)) = (
260 self.answers.as_mut_slice().get_mut(id),
261 self.next.as_mut().and_then(|n| n.as_mut_slice().get_mut(i)),
262 ) {
263 *slot = answer.head;
264 answer.head = i;
265 }
266 Ok(())
267 }
268
269 /// Puts back into `pending` every table index listed under an answer
270 /// whose flag a load set since the last call, any thread's, each
271 /// counted on `meter` first ([`reopen_work`]); the lists are taken, so
272 /// an index is put back once per dereference.
273 fn reopen(&mut self, pending: &mut Pending, meter: &Meter) -> Result<(), Error> {
274 let Self {
275 walk,
276 answers,
277 next,
278 ..
279 } = self;
280 let links = next.as_ref().map(MeteredVec::as_slice);
281 for id in walk.fired() {
282 let Some(answer) = answers.as_mut_slice().get_mut(id) else {
283 continue;
284 };
285 let mut i = std::mem::replace(&mut answer.head, NONE);
286 while i != NONE {
287 meter.count_work(reopen_work(pending))?;
288 pending.insert(i);
289 i = links.and_then(|l| l.get(i)).copied().unwrap_or(NONE);
290 }
291 }
292 Ok(())
293 }
294}
295
296/// A dereference failure as a file-level error (the container preload's
297/// raise): a load failure as it is, any other as [`Error::Deref`].
298fn deref_error(e: DerefError) -> Error {
299 match e {
300 DerefError::Load(e) => e,
301 other => Error::Deref(Arc::new(other)),
302 }
303}
304
305impl Environment {
306 /// `container`: every non-dependency CAB's container rows, CABs in
307 /// registry order, after the preload index is built (`environment.py:208-226`),
308 /// as an iterator. It walks the registry, as UnityPy walks `env.cabs`, so
309 /// a CAB a dependency's replaced under its key is not listed.
310 pub fn container(&self) -> Result<ContainerItems<'_>, Error> {
311 self.build_container_index()?;
312 Ok(ContainerItems {
313 env: self,
314 after: None,
315 current: None,
316 })
317 }
318
319 /// `obj.container` (`files/ObjectReader.py:199-204`): the container index
320 /// built first ([`Self::container`]'s pass, whose failure is this call's),
321 /// then the path the object's file's `path_dict` holds for its path id
322 /// ([`crate::container`]): a row of the file's own container whose asset
323 /// is the object, the last such row winning, else the first row whose
324 /// preload span reached it. The paths live in the file the object was
325 /// read from, as UnityPy's live in its `assets_file`, so a second handle
326 /// a caller opened on the same bytes holds only its own rows'.
327 ///
328 /// Where a dependency may be evicted (every option but
329 /// [`EnvOptions::unitypy`]'s, [`EnvOptions::dep_cache_bytes`]), an
330 /// object of a dependency's file is refused with
331 /// [`Unsupported::EvictableContainerPath`]: an eviction drops the paths
332 /// the preload set in that file and its reopen does not walk again, so
333 /// the answer would depend on when the LRU evicted (`PARITY.md` section
334 /// 19.3). The preload sets no path in such a file.
335 ///
336 /// [`EnvOptions::unitypy`]: super::EnvOptions::unitypy
337 /// [`EnvOptions::dep_cache_bytes`]: super::EnvOptions::dep_cache_bytes
338 pub fn container_path(&self, obj: &ObjectRef) -> Result<Option<ContainerPath>, Error> {
339 let sf = obj.file();
340 if self.may_reopen(sf) {
341 return Err(Error::Unsupported(Unsupported::EvictableContainerPath));
342 }
343 self.build_container_index()?;
344 sf.container()?.path_of(obj.path_id(), sf.meter())
345 }
346
347 /// Whether `sf` is a dependency's file the LRU may evict and the
348 /// Environment open again.
349 fn may_reopen(&self, sf: &SerializedFile) -> bool {
350 !never_evicts(&self.inner.options) && sf.meter().reservation() == Reservation::Dependency
351 }
352
353 /// `_build_container_index` (`environment.py:208-215`): once until the
354 /// next load's registration, every registered `SerializedFile`'s preload
355 /// table, dependencies' included, in registry order. UnityPy sets its
356 /// flag before the loop and each registration clears it, so a pass that
357 /// fails leaves the index built unless the pass itself registered
358 /// something; here a pass marks built the registry generation it started
359 /// at when it ends or fails with an error the input decides, and any
360 /// load's registration since starts a new generation. A refusal (a
361 /// budget, meter or I/O failure, [`crate::once::CellError::is_transient`])
362 /// is never kept (plan 4.1): the pass is not marked built, and the next
363 /// ask walks again (review round 5: a pool refused once left the preload
364 /// unwalked for the generation). What a walk the file's own meter refused
365 /// was granted counts as that open's work, so a refusal the next ask
366 /// would repeat stops at [`crate::LimitKind::DecodeWork`], which is kept
367 /// ([`Self::walk_counted`]). A slot's table is marked walked only when its
368 /// walk ends (UnityPy clears `_preload_table` then). A pass whose own
369 /// dereferences make a new key fails as UnityPy's loop over
370 /// `cabs.values()` does (`RuntimeError: dictionary changed size during
371 /// iteration`), at the step after; a key a nested container's entries
372 /// make under [`LoadMode::Lazy`] is not one (an eager load made it at
373 /// load), nor is one another thread's load makes. Under `Lazy` the pass
374 /// first reaches every registered entry and every dropped one, an
375 /// evicted dependency's opened again (the registry's module docs; plan
376 /// section 9, Q13), so it walks what an eager load registered; a slot
377 /// dropped after a pass started is dropped by a later load, whose
378 /// registration starts a new generation in the step that lists the
379 /// slot whether or not it changes a key (review round 16: one ranked
380 /// between the key's position and its value changed none, started
381 /// none, and the entries its slot's reach registered were never
382 /// walked), or by the reach of such a load's slot, so a pass finished
383 /// at the current generation leaves none unreached but for a kept
384 /// refusal's. Two passes may run at once: each walks every table not yet marked, and each `path_dict`
385 /// entry keeps the least stamp ([`crate::container`]), so both leave
386 /// what one after the other would.
387 ///
388 /// [`LoadMode::Lazy`]: super::LoadMode::Lazy
389 fn build_container_index(&self) -> Result<(), Error> {
390 let reg = &self.inner.registry;
391 // Every slot dropped before this call is reached first, as by every
392 // other reader (plan section 3.16, R3); a refusal is the walk's to
393 // meet, or the next call's.
394 reg.settle().ok();
395 let generation = reg.generation();
396 if reg.built(generation) {
397 return Ok(());
398 }
399 let result = self.container_pass(generation);
400 if result.as_ref().err().is_none_or(|e| !e.is_transient()) {
401 reg.publish_built(generation);
402 }
403 result
404 }
405
406 /// [`Self::preload`] of `sf`'s table, as a lazy computation of `sf`'s
407 /// open (`crate::once`'s rule on work): when `sf`'s own meter refuses
408 /// it, what the meter granted it on this thread counts as the open's
409 /// work, and once that work has passed the allowance the refusal is
410 /// [`crate::LimitKind::DecodeWork`], which the input decides, so the pass
411 /// keeps it.
412 fn walk_counted(
413 &self,
414 sf: &Arc<SerializedFile>,
415 container: &Container,
416 at: (u64, &Rank),
417 siblings: &mut Siblings,
418 ) -> Result<(), Error> {
419 let meter = sf.meter();
420 let scope = GrantScope::enter(meter);
421 let walked = self.preload(sf, container, at, siblings);
422 let own = scope.finish(meter);
423 match walked {
424 Err(e) if e.is_meter_refusal() => {
425 meter.add_work(own);
426 Err(meter.work_refused().map_or(e, Error::Limit))
427 }
428 other => other,
429 }
430 }
431
432 fn container_pass(&self, generation: u64) -> Result<(), Error> {
433 let reg = &self.inner.registry;
434 let changed = || Error::Malformed {
435 what: "container",
436 detail: "dictionary changed size during iteration".into(),
437 };
438 // Under `Lazy`, every registered entry is reached before the walk
439 // (the registry's module docs), so a nested container's entries are
440 // walked at the positions and with the values an eager load gives
441 // them, whichever container the walk would meet first (review round
442 // 4: a later container's entry replaced the value of a key the walk
443 // had passed, and the walk had read the earlier one).
444 if self.inner.options.load == LoadMode::Lazy {
445 let mut after: Option<Rank> = None;
446 while let Some((rank, slot)) = reg.slot_after(after.as_ref()) {
447 reg.reach(&slot, None)?;
448 after = Some(rank);
449 }
450 // Every slot a registration dropped, those these reaches dropped
451 // included, and those of an evicted dependency, opened again
452 // (plan section 9, Q13); a refusal there is the pass's.
453 reg.settle_for_pass()?;
454 }
455 let mut siblings = Siblings::default();
456 let mut after: Option<Rank> = None;
457 let mut grew = false;
458 while let Some((rank, slot)) = reg.slot_after(after.as_ref()) {
459 if grew {
460 return Err(changed());
461 }
462 // The dependency's file is held while its table is walked, so
463 // an eviction the walk makes cannot drop the CAB's bundle.
464 let Some(held) = reg.resolve_held(&slot)? else {
465 after = Some(rank);
466 continue;
467 };
468 let Cab::Serialized(sf) = &held.cab else {
469 after = Some(rank);
470 continue;
471 };
472 if !slot.walked() {
473 let before = load_keys_here();
474 let mut walk = || {
475 let container = sf.container()?;
476 self.walk_counted(sf, container, (generation, &rank), &mut siblings)
477 };
478 // A dependency's table is walked on its meter: a walk the
479 // pool refuses evicts the others and walks again
480 // (`registry::with_room`; review round 7: under either mode
481 // a pool with no room for the walk failed every pass). A
482 // walk is idempotent: each `path_dict` entry keeps the
483 // least stamp.
484 match &slot.kind {
485 SlotKind::Dependency { dep } => dep.with_room(walk)?,
486 SlotKind::Entry { .. } | SlotKind::Ready(_) => walk()?,
487 }
488 slot.set_walked();
489 grew = load_keys_here() != before;
490 }
491 after = Some(rank);
492 drop(held);
493 }
494 if grew {
495 return Err(changed());
496 }
497 Ok(())
498 }
499
500 /// `ContainerHelper.parse_preload_table`
501 /// (`helpers/ContainerHelper.py:31-49`): every row's preload pointers
502 /// dereferenced, and each target's path id given the row's path in its
503 /// file's `path_dict` unless it has one (`setdefault`);
504 /// `FileNotFoundError` and `KeyError` skipped, any other failure raised.
505 ///
506 /// UnityPy's loop is quadratic (every row walks its whole span). Here a
507 /// table index is dereferenced at the first row whose span reaches it,
508 /// and again, from the walk's next search for an index on, once a load
509 /// has changed a key its external's lookup read since its last
510 /// dereference, the walk's own or another thread's ([`Pending`], and the
511 /// lists of [`Targets`]): UnityPy's dereference that gives the file the
512 /// last one gave sets nothing (`setdefault`), so each file a dereference
513 /// reaches gets the path of the first row that reaches it there, as in
514 /// UnityPy (review round 13: each index was dereferenced once, so the
515 /// file a load in between put under its external's key got no later
516 /// row's path). The walk takes what changed before each search, so a
517 /// load another thread makes gives what UnityPy's walk gives with each
518 /// key the load changes changed at the point of the walk at which it
519 /// was published, a point between two searches or dereferences (a load
520 /// publishes its keys one at a time; `super::watch`'s module docs;
521 /// review round 14: another thread's change put back no index, and set
522 /// a flag the walk's own later change of the key then found gone), but
523 /// for a key the load adds, which does not fail the pass (below).
524 /// Each external's answer is kept while it holds ([`Targets`]), so an
525 /// external is looked up once, and once more per change to a key its
526 /// lookup read, under every option set and whatever other keys other
527 /// threads change meanwhile (`EnvStats::preload_lookups` counts them);
528 /// each bundle's names indexed once per pass ([`Siblings`]), and each
529 /// row's path copied once, charged to this file's meter. So a walk no
530 /// load changes is linear in the rows, the table and the externals
531 /// (plan decision 11, counting no work), and what changes make it do
532 /// again (an index put back, an external looked up again) is counted as
533 /// work on this file's meter, whose allowance bounds a table whose
534 /// answers loads change over and over ([`crate::LimitKind::DecodeWork`]).
535 ///
536 /// A dereference of an external answered before answers as UnityPy's
537 /// would: `find_file` registers what it loads under the path it found,
538 /// whose basename is the name asked when the walk found that name, and a
539 /// longer one when it matched a file name by suffix (`ab.assets` finds
540 /// `xab.assets`, `environment.py:345-354`), so every dereference of that
541 /// external loads it again and the last copy keeps its key. The index
542 /// that loaded such a copy is dereferenced again only once a change to a
543 /// key its lookup read makes the name it asked findable; until then,
544 /// where rows' spans overlap, the external is loaded once per index, not
545 /// once per row and index (`PARITY.md` section 19.3; plan section 9,
546 /// Q11).
547 fn preload(
548 &self,
549 sf: &Arc<SerializedFile>,
550 container: &Container,
551 at: (u64, &Rank),
552 siblings: &mut Siblings,
553 ) -> Result<(), Error> {
554 if matches!(container.preload_table(), Ok(None)) {
555 return Ok(());
556 }
557 let malformed = |detail: &'static str| Error::Malformed {
558 what: "container",
559 detail: detail.into(),
560 };
561 let meter = sf.meter();
562 let mut pending: Option<Pending> = None;
563 let mut targets = Targets {
564 known: HashMap::new(),
565 answers: MeteredVec::new(meter),
566 next: None,
567 walk: self
568 .inner
569 .registry
570 .watches()
571 .start(meter, [&self.inner.registry, &self.inner.files]),
572 charge: Charge::zero(meter),
573 };
574 for (r, row) in container.rows().iter().enumerate() {
575 // `start = info.preloadIndex`, `size = info.preloadSize`: a
576 // member the AssetInfo lacks raises; one that is no `int` raises
577 // here too (PARITY.md section 19.3).
578 let (Some(start), Some(size)) = (row.preload_index, row.preload_size) else {
579 return Err(malformed("an AssetInfo index is not an int"));
580 };
581 if start < 0 || size <= 0 {
582 continue;
583 }
584 // `len(self._preload_table)`: a table that is not a list raises.
585 let table = container.preload_table()?.unwrap_or_default();
586 if start + size > i128::try_from(table.len()).unwrap_or(i128::MAX) {
587 continue;
588 }
589 let (Ok(s), Ok(e)) = (usize::try_from(start), usize::try_from(start + size)) else {
590 continue;
591 };
592 let pending = match &mut pending {
593 Some(u) => u,
594 None => pending.insert(Pending::full(table.len(), meter)?),
595 };
596 let mut path: Option<Option<RawStr>> = None;
597 let mut from = s;
598 loop {
599 #[cfg(test)]
600 self.inner.registry.hook(super::registry::Hook::Search {
601 row: r,
602 index: from,
603 });
604 // What a load changed since the last search, any thread's
605 // (the last dereference's own included, this index's
606 // answer too), is dereferenced again from here on in this
607 // row, or from the next row on: UnityPy's walk, which
608 // dereferences every element of every span, would find the
609 // change at each of those.
610 targets.reopen(pending, meter)?;
611 let Some(j) = pending.first_from(from).filter(|&j| j < e) else {
612 break;
613 };
614 pending.remove(j);
615 from = j.saturating_add(1);
616 #[cfg(test)]
617 self.inner
618 .registry
619 .hook(super::registry::Hook::Step { row: r, index: j });
620 // An element that is no pointer raises (PARITY.md 19.3).
621 let Some(p) = table.get(j).copied().flatten() else {
622 return Err(malformed("a preload table element is not a pointer"));
623 };
624 if !p.is_set() {
625 continue;
626 }
627 self.inner.preload_derefs.fetch_add(1, Ordering::Relaxed);
628 let (target, answer) = self.preload_target(sf, p, &mut targets, siblings)?;
629 if let Some(id) = answer {
630 targets.link(j, id, table.len(), meter)?;
631 }
632 if let Some(target) = target
633 && !self.may_reopen(target.file())
634 {
635 let tf = target.file();
636 let path = match &path {
637 Some(p) => p.clone(),
638 None => path
639 .insert(
640 row.path
641 .as_ref()
642 .map(|p| RawStr::charged(p.as_bytes(), meter))
643 .transpose()?,
644 )
645 .clone(),
646 };
647 let stamp = (
648 at.0,
649 at.1.clone(),
650 u32::try_from(r).unwrap_or(u32::MAX),
651 u32::try_from(j).unwrap_or(u32::MAX),
652 );
653 tf.container()?
654 .set_default(p.path_id, path.as_ref(), &stamp, tf.meter())?;
655 }
656 }
657 }
658 Ok(())
659 }
660
661 /// `pptr.deref()` for a preload element: the object, or `None` where the
662 /// preload skips (`FileNotFoundError`, `KeyError`); and the number of
663 /// the flag whose setting changes the answer (`None`: an answer for the
664 /// walk, the own file's, a sibling's or a file id past the end). An
665 /// external's answer is kept in `targets` while it holds (charged to its
666 /// charge); a lookup of an external the walk looked up before is
667 /// counted as work on `sf`'s meter first (`STEP_WORK` and the bytes of
668 /// the external's path, which the lookup cleans, simplifies and hashes).
669 fn preload_target(
670 &self,
671 sf: &Arc<SerializedFile>,
672 p: PPtr,
673 targets: &mut Targets,
674 siblings: &mut Siblings,
675 ) -> Result<(Option<ObjectRef>, Option<usize>), Error> {
676 let (found, answer) = if p.file_id == 0 {
677 (Some(Found::File(File::Serialized(sf.clone()))), None)
678 } else if let Some(k) = targets.kept(p.file_id) {
679 (k.found.clone(), k.flag.as_ref().map(|f| f.id()))
680 } else {
681 if targets.known.contains_key(&p.file_id) {
682 let bytes = Self::external_path(sf, p.file_id).map_or(0, <[u8]>::len);
683 sf.meter().count_work(
684 STEP_WORK.saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX)),
685 )?;
686 }
687 self.inner.preload_lookups.fetch_add(1, Ordering::Relaxed);
688 // Watched before the lookup reads, so no change it misses is
689 // missed by the flag; its answer's list made first, so the flag's
690 // number is its place there.
691 let keys = Self::lookup_keys(sf, p.file_id, &targets.walk)?;
692 targets.answers.reserve(1)?;
693 let flag = targets.walk.watch(&keys)?;
694 let id = flag.id();
695 targets.answers.push(Answer { head: NONE })?;
696 #[cfg(test)]
697 self.inner.registry.hook(super::registry::Hook::Watched);
698 // What to record: kept until a change, kept for the walk, or
699 // not kept (the resolver's load).
700 let (found, kept, flag) = match self.external(sf, p.file_id, Some(siblings), None) {
701 Ok((t, Reached::Sibling)) => (Some(t), true, None),
702 Ok((t, Reached::Lookup)) => (Some(t), true, Some(flag)),
703 Ok((t, Reached::Load)) => (Some(t), false, Some(flag)),
704 Err(DerefError::FileIdPastEnd { .. }) => (None, true, None),
705 Err(DerefError::NotFound { .. }) => (None, true, Some(flag)),
706 Err(e) => return Err(deref_error(e)),
707 };
708 let answer = flag.is_some().then_some(id);
709 targets.keep(p.file_id, found.clone(), kept, flag)?;
710 (found, answer)
711 };
712 let Some(found) = found else {
713 return Ok((None, answer));
714 };
715 match Self::object_of(&found, p.path_id) {
716 Ok(o) => Ok((Some(o), answer)),
717 Err(DerefError::MissingObject { .. }) => Ok((None, answer)),
718 Err(e) => Err(deref_error(e)),
719 }
720 }
721
722 /// External `file_id`'s stored path, if `sf` has that external.
723 fn external_path(sf: &SerializedFile, file_id: i32) -> Option<&[u8]> {
724 let externals = sf.externals();
725 pptr::external_index(file_id, externals.len())
726 .ok()
727 .and_then(|i| externals.get(i))
728 .map(|e| e.path.as_bytes())
729 }
730
731 /// The keys external `file_id`'s lookup read past `sf`'s bundle names
732 /// ([`Self::external`]), each with the lock it is published under: the
733 /// registry key `find_file` asks (`simplify_name` twice), and for a
734 /// top-level file the lowered `files` key its sibling scan matches.
735 fn lookup_keys(
736 sf: &SerializedFile,
737 file_id: i32,
738 walk: &Walk,
739 ) -> Result<Vec<(usize, u64)>, Error> {
740 let externals = sf.externals();
741 let Some(external) = pptr::external_index(file_id, externals.len())
742 .ok()
743 .and_then(|i| externals.get(i))
744 else {
745 return Ok(Vec::new());
746 };
747 let meter = sf.meter();
748 let clean = clean_external_path(&external.path, meter)?;
749 let simple = simplify_name(&simplify_name(&clean, meter)?, meter)?;
750 let mut keys = vec![(REGISTRY, walk.hash(simple.as_bytes()))];
751 if sf.parent().is_none() {
752 keys.push((FILES, walk.hash(clean.as_bytes())));
753 }
754 Ok(keys)
755 }
756}
757
758#[cfg(test)]
759mod drop_tests;