unityrs_core/env/load.rs
1// Ported from UnityPy 1.25.2 (MIT): UnityPy/environment.py, UnityPy/files/BundleFile.py, UnityPy/files/File.py. See THIRD_PARTY_LICENSES.
2//! Loads: [`Environment::load`], [`Environment::load_dependency`] and the
3//! files the resolver returns, and the registration of their CABs.
4
5use std::sync::atomic::{AtomicUsize, Ordering};
6use std::sync::{Arc, Mutex};
7
8use super::listing::Listing;
9#[cfg(test)]
10use super::registry::Hook;
11use super::registry::{DepCache, Dependency, Open, Stage, open_source, with_room};
12use super::{Cab, Environment, LoadMode, RawCab, Source, source_name};
13use crate::budget::Reservation;
14use crate::counters::LoadId;
15use crate::error::Error;
16use crate::files::{BundleFile, File};
17use crate::limits::{Charge, Meter};
18use crate::raw::RawStr;
19
20/// Reopens a registered dependency without pinning its files in the cache.
21pub struct DependencyFile(Arc<Dependency>);
22
23const _: () = crate::send_sync::<DependencyFile>();
24
25impl DependencyFile {
26 /// Read through the registered open, retrying with room if the pool refuses.
27 pub fn read<T>(&self, mut read: impl FnMut(File) -> Result<T, Error>) -> Result<T, Error> {
28 self.0.with_room(|| {
29 let file = self.0.handle()?;
30 if self.0.unreachable()
31 && let Some(cache) = self.0.cache.upgrade()
32 {
33 cache.forget(&self.0);
34 }
35 read(file)
36 })
37 }
38
39 /// Whether a file belongs to this dependency, including across reopens.
40 #[must_use]
41 pub fn owns(&self, file: &crate::files::SerializedFile) -> bool {
42 self.0.meter().same_input(file.meter())
43 }
44}
45
46/// Whose CABs a registration makes: a caller's file (slots that may hold
47/// it), or a dependency's (slots that reopen it), registered one at a time
48/// or staged, every one charged before any is published (the registry's
49/// module docs, "A failed dependency load").
50pub(super) enum Target<'a> {
51 Caller,
52 Dependency(&'a Arc<Dependency>),
53 Staged(&'a Arc<Dependency>, &'a mut Stage),
54}
55
56/// The internal error of a dependency's registration under `Lazy` made one
57/// key at a time ([`Target::Dependency`]), which no load makes.
58fn unstaged() -> Error {
59 Error::Malformed {
60 what: "registry",
61 detail: "a dependency's lazy load registered unstaged".into(),
62 }
63}
64
65/// Who asked the resolver for the file [`Environment::load_resolved`]
66/// loads, which decides how it is loaded and keyed.
67#[derive(Clone, Copy, Debug)]
68pub(super) enum Asked<'a> {
69 /// `find_file(name, is_dependency)`: keyed by the source's name (UnityPy
70 /// keys the path it found, `load_file` keying a path by itself).
71 Find {
72 /// UnityPy's flag (a dereference's `find_file` takes its default,
73 /// `True`).
74 is_dependency: bool,
75 },
76 /// A resource read's fallback, `load_file(name, True)`: keyed by the
77 /// name it asked for.
78 Fallback(&'a RawStr),
79}
80
81impl Environment {
82 /// `load_file`: detect, parse, register the CABs, add to `files` under the
83 /// source's name (a repeated name replaces the value in place, as a dict).
84 // The source is handed over (the frozen signature); a caller's load
85 // keeps nothing of it but its name.
86 #[allow(clippy::needless_pass_by_value)]
87 pub fn load(&self, source: Source) -> Result<File, Error> {
88 let name = source_name(&source);
89 let load_id = LoadId::draw(&name);
90 let file = self.load_listed(&name, &source, false, load_id)?;
91 self.inner.loaded.fetch_add(1, Ordering::Relaxed);
92 Ok(file)
93 }
94
95 pub(super) fn load_data(
96 &self,
97 name: RawStr,
98 data: bytes::Bytes,
99 reader: crate::stream::ReaderKind,
100 meter: &Arc<Meter>,
101 ) -> Result<Option<File>, Error> {
102 let ty = crate::detect::check_file_type_with_reader(&data, &self.ctx().limits, reader)?;
103 if ty == crate::detect::FileType::Zip {
104 self.load_zip_data(data, meter)?;
105 self.list_zip_none(name, meter)?;
106 return Ok(None);
107 }
108 self.load_detected(name, data, reader, ty, meter, 0)
109 }
110
111 pub(super) fn load_detected(
112 &self,
113 name: RawStr,
114 data: bytes::Bytes,
115 reader: crate::stream::ReaderKind,
116 ty: crate::detect::FileType,
117 meter: &Arc<Meter>,
118 depth: u32,
119 ) -> Result<Option<File>, Error> {
120 let file = crate::files::parse_file_at_depth(
121 name.clone(),
122 data,
123 ty,
124 None,
125 meter,
126 self.ctx(),
127 reader,
128 depth,
129 )?;
130 let load = self.register_file(&name, &file, &mut Target::Caller)?;
131 self.list(name, file.clone(), false, load, meter)?;
132 self.inner.loaded.fetch_add(1, Ordering::Relaxed);
133 Ok(Some(file))
134 }
135
136 /// A load UnityPy's `load_file` makes (the caller's, or a resource
137 /// fallback's under `keep_resolver_loads`): its own input meter, its
138 /// CABs registered as a caller's, and a `files` entry under `name` at
139 /// the load's number, as its registrations are ranked, charged to the
140 /// file's meter (a resolver's name copied into its key first; a
141 /// caller's own name is shared, not copied). Its bundle's reads are
142 /// keyed by `load_id`, which the load drew before its first attempt
143 /// (CR8's test feature, `crate::testing`).
144 fn load_listed(
145 &self,
146 name: &RawStr,
147 source: &Source,
148 resolved: bool,
149 load_id: LoadId,
150 ) -> Result<File, Error> {
151 let ctx = self.ctx();
152 let meter = Meter::for_input(&ctx.limits, ctx.budget.clone(), Reservation::Input);
153 // UnityPy environment.py:139-160
154 let file = open_source(name, source, &meter, ctx, load_id)?;
155 let load = self.register_file(name, &file, &mut Target::Caller)?;
156 let key = if resolved {
157 RawStr::charged(name.as_bytes(), &meter)?
158 } else {
159 name.clone()
160 };
161 self.list(key, file.clone(), false, load, &meter)?;
162 Ok(file)
163 }
164
165 /// Registers the CABs of a top-level `file` loaded under `name`: the
166 /// file itself, or a bundle's entries (`Eager`: as UnityPy parses them;
167 /// `Lazy`: every directory entry's name now, parsed on first use), or
168 /// stages them ([`Target::Staged`]: room for a lazy bundle's directory
169 /// reserved first). Every registration is one load's, ranked by its
170 /// directory path ([`registry`]); returns the load's number, which its
171 /// `files` entry takes ([`Self::list`]).
172 ///
173 /// [`registry`]: super::registry
174 pub(super) fn register_file(
175 &self,
176 name: &RawStr,
177 file: &File,
178 target: &mut Target<'_>,
179 ) -> Result<u64, Error> {
180 // Never: a dependency's load under `Lazy` is staged, so that it
181 // publishes nothing when it fails (`try_load_dep`); one that came
182 // this way would publish one key at a time (CR1's review, round 2:
183 // only a lazy bundle's entries were checked).
184 if self.inner.options.load == LoadMode::Lazy && matches!(target, Target::Dependency(_)) {
185 return Err(unstaged());
186 }
187 let reg = &self.inner.registry;
188 let load = reg.next_load();
189 let registered = match file {
190 File::Serialized(sf) => self.register_cab(
191 name,
192 Cab::Serialized(sf.clone()),
193 file.meter(),
194 target,
195 load,
196 &[],
197 ),
198 File::Raw(raw) => self.register_cab(
199 name,
200 Cab::Raw(RawCab::File(raw.clone())),
201 file.meter(),
202 target,
203 load,
204 &[],
205 ),
206 File::Bundle(b) => match self.inner.options.load {
207 LoadMode::Eager => self.register_eager(b, target, load, &mut Vec::new()),
208 LoadMode::Lazy => {
209 if let Target::Staged(_, stage) = target {
210 stage.reserve(b.directory().len())?;
211 }
212 for (i, d) in b.directory().iter().enumerate() {
213 match target {
214 Target::Caller => {
215 reg.register_entry(&d.path, b.clone(), i, load, &[i], b.meter())?;
216 }
217 // Never: refused at the top.
218 Target::Dependency(_) => return Err(unstaged()),
219 Target::Staged(dep, stage) => {
220 reg.stage(stage, &d.path, dep, load, &[i], b.meter(), true)?;
221 }
222 }
223 }
224 Ok(())
225 }
226 },
227 File::Web(w) => self.register_web(w, target, load, &mut Vec::new()),
228 };
229 registered.map(|()| load)
230 }
231
232 fn register_cab(
233 &self,
234 name: &RawStr,
235 cab: Cab,
236 meter: &Arc<Meter>,
237 target: &mut Target<'_>,
238 load: u64,
239 path: &[usize],
240 ) -> Result<(), Error> {
241 let reg = &self.inner.registry;
242 match target {
243 Target::Caller => reg.register_ready(name, cab, load, path, meter),
244 Target::Dependency(dep) => reg.register_dependency(name, dep, load, path, meter),
245 Target::Staged(dep, stage) => reg.stage(stage, name, dep, load, path, meter, false),
246 }
247 }
248
249 /// UnityPy's eager bundle: every block decoded, then every entry parsed
250 /// in directory order and each `SerializedFile` and raw entry registered
251 /// as it is parsed (`files/BundleFile.py:168-186`, `files/File.py:76-90`),
252 /// a nested bundle's entries in place. A failing entry fails the bundle
253 /// with the CABs before it registered; a block that fails registers
254 /// nothing (plan section 3.13). [`BundleFile::load_all`] decodes and
255 /// parses first, so this walk only reads its cells: when it failed with
256 /// an entry's error, the blocks all decoded and the entries before the
257 /// failing one are registered; any other failure is the decode's.
258 fn register_eager(
259 &self,
260 b: &Arc<BundleFile>,
261 target: &mut Target<'_>,
262 load: u64,
263 path: &mut Vec<usize>,
264 ) -> Result<(), Error> {
265 let all = b.load_all();
266 if let Err(e) = &all
267 && !matches!(e, Error::Entry { .. })
268 {
269 return all;
270 }
271 for (i, d) in b.directory().iter().enumerate() {
272 path.push(i);
273 let step = b.entry(i).and_then(|f| {
274 self.inner.eager_parsed.fetch_add(1, Ordering::Relaxed);
275 match f {
276 File::Bundle(nested) => self.register_eager(&nested, target, load, path),
277 File::Serialized(sf) => self.register_cab(
278 &d.path,
279 Cab::Serialized(sf),
280 b.meter(),
281 target,
282 load,
283 path,
284 ),
285 File::Raw(_) => {
286 let cab = Cab::Raw(RawCab::InBundle {
287 bundle: b.clone(),
288 dir_index: i,
289 });
290 self.register_cab(&d.path, cab, b.meter(), target, load, path)
291 }
292 File::Web(w) => self.register_web(&w, target, load, path),
293 }
294 });
295 path.pop();
296 if let Err(e) = step {
297 return Err(all.err().unwrap_or(e));
298 }
299 }
300 all
301 }
302
303 // files/File.py:76-90: WebFile members register just like bundle members.
304 fn register_web(
305 &self,
306 w: &Arc<crate::files::WebFile>,
307 target: &mut Target<'_>,
308 load: u64,
309 path: &mut Vec<usize>,
310 ) -> Result<(), Error> {
311 for (i, d) in w.directory().iter().enumerate() {
312 path.push(i);
313 let result = w.entry(i).and_then(|f| match f {
314 File::Bundle(b) => self.register_eager(&b, target, load, path),
315 File::Web(w) => self.register_web(&w, target, load, path),
316 File::Serialized(sf) => {
317 self.register_cab(&d.path, Cab::Serialized(sf), w.meter(), target, load, path)
318 }
319 File::Raw(raw) => self.register_cab(
320 &d.path,
321 Cab::Raw(RawCab::File(raw)),
322 w.meter(),
323 target,
324 load,
325 path,
326 ),
327 });
328 path.pop();
329 result?;
330 }
331 Ok(())
332 }
333
334 /// Loads `source` as a dependency, as `find_file` loads one through the
335 /// resolver: its own meter under [`crate::budget::Reservation::Dependency`]
336 /// (so a refusal names the dependency pool), its CABs registered (lazily
337 /// under [`LoadMode::Lazy`]) without a strong handle, kept in the LRU and
338 /// evictable, and not added to [`Self::files`] (UnityPy has no such call;
339 /// a batch engine registers its inputs this way). The returned handle is
340 /// the caller's: an eviction drops only the LRU's, so while the caller
341 /// holds it the slots reach the same open file through their weak handle.
342 ///
343 /// Under [`LoadMode::Lazy`] a load that fails publishes nothing: every
344 /// registration is charged, with the LRU's entry, before the first is
345 /// published, so a refusal at any of them leaves the registry as it was
346 /// (the registry's module docs, "A failed dependency load"). Under
347 /// [`LoadMode::Eager`] the CABs are registered as they are parsed, as
348 /// UnityPy's `load_file` registers them, so a load that fails keeps
349 /// those registered before its failure (the partial registration of
350 /// the module docs).
351 // The source is handed over (the frozen signature) and kept by a clone.
352 pub fn load_dependency(&self, source: Source) -> Result<File, Error> {
353 self.load_dependency_with_handle(source)
354 .map(|(file, _)| file)
355 }
356
357 /// Load a dependency and give the caller an evictable handle to its exact
358 /// occurrence, even if another dependency registers the same CAB name.
359 /// Uses the same dependency pool and eviction retry as
360 /// [`Self::load_dependency`].
361 #[allow(clippy::needless_pass_by_value)]
362 pub fn load_dependency_with_handle(
363 &self,
364 source: Source,
365 ) -> Result<(File, DependencyFile), Error> {
366 let name = source_name(&source);
367 // Drawn once, so the retry keeps it (CR8's test feature).
368 let load_id = LoadId::draw(&name);
369 // A pool refusal evicts the others and retries once (plan 3.13); a
370 // second refusal fails naming the pool, and nothing of the
371 // dependency is kept (its first attempt's charges returned with its
372 // handle; under `Lazy` neither attempt published a key).
373 with_room(Some(&self.inner.cache), None, || {
374 self.try_load_dep(&name, &source, false, load_id)
375 .map(|(file, _, _, dep)| (file, DependencyFile(dep)))
376 })
377 }
378
379 /// One attempt of [`Self::load_dependency_with_handle`], on a meter of its own: the file,
380 /// the load's number and, with `listed` under `Lazy`, its `files`
381 /// entry under `name` charged and not yet published. Under `Lazy` its
382 /// registrations are staged, every one charged, and the LRU's entry and
383 /// the listing charged, before the first is published
384 /// ([`Self::register_staged`]); under `Eager` each is published as it
385 /// is registered, and a listing is the caller's to charge. Its
386 /// bundle's reads, and every reopen's, are keyed by `load_id` (CR8's
387 /// test feature).
388 fn try_load_dep(
389 &self,
390 name: &RawStr,
391 source: &Source,
392 listed: bool,
393 load_id: LoadId,
394 ) -> Result<(File, u64, Option<Listing>, Arc<Dependency>), Error> {
395 let ctx = self.ctx();
396 let meter = Meter::for_input(&ctx.limits, ctx.budget.clone(), Reservation::Dependency);
397 let file = open_source(name, source, &meter, ctx, load_id)?;
398 let dep = Arc::new(Dependency {
399 name: name.clone(),
400 source: source.clone(),
401 open: Mutex::new(Open::of(&file, meter)),
402 ctx: ctx.clone(),
403 cache: Arc::downgrade(&self.inner.cache),
404 slots: AtomicUsize::new(0),
405 #[cfg(feature = "test-counters")]
406 load_id,
407 });
408 let (load, room, listing) = match self.inner.options.load {
409 LoadMode::Lazy => {
410 let listed = listed.then_some(name);
411 let (load, room, listing) = self.register_staged(&dep, &file, listed)?;
412 (load, Some(room), listing)
413 }
414 LoadMode::Eager => {
415 let load = self.register_file(&dep.name, &file, &mut Target::Dependency(&dep))?;
416 (load, None, None)
417 }
418 };
419 #[cfg(test)]
420 self.inner.cache.hook(Hook::LoadRegistered);
421 self.inner.cache.loaded.fetch_add(1, Ordering::Relaxed);
422 match room {
423 Some(room) => self.inner.cache.touch_paid(&dep, &file, room),
424 None => self.inner.cache.touch(&dep, &file)?,
425 }
426 // Another load registered every key this one did meanwhile: no slot
427 // reaches it, and the LRU lets go of it ([`registry`]). The last
428 // slot to go before the touch told an LRU that did not hold it yet;
429 // one that goes after it tells the LRU itself (`DepSlot`).
430 if dep.unreachable() {
431 self.inner.cache.forget(&dep);
432 }
433 #[cfg(test)]
434 self.inner.cache.hook(Hook::LoadChecked);
435 Ok((file, load, listing, dep))
436 }
437
438 /// A dependency's registrations under `Lazy`, staged ([`Stage`]: the
439 /// list that holds them, room for a bundle's whole directory charged
440 /// first, then each key, slot, new key's share and copy, and position's
441 /// path, charged to the dependency's meter), then the LRU's entry
442 /// charged ([`DepCache::room`]) and, with `listed`, the `files` entry
443 /// under that name (its key's copy, its share and its lowered key,
444 /// [`Self::charge_listing`]), then every registration published in the
445 /// order it was staged ([`super::registry::Registry::commit`]). A
446 /// refusal before the commit drops the stage, which returns every
447 /// charge and was never in the registry, so no lookup, on this thread
448 /// or another, can have found any of it. Nothing after the commit's
449 /// start can fail the load: the `settle` each registration runs keeps a
450 /// refusal to itself and leaves the slot it could not reach listed for
451 /// a later call, as every `settle` does. The load's number, the LRU
452 /// entry's charge for [`DepCache::touch_paid`], and the listing.
453 fn register_staged(
454 &self,
455 dep: &Arc<Dependency>,
456 file: &File,
457 listed: Option<&RawStr>,
458 ) -> Result<(u64, Charge, Option<Listing>), Error> {
459 let mut stage = Stage::new(file.meter());
460 let load = self.register_file(&dep.name, file, &mut Target::Staged(dep, &mut stage))?;
461 let room = DepCache::room(file)?;
462 let listing = match listed {
463 Some(name) => {
464 let key = RawStr::charged(name.as_bytes(), file.meter())?;
465 Some(Self::charge_listing(key, file.meter())?)
466 }
467 None => None,
468 };
469 self.inner.registry.commit(stage)?;
470 Ok((load, room, listing))
471 }
472
473 /// A file the resolver returned (module docs, "Resolver loads in
474 /// `files`"). With `keep_resolver_loads`: `find_file`'s
475 /// ([`Asked::Find`]) with `is_dependency` is a dependency, listed in
476 /// `files` under the source's name (flagged a dependency unless it is a
477 /// raw reader, which UnityPy gives no flag); without `is_dependency` a caller's
478 /// load under that name (UnityPy's `load_file(fp, name=name,
479 /// is_dependency=False)`, `environment.py:332`, `:354`); a resource
480 /// fallback's ([`Asked::Fallback`]) is a caller's load under the name it
481 /// asked for. Without `keep_resolver_loads` every one is a dependency in
482 /// the LRU only (`PARITY.md` section 19.3's row "resolver loads under
483 /// `keep_resolver_loads` false", and section 19.2's
484 /// `SerializedFile.load_dependencies` row).
485 ///
486 /// Every one of these loads, whatever its listing, is one run of
487 /// [`with_room`]: a pool refusal anywhere in it (the source's fetcher's
488 /// own included, counted as it returns: `registry::from_callers`)
489 /// evicts the other dependencies and loads once more, as
490 /// [`Self::load_dep`] does (review round 12: a load listed as no
491 /// dependency, `find_file`'s with `is_dependency` false or a resource
492 /// fallback's under `keep_resolver_loads`, returned its fetcher's
493 /// refusal untried, unless the fallback's candidate step around it
494 /// retried it). A listing is charged to the loaded file's meter inside
495 /// that run, so a second run loads and lists again (review round 8: a
496 /// refused listing failed the load, whose file then stayed registered
497 /// and unlisted).
498 pub(super) fn load_resolved(&self, source: &Source, asked: Asked<'_>) -> Result<File, Error> {
499 let keep = self.inner.options.keep_resolver_loads;
500 let resolved;
501 let name = match asked {
502 Asked::Fallback(name) => name,
503 Asked::Find { .. } => {
504 resolved = source_name(source);
505 &resolved
506 }
507 };
508 // Drawn once, so the retry keeps it (CR8's test feature).
509 let load_id = LoadId::draw(name);
510 with_room(Some(&self.inner.cache), None, || match asked {
511 _ if !keep => self
512 .try_load_dep(name, source, false, load_id)
513 .map(|(file, ..)| file),
514 Asked::Fallback(_)
515 | Asked::Find {
516 is_dependency: false,
517 } => self.load_listed(name, source, true, load_id),
518 Asked::Find {
519 is_dependency: true,
520 } => {
521 // Under `Lazy` the listing is charged before the keys are
522 // published, so a refused one publishes none of them
523 // (`register_staged`); under `Eager`, after.
524 let (file, load, listing, _) = self.try_load_dep(name, source, true, load_id)?;
525 let listing = if let Some(listing) = listing {
526 listing
527 } else {
528 let meter = file.meter();
529 let key = RawStr::charged(name.as_bytes(), meter)?;
530 Self::charge_listing(key, meter)?
531 };
532 // A reader has no `is_dependency`: `parse_file` returns the
533 // reader itself (`helpers/ImportHelper.py:136-156`), which
534 // `getattr(f, "is_dependency", False)` reads false.
535 let flag = !matches!(file, File::Raw(_));
536 self.publish_listing(listing, file.clone(), flag, load);
537 Ok(file)
538 }
539 })
540 }
541}