unityrs-core 1.0.0

Read and write Unity asset bundles, serialized files and typetrees.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
//! The LRU lets go of a dependency no registry slot reaches, whatever the
//! order in which two threads drop its slots or load it (review round 2: a
//! dependency whose last two slots dropped at once stayed in the LRU for
//! the Environment's life under `EnvOptions::unitypy()`, which never
//! evicts). Each interleaving is forced with a hook ([`Hook`]), never by
//! repetition; a stress run follows as a second check.

// The source scan behind "the core starts no thread and never waits"
// (`unityrs_testkit::waits`) takes only an item under `#[cfg(test)]` in its
// own file for test code (this file's `#[cfg(test)] mod tests;` is in
// another), so every item here that names a thread, a channel or a barrier
// carries one.
#[cfg(test)]
use std::sync::{Arc, Barrier, Mutex, mpsc};
#[cfg(test)]
use std::thread::ThreadId;

use super::{By, Cab, Hook, HookFn, Loads, Rank, RawCab, Registry, SlotKind};
use crate::detect::FileType;
use crate::env::heap::{self, Noted};
use crate::env::watch::REGISTRY;
use crate::env::{EnvOptions, Environment, Source, held};
use crate::files::RawFile;
use crate::limits::Meter;
use crate::raw::RawStr;
use bytes::Bytes;
use unityrs_testkit::env::{Obj, bundle, res_bytes, serialized};

/// A dependency bundle that registers two keys, `cab-x` and `cab-x.ress`.
fn two_keys() -> Source {
    Source::Bytes {
        name: RawStr::from("d.bundle"),
        data: Bytes::from(bundle(&[
            ("CAB-X", serialized(&[], &[Obj::named(1, "x1", &[])])),
            ("CAB-X.resS", res_bytes(64, 3)),
        ])),
    }
}

/// Stops the thread that arms it when it reaches `at`, until the test lets
/// it go; every other thread passes.
#[cfg(test)]
struct Pause {
    at: Hook,
    armed: Mutex<Option<ThreadId>>,
    stopped: Mutex<Option<mpsc::Sender<()>>>,
    go: Mutex<Option<mpsc::Receiver<()>>>,
}

/// The test's side of a [`Pause`].
#[cfg(test)]
struct Held {
    pause: Arc<Pause>,
    stopped: mpsc::Receiver<()>,
    go: mpsc::Sender<()>,
}

#[cfg(test)]
impl Held {
    fn install(env: &Environment, at: Hook) -> Self {
        let (stopped_tx, stopped) = mpsc::channel();
        let (go, go_rx) = mpsc::channel();
        let pause = Arc::new(Pause {
            at,
            armed: Mutex::new(None),
            stopped: Mutex::new(Some(stopped_tx)),
            go: Mutex::new(Some(go_rx)),
        });
        let p = pause.clone();
        *env.inner.cache.hook.lock().unwrap() = Some(HookFn(Arc::new(move |h| p.reach(h))));
        Self { pause, stopped, go }
    }

    fn wait_stopped(&self) {
        unityrs_testkit::guard::wait(&self.stopped, "the armed thread to reach its hook");
    }

    fn release(&self) {
        self.go.send(()).unwrap();
    }
}

#[cfg(test)]
impl Pause {
    /// Arms the calling thread.
    fn arm(&self) {
        *self.armed.lock().unwrap() = Some(std::thread::current().id());
    }

    fn reach(&self, h: Hook) {
        if h != self.at {
            return;
        }
        let me = std::thread::current().id();
        let stop = {
            let mut armed = self.armed.lock().unwrap();
            let stop = *armed == Some(me);
            if stop {
                *armed = None;
            }
            stop
        };
        if !stop {
            return;
        }
        if let Some(tx) = self.stopped.lock().unwrap().take() {
            let _ = tx.send(());
        }
        let go = self.go.lock().unwrap().take();
        if let Some(rx) = go {
            unityrs_testkit::guard::hold(&rx, "the test to let the held thread go");
        }
    }
}

/// The meter of the dependency the registry's keys name now: all the LRU
/// should hold.
fn current_dep_bytes(env: &Environment) -> u64 {
    let held = env
        .get_cab_held(&RawStr::from("cab-x"), &Meter::unlimited())
        .unwrap()
        .unwrap();
    held.hold.unwrap().meter().used()
}

/// Two threads each hold one of a replaced dependency's two slots and drop
/// them at once: the first is stopped after it counts its slot out and
/// before it acts, the second drops its slot meanwhile. Exactly one of them
/// is the last, so the LRU lets go (the old test of the `Arc` count saw 3
/// on both threads, and neither did).
#[cfg(test)]
#[test]
fn two_slots_dropped_at_once_leave_the_lru_once() {
    let env = Environment::new(EnvOptions::unitypy());
    let m = Meter::unlimited();
    env.load_dependency(two_keys()).unwrap();
    let a = env
        .registry()
        .slot(&RawStr::from("cab-x"), &m)
        .unwrap()
        .unwrap();
    let b = env
        .registry()
        .slot(&RawStr::from("cab-x.ress"), &m)
        .unwrap()
        .unwrap();
    // A `find_file` reload: both keys now name a new dependency, and only
    // `a` and `b` reach the old one.
    env.load_dependency(two_keys()).unwrap();
    assert_eq!(env.inner.cache.len(), 2);
    let held = Held::install(&env, Hook::SlotReleased);
    let first = {
        let held_pause = held.pause.clone();
        std::thread::spawn(move || {
            held_pause.arm();
            drop(a);
        })
    };
    held.wait_stopped();
    drop(b);
    held.release();
    first.join().unwrap();
    assert_eq!(env.inner.cache.len(), 1, "the replaced dependency left");
    assert_eq!(env.stats().dependency_bytes, current_dep_bytes(&env));
}

/// A dependency load stopped after it touched the LRU and found its slots
/// alive, before it returns, while another load replaces every key it
/// registered: the last of its slots to go makes the LRU let go (the old
/// test of the `Arc` count counted the stopped load's own handle, so
/// neither slot did, nor the load).
#[cfg(test)]
#[test]
fn a_load_whose_keys_are_replaced_before_it_returns_leaves_the_lru() {
    let env = Environment::new(EnvOptions::unitypy());
    let held = Held::install(&env, Hook::LoadChecked);
    let first = {
        let env = env.clone();
        let held_pause = held.pause.clone();
        std::thread::spawn(move || {
            held_pause.arm();
            env.load_dependency(two_keys()).unwrap();
        })
    };
    held.wait_stopped();
    env.load_dependency(two_keys()).unwrap();
    held.release();
    first.join().unwrap();
    assert_eq!(env.inner.cache.len(), 1);
    assert_eq!(env.stats().dependency_bytes, current_dep_bytes(&env));
}

/// A dependency load stopped after it registered and before it touches the
/// LRU, while another load replaces every key: its slots went before the
/// LRU held it, so the load itself makes the LRU let go after its touch.
#[cfg(test)]
#[test]
fn a_load_whose_keys_are_replaced_before_its_touch_leaves_the_lru() {
    let env = Environment::new(EnvOptions::unitypy());
    let held = Held::install(&env, Hook::LoadRegistered);
    let first = {
        let env = env.clone();
        let held_pause = held.pause.clone();
        std::thread::spawn(move || {
            held_pause.arm();
            env.load_dependency(two_keys()).unwrap();
        })
    };
    held.wait_stopped();
    env.load_dependency(two_keys()).unwrap();
    held.release();
    first.join().unwrap();
    assert_eq!(env.inner.cache.len(), 1);
    assert_eq!(env.stats().dependency_bytes, current_dep_bytes(&env));
}

/// The same under `LoadMode::Lazy`, where the load charged its LRU entry
/// before its commit and is stopped after the commit, before it hands
/// that charge to the LRU (`DepCache::touch_paid`): its slots went before
/// the LRU held it, so the load makes the LRU let go after the touch, and
/// the entry's charge goes with it (CR1's review, round 1).
#[cfg(test)]
#[test]
fn a_lazy_load_whose_keys_are_replaced_before_its_paid_touch_leaves_the_lru() {
    let mut options = EnvOptions::unitypy();
    options.load = crate::env::LoadMode::Lazy;
    let env = Environment::new(options);
    let held = Held::install(&env, Hook::LoadRegistered);
    let first = {
        let env = env.clone();
        let held_pause = held.pause.clone();
        std::thread::spawn(move || {
            held_pause.arm();
            env.load_dependency(two_keys()).unwrap();
        })
    };
    held.wait_stopped();
    env.load_dependency(two_keys()).unwrap();
    held.release();
    first.join().unwrap();
    assert_eq!(env.inner.cache.len(), 1);
    // The current dependency's meter read after the lookup parses its CAB.
    let current = current_dep_bytes(&env);
    assert_eq!(env.stats().dependency_bytes, current);
}

/// A staged registration is 240 bytes on a 64-bit target, the figure the
/// docs give for a staged row (`docs/plans/core.md` 3.13,
/// `docs/plans/batch.md` 5.2, 5.5 and A4): its key (32), position (24),
/// slot (8), a new key's share (88) and its path's charge (88). CR1's
/// review, round 2: the figure was measured once and pinned by nothing.
#[cfg(target_pointer_width = "64")]
#[test]
fn a_staged_registration_is_240_bytes() {
    assert_eq!(std::mem::size_of::<super::register::Staged>(), 240);
}

/// The stress run: many rounds of two slots of a replaced dependency
/// dropped by two threads released together. Passes on any schedule; the
/// old code left about one round in a thousand in the LRU.
#[cfg(test)]
#[test]
fn slots_dropped_together_never_leave_a_dependency_behind() {
    const ROUNDS: usize = 2_000;
    let env = Environment::new(EnvOptions::unitypy());
    let m = Meter::unlimited();
    env.load_dependency(two_keys()).unwrap();
    for round in 0..ROUNDS {
        let a = env
            .registry()
            .slot(&RawStr::from("cab-x"), &m)
            .unwrap()
            .unwrap();
        let b = env
            .registry()
            .slot(&RawStr::from("cab-x.ress"), &m)
            .unwrap()
            .unwrap();
        env.load_dependency(two_keys()).unwrap();
        let start = Arc::new(Barrier::new(2));
        let other = {
            let start = start.clone();
            std::thread::spawn(move || {
                start.wait();
                drop(b);
            })
        };
        start.wait();
        drop(a);
        other.join().unwrap();
        assert_eq!(env.inner.cache.len(), 1, "round {round}");
    }
}

/// How many dropped slots wait to be reached, on either list.
fn listed(env: &Environment) -> usize {
    let reg = &env.inner.registry;
    reg.pending_len.load(std::sync::atomic::Ordering::Acquire)
        + reg.evicted_len.load(std::sync::atomic::Ordering::Acquire)
}

/// What the key `foo` holds: its value's slot and its position.
fn foo(env: &Environment) -> Option<(usize, Rank)> {
    let inner = env.inner.registry.read();
    let entry = inner.slots.get(b"foo".as_slice())?;
    Some((Arc::as_ptr(&entry.slot) as usize, entry.first.clone()))
}

/// A registration's change to a key, the setting of the walks' flags under
/// it and a load's new container index generation are one step, under the
/// registry's lock (W1 and P1, plan section 3.16); so are a load's listing
/// of the lazy slot it dropped and a new generation, whether or not it
/// changes the key (review round 16: a load between the key's position and
/// its value listed its slot and started none). Checked as each lock the
/// registering thread takes is released ([`held::after_each_release`]), so
/// whichever part of the step leaves the lock, ahead of the change or
/// behind it, is seen at the release in between, where another thread could
/// see it (review round 15: the generation was bumped after the lock, and
/// the notify was pinned only at a hook after it). By a load and by a reach,
/// which starts no generation (`Registry::insert`), of slots read at once
/// and of lazy ones, for each thing a registration can do to a key:
/// `before` are the loads registered first, `at` the one that runs,
/// `changes` whether it changes the key and `drops` whether it drops a
/// slot of another rank (which a lazy one lists).
#[test]
fn at_each_release_a_registrations_key_flags_and_generation_agree() {
    let cases = [
        ("a new key", &[][..], 5, true, false),
        ("a later load replaces the value", &[5][..], 9, true, true),
        ("an earlier load moves the key", &[5][..], 3, true, true),
        ("an equal rank is no change", &[5][..], 5, false, false),
        (
            "a load between its position and its value",
            &[3, 9][..],
            5,
            false,
            true,
        ),
    ];
    for (by, lazy) in [
        (By::Load, false),
        (By::Reach, false),
        (By::Load, true),
        (By::Reach, true),
    ] {
        for (what, before, at, changes, drops) in cases {
            let what = format!("{what}, lazy {lazy}");
            let env = Environment::new(EnvOptions::unitypy());
            env.load(Source::Bytes {
                name: RawStr::from("seed"),
                data: Bytes::from(serialized(&[], &[Obj::named(1, "s1", &[])])),
            })
            .unwrap();
            let meter = Meter::unlimited();
            let name = RawStr::from("foo");
            let cab: Cab = env.get_cab(&RawStr::from("seed"), &meter).unwrap().unwrap();
            let register = |load: u64, by: By| {
                let kind = SlotKind::Ready(cab.clone());
                let reg = &env.inner.registry;
                reg.insert(&name, kind, Loads::at(load), &[], &meter, by, lazy)
                    .unwrap();
            };
            for &load in before {
                register(load, By::Load);
            }
            let reg = &env.inner.registry;
            let mut walk = reg.watches().start(&meter, [reg, &env.inner.files]);
            let hash = walk.hash(b"foo");
            let flag = walk.watch(&[(REGISTRY, hash)]).unwrap();
            let (key_before, generation_before) = (foo(&env), reg.generation());
            assert_eq!(listed(&env), 0, "{what}: a slot is listed before");
            let seen = std::rc::Rc::new(std::cell::Cell::new((0_u32, 0_u32, 0_u32)));
            // Whether the slot it drops was listed by now (a load's `settle`
            // takes it off the list again).
            let was_listed = std::rc::Rc::new(std::cell::Cell::new(false));
            held::after_each_release(Some(std::rc::Rc::new({
                let (env, flag, key_before, seen, was_listed) = (
                    env.clone(),
                    flag.clone(),
                    key_before.clone(),
                    seen.clone(),
                    was_listed.clone(),
                );
                let by_load = by == By::Load;
                let what = what.clone();
                move || {
                    let changed = foo(&env) != key_before;
                    let generation = env.inner.registry.generation();
                    was_listed.set(was_listed.get() || listed(&env) != 0);
                    let listed = was_listed.get();
                    assert_eq!(
                        flag.changed(),
                        changed,
                        "{what}: at a release the key's change and the walks' flag differ (W1)"
                    );
                    assert_eq!(
                        generation != generation_before,
                        (changed || listed) && by_load,
                        "{what}: at a release the key's change or the slot's listing and the \
                         generation differ (P1)"
                    );
                    let (releases, with_change, with_listing) = seen.get();
                    seen.set((
                        releases + 1,
                        with_change + u32::from(changed),
                        with_listing + u32::from(listed),
                    ));
                }
            })));
            register(at, by);
            held::after_each_release(None);
            let (releases, with_change, with_listing) = seen.get();
            assert_ne!(releases, 0, "{what}: no lock was released");
            assert_eq!(with_change != 0, changes, "{what}");
            assert_eq!(with_listing != 0, drops && lazy, "{what}: the listing");
        }
    }
}

/// A key's entry, taken out of the registry and dropped alone, with the
/// charge for its position's path on a meter of its own: the charge is
/// released after the path is freed. The entry's fields drop in order, its
/// slot, its position, its share, then that charge (fix pass 17's sweep: no
/// test failed with the charge declared before the position). The heap is
/// counted on the test's thread ([`crate::env::heap`]).
#[test]
fn a_keys_entry_releases_its_paths_charge_after_the_path() {
    heap::count();
    let reg = Registry::default();
    let meter = Meter::unlimited();
    let raw = RawFile::new(
        RawStr::from("CAB-K.resS"),
        Bytes::from_static(b"raw"),
        FileType::ResourceFile,
        &meter,
    )
    .unwrap();
    reg.register(&RawStr::from("CAB-K"), Cab::Raw(RawCab::File(raw)), &meter)
        .unwrap();
    let mut entry = {
        let mut inner = reg.write();
        let entry = inner.slots.remove(b"cab-k".as_slice()).expect("the key");
        // The order's copy of the position shares its path.
        inner.order.remove(&entry.first);
        entry
    };
    let (noted_meter, noted) = Noted::meter();
    let paid = noted_meter
        .charge(Rank::heap(entry.first.path().len()))
        .unwrap();
    drop(std::mem::replace(&mut entry.first_charge, paid));
    let from = noted.releases();
    let before = heap::live();
    drop(entry);
    let after = heap::live();
    assert!(before > after, "the drop freed nothing");
    heap::released_last(&noted, from, after, "the path's charge");
}