1use std::collections::HashMap;
54use std::mem::size_of;
55use std::sync::{Arc, Mutex, PoisonError};
56
57use crate::env::PPtr;
58use crate::env::registry::Rank;
59use crate::error::Error;
60use crate::files::SerializedFile;
61use crate::limits::{Charge, ChargeError, HASH_TABLE_SHARES, Meter, MeteredVec};
62use crate::once::{ResultCell, get_or_publish_result};
63use crate::raw::RawStr;
64use crate::typetree::{NodeId, NodeList};
65use crate::value::{Map, Parsed, Value};
66
67#[derive(Clone, Debug, PartialEq, Eq)]
72pub struct ContainerRow {
73 pub path: Option<RawStr>,
75 pub preload_index: Option<i128>,
77 pub preload_size: Option<i128>,
79 pub file_id: Option<i128>,
81 pub path_id: Option<i128>,
83}
84
85impl ContainerRow {
86 #[must_use]
88 pub fn asset(&self) -> Option<PPtr> {
89 Some(PPtr {
90 file_id: i32::try_from(self.file_id?).ok()?,
91 path_id: i64::try_from(self.path_id?).ok()?,
92 })
93 }
94}
95
96#[derive(Debug, PartialEq, Eq)]
98enum Preload {
99 None,
101 List(Box<[Option<PPtr>]>),
104 NotAList,
106}
107
108#[derive(Clone, Debug, PartialEq, Eq)]
111pub enum ContainerPath {
112 Str(RawStr),
114 NotAString,
116}
117
118const _: () = crate::send_sync::<ContainerPath>();
119
120impl ContainerPath {
121 fn of(path: Option<&RawStr>) -> Self {
122 path.map_or(Self::NotAString, |p| Self::Str(p.clone()))
123 }
124}
125
126pub(crate) type Stamp = (u64, Rank, u32, u32);
130
131#[derive(Debug)]
133struct Preloaded {
134 stamp: Stamp,
135 path: Option<RawStr>,
136 _charge: Charge,
137}
138
139const PRELOADED_SHARE: u64 =
141 HASH_TABLE_SHARES * (size_of::<i64>() + size_of::<Preloaded>() + 1) as u64;
142
143#[derive(Debug)]
145struct PathIndex {
146 rows: HashMap<i64, u32>,
147 _charge: Charge,
148}
149
150#[derive(Debug)]
152pub struct Container {
153 rows: Box<[ContainerRow]>,
154 keys: Box<[Option<i64>]>,
157 preload: Preload,
158 charge: Charge,
159 index: ResultCell<PathIndex, Error>,
161 preloaded: Mutex<HashMap<i64, Preloaded>>,
163}
164
165impl PartialEq for Container {
166 fn eq(&self, other: &Self) -> bool {
167 self.rows == other.rows && self.preload == other.preload
168 }
169}
170
171impl Eq for Container {}
172
173impl Container {
174 #[must_use]
176 pub fn empty(meter: &Arc<Meter>) -> Self {
177 Self::parts(
178 Box::new([]),
179 Box::new([]),
180 Preload::None,
181 Charge::zero(meter),
182 )
183 }
184
185 fn parts(
186 rows: Box<[ContainerRow]>,
187 keys: Box<[Option<i64>]>,
188 preload: Preload,
189 charge: Charge,
190 ) -> Self {
191 Self {
192 rows,
193 keys,
194 preload,
195 charge,
196 index: ResultCell::new(),
197 preloaded: Mutex::new(HashMap::new()),
198 }
199 }
200
201 fn int_keys(rows: &MeteredVec<ContainerRow>) -> Result<MeteredVec<Option<i64>>, ChargeError> {
204 let mut keys = MeteredVec::with_capacity(rows.charge().meter(), rows.len())?;
205 for r in rows.as_slice() {
206 let own = r.file_id == Some(0);
207 keys.push(
208 r.path_id
209 .filter(|_| own)
210 .and_then(|p| i64::try_from(p).ok()),
211 )?;
212 }
213 Ok(keys)
214 }
215
216 pub fn new(
220 rows: MeteredVec<ContainerRow>,
221 preload_table: Option<MeteredVec<Option<PPtr>>>,
222 ) -> Result<Self, ChargeError> {
223 let keys = Self::int_keys(&rows)?;
224 Self::keyed(
225 rows,
226 keys,
227 preload_table.map_or(Ok((Preload::None, None)), list),
228 )
229 }
230
231 pub fn with_preload_not_a_list(rows: MeteredVec<ContainerRow>) -> Result<Self, ChargeError> {
233 let keys = Self::int_keys(&rows)?;
234 Self::keyed(rows, keys, Ok((Preload::NotAList, None)))
235 }
236
237 fn keyed(
238 rows: MeteredVec<ContainerRow>,
239 keys: MeteredVec<Option<i64>>,
240 preload: Result<(Preload, Option<Charge>), ChargeError>,
241 ) -> Result<Self, ChargeError> {
242 let (rows, mut charge) = rows.into_boxed_slice()?;
243 let (keys, c) = keys.into_boxed_slice()?;
244 charge.absorb(c);
245 let (preload, c) = preload?;
246 if let Some(c) = c {
247 charge.absorb(c);
248 }
249 Ok(Self::parts(rows, keys, preload, charge))
250 }
251
252 #[must_use]
254 pub fn rows(&self) -> &[ContainerRow] {
255 &self.rows
256 }
257
258 pub fn preload_table(&self) -> Result<Option<&[Option<PPtr>]>, Error> {
262 match &self.preload {
263 Preload::None => Ok(None),
264 Preload::List(t) => Ok(Some(t)),
265 Preload::NotAList => Err(malformed("m_PreloadTable is not a list")),
266 }
267 }
268
269 #[must_use]
271 pub fn charge(&self) -> &Charge {
272 &self.charge
273 }
274
275 #[must_use]
277 pub fn is_empty(&self) -> bool {
278 self.rows.is_empty()
279 }
280
281 pub(crate) fn path_of(
285 &self,
286 path_id: i64,
287 meter: &Arc<Meter>,
288 ) -> Result<Option<ContainerPath>, Error> {
289 let index = get_or_publish_result(&self.index, meter, || self.build_index(meter))?;
290 if let Some(&row) = index.rows.get(&path_id) {
291 let row = usize::try_from(row).ok().and_then(|r| self.rows.get(r));
292 return Ok(row.map(|r| ContainerPath::of(r.path.as_ref())));
293 }
294 let set = self
295 .preloaded
296 .lock()
297 .unwrap_or_else(PoisonError::into_inner);
298 Ok(set
299 .get(&path_id)
300 .map(|p| ContainerPath::of(p.path.as_ref())))
301 }
302
303 fn build_index(&self, meter: &Arc<Meter>) -> Result<PathIndex, Error> {
306 let n = self.keys.iter().flatten().count();
307 let charge = meter.charge(
308 HASH_TABLE_SHARES
309 .saturating_mul((size_of::<i64>() + size_of::<u32>() + 1) as u64)
310 .saturating_mul(u64::try_from(n).unwrap_or(u64::MAX)),
311 )?;
312 let mut rows = HashMap::with_capacity(n);
313 for (i, key) in self.keys.iter().enumerate() {
314 if let (Some(key), Ok(i)) = (key, u32::try_from(i)) {
315 rows.insert(*key, i);
316 }
317 }
318 Ok(PathIndex {
319 rows,
320 _charge: charge,
321 })
322 }
323
324 pub(crate) fn set_default(
329 &self,
330 path_id: i64,
331 path: Option<&RawStr>,
332 stamp: &Stamp,
333 meter: &Arc<Meter>,
334 ) -> Result<(), Error> {
335 let index = get_or_publish_result(&self.index, meter, || self.build_index(meter))?;
336 if index.rows.contains_key(&path_id) {
337 return Ok(());
338 }
339 let lose =
340 |set: &HashMap<i64, Preloaded>| set.get(&path_id).is_some_and(|p| p.stamp <= *stamp);
341 if lose(
342 &self
343 .preloaded
344 .lock()
345 .unwrap_or_else(PoisonError::into_inner),
346 ) {
347 return Ok(());
348 }
349 let charge = meter.charge(PRELOADED_SHARE)?;
350 let entry = Preloaded {
351 stamp: stamp.clone(),
352 path: path.cloned(),
353 _charge: charge,
354 };
355 let replaced = {
356 let mut set = self
357 .preloaded
358 .lock()
359 .unwrap_or_else(PoisonError::into_inner);
360 if lose(&set) {
361 Some(entry)
362 } else {
363 set.insert(path_id, entry)
364 }
365 };
366 drop(replaced);
368 Ok(())
369 }
370}
371
372fn list(t: MeteredVec<Option<PPtr>>) -> Result<(Preload, Option<Charge>), ChargeError> {
374 let (t, c) = t.into_boxed_slice()?;
375 Ok((Preload::List(t), Some(c)))
376}
377
378pub fn read_container(file: &Arc<SerializedFile>) -> Result<Container, Error> {
383 let meter = file.meter();
386 let Some(obj) = file.objects().find(|o| o.class_id() == ASSET_BUNDLE) else {
387 return Ok(Container::empty(meter));
388 };
389 let parsed = obj
390 .read_typetree(None, true)
391 .map_err(|e| Error::AssetBundle(Arc::new(e)))?;
392 let Some(view) = View::new(&parsed) else {
393 return Err(malformed("no node list"));
394 };
395 let root = view.root();
399 let preload = view.attr(root, b"m_PreloadTable")?;
400 let items = view.attr(root, b"m_Container")?;
401 let mut rows = MeteredVec::new(meter);
402 let mut keys = MeteredVec::new(meter);
403 for item in view.sequence(items)?.nodes() {
407 let Some([key, info]) = view.unpack2(item) else {
408 return Err(malformed("an m_Container item is not a pair"));
409 };
410 let asset = view.attr(info, b"asset")?;
411 let (file_id, path_id) = match view.pptr_object(asset) {
412 Some((file_id, path_id)) => (Some(file_id), path_id),
413 None => (
415 view.attr(asset, b"m_FileID").ok(),
416 view.attr(asset, b"path_id")?,
417 ),
418 };
419 if !view.hashable(key) || !view.hashable(path_id) {
420 return Err(malformed("an m_Container key or path_id is unhashable"));
421 }
422 let own = file_id.and_then(|f| py_int(f.value)) == Some(0);
425 keys.push(if own { py_int(path_id.value) } else { None })?;
426 let path_id = match view.pptr_object(asset) {
428 Some(_) => Some(path_id),
429 None => view.attr(asset, b"m_PathID").ok(),
430 };
431 let int = |n: Option<Node<'_>>| n.and_then(|n| int(n.value));
432 rows.push(ContainerRow {
433 path: match key.value {
434 Value::Str(s) => Some(s.clone()),
435 _ => None,
436 },
437 preload_index: int(view.attr(info, b"preloadIndex").ok()),
438 preload_size: int(view.attr(info, b"preloadSize").ok()),
439 file_id: int(file_id),
440 path_id: int(path_id),
441 })?;
442 }
443 let mut table = MeteredVec::new(meter);
445 match preload.value {
446 Value::List(_) => {
447 for p in view.sequence(preload)?.nodes() {
448 table.push(view.pointer(p))?;
449 }
450 }
451 Value::Prims(p) => {
453 for _ in 0..p.len() {
454 table.push(None)?;
455 }
456 }
457 _ => return Ok(Container::keyed(rows, keys, Ok((Preload::NotAList, None)))?),
458 }
459 Ok(Container::keyed(rows, keys, list(table))?)
460}
461
462fn py_int(v: &Value) -> Option<i64> {
466 let float = |f: f64| {
467 #[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)]
469 (f.fract() == 0.0
470 && (-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0).contains(&f))
471 .then_some(f as i64)
472 };
473 match v {
474 Value::Int(i) => Some(*i),
475 Value::UInt(u) => i64::try_from(*u).ok(),
476 Value::Bool(b) => Some(i64::from(*b)),
477 Value::F32(bits) => float(f64::from(f32::from_bits(*bits))),
478 Value::F64(bits) => float(f64::from_bits(*bits)),
479 _ => None,
480 }
481}
482
483fn int(v: &Value) -> Option<i128> {
485 match v {
486 Value::Int(i) => Some(i128::from(*i)),
487 Value::UInt(u) => Some(i128::from(*u)),
488 _ => None,
489 }
490}
491
492const ASSET_BUNDLE: i32 = 142;
494
495fn malformed(detail: &str) -> Error {
496 Error::Malformed {
497 what: "AssetBundle",
498 detail: detail.into(),
499 }
500}
501
502#[derive(Clone, Copy)]
504pub(crate) struct Node<'a> {
505 id: NodeId,
506 value: &'a Value,
507}
508
509impl<'a> Node<'a> {
510 fn new(id: NodeId, value: &'a Value) -> Self {
511 Self { id, value }
512 }
513
514 pub(crate) fn value(&self) -> &'a Value {
516 self.value
517 }
518}
519
520pub(crate) struct View<'a> {
527 parsed: &'a Parsed,
528 list: &'a NodeList,
529}
530
531impl<'a> View<'a> {
532 pub(crate) fn new(parsed: &'a Parsed) -> Option<Self> {
534 let list = parsed.lists().first()?;
535 Some(Self { parsed, list })
536 }
537
538 pub(crate) fn root(&self) -> Node<'a> {
540 Node::new(self.list.root(), self.parsed.value())
541 }
542
543 pub(crate) fn attr(&self, node: Node<'a>, name: &[u8]) -> Result<Node<'a>, Error> {
547 let Value::Dict(map) = node.value else {
548 return Err(malformed("an attribute of a value that is not a class"));
549 };
550 let child = self
551 .list
552 .children(node.id)
553 .iter()
554 .rev()
555 .find(|c| clean_name_is(self.list.name(**c), name))
556 .copied();
557 let found = child.and_then(|c| self.field(map, self.list.name(c)).map(|v| (c, v)));
558 let Some((id, value)) = found else {
559 return Err(Error::Malformed {
560 what: "AssetBundle",
561 detail: format!("no attribute {}", name.escape_ascii()).into(),
562 });
563 };
564 Ok(Node { id, value })
565 }
566
567 fn field(&self, map: &'a Map, raw_name: &[u8]) -> Option<&'a Value> {
568 self.parsed.field(map, raw_name)
569 }
570
571 pub(crate) fn pptr(&self, node: Node<'a>) -> Result<PPtr, Error> {
573 if !self.list.type_name(node.id).starts_with(b"PPtr<")
577 || (self.parsed.read_by() == crate::value::ReadBy::Boost
578 && self.pptr_object(node).is_none())
579 {
580 return Err(malformed("the script pointer has no dereference method"));
581 }
582 let id = |v: &Value| match v {
583 Value::Bool(b) => Some(i64::from(*b)),
584 Value::F32(_) | Value::F64(_) if v.as_f64() == Some(0.0) => Some(0),
585 _ => v.as_i64(),
586 };
587 let file_id = id(self.attr(node, b"m_FileID")?.value).and_then(|i| i32::try_from(i).ok());
588 let path_id = id(self.attr(node, b"m_PathID")?.value);
589 match (file_id, path_id) {
590 (Some(file_id), Some(path_id)) => Ok(PPtr { file_id, path_id }),
591 _ => Err(malformed("a pointer is not (int32, int64)")),
592 }
593 }
594
595 fn pptr_object(&self, node: Node<'a>) -> Option<(Node<'a>, Node<'a>)> {
603 const FIELDS: [&[u8]; 3] = [b"m_FileID", b"m_PathID", b"assetsfile"];
604 if !matches!(node.value, Value::Dict(_))
605 || !self.list.type_name(node.id).starts_with(b"PPtr<")
606 || !self.list.children(node.id).iter().all(|&c| {
607 let name = self.list.name(c);
608 FIELDS.iter().any(|f| clean_name_is(name, f))
609 })
610 {
611 return None;
612 }
613 Some((
614 self.attr(node, b"m_FileID").ok()?,
615 self.attr(node, b"m_PathID").ok()?,
616 ))
617 }
618
619 fn pointer(&self, node: Node<'a>) -> Option<PPtr> {
622 let (f, p) = self.pptr_object(node)?;
623 Some(PPtr {
624 file_id: i32::try_from(int(f.value)?).ok()?,
625 path_id: i64::try_from(int(p.value)?).ok()?,
626 })
627 }
628
629 fn hashable(&self, node: Node<'a>) -> bool {
634 match node.value {
635 Value::List(_) | Value::Prims(_) => false,
636 Value::Tuple(items) => Seq::Tuple(self.list.children(node.id), node.id, items)
637 .nodes()
638 .all(|n| self.hashable(n)),
639 Value::Dict(_) => self
640 .pptr_object(node)
641 .is_none_or(|(f, p)| self.hashable(f) && self.hashable(p)),
642 _ => true,
643 }
644 }
645
646 fn sequence(&self, node: Node<'a>) -> Result<Seq<'a>, Error> {
651 match node.value {
652 Value::List(items) => Ok(Seq::List(self.element(node.id), items)),
653 Value::Tuple(items) => Ok(Seq::Tuple(self.list.children(node.id), node.id, items)),
654 Value::Prims(p) if p.is_empty() => Ok(Seq::List(node.id, &[])),
655 Value::Str(s) if s.as_bytes().is_empty() => Ok(Seq::List(node.id, &[])),
656 Value::Bytes(b) if b.is_empty() => Ok(Seq::List(node.id, &[])),
657 Value::Prims(_) | Value::Str(_) | Value::Bytes(_) => {
658 Err(malformed("an m_Container item is not a pair"))
659 }
660 _ => Err(malformed("not iterable")),
661 }
662 }
663
664 fn unpack2(&self, item: Node<'a>) -> Option<[Node<'a>; 2]> {
667 if !matches!(item.value, Value::List(_) | Value::Tuple(_)) {
668 return None;
669 }
670 let mut it = self.sequence(item).ok()?.nodes();
671 match (it.next(), it.next(), it.next()) {
672 (Some(k), Some(v), None) => Some([k, v]),
673 _ => None,
674 }
675 }
676
677 fn element(&self, id: NodeId) -> NodeId {
680 let array = match self.list.children(id).first() {
681 Some(&a) if self.list.type_name(a) == b"Array" => a,
682 _ => id,
683 };
684 self.list.children(array).get(1).copied().unwrap_or(array)
685 }
686}
687
688#[derive(Clone, Copy)]
690enum Seq<'a> {
691 List(NodeId, &'a [Value]),
693 Tuple(&'a [NodeId], NodeId, &'a [Value]),
695}
696
697impl<'a> Seq<'a> {
698 fn nodes(self) -> impl Iterator<Item = Node<'a>> {
700 let items = match self {
701 Self::List(_, v) | Self::Tuple(_, _, v) => v,
702 };
703 items.iter().enumerate().map(move |(i, v)| {
704 let id = match self {
705 Self::List(elem, _) => elem,
706 Self::Tuple(kids, id, _) => kids.get(i).copied().unwrap_or(id),
707 };
708 Node::new(id, v)
709 })
710 }
711}
712
713fn clean_name_is(name: &[u8], target: &[u8]) -> bool {
718 let name = name.strip_prefix(b"(int&)").unwrap_or(name);
719 let name = name.strip_suffix(b"?").unwrap_or(name);
720 name.len() == target.len()
721 && name.iter().zip(target).all(|(&a, &b)| {
722 let a = if b" .:-[]".contains(&a) { b'_' } else { a };
723 a == b
724 })
725}
726
727#[cfg(test)]
728mod tests {
729 use std::sync::Arc;
730
731 use super::{Container, ContainerRow, clean_name_is};
732 use crate::budget::{Counting, Reservation};
733 use crate::env::PPtr;
734 use crate::limits::{Meter, MeteredVec, heap_cost};
735
736 fn row(i: i64) -> ContainerRow {
737 ContainerRow {
738 path: Some("a".into()),
739 preload_index: Some(0),
740 preload_size: Some(0),
741 file_id: Some(0),
742 path_id: Some(i128::from(i)),
743 }
744 }
745
746 #[test]
747 fn the_tables_are_charged_at_their_exact_size() {
748 let budget = Arc::new(Counting::new(u64::MAX));
751 let m = Meter::new(u64::MAX, budget.clone(), Reservation::Input);
752 let mut rows = MeteredVec::new(&m);
753 let mut preload = MeteredVec::new(&m);
754 for i in 0..100 {
755 rows.push(row(i)).unwrap();
756 preload
757 .push(Some(PPtr {
758 file_id: 0,
759 path_id: i,
760 }))
761 .unwrap();
762 }
763 let c = Container::new(rows, Some(preload)).unwrap();
764 let exact = |n: usize| heap_cost(u64::try_from(n).unwrap());
765 assert_eq!(
766 m.used(),
767 exact(100 * std::mem::size_of::<ContainerRow>())
768 + exact(100 * std::mem::size_of::<Option<i64>>())
769 + exact(100 * std::mem::size_of::<Option<PPtr>>())
770 );
771 assert_eq!(
772 (
773 c.rows().len(),
774 c.preload_table().unwrap().map(<[Option<PPtr>]>::len)
775 ),
776 (100, Some(100))
777 );
778 assert_eq!(c.charge().records(), 0);
779 drop(c);
780 assert_eq!((m.used(), budget.used()), (0, 0));
781 }
782
783 #[test]
784 fn clean_names_match_as_python_cleans_them() {
785 assert!(clean_name_is(b"m_Container", b"m_Container"));
786 assert!(clean_name_is(b"m Container", b"m_Container"));
787 assert!(clean_name_is(b"(int&)m.Container?", b"m_Container"));
788 assert!(clean_name_is(b"m-Container", b"m_Container"));
789 assert!(!clean_name_is(b"m_Container??", b"m_Container"));
790 assert!(!clean_name_is(b"m_container", b"m_Container"));
791 assert!(!clean_name_is(b"", b"asset"));
792 }
793}