use std::collections::HashMap;
use std::mem::size_of;
use std::sync::{Arc, Mutex, PoisonError};
use crate::env::PPtr;
use crate::env::registry::Rank;
use crate::error::Error;
use crate::files::SerializedFile;
use crate::limits::{Charge, ChargeError, HASH_TABLE_SHARES, Meter, MeteredVec};
use crate::once::{ResultCell, get_or_publish_result};
use crate::raw::RawStr;
use crate::typetree::{NodeId, NodeList};
use crate::value::{Map, Parsed, Value};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ContainerRow {
pub path: Option<RawStr>,
pub preload_index: Option<i128>,
pub preload_size: Option<i128>,
pub file_id: Option<i128>,
pub path_id: Option<i128>,
}
impl ContainerRow {
#[must_use]
pub fn asset(&self) -> Option<PPtr> {
Some(PPtr {
file_id: i32::try_from(self.file_id?).ok()?,
path_id: i64::try_from(self.path_id?).ok()?,
})
}
}
#[derive(Debug, PartialEq, Eq)]
enum Preload {
None,
List(Box<[Option<PPtr>]>),
NotAList,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ContainerPath {
Str(RawStr),
NotAString,
}
const _: () = crate::send_sync::<ContainerPath>();
impl ContainerPath {
fn of(path: Option<&RawStr>) -> Self {
path.map_or(Self::NotAString, |p| Self::Str(p.clone()))
}
}
pub(crate) type Stamp = (u64, Rank, u32, u32);
#[derive(Debug)]
struct Preloaded {
stamp: Stamp,
path: Option<RawStr>,
_charge: Charge,
}
const PRELOADED_SHARE: u64 =
HASH_TABLE_SHARES * (size_of::<i64>() + size_of::<Preloaded>() + 1) as u64;
#[derive(Debug)]
struct PathIndex {
rows: HashMap<i64, u32>,
_charge: Charge,
}
#[derive(Debug)]
pub struct Container {
rows: Box<[ContainerRow]>,
keys: Box<[Option<i64>]>,
preload: Preload,
charge: Charge,
index: ResultCell<PathIndex, Error>,
preloaded: Mutex<HashMap<i64, Preloaded>>,
}
impl PartialEq for Container {
fn eq(&self, other: &Self) -> bool {
self.rows == other.rows && self.preload == other.preload
}
}
impl Eq for Container {}
impl Container {
#[must_use]
pub fn empty(meter: &Arc<Meter>) -> Self {
Self::parts(
Box::new([]),
Box::new([]),
Preload::None,
Charge::zero(meter),
)
}
fn parts(
rows: Box<[ContainerRow]>,
keys: Box<[Option<i64>]>,
preload: Preload,
charge: Charge,
) -> Self {
Self {
rows,
keys,
preload,
charge,
index: ResultCell::new(),
preloaded: Mutex::new(HashMap::new()),
}
}
fn int_keys(rows: &MeteredVec<ContainerRow>) -> Result<MeteredVec<Option<i64>>, ChargeError> {
let mut keys = MeteredVec::with_capacity(rows.charge().meter(), rows.len())?;
for r in rows.as_slice() {
let own = r.file_id == Some(0);
keys.push(
r.path_id
.filter(|_| own)
.and_then(|p| i64::try_from(p).ok()),
)?;
}
Ok(keys)
}
pub fn new(
rows: MeteredVec<ContainerRow>,
preload_table: Option<MeteredVec<Option<PPtr>>>,
) -> Result<Self, ChargeError> {
let keys = Self::int_keys(&rows)?;
Self::keyed(
rows,
keys,
preload_table.map_or(Ok((Preload::None, None)), list),
)
}
pub fn with_preload_not_a_list(rows: MeteredVec<ContainerRow>) -> Result<Self, ChargeError> {
let keys = Self::int_keys(&rows)?;
Self::keyed(rows, keys, Ok((Preload::NotAList, None)))
}
fn keyed(
rows: MeteredVec<ContainerRow>,
keys: MeteredVec<Option<i64>>,
preload: Result<(Preload, Option<Charge>), ChargeError>,
) -> Result<Self, ChargeError> {
let (rows, mut charge) = rows.into_boxed_slice()?;
let (keys, c) = keys.into_boxed_slice()?;
charge.absorb(c);
let (preload, c) = preload?;
if let Some(c) = c {
charge.absorb(c);
}
Ok(Self::parts(rows, keys, preload, charge))
}
#[must_use]
pub fn rows(&self) -> &[ContainerRow] {
&self.rows
}
pub fn preload_table(&self) -> Result<Option<&[Option<PPtr>]>, Error> {
match &self.preload {
Preload::None => Ok(None),
Preload::List(t) => Ok(Some(t)),
Preload::NotAList => Err(malformed("m_PreloadTable is not a list")),
}
}
#[must_use]
pub fn charge(&self) -> &Charge {
&self.charge
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.rows.is_empty()
}
pub(crate) fn path_of(
&self,
path_id: i64,
meter: &Arc<Meter>,
) -> Result<Option<ContainerPath>, Error> {
let index = get_or_publish_result(&self.index, meter, || self.build_index(meter))?;
if let Some(&row) = index.rows.get(&path_id) {
let row = usize::try_from(row).ok().and_then(|r| self.rows.get(r));
return Ok(row.map(|r| ContainerPath::of(r.path.as_ref())));
}
let set = self
.preloaded
.lock()
.unwrap_or_else(PoisonError::into_inner);
Ok(set
.get(&path_id)
.map(|p| ContainerPath::of(p.path.as_ref())))
}
fn build_index(&self, meter: &Arc<Meter>) -> Result<PathIndex, Error> {
let n = self.keys.iter().flatten().count();
let charge = meter.charge(
HASH_TABLE_SHARES
.saturating_mul((size_of::<i64>() + size_of::<u32>() + 1) as u64)
.saturating_mul(u64::try_from(n).unwrap_or(u64::MAX)),
)?;
let mut rows = HashMap::with_capacity(n);
for (i, key) in self.keys.iter().enumerate() {
if let (Some(key), Ok(i)) = (key, u32::try_from(i)) {
rows.insert(*key, i);
}
}
Ok(PathIndex {
rows,
_charge: charge,
})
}
pub(crate) fn set_default(
&self,
path_id: i64,
path: Option<&RawStr>,
stamp: &Stamp,
meter: &Arc<Meter>,
) -> Result<(), Error> {
let index = get_or_publish_result(&self.index, meter, || self.build_index(meter))?;
if index.rows.contains_key(&path_id) {
return Ok(());
}
let lose =
|set: &HashMap<i64, Preloaded>| set.get(&path_id).is_some_and(|p| p.stamp <= *stamp);
if lose(
&self
.preloaded
.lock()
.unwrap_or_else(PoisonError::into_inner),
) {
return Ok(());
}
let charge = meter.charge(PRELOADED_SHARE)?;
let entry = Preloaded {
stamp: stamp.clone(),
path: path.cloned(),
_charge: charge,
};
let replaced = {
let mut set = self
.preloaded
.lock()
.unwrap_or_else(PoisonError::into_inner);
if lose(&set) {
Some(entry)
} else {
set.insert(path_id, entry)
}
};
drop(replaced);
Ok(())
}
}
fn list(t: MeteredVec<Option<PPtr>>) -> Result<(Preload, Option<Charge>), ChargeError> {
let (t, c) = t.into_boxed_slice()?;
Ok((Preload::List(t), Some(c)))
}
pub fn read_container(file: &Arc<SerializedFile>) -> Result<Container, Error> {
let meter = file.meter();
let Some(obj) = file.objects().find(|o| o.class_id() == ASSET_BUNDLE) else {
return Ok(Container::empty(meter));
};
let parsed = obj
.read_typetree(None, true)
.map_err(|e| Error::AssetBundle(Arc::new(e)))?;
let Some(view) = View::new(&parsed) else {
return Err(malformed("no node list"));
};
let root = view.root();
let preload = view.attr(root, b"m_PreloadTable")?;
let items = view.attr(root, b"m_Container")?;
let mut rows = MeteredVec::new(meter);
let mut keys = MeteredVec::new(meter);
for item in view.sequence(items)?.nodes() {
let Some([key, info]) = view.unpack2(item) else {
return Err(malformed("an m_Container item is not a pair"));
};
let asset = view.attr(info, b"asset")?;
let (file_id, path_id) = match view.pptr_object(asset) {
Some((file_id, path_id)) => (Some(file_id), path_id),
None => (
view.attr(asset, b"m_FileID").ok(),
view.attr(asset, b"path_id")?,
),
};
if !view.hashable(key) || !view.hashable(path_id) {
return Err(malformed("an m_Container key or path_id is unhashable"));
}
let own = file_id.and_then(|f| py_int(f.value)) == Some(0);
keys.push(if own { py_int(path_id.value) } else { None })?;
let path_id = match view.pptr_object(asset) {
Some(_) => Some(path_id),
None => view.attr(asset, b"m_PathID").ok(),
};
let int = |n: Option<Node<'_>>| n.and_then(|n| int(n.value));
rows.push(ContainerRow {
path: match key.value {
Value::Str(s) => Some(s.clone()),
_ => None,
},
preload_index: int(view.attr(info, b"preloadIndex").ok()),
preload_size: int(view.attr(info, b"preloadSize").ok()),
file_id: int(file_id),
path_id: int(path_id),
})?;
}
let mut table = MeteredVec::new(meter);
match preload.value {
Value::List(_) => {
for p in view.sequence(preload)?.nodes() {
table.push(view.pointer(p))?;
}
}
Value::Prims(p) => {
for _ in 0..p.len() {
table.push(None)?;
}
}
_ => return Ok(Container::keyed(rows, keys, Ok((Preload::NotAList, None)))?),
}
Ok(Container::keyed(rows, keys, list(table))?)
}
fn py_int(v: &Value) -> Option<i64> {
let float = |f: f64| {
#[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)]
(f.fract() == 0.0
&& (-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0).contains(&f))
.then_some(f as i64)
};
match v {
Value::Int(i) => Some(*i),
Value::UInt(u) => i64::try_from(*u).ok(),
Value::Bool(b) => Some(i64::from(*b)),
Value::F32(bits) => float(f64::from(f32::from_bits(*bits))),
Value::F64(bits) => float(f64::from_bits(*bits)),
_ => None,
}
}
fn int(v: &Value) -> Option<i128> {
match v {
Value::Int(i) => Some(i128::from(*i)),
Value::UInt(u) => Some(i128::from(*u)),
_ => None,
}
}
const ASSET_BUNDLE: i32 = 142;
fn malformed(detail: &str) -> Error {
Error::Malformed {
what: "AssetBundle",
detail: detail.into(),
}
}
#[derive(Clone, Copy)]
pub(crate) struct Node<'a> {
id: NodeId,
value: &'a Value,
}
impl<'a> Node<'a> {
fn new(id: NodeId, value: &'a Value) -> Self {
Self { id, value }
}
pub(crate) fn value(&self) -> &'a Value {
self.value
}
}
pub(crate) struct View<'a> {
parsed: &'a Parsed,
list: &'a NodeList,
}
impl<'a> View<'a> {
pub(crate) fn new(parsed: &'a Parsed) -> Option<Self> {
let list = parsed.lists().first()?;
Some(Self { parsed, list })
}
pub(crate) fn root(&self) -> Node<'a> {
Node::new(self.list.root(), self.parsed.value())
}
pub(crate) fn attr(&self, node: Node<'a>, name: &[u8]) -> Result<Node<'a>, Error> {
let Value::Dict(map) = node.value else {
return Err(malformed("an attribute of a value that is not a class"));
};
let child = self
.list
.children(node.id)
.iter()
.rev()
.find(|c| clean_name_is(self.list.name(**c), name))
.copied();
let found = child.and_then(|c| self.field(map, self.list.name(c)).map(|v| (c, v)));
let Some((id, value)) = found else {
return Err(Error::Malformed {
what: "AssetBundle",
detail: format!("no attribute {}", name.escape_ascii()).into(),
});
};
Ok(Node { id, value })
}
fn field(&self, map: &'a Map, raw_name: &[u8]) -> Option<&'a Value> {
self.parsed.field(map, raw_name)
}
pub(crate) fn pptr(&self, node: Node<'a>) -> Result<PPtr, Error> {
if !self.list.type_name(node.id).starts_with(b"PPtr<")
|| (self.parsed.read_by() == crate::value::ReadBy::Boost
&& self.pptr_object(node).is_none())
{
return Err(malformed("the script pointer has no dereference method"));
}
let id = |v: &Value| match v {
Value::Bool(b) => Some(i64::from(*b)),
Value::F32(_) | Value::F64(_) if v.as_f64() == Some(0.0) => Some(0),
_ => v.as_i64(),
};
let file_id = id(self.attr(node, b"m_FileID")?.value).and_then(|i| i32::try_from(i).ok());
let path_id = id(self.attr(node, b"m_PathID")?.value);
match (file_id, path_id) {
(Some(file_id), Some(path_id)) => Ok(PPtr { file_id, path_id }),
_ => Err(malformed("a pointer is not (int32, int64)")),
}
}
fn pptr_object(&self, node: Node<'a>) -> Option<(Node<'a>, Node<'a>)> {
const FIELDS: [&[u8]; 3] = [b"m_FileID", b"m_PathID", b"assetsfile"];
if !matches!(node.value, Value::Dict(_))
|| !self.list.type_name(node.id).starts_with(b"PPtr<")
|| !self.list.children(node.id).iter().all(|&c| {
let name = self.list.name(c);
FIELDS.iter().any(|f| clean_name_is(name, f))
})
{
return None;
}
Some((
self.attr(node, b"m_FileID").ok()?,
self.attr(node, b"m_PathID").ok()?,
))
}
fn pointer(&self, node: Node<'a>) -> Option<PPtr> {
let (f, p) = self.pptr_object(node)?;
Some(PPtr {
file_id: i32::try_from(int(f.value)?).ok()?,
path_id: i64::try_from(int(p.value)?).ok()?,
})
}
fn hashable(&self, node: Node<'a>) -> bool {
match node.value {
Value::List(_) | Value::Prims(_) => false,
Value::Tuple(items) => Seq::Tuple(self.list.children(node.id), node.id, items)
.nodes()
.all(|n| self.hashable(n)),
Value::Dict(_) => self
.pptr_object(node)
.is_none_or(|(f, p)| self.hashable(f) && self.hashable(p)),
_ => true,
}
}
fn sequence(&self, node: Node<'a>) -> Result<Seq<'a>, Error> {
match node.value {
Value::List(items) => Ok(Seq::List(self.element(node.id), items)),
Value::Tuple(items) => Ok(Seq::Tuple(self.list.children(node.id), node.id, items)),
Value::Prims(p) if p.is_empty() => Ok(Seq::List(node.id, &[])),
Value::Str(s) if s.as_bytes().is_empty() => Ok(Seq::List(node.id, &[])),
Value::Bytes(b) if b.is_empty() => Ok(Seq::List(node.id, &[])),
Value::Prims(_) | Value::Str(_) | Value::Bytes(_) => {
Err(malformed("an m_Container item is not a pair"))
}
_ => Err(malformed("not iterable")),
}
}
fn unpack2(&self, item: Node<'a>) -> Option<[Node<'a>; 2]> {
if !matches!(item.value, Value::List(_) | Value::Tuple(_)) {
return None;
}
let mut it = self.sequence(item).ok()?.nodes();
match (it.next(), it.next(), it.next()) {
(Some(k), Some(v), None) => Some([k, v]),
_ => None,
}
}
fn element(&self, id: NodeId) -> NodeId {
let array = match self.list.children(id).first() {
Some(&a) if self.list.type_name(a) == b"Array" => a,
_ => id,
};
self.list.children(array).get(1).copied().unwrap_or(array)
}
}
#[derive(Clone, Copy)]
enum Seq<'a> {
List(NodeId, &'a [Value]),
Tuple(&'a [NodeId], NodeId, &'a [Value]),
}
impl<'a> Seq<'a> {
fn nodes(self) -> impl Iterator<Item = Node<'a>> {
let items = match self {
Self::List(_, v) | Self::Tuple(_, _, v) => v,
};
items.iter().enumerate().map(move |(i, v)| {
let id = match self {
Self::List(elem, _) => elem,
Self::Tuple(kids, id, _) => kids.get(i).copied().unwrap_or(id),
};
Node::new(id, v)
})
}
}
fn clean_name_is(name: &[u8], target: &[u8]) -> bool {
let name = name.strip_prefix(b"(int&)").unwrap_or(name);
let name = name.strip_suffix(b"?").unwrap_or(name);
name.len() == target.len()
&& name.iter().zip(target).all(|(&a, &b)| {
let a = if b" .:-[]".contains(&a) { b'_' } else { a };
a == b
})
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use super::{Container, ContainerRow, clean_name_is};
use crate::budget::{Counting, Reservation};
use crate::env::PPtr;
use crate::limits::{Meter, MeteredVec, heap_cost};
fn row(i: i64) -> ContainerRow {
ContainerRow {
path: Some("a".into()),
preload_index: Some(0),
preload_size: Some(0),
file_id: Some(0),
path_id: Some(i128::from(i)),
}
}
#[test]
fn the_tables_are_charged_at_their_exact_size() {
let budget = Arc::new(Counting::new(u64::MAX));
let m = Meter::new(u64::MAX, budget.clone(), Reservation::Input);
let mut rows = MeteredVec::new(&m);
let mut preload = MeteredVec::new(&m);
for i in 0..100 {
rows.push(row(i)).unwrap();
preload
.push(Some(PPtr {
file_id: 0,
path_id: i,
}))
.unwrap();
}
let c = Container::new(rows, Some(preload)).unwrap();
let exact = |n: usize| heap_cost(u64::try_from(n).unwrap());
assert_eq!(
m.used(),
exact(100 * std::mem::size_of::<ContainerRow>())
+ exact(100 * std::mem::size_of::<Option<i64>>())
+ exact(100 * std::mem::size_of::<Option<PPtr>>())
);
assert_eq!(
(
c.rows().len(),
c.preload_table().unwrap().map(<[Option<PPtr>]>::len)
),
(100, Some(100))
);
assert_eq!(c.charge().records(), 0);
drop(c);
assert_eq!((m.used(), budget.used()), (0, 0));
}
#[test]
fn clean_names_match_as_python_cleans_them() {
assert!(clean_name_is(b"m_Container", b"m_Container"));
assert!(clean_name_is(b"m Container", b"m_Container"));
assert!(clean_name_is(b"(int&)m.Container?", b"m_Container"));
assert!(clean_name_is(b"m-Container", b"m_Container"));
assert!(!clean_name_is(b"m_Container??", b"m_Container"));
assert!(!clean_name_is(b"m_container", b"m_Container"));
assert!(!clean_name_is(b"", b"asset"));
}
}