use std::sync::Arc;
use bytes::Bytes;
use super::dict::{FindError, RefIndex, steps, visit};
use super::{
OpenWork, RefTypes, VALUE_WORK, WriteCtx, WriteError, WriteRules, py_size_field_is_zero,
};
use crate::budget::DecodeBuf;
use crate::raw::RawStr;
use crate::typetree::{Kind, NodeId, Tree, TreeError};
use crate::value::{Map, Parsed, ReadBy, Value};
mod bounds;
mod lookup;
mod pyobj;
mod run;
use lookup::{KeptTables, Lookup};
use pyobj::{CLEAN_MAX, Obj, chars, clean_name, narrow};
pub use pyobj::{cleans_to, python_attributes};
pub fn write(
parsed: &Parsed,
tree: &Tree,
ctx: &WriteCtx<'_>,
out: &mut DecodeBuf,
) -> Result<(), WriteError> {
write_at(parsed, tree, ctx, out, out.len())
}
pub fn write_at(
parsed: &Parsed,
tree: &Tree,
ctx: &WriteCtx<'_>,
out: &mut DecodeBuf,
alignment_origin: usize,
) -> Result<(), WriteError> {
write_at_with_lengths(parsed, tree, ctx, out, alignment_origin, &|_| None)
}
pub fn write_at_with_lengths(
parsed: &Parsed,
tree: &Tree,
ctx: &WriteCtx<'_>,
out: &mut DecodeBuf,
alignment_origin: usize,
lengths: &dyn Fn(&Value) -> Option<usize>,
) -> Result<(), WriteError> {
if alignment_origin > out.len() {
return Err(WriteError::OutOfRange {
node: RawStr::from_static(b"writer alignment origin"),
});
}
let meter = out.meter().clone();
let open = OpenWork::start(&meter).map_err(WriteError::Limit)?;
let mut w = Writer {
parsed,
lengths,
out,
base: alignment_origin,
big: ctx.endian == crate::stream::Endian::Big,
mirror: match ctx.rules {
WriteRules::UnityPy => None,
WriteRules::MirrorReader => Some(parsed.read_by()),
},
ref_types: ctx.ref_types,
refs: RefIndex::new(ctx.ref_types),
max_depth: ctx.limits.max_value_depth,
limits: *ctx.limits,
depth: 0,
values: 0,
repeated: 0,
open,
has_registry: false,
shared: 0,
kept: KeptTables::new(parsed.charge().meter()),
};
let written = w.value(tree, tree.root(), Obj::V(parsed.value()));
let ended = w.open.finish();
drop(w);
written?;
ended.map_err(WriteError::Limit)
}
struct Writer<'a, 'o> {
parsed: &'a Parsed,
lengths: &'o dyn Fn(&Value) -> Option<usize>,
out: &'o mut DecodeBuf,
base: usize,
big: bool,
mirror: Option<ReadBy>,
ref_types: RefTypes<'a>,
refs: RefIndex<'a>,
limits: crate::limits::Limits,
max_depth: u32,
depth: u32,
values: u64,
repeated: u32,
open: OpenWork<'o>,
has_registry: bool,
shared: u32,
kept: KeptTables<'a>,
}
#[derive(Clone)]
enum Data {
Skip,
Through(Option<Arc<Tree>>),
}
impl<'a> Writer<'a, '_> {
fn logical_len(&self, value: Obj<'_>) -> Option<usize> {
if let Obj::V(value) = value
&& let Some(len) = (self.lengths)(value)
{
return Some(len);
}
Self::len(value)
}
fn name(tree: &Tree, node: NodeId) -> RawStr {
tree.list()
.row(node)
.map(|r| r.name.clone())
.unwrap_or_default()
}
fn mismatch(tree: &Tree, node: NodeId, expected: &'static str) -> WriteError {
WriteError::TypeMismatch {
node: Self::name(tree, node),
expected,
}
}
fn put(&mut self, bytes: &[u8]) -> Result<(), WriteError> {
self.out
.extend_from_slice(bytes)
.map_err(WriteError::Charge)
}
fn str_work(&mut self, o: Obj<'_>) -> Result<(), WriteError> {
let n = match o {
Obj::V(Value::Str(s)) => s.len(),
Obj::Str(s) => s.len(),
_ => return Ok(()),
};
let n = u64::try_from(n).unwrap_or(u64::MAX);
self.open.add(n).map_err(WriteError::Limit)
}
fn index(&mut self, o: Obj<'a>, i: usize) -> Result<Option<Obj<'a>>, WriteError> {
self.str_work(o)?;
Ok(Self::item(o, i))
}
fn put_counted(&mut self, bytes: &[u8]) -> Result<(), WriteError> {
let n = u64::try_from(bytes.len()).unwrap_or(u64::MAX);
self.open.add(n).map_err(WriteError::Limit)?;
self.put(bytes)
}
fn align(&mut self) -> Result<(), WriteError> {
let pos = self.out.len().saturating_sub(self.base);
let pad = (4 - pos % 4) % 4;
self.put([0; 4].get(..pad).unwrap_or_default())
}
fn write_int(&mut self, n: usize, tree: &Tree, node: NodeId) -> Result<(), WriteError> {
let n = i32::try_from(n).map_err(|_| WriteError::OutOfRange {
node: Self::name(tree, node),
})?;
let b = if self.big {
n.to_be_bytes()
} else {
n.to_le_bytes()
};
self.put(&b)
}
fn field(&mut self, m: &'a Map, name: &[u8]) -> Result<Option<&'a Value>, WriteError> {
let pos = if self.shared > 0 {
steps(1);
self.kept.position(self.parsed, m, name, &mut self.open)?
} else {
let mut found = None;
for (i, (n, _)) in self.parsed.entries(m).enumerate() {
steps(1);
self.open.steps(1).map_err(WriteError::Limit)?;
if n == name {
found = Some(i);
break;
}
}
found
};
Ok(pos.and_then(|p| m.entries().get(p)).map(|(_, v)| v))
}
fn children(
&mut self,
tree: &Tree,
node: NodeId,
f: impl FnOnce(&mut Self) -> Result<(), WriteError>,
) -> Result<(), WriteError> {
let dup = tree.info(node).is_some_and(|i| i.dup_names);
if dup {
self.shared = self.shared.saturating_add(1);
}
let r = f(self);
if dup {
self.shared = self.shared.saturating_sub(1);
if self.shared == 0 {
self.kept = KeptTables::new(self.parsed.charge().meter());
}
}
r
}
fn find(
&mut self,
find: &mut Lookup<'a>,
tree: &Tree,
child: NodeId,
) -> Result<Option<&'a Value>, WriteError> {
let kept = (self.shared > 0).then_some(&mut self.kept);
find.get(self.parsed, tree, child, kept, &mut self.open)
}
fn value(&mut self, tree: &Tree, node: NodeId, o: Obj<'a>) -> Result<(), WriteError> {
self.count(self.cost(tree, node))?;
self.frame(|w| w.value_inner(tree, node, o))
}
fn cost(&self, tree: &Tree, node: NodeId) -> u64 {
self.kind(tree, node)
.prim_size()
.map_or(VALUE_WORK, u64::from)
}
fn kind(&self, tree: &Tree, node: NodeId) -> Kind {
match self.mirror {
Some(ReadBy::Boost) => tree.info(node).map_or(Kind::Unknown, |i| i.kind),
None | Some(ReadBy::Python) => tree.py_kind(node),
}
}
fn string(&mut self, tree: &Tree, node: NodeId, o: Obj<'a>) -> Result<(), WriteError> {
let s = Self::as_str(o).ok_or_else(|| Self::mismatch(tree, node, "str"))?;
let len = match o {
Obj::V(value) => (self.lengths)(value),
_ => None,
}
.unwrap_or(s.len());
self.write_int(len, tree, node)?;
self.put_counted(s)?;
if s.is_empty() && self.mirror == Some(ReadBy::Python) {
return Ok(());
}
self.align()
}
fn bytes(&mut self, tree: &Tree, node: NodeId, o: Obj<'a>) -> Result<(), WriteError> {
let Obj::V(Value::Bytes(b)) = o else {
return Err(Self::mismatch(tree, node, "bytes"));
};
let len = match o {
Obj::V(value) => (self.lengths)(value),
_ => None,
}
.unwrap_or(b.len());
self.write_int(len, tree, node)?;
self.put_counted(b)
}
fn value_inner(&mut self, tree: &Tree, node: NodeId, o: Obj<'a>) -> Result<(), WriteError> {
let mut align = tree.info(node).is_some_and(|i| i.align);
let kind = self.kind(tree, node);
let kids = tree.list().children(node);
if kind.prim_size().is_some() {
self.prim(tree, node, kind, o)?;
} else if kind == Kind::Str {
self.string(tree, node, o)?;
} else if kind == Kind::Bytes {
self.bytes(tree, node, o)?;
} else if kind == Kind::Pair {
for i in 0..2 {
let item = self
.index(o, i)?
.ok_or_else(|| Self::mismatch(tree, node, "a sequence of two"))?;
let child = kids
.get(i)
.copied()
.ok_or_else(|| WriteError::Malformed(Self::name(tree, node)))?;
self.value(tree, child, item)?;
}
} else if kind == Kind::ReferencedObject {
self.ref_object(tree, node, o)?;
} else if let Some(array) = kids
.first()
.copied()
.filter(|c| self.kind(tree, *c) == Kind::Array)
{
align |= tree.info(array).is_some_and(|i| i.align);
self.str_work(o)?;
let n = self
.logical_len(o)
.ok_or_else(|| Self::mismatch(tree, node, "a sized iterable"))?;
self.write_int(n, tree, node)?;
let elem = tree
.list()
.children(array)
.get(1)
.copied()
.ok_or_else(|| WriteError::Malformed(Self::name(tree, node)))?;
self.frame(|w| w.elements(tree, elem, o))?;
} else {
self.class(tree, node, o, true)?;
}
if align {
self.align()?;
}
Ok(())
}
fn prim(
&mut self,
tree: &Tree,
node: NodeId,
kind: Kind,
o: Obj<'a>,
) -> Result<(), WriteError> {
let big = self.big;
let range = || WriteError::OutOfRange {
node: Self::name(tree, node),
};
let mut buf = [0_u8; 8];
macro_rules! put {
($v:expr) => {{
let v = $v;
let b = if big {
v.to_be_bytes()
} else {
v.to_le_bytes()
};
let n = b.len();
buf.get_mut(..n).unwrap_or_default().copy_from_slice(&b);
n
}};
}
macro_rules! int {
($t:ty) => {{
let i = Self::as_int(o).ok_or_else(|| Self::mismatch(tree, node, "int"))?;
put!(<$t>::try_from(i).map_err(|_| range())?)
}};
}
let n = match kind {
Kind::Bool => put!(u8::from(Self::truthy(o))),
Kind::F32 => put!(match o {
Obj::V(Value::F32(b)) => *b,
Obj::F32(b) => b,
_ => {
let x =
Self::as_double(o).ok_or_else(|| Self::mismatch(tree, node, "float"))?;
narrow(x).ok_or_else(range)?
}
}),
Kind::F64 => {
let x = Self::as_double(o).ok_or_else(|| Self::mismatch(tree, node, "float"))?;
put!(x.to_bits())
}
Kind::S8 => int!(i8),
Kind::U8 => int!(u8),
Kind::S16 => int!(i16),
Kind::U16 => int!(u16),
Kind::S32 => int!(i32),
Kind::U32 => int!(u32),
Kind::S64 => int!(i64),
_ => int!(u64),
};
let (bytes, _) = buf.split_at(n.min(8));
self.put(bytes)
}
#[inline(never)]
fn class(
&mut self,
tree: &Tree,
node: NodeId,
o: Obj<'a>,
registry: bool,
) -> Result<(), WriteError> {
if let Obj::V(Value::Dict(m)) = o {
return self.class_dict(tree, node, m, registry);
}
let outer = self.has_registry;
let kids = tree.list().children(node);
for c in kids {
visit();
if registry && self.kind(tree, *c) == Kind::ManagedReferencesRegistry {
if self.has_registry {
self.skip()?;
continue;
}
self.has_registry = true;
}
let name = tree.list().name(*c);
let mut buf = [0_u8; CLEAN_MAX];
let clean = clean_name(name, &mut buf).and_then(|n| buf.get(..n));
let attr = match clean {
Some(clean) => Self::getattr(o, clean),
None => Ok(None),
};
let item = match attr {
Ok(Some(a)) => a,
Ok(None) => {
return Err(Self::mismatch(tree, *c, "an object with this attribute"));
}
Err(()) => {
return Err(Self::mismatch(
tree,
*c,
"a modelled (non-dunder) attribute",
));
}
};
let repeated = tree.info(*c).is_some_and(|i| i.repeated);
let before = self.out.len();
self.child(repeated, |w| w.value(tree, *c, item))?;
if self.out.len() == before {
self.bound()?;
}
}
self.has_registry = outer;
Ok(())
}
#[inline(never)]
fn class_dict(
&mut self,
tree: &Tree,
node: NodeId,
m: &'a Map,
registry: bool,
) -> Result<(), WriteError> {
let outer = self.has_registry;
let mut find = Lookup::new(self.parsed, tree, node, m);
self.children(tree, node, |w| {
for c in tree.list().children(node) {
visit();
if registry && w.kind(tree, *c) == Kind::ManagedReferencesRegistry {
if w.has_registry {
w.skip()?;
continue;
}
w.has_registry = true;
}
let v = w
.find(&mut find, tree, *c)?
.ok_or_else(|| WriteError::MissingKey(Self::name(tree, *c)))?;
let repeated = tree.info(*c).is_some_and(|i| i.repeated);
w.child(repeated, |w| w.value(tree, *c, Obj::V(v)))?;
}
Ok(())
})?;
self.has_registry = outer;
Ok(())
}
#[inline(never)]
fn ref_object(&mut self, tree: &Tree, node: NodeId, o: Obj<'a>) -> Result<(), WriteError> {
let kids = tree.list().children(node);
if kids.is_empty() {
return Ok(());
}
let Obj::V(Value::Dict(m)) = o else {
return Err(Self::mismatch(tree, node, "dict"));
};
let mut find = Lookup::new(self.parsed, tree, node, m);
let mut data: Option<Data> = None;
self.children(tree, node, |w| {
for c in kids {
visit();
let repeated = tree.info(*c).is_some_and(|i| i.repeated);
if w.kind(tree, *c) != Kind::ReferencedObjectData {
let v = w
.find(&mut find, tree, *c)?
.ok_or_else(|| WriteError::MissingKey(Self::name(tree, *c)))?;
w.child(repeated, |w| w.value(tree, *c, Obj::V(v)))?;
continue;
}
let through = if let Some(d) = &data {
d.clone()
} else {
let d = match w.mirror {
None => Data::Through(w.ref_node_py(m)?),
Some(ReadBy::Boost) => w
.ref_node_boost(m)?
.map_or(Data::Skip, |t| Data::Through(Some(t))),
Some(ReadBy::Python) => w
.ref_node_py(m)?
.map_or(Data::Skip, |t| Data::Through(Some(t))),
};
data.insert(d).clone()
};
let Data::Through(ref_tree) = through else {
w.skip()?;
continue;
};
let v = w
.find(&mut find, tree, *c)?
.ok_or_else(|| WriteError::MissingKey(Self::name(tree, *c)))?;
let Some(rt) = ref_tree else {
return Err(Self::mismatch(tree, *c, "a ref type with a tree"));
};
w.child(repeated, |w| w.value(&rt, rt.root(), Obj::V(v)))?;
}
Ok(())
})
}
fn tree_error(e: TreeError) -> WriteError {
match e {
TreeError::Limit(l) => WriteError::Limit(l),
TreeError::Charge(c) => WriteError::Charge(c),
_ => WriteError::Malformed(RawStr::new(Bytes::from_static(b"ReferencedObjectData"))),
}
}
fn names(&mut self, m: &'a Map) -> Result<(&'a Value, &'a Value, &'a Value), WriteError> {
let key = |k: &'static [u8]| RawStr::new(Bytes::from_static(k));
let t = match self.field(m, b"type")? {
None => return Err(WriteError::MissingKey(key(b"type"))),
Some(Value::Dict(t)) => t,
Some(_) => {
return Err(WriteError::TypeMismatch {
node: key(b"type"),
expected: "a dict or an object with class, ns and asm",
});
}
};
let mut get = |k: &'static [u8]| {
self.field(t, k)?
.ok_or_else(|| WriteError::MissingKey(key(k)))
};
Ok((get(b"class")?, get(b"ns")?, get(b"asm")?))
}
fn find_ref(
&mut self,
(cls, ns, asm): (&Value, &Value, &Value),
) -> Result<Option<Arc<Tree>>, WriteError> {
let r = self
.refs
.find(self.parsed.charge().meter(), &mut self.open, cls, ns, asm)
.map_err(|e| match e {
FindError::Charge(e) => WriteError::Charge(e),
FindError::Work(l) => WriteError::Limit(l),
})?
.ok_or(WriteError::RefTypeNotFound)?;
r.tree(&self.limits).transpose().map_err(Self::tree_error)
}
fn ref_node_py(&mut self, m: &'a Map) -> Result<Option<Arc<Tree>>, WriteError> {
let (cls, ns, asm) = self.names(m)?;
match self.ref_types {
RefTypes::List(l) if !l.is_empty() => {}
RefTypes::Absent => return Err(WriteError::NoRefTypesAttribute),
RefTypes::NoFile | RefTypes::List(_) => return Err(WriteError::NoRefTypes),
}
if matches!(cls, Value::Str(s) if s.as_bytes().is_empty()) {
return Ok(None);
}
self.find_ref((cls, ns, asm))
}
fn ref_node_boost(&mut self, m: &'a Map) -> Result<Option<Arc<Tree>>, WriteError> {
match self.ref_types {
RefTypes::List(_) => {}
RefTypes::Absent => return Err(WriteError::NoRefTypesAttribute),
RefTypes::NoFile => return Err(WriteError::NoRefTypes),
}
let (cls, ns, asm) = self.names(m)?;
if py_size_field_is_zero(cls) {
return Ok(None);
}
self.find_ref((cls, ns, asm))
}
}
#[cfg(test)]
mod tests;
#[cfg(test)]
mod work_tests;