use std::sync::Arc;
use bytes::Bytes;
use super::dict::{Dict, FindError, RefIndex, RefNode, Trees, ref_tree, visit};
use super::{
FloatMode, Out, ReadCtx, ReadDetail, ReadError, ReadErrorKind, RefTypes, Work, charge_error,
f32_bits, int, limit_refused, stream_refused, uint,
};
use crate::raw::RawStr;
use crate::stream::{Endian, ReaderKind};
use crate::typetree::{Kind, NodeId, Tree};
use crate::value::{ListIx, Parsed, ReadBy, Value, ValueBuilder};
pub fn read(
file: &Bytes,
start: i128,
tree: &Arc<Tree>,
ctx: &ReadCtx<'_>,
) -> Result<Parsed, ReadError> {
read_with_reader(file, start, tree, ctx, ReaderKind::Memory)
}
pub fn read_with_reader(
file: &Bytes,
start: i128,
tree: &Arc<Tree>,
ctx: &ReadCtx<'_>,
kind: ReaderKind,
) -> Result<Parsed, ReadError> {
read_with_mode(file, start, tree, ctx, kind, (false, false))
}
pub(crate) fn read_with_mode(
file: &Bytes,
start: i128,
tree: &Arc<Tree>,
ctx: &ReadCtx<'_>,
kind: ReaderKind,
(head, object_mode): (bool, bool),
) -> Result<Parsed, ReadError> {
read_with_fields(file, start, tree, ctx, kind, (head, object_mode), None)
}
pub(crate) fn read_with_fields(
file: &Bytes,
start: i128,
tree: &Arc<Tree>,
ctx: &ReadCtx<'_>,
kind: ReaderKind,
(head, object_mode): (bool, bool),
progress: Option<&mut super::FieldProgress>,
) -> Result<Parsed, ReadError> {
let work = Work::start(ctx.options.limits, ctx.meter)?;
let builder =
ValueBuilder::new(ctx.meter, tree.list().clone()).map_err(|e| charge_error(0, e))?;
let trees = Trees::new(&builder, tree)?;
let mut r = Reader {
data: file,
bytes: file.as_ref(),
pos: start,
start,
kind,
head,
object_mode,
progress,
big: ctx.endian == Endian::Big,
floats: ctx.options.floats,
ref_types: ctx.ref_types,
refs: RefIndex::new(ctx.ref_types),
builder,
trees,
work,
has_registry: false,
};
let value = r.value(tree, 0, tree.root());
let at = r.at();
let ended = r.work.finish(at);
let value = value?;
ended?;
let built = value.into_built(&r.builder, at)?;
let read = r.pos.saturating_sub(start);
let bytes_read = i64::try_from(read).unwrap_or(if read < 0 { i64::MIN } else { i64::MAX });
let Reader { builder, .. } = r;
Ok(builder
.finish(built, bytes_read)
.with_read_by(ReadBy::Python))
}
#[allow(clippy::struct_excessive_bools)] struct Reader<'a, 'c, 'p> {
data: &'a Bytes,
bytes: &'a [u8],
pos: i128,
start: i128,
kind: ReaderKind,
head: bool,
object_mode: bool,
progress: Option<&'p mut super::FieldProgress>,
big: bool,
floats: FloatMode,
ref_types: RefTypes<'c>,
refs: RefIndex<'c>,
builder: ValueBuilder,
trees: Trees,
work: Work<'c>,
has_registry: bool,
}
macro_rules! num {
($big:expr, $t:ty, $b:expr) => {
if $big {
<$t>::from_be_bytes($b)
} else {
<$t>::from_le_bytes($b)
}
};
}
mod array;
#[derive(Clone, Copy)]
enum Py {
Struct(u64),
Index,
Other(&'static str),
Limit(crate::error::LimitError),
}
impl Reader<'_, '_, '_> {
fn at(&self) -> u64 {
u64::try_from(self.pos.saturating_sub(self.start)).unwrap_or(0)
}
fn fail(&self, kind: ReadErrorKind, detail: ReadDetail) -> ReadError {
ReadError {
kind,
at: self.at(),
detail,
}
}
fn raise(&self, e: Py) -> ReadError {
match e {
Py::Struct(need) => self.fail(
ReadErrorKind::Other,
ReadDetail::Eof {
need,
exception: "error",
},
),
Py::Index => self.fail(ReadErrorKind::Other, ReadDetail::Python("IndexError")),
Py::Other(m) => self.fail(ReadErrorKind::Other, ReadDetail::Python(m)),
Py::Limit(l) => limit_refused(self.at(), l),
}
}
fn read(&mut self, size: i64) -> Result<std::ops::Range<usize>, ReadError> {
let (range, next) = self
.kind
.read(self.bytes.len(), self.pos, size)
.map_err(|e| stream_refused(self.at(), e))?;
if next < self.pos {
self.work.discard();
}
self.pos = next;
Ok(range)
}
fn read_n<const N: usize>(&mut self) -> Result<[u8; N], ReadError> {
let range = self.read(i64::try_from(N).unwrap_or(i64::MAX))?;
self.bytes
.get(range)
.and_then(|s| <[u8; N]>::try_from(s).ok())
.ok_or_else(|| self.raise(Py::Struct(N as u64)))
}
fn read1(&mut self) -> Result<u8, ReadError> {
let [b] = self.read_n::<1>()?;
Ok(b)
}
fn unpack_from<const N: usize>(&mut self) -> Result<[u8; N], ReadError> {
let (at, next) = self.kind.unpack(self.bytes.len(), self.pos, N);
let b = at
.and_then(|at| self.bytes.get(at..at.checked_add(N)?))
.and_then(|s| <[u8; N]>::try_from(s).ok());
let Some(b) = b else {
let e = self.raise(Py::Struct(N as u64));
self.pos = next;
return Err(e);
};
self.pos = next;
Ok(b)
}
fn read_int(&mut self) -> Result<i32, ReadError> {
let big = self.big;
Ok(num!(big, i32, self.unpack_from()?))
}
fn frame<R>(
&mut self,
f: impl FnOnce(&mut Self) -> Result<R, ReadError>,
) -> Result<R, ReadError> {
self.work.enter(self.at())?;
let out = f(self);
self.work.leave();
out
}
fn align(&mut self) -> Result<(), ReadError> {
self.pos = self
.kind
.align(self.pos, 4)
.map_err(|e| stream_refused(self.at(), e))?;
Ok(())
}
fn prim(&mut self, kind: Kind) -> Result<Value, ReadError> {
let big = self.big;
Ok(match kind {
Kind::S8 => int(i64::from(self.read1()?.cast_signed())),
Kind::U8 => int(i64::from(self.read1()?)),
Kind::Bool => Value::Bool(self.read1()? != 0),
Kind::S16 => int(i64::from(num!(big, i16, self.read_n()?))),
Kind::U16 => int(i64::from(num!(big, u16, self.read_n()?))),
Kind::S32 => int(i64::from(num!(big, i32, self.read_n()?))),
Kind::U32 => int(i64::from(num!(big, u32, self.read_n()?))),
Kind::S64 => int(num!(big, i64, self.read_n()?)),
Kind::U64 => uint(num!(big, u64, self.read_n()?)),
Kind::F32 => Value::F32(f32_bits(num!(big, u32, self.read_n()?), self.floats)),
Kind::F64 => Value::F64(num!(big, u64, self.read_n()?)),
_ => return Err(self.raise(Py::Other("not a primitive"))),
})
}
fn string(&mut self) -> Result<Value, ReadError> {
let len = self.read_int()?;
let room = i128::try_from(self.bytes.len())
.unwrap_or(i128::MAX)
.saturating_sub(self.pos);
if len > 0 && i128::from(len) <= room {
let range = self.read(i64::from(len))?;
let s = RawStr::from_shared(self.data.slice(range));
self.align()?;
return Ok(Value::Str(s));
}
Ok(Value::Str(RawStr::default()))
}
fn typeless(&mut self) -> Result<Value, ReadError> {
let len = self.read_int()?;
let range = self.read(i64::from(len))?;
Ok(Value::Bytes(self.data.slice(range)))
}
fn value(&mut self, t: &Arc<Tree>, lx: ListIx, node: NodeId) -> Result<Out, ReadError> {
if self.progress.as_ref().is_some_and(|p| p.error.is_some()) {
return Ok(Out::Plain(Value::Int(0)));
}
let at = self.at();
self.work.enter(at)?;
let out = self.value_inner(t, lx, node);
self.work.leave();
self.recover(out)
}
fn recover(&mut self, out: Result<Out, ReadError>) -> Result<Out, ReadError> {
match (out, &mut self.progress) {
(Err(error), Some(progress)) => progress.recover(error),
(out, _) => out,
}
}
fn child(t: &Tree, node: NodeId, i: usize) -> Result<NodeId, Py> {
t.list().children(node).get(i).copied().ok_or(Py::Index)
}
fn array_child(t: &Tree, node: NodeId) -> Option<NodeId> {
let first = *t.list().children(node).first()?;
(t.py_kind(first) == Kind::Array).then_some(first)
}
fn value_inner(&mut self, t: &Arc<Tree>, lx: ListIx, node: NodeId) -> Result<Out, ReadError> {
self.work.values(1, self.at())?;
let mut align = t.info(node).is_some_and(|i| i.align);
let kind = t.py_kind(node);
let out = if kind.prim_size().is_some() {
Out::Plain(self.prim(kind)?)
} else if kind == Kind::Str {
Out::Plain(self.string()?)
} else if kind == Kind::Bytes {
Out::Plain(self.typeless()?)
} else if kind == Kind::Pair {
let a = Self::child(t, node, 0).map_err(|e| self.raise(e))?;
let first = self.value(t, lx, a)?;
let second = match Self::child(t, node, 1) {
Ok(b) => self.value(t, lx, b)?,
Err(error) => {
let error = self.raise(error);
self.recover(Err(error))?;
return self.tuple([first]);
}
};
self.tuple(vec_of2(first, second))?
} else if kind == Kind::ReferencedObject {
self.ref_object(t, lx, node, false)?
} else if let Some(array) = Self::array_child(t, node) {
align |= t.info(array).is_some_and(|i| i.align);
let size = self.read_int()?;
let Ok(count) = usize::try_from(size) else {
return Err(self.fail(
ReadErrorKind::NegativeLength,
ReadDetail::NegativeLength(i64::from(size)),
));
};
let elem = Self::child(t, array, 1).map_err(|e| self.raise(e))?;
if t.info(elem).is_some_and(|i| i.align) {
self.array(t, lx, elem, i64::from(size))?
} else {
self.frame(|r| r.list(t, lx, elem, count))?
}
} else {
self.class(t, lx, node)?
};
if align {
self.align()?;
}
Ok(out)
}
fn tuple<const N: usize>(&mut self, items: [Out; N]) -> Result<Out, ReadError> {
let at = self.at();
let mut v = self.builder.vec();
v.reserve_exact(N).map_err(|e| charge_error(at, e))?;
for item in items {
item.push_to(&mut v).map_err(|e| charge_error(at, e))?;
}
let built = self.builder.tuple(v).map_err(|e| charge_error(at, e))?;
Ok(Out::Built(built))
}
fn remaining(&self) -> usize {
let len = i128::try_from(self.bytes.len()).unwrap_or(i128::MAX);
usize::try_from(len.saturating_sub(self.pos).max(0)).unwrap_or(usize::MAX)
}
fn class(&mut self, t: &Arc<Tree>, lx: ListIx, node: NodeId) -> Result<Out, ReadError> {
self.dict(t, lx, node, true)
}
fn plain_class(&mut self, t: &Arc<Tree>, lx: ListIx, node: NodeId) -> Result<Out, ReadError> {
self.frame(|r| r.dict(t, lx, node, false))
}
fn dict(
&mut self,
t: &Arc<Tree>,
lx: ListIx,
node: NodeId,
registry: bool,
) -> Result<Out, ReadError> {
let at = self.at();
let kids = t.list().children(node);
let is_registry = |c: NodeId| registry && t.py_kind(c) == Kind::ManagedReferencesRegistry;
let dup = t.info(node).is_some_and(|i| i.dup_names);
let mut d = Dict::new(
&self.builder,
kids.len(),
dup && self.progress.is_none(),
at,
)?;
let outer = self.has_registry;
for c in kids {
visit();
if is_registry(*c) {
if self.has_registry {
self.work.skipped(self.at())?;
continue;
}
self.has_registry = true;
}
let v = self.value(t, lx, *c)?;
let (key, slot) = t.info(*c).map_or((*c, 0), |i| (i.key, i.slot));
let at = self.at();
d.set(
&mut self.work,
if self.progress.is_some() { *c } else { key },
slot,
v,
at,
)?;
if self.progress.as_ref().is_some_and(|p| p.error.is_some()) {
break;
}
}
self.has_registry = outer;
if self.object_mode
&& self.progress.is_none()
&& t.list().type_name(node).starts_with(b"PPtr<")
{
self.work
.open
.steps(kids.len())
.map_err(|e| limit_refused(self.at(), e))?;
let has = |name: &[u8]| {
kids.iter().any(|&c| {
if registry {
crate::walk::write::cleans_to(t.list().name(c), name)
} else {
t.list().name(c) == name
}
})
};
if !has(b"m_FileID") || !has(b"m_PathID") {
return Err(self.raise(Py::Other(if !registry {
"TypeError"
} else if !has(b"m_FileID") {
"KeyError: m_FileID"
} else {
"KeyError: m_PathID"
})));
}
if !registry
&& kids
.iter()
.any(|&c| !matches!(t.list().name(c), b"m_FileID" | b"m_PathID"))
{
return Err(self.raise(Py::Other("TypeError")));
}
}
let result = d.finish(&self.builder, lx, self.at())?;
if let Some(progress) = &mut self.progress {
progress.finish_class();
}
Ok(result)
}
fn ref_object(
&mut self,
t: &Arc<Tree>,
lx: ListIx,
node: NodeId,
in_array: bool,
) -> Result<Out, ReadError> {
let at = self.at();
let kids = t.list().children(node);
let mut d = Dict::new(&self.builder, kids.len(), true, at)?;
d.preserve_fields(self.progress.is_some());
let mut type_slot = None;
let mut object_type = false;
let mut resolved: Option<Option<RefNode>> = None;
for c in kids {
visit();
let (key, slot) = t.info(*c).map_or((*c, 0), |i| (i.key, i.slot));
if type_slot.is_none() && t.info(*c).is_some() && t.list().name(*c) == b"type" {
type_slot = Some(slot);
}
if t.list().name(*c) == b"type" {
object_type = self.object_mode && t.py_kind(*c) != Kind::ReferencedObject;
}
let v = if t.py_kind(*c) == Kind::ReferencedObjectData {
let node = if let Some(r) = &resolved {
r.clone()
} else {
let ty = type_slot
.and_then(|s| d.position(s))
.and_then(|p| d.entries.as_slice().get(p))
.map(|(_, v)| v);
let found = match self.ref_node(ty, object_type) {
Ok(found) => found,
Err(error) => {
if let Some(progress) = &mut self.progress {
progress.recover(error)?;
break;
}
return Err(error);
}
};
resolved.insert(found).clone()
};
match (node, in_array) {
(Some(RefNode::Tree(rt)), _) => {
let at = self.at();
let rlx = self.trees.index(&mut self.builder, &rt, at)?;
self.value(&rt, rlx, rt.root())?
}
(None | Some(RefNode::NoTree), false) => {
self.work.skipped(self.at())?;
continue;
}
(None | Some(RefNode::NoTree), true) => {
let error = self.fail(ReadErrorKind::Other, ReadDetail::RefTypeNoTree);
if let Some(progress) = &mut self.progress {
progress.recover(error)?;
break;
}
return Err(error);
}
}
} else {
self.value(t, lx, *c)?
};
let at = self.at();
d.set(
&mut self.work,
if self.progress.is_some() { *c } else { key },
slot,
v,
at,
)?;
if self.progress.as_ref().is_some_and(|p| p.error.is_some()) {
break;
}
if Some(slot) == type_slot {
resolved = None;
}
}
let result = d.finish(&self.builder, lx, self.at())?;
if let Some(progress) = &mut self.progress {
progress.finish_class();
}
Ok(result)
}
fn ref_node(
&mut self,
ty: Option<&Value>,
object_type: bool,
) -> Result<Option<RefNode>, ReadError> {
let Some(ty) = ty else {
return Err(self.raise(Py::Other("KeyError: 'type'")));
};
let Value::Dict(m) = ty else {
return Err(self.raise(Py::Other("AttributeError: no attribute 'class'")));
};
let at = self.at();
let mut field = |name: &'static [u8], msg: &'static str| match self.trees.field(
m,
name,
&mut self.work.open,
object_type,
) {
Ok(Some(v)) => Ok(v),
Ok(None) if object_type => Err(Py::Other("AttributeError")),
Ok(None) => Err(Py::Other(msg)),
Err(l) => Err(Py::Limit(l)),
};
let fields = (|| {
Ok((
field(b"class", "KeyError: 'class'")?,
field(b"ns", "KeyError: 'ns'")?,
field(b"asm", "KeyError: 'asm'")?,
))
})();
let (cls, ns, asm) = fields.map_err(|e| match e {
Py::Limit(l) => limit_refused(at, l),
e => self.raise(e),
})?;
if !matches!(self.ref_types, RefTypes::List(l) if !l.is_empty()) {
return Err(self.fail(
ReadErrorKind::RefType,
ReadDetail::NoRefTypes {
reader: ReadBy::Python,
absent: matches!(self.ref_types, RefTypes::Absent),
},
));
}
if matches!(cls, Value::Str(s) if s.is_empty()) {
return Ok(None);
}
let found = self
.refs
.find(self.builder.meter(), &mut self.work.open, cls, ns, asm)
.map_err(|e| match e {
FindError::Charge(e) => charge_error(at, e),
FindError::Work(l) => limit_refused(at, l),
})?;
if let Some(r) = found {
Ok(Some(ref_tree(r, &self.work.limits, at)?))
} else {
if let (Some(progress), Value::Str(cls), Value::Str(ns), Value::Str(asm)) =
(&mut self.progress, cls, ns, asm)
{
progress.reference_names = Some([cls.clone(), ns.clone(), asm.clone()]);
}
Err(self.fail(ReadErrorKind::RefType, ReadDetail::RefTypeNotFound))
}
}
}
fn vec_of2(a: Out, b: Out) -> [Out; 2] {
[a, b]
}