use std::path::Path;
use std::sync::Arc;
use unityrs_core::budget::{Reservation, Unbounded};
use unityrs_core::detect::{FileType, check_file_type, check_file_type_with_reader};
use unityrs_core::env::{DependencyFile, EnvOptions, Environment, Source, read_path};
use unityrs_core::files::{BundleFile, File, parse_file_with_reader};
use unityrs_core::limits::{Charge, Limits, Meter, MeteredVec, heap_cost};
use unityrs_core::pystr::simplify_name;
use unityrs_core::stream::ReaderKind;
use unityrs_core::{Bytes, Error, FileContext, LimitError, LimitKind, RawStr};
use super::classify::{self, Class, Reason};
use super::seam::{self, Hooks, Registered};
use super::table;
use super::{Failure, View};
#[derive(Clone, Copy, Debug)]
pub enum Data<'a> {
Path(&'a Path),
Bytes(&'a Bytes),
}
#[derive(Clone, Copy, Debug)]
pub struct Staged<'a> {
pub id: &'a str,
pub name: &'a RawStr,
pub data: Data<'a>,
}
#[derive(Debug)]
pub struct Bits {
rows: u32,
packed: MeteredVec<u8>,
}
impl Bits {
pub fn charged(rows: u32, meter: &Arc<Meter>) -> Result<Self, Class> {
let n = usize::try_from(rows.div_ceil(8)).unwrap_or(usize::MAX);
let mut packed = table::with_capacity(meter, n)?;
for _ in 0..n {
table::push(&mut packed, 0u8)?;
}
Ok(Self { rows, packed })
}
pub fn copied(bits: &Self, meter: &Arc<Meter>, hooks: &dyn Hooks) -> Result<Self, Failure> {
let mut out = seam::probe(hooks, || Self::charged(bits.rows, meter)).map_err(own)?;
out.bytes_mut().copy_from_slice(bits.bytes());
Ok(out)
}
#[must_use]
pub fn rows(&self) -> u32 {
self.rows
}
#[must_use]
pub fn bytes(&self) -> &[u8] {
self.packed.as_slice()
}
pub fn bytes_mut(&mut self) -> &mut [u8] {
self.packed.as_mut_slice()
}
#[must_use]
pub fn get(&self, i: usize) -> bool {
self.packed
.as_slice()
.get(i / 8)
.is_some_and(|b| b & (1 << (i % 8)) != 0)
}
pub fn set(&mut self, i: usize) {
if let Some(b) = self.packed.as_mut_slice().get_mut(i / 8) {
*b |= 1 << (i % 8);
}
}
}
pub trait PeekBits {
fn load(&self, id: &str, meter: &Arc<Meter>, hooks: &dyn Hooks) -> Result<Bits, Failure>;
}
const PEEK: u64 = 21;
pub(super) fn dependency_failed(id: &str, c: Class) -> Failure {
let inner = match c {
Class::Run(r) => return Failure::Run(r),
Class::Skip(r) | Class::Input(r) => container(r),
};
Failure::Input(Reason {
reason: format!("dependency_failed({id}, {})", inner.reason),
detail: inner.detail,
cut: inner.cut,
})
}
fn container(r: Reason) -> Reason {
if r.reason == "decode_error" {
Reason {
reason: "container_error".to_owned(),
detail: r.detail,
cut: r.cut,
}
} else {
r
}
}
fn opened(e: &Error) -> Class {
match classify::error(e) {
Class::Skip(r) | Class::Input(r) => Class::Input(container(r)),
run @ Class::Run(_) => run,
}
}
fn own(c: Class) -> Failure {
Failure::from(c)
}
fn internal() -> Failure {
Failure::Run(Reason::plain("internal"))
}
fn open(
name: &RawStr,
data: Data<'_>,
context: &Arc<FileContext>,
meter: &Arc<Meter>,
hooks: &dyn Hooks,
) -> Result<(Bytes, Option<Charge>, Arc<BundleFile>), Class> {
let limits = &context.limits;
let (bytes, charge) = match data {
Data::Path(p) => {
let read = seam::probe(hooks, || {
read_path(p, limits, meter).map_err(|e| opened(&e))
});
(read?, None)
}
Data::Bytes(b) => {
let len = u64::try_from(b.len()).unwrap_or(u64::MAX);
if len > limits.max_input_bytes {
let e = Error::Limit(LimitError::refused(
LimitKind::InputBytes,
len,
limits.max_input_bytes,
));
return Err(opened(&e));
}
let charge = seam::probe(hooks, || {
meter.charge(heap_cost(len)).map_err(|e| table::refused(&e))
})?;
(b.clone(), Some(charge))
}
};
let ty = seam::probe(hooks, || {
check_file_type_with_reader(&bytes, limits, ReaderKind::Stream)
.map_err(|e| opened(&e.into()))
})?;
let file = seam::probe(hooks, || {
parse_file_with_reader(
name.clone(),
bytes.clone(),
ty,
None,
meter,
context,
ReaderKind::Stream,
)
.map_err(|e| opened(&e))
})?;
match file {
File::Bundle(b) => Ok((bytes, charge, b)),
_ => Err(Class::Input(Reason::plain("not_a_bundle"))),
}
}
fn is_bundle_entry(b: &BundleFile, i: usize) -> Result<bool, Error> {
match b.entry_range(i, 0..PEEK) {
Ok(head) => match check_file_type(&head, &Limits::default()) {
Ok(ty) => Ok(ty == FileType::BundleFile),
Err(e @ unityrs_core::detect::DetectError::Allocator(_)) => Err(e.into()),
Err(_) => Ok(false),
},
Err(e @ (Error::Limit(_) | Error::Io(_) | Error::Budget(_))) => Err(e),
Err(e) => match classify::error(&e) {
Class::Run(_) => Err(e),
Class::Skip(_) | Class::Input(_) => Ok(false),
},
}
}
fn peeked(c: Class) -> Failure {
match c {
Class::Run(r) => Failure::Run(r),
Class::Skip(r) | Class::Input(r) if r.reason.starts_with("budget(") => {
Failure::Run(Reason::plain("host_memory"))
}
Class::Skip(r) | Class::Input(r) => Failure::Input(r),
}
}
pub fn peek(
name: &RawStr,
data: Data<'_>,
context: &Arc<FileContext>,
meter: &Arc<Meter>,
hooks: &dyn Hooks,
) -> Result<Bits, Failure> {
let (_, _held, b) = open(name, data, context, meter, hooks).map_err(peeked)?;
let n = b.directory().len();
let rows = u32::try_from(n).map_err(|_| internal())?;
let mut bits = seam::probe(hooks, || Bits::charged(rows, meter)).map_err(peeked)?;
for i in 0..n {
if hooks.stopped() {
return Err(Failure::Stopped);
}
if seam::probe(hooks, || is_bundle_entry(&b, i).map_err(|e| opened(&e))).map_err(peeked)? {
bits.set(i);
}
}
Ok(bits)
}
fn call<T>(
env: &Environment,
admit: &Admit<'_>,
f: impl FnOnce() -> Result<T, Error>,
) -> Result<Result<T, Error>, Failure> {
seam::allocator_check(admit.hooks).map_err(Failure::from)?;
let before = admit.hooks.refusals();
let out = f();
#[cfg(feature = "test-hooks")]
let out = match admit.hooks.environment_fault(env) {
Some(error) => Err(error),
None => out,
};
admit.hooks.environment(env);
seam::allocator_check(admit.hooks).map_err(Failure::from)?;
if let Err(e) = &out
&& let Class::Run(r) = seam::probed(admit.hooks, before, classify::error(e))
{
return Err(Failure::Run(r));
}
seam::evictions(env, admit.max_evictions).map_err(own)?;
Ok(out)
}
#[derive(Clone, Copy)]
enum Whose<'a> {
Own,
Member(&'a str),
}
impl Whose<'_> {
fn failed(self, e: &Error, path: &RawStr, admit: &Admit<'_>, before: Option<u64>) -> Failure {
let c = seam::probed(admit.hooks, before, classify::error(e));
match (self, c) {
(_, Class::Run(r)) => Failure::Run(r),
(Self::Own, _) => own(seam::probed(
admit.hooks,
before,
classify::entry_error(e, path),
)),
(Self::Member(id), c) => dependency_failed(id, c),
}
}
}
fn reach(
env: &Environment,
b: &BundleFile,
bits: &Bits,
whose: Whose<'_>,
admit: &Admit<'_>,
) -> Result<(), Failure> {
let meter = admit.meter;
let dir = b.directory();
let _sample = super::read::Sample(b.meter(), admit.hooks);
let n = dir.len();
if u32::try_from(n).ok() != Some(bits.rows()) {
return Err(internal());
}
let hooks = admit.hooks;
let mut keys: MeteredVec<u8> = MeteredVec::new(meter);
let mut ends: MeteredVec<usize> =
seam::probe(hooks, || table::with_capacity(meter, n)).map_err(own)?;
for row in dir {
let key = seam::probe(hooks, || {
simplify_name(&row.path, meter).map_err(|e| table::refused(&e))
})
.map_err(own)?;
seam::probe(hooks, || {
keys.extend_from_slice(key.as_bytes())
.map_err(|e| table::refused(&e))
})
.map_err(own)?;
let end = keys.len();
seam::probe(hooks, || table::push(&mut ends, end)).map_err(own)?;
}
let key = |i: u32| {
let i = usize::try_from(i).unwrap_or(usize::MAX);
let start = i
.checked_sub(1)
.and_then(|p| ends.as_slice().get(p))
.copied();
let end = ends.as_slice().get(i).copied().unwrap_or(0);
keys.as_slice()
.get(start.unwrap_or(0)..end)
.unwrap_or_default()
};
let mut order: MeteredVec<u32> =
seam::probe(hooks, || table::with_capacity(meter, n)).map_err(own)?;
for i in 0..bits.rows() {
seam::probe(hooks, || table::push(&mut order, i)).map_err(own)?;
}
order
.as_mut_slice()
.sort_unstable_by(|a, b| key(*a).cmp(key(*b)).then(a.cmp(b)));
let mut dropped = seam::probe(hooks, || Bits::charged(bits.rows(), meter)).map_err(own)?;
for pair in order.as_slice().windows(2) {
if let [a, b] = pair
&& key(*a) == key(*b)
{
dropped.set(usize::try_from(*a).unwrap_or(usize::MAX));
}
}
drop(order);
for (i, row) in dir.iter().enumerate() {
if !bits.get(i) || dropped.get(i) {
continue;
}
let before = admit.hooks.refusals();
if let Err(e) = call(env, admit, || env.get_cab(&row.path, meter))? {
return Err(whose.failed(&e, &row.path, admit, before));
}
}
Ok(())
}
type Loaded<'a> = dyn FnMut(&str, &Arc<BundleFile>) -> Result<(), Failure> + 'a;
type WithHandle<'a> = dyn FnMut(&str, &Arc<BundleFile>, DependencyFile) -> Result<(), Failure> + 'a;
#[derive(Clone, Copy)]
pub struct Admit<'a> {
pub env: &'a EnvOptions,
pub meter: &'a Arc<Meter>,
pub hooks: &'a dyn Hooks,
pub max_evictions: u64,
}
pub struct Registration<'a> {
admit: Admit<'a>,
input: Staged<'a>,
env: Environment,
own: Arc<BundleFile>,
bytes: Bytes,
held: Option<Charge>,
ids: &'a [&'a str],
next: usize,
}
fn in_order(ids: &[&str], x: &str) -> bool {
ids.windows(2)
.all(|w| matches!(w, [a, b] if a.as_bytes() < b.as_bytes()))
&& !ids.contains(&x)
}
impl<'a> Registration<'a> {
pub fn open(input: Staged<'a>, ids: &'a [&'a str], admit: Admit<'a>) -> Result<Self, Failure> {
if !in_order(ids, input.id) {
return Err(internal());
}
if admit.hooks.stopped() {
return Err(Failure::Stopped);
}
let context = &admit.env.context;
let (bytes, held, own) = open(input.name, input.data, context, admit.meter, admit.hooks)?;
Ok(Self {
admit,
input,
env: Environment::new(admit.env.clone()),
own,
bytes,
held,
ids,
next: 0,
})
}
#[must_use]
pub fn stats(&self) -> unityrs_core::env::EnvStats {
self.env.stats()
}
#[must_use]
pub fn own(&self) -> &Arc<BundleFile> {
&self.own
}
pub fn member(
&mut self,
m: &Staged<'a>,
bits: &dyn PeekBits,
loaded: &mut Loaded<'_>,
) -> Result<(), Failure> {
self.member_with_handle(m, bits, &mut |id, bundle, _| loaded(id, bundle))
}
pub fn member_with_handle(
&mut self,
m: &Staged<'a>,
bits: &dyn PeekBits,
loaded: &mut WithHandle<'_>,
) -> Result<(), Failure> {
let admit = self.admit;
if admit.hooks.stopped() {
return Err(Failure::Stopped);
}
if self.ids.get(self.next) != Some(&m.id) {
return Err(internal());
}
self.next += 1;
let whose = Whose::Member(m.id);
let env = &self.env;
let before = admit.hooks.refusals();
let source = match m.data {
Data::Bytes(b) => Source::Bytes {
name: m.name.clone(),
data: b.clone(),
},
Data::Path(p) => Source::Path(p.to_path_buf()),
};
let (file, handle) = call(env, &admit, || env.load_dependency_with_handle(source))?
.map_err(|e| whose.failed(&e, m.name, &admit, before))?;
let File::Bundle(b) = file else {
return Err(dependency_failed(
m.id,
Class::Input(Reason::plain("not_a_bundle")),
));
};
loaded(m.id, &b, handle)?;
let bits = bits.load(m.id, admit.meter, admit.hooks)?;
reach(env, &b, &bits, whose, &admit)
}
pub fn finish(
self,
bits: &dyn PeekBits,
loaded: &mut Loaded<'_>,
) -> Result<Registered, Failure> {
self.finish_with_handle(bits, &mut |id, bundle, _| loaded(id, bundle))
}
pub fn finish_with_handle(
self,
bits: &dyn PeekBits,
loaded: &mut WithHandle<'_>,
) -> Result<Registered, Failure> {
let admit = self.admit;
if self.next != self.ids.len() {
return Err(internal());
}
if admit.hooks.stopped() {
return Err(Failure::Stopped);
}
let env = &self.env;
let before = admit.hooks.refusals();
let source = Source::Bytes {
name: self.input.name.clone(),
data: self.bytes.clone(),
};
let (file, handle) = call(env, &admit, || env.load_dependency_with_handle(source))?
.map_err(
|e| match seam::probed(admit.hooks, before, classify::error(&e)) {
Class::Skip(r) | Class::Input(r) => Failure::Input(container(r)),
Class::Run(r) => Failure::Run(r),
},
)?;
let File::Bundle(slot) = file else {
return Err(Failure::Input(Reason::plain("not_a_bundle")));
};
same_open(&slot, &self.own)?;
loaded(self.input.id, &slot, handle)?;
let bits = bits.load(self.input.id, admit.meter, admit.hooks)?;
reach(env, &slot, &bits, Whose::Own, &admit)?;
drop(slot);
Ok(Registered {
env: self.env,
own: self.own,
_held: self.held,
})
}
}
pub fn same_open(slot: &BundleFile, own: &BundleFile) -> Result<(), Failure> {
if slot.header() != own.header() || slot.directory() != own.directory() {
return Err(Failure::Input(container(Reason::decode_error("malformed"))));
}
Ok(())
}
pub fn register(
input: Staged<'_>,
members: &[Staged<'_>],
bits: &dyn PeekBits,
admit: Admit<'_>,
loaded: &mut Loaded<'_>,
) -> Result<Registered, Failure> {
let mut sorted: Vec<&Staged<'_>> = members.iter().filter(|m| m.id != input.id).collect();
sorted.sort_by(|a, b| a.id.as_bytes().cmp(b.id.as_bytes()));
let ids: Vec<&str> = sorted.iter().map(|m| m.id).collect();
let mut reg = Registration::open(input, &ids, admit)?;
for m in sorted {
reg.member(m, bits, loaded)?;
}
reg.finish(bits, loaded)
}
struct Peeked<'a> {
ids: Vec<&'a str>,
bits: Vec<Bits>,
}
impl PeekBits for Peeked<'_> {
fn load(&self, id: &str, meter: &Arc<Meter>, hooks: &dyn Hooks) -> Result<Bits, Failure> {
let Some(i) = self.ids.iter().position(|x| *x == id) else {
return Err(internal());
};
let Some(peeked) = self.bits.get(i) else {
return Err(internal());
};
Bits::copied(peeked, meter, hooks)
}
}
pub fn register_bytes(view: View<'_>, admit: Admit<'_>) -> Result<Registered, Failure> {
let context = &admit.env.context;
let mut sorted: Vec<&super::Member> = view
.members
.iter()
.filter(|m| m.id != view.input.id)
.collect();
sorted.sort_by(|a, b| a.id.as_bytes().cmp(b.id.as_bytes()));
let ids: Vec<&str> = sorted.iter().map(|m| m.id.as_str()).collect();
if !in_order(&ids, &view.input.id) {
return Err(internal());
}
let peek_of = |m: &super::Member| {
let meter = Meter::for_input(&context.limits, Arc::new(Unbounded), Reservation::Input);
peek(
&RawStr::from(m.id.as_str()),
Data::Bytes(&m.data),
context,
&meter,
admit.hooks,
)
};
let mut peeked = Peeked {
ids: vec![view.input.id.as_str()],
bits: vec![peek_of(view.input)?],
};
for m in &sorted {
match peek_of(m) {
Ok(b) => {
peeked.ids.push(&m.id);
peeked.bits.push(b);
}
Err(Failure::Input(r)) => return Err(dependency_failed(&m.id, Class::Input(r))),
Err(f) => return Err(f),
}
}
let names: Vec<RawStr> = sorted.iter().map(|m| RawStr::from(m.id.as_str())).collect();
let staged: Vec<Staged<'_>> = sorted
.iter()
.zip(&names)
.map(|(m, name)| Staged {
id: &m.id,
name,
data: Data::Bytes(&m.data),
})
.collect();
let x_name = RawStr::from(view.input.id.as_str());
let input = Staged {
id: &view.input.id,
name: &x_name,
data: Data::Bytes(&view.input.data),
};
register(input, &staged, &peeked, admit, &mut |_, _| Ok(()))
}