use crate::container::{ObjectSource, ParsedDescriptor};
use crate::error::{Error, Result};
use crate::limits::Limits;
use crate::materialize::observation::{select_ops, selection_references, serve_selection};
use crate::store::{NodeId, SeedStore};
use super::derive;
use super::node::{NodeKind, SeedNode, read_object_params, read_span_params, read_u32_params};
pub trait SourceServer {
fn serve_range(&self, offset: u64, len: u64, limits: Limits) -> Result<Vec<u8>>;
fn serve_document(&self, limits: Limits) -> Result<Vec<u8>>;
}
impl SourceServer for ParsedDescriptor {
fn serve_range(&self, offset: u64, len: u64, limits: Limits) -> Result<Vec<u8>> {
serve_source_range(self, offset, len, limits)
}
fn serve_document(&self, limits: Limits) -> Result<Vec<u8>> {
crate::materialize::materialize(self, limits)
}
}
pub const MAX_EVAL_NODES: u64 = 1 << 20;
pub const MAX_EVAL_BYTES: u64 = 1 << 32;
#[derive(Debug, Clone)]
pub struct EvalBudget {
pub max_nodes: u64,
pub max_bytes: u64,
pub nodes: u64,
pub produced: u64,
}
impl Default for EvalBudget {
fn default() -> Self {
EvalBudget {
max_nodes: MAX_EVAL_NODES,
max_bytes: MAX_EVAL_BYTES,
nodes: 0,
produced: 0,
}
}
}
impl EvalBudget {
fn charge_node(&mut self) -> Result<()> {
self.nodes = self
.nodes
.checked_add(1)
.ok_or_else(|| Error::resource_limit("seed evaluation node count overflow"))?;
if self.nodes > self.max_nodes {
return Err(Error::resource_limit(format!(
"seed evaluation exceeded {} nodes",
self.max_nodes
)));
}
Ok(())
}
pub(crate) fn charge_bytes(&mut self, n: u64) -> Result<()> {
self.produced = self
.produced
.checked_add(n)
.ok_or_else(|| Error::resource_limit("seed evaluation byte count overflow"))?;
if self.produced > self.max_bytes {
return Err(Error::resource_limit(format!(
"seed evaluation exceeded {} bytes",
self.max_bytes
)));
}
Ok(())
}
}
pub fn load_node(store: &dyn SeedStore, id: &NodeId) -> Result<SeedNode> {
let bytes = store.get_node(id)?;
let node = SeedNode::decode_canonical(&bytes)?;
if node.content_id() != *id {
return Err(Error::integrity_mismatch(format!(
"seed node {id} decoded to a different content id"
)));
}
node.check_limits(&Limits::DEFAULT)?;
Ok(node)
}
pub fn deps_of_canonical(bytes: &[u8]) -> Result<Vec<NodeId>> {
Ok(SeedNode::decode_canonical(bytes)?.deps)
}
fn serve_source_range(
parsed: &ParsedDescriptor,
offset: u64,
len: u64,
limits: Limits,
) -> Result<Vec<u8>> {
let d = &parsed.descriptor;
if d.objects
.iter()
.any(|o| matches!(o, ObjectSource::External { .. }))
{
return Err(Error::unsupported_feature(
"field v1 requires an inline descriptor (no external objects)",
));
}
let end = offset
.checked_add(len)
.ok_or_else(|| Error::usage("source slice end overflows"))?;
if end > d.source_len {
return Err(Error::usage(format!(
"source slice {offset}..{end} exceeds source length {}",
d.source_len
)));
}
let objects: Vec<Vec<u8>> = d
.objects
.iter()
.map(|o| o.as_inline().unwrap_or(&[]).to_vec())
.collect();
let object_lens: Vec<u64> = objects.iter().map(|o| o.len() as u64).collect();
let channel_lens: Vec<u64> = d.channels.iter().map(|c| c.decoded_length).collect();
let window = select_ops(&d.program, &object_lens, &channel_lens, offset, end, limits)?;
let (objects_used, channels_used) =
selection_references(&window.ops, objects.len(), d.channels.len());
let served = serve_selection(
&objects,
&d.channels,
&d.models,
window,
&objects_used,
&channels_used,
offset,
end,
limits,
)?;
Ok(served.bytes)
}
pub trait OutputCache {
fn get(&self, id: &NodeId) -> Result<Option<Vec<u8>>>;
fn put(&mut self, id: &NodeId, bytes: &[u8]) -> Result<()>;
}
#[derive(Debug, Clone, Copy, Default)]
pub struct NoCache;
impl OutputCache for NoCache {
fn get(&self, _id: &NodeId) -> Result<Option<Vec<u8>>> {
Ok(None)
}
fn put(&mut self, _id: &NodeId, _bytes: &[u8]) -> Result<()> {
Ok(())
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ReuseStats {
pub nodes_executed: u64,
pub nodes_reused: u64,
pub cache_bytes_written: u64,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct InverseWork {
pub node_executions: u64,
pub input_bytes: u64,
}
impl InverseWork {
pub const fn new(node_executions: u64, input_bytes: u64) -> Self {
InverseWork {
node_executions,
input_bytes,
}
}
pub const fn units(self) -> u64 {
self.node_executions.saturating_add(self.input_bytes)
}
}
pub fn retained_inverse_work_fraction(cold: InverseWork, warm: InverseWork) -> f64 {
let total = cold.units();
if total == 0 {
return 1.0;
}
let reused = cold
.node_executions
.saturating_sub(warm.node_executions)
.saturating_add(cold.input_bytes.saturating_sub(warm.input_bytes));
reused as f64 / total as f64
}
pub fn materialize_node(
parsed: &ParsedDescriptor,
store: &dyn SeedStore,
node: &SeedNode,
limits: Limits,
budget: &mut EvalBudget,
depth: u16,
) -> Result<Vec<u8>> {
let mut cache = NoCache;
let mut reuse = ReuseStats::default();
materialize_inner(
parsed, store, &mut cache, node, limits, budget, depth, &mut reuse,
)
}
pub fn materialize_node_with(
source: &dyn SourceServer,
store: &dyn SeedStore,
node: &SeedNode,
limits: Limits,
budget: &mut EvalBudget,
depth: u16,
) -> Result<Vec<u8>> {
let mut cache = NoCache;
let mut reuse = ReuseStats::default();
materialize_inner(
source, store, &mut cache, node, limits, budget, depth, &mut reuse,
)
}
#[allow(clippy::too_many_arguments)]
pub fn materialize_node_cached(
parsed: &ParsedDescriptor,
store: &dyn SeedStore,
cache: &mut dyn OutputCache,
node: &SeedNode,
limits: Limits,
budget: &mut EvalBudget,
depth: u16,
reuse: &mut ReuseStats,
) -> Result<Vec<u8>> {
materialize_inner(parsed, store, cache, node, limits, budget, depth, reuse)
}
#[allow(clippy::too_many_arguments)]
pub fn materialize_node_cached_with(
source: &dyn SourceServer,
store: &dyn SeedStore,
cache: &mut dyn OutputCache,
node: &SeedNode,
limits: Limits,
budget: &mut EvalBudget,
depth: u16,
reuse: &mut ReuseStats,
) -> Result<Vec<u8>> {
materialize_inner(source, store, cache, node, limits, budget, depth, reuse)
}
#[allow(clippy::too_many_arguments)]
fn materialize_inner(
source: &dyn SourceServer,
store: &dyn SeedStore,
cache: &mut dyn OutputCache,
node: &SeedNode,
limits: Limits,
budget: &mut EvalBudget,
depth: u16,
reuse: &mut ReuseStats,
) -> Result<Vec<u8>> {
if depth == 0 {
return Err(Error::resource_limit("seed DAG exceeded its depth bound"));
}
let id = node.content_id();
if let Ok(Some(bytes)) = cache.get(&id)
&& bytes.len() as u64 <= node.limits.max_output_bytes
{
reuse.nodes_reused = reuse.nodes_reused.saturating_add(1);
budget.charge_bytes(bytes.len() as u64)?;
return Ok(bytes);
}
budget.charge_node()?;
reuse.nodes_executed = reuse.nodes_executed.saturating_add(1);
let out = match node.kind {
NodeKind::DocumentExact => source.serve_document(limits)?,
NodeKind::SourceSlice | NodeKind::ResourceRef => {
let (offset, len) = read_span_params(&node.params)?;
source.serve_range(offset, len, limits)?
}
NodeKind::PdfRevision | NodeKind::PdfObject | NodeKind::PdfStreamEncoded => {
let (_number, _generation, extra) = read_object_params(&node.params)?;
let offset = extra >> 32;
let len = extra & 0xFFFF_FFFF;
source.serve_range(offset, len, limits)?
}
NodeKind::Concat | NodeKind::PageContent => {
let mut out = Vec::new();
for dep in &node.deps {
let child = load_node(store, dep)?;
let bytes = materialize_inner(
source,
store,
cache,
&child,
limits,
budget,
depth - 1,
reuse,
)?;
budget.charge_bytes(bytes.len() as u64)?;
out.extend_from_slice(&bytes);
}
out
}
NodeKind::Literal => node.params.clone(),
NodeKind::ResourceBlob => node.params.clone(),
NodeKind::PackageRoot => source.serve_document(limits)?,
NodeKind::PackageMemberRaw => {
let (offset, len) = read_span_params(&node.params)?;
source.serve_range(offset, len, limits)?
}
NodeKind::PackageMemberDecoded => {
let (_ordinal, method, _extra) = read_object_params(&node.params)?;
let dep = node
.deps
.first()
.ok_or_else(|| Error::usage("PackageMemberDecoded has no dependency"))?;
let child = load_node(store, dep)?;
let encoded = materialize_inner(
source,
store,
cache,
&child,
limits,
budget,
depth - 1,
reuse,
)?;
match method {
0 => {
if encoded.len() as u64 != node.logical_output_len {
return Err(Error::reconstruction_mismatch(format!(
"stored member is {} bytes but the node declared {}",
encoded.len(),
node.logical_output_len
)));
}
encoded
}
8 => derive::inflate_raw_deflate(&encoded, node.logical_output_len, limits)?,
other => {
return Err(Error::unsupported_feature(format!(
"zip member compression method {other} has no decoded representation"
)));
}
}
}
NodeKind::PackageOpcModel => {
#[cfg(feature = "opc")]
{
let dep = node
.deps
.first()
.ok_or_else(|| Error::usage("PackageOpcModel has no dependency"))?;
let child = load_node(store, dep)?;
let source = materialize_inner(
source,
store,
cache,
&child,
limits,
budget,
depth - 1,
reuse,
)?;
crate::field::opc::build_opc_model(&source, limits)?
}
#[cfg(not(feature = "opc"))]
{
return Err(Error::unsupported_feature(
"OPC support is not compiled in (feature `opc`)",
));
}
}
NodeKind::DocxModel => {
#[cfg(feature = "docx")]
{
let dep = node
.deps
.first()
.ok_or_else(|| Error::usage("DocxModel has no dependency"))?;
let child = load_node(store, dep)?;
let opc_bytes = materialize_inner(
source,
store,
cache,
&child,
limits,
budget,
depth - 1,
reuse,
)?;
crate::adapter::docx::build_docx_model(&opc_bytes, limits)?
}
#[cfg(not(feature = "docx"))]
{
return Err(Error::unsupported_feature(
"DOCX support is not compiled in (feature `docx`)",
));
}
}
NodeKind::DocxStory => {
#[cfg(feature = "docx")]
{
let (story, part_name, profile) =
crate::adapter::docx::read_story_params(&node.params)?;
let part_dep = node
.deps
.first()
.ok_or_else(|| Error::usage("DocxStory has no part dependency"))?;
let part_node = load_node(store, part_dep)?;
let part_bytes = materialize_inner(
source,
store,
cache,
&part_node,
limits,
budget,
depth - 1,
reuse,
)?;
let styles = match node.deps.get(1) {
Some(styles_dep) => {
let styles_node = load_node(store, styles_dep)?;
let styles_bytes = materialize_inner(
source,
store,
cache,
&styles_node,
limits,
budget,
depth - 1,
reuse,
)?;
Some(crate::adapter::docx::parse_styles(&styles_bytes, limits)?)
}
None => None,
};
crate::adapter::docx::wml::parse_story(
&part_bytes,
&part_name,
story,
&profile,
styles.as_ref(),
limits,
)?
.encode()
}
#[cfg(not(feature = "docx"))]
{
return Err(Error::unsupported_feature(
"DOCX support is not compiled in (feature `docx`)",
));
}
}
NodeKind::EpubModel => {
#[cfg(feature = "epub")]
{
let dep = node
.deps
.first()
.ok_or_else(|| Error::usage("EpubModel has no dependency"))?;
let child = load_node(store, dep)?;
let source = materialize_inner(
source,
store,
cache,
&child,
limits,
budget,
depth - 1,
reuse,
)?;
crate::adapter::epub::build_epub_model(&source, limits)?
}
#[cfg(not(feature = "epub"))]
{
return Err(Error::unsupported_feature(
"EPUB support is not compiled in (feature `epub`)",
));
}
}
NodeKind::EpubContent => {
#[cfg(feature = "epub")]
{
let (_spine, _ordinal, base_dir, _profile) =
crate::adapter::epub::read_content_params(&node.params)?;
let dep = node
.deps
.first()
.ok_or_else(|| Error::usage("EpubContent has no part dependency"))?;
let child = load_node(store, dep)?;
let bytes = materialize_inner(
source,
store,
cache,
&child,
limits,
budget,
depth - 1,
reuse,
)?;
crate::adapter::epub::parse_content(&bytes, &base_dir, limits)?.encode()
}
#[cfg(not(feature = "epub"))]
{
return Err(Error::unsupported_feature(
"EPUB support is not compiled in (feature `epub`)",
));
}
}
NodeKind::PdfStreamDecoded => {
let dep = node
.deps
.first()
.ok_or_else(|| Error::usage("PdfStreamDecoded has no dependency"))?;
let child = load_node(store, dep)?;
let encoded = materialize_inner(
source,
store,
cache,
&child,
limits,
budget,
depth - 1,
reuse,
)?;
derive::inflate_zlib(&encoded, node.logical_output_len, limits)?
}
NodeKind::ContentOperators => {
let dep = node
.deps
.first()
.ok_or_else(|| Error::usage("ContentOperators has no dependency"))?;
let child = load_node(store, dep)?;
let decoded = materialize_inner(
source,
store,
cache,
&child,
limits,
budget,
depth - 1,
reuse,
)?;
derive::content_operators(&decoded, limits)?
}
NodeKind::TextRuns => {
let dep = node
.deps
.first()
.ok_or_else(|| Error::usage("TextRuns has no dependency"))?;
let child = load_node(store, dep)?;
let ops = materialize_inner(
source,
store,
cache,
&child,
limits,
budget,
depth - 1,
reuse,
)?;
derive::text_runs(&ops, limits)?
}
NodeKind::PagePreview => {
let dep = node
.deps
.first()
.ok_or_else(|| Error::usage("PagePreview has no dependency"))?;
let child = load_node(store, dep)?;
let content = materialize_inner(
source,
store,
cache,
&child,
limits,
budget,
depth - 1,
reuse,
)?;
let page = read_u32_params(&node.params)?;
derive::page_preview(page, &content, limits)?
}
};
if out.len() as u64 > node.limits.max_output_bytes {
return Err(Error::resource_limit(format!(
"node {} produced {} bytes > its cap {}",
node.kind.name(),
out.len(),
node.limits.max_output_bytes
)));
}
budget.charge_bytes(out.len() as u64)?;
if cache.put(&id, &out).is_ok() {
reuse.cache_bytes_written = reuse.cache_bytes_written.saturating_add(out.len() as u64);
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::container::ObjectSource;
use crate::dra::{Op, Program};
use crate::integrity::sha256;
use crate::{EXACTNESS_PROFILE_EXACT_BYTES, SOURCE_FORMAT_OPAQUE};
fn parsed_for(source: &[u8]) -> ParsedDescriptor {
let d = crate::container::Descriptor {
universe: crate::container::UNIVERSE.to_string(),
source_format: SOURCE_FORMAT_OPAQUE,
format_basis: "opaque:test".to_string(),
models: vec![],
channels: vec![],
objects: vec![ObjectSource::Inline(source.to_vec())],
program: Program::new(vec![Op::EmitObject { object_id: 0 }]),
observation_index: None,
seek_directory: false,
source_sha256: sha256(source),
source_len: source.len() as u64,
};
let (bytes, _cost) = d.serialize().unwrap();
let _ = EXACTNESS_PROFILE_EXACT_BYTES;
crate::container::Descriptor::parse(&bytes, Limits::DEFAULT).unwrap()
}
#[test]
fn source_slice_matches_exact_bytes() {
let parsed = parsed_for(b"hello field world");
let mut budget = EvalBudget::default();
let (off, len) = (6u64, 5u64);
let node = SeedNode::new(
NodeKind::SourceSlice,
len,
crate::field::node::span_params(off, len),
vec![],
"test",
);
let bytes = materialize_node(
&parsed,
&crate::store::FsSeedStore::open(
std::env::temp_dir().join(format!("vole-dag-{}", std::process::id())),
)
.unwrap(),
&node,
Limits::DEFAULT,
&mut budget,
8,
)
.unwrap();
assert_eq!(bytes, b"field");
}
#[test]
fn literal_roundtrips() {
let parsed = parsed_for(b"x");
let mut budget = EvalBudget::default();
let node = SeedNode::new(NodeKind::Literal, 3, b"abc".to_vec(), vec![], "test");
let bytes = materialize_node(
&parsed,
&crate::store::FsSeedStore::open(
std::env::temp_dir().join(format!("vole-lit-{}", std::process::id())),
)
.unwrap(),
&node,
Limits::DEFAULT,
&mut budget,
4,
)
.unwrap();
assert_eq!(bytes, b"abc");
}
}