1use crate::container::{ObjectSource, ParsedDescriptor};
15use crate::error::{Error, Result};
16use crate::limits::Limits;
17use crate::materialize::observation::{select_ops, selection_references, serve_selection};
18use crate::store::{NodeId, SeedStore};
19
20use super::derive;
21use super::node::{NodeKind, SeedNode, read_object_params, read_span_params, read_u32_params};
22
23pub trait SourceServer {
31 fn serve_range(&self, offset: u64, len: u64, limits: Limits) -> Result<Vec<u8>>;
33 fn serve_document(&self, limits: Limits) -> Result<Vec<u8>>;
35}
36
37impl SourceServer for ParsedDescriptor {
38 fn serve_range(&self, offset: u64, len: u64, limits: Limits) -> Result<Vec<u8>> {
39 serve_source_range(self, offset, len, limits)
40 }
41
42 fn serve_document(&self, limits: Limits) -> Result<Vec<u8>> {
43 crate::materialize::materialize(self, limits)
44 }
45}
46
47pub const MAX_EVAL_NODES: u64 = 1 << 20;
49pub const MAX_EVAL_BYTES: u64 = 1 << 32;
51
52#[derive(Debug, Clone)]
54pub struct EvalBudget {
55 pub max_nodes: u64,
57 pub max_bytes: u64,
59 pub nodes: u64,
61 pub produced: u64,
63}
64
65impl Default for EvalBudget {
66 fn default() -> Self {
67 EvalBudget {
68 max_nodes: MAX_EVAL_NODES,
69 max_bytes: MAX_EVAL_BYTES,
70 nodes: 0,
71 produced: 0,
72 }
73 }
74}
75
76impl EvalBudget {
77 fn charge_node(&mut self) -> Result<()> {
78 self.nodes = self
79 .nodes
80 .checked_add(1)
81 .ok_or_else(|| Error::resource_limit("seed evaluation node count overflow"))?;
82 if self.nodes > self.max_nodes {
83 return Err(Error::resource_limit(format!(
84 "seed evaluation exceeded {} nodes",
85 self.max_nodes
86 )));
87 }
88 Ok(())
89 }
90
91 pub(crate) fn charge_bytes(&mut self, n: u64) -> Result<()> {
92 self.produced = self
93 .produced
94 .checked_add(n)
95 .ok_or_else(|| Error::resource_limit("seed evaluation byte count overflow"))?;
96 if self.produced > self.max_bytes {
97 return Err(Error::resource_limit(format!(
98 "seed evaluation exceeded {} bytes",
99 self.max_bytes
100 )));
101 }
102 Ok(())
103 }
104}
105
106pub fn load_node(store: &dyn SeedStore, id: &NodeId) -> Result<SeedNode> {
108 let bytes = store.get_node(id)?;
109 let node = SeedNode::decode_canonical(&bytes)?;
110 if node.content_id() != *id {
111 return Err(Error::integrity_mismatch(format!(
112 "seed node {id} decoded to a different content id"
113 )));
114 }
115 node.check_limits(&Limits::DEFAULT)?;
116 Ok(node)
117}
118
119pub fn deps_of_canonical(bytes: &[u8]) -> Result<Vec<NodeId>> {
121 Ok(SeedNode::decode_canonical(bytes)?.deps)
122}
123
124fn serve_source_range(
128 parsed: &ParsedDescriptor,
129 offset: u64,
130 len: u64,
131 limits: Limits,
132) -> Result<Vec<u8>> {
133 let d = &parsed.descriptor;
134 if d.objects
135 .iter()
136 .any(|o| matches!(o, ObjectSource::External { .. }))
137 {
138 return Err(Error::unsupported_feature(
139 "field v1 requires an inline descriptor (no external objects)",
140 ));
141 }
142 let end = offset
143 .checked_add(len)
144 .ok_or_else(|| Error::usage("source slice end overflows"))?;
145 if end > d.source_len {
146 return Err(Error::usage(format!(
147 "source slice {offset}..{end} exceeds source length {}",
148 d.source_len
149 )));
150 }
151 let objects: Vec<Vec<u8>> = d
152 .objects
153 .iter()
154 .map(|o| o.as_inline().unwrap_or(&[]).to_vec())
155 .collect();
156 let object_lens: Vec<u64> = objects.iter().map(|o| o.len() as u64).collect();
157 let channel_lens: Vec<u64> = d.channels.iter().map(|c| c.decoded_length).collect();
158 let window = select_ops(&d.program, &object_lens, &channel_lens, offset, end, limits)?;
159 let (objects_used, channels_used) =
160 selection_references(&window.ops, objects.len(), d.channels.len());
161 let served = serve_selection(
162 &objects,
163 &d.channels,
164 &d.models,
165 window,
166 &objects_used,
167 &channels_used,
168 offset,
169 end,
170 limits,
171 )?;
172 Ok(served.bytes)
173}
174
175pub trait OutputCache {
183 fn get(&self, id: &NodeId) -> Result<Option<Vec<u8>>>;
186 fn put(&mut self, id: &NodeId, bytes: &[u8]) -> Result<()>;
189}
190
191#[derive(Debug, Clone, Copy, Default)]
194pub struct NoCache;
195
196impl OutputCache for NoCache {
197 fn get(&self, _id: &NodeId) -> Result<Option<Vec<u8>>> {
198 Ok(None)
199 }
200
201 fn put(&mut self, _id: &NodeId, _bytes: &[u8]) -> Result<()> {
202 Ok(())
203 }
204}
205
206#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
210pub struct ReuseStats {
211 pub nodes_executed: u64,
213 pub nodes_reused: u64,
215 pub cache_bytes_written: u64,
217}
218
219#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
227pub struct InverseWork {
228 pub node_executions: u64,
230 pub input_bytes: u64,
232}
233
234impl InverseWork {
235 pub const fn new(node_executions: u64, input_bytes: u64) -> Self {
237 InverseWork {
238 node_executions,
239 input_bytes,
240 }
241 }
242
243 pub const fn units(self) -> u64 {
245 self.node_executions.saturating_add(self.input_bytes)
246 }
247}
248
249pub fn retained_inverse_work_fraction(cold: InverseWork, warm: InverseWork) -> f64 {
264 let total = cold.units();
265 if total == 0 {
266 return 1.0;
267 }
268 let reused = cold
269 .node_executions
270 .saturating_sub(warm.node_executions)
271 .saturating_add(cold.input_bytes.saturating_sub(warm.input_bytes));
272 reused as f64 / total as f64
273}
274
275pub fn materialize_node(
277 parsed: &ParsedDescriptor,
278 store: &dyn SeedStore,
279 node: &SeedNode,
280 limits: Limits,
281 budget: &mut EvalBudget,
282 depth: u16,
283) -> Result<Vec<u8>> {
284 let mut cache = NoCache;
285 let mut reuse = ReuseStats::default();
286 materialize_inner(
287 parsed, store, &mut cache, node, limits, budget, depth, &mut reuse,
288 )
289}
290
291pub fn materialize_node_with(
296 source: &dyn SourceServer,
297 store: &dyn SeedStore,
298 node: &SeedNode,
299 limits: Limits,
300 budget: &mut EvalBudget,
301 depth: u16,
302) -> Result<Vec<u8>> {
303 let mut cache = NoCache;
304 let mut reuse = ReuseStats::default();
305 materialize_inner(
306 source, store, &mut cache, node, limits, budget, depth, &mut reuse,
307 )
308}
309
310#[allow(clippy::too_many_arguments)]
318pub fn materialize_node_cached(
319 parsed: &ParsedDescriptor,
320 store: &dyn SeedStore,
321 cache: &mut dyn OutputCache,
322 node: &SeedNode,
323 limits: Limits,
324 budget: &mut EvalBudget,
325 depth: u16,
326 reuse: &mut ReuseStats,
327) -> Result<Vec<u8>> {
328 materialize_inner(parsed, store, cache, node, limits, budget, depth, reuse)
329}
330
331#[allow(clippy::too_many_arguments)]
333pub fn materialize_node_cached_with(
334 source: &dyn SourceServer,
335 store: &dyn SeedStore,
336 cache: &mut dyn OutputCache,
337 node: &SeedNode,
338 limits: Limits,
339 budget: &mut EvalBudget,
340 depth: u16,
341 reuse: &mut ReuseStats,
342) -> Result<Vec<u8>> {
343 materialize_inner(source, store, cache, node, limits, budget, depth, reuse)
344}
345
346#[allow(clippy::too_many_arguments)]
347fn materialize_inner(
348 source: &dyn SourceServer,
349 store: &dyn SeedStore,
350 cache: &mut dyn OutputCache,
351 node: &SeedNode,
352 limits: Limits,
353 budget: &mut EvalBudget,
354 depth: u16,
355 reuse: &mut ReuseStats,
356) -> Result<Vec<u8>> {
357 if depth == 0 {
358 return Err(Error::resource_limit("seed DAG exceeded its depth bound"));
359 }
360
361 let id = node.content_id();
362 if let Ok(Some(bytes)) = cache.get(&id)
366 && bytes.len() as u64 <= node.limits.max_output_bytes
367 {
368 reuse.nodes_reused = reuse.nodes_reused.saturating_add(1);
369 budget.charge_bytes(bytes.len() as u64)?;
370 return Ok(bytes);
371 }
372
373 budget.charge_node()?;
374 reuse.nodes_executed = reuse.nodes_executed.saturating_add(1);
375
376 let out = match node.kind {
377 NodeKind::DocumentExact => source.serve_document(limits)?,
378 NodeKind::SourceSlice | NodeKind::ResourceRef => {
379 let (offset, len) = read_span_params(&node.params)?;
380 source.serve_range(offset, len, limits)?
381 }
382 NodeKind::PdfRevision | NodeKind::PdfObject | NodeKind::PdfStreamEncoded => {
383 let (_number, _generation, extra) = read_object_params(&node.params)?;
384 let offset = extra >> 32;
386 let len = extra & 0xFFFF_FFFF;
387 source.serve_range(offset, len, limits)?
388 }
389 NodeKind::Concat | NodeKind::PageContent => {
390 let mut out = Vec::new();
391 for dep in &node.deps {
392 let child = load_node(store, dep)?;
393 let bytes = materialize_inner(
394 source,
395 store,
396 cache,
397 &child,
398 limits,
399 budget,
400 depth - 1,
401 reuse,
402 )?;
403 budget.charge_bytes(bytes.len() as u64)?;
404 out.extend_from_slice(&bytes);
405 }
406 out
407 }
408 NodeKind::Literal => node.params.clone(),
409 NodeKind::ResourceBlob => node.params.clone(),
412 NodeKind::PackageRoot => source.serve_document(limits)?,
413 NodeKind::PackageMemberRaw => {
414 let (offset, len) = read_span_params(&node.params)?;
415 source.serve_range(offset, len, limits)?
416 }
417 NodeKind::PackageMemberDecoded => {
418 let (_ordinal, method, _extra) = read_object_params(&node.params)?;
419 let dep = node
420 .deps
421 .first()
422 .ok_or_else(|| Error::usage("PackageMemberDecoded has no dependency"))?;
423 let child = load_node(store, dep)?;
424 let encoded = materialize_inner(
425 source,
426 store,
427 cache,
428 &child,
429 limits,
430 budget,
431 depth - 1,
432 reuse,
433 )?;
434 match method {
435 0 => {
437 if encoded.len() as u64 != node.logical_output_len {
438 return Err(Error::reconstruction_mismatch(format!(
439 "stored member is {} bytes but the node declared {}",
440 encoded.len(),
441 node.logical_output_len
442 )));
443 }
444 encoded
445 }
446 8 => derive::inflate_raw_deflate(&encoded, node.logical_output_len, limits)?,
448 other => {
450 return Err(Error::unsupported_feature(format!(
451 "zip member compression method {other} has no decoded representation"
452 )));
453 }
454 }
455 }
456 NodeKind::PackageOpcModel => {
457 #[cfg(feature = "opc")]
461 {
462 let dep = node
463 .deps
464 .first()
465 .ok_or_else(|| Error::usage("PackageOpcModel has no dependency"))?;
466 let child = load_node(store, dep)?;
467 let source = materialize_inner(
468 source,
469 store,
470 cache,
471 &child,
472 limits,
473 budget,
474 depth - 1,
475 reuse,
476 )?;
477 crate::field::opc::build_opc_model(&source, limits)?
478 }
479 #[cfg(not(feature = "opc"))]
480 {
481 return Err(Error::unsupported_feature(
482 "OPC support is not compiled in (feature `opc`)",
483 ));
484 }
485 }
486 NodeKind::DocxModel => {
487 #[cfg(feature = "docx")]
488 {
489 let dep = node
490 .deps
491 .first()
492 .ok_or_else(|| Error::usage("DocxModel has no dependency"))?;
493 let child = load_node(store, dep)?;
494 let opc_bytes = materialize_inner(
495 source,
496 store,
497 cache,
498 &child,
499 limits,
500 budget,
501 depth - 1,
502 reuse,
503 )?;
504 crate::adapter::docx::build_docx_model(&opc_bytes, limits)?
505 }
506 #[cfg(not(feature = "docx"))]
507 {
508 return Err(Error::unsupported_feature(
509 "DOCX support is not compiled in (feature `docx`)",
510 ));
511 }
512 }
513 NodeKind::DocxStory => {
514 #[cfg(feature = "docx")]
515 {
516 let (story, part_name, profile) =
517 crate::adapter::docx::read_story_params(&node.params)?;
518 let part_dep = node
519 .deps
520 .first()
521 .ok_or_else(|| Error::usage("DocxStory has no part dependency"))?;
522 let part_node = load_node(store, part_dep)?;
523 let part_bytes = materialize_inner(
524 source,
525 store,
526 cache,
527 &part_node,
528 limits,
529 budget,
530 depth - 1,
531 reuse,
532 )?;
533 let styles = match node.deps.get(1) {
534 Some(styles_dep) => {
535 let styles_node = load_node(store, styles_dep)?;
536 let styles_bytes = materialize_inner(
537 source,
538 store,
539 cache,
540 &styles_node,
541 limits,
542 budget,
543 depth - 1,
544 reuse,
545 )?;
546 Some(crate::adapter::docx::parse_styles(&styles_bytes, limits)?)
547 }
548 None => None,
549 };
550 crate::adapter::docx::wml::parse_story(
551 &part_bytes,
552 &part_name,
553 story,
554 &profile,
555 styles.as_ref(),
556 limits,
557 )?
558 .encode()
559 }
560 #[cfg(not(feature = "docx"))]
561 {
562 return Err(Error::unsupported_feature(
563 "DOCX support is not compiled in (feature `docx`)",
564 ));
565 }
566 }
567 NodeKind::EpubModel => {
568 #[cfg(feature = "epub")]
573 {
574 let dep = node
575 .deps
576 .first()
577 .ok_or_else(|| Error::usage("EpubModel has no dependency"))?;
578 let child = load_node(store, dep)?;
579 let source = materialize_inner(
580 source,
581 store,
582 cache,
583 &child,
584 limits,
585 budget,
586 depth - 1,
587 reuse,
588 )?;
589 crate::adapter::epub::build_epub_model(&source, limits)?
590 }
591 #[cfg(not(feature = "epub"))]
592 {
593 return Err(Error::unsupported_feature(
594 "EPUB support is not compiled in (feature `epub`)",
595 ));
596 }
597 }
598 NodeKind::EpubContent => {
599 #[cfg(feature = "epub")]
604 {
605 let (_spine, _ordinal, base_dir, _profile) =
606 crate::adapter::epub::read_content_params(&node.params)?;
607 let dep = node
608 .deps
609 .first()
610 .ok_or_else(|| Error::usage("EpubContent has no part dependency"))?;
611 let child = load_node(store, dep)?;
612 let bytes = materialize_inner(
613 source,
614 store,
615 cache,
616 &child,
617 limits,
618 budget,
619 depth - 1,
620 reuse,
621 )?;
622 crate::adapter::epub::parse_content(&bytes, &base_dir, limits)?.encode()
623 }
624 #[cfg(not(feature = "epub"))]
625 {
626 return Err(Error::unsupported_feature(
627 "EPUB support is not compiled in (feature `epub`)",
628 ));
629 }
630 }
631 NodeKind::OdtModel => {
632 #[cfg(feature = "odt")]
637 {
638 let dep = node
639 .deps
640 .first()
641 .ok_or_else(|| Error::usage("OdtModel has no dependency"))?;
642 let child = load_node(store, dep)?;
643 let source = materialize_inner(
644 source,
645 store,
646 cache,
647 &child,
648 limits,
649 budget,
650 depth - 1,
651 reuse,
652 )?;
653 crate::adapter::odt::build_odt_model(&source, limits)?
654 }
655 #[cfg(not(feature = "odt"))]
656 {
657 return Err(Error::unsupported_feature(
658 "ODT support is not compiled in (feature `odt`)",
659 ));
660 }
661 }
662 NodeKind::OdtContent => {
663 #[cfg(feature = "odt")]
668 {
669 let (_ordinal, part_name, profile) =
670 crate::adapter::odt::read_content_params(&node.params)?;
671 let dep = node
672 .deps
673 .first()
674 .ok_or_else(|| Error::usage("OdtContent has no part dependency"))?;
675 let child = load_node(store, dep)?;
676 let bytes = materialize_inner(
677 source,
678 store,
679 cache,
680 &child,
681 limits,
682 budget,
683 depth - 1,
684 reuse,
685 )?;
686 crate::adapter::odt::parse_content(&bytes, &part_name, &profile, limits)?.encode()
687 }
688 #[cfg(not(feature = "odt"))]
689 {
690 return Err(Error::unsupported_feature(
691 "ODT support is not compiled in (feature `odt`)",
692 ));
693 }
694 }
695 NodeKind::PdfStreamDecoded => {
696 let dep = node
697 .deps
698 .first()
699 .ok_or_else(|| Error::usage("PdfStreamDecoded has no dependency"))?;
700 let child = load_node(store, dep)?;
701 let encoded = materialize_inner(
702 source,
703 store,
704 cache,
705 &child,
706 limits,
707 budget,
708 depth - 1,
709 reuse,
710 )?;
711 derive::inflate_zlib(&encoded, node.logical_output_len, limits)?
712 }
713 NodeKind::ContentOperators => {
714 let dep = node
715 .deps
716 .first()
717 .ok_or_else(|| Error::usage("ContentOperators has no dependency"))?;
718 let child = load_node(store, dep)?;
719 let decoded = materialize_inner(
720 source,
721 store,
722 cache,
723 &child,
724 limits,
725 budget,
726 depth - 1,
727 reuse,
728 )?;
729 derive::content_operators(&decoded, limits)?
730 }
731 NodeKind::TextRuns => {
732 let dep = node
733 .deps
734 .first()
735 .ok_or_else(|| Error::usage("TextRuns has no dependency"))?;
736 let child = load_node(store, dep)?;
737 let ops = materialize_inner(
738 source,
739 store,
740 cache,
741 &child,
742 limits,
743 budget,
744 depth - 1,
745 reuse,
746 )?;
747 derive::text_runs(&ops, limits)?
748 }
749 NodeKind::PagePreview => {
750 let dep = node
751 .deps
752 .first()
753 .ok_or_else(|| Error::usage("PagePreview has no dependency"))?;
754 let child = load_node(store, dep)?;
755 let content = materialize_inner(
756 source,
757 store,
758 cache,
759 &child,
760 limits,
761 budget,
762 depth - 1,
763 reuse,
764 )?;
765 let page = read_u32_params(&node.params)?;
766 derive::page_preview(page, &content, limits)?
767 }
768 };
769
770 if out.len() as u64 > node.limits.max_output_bytes {
771 return Err(Error::resource_limit(format!(
772 "node {} produced {} bytes > its cap {}",
773 node.kind.name(),
774 out.len(),
775 node.limits.max_output_bytes
776 )));
777 }
778 budget.charge_bytes(out.len() as u64)?;
779 if cache.put(&id, &out).is_ok() {
781 reuse.cache_bytes_written = reuse.cache_bytes_written.saturating_add(out.len() as u64);
782 }
783 Ok(out)
784}
785
786#[cfg(test)]
787mod tests {
788 use super::*;
789 use crate::container::ObjectSource;
790 use crate::dra::{Op, Program};
791 use crate::integrity::sha256;
792 use crate::{EXACTNESS_PROFILE_EXACT_BYTES, SOURCE_FORMAT_OPAQUE};
793
794 fn parsed_for(source: &[u8]) -> ParsedDescriptor {
795 let d = crate::container::Descriptor {
796 universe: crate::container::UNIVERSE.to_string(),
797 source_format: SOURCE_FORMAT_OPAQUE,
798 format_basis: "opaque:test".to_string(),
799 models: vec![],
800 channels: vec![],
801 objects: vec![ObjectSource::Inline(source.to_vec())],
802 program: Program::new(vec![Op::EmitObject { object_id: 0 }]),
803 observation_index: None,
804 seek_directory: false,
805 checkpoints: None,
806 source_sha256: sha256(source),
807 source_len: source.len() as u64,
808 };
809 let (bytes, _cost) = d.serialize().unwrap();
810 let _ = EXACTNESS_PROFILE_EXACT_BYTES;
811 crate::container::Descriptor::parse(&bytes, Limits::DEFAULT).unwrap()
812 }
813
814 #[test]
815 fn source_slice_matches_exact_bytes() {
816 let parsed = parsed_for(b"hello field world");
817 let mut budget = EvalBudget::default();
818 let (off, len) = (6u64, 5u64);
819 let node = SeedNode::new(
820 NodeKind::SourceSlice,
821 len,
822 crate::field::node::span_params(off, len),
823 vec![],
824 "test",
825 );
826 let bytes = materialize_node(
827 &parsed,
828 &crate::store::FsSeedStore::open(
829 std::env::temp_dir().join(format!("vole-dag-{}", std::process::id())),
830 )
831 .unwrap(),
832 &node,
833 Limits::DEFAULT,
834 &mut budget,
835 8,
836 )
837 .unwrap();
838 assert_eq!(bytes, b"field");
839 }
840
841 #[test]
842 fn literal_roundtrips() {
843 let parsed = parsed_for(b"x");
844 let mut budget = EvalBudget::default();
845 let node = SeedNode::new(NodeKind::Literal, 3, b"abc".to_vec(), vec![], "test");
846 let bytes = materialize_node(
847 &parsed,
848 &crate::store::FsSeedStore::open(
849 std::env::temp_dir().join(format!("vole-lit-{}", std::process::id())),
850 )
851 .unwrap(),
852 &node,
853 Limits::DEFAULT,
854 &mut budget,
855 4,
856 )
857 .unwrap();
858 assert_eq!(bytes, b"abc");
859 }
860}