1use crate::container::{ObjectSource, ParsedDescriptor};
15use crate::error::{Error, Result};
16use crate::limits::Limits;
17use crate::materialize::observation::{select_ops, selection_references, serve_selection};
18use crate::store::{NodeId, SeedStore};
19
20use super::derive;
21use super::node::{NodeKind, SeedNode, read_object_params, read_span_params, read_u32_params};
22
23pub trait SourceServer {
31 fn serve_range(&self, offset: u64, len: u64, limits: Limits) -> Result<Vec<u8>>;
33 fn serve_document(&self, limits: Limits) -> Result<Vec<u8>>;
35}
36
37impl SourceServer for ParsedDescriptor {
38 fn serve_range(&self, offset: u64, len: u64, limits: Limits) -> Result<Vec<u8>> {
39 serve_source_range(self, offset, len, limits)
40 }
41
42 fn serve_document(&self, limits: Limits) -> Result<Vec<u8>> {
43 crate::materialize::materialize(self, limits)
44 }
45}
46
47pub const MAX_EVAL_NODES: u64 = 1 << 20;
49pub const MAX_EVAL_BYTES: u64 = 1 << 32;
51
52#[derive(Debug, Clone)]
54pub struct EvalBudget {
55 pub max_nodes: u64,
57 pub max_bytes: u64,
59 pub nodes: u64,
61 pub produced: u64,
63}
64
65impl Default for EvalBudget {
66 fn default() -> Self {
67 EvalBudget {
68 max_nodes: MAX_EVAL_NODES,
69 max_bytes: MAX_EVAL_BYTES,
70 nodes: 0,
71 produced: 0,
72 }
73 }
74}
75
76impl EvalBudget {
77 fn charge_node(&mut self) -> Result<()> {
78 self.nodes = self
79 .nodes
80 .checked_add(1)
81 .ok_or_else(|| Error::resource_limit("seed evaluation node count overflow"))?;
82 if self.nodes > self.max_nodes {
83 return Err(Error::resource_limit(format!(
84 "seed evaluation exceeded {} nodes",
85 self.max_nodes
86 )));
87 }
88 Ok(())
89 }
90
91 pub(crate) fn charge_bytes(&mut self, n: u64) -> Result<()> {
92 self.produced = self
93 .produced
94 .checked_add(n)
95 .ok_or_else(|| Error::resource_limit("seed evaluation byte count overflow"))?;
96 if self.produced > self.max_bytes {
97 return Err(Error::resource_limit(format!(
98 "seed evaluation exceeded {} bytes",
99 self.max_bytes
100 )));
101 }
102 Ok(())
103 }
104}
105
106pub fn load_node(store: &dyn SeedStore, id: &NodeId) -> Result<SeedNode> {
108 let bytes = store.get_node(id)?;
109 let node = SeedNode::decode_canonical(&bytes)?;
110 if node.content_id() != *id {
111 return Err(Error::integrity_mismatch(format!(
112 "seed node {id} decoded to a different content id"
113 )));
114 }
115 node.check_limits(&Limits::DEFAULT)?;
116 Ok(node)
117}
118
119pub fn deps_of_canonical(bytes: &[u8]) -> Result<Vec<NodeId>> {
121 Ok(SeedNode::decode_canonical(bytes)?.deps)
122}
123
124fn serve_source_range(
128 parsed: &ParsedDescriptor,
129 offset: u64,
130 len: u64,
131 limits: Limits,
132) -> Result<Vec<u8>> {
133 let d = &parsed.descriptor;
134 if d.objects
135 .iter()
136 .any(|o| matches!(o, ObjectSource::External { .. }))
137 {
138 return Err(Error::unsupported_feature(
139 "field v1 requires an inline descriptor (no external objects)",
140 ));
141 }
142 let end = offset
143 .checked_add(len)
144 .ok_or_else(|| Error::usage("source slice end overflows"))?;
145 if end > d.source_len {
146 return Err(Error::usage(format!(
147 "source slice {offset}..{end} exceeds source length {}",
148 d.source_len
149 )));
150 }
151 let objects: Vec<Vec<u8>> = d
152 .objects
153 .iter()
154 .map(|o| o.as_inline().unwrap_or(&[]).to_vec())
155 .collect();
156 let object_lens: Vec<u64> = objects.iter().map(|o| o.len() as u64).collect();
157 let channel_lens: Vec<u64> = d.channels.iter().map(|c| c.decoded_length).collect();
158 let window = select_ops(&d.program, &object_lens, &channel_lens, offset, end, limits)?;
159 let (objects_used, channels_used) =
160 selection_references(&window.ops, objects.len(), d.channels.len());
161 let served = serve_selection(
162 &objects,
163 &d.channels,
164 &d.models,
165 window,
166 &objects_used,
167 &channels_used,
168 offset,
169 end,
170 limits,
171 )?;
172 Ok(served.bytes)
173}
174
175#[derive(Debug, Clone, Copy, PartialEq, Eq)]
177pub enum CacheEffect {
178 Hit,
180 Stored,
182}
183
184#[derive(Debug, Clone, Copy)]
193pub struct CacheNote<'a> {
194 pub effect: CacheEffect,
196 pub kind: NodeKind,
198 pub id: &'a NodeId,
200 pub bytes: u64,
202 pub wall_micros: u64,
204}
205
206pub trait OutputCache {
214 fn get(&self, id: &NodeId) -> Result<Option<Vec<u8>>>;
217 fn put(&mut self, id: &NodeId, bytes: &[u8]) -> Result<()>;
220 fn note(&mut self, _note: CacheNote<'_>) {}
225}
226
227#[derive(Debug, Clone, Copy, Default)]
230pub struct NoCache;
231
232impl OutputCache for NoCache {
233 fn get(&self, _id: &NodeId) -> Result<Option<Vec<u8>>> {
234 Ok(None)
235 }
236
237 fn put(&mut self, _id: &NodeId, _bytes: &[u8]) -> Result<()> {
238 Ok(())
239 }
240}
241
242#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
246pub struct ReuseStats {
247 pub nodes_executed: u64,
249 pub nodes_reused: u64,
251 pub cache_bytes_written: u64,
253}
254
255#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
263pub struct InverseWork {
264 pub node_executions: u64,
266 pub input_bytes: u64,
268}
269
270impl InverseWork {
271 pub const fn new(node_executions: u64, input_bytes: u64) -> Self {
273 InverseWork {
274 node_executions,
275 input_bytes,
276 }
277 }
278
279 pub const fn units(self) -> u64 {
281 self.node_executions.saturating_add(self.input_bytes)
282 }
283}
284
285pub fn retained_inverse_work_fraction(cold: InverseWork, warm: InverseWork) -> f64 {
300 let total = cold.units();
301 if total == 0 {
302 return 1.0;
303 }
304 let reused = cold
305 .node_executions
306 .saturating_sub(warm.node_executions)
307 .saturating_add(cold.input_bytes.saturating_sub(warm.input_bytes));
308 reused as f64 / total as f64
309}
310
311pub fn materialize_node(
313 parsed: &ParsedDescriptor,
314 store: &dyn SeedStore,
315 node: &SeedNode,
316 limits: Limits,
317 budget: &mut EvalBudget,
318 depth: u16,
319) -> Result<Vec<u8>> {
320 let mut cache = NoCache;
321 let mut reuse = ReuseStats::default();
322 materialize_inner(
323 parsed, store, &mut cache, node, limits, budget, depth, &mut reuse,
324 )
325}
326
327pub fn materialize_node_with(
332 source: &dyn SourceServer,
333 store: &dyn SeedStore,
334 node: &SeedNode,
335 limits: Limits,
336 budget: &mut EvalBudget,
337 depth: u16,
338) -> Result<Vec<u8>> {
339 let mut cache = NoCache;
340 let mut reuse = ReuseStats::default();
341 materialize_inner(
342 source, store, &mut cache, node, limits, budget, depth, &mut reuse,
343 )
344}
345
346#[allow(clippy::too_many_arguments)]
354pub fn materialize_node_cached(
355 parsed: &ParsedDescriptor,
356 store: &dyn SeedStore,
357 cache: &mut dyn OutputCache,
358 node: &SeedNode,
359 limits: Limits,
360 budget: &mut EvalBudget,
361 depth: u16,
362 reuse: &mut ReuseStats,
363) -> Result<Vec<u8>> {
364 materialize_inner(parsed, store, cache, node, limits, budget, depth, reuse)
365}
366
367#[allow(clippy::too_many_arguments)]
369pub fn materialize_node_cached_with(
370 source: &dyn SourceServer,
371 store: &dyn SeedStore,
372 cache: &mut dyn OutputCache,
373 node: &SeedNode,
374 limits: Limits,
375 budget: &mut EvalBudget,
376 depth: u16,
377 reuse: &mut ReuseStats,
378) -> Result<Vec<u8>> {
379 materialize_inner(source, store, cache, node, limits, budget, depth, reuse)
380}
381
382#[allow(clippy::too_many_arguments)]
383fn materialize_inner(
384 source: &dyn SourceServer,
385 store: &dyn SeedStore,
386 cache: &mut dyn OutputCache,
387 node: &SeedNode,
388 limits: Limits,
389 budget: &mut EvalBudget,
390 depth: u16,
391 reuse: &mut ReuseStats,
392) -> Result<Vec<u8>> {
393 if depth == 0 {
394 return Err(Error::resource_limit("seed DAG exceeded its depth bound"));
395 }
396
397 let id = node.content_id();
398 if let Ok(Some(bytes)) = cache.get(&id)
402 && bytes.len() as u64 <= node.limits.max_output_bytes
403 {
404 reuse.nodes_reused = reuse.nodes_reused.saturating_add(1);
405 budget.charge_bytes(bytes.len() as u64)?;
406 cache.note(CacheNote {
407 effect: CacheEffect::Hit,
408 kind: node.kind,
409 id: &id,
410 bytes: bytes.len() as u64,
411 wall_micros: 0,
412 });
413 return Ok(bytes);
414 }
415
416 budget.charge_node()?;
417 reuse.nodes_executed = reuse.nodes_executed.saturating_add(1);
418
419 let out = match node.kind {
420 NodeKind::DocumentExact => source.serve_document(limits)?,
421 NodeKind::SourceSlice | NodeKind::ResourceRef => {
422 let (offset, len) = read_span_params(&node.params)?;
423 source.serve_range(offset, len, limits)?
424 }
425 NodeKind::PdfRevision | NodeKind::PdfObject | NodeKind::PdfStreamEncoded => {
426 let (_number, _generation, extra) = read_object_params(&node.params)?;
427 let offset = extra >> 32;
429 let len = extra & 0xFFFF_FFFF;
430 source.serve_range(offset, len, limits)?
431 }
432 NodeKind::Concat | NodeKind::PageContent => {
433 let mut out = Vec::new();
434 for dep in &node.deps {
435 let child = load_node(store, dep)?;
436 let bytes = materialize_inner(
437 source,
438 store,
439 cache,
440 &child,
441 limits,
442 budget,
443 depth - 1,
444 reuse,
445 )?;
446 budget.charge_bytes(bytes.len() as u64)?;
447 out.extend_from_slice(&bytes);
448 }
449 out
450 }
451 NodeKind::Literal => node.params.clone(),
452 NodeKind::ResourceBlob => node.params.clone(),
455 NodeKind::PackageRoot => source.serve_document(limits)?,
456 NodeKind::PackageMemberRaw => {
457 let (offset, len) = read_span_params(&node.params)?;
458 source.serve_range(offset, len, limits)?
459 }
460 NodeKind::PackageMemberDecoded => {
461 let (_ordinal, method, _extra) = read_object_params(&node.params)?;
462 let dep = node
463 .deps
464 .first()
465 .ok_or_else(|| Error::usage("PackageMemberDecoded has no dependency"))?;
466 let child = load_node(store, dep)?;
467 let encoded = materialize_inner(
468 source,
469 store,
470 cache,
471 &child,
472 limits,
473 budget,
474 depth - 1,
475 reuse,
476 )?;
477 match method {
478 0 => {
480 if encoded.len() as u64 != node.logical_output_len {
481 return Err(Error::reconstruction_mismatch(format!(
482 "stored member is {} bytes but the node declared {}",
483 encoded.len(),
484 node.logical_output_len
485 )));
486 }
487 encoded
488 }
489 8 => derive::inflate_raw_deflate(&encoded, node.logical_output_len, limits)?,
491 other => {
493 return Err(Error::unsupported_feature(format!(
494 "zip member compression method {other} has no decoded representation"
495 )));
496 }
497 }
498 }
499 NodeKind::PackageOpcModel => {
500 #[cfg(feature = "opc")]
504 {
505 let dep = node
506 .deps
507 .first()
508 .ok_or_else(|| Error::usage("PackageOpcModel has no dependency"))?;
509 let child = load_node(store, dep)?;
510 let source = materialize_inner(
511 source,
512 store,
513 cache,
514 &child,
515 limits,
516 budget,
517 depth - 1,
518 reuse,
519 )?;
520 crate::field::opc::build_opc_model(&source, limits)?
521 }
522 #[cfg(not(feature = "opc"))]
523 {
524 return Err(Error::unsupported_feature(
525 "OPC support is not compiled in (feature `opc`)",
526 ));
527 }
528 }
529 NodeKind::DocxModel => {
530 #[cfg(feature = "docx")]
531 {
532 let dep = node
533 .deps
534 .first()
535 .ok_or_else(|| Error::usage("DocxModel has no dependency"))?;
536 let child = load_node(store, dep)?;
537 let opc_bytes = materialize_inner(
538 source,
539 store,
540 cache,
541 &child,
542 limits,
543 budget,
544 depth - 1,
545 reuse,
546 )?;
547 crate::adapter::docx::build_docx_model(&opc_bytes, limits)?
548 }
549 #[cfg(not(feature = "docx"))]
550 {
551 return Err(Error::unsupported_feature(
552 "DOCX support is not compiled in (feature `docx`)",
553 ));
554 }
555 }
556 NodeKind::DocxStory => {
557 #[cfg(feature = "docx")]
558 {
559 let (story, part_name, profile) =
560 crate::adapter::docx::read_story_params(&node.params)?;
561 let part_dep = node
562 .deps
563 .first()
564 .ok_or_else(|| Error::usage("DocxStory has no part dependency"))?;
565 let part_node = load_node(store, part_dep)?;
566 let part_bytes = materialize_inner(
567 source,
568 store,
569 cache,
570 &part_node,
571 limits,
572 budget,
573 depth - 1,
574 reuse,
575 )?;
576 let styles = match node.deps.get(1) {
577 Some(styles_dep) => {
578 let styles_node = load_node(store, styles_dep)?;
579 let styles_bytes = materialize_inner(
580 source,
581 store,
582 cache,
583 &styles_node,
584 limits,
585 budget,
586 depth - 1,
587 reuse,
588 )?;
589 Some(crate::adapter::docx::parse_styles(&styles_bytes, limits)?)
590 }
591 None => None,
592 };
593 crate::adapter::docx::wml::parse_story(
594 &part_bytes,
595 &part_name,
596 story,
597 &profile,
598 styles.as_ref(),
599 limits,
600 )?
601 .encode()
602 }
603 #[cfg(not(feature = "docx"))]
604 {
605 return Err(Error::unsupported_feature(
606 "DOCX support is not compiled in (feature `docx`)",
607 ));
608 }
609 }
610 NodeKind::EpubModel => {
611 #[cfg(feature = "epub")]
616 {
617 let dep = node
618 .deps
619 .first()
620 .ok_or_else(|| Error::usage("EpubModel has no dependency"))?;
621 let child = load_node(store, dep)?;
622 let source = materialize_inner(
623 source,
624 store,
625 cache,
626 &child,
627 limits,
628 budget,
629 depth - 1,
630 reuse,
631 )?;
632 crate::adapter::epub::build_epub_model(&source, limits)?
633 }
634 #[cfg(not(feature = "epub"))]
635 {
636 return Err(Error::unsupported_feature(
637 "EPUB support is not compiled in (feature `epub`)",
638 ));
639 }
640 }
641 NodeKind::EpubContent => {
642 #[cfg(feature = "epub")]
647 {
648 let (_spine, _ordinal, base_dir, _profile) =
649 crate::adapter::epub::read_content_params(&node.params)?;
650 let dep = node
651 .deps
652 .first()
653 .ok_or_else(|| Error::usage("EpubContent has no part dependency"))?;
654 let child = load_node(store, dep)?;
655 let bytes = materialize_inner(
656 source,
657 store,
658 cache,
659 &child,
660 limits,
661 budget,
662 depth - 1,
663 reuse,
664 )?;
665 crate::adapter::epub::parse_content(&bytes, &base_dir, limits)?.encode()
666 }
667 #[cfg(not(feature = "epub"))]
668 {
669 return Err(Error::unsupported_feature(
670 "EPUB support is not compiled in (feature `epub`)",
671 ));
672 }
673 }
674 NodeKind::OdtModel => {
675 #[cfg(feature = "odt")]
680 {
681 let dep = node
682 .deps
683 .first()
684 .ok_or_else(|| Error::usage("OdtModel has no dependency"))?;
685 let child = load_node(store, dep)?;
686 let source = materialize_inner(
687 source,
688 store,
689 cache,
690 &child,
691 limits,
692 budget,
693 depth - 1,
694 reuse,
695 )?;
696 crate::adapter::odt::build_odt_model(&source, limits)?
697 }
698 #[cfg(not(feature = "odt"))]
699 {
700 return Err(Error::unsupported_feature(
701 "ODT support is not compiled in (feature `odt`)",
702 ));
703 }
704 }
705 NodeKind::OdtContent => {
706 #[cfg(feature = "odt")]
711 {
712 let (_ordinal, part_name, profile) =
713 crate::adapter::odt::read_content_params(&node.params)?;
714 let dep = node
715 .deps
716 .first()
717 .ok_or_else(|| Error::usage("OdtContent has no part dependency"))?;
718 let child = load_node(store, dep)?;
719 let bytes = materialize_inner(
720 source,
721 store,
722 cache,
723 &child,
724 limits,
725 budget,
726 depth - 1,
727 reuse,
728 )?;
729 crate::adapter::odt::parse_content(&bytes, &part_name, &profile, limits)?.encode()
730 }
731 #[cfg(not(feature = "odt"))]
732 {
733 return Err(Error::unsupported_feature(
734 "ODT support is not compiled in (feature `odt`)",
735 ));
736 }
737 }
738 NodeKind::PdfStreamDecoded => {
739 let dep = node
740 .deps
741 .first()
742 .ok_or_else(|| Error::usage("PdfStreamDecoded has no dependency"))?;
743 let child = load_node(store, dep)?;
744 let encoded = materialize_inner(
745 source,
746 store,
747 cache,
748 &child,
749 limits,
750 budget,
751 depth - 1,
752 reuse,
753 )?;
754 derive::inflate_zlib(&encoded, node.logical_output_len, limits)?
755 }
756 NodeKind::ContentOperators => {
757 let dep = node
758 .deps
759 .first()
760 .ok_or_else(|| Error::usage("ContentOperators has no dependency"))?;
761 let child = load_node(store, dep)?;
762 let decoded = materialize_inner(
763 source,
764 store,
765 cache,
766 &child,
767 limits,
768 budget,
769 depth - 1,
770 reuse,
771 )?;
772 derive::content_operators(&decoded, limits)?
773 }
774 NodeKind::TextRuns => {
775 let dep = node
776 .deps
777 .first()
778 .ok_or_else(|| Error::usage("TextRuns has no dependency"))?;
779 let child = load_node(store, dep)?;
780 let ops = materialize_inner(
781 source,
782 store,
783 cache,
784 &child,
785 limits,
786 budget,
787 depth - 1,
788 reuse,
789 )?;
790 derive::text_runs(&ops, limits)?
791 }
792 NodeKind::PagePreview => {
793 let dep = node
794 .deps
795 .first()
796 .ok_or_else(|| Error::usage("PagePreview has no dependency"))?;
797 let child = load_node(store, dep)?;
798 let content = materialize_inner(
799 source,
800 store,
801 cache,
802 &child,
803 limits,
804 budget,
805 depth - 1,
806 reuse,
807 )?;
808 let page = read_u32_params(&node.params)?;
809 derive::page_preview(page, &content, limits)?
810 }
811 };
812
813 if out.len() as u64 > node.limits.max_output_bytes {
814 return Err(Error::resource_limit(format!(
815 "node {} produced {} bytes > its cap {}",
816 node.kind.name(),
817 out.len(),
818 node.limits.max_output_bytes
819 )));
820 }
821 budget.charge_bytes(out.len() as u64)?;
822 if cache.put(&id, &out).is_ok() {
824 reuse.cache_bytes_written = reuse.cache_bytes_written.saturating_add(out.len() as u64);
825 cache.note(CacheNote {
826 effect: CacheEffect::Stored,
827 kind: node.kind,
828 id: &id,
829 bytes: out.len() as u64,
830 wall_micros: 0,
831 });
832 }
833 Ok(out)
834}
835
836#[cfg(test)]
837mod tests {
838 use super::*;
839 use crate::container::ObjectSource;
840 use crate::dra::{Op, Program};
841 use crate::integrity::sha256;
842 use crate::{EXACTNESS_PROFILE_EXACT_BYTES, SOURCE_FORMAT_OPAQUE};
843
844 fn parsed_for(source: &[u8]) -> ParsedDescriptor {
845 let d = crate::container::Descriptor {
846 universe: crate::container::UNIVERSE.to_string(),
847 source_format: SOURCE_FORMAT_OPAQUE,
848 format_basis: "opaque:test".to_string(),
849 models: vec![],
850 channels: vec![],
851 objects: vec![ObjectSource::Inline(source.to_vec())],
852 program: Program::new(vec![Op::EmitObject { object_id: 0 }]),
853 observation_index: None,
854 seek_directory: false,
855 checkpoints: None,
856 source_sha256: sha256(source),
857 source_len: source.len() as u64,
858 };
859 let (bytes, _cost) = d.serialize().unwrap();
860 let _ = EXACTNESS_PROFILE_EXACT_BYTES;
861 crate::container::Descriptor::parse(&bytes, Limits::DEFAULT).unwrap()
862 }
863
864 #[test]
865 fn source_slice_matches_exact_bytes() {
866 let parsed = parsed_for(b"hello field world");
867 let mut budget = EvalBudget::default();
868 let (off, len) = (6u64, 5u64);
869 let node = SeedNode::new(
870 NodeKind::SourceSlice,
871 len,
872 crate::field::node::span_params(off, len),
873 vec![],
874 "test",
875 );
876 let bytes = materialize_node(
877 &parsed,
878 &crate::store::FsSeedStore::open(
879 std::env::temp_dir().join(format!("vole-dag-{}", std::process::id())),
880 )
881 .unwrap(),
882 &node,
883 Limits::DEFAULT,
884 &mut budget,
885 8,
886 )
887 .unwrap();
888 assert_eq!(bytes, b"field");
889 }
890
891 #[test]
892 fn literal_roundtrips() {
893 let parsed = parsed_for(b"x");
894 let mut budget = EvalBudget::default();
895 let node = SeedNode::new(NodeKind::Literal, 3, b"abc".to_vec(), vec![], "test");
896 let bytes = materialize_node(
897 &parsed,
898 &crate::store::FsSeedStore::open(
899 std::env::temp_dir().join(format!("vole-lit-{}", std::process::id())),
900 )
901 .unwrap(),
902 &node,
903 Limits::DEFAULT,
904 &mut budget,
905 4,
906 )
907 .unwrap();
908 assert_eq!(bytes, b"abc");
909 }
910}