1pub mod cache;
16pub mod capabilities;
17pub mod dag;
18pub mod derive;
19pub mod document_format;
20pub mod edit;
21pub mod explain;
22pub mod index;
23pub(crate) mod inflate;
26pub mod ingest;
27#[cfg(feature = "package")]
28pub mod ingest_package;
29pub mod manifest;
30pub mod node;
31pub mod observe;
32#[cfg(feature = "opc")]
33pub mod opc;
34pub mod partial;
35pub mod plan;
36pub mod promote;
37pub mod provenance;
38pub mod resource;
39pub mod session;
40pub mod share;
41
42pub use manifest::{FieldId, FieldRoot};
43
44use std::fs;
45use std::path::{Path, PathBuf};
46use std::rc::Rc;
47
48#[cfg(feature = "entropyfs-store")]
49use std::sync::Arc;
50
51#[cfg(feature = "entropyfs-store")]
52use entropyfs::engine::BlobId;
53
54use crate::container::ParsedDescriptor;
55use crate::error::{Error, Result};
56use crate::limits::Limits;
57#[cfg(feature = "entropyfs-store")]
58use crate::store::{EntropyFsStore, map_engine_error};
59use crate::store::{FsSeedStore, Id, IoCounters, IoSnapshot, NodeId, PackedSeedStore, SeedStore};
60
61#[cfg(feature = "entropyfs-store")]
62use self::manifest::FIELD_ROOT_DOMAIN;
63use self::node::{NodeKind, SeedNode};
64
65pub const FIELD_UNIVERSE: &str = "vole-document;universe;phase11;exact-bytes;dra-8;opaque+entropy+pdf+channels+offsets+packed+packed-channels+deflate-replay-preflate-0.7.6-experimental+observation-index-v1+seek-directory-v1+external-objects-v1+procedural-seed-field-v1+hier-index-v1";
67
68#[cfg(feature = "package")]
75pub const PACKAGE_UNIVERSE: &str = "vole-document;universe;phase11;exact-bytes;dra-8;opaque+entropy+pdf+channels+offsets+packed+packed-channels+deflate-replay-preflate-0.7.6-experimental+observation-index-v1+seek-directory-v1+external-objects-v1+procedural-seed-field-v1+hier-index-v1+package-v1";
76
77#[derive(Clone)]
86pub(crate) enum SeedSubstrate {
87 Fs { root: PathBuf, io: IoCounters },
89 Packed { store: Rc<PackedSeedStore> },
91 #[cfg(feature = "entropyfs-store")]
93 EntropyFs {
94 store: Arc<EntropyFsStore>,
95 io: IoCounters,
96 },
97}
98
99impl SeedStore for SeedSubstrate {
100 fn put_node(&mut self, canonical: &[u8]) -> Result<NodeId> {
101 match self {
102 SeedSubstrate::Fs { root, io } => {
103 FsSeedStore::open_with_io(root, io.handle())?.put_node(canonical)
104 }
105 SeedSubstrate::Packed { store } => store.insert(canonical),
106 #[cfg(feature = "entropyfs-store")]
107 SeedSubstrate::EntropyFs { store, .. } => store.seed_put(canonical),
108 }
109 }
110
111 fn get_node(&self, id: &NodeId) -> Result<Vec<u8>> {
112 match self {
113 SeedSubstrate::Fs { root, io } => {
114 FsSeedStore::open_with_io(root, io.handle())?.get_node(id)
115 }
116 SeedSubstrate::Packed { store } => store.fetch(id),
117 #[cfg(feature = "entropyfs-store")]
118 SeedSubstrate::EntropyFs { store, io } => {
119 let bytes = store.seed_get(id)?;
120 io.add_seed(bytes.len() as u64);
121 Ok(bytes)
122 }
123 }
124 }
125
126 fn get_node_range(&self, id: &NodeId, offset: u64, len: u64) -> Result<Vec<u8>> {
127 match self {
128 SeedSubstrate::Fs { root, io } => {
129 FsSeedStore::open_with_io(root, io.handle())?.get_node_range(id, offset, len)
130 }
131 SeedSubstrate::Packed { store } => store.fetch_range(id, offset, len),
132 #[cfg(feature = "entropyfs-store")]
133 SeedSubstrate::EntropyFs { store, io } => {
134 let bytes = store.seed_get_range(id, offset, len)?;
135 io.add_seed(bytes.len() as u64);
136 Ok(bytes)
137 }
138 }
139 }
140
141 fn contains_node(&self, id: &NodeId) -> Result<bool> {
142 match self {
143 SeedSubstrate::Fs { root, io } => {
144 FsSeedStore::open_with_io(root, io.handle())?.contains_node(id)
145 }
146 SeedSubstrate::Packed { store } => store.has(id),
147 #[cfg(feature = "entropyfs-store")]
148 SeedSubstrate::EntropyFs { store, .. } => store.seed_contains(id),
149 }
150 }
151
152 fn list_nodes(&self) -> Result<Vec<(NodeId, u64)>> {
153 match self {
154 SeedSubstrate::Fs { root, io } => {
155 FsSeedStore::open_with_io(root, io.handle())?.list_nodes()
156 }
157 SeedSubstrate::Packed { store } => store.entries(),
158 #[cfg(feature = "entropyfs-store")]
159 SeedSubstrate::EntropyFs { store, .. } => store.seed_list(),
160 }
161 }
162}
163
164enum BlobBackend {
172 Fs,
174 #[cfg(feature = "entropyfs-store")]
180 EntropyFs(Arc<EntropyFsStore>),
181}
182
183impl BlobBackend {
184 fn descriptor_put(&self, root: &Path, bytes: &[u8]) -> Result<Id> {
185 match self {
186 BlobBackend::Fs => {
187 let id = Id::of(bytes);
188 let path = root.join("descriptor").join(id.to_hex());
189 if !path.exists() {
190 write_atomic(&path, bytes)?;
191 }
192 Ok(id)
193 }
194 #[cfg(feature = "entropyfs-store")]
195 BlobBackend::EntropyFs(store) => {
196 let id = Id::of(bytes);
197 let blob = store
198 .engine()
199 .put_blob(bytes)
200 .map_err(|e| map_engine_error("put_blob", &e))?;
201 debug_assert_eq!(blob.as_bytes(), id.as_bytes());
202 Ok(id)
203 }
204 }
205 }
206
207 fn descriptor_get(&self, root: &Path, id: &Id) -> Result<Vec<u8>> {
208 match self {
209 BlobBackend::Fs => {
210 let path = root.join("descriptor").join(id.to_hex());
211 fs::read(&path).map_err(|e| {
212 if e.kind() == std::io::ErrorKind::NotFound {
213 Error::missing_external_object(format!(
214 "descriptor blob {id} is not present"
215 ))
216 } else {
217 Error::io(format!("reading descriptor blob {id}: {e}"))
218 }
219 })
220 }
221 #[cfg(feature = "entropyfs-store")]
222 BlobBackend::EntropyFs(store) => store
223 .engine()
224 .get_blob(BlobId::new(*id.as_bytes()))
225 .map_err(|e| map_engine_error("get_blob", &e)),
226 }
227 }
228
229 fn field_put(&self, root: &Path, manifest: &FieldRoot) -> Result<FieldId> {
230 let bytes = manifest.encode_canonical();
231 let id = FieldId::of_manifest(&bytes);
232 match self {
233 BlobBackend::Fs => {
234 let path = root.join("field").join(id.to_hex());
235 if !path.exists() {
236 write_atomic(&path, &bytes)?;
237 }
238 Ok(id)
239 }
240 #[cfg(feature = "entropyfs-store")]
241 BlobBackend::EntropyFs(store) => {
242 let mut prefixed = Vec::with_capacity(FIELD_ROOT_DOMAIN.len() + bytes.len());
243 prefixed.extend_from_slice(FIELD_ROOT_DOMAIN);
244 prefixed.extend_from_slice(&bytes);
245 let blob = store
246 .engine()
247 .put_blob(&prefixed)
248 .map_err(|e| map_engine_error("put_blob", &e))?;
249 debug_assert_eq!(blob.as_bytes(), id.as_bytes());
250 Ok(id)
251 }
252 }
253 }
254
255 fn field_get(&self, root: &Path, id: &FieldId) -> Result<Vec<u8>> {
256 match self {
257 BlobBackend::Fs => {
258 let path = root.join("field").join(id.to_hex());
259 fs::read(&path).map_err(|e| {
260 if e.kind() == std::io::ErrorKind::NotFound {
261 Error::missing_external_object(format!("field {id} is not present"))
262 } else {
263 Error::io(format!("reading field {id}: {e}"))
264 }
265 })
266 }
267 #[cfg(feature = "entropyfs-store")]
268 BlobBackend::EntropyFs(store) => {
269 let blob = store
270 .engine()
271 .get_blob(BlobId::new(*id.as_bytes()))
272 .map_err(|e| map_engine_error("get_blob", &e))?;
273 let rest = blob.strip_prefix(FIELD_ROOT_DOMAIN).ok_or_else(|| {
274 Error::integrity_mismatch(format!(
275 "field manifest {id} is missing its domain prefix"
276 ))
277 })?;
278 Ok(rest.to_vec())
279 }
280 }
281 }
282}
283
284#[cfg(feature = "entropyfs-store")]
313#[derive(Debug, Clone, Copy, PartialEq, Eq)]
314pub struct EngineStoreStats {
315 pub blob_count: u64,
317 pub logical_bytes: u64,
319 pub physical_used_bytes: u64,
321}
322
323pub struct FieldStore {
324 root: PathBuf,
325 backend: BlobBackend,
326 seeds: SeedSubstrate,
327 io: IoCounters,
328 promote: promote::PromotePolicy,
330 governor: promote::Governor,
332}
333
334impl std::fmt::Debug for FieldStore {
335 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
336 f.debug_struct("FieldStore")
337 .field("root", &self.root)
338 .finish_non_exhaustive()
339 }
340}
341
342impl FieldStore {
343 pub fn open(root: impl AsRef<Path>) -> Result<Self> {
345 let root = root.as_ref().to_path_buf();
346 fs::create_dir_all(root.join("descriptor"))?;
347 fs::create_dir_all(root.join("field"))?;
348 fs::create_dir_all(root.join("index"))?;
349 fs::create_dir_all(root.join("cache"))?;
350 let io = IoCounters::new();
351 let seeds = SeedSubstrate::Fs {
352 root: root.clone(),
353 io: io.handle(),
354 };
355 Ok(FieldStore {
356 root,
357 backend: BlobBackend::Fs,
358 seeds,
359 io,
360 promote: promote::PromotePolicy::default(),
361 governor: promote::Governor::default(),
362 })
363 }
364
365 #[cfg(feature = "entropyfs-store")]
375 pub fn open_entropyfs(root: impl AsRef<Path>) -> Result<Self> {
376 let root = root.as_ref().to_path_buf();
377 fs::create_dir_all(root.join("index"))?;
378 fs::create_dir_all(root.join("cache"))?;
379 let engine_root = root.join("entropyfs");
380 fs::create_dir_all(&engine_root)?;
381 let fresh = fs::read_dir(&engine_root)?.next().is_none();
384 let engine = if fresh {
385 EntropyFsStore::create(&engine_root)?
386 } else {
387 EntropyFsStore::open(&engine_root)?
388 };
389 let engine = Arc::new(engine);
390 let io = IoCounters::new();
391 let seeds = SeedSubstrate::EntropyFs {
392 store: Arc::clone(&engine),
393 io: io.handle(),
394 };
395 Ok(FieldStore {
396 root,
397 backend: BlobBackend::EntropyFs(engine),
398 seeds,
399 io,
400 promote: promote::PromotePolicy::default(),
401 governor: promote::Governor::default(),
402 })
403 }
404
405 pub fn open_packed(root: impl AsRef<Path>) -> Result<Self> {
414 let root = root.as_ref().to_path_buf();
415 fs::create_dir_all(root.join("descriptor"))?;
416 fs::create_dir_all(root.join("field"))?;
417 fs::create_dir_all(root.join("index"))?;
418 fs::create_dir_all(root.join("cache"))?;
419 let io = IoCounters::new();
421 let packed = Rc::new(PackedSeedStore::open_write(&root, io.handle())?);
422 let seeds = SeedSubstrate::Packed { store: packed };
423 Ok(FieldStore {
424 root,
425 backend: BlobBackend::Fs,
426 seeds,
427 io,
428 promote: promote::PromotePolicy::default(),
429 governor: promote::Governor::default(),
430 })
431 }
432
433 pub fn io(&self) -> &IoCounters {
439 &self.io
440 }
441
442 pub fn root(&self) -> &Path {
444 &self.root
445 }
446
447 pub(crate) fn seed_substrate(&self) -> SeedSubstrate {
450 self.seeds.clone()
451 }
452
453 pub(crate) fn supports_partial_descriptor(&self) -> bool {
457 matches!(self.backend, BlobBackend::Fs)
458 }
459
460 pub fn seeds(&self) -> &dyn SeedStore {
462 &self.seeds
463 }
464
465 pub fn seeds_mut(&mut self) -> &mut dyn SeedStore {
467 &mut self.seeds
468 }
469
470 pub fn cache(&self) -> Result<cache::DerivedCache> {
475 cache::DerivedCache::open(self.root.join("cache"))
476 }
477
478 pub fn set_promote(&mut self, policy: promote::PromotePolicy) {
482 self.governor.set_policy(policy);
485 self.promote = policy;
486 }
487
488 pub fn promote_policy(&self) -> promote::PromotePolicy {
490 self.promote
491 }
492
493 pub(crate) fn governor(&self) -> promote::Governor {
495 self.governor.clone()
496 }
497
498 pub fn promoted(&self) -> Result<promote::PromotedStore> {
502 promote::PromotedStore::open(self.root.join("promoted"))
503 }
504
505 pub fn sync(&self) -> Result<()> {
512 match &self.backend {
513 BlobBackend::Fs => {}
514 #[cfg(feature = "entropyfs-store")]
515 BlobBackend::EntropyFs(store) => store.sync()?,
516 }
517 if let SeedSubstrate::Packed { store } = &self.seeds {
520 store.seal()?;
521 }
522 Ok(())
523 }
524
525 #[cfg(feature = "entropyfs-store")]
533 pub fn engine_stats(&self) -> Result<Option<EngineStoreStats>> {
534 match &self.backend {
535 BlobBackend::Fs => Ok(None),
536 BlobBackend::EntropyFs(store) => {
537 let m = store
538 .engine()
539 .metrics()
540 .map_err(|e| map_engine_error("metrics", &e))?;
541 Ok(Some(EngineStoreStats {
542 blob_count: m.accounting.blob_count,
543 logical_bytes: m.accounting.logical_bytes,
544 physical_used_bytes: m.accounting.physical_used_bytes,
545 }))
546 }
547 }
548 }
549
550 pub(crate) fn descriptor_path(&self, id: &Id) -> Option<PathBuf> {
553 match self.backend {
554 BlobBackend::Fs => Some(self.root.join("descriptor").join(id.to_hex())),
555 #[cfg(feature = "entropyfs-store")]
556 BlobBackend::EntropyFs(_) => None,
557 }
558 }
559
560 pub fn put_descriptor(&mut self, bytes: &[u8]) -> Result<Id> {
565 self.backend.descriptor_put(&self.root, bytes)
566 }
567
568 pub fn get_descriptor(&self, id: &Id) -> Result<Vec<u8>> {
570 let bytes = self.backend.descriptor_get(&self.root, id)?;
571 let actual = Id::of(&bytes);
572 if actual != *id {
573 return Err(Error::integrity_mismatch(format!(
574 "descriptor blob {id} hashes to {actual}"
575 )));
576 }
577 self.io.add_descriptor(bytes.len() as u64);
578 Ok(bytes)
579 }
580
581 pub fn put_field(&mut self, manifest: &FieldRoot) -> Result<FieldId> {
583 self.backend.field_put(&self.root, manifest)
584 }
585
586 pub fn get_field(&self, id: &FieldId) -> Result<FieldRoot> {
588 let bytes = self.backend.field_get(&self.root, id)?;
589 let manifest = FieldRoot::decode_canonical(&bytes)?;
590 if manifest.content_id() != *id {
591 return Err(Error::integrity_mismatch(format!(
592 "field {id} manifest content id mismatch"
593 )));
594 }
595 self.io.add_manifest(bytes.len() as u64);
596 Ok(manifest)
597 }
598
599 pub fn list_fields(&self) -> Result<Vec<FieldId>> {
606 if !matches!(self.backend, BlobBackend::Fs) {
607 return Err(Error::unsupported_feature(
608 "EntropyFS exposes no per-blob enumeration; a field store backed by it \
609 cannot list its manifests (open a field by its FieldId instead)",
610 ));
611 }
612 let dir = self.root.join("field");
613 let mut out = Vec::new();
614 for entry in fs::read_dir(&dir)?.flatten() {
615 if let Some(name) = entry.file_name().to_str()
616 && let Ok(id) = FieldId::from_hex(name)
617 {
618 out.push(id);
619 }
620 }
621 out.sort_unstable();
622 Ok(out)
623 }
624
625 pub fn ingest(&mut self, descriptor_bytes: &[u8], limits: Limits) -> Result<FieldId> {
631 let parsed = crate::container::Descriptor::parse(descriptor_bytes, limits)?;
634 let source = crate::materialize::materialize(&parsed, limits)?;
635 let descriptor_id = self.put_descriptor(descriptor_bytes)?;
636
637 let root = SeedNode::new(
638 NodeKind::DocumentExact,
639 source.len() as u64,
640 Vec::new(),
641 Vec::new(),
642 "field:document-exact",
643 );
644 let root_id = self.seeds.put_node(&root.encode_canonical())?;
645
646 let manifest = FieldRoot {
647 universe_id: crate::container::universe_id_from_str(FIELD_UNIVERSE),
648 source_sha256: parsed.descriptor.source_sha256,
649 source_len: parsed.descriptor.source_len,
650 descriptor_id: *descriptor_id.as_bytes(),
651 root_node: root_id,
652 index_root: manifest::ABSENT_ROOT,
653 node_count: 1,
654 index_node_count: 0,
655 provenance: format!("field:ingest;{}", parsed.descriptor.format_basis),
656 };
657 self.put_field(&manifest)
658 }
659}
660
661pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {
663 use std::io::Write;
664 let dir = path
665 .parent()
666 .ok_or_else(|| Error::internal_invariant("atomic write path has no parent"))?;
667 fs::create_dir_all(dir)?;
668 let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("blob");
669 let tmp = dir.join(format!(".{name}.tmp-{}", std::process::id()));
670 {
671 let mut f = fs::File::create(&tmp)?;
672 f.write_all(bytes)?;
673 f.sync_all()?;
674 }
675 fs::rename(&tmp, path)?;
676 Ok(())
677}
678
679pub struct Field {
681 store_root: PathBuf,
682 manifest: FieldRoot,
683 parsed: ParsedDescriptor,
684 descriptor_bytes: Vec<u8>,
685 seeds: SeedSubstrate,
689 open_io: IoSnapshot,
693}
694
695impl std::fmt::Debug for Field {
696 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
697 f.debug_struct("Field")
698 .field("manifest", &self.manifest.content_id())
699 .field("source_len", &self.manifest.source_len)
700 .finish_non_exhaustive()
701 }
702}
703
704impl Field {
705 pub fn open(store: &FieldStore, id: &FieldId, limits: Limits) -> Result<Field> {
707 let io_before = store.io().snapshot();
708 let manifest = store.get_field(id)?;
709 Field::open_after_manifest(store, manifest, io_before, limits)
710 }
711
712 pub(crate) fn open_after_manifest(
717 store: &FieldStore,
718 manifest: FieldRoot,
719 io_before: IoSnapshot,
720 limits: Limits,
721 ) -> Result<Field> {
722 let descriptor_bytes = store.get_descriptor(&Id::from_bytes(manifest.descriptor_id))?;
723 let open_io = io_before.delta(&store.io().snapshot());
724 let mut parsed = crate::container::Descriptor::parse(&descriptor_bytes, limits)?;
725 parsed.universe_id = manifest.universe_id;
726 if parsed.descriptor.source_len != manifest.source_len
728 || parsed.descriptor.source_sha256 != manifest.source_sha256
729 {
730 return Err(Error::integrity_mismatch(
731 "field manifest does not match its descriptor's declared source",
732 ));
733 }
734 Ok(Field {
735 store_root: store.root().to_path_buf(),
736 manifest,
737 parsed,
738 descriptor_bytes,
739 seeds: store.seed_substrate(),
740 open_io,
741 })
742 }
743
744 pub(crate) fn open_io(&self) -> IoSnapshot {
746 self.open_io
747 }
748
749 pub fn manifest(&self) -> &FieldRoot {
751 &self.manifest
752 }
753
754 pub fn id(&self) -> FieldId {
756 self.manifest.content_id()
757 }
758
759 pub fn parsed(&self) -> &ParsedDescriptor {
761 &self.parsed
762 }
763
764 pub fn descriptor_bytes(&self) -> &[u8] {
766 &self.descriptor_bytes
767 }
768
769 pub fn store_root(&self) -> &Path {
771 &self.store_root
772 }
773
774 pub fn materialize_exact(&self, limits: Limits) -> Result<Vec<u8>> {
780 crate::materialize::materialize(&self.parsed, limits)
781 }
782
783 pub fn materialize_node(
786 &self,
787 id: &NodeId,
788 limits: Limits,
789 budget: &mut dag::EvalBudget,
790 ) -> Result<Vec<u8>> {
791 let node = dag::load_node(&self.seeds, id)?;
792 dag::materialize_node(
793 &self.parsed,
794 &self.seeds,
795 &node,
796 limits,
797 budget,
798 node.limits.max_depth,
799 )
800 }
801}
802
803#[cfg(test)]
804mod tests {
805 use super::*;
806
807 fn temp_root(label: &str) -> PathBuf {
808 let mut p = std::env::temp_dir();
809 p.push(format!(
810 "vole-field-{label}-{}-{}",
811 std::process::id(),
812 std::time::SystemTime::now()
813 .duration_since(std::time::UNIX_EPOCH)
814 .unwrap()
815 .as_nanos()
816 ));
817 p
818 }
819
820 fn tiny_descriptor() -> Vec<u8> {
821 use crate::container::{Descriptor, ObjectSource};
822 use crate::dra::{Op, Program};
823 let source = b"the exact field bytes";
824 let d = Descriptor {
825 universe: crate::container::UNIVERSE.to_string(),
826 source_format: crate::SOURCE_FORMAT_OPAQUE,
827 format_basis: "opaque:field-test".to_string(),
828 models: vec![],
829 channels: vec![],
830 objects: vec![ObjectSource::Inline(source.to_vec())],
831 program: Program::new(vec![Op::EmitObject { object_id: 0 }]),
832 observation_index: None,
833 seek_directory: false,
834 checkpoints: None,
835 source_sha256: crate::integrity::sha256(source),
836 source_len: source.len() as u64,
837 };
838 d.serialize().unwrap().0
839 }
840
841 #[test]
842 fn ingest_then_materialize_is_exact() {
843 let root = temp_root("rt");
844 let mut store = FieldStore::open(&root).unwrap();
845 let bytes = tiny_descriptor();
846 let id = store.ingest(&bytes, Limits::DEFAULT).unwrap();
847 let field = Field::open(&store, &id, Limits::DEFAULT).unwrap();
848 assert_eq!(
849 field.materialize_exact(Limits::DEFAULT).unwrap(),
850 b"the exact field bytes"
851 );
852 let mut budget = dag::EvalBudget::default();
854 let out = field
855 .materialize_node(&field.manifest.root_node, Limits::DEFAULT, &mut budget)
856 .unwrap();
857 assert_eq!(out, b"the exact field bytes");
858 fs::remove_dir_all(&root).ok();
859 }
860
861 #[test]
862 fn field_id_is_stable_and_reopenable() {
863 let root = temp_root("reopen");
864 let bytes = tiny_descriptor();
865 let id1 = {
866 let mut store = FieldStore::open(&root).unwrap();
867 store.ingest(&bytes, Limits::DEFAULT).unwrap()
868 };
869 let store = FieldStore::open(&root).unwrap();
870 let m = store.get_field(&id1).unwrap();
871 assert_eq!(m.content_id(), id1);
872 assert_eq!(store.list_fields().unwrap(), vec![id1]);
873 fs::remove_dir_all(&root).ok();
874 }
875
876 #[test]
877 fn promotion_is_a_noop_by_default() {
878 use crate::field::observe::{self, ObserveRequest, Representation, Selector};
879 let root = temp_root("promote-off");
880 let mut store = FieldStore::open(&root).unwrap();
881 let id = store.ingest(&tiny_descriptor(), Limits::DEFAULT).unwrap();
882 let req = ObserveRequest::new(
883 Selector::ByteRange { offset: 4, len: 5 },
884 Representation::ExactBytes,
885 );
886 let (a1, _, _) = observe::observe(&mut store, &id, &req, Limits::DEFAULT).unwrap();
887 let (a2, _, _) = observe::observe(&mut store, &id, &req, Limits::DEFAULT).unwrap();
888 assert_eq!(a1, a2);
889 assert!(!root.join("promoted").exists(), "promotion must be opt-in");
891 fs::remove_dir_all(&root).ok();
892 }
893
894 #[test]
895 fn promotion_preserves_the_answer_and_the_durable_store_serves_it() {
896 use crate::field::observe::{self, ObserveRequest, Representation, Selector};
897 use crate::field::promote::PromotePolicy;
898 let root = temp_root("promote-on");
899 let mut store = FieldStore::open(&root).unwrap();
900 store.set_promote(PromotePolicy {
902 enabled: true,
903 min_hits: 1,
904 rent_per_byte: 0,
905 ..PromotePolicy::default()
906 });
907 let id = store.ingest(&tiny_descriptor(), Limits::DEFAULT).unwrap();
908 let req = ObserveRequest::new(
909 Selector::ByteRange { offset: 4, len: 5 },
910 Representation::ExactBytes,
911 );
912 let (cold, _, _) = observe::observe(&mut store, &id, &req, Limits::DEFAULT).unwrap();
913 let (warm, _, _) = observe::observe(&mut store, &id, &req, Limits::DEFAULT).unwrap();
914 assert_eq!(cold, warm, "promotion must not change the answer");
915 let promoted = store.promoted().unwrap();
917 let ob = promoted.output_bytes().unwrap();
918 assert!(
919 ob > 0,
920 "the reused node must be in the durable store (output_bytes={ob}, promoted_dir_exists={})",
921 root.join("promoted").exists()
922 );
923 crate::field::cache::DerivedCache::open(root.join("cache"))
925 .unwrap()
926 .clear()
927 .unwrap();
928 let (after, _, _) = observe::observe(&mut store, &id, &req, Limits::DEFAULT).unwrap();
929 assert_eq!(cold, after, "the promoted store must yield the same answer");
930 fs::remove_dir_all(&root).ok();
931 }
932}
933
934#[cfg(all(test, feature = "entropyfs-store"))]
937mod entropyfs_field_tests {
938 use super::*;
939 use crate::field::observe::{self, ObserveRequest, Representation, Selector};
940 use crate::field::provenance::AnswerValue;
941 use crate::store::seed_closure;
942
943 fn temp_root(label: &str) -> PathBuf {
944 let mut p = std::env::temp_dir();
945 p.push(format!(
946 "vole-field-entropyfs-{label}-{}-{}",
947 std::process::id(),
948 std::time::SystemTime::now()
949 .duration_since(std::time::UNIX_EPOCH)
950 .unwrap()
951 .as_nanos()
952 ));
953 p
954 }
955
956 fn tiny_descriptor() -> Vec<u8> {
957 use crate::container::{Descriptor, ObjectSource};
958 use crate::dra::{Op, Program};
959 let source = b"the exact field bytes";
960 let d = Descriptor {
961 universe: crate::container::UNIVERSE.to_string(),
962 source_format: crate::SOURCE_FORMAT_OPAQUE,
963 format_basis: "opaque:field-entropyfs-test".to_string(),
964 models: vec![],
965 channels: vec![],
966 objects: vec![ObjectSource::Inline(source.to_vec())],
967 program: Program::new(vec![Op::EmitObject { object_id: 0 }]),
968 observation_index: None,
969 seek_directory: false,
970 checkpoints: None,
971 source_sha256: crate::integrity::sha256(source),
972 source_len: source.len() as u64,
973 };
974 d.serialize().unwrap().0
975 }
976
977 #[test]
978 fn ingest_observe_and_exact_materialize_through_the_engine() {
979 let root = temp_root("rt");
980 let descriptor = tiny_descriptor();
981 let expected = b"the exact field bytes";
982 let mut store = FieldStore::open_entropyfs(&root).unwrap();
983 let id = store.ingest(&descriptor, Limits::DEFAULT).unwrap();
984 let field = Field::open(&store, &id, Limits::DEFAULT).unwrap();
985 assert_eq!(field.materialize_exact(Limits::DEFAULT).unwrap(), expected);
986 let mut budget = dag::EvalBudget::default();
988 assert_eq!(
989 field
990 .materialize_node(&field.manifest().root_node, Limits::DEFAULT, &mut budget)
991 .unwrap(),
992 expected
993 );
994 let req = ObserveRequest::new(
996 Selector::ByteRange { offset: 4, len: 5 },
997 Representation::ExactBytes,
998 );
999 let (answer, _stats, _) = observe::observe(&mut store, &id, &req, Limits::DEFAULT).unwrap();
1000 match answer.value {
1001 AnswerValue::Bytes(b) => assert_eq!(b, b"exact"),
1002 other => panic!("expected bytes, got {other:?}"),
1003 }
1004 assert_eq!(
1007 store.list_fields().unwrap_err().class(),
1008 crate::ErrorClass::UnsupportedFeature
1009 );
1010 std::fs::remove_dir_all(&root).ok();
1011 }
1012
1013 #[test]
1014 fn cross_process_reopen_reads_what_a_previous_handle_wrote() {
1015 let root = temp_root("reopen");
1016 let descriptor = tiny_descriptor();
1017 let (id, root_node) = {
1020 let mut store = FieldStore::open_entropyfs(&root).unwrap();
1021 let id = store.ingest(&descriptor, Limits::DEFAULT).unwrap();
1022 let node = store.get_field(&id).unwrap().root_node;
1023 store.sync().unwrap();
1025 (id, node)
1026 };
1027 let store = FieldStore::open_entropyfs(&root).unwrap();
1029 let manifest = store.get_field(&id).unwrap();
1030 assert_eq!(manifest.content_id(), id);
1031 let field = Field::open(&store, &id, Limits::DEFAULT).unwrap();
1032 assert_eq!(
1033 field.materialize_exact(Limits::DEFAULT).unwrap(),
1034 b"the exact field bytes"
1035 );
1036 let mut budget = dag::EvalBudget::default();
1037 assert_eq!(
1038 field
1039 .materialize_node(&root_node, Limits::DEFAULT, &mut budget)
1040 .unwrap(),
1041 b"the exact field bytes"
1042 );
1043 std::fs::remove_dir_all(&root).ok();
1044 }
1045
1046 #[test]
1047 fn list_nodes_declines_so_gc_cannot_sweep() {
1048 let root = temp_root("gc");
1049 let mut store = FieldStore::open_entropyfs(&root).unwrap();
1050 let id = store.ingest(&tiny_descriptor(), Limits::DEFAULT).unwrap();
1051 let manifest = store.get_field(&id).unwrap();
1052 let e = store.seeds().list_nodes().unwrap_err();
1054 assert_eq!(e.class(), crate::ErrorClass::UnsupportedFeature);
1055 let reachable =
1058 seed_closure(store.seeds(), &[manifest.root_node], |_| Ok(Vec::new())).unwrap();
1059 assert!(reachable.contains(&manifest.root_node));
1060 std::fs::remove_dir_all(&root).ok();
1061 }
1062}
1063
1064#[cfg(test)]
1068mod packed_field_tests {
1069 use super::*;
1070 use crate::field::observe::{self, ObserveRequest, Representation, Selector};
1071 use crate::field::provenance::AnswerValue;
1072
1073 fn temp_root(label: &str) -> PathBuf {
1074 let mut p = std::env::temp_dir();
1075 p.push(format!(
1076 "vole-field-packed-{label}-{}-{}",
1077 std::process::id(),
1078 std::time::SystemTime::now()
1079 .duration_since(std::time::UNIX_EPOCH)
1080 .unwrap()
1081 .as_nanos()
1082 ));
1083 p
1084 }
1085
1086 fn tiny_descriptor() -> Vec<u8> {
1087 use crate::container::{Descriptor, ObjectSource};
1088 use crate::dra::{Op, Program};
1089 let source = b"the exact field bytes";
1090 let d = Descriptor {
1091 universe: crate::container::UNIVERSE.to_string(),
1092 source_format: crate::SOURCE_FORMAT_OPAQUE,
1093 format_basis: "opaque:field-packed-test".to_string(),
1094 models: vec![],
1095 channels: vec![],
1096 objects: vec![ObjectSource::Inline(source.to_vec())],
1097 program: Program::new(vec![Op::EmitObject { object_id: 0 }]),
1098 observation_index: None,
1099 seek_directory: false,
1100 checkpoints: None,
1101 source_sha256: crate::integrity::sha256(source),
1102 source_len: source.len() as u64,
1103 };
1104 d.serialize().unwrap().0
1105 }
1106
1107 #[test]
1108 fn packed_matches_fs_for_ingest_observe_and_exact_materialize() {
1109 let fs_root = temp_root("fs");
1110 let pk_root = temp_root("packed");
1111 let descriptor = tiny_descriptor();
1112 let expected = b"the exact field bytes";
1113
1114 let mut fs_store = FieldStore::open(&fs_root).unwrap();
1115 let fs_id = fs_store.ingest(&descriptor, Limits::DEFAULT).unwrap();
1116 fs_store.sync().unwrap();
1117
1118 let mut pk_store = FieldStore::open_packed(&pk_root).unwrap();
1119 let pk_id = pk_store.ingest(&descriptor, Limits::DEFAULT).unwrap();
1120 pk_store.sync().unwrap();
1121
1122 assert_eq!(
1125 fs_id, pk_id,
1126 "the packed backend must not change the FieldId"
1127 );
1128 assert_eq!(
1129 fs_store.get_field(&fs_id).unwrap().root_node,
1130 pk_store.get_field(&pk_id).unwrap().root_node,
1131 "node identity is content-derived and backend-independent"
1132 );
1133
1134 {
1136 let f = Field::open(&fs_store, &fs_id, Limits::DEFAULT).unwrap();
1137 assert_eq!(f.materialize_exact(Limits::DEFAULT).unwrap(), expected);
1138 let p = Field::open(&pk_store, &pk_id, Limits::DEFAULT).unwrap();
1139 assert_eq!(p.materialize_exact(Limits::DEFAULT).unwrap(), expected);
1140 }
1141
1142 let req = ObserveRequest::new(
1144 Selector::ByteRange { offset: 4, len: 5 },
1145 Representation::ExactBytes,
1146 );
1147 let (fs_ans, _, _) =
1148 observe::observe(&mut fs_store, &fs_id, &req, Limits::DEFAULT).unwrap();
1149 let (pk_ans, _, _) =
1150 observe::observe(&mut pk_store, &pk_id, &req, Limits::DEFAULT).unwrap();
1151 match (&fs_ans.value, &pk_ans.value) {
1152 (AnswerValue::Bytes(a), AnswerValue::Bytes(b)) => {
1153 assert_eq!(a, b, "packed and fs observations must agree");
1154 assert_eq!(b, b"exact");
1155 }
1156 other => panic!("expected byte answers, got {other:?}"),
1157 }
1158
1159 std::fs::remove_dir_all(&fs_root).ok();
1160 std::fs::remove_dir_all(&pk_root).ok();
1161 }
1162
1163 #[test]
1164 fn packed_reopen_serves_the_same_field() {
1165 let root = temp_root("reopen");
1166 let descriptor = tiny_descriptor();
1167 let id = {
1168 let mut store = FieldStore::open_packed(&root).unwrap();
1169 let id = store.ingest(&descriptor, Limits::DEFAULT).unwrap();
1170 store.sync().unwrap();
1172 id
1173 };
1174 let store = FieldStore::open_packed(&root).unwrap();
1175 let field = Field::open(&store, &id, Limits::DEFAULT).unwrap();
1176 assert_eq!(
1177 field.materialize_exact(Limits::DEFAULT).unwrap(),
1178 b"the exact field bytes"
1179 );
1180 std::fs::remove_dir_all(&root).ok();
1181 }
1182}