vole_document/field/
mod.rs

1//! The persistent procedural document field (Phase 11).
2//!
3//! A [`Field`] is the queryable, content-addressed procedural substrate behind a
4//! document. It is opened from a [`FieldStore`] by a [`FieldId`] and can:
5//!
6//! * materialize the exact original source bytes ([`Field::materialize_exact`]),
7//!   exactly as the standalone `.voldoc` form does, and
8//! * serve derived/structured observations from the persisted seed DAG without
9//!   reconstructing the whole document.
10//!
11//! The exact archival authority is the serialized `.voldoc` descriptor, stored as
12//! one blob; the seed DAG and the hierarchical index are additional persisted
13//! state. A field never weakens `materialize(root) == original_bytes` (ADR-0024).
14
15pub mod cache;
16pub mod dag;
17pub mod derive;
18pub mod edit;
19pub mod explain;
20pub mod index;
21pub mod ingest;
22pub mod manifest;
23pub mod node;
24pub mod observe;
25pub mod partial;
26pub mod plan;
27pub mod provenance;
28pub mod share;
29
30pub use manifest::{FieldId, FieldRoot};
31
32use std::fs;
33use std::path::{Path, PathBuf};
34
35#[cfg(feature = "entropyfs-store")]
36use std::sync::Arc;
37
38#[cfg(feature = "entropyfs-store")]
39use entropyfs::engine::BlobId;
40
41use crate::container::ParsedDescriptor;
42use crate::error::{Error, Result};
43use crate::limits::Limits;
44#[cfg(feature = "entropyfs-store")]
45use crate::store::{EntropyFsStore, map_engine_error};
46use crate::store::{FsSeedStore, Id, IoCounters, IoSnapshot, NodeId, SeedStore};
47
48use self::manifest::FIELD_ROOT_DOMAIN;
49use self::node::{NodeKind, SeedNode};
50
51/// The canonical universe string for a Phase-11 field.
52pub const FIELD_UNIVERSE: &str = "vole-document;universe;phase11;exact-bytes;dra-8;opaque+entropy+pdf+channels+offsets+packed+packed-channels+deflate-replay-preflate-0.7.6-experimental+observation-index-v1+seek-directory-v1+external-objects-v1+procedural-seed-field-v1+hier-index-v1";
53
54/// A cheaply cloneable handle to a field's seed substrate.
55///
56/// Cloning shares the *same* underlying substrate and I/O counters, so an
57/// observation can mint a detached seed handle without borrowing the
58/// [`FieldStore`] that owns it (the evaluation core holds `&mut FieldStore`).
59/// Every variant owns a reference-counted engine or a path — never a second lock:
60/// an `entropyfs-store` field uses exactly **one** EntropyFS engine for its
61/// descriptors, manifests, and seed nodes, so there is no per-observation reopen.
62#[derive(Clone)]
63pub(crate) enum SeedSubstrate {
64    /// Plain files under `<root>/seed` ([`FsSeedStore`]).
65    Fs { root: PathBuf, io: IoCounters },
66    /// One engine blob per node, through the store's shared EntropyFS engine.
67    #[cfg(feature = "entropyfs-store")]
68    EntropyFs {
69        store: Arc<EntropyFsStore>,
70        io: IoCounters,
71    },
72}
73
74impl SeedStore for SeedSubstrate {
75    fn put_node(&mut self, canonical: &[u8]) -> Result<NodeId> {
76        match self {
77            SeedSubstrate::Fs { root, io } => {
78                FsSeedStore::open_with_io(root, io.handle())?.put_node(canonical)
79            }
80            #[cfg(feature = "entropyfs-store")]
81            SeedSubstrate::EntropyFs { store, .. } => store.seed_put(canonical),
82        }
83    }
84
85    fn get_node(&self, id: &NodeId) -> Result<Vec<u8>> {
86        match self {
87            SeedSubstrate::Fs { root, io } => {
88                FsSeedStore::open_with_io(root, io.handle())?.get_node(id)
89            }
90            #[cfg(feature = "entropyfs-store")]
91            SeedSubstrate::EntropyFs { store, io } => {
92                let bytes = store.seed_get(id)?;
93                io.add_seed(bytes.len() as u64);
94                Ok(bytes)
95            }
96        }
97    }
98
99    fn get_node_range(&self, id: &NodeId, offset: u64, len: u64) -> Result<Vec<u8>> {
100        match self {
101            SeedSubstrate::Fs { root, io } => {
102                FsSeedStore::open_with_io(root, io.handle())?.get_node_range(id, offset, len)
103            }
104            #[cfg(feature = "entropyfs-store")]
105            SeedSubstrate::EntropyFs { store, io } => {
106                let bytes = store.seed_get_range(id, offset, len)?;
107                io.add_seed(bytes.len() as u64);
108                Ok(bytes)
109            }
110        }
111    }
112
113    fn contains_node(&self, id: &NodeId) -> Result<bool> {
114        match self {
115            SeedSubstrate::Fs { root, io } => {
116                FsSeedStore::open_with_io(root, io.handle())?.contains_node(id)
117            }
118            #[cfg(feature = "entropyfs-store")]
119            SeedSubstrate::EntropyFs { store, .. } => store.seed_contains(id),
120        }
121    }
122
123    fn list_nodes(&self) -> Result<Vec<(NodeId, u64)>> {
124        match self {
125            SeedSubstrate::Fs { root, io } => {
126                FsSeedStore::open_with_io(root, io.handle())?.list_nodes()
127            }
128            #[cfg(feature = "entropyfs-store")]
129            SeedSubstrate::EntropyFs { store, .. } => store.seed_list(),
130        }
131    }
132}
133
134/// Where a field's descriptor and manifest blobs live.
135///
136/// The `FieldStore` API never leaks which backend is in use: both store and fetch
137/// by content id (`Id` for descriptors, `FieldId` for manifests). Only [`stats`]
138/// distinguish them.
139///
140/// [`stats`]: crate::field::observe::ObserveStats
141enum BlobBackend {
142    /// Plain files: `descriptor/` and `field/` under the store root.
143    Fs,
144    /// One engine: a descriptor is stored raw (so the engine's `BlobId` equals the
145    /// descriptor `Id`), a manifest is stored domain-prefixed with
146    /// [`FIELD_ROOT_DOMAIN`] (so the engine's `BlobId` equals the manifest
147    /// `FieldId`). Both namespaces are content-addressed and cannot collide with a
148    /// seed node, whose bytes carry a different domain prefix.
149    #[cfg(feature = "entropyfs-store")]
150    EntropyFs(Arc<EntropyFsStore>),
151}
152
153impl BlobBackend {
154    fn descriptor_put(&self, root: &Path, bytes: &[u8]) -> Result<Id> {
155        match self {
156            BlobBackend::Fs => {
157                let id = Id::of(bytes);
158                let path = root.join("descriptor").join(id.to_hex());
159                if !path.exists() {
160                    write_atomic(&path, bytes)?;
161                }
162                Ok(id)
163            }
164            #[cfg(feature = "entropyfs-store")]
165            BlobBackend::EntropyFs(store) => {
166                let id = Id::of(bytes);
167                let blob = store
168                    .engine()
169                    .put_blob(bytes)
170                    .map_err(|e| map_engine_error("put_blob", &e))?;
171                debug_assert_eq!(blob.as_bytes(), id.as_bytes());
172                Ok(id)
173            }
174        }
175    }
176
177    fn descriptor_get(&self, root: &Path, id: &Id) -> Result<Vec<u8>> {
178        match self {
179            BlobBackend::Fs => {
180                let path = root.join("descriptor").join(id.to_hex());
181                fs::read(&path).map_err(|e| {
182                    if e.kind() == std::io::ErrorKind::NotFound {
183                        Error::missing_external_object(format!(
184                            "descriptor blob {id} is not present"
185                        ))
186                    } else {
187                        Error::io(format!("reading descriptor blob {id}: {e}"))
188                    }
189                })
190            }
191            #[cfg(feature = "entropyfs-store")]
192            BlobBackend::EntropyFs(store) => store
193                .engine()
194                .get_blob(BlobId::new(*id.as_bytes()))
195                .map_err(|e| map_engine_error("get_blob", &e)),
196        }
197    }
198
199    fn field_put(&self, root: &Path, manifest: &FieldRoot) -> Result<FieldId> {
200        let bytes = manifest.encode_canonical();
201        let id = FieldId::of_manifest(&bytes);
202        match self {
203            BlobBackend::Fs => {
204                let path = root.join("field").join(id.to_hex());
205                if !path.exists() {
206                    write_atomic(&path, &bytes)?;
207                }
208                Ok(id)
209            }
210            #[cfg(feature = "entropyfs-store")]
211            BlobBackend::EntropyFs(store) => {
212                let mut prefixed = Vec::with_capacity(FIELD_ROOT_DOMAIN.len() + bytes.len());
213                prefixed.extend_from_slice(FIELD_ROOT_DOMAIN);
214                prefixed.extend_from_slice(&bytes);
215                let blob = store
216                    .engine()
217                    .put_blob(&prefixed)
218                    .map_err(|e| map_engine_error("put_blob", &e))?;
219                debug_assert_eq!(blob.as_bytes(), id.as_bytes());
220                Ok(id)
221            }
222        }
223    }
224
225    fn field_get(&self, root: &Path, id: &FieldId) -> Result<Vec<u8>> {
226        match self {
227            BlobBackend::Fs => {
228                let path = root.join("field").join(id.to_hex());
229                fs::read(&path).map_err(|e| {
230                    if e.kind() == std::io::ErrorKind::NotFound {
231                        Error::missing_external_object(format!("field {id} is not present"))
232                    } else {
233                        Error::io(format!("reading field {id}: {e}"))
234                    }
235                })
236            }
237            #[cfg(feature = "entropyfs-store")]
238            BlobBackend::EntropyFs(store) => {
239                let blob = store
240                    .engine()
241                    .get_blob(BlobId::new(*id.as_bytes()))
242                    .map_err(|e| map_engine_error("get_blob", &e))?;
243                let rest = blob.strip_prefix(FIELD_ROOT_DOMAIN).ok_or_else(|| {
244                    Error::integrity_mismatch(format!(
245                        "field manifest {id} is missing its domain prefix"
246                    ))
247                })?;
248                Ok(rest.to_vec())
249            }
250        }
251    }
252}
253
254/// On-disk layout of a field store. Every namespace is content-addressed.
255///
256/// ```text
257/// <root>/
258///   descriptor/<64-hex>       serialized .voldoc descriptor blobs
259///   field/<64-hex>            canonical field manifests
260///   seed/<aa>/<bb>/<64-hex>   canonical seed nodes (FsSeedStore)
261///   index/<64-hex>            hierarchical index nodes (11.3)
262///   cache/                    derived observation cache (11.8)
263/// ```
264///
265/// An EntropyFS-backed store ([`FieldStore::open_entropyfs`], feature
266/// `entropyfs-store`) keeps the descriptor, manifest, and seed namespaces in
267/// `entropyfs/` (one engine blob each) and `index/`/`cache/` as files. The seed
268/// DAG is therefore persisted **one node per engine blob**, never as one coarse
269/// blob; EntropyFS sees only opaque bytes and VOLE owns node semantics (ADR-0025).
270/// Advisory engine accounting for an EntropyFS-backed field store.
271///
272/// These are EntropyFS's own numbers, not VOLE's; they are reported so a court can
273/// witness that the seed DAG is many individual engine blobs rather than one. They
274/// carry **no** decoder authority and no exactness meaning.
275#[cfg(feature = "entropyfs-store")]
276#[derive(Debug, Clone, Copy, PartialEq, Eq)]
277pub struct EngineStoreStats {
278    /// Files in the engine blob namespace (one per descriptor / manifest / node).
279    pub blob_count: u64,
280    /// Sum of materialized logical bytes across reachable inodes.
281    pub logical_bytes: u64,
282    /// Sum of segment-file lengths (physical store bytes).
283    pub physical_used_bytes: u64,
284}
285
286pub struct FieldStore {
287    root: PathBuf,
288    backend: BlobBackend,
289    seeds: SeedSubstrate,
290    io: IoCounters,
291}
292
293impl std::fmt::Debug for FieldStore {
294    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
295        f.debug_struct("FieldStore")
296            .field("root", &self.root)
297            .finish_non_exhaustive()
298    }
299}
300
301impl FieldStore {
302    /// Create or open a field store rooted at `root` (the filesystem backend).
303    pub fn open(root: impl AsRef<Path>) -> Result<Self> {
304        let root = root.as_ref().to_path_buf();
305        fs::create_dir_all(root.join("descriptor"))?;
306        fs::create_dir_all(root.join("field"))?;
307        fs::create_dir_all(root.join("index"))?;
308        fs::create_dir_all(root.join("cache"))?;
309        let io = IoCounters::new();
310        let seeds = SeedSubstrate::Fs {
311            root: root.clone(),
312            io: io.handle(),
313        };
314        Ok(FieldStore {
315            root,
316            backend: BlobBackend::Fs,
317            seeds,
318            io,
319        })
320    }
321
322    /// Create or open a field store whose descriptor, manifest, and seed
323    /// namespaces are served by one embedded EntropyFS engine at
324    /// `root/entropyfs` (feature `entropyfs-store`).
325    ///
326    /// The hierarchical observation index and the disposable derived cache remain
327    /// files under `<root>/index` and `<root>/cache`. Exactly one engine is opened;
328    /// the seed substrate and every observation share it by reference count, so a
329    /// field never opens a second engine over the same directory (which would
330    /// deadlock on the engine's exclusive lock).
331    #[cfg(feature = "entropyfs-store")]
332    pub fn open_entropyfs(root: impl AsRef<Path>) -> Result<Self> {
333        let root = root.as_ref().to_path_buf();
334        fs::create_dir_all(root.join("index"))?;
335        fs::create_dir_all(root.join("cache"))?;
336        let engine_root = root.join("entropyfs");
337        fs::create_dir_all(&engine_root)?;
338        // An empty engine directory is a fresh store; anything else is an
339        // existing one. (`Engine::open` cannot open a directory with no store.)
340        let fresh = fs::read_dir(&engine_root)?.next().is_none();
341        let engine = if fresh {
342            EntropyFsStore::create(&engine_root)?
343        } else {
344            EntropyFsStore::open(&engine_root)?
345        };
346        let engine = Arc::new(engine);
347        let io = IoCounters::new();
348        let seeds = SeedSubstrate::EntropyFs {
349            store: Arc::clone(&engine),
350            io: io.handle(),
351        };
352        Ok(FieldStore {
353            root,
354            backend: BlobBackend::EntropyFs(engine),
355            seeds,
356            io,
357        })
358    }
359
360    /// The physical-I/O counters shared by this store and its seed substrate.
361    ///
362    /// Every descriptor/manifest/index/seed read made through any handle derived
363    /// from this store is attributed here; an observation snapshots it before and
364    /// after to report its own physical bytes (review fix #1).
365    pub fn io(&self) -> &IoCounters {
366        &self.io
367    }
368
369    /// The store root directory.
370    pub fn root(&self) -> &Path {
371        &self.root
372    }
373
374    /// A detached, cheaply cloned seed handle that shares this store's substrate
375    /// and I/O counters (used by observations).
376    pub(crate) fn seed_substrate(&self) -> SeedSubstrate {
377        self.seeds.clone()
378    }
379
380    /// Whether the descriptor blob is a filesystem file supporting seek-based
381    /// partial reads. The EntropyFS backend stores it as an engine blob, so the
382    /// partial lane is unavailable and observations read the full descriptor.
383    pub(crate) fn supports_partial_descriptor(&self) -> bool {
384        matches!(self.backend, BlobBackend::Fs)
385    }
386
387    /// The seed store (for advanced callers and courts).
388    pub fn seeds(&self) -> &dyn SeedStore {
389        &self.seeds
390    }
391
392    /// Mutable seed store access.
393    pub fn seeds_mut(&mut self) -> &mut dyn SeedStore {
394        &mut self.seeds
395    }
396
397    /// Open the disposable derived observation cache (11.8) under this store.
398    ///
399    /// The cache is never normative: it can always be deleted and observations
400    /// remain correct by recomputation (ADR-0027).
401    pub fn cache(&self) -> Result<cache::DerivedCache> {
402        cache::DerivedCache::open(self.root.join("cache"))
403    }
404
405    /// Make every acknowledged write power-durable before the handle is dropped.
406    ///
407    /// The EntropyFS engine acks a `put_blob` at rename but does not barrier, so a
408    /// fresh open in another process can miss an unpublishied epoch. Calling this
409    /// after a mutation closes that gap. The filesystem backend writes atomically
410    /// (`tmp -> fsync -> rename`) and needs no barrier.
411    pub fn sync(&self) -> Result<()> {
412        match &self.backend {
413            BlobBackend::Fs => Ok(()),
414            #[cfg(feature = "entropyfs-store")]
415            BlobBackend::EntropyFs(store) => store.sync(),
416        }
417    }
418
419    /// Advisory engine accounting for an EntropyFS-backed store, or `None` for
420    /// the filesystem backend.
421    ///
422    /// `blob_count` is the number of files in the engine's blob namespace: every
423    /// descriptor, manifest, and seed node is exactly one blob, so a seed DAG of
424    /// `n` nodes adds `n` blobs (never one coarse blob). This is a snapshot, not a
425    /// claim that the engine understands procedural state (ADR-0025).
426    #[cfg(feature = "entropyfs-store")]
427    pub fn engine_stats(&self) -> Result<Option<EngineStoreStats>> {
428        match &self.backend {
429            BlobBackend::Fs => Ok(None),
430            BlobBackend::EntropyFs(store) => {
431                let m = store
432                    .engine()
433                    .metrics()
434                    .map_err(|e| map_engine_error("metrics", &e))?;
435                Ok(Some(EngineStoreStats {
436                    blob_count: m.accounting.blob_count,
437                    logical_bytes: m.accounting.logical_bytes,
438                    physical_used_bytes: m.accounting.physical_used_bytes,
439                }))
440            }
441        }
442    }
443
444    /// The filesystem path of a descriptor blob, or `None` when the backend keeps
445    /// it as an engine blob (in which case the partial lane is unavailable).
446    pub(crate) fn descriptor_path(&self, id: &Id) -> Option<PathBuf> {
447        match self.backend {
448            BlobBackend::Fs => Some(self.root.join("descriptor").join(id.to_hex())),
449            #[cfg(feature = "entropyfs-store")]
450            BlobBackend::EntropyFs(_) => None,
451        }
452    }
453
454    /// Store a serialized `.voldoc` descriptor as a content-addressed blob.
455    ///
456    /// Returns the blob's [`Id`] (`BLAKE3-256(bytes)`), which is what the field
457    /// manifest binds. Idempotent.
458    pub fn put_descriptor(&mut self, bytes: &[u8]) -> Result<Id> {
459        self.backend.descriptor_put(&self.root, bytes)
460    }
461
462    /// Fetch a descriptor blob, verifying its content id.
463    pub fn get_descriptor(&self, id: &Id) -> Result<Vec<u8>> {
464        let bytes = self.backend.descriptor_get(&self.root, id)?;
465        let actual = Id::of(&bytes);
466        if actual != *id {
467            return Err(Error::integrity_mismatch(format!(
468                "descriptor blob {id} hashes to {actual}"
469            )));
470        }
471        self.io.add_descriptor(bytes.len() as u64);
472        Ok(bytes)
473    }
474
475    /// Store a canonical field manifest.
476    pub fn put_field(&mut self, manifest: &FieldRoot) -> Result<FieldId> {
477        self.backend.field_put(&self.root, manifest)
478    }
479
480    /// Fetch a field manifest, verifying its content id and universe.
481    pub fn get_field(&self, id: &FieldId) -> Result<FieldRoot> {
482        let bytes = self.backend.field_get(&self.root, id)?;
483        let manifest = FieldRoot::decode_canonical(&bytes)?;
484        if manifest.content_id() != *id {
485            return Err(Error::integrity_mismatch(format!(
486                "field {id} manifest content id mismatch"
487            )));
488        }
489        self.io.add_manifest(bytes.len() as u64);
490        Ok(manifest)
491    }
492
493    /// List every stored field manifest id.
494    ///
495    /// The filesystem backend reads the `field/` directory. The EntropyFS backend
496    /// **declines** with `UnsupportedFeature`: the engine exposes no per-blob
497    /// enumeration, so a manifest cannot be listed — but any manifest remains
498    /// openable by its `FieldId` (`get_field`).
499    pub fn list_fields(&self) -> Result<Vec<FieldId>> {
500        if !matches!(self.backend, BlobBackend::Fs) {
501            return Err(Error::unsupported_feature(
502                "EntropyFS exposes no per-blob enumeration; a field store backed by it \
503                 cannot list its manifests (open a field by its FieldId instead)",
504            ));
505        }
506        let dir = self.root.join("field");
507        let mut out = Vec::new();
508        for entry in fs::read_dir(&dir)?.flatten() {
509            if let Some(name) = entry.file_name().to_str()
510                && let Ok(id) = FieldId::from_hex(name)
511            {
512                out.push(id);
513            }
514        }
515        out.sort_unstable();
516        Ok(out)
517    }
518
519    /// Ingest a serialized `.voldoc` descriptor as a new field.
520    ///
521    /// Stage A (durable exact capture): store the descriptor blob and an exact
522    /// `DocumentExact` root node, then write the manifest. Deeper Stage-B
523    /// procedural nodes are added by [`crate::field::ingest`].
524    pub fn ingest(&mut self, descriptor_bytes: &[u8], limits: Limits) -> Result<FieldId> {
525        // The descriptor must parse and materialize exactly: ingest never invents
526        // authority it cannot reproduce.
527        let parsed = crate::container::Descriptor::parse(descriptor_bytes, limits)?;
528        let source = crate::materialize::materialize(&parsed, limits)?;
529        let descriptor_id = self.put_descriptor(descriptor_bytes)?;
530
531        let root = SeedNode::new(
532            NodeKind::DocumentExact,
533            source.len() as u64,
534            Vec::new(),
535            Vec::new(),
536            "field:document-exact",
537        );
538        let root_id = self.seeds.put_node(&root.encode_canonical())?;
539
540        let manifest = FieldRoot {
541            universe_id: crate::container::universe_id_from_str(FIELD_UNIVERSE),
542            source_sha256: parsed.descriptor.source_sha256,
543            source_len: parsed.descriptor.source_len,
544            descriptor_id: *descriptor_id.as_bytes(),
545            root_node: root_id,
546            index_root: manifest::ABSENT_ROOT,
547            node_count: 1,
548            index_node_count: 0,
549            provenance: format!("field:ingest;{}", parsed.descriptor.format_basis),
550        };
551        self.put_field(&manifest)
552    }
553}
554
555/// Write bytes to `path` atomically (`tmp -> fsync -> rename`).
556pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<()> {
557    use std::io::Write;
558    let dir = path
559        .parent()
560        .ok_or_else(|| Error::internal_invariant("atomic write path has no parent"))?;
561    fs::create_dir_all(dir)?;
562    let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("blob");
563    let tmp = dir.join(format!(".{name}.tmp-{}", std::process::id()));
564    {
565        let mut f = fs::File::create(&tmp)?;
566        f.write_all(bytes)?;
567        f.sync_all()?;
568    }
569    fs::rename(&tmp, path)?;
570    Ok(())
571}
572
573/// An opened field: the manifest plus its parsed exact descriptor.
574pub struct Field {
575    store_root: PathBuf,
576    manifest: FieldRoot,
577    parsed: ParsedDescriptor,
578    descriptor_bytes: Vec<u8>,
579    /// A detached seed handle sharing the store's substrate, so a field opened
580    /// from an EntropyFS-backed store materializes its seed nodes through the
581    /// same engine (never a second opener).
582    seeds: SeedSubstrate,
583    /// The physical bytes this `open` fetched to load the manifest and the
584    /// descriptor blob. An observation attributes exactly these to its own
585    /// `descriptor_bytes_read`/`manifest_bytes_read` (review fix #1/#2).
586    open_io: IoSnapshot,
587}
588
589impl std::fmt::Debug for Field {
590    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
591        f.debug_struct("Field")
592            .field("manifest", &self.manifest.content_id())
593            .field("source_len", &self.manifest.source_len)
594            .finish_non_exhaustive()
595    }
596}
597
598impl Field {
599    /// Open a field by id, loading and verifying its descriptor.
600    pub fn open(store: &FieldStore, id: &FieldId, limits: Limits) -> Result<Field> {
601        let io_before = store.io().snapshot();
602        let manifest = store.get_field(id)?;
603        Field::open_after_manifest(store, manifest, io_before, limits)
604    }
605
606    /// The body of [`Field::open`] from an already-read manifest. `io_before` is
607    /// the snapshot `open_io` is measured from: pass one taken before the
608    /// manifest read to charge it here (the ordinary path), or after it to charge
609    /// it elsewhere (the narrow probe, which already counted it in `base_io`).
610    pub(crate) fn open_after_manifest(
611        store: &FieldStore,
612        manifest: FieldRoot,
613        io_before: IoSnapshot,
614        limits: Limits,
615    ) -> Result<Field> {
616        let descriptor_bytes = store.get_descriptor(&Id::from_bytes(manifest.descriptor_id))?;
617        let open_io = io_before.delta(&store.io().snapshot());
618        let mut parsed = crate::container::Descriptor::parse(&descriptor_bytes, limits)?;
619        parsed.universe_id = manifest.universe_id;
620        // The manifest must agree with the descriptor it binds.
621        if parsed.descriptor.source_len != manifest.source_len
622            || parsed.descriptor.source_sha256 != manifest.source_sha256
623        {
624            return Err(Error::integrity_mismatch(
625                "field manifest does not match its descriptor's declared source",
626            ));
627        }
628        Ok(Field {
629            store_root: store.root().to_path_buf(),
630            manifest,
631            parsed,
632            descriptor_bytes,
633            seeds: store.seed_substrate(),
634            open_io,
635        })
636    }
637
638    /// The physical bytes fetched to open this field (manifest + descriptor).
639    pub(crate) fn open_io(&self) -> IoSnapshot {
640        self.open_io
641    }
642
643    /// The field manifest.
644    pub fn manifest(&self) -> &FieldRoot {
645        &self.manifest
646    }
647
648    /// The field id.
649    pub fn id(&self) -> FieldId {
650        self.manifest.content_id()
651    }
652
653    /// The parsed exact descriptor.
654    pub fn parsed(&self) -> &ParsedDescriptor {
655        &self.parsed
656    }
657
658    /// The raw serialized descriptor bytes.
659    pub fn descriptor_bytes(&self) -> &[u8] {
660        &self.descriptor_bytes
661    }
662
663    /// The store root this field was opened from.
664    pub fn store_root(&self) -> &Path {
665        &self.store_root
666    }
667
668    /// Materialize the exact original source bytes.
669    ///
670    /// This is the archival authority: it is byte-identical to the standalone
671    /// `.voldoc` materialization and is the only path that verifies the
672    /// whole-source SHA-256.
673    pub fn materialize_exact(&self, limits: Limits) -> Result<Vec<u8>> {
674        crate::materialize::materialize(&self.parsed, limits)
675    }
676
677    /// Materialize a seed node's output by id, using the field's own seed handle
678    /// (which for an EntropyFS-backed store is the shared engine, not a reopen).
679    pub fn materialize_node(
680        &self,
681        id: &NodeId,
682        limits: Limits,
683        budget: &mut dag::EvalBudget,
684    ) -> Result<Vec<u8>> {
685        let node = dag::load_node(&self.seeds, id)?;
686        dag::materialize_node(
687            &self.parsed,
688            &self.seeds,
689            &node,
690            limits,
691            budget,
692            node.limits.max_depth,
693        )
694    }
695}
696
697#[cfg(test)]
698mod tests {
699    use super::*;
700
701    fn temp_root(label: &str) -> PathBuf {
702        let mut p = std::env::temp_dir();
703        p.push(format!(
704            "vole-field-{label}-{}-{}",
705            std::process::id(),
706            std::time::SystemTime::now()
707                .duration_since(std::time::UNIX_EPOCH)
708                .unwrap()
709                .as_nanos()
710        ));
711        p
712    }
713
714    fn tiny_descriptor() -> Vec<u8> {
715        use crate::container::{Descriptor, ObjectSource};
716        use crate::dra::{Op, Program};
717        let source = b"the exact field bytes";
718        let d = Descriptor {
719            universe: crate::container::UNIVERSE.to_string(),
720            source_format: crate::SOURCE_FORMAT_OPAQUE,
721            format_basis: "opaque:field-test".to_string(),
722            models: vec![],
723            channels: vec![],
724            objects: vec![ObjectSource::Inline(source.to_vec())],
725            program: Program::new(vec![Op::EmitObject { object_id: 0 }]),
726            observation_index: None,
727            seek_directory: false,
728            source_sha256: crate::integrity::sha256(source),
729            source_len: source.len() as u64,
730        };
731        d.serialize().unwrap().0
732    }
733
734    #[test]
735    fn ingest_then_materialize_is_exact() {
736        let root = temp_root("rt");
737        let mut store = FieldStore::open(&root).unwrap();
738        let bytes = tiny_descriptor();
739        let id = store.ingest(&bytes, Limits::DEFAULT).unwrap();
740        let field = Field::open(&store, &id, Limits::DEFAULT).unwrap();
741        assert_eq!(
742            field.materialize_exact(Limits::DEFAULT).unwrap(),
743            b"the exact field bytes"
744        );
745        // The root node also materializes the exact source.
746        let mut budget = dag::EvalBudget::default();
747        let out = field
748            .materialize_node(&field.manifest.root_node, Limits::DEFAULT, &mut budget)
749            .unwrap();
750        assert_eq!(out, b"the exact field bytes");
751        fs::remove_dir_all(&root).ok();
752    }
753
754    #[test]
755    fn field_id_is_stable_and_reopenable() {
756        let root = temp_root("reopen");
757        let bytes = tiny_descriptor();
758        let id1 = {
759            let mut store = FieldStore::open(&root).unwrap();
760            store.ingest(&bytes, Limits::DEFAULT).unwrap()
761        };
762        let store = FieldStore::open(&root).unwrap();
763        let m = store.get_field(&id1).unwrap();
764        assert_eq!(m.content_id(), id1);
765        assert_eq!(store.list_fields().unwrap(), vec![id1]);
766        fs::remove_dir_all(&root).ok();
767    }
768}
769
770/// The EntropyFS-backed field store: the seed DAG persisted one engine blob per
771/// node, through the same engine as the descriptor and manifest namespaces.
772#[cfg(all(test, feature = "entropyfs-store"))]
773mod entropyfs_field_tests {
774    use super::*;
775    use crate::field::observe::{self, ObserveRequest, Representation, Selector};
776    use crate::field::provenance::AnswerValue;
777    use crate::store::seed_closure;
778
779    fn temp_root(label: &str) -> PathBuf {
780        let mut p = std::env::temp_dir();
781        p.push(format!(
782            "vole-field-entropyfs-{label}-{}-{}",
783            std::process::id(),
784            std::time::SystemTime::now()
785                .duration_since(std::time::UNIX_EPOCH)
786                .unwrap()
787                .as_nanos()
788        ));
789        p
790    }
791
792    fn tiny_descriptor() -> Vec<u8> {
793        use crate::container::{Descriptor, ObjectSource};
794        use crate::dra::{Op, Program};
795        let source = b"the exact field bytes";
796        let d = Descriptor {
797            universe: crate::container::UNIVERSE.to_string(),
798            source_format: crate::SOURCE_FORMAT_OPAQUE,
799            format_basis: "opaque:field-entropyfs-test".to_string(),
800            models: vec![],
801            channels: vec![],
802            objects: vec![ObjectSource::Inline(source.to_vec())],
803            program: Program::new(vec![Op::EmitObject { object_id: 0 }]),
804            observation_index: None,
805            seek_directory: false,
806            source_sha256: crate::integrity::sha256(source),
807            source_len: source.len() as u64,
808        };
809        d.serialize().unwrap().0
810    }
811
812    #[test]
813    fn ingest_observe_and_exact_materialize_through_the_engine() {
814        let root = temp_root("rt");
815        let descriptor = tiny_descriptor();
816        let expected = b"the exact field bytes";
817        let mut store = FieldStore::open_entropyfs(&root).unwrap();
818        let id = store.ingest(&descriptor, Limits::DEFAULT).unwrap();
819        let field = Field::open(&store, &id, Limits::DEFAULT).unwrap();
820        assert_eq!(field.materialize_exact(Limits::DEFAULT).unwrap(), expected);
821        // The DocumentExact seed node materializes through the engine itself.
822        let mut budget = dag::EvalBudget::default();
823        assert_eq!(
824            field
825                .materialize_node(&field.manifest().root_node, Limits::DEFAULT, &mut budget)
826                .unwrap(),
827            expected
828        );
829        // A narrow observation resolves against the engine-backed seed substrate.
830        let req = ObserveRequest::new(
831            Selector::ByteRange { offset: 4, len: 5 },
832            Representation::ExactBytes,
833        );
834        let (answer, _stats, _) = observe::observe(&mut store, &id, &req, Limits::DEFAULT).unwrap();
835        match answer.value {
836            AnswerValue::Bytes(b) => assert_eq!(b, b"exact"),
837            other => panic!("expected bytes, got {other:?}"),
838        }
839        // `list_fields` cannot enumerate the engine namespace, but the manifest is
840        // openable by id.
841        assert_eq!(
842            store.list_fields().unwrap_err().class(),
843            crate::ErrorClass::UnsupportedFeature
844        );
845        std::fs::remove_dir_all(&root).ok();
846    }
847
848    #[test]
849    fn cross_process_reopen_reads_what_a_previous_handle_wrote() {
850        let root = temp_root("reopen");
851        let descriptor = tiny_descriptor();
852        // The first handle (a first process) writes, then is dropped, releasing the
853        // engine's exclusive lock.
854        let (id, root_node) = {
855            let mut store = FieldStore::open_entropyfs(&root).unwrap();
856            let id = store.ingest(&descriptor, Limits::DEFAULT).unwrap();
857            let node = store.get_field(&id).unwrap().root_node;
858            // Barrier so the engine publishes the epoch before this handle drops.
859            store.sync().unwrap();
860            (id, node)
861        };
862        // A brand-new handle over the same directory (a second process) reads it.
863        let store = FieldStore::open_entropyfs(&root).unwrap();
864        let manifest = store.get_field(&id).unwrap();
865        assert_eq!(manifest.content_id(), id);
866        let field = Field::open(&store, &id, Limits::DEFAULT).unwrap();
867        assert_eq!(
868            field.materialize_exact(Limits::DEFAULT).unwrap(),
869            b"the exact field bytes"
870        );
871        let mut budget = dag::EvalBudget::default();
872        assert_eq!(
873            field
874                .materialize_node(&root_node, Limits::DEFAULT, &mut budget)
875                .unwrap(),
876            b"the exact field bytes"
877        );
878        std::fs::remove_dir_all(&root).ok();
879    }
880
881    #[test]
882    fn list_nodes_declines_so_gc_cannot_sweep() {
883        let root = temp_root("gc");
884        let mut store = FieldStore::open_entropyfs(&root).unwrap();
885        let id = store.ingest(&tiny_descriptor(), Limits::DEFAULT).unwrap();
886        let manifest = store.get_field(&id).unwrap();
887        // A mark-and-sweep needs `list_nodes`, which the engine cannot provide.
888        let e = store.seeds().list_nodes().unwrap_err();
889        assert_eq!(e.class(), crate::ErrorClass::UnsupportedFeature);
890        // The reachable closure is still derivable, because it descends by explicit
891        // id and never enumerates the store.
892        let reachable =
893            seed_closure(store.seeds(), &[manifest.root_node], |_| Ok(Vec::new())).unwrap();
894        assert!(reachable.contains(&manifest.root_node));
895        std::fs::remove_dir_all(&root).ok();
896    }
897}
vole_document/field/mod.rs

vole_document/field/
mod.rs
