Skip to main content

datumesh_engine/
engine.rs

1//! [`Engine`], [`EngineBuilder`] and [`Answer`].
2
3use std::borrow::Cow;
4use std::collections::HashMap;
5use std::fmt;
6use std::path::PathBuf;
7use std::sync::{Arc, Mutex};
8
9use datumesh_core::{lexical, Cell, CellRef, Descriptor, KeyDoc, Resolver, Scalar, SigningKey};
10use datumesh_store::Store;
11
12use crate::cache::{Cache, Parsed};
13use crate::check::{check_args, check_no_slice};
14use crate::clock::{Clock, SystemClock};
15use crate::error::{ConfigError, EngineError, EvalError, StoreError};
16use crate::eval::{is_instant, parse_stored, Done, Req};
17use crate::memo::Memo;
18use crate::runtime::Runtime;
19
20/// The default depth limit: frames on the evaluation stack.
21pub const DEFAULT_DEPTH_LIMIT: usize = 10_000;
22/// The default fuel of a request: runtime steps plus input reads.
23pub const DEFAULT_FUEL: u64 = 10_000_000;
24/// The default memo capacity, in entries.
25pub const DEFAULT_MEMO_CAPACITY: usize = 1_000_000;
26
27/// The answer to "the value of this cell as of A".
28#[derive(Clone, Debug, PartialEq, Eq)]
29pub enum Answer {
30    /// The receipt that answers, canonical JSON text: for a stored cell the publisher's
31    /// receipt as the store holds it; for a computed cell one signed by the engine.
32    Receipt(Vec<u8>),
33    /// Nothing answers as of A: the function is unknown then, a stored cell has no version
34    /// answering then, or a computed cell reads such an input (and its runtime did not handle
35    /// it).
36    NotCovered,
37}
38
39/// The function computing a computed receipt's `summary`.
40pub type SummaryFn = dyn Fn(&Cell) -> String + Send + Sync;
41
42/// Builds an [`Engine`]: [`Engine::builder`].
43pub struct EngineBuilder {
44    pub(crate) store: Arc<dyn Store + Send + Sync>,
45    pub(crate) host: String,
46    pub(crate) key: Option<SigningKey>,
47    pub(crate) key_file: Option<PathBuf>,
48    pub(crate) runtimes: Vec<Arc<dyn Runtime>>,
49    pub(crate) depth_limit: usize,
50    pub(crate) fuel: u64,
51    pub(crate) memo_capacity: usize,
52    pub(crate) licence: String,
53    pub(crate) summary: Option<Arc<SummaryFn>>,
54    pub(crate) clock: Arc<dyn Clock>,
55}
56
57impl EngineBuilder {
58    /// The mesh's signing key.
59    pub fn signing_key(mut self, key: SigningKey) -> Self {
60        self.key = Some(key);
61        self.key_file = None;
62        self
63    }
64
65    /// The mesh's signing key, from a file holding its 32-byte seed: the 32 bytes, or 64 hex
66    /// digits (an optional final newline). The only file the engine reads.
67    pub fn signing_key_file(mut self, path: impl Into<PathBuf>) -> Self {
68        self.key_file = Some(path.into());
69        self.key = None;
70        self
71    }
72
73    /// Adds a runtime, for the language its [`Runtime::lang`] names.
74    pub fn runtime(mut self, runtime: Arc<dyn Runtime>) -> Self {
75        self.runtimes.push(runtime);
76        self
77    }
78
79    /// The depth limit: frames on the evaluation stack (default 10,000).
80    pub fn depth_limit(mut self, limit: usize) -> Self {
81        self.depth_limit = limit;
82        self
83    }
84
85    /// The fuel of each request: runtime steps plus input reads (default 10,000,000).
86    pub fn fuel(mut self, fuel: u64) -> Self {
87        self.fuel = fuel;
88        self
89    }
90
91    /// How many answers the memo keeps (default 1,000,000); when full it starts afresh. 0
92    /// turns the memo off: every request then validates through the store.
93    pub fn memo_capacity(mut self, entries: usize) -> Self {
94        self.memo_capacity = entries;
95        self
96    }
97
98    /// The `licence` of computed receipts: an SPDX identifier or an https URL (default
99    /// `CC-BY-4.0`).
100    pub fn licence(mut self, licence: impl Into<String>) -> Self {
101        self.licence = licence.into();
102        self
103    }
104
105    /// The `summary` of a computed receipt, from its cell. The default says which function
106    /// the mesh computed: `"<namespace>/<function> computed by <mesh host>."`.
107    pub fn summary(mut self, f: impl Fn(&Cell) -> String + Send + Sync + 'static) -> Self {
108        self.summary = Some(Arc::new(f));
109        self
110    }
111
112    /// The clock (default [`SystemClock`]).
113    pub fn clock(mut self, clock: Arc<dyn Clock>) -> Self {
114        self.clock = clock;
115        self
116    }
117
118    /// Checks the configuration and builds the engine.
119    pub fn build(self) -> Result<Engine, ConfigError> {
120        lexical::check_dns_name(&self.host, "$.host").map_err(ConfigError::Host)?;
121        let key = match (self.key, &self.key_file) {
122            (Some(k), _) => k,
123            (None, Some(path)) => read_seed(path)?,
124            (None, None) => return Err(ConfigError::NoSigningKey),
125        };
126        lexical::check_licence(&self.licence, "$.licence").map_err(ConfigError::Licence)?;
127        let mut runtimes: HashMap<String, Arc<dyn Runtime>> = HashMap::new();
128        for rt in self.runtimes {
129            let lang = rt.lang().to_string();
130            if lang == STORED {
131                return Err(ConfigError::Runtime("stored is answered by the engine, not a runtime".into()));
132            }
133            if runtimes.insert(lang.clone(), rt).is_some() {
134                return Err(ConfigError::Runtime(format!("two runtimes for {lang}")));
135            }
136        }
137        let published = match self.store.namespace(&self.host) {
138            Ok(Some(ns)) => KeyDoc::from_json(&self.host, &ns.keydoc)
139                .ok()
140                .and_then(|doc| doc.key(key.key_id()).map(|k| k.public == key.public_key()))
141                .unwrap_or(false),
142            Ok(None) => false,
143            Err(e) => return Err(ConfigError::Store(StoreError::new("namespace", e))),
144        };
145        if !published {
146            return Err(ConfigError::KeyNotPublished { host: self.host, key_id: key.key_id().to_string() });
147        }
148        Ok(Engine {
149            store: self.store,
150            host: self.host,
151            key,
152            runtimes,
153            depth_limit: self.depth_limit,
154            fuel: self.fuel,
155            licence: self.licence,
156            summary: self.summary,
157            clock: self.clock,
158            memo: Memo::new(self.memo_capacity),
159            descriptors: Cache::new(DESCRIPTOR_CACHE),
160            receipts: Cache::new(RECEIPT_CACHE),
161            evaluator: Mutex::new(()),
162        })
163    }
164}
165
166impl fmt::Debug for EngineBuilder {
167    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
168        f.debug_struct("EngineBuilder").field("host", &self.host).finish_non_exhaustive()
169    }
170}
171
172/// The engine: answers the value of a cell as of a time. See the crate documentation.
173pub struct Engine {
174    pub(crate) store: Arc<dyn Store + Send + Sync>,
175    pub(crate) host: String,
176    pub(crate) key: SigningKey,
177    pub(crate) runtimes: HashMap<String, Arc<dyn Runtime>>,
178    pub(crate) depth_limit: usize,
179    pub(crate) fuel: u64,
180    pub(crate) licence: String,
181    pub(crate) summary: Option<Arc<SummaryFn>>,
182    pub(crate) clock: Arc<dyn Clock>,
183    pub(crate) memo: Memo,
184    /// Parsed descriptors by function hash (immutable: content-addressed).
185    pub(crate) descriptors: Cache<Arc<Descriptor>>,
186    /// Parsed receipts by hash (immutable: content-addressed).
187    pub(crate) receipts: Cache<Arc<Parsed>>,
188    /// Evaluations run one at a time (single-flight); answers from the memo do not wait.
189    pub(crate) evaluator: Mutex<()>,
190}
191
192/// The language of stored functions, answered by the engine itself.
193pub(crate) const STORED: &str = "stored";
194const DESCRIPTOR_CACHE: usize = 10_000;
195const RECEIPT_CACHE: usize = 100_000;
196
197/// The seed in a key file: 32 bytes, or 64 lowercase hex digits and an optional newline.
198fn read_seed(path: &std::path::Path) -> Result<SigningKey, ConfigError> {
199    let bytes = std::fs::read(path).map_err(|e| ConfigError::KeyFile(format!("{}: {e}", path.display())))?;
200    if let Ok(seed) = <[u8; 32]>::try_from(bytes.as_slice()) {
201        return Ok(SigningKey::from_seed(&seed));
202    }
203    let text = bytes.strip_suffix(b"\n").unwrap_or(&bytes);
204    let hex = |c: u8| match c {
205        b'0'..=b'9' => Some(c - b'0'),
206        b'a'..=b'f' => Some(c - b'a' + 10),
207        _ => None,
208    };
209    if text.len() == 64 {
210        let mut seed = [0u8; 32];
211        let mut ok = true;
212        for (i, pair) in text.chunks(2).enumerate() {
213            match (hex(pair[0]), hex(pair[1])) {
214                (Some(h), Some(l)) => seed[i] = h << 4 | l,
215                _ => ok = false,
216            }
217        }
218        if ok {
219            return Ok(SigningKey::from_seed(&seed));
220        }
221    }
222    Err(ConfigError::KeyFile(format!("{}: not a seed (32 bytes, or 64 lowercase hex digits)", path.display())))
223}
224
225impl fmt::Debug for Engine {
226    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
227        f.debug_struct("Engine").finish_non_exhaustive()
228    }
229}
230
231/// Sizes of the memo and of the dirty set ([`Engine::memo_stats`]).
232#[non_exhaustive]
233#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
234pub struct MemoStats {
235    /// Answers held.
236    pub entries: usize,
237    /// Cells with at least one answer held.
238    pub cells: usize,
239    /// Edges of the in-memory dependents index (input cell to reading cell).
240    pub dependent_edges: usize,
241    /// Cells marked dirty and kept to refuse answers of evaluations already running.
242    pub dirty: usize,
243    /// Answers dropped because a stored version they rested on changed, kept to be rechecked
244    /// from memory (early cutoff without store queries for the unchanged part of a graph).
245    pub stale: usize,
246    /// An estimate of the bytes these take.
247    pub approx_bytes: usize,
248}
249
250impl Engine {
251    /// A builder for an engine on `store`, signing as the mesh `host`.
252    pub fn builder(store: Arc<dyn Store + Send + Sync>, host: impl Into<String>) -> EngineBuilder {
253        EngineBuilder {
254            store,
255            host: host.into(),
256            key: None,
257            key_file: None,
258            runtimes: Vec::new(),
259            depth_limit: DEFAULT_DEPTH_LIMIT,
260            fuel: DEFAULT_FUEL,
261            memo_capacity: DEFAULT_MEMO_CAPACITY,
262            licence: "CC-BY-4.0".into(),
263            summary: None,
264            clock: Arc::new(SystemClock),
265        }
266    }
267
268    /// The value of the cell `cell` denotes, as of `asof` (else `cell.asof`, else now).
269    ///
270    /// The function name resolves as of the asof; with no binding then, the answer is
271    /// [`Answer::NotCovered`]. See the crate documentation for the answers and errors.
272    pub fn get(&self, cell: &CellRef, asof: Option<&str>) -> Result<Answer, EngineError> {
273        let now = self.clock.now();
274        let a = asof.or(cell.asof.as_deref()).unwrap_or(&now).to_string();
275        lexical::check_instant(&a, "$.asof").map_err(EngineError::Invalid)?;
276        check_no_slice(&cell.args)?;
277        let fn_name = format!("{}/{}", cell.ns, cell.fn_name);
278        let Some(fn_hash) = self.store.resolve_name(&fn_name, &a).map_err(|e| StoreError::new("resolve_name", e))?
279        else {
280            return Ok(Answer::NotCovered);
281        };
282        let resolved = cell.resolve(&fn_hash).map_err(EngineError::Invalid)?;
283        self.answer(&resolved, &a, &now)
284    }
285
286    /// The value of `cell` (its function pinned by `fn_hash`), as of `asof` (else now). The
287    /// form for a reference that already carries the function hash. An unknown function
288    /// hash is [`Answer::NotCovered`].
289    pub fn get_cell(&self, cell: &Cell, asof: Option<&str>) -> Result<Answer, EngineError> {
290        let now = self.clock.now();
291        let a = asof.unwrap_or(&now).to_string();
292        lexical::check_instant(&a, "$.asof").map_err(EngineError::Invalid)?;
293        check_no_slice(&cell.args)?;
294        self.answer(cell, &a, &now)
295    }
296
297    /// The answer for a resolved cell as of `a`; `now` is the request's clock reading.
298    fn answer(&self, cell: &Cell, a: &str, now: &str) -> Result<Answer, EngineError> {
299        let Some(desc) = self.descriptor_of(&cell.fn_hash)? else {
300            return Ok(Answer::NotCovered);
301        };
302        check_args(cell, &desc).map_err(EngineError::Invalid)?;
303        let id = cell.id();
304        let store_err = |e: EvalError| match e {
305            EvalError::Store(e) => EngineError::Store(e),
306            other => EngineError::Runtime { cell_id: id.clone(), message: other.to_string() },
307        };
308        // From the memo, without waiting for the evaluator.
309        if let Some(ans) = Req::new(self, a, now).memo_hit(&id).map_err(store_err)? {
310            return Ok(Answer::Receipt(ans.bytes.to_vec()));
311        }
312        let done = if desc.lang == STORED {
313            let start = self.memo.begin();
314            let mut req = Req::new(self, a, now);
315            match req.stored(cell, &id) {
316                Ok(d) => {
317                    self.memo.end(start, &req.learned);
318                    d
319                }
320                Err(e) => {
321                    self.memo.abort(start);
322                    return Err(store_err(e));
323                }
324            }
325        } else {
326            let _single = self.evaluator.lock().unwrap_or_else(|p| p.into_inner());
327            self.evaluate_cell(cell, desc, a, now, false)?
328        };
329        Ok(match done {
330            Done::Value(ans) => Answer::Receipt(ans.bytes.to_vec()),
331            Done::NotCovered | Done::Unsigned(_) => Answer::NotCovered,
332        })
333    }
334
335    /// Evaluates a computed cell in a request of its own (the caller holds the evaluator):
336    /// memo first, then the store's version validated, else the function; writes what is new
337    /// and keeps the answers in the memo.
338    fn evaluate_cell(
339        &self,
340        cell: &Cell,
341        desc: Arc<Descriptor>,
342        a: &str,
343        now: &str,
344        unsigned: bool,
345    ) -> Result<Done, EngineError> {
346        let start = self.memo.begin();
347        let mut req = Req::new(self, a, now);
348        let result = (|| {
349            if !unsigned {
350                if let Some(ans) = req.memo_hit(&cell.id()).map_err(|e| crate::eval::engine_error(e, &cell.id()))? {
351                    return Ok(Done::Value(ans));
352                }
353            }
354            let done = req.run(cell.clone(), desc, unsigned)?;
355            req.flush()?;
356            Ok(done)
357        })();
358        match &result {
359            Ok(_) => self.memo.end(start, &req.learned),
360            Err(_) => self.memo.abort(start),
361        }
362        result
363    }
364
365    /// A receipt the store returned, parsed (cached by hash).
366    pub(crate) fn parsed(&self, hash: &str, bytes: &[u8]) -> Result<Arc<Parsed>, StoreError> {
367        if let Some(p) = self.receipts.get(hash) {
368            return Ok(p);
369        }
370        let p = Arc::new(parse_stored(hash, bytes)?);
371        self.receipts.insert(hash.to_string(), p.clone());
372        Ok(p)
373    }
374
375    /// The value of `cell` as of `asof`, evaluated (a computed cell's function is run, not
376    /// looked up); for `Resolver::evaluate`.
377    fn evaluate_value(&self, cell: &Cell, asof: &str) -> Result<Option<Scalar>, EngineError> {
378        if !is_instant(asof) || cell.is_slice() {
379            return Ok(None);
380        }
381        let Some(desc) = self.descriptor_of(&cell.fn_hash)? else { return Ok(None) };
382        if check_args(cell, &desc).is_err() {
383            return Ok(None);
384        }
385        let now = self.clock.now();
386        let done = if desc.lang == STORED {
387            Req::new(self, asof, &now)
388                .stored(cell, &cell.id())
389                .map_err(|e| EngineError::Runtime { cell_id: cell.id(), message: e.to_string() })?
390        } else {
391            let _single = self.evaluator.lock().unwrap_or_else(|p| p.into_inner());
392            self.evaluate_cell(cell, desc, asof, &now, true)?
393        };
394        Ok(match done {
395            Done::Value(ans) => Some(ans.value.clone()),
396            Done::Unsigned(v) => Some(v),
397            Done::NotCovered => None,
398        })
399    }
400
401    /// The parsed descriptor of `fn_hash`, cached.
402    pub(crate) fn descriptor_of(&self, fn_hash: &str) -> Result<Option<Arc<Descriptor>>, EngineError> {
403        if let Some(d) = self.descriptors.get(fn_hash) {
404            return Ok(Some(d));
405        }
406        let Some(bytes) = self.store.descriptor(fn_hash) else { return Ok(None) };
407        let d = Arc::new(Descriptor::from_json(&bytes).map_err(EngineError::Invalid)?);
408        self.descriptors.insert(fn_hash.to_string(), d.clone());
409        Ok(Some(d))
410    }
411
412    /// Tells the engine that a stored version of `cell_id` was added: the memoised answers
413    /// that depend on the cell are dropped.
414    pub fn stored_version_added(&self, cell_id: &str) {
415        self.memo.invalidate(cell_id);
416    }
417
418    /// Drops every memoised answer.
419    pub fn clear_memo(&self) {
420        self.memo.clear();
421    }
422
423    /// Sizes of the memo and of the dirty set.
424    pub fn memo_stats(&self) -> MemoStats {
425        self.memo.stats()
426    }
427}
428
429impl Resolver for Engine {
430    fn descriptor(&self, fn_hash: &str) -> Option<Cow<'_, [u8]>> {
431        self.store.descriptor(fn_hash)
432    }
433
434    fn receipt(&self, hash: &str) -> Option<Cow<'_, [u8]>> {
435        self.store.receipt(hash)
436    }
437
438    /// The value of `cell` as of `asof` by the engine's answering rule (a receipt answers at
439    /// A when its tx and its asof are both at or before A): a stored cell's is looked up; a
440    /// computed cell's function is evaluated (its inputs answered as for `get`, and stored
441    /// when new), never read back from its own receipts. `None` when there is none or the
442    /// evaluation fails (logged with `tracing`, target `datumesh_engine`).
443    fn evaluate(&self, cell: &Cell, asof: &str) -> Option<Scalar> {
444        match self.evaluate_value(cell, asof) {
445            Ok(v) => v,
446            Err(e) => {
447                tracing::debug!(target: "datumesh_engine", cell_id = %cell.id(), asof, error = %e, "evaluate failed");
448                None
449            }
450        }
451    }
452}