1use std::borrow::Cow;
4use std::collections::HashMap;
5use std::fmt;
6use std::path::PathBuf;
7use std::sync::{Arc, Mutex};
8
9use datumesh_core::{
10 lexical, Cell, CellRef, Evaluation, Fault, FunctionDescriptor, KeyDoc, Resolver, Scalar, SigningKey,
11};
12use datumesh_store::Store;
13
14use crate::cache::{Cache, Parsed};
15use crate::check::{check_args, check_no_slice};
16use crate::clock::{Clock, SystemClock};
17use crate::error::{ConfigError, EngineError, EvalError, StoreError};
18use crate::eval::{is_instant, parse_stored, Done, Req};
19use crate::memo::Memo;
20use crate::runtime::Runtime;
21
22pub const DEFAULT_DEPTH_LIMIT: usize = 10_000;
24pub const DEFAULT_FUEL: u64 = 10_000_000;
26pub const DEFAULT_WORK_FACTOR: u64 = 16;
28pub const DEFAULT_MEMO_CAPACITY: usize = 1_000_000;
30
31#[non_exhaustive]
33#[derive(Clone, Debug, PartialEq, Eq)]
34pub enum Answer {
35 Receipt(Vec<u8>),
38 NotCovered {
45 fault: Option<Fault>,
47 },
48}
49
50impl Answer {
51 pub const NOT_COVERED: Answer = Answer::NotCovered { fault: None };
53}
54
55pub type SummaryFn = dyn Fn(&Cell) -> String + Send + Sync;
57
58pub struct EngineBuilder {
60 pub(crate) store: Arc<dyn Store + Send + Sync>,
61 pub(crate) host: String,
62 pub(crate) key: Option<SigningKey>,
63 pub(crate) key_file: Option<PathBuf>,
64 pub(crate) runtimes: Vec<Arc<dyn Runtime>>,
65 pub(crate) depth_limit: usize,
66 pub(crate) fuel: u64,
67 pub(crate) work_limit: Option<u64>,
68 pub(crate) memo_capacity: usize,
69 pub(crate) licence: String,
70 pub(crate) summary: Option<Arc<SummaryFn>>,
71 pub(crate) clock: Arc<dyn Clock>,
72}
73
74impl EngineBuilder {
75 pub fn signing_key(mut self, key: SigningKey) -> Self {
77 self.key = Some(key);
78 self.key_file = None;
79 self
80 }
81
82 pub fn signing_key_file(mut self, path: impl Into<PathBuf>) -> Self {
85 self.key_file = Some(path.into());
86 self.key = None;
87 self
88 }
89
90 pub fn runtime(mut self, runtime: Arc<dyn Runtime>) -> Self {
92 self.runtimes.push(runtime);
93 self
94 }
95
96 pub fn depth_limit(mut self, limit: usize) -> Self {
98 self.depth_limit = limit;
99 self
100 }
101
102 pub fn fuel(mut self, fuel: u64) -> Self {
104 self.fuel = fuel;
105 self
106 }
107
108 pub fn work_limit(mut self, units: u64) -> Self {
113 self.work_limit = Some(units);
114 self
115 }
116
117 pub fn memo_capacity(mut self, entries: usize) -> Self {
120 self.memo_capacity = entries;
121 self
122 }
123
124 pub fn licence(mut self, licence: impl Into<String>) -> Self {
127 self.licence = licence.into();
128 self
129 }
130
131 pub fn summary(mut self, f: impl Fn(&Cell) -> String + Send + Sync + 'static) -> Self {
134 self.summary = Some(Arc::new(f));
135 self
136 }
137
138 pub fn clock(mut self, clock: Arc<dyn Clock>) -> Self {
140 self.clock = clock;
141 self
142 }
143
144 pub fn build(self) -> Result<Engine, ConfigError> {
146 lexical::check_dns_name(&self.host, "$.host").map_err(ConfigError::Host)?;
147 let key = match (self.key, &self.key_file) {
148 (Some(k), _) => k,
149 (None, Some(path)) => read_seed(path)?,
150 (None, None) => return Err(ConfigError::NoSigningKey),
151 };
152 lexical::check_licence(&self.licence, "$.licence").map_err(ConfigError::Licence)?;
153 let mut runtimes: HashMap<String, Arc<dyn Runtime>> = HashMap::new();
154 for rt in self.runtimes {
155 let lang = rt.lang().to_string();
156 if lang == STORED {
157 return Err(ConfigError::Runtime("stored is answered by the engine, not a runtime".into()));
158 }
159 if runtimes.insert(lang.clone(), rt).is_some() {
160 return Err(ConfigError::Runtime(format!("two runtimes for {lang}")));
161 }
162 }
163 let published = match self.store.namespace(&self.host) {
164 Ok(Some(ns)) => KeyDoc::from_json(&self.host, &ns.keydoc)
165 .ok()
166 .and_then(|doc| doc.key(key.key_id()).map(|k| k.public == key.public_key()))
167 .unwrap_or(false),
168 Ok(None) => false,
169 Err(e) => return Err(ConfigError::Store(StoreError::new("namespace", e))),
170 };
171 if !published {
172 return Err(ConfigError::KeyNotPublished { host: self.host, key_id: key.key_id().to_string() });
173 }
174 Ok(Engine {
175 store: self.store,
176 host: self.host,
177 key,
178 runtimes,
179 depth_limit: self.depth_limit,
180 fuel: self.fuel,
181 work_limit: self.work_limit.unwrap_or(self.fuel.saturating_mul(DEFAULT_WORK_FACTOR)),
182 licence: self.licence,
183 summary: self.summary,
184 clock: self.clock,
185 memo: Memo::new(self.memo_capacity),
186 descriptors: Cache::new(DESCRIPTOR_CACHE),
187 receipts: Cache::new(RECEIPT_CACHE),
188 evaluator: Mutex::new(()),
189 })
190 }
191}
192
193impl fmt::Debug for EngineBuilder {
194 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
195 f.debug_struct("EngineBuilder").field("host", &self.host).finish_non_exhaustive()
196 }
197}
198
199pub struct Engine {
201 pub(crate) store: Arc<dyn Store + Send + Sync>,
202 pub(crate) host: String,
203 pub(crate) key: SigningKey,
204 pub(crate) runtimes: HashMap<String, Arc<dyn Runtime>>,
205 pub(crate) depth_limit: usize,
206 pub(crate) fuel: u64,
207 pub(crate) work_limit: u64,
208 pub(crate) licence: String,
209 pub(crate) summary: Option<Arc<SummaryFn>>,
210 pub(crate) clock: Arc<dyn Clock>,
211 pub(crate) memo: Memo,
212 pub(crate) descriptors: Cache<Arc<FunctionDescriptor>>,
214 pub(crate) receipts: Cache<Arc<Parsed>>,
216 pub(crate) evaluator: Mutex<()>,
218}
219
220pub(crate) const STORED: &str = "stored";
222const DESCRIPTOR_CACHE: usize = 10_000;
223const RECEIPT_CACHE: usize = 100_000;
224
225fn read_seed(path: &std::path::Path) -> Result<SigningKey, ConfigError> {
227 let bytes = std::fs::read(path).map_err(|e| ConfigError::KeyFile(format!("{}: {e}", path.display())))?;
228 if let Ok(seed) = <[u8; 32]>::try_from(bytes.as_slice()) {
229 return Ok(SigningKey::from_seed(&seed));
230 }
231 let text = bytes.strip_suffix(b"\n").unwrap_or(&bytes);
232 let hex = |c: u8| match c {
233 b'0'..=b'9' => Some(c - b'0'),
234 b'a'..=b'f' => Some(c - b'a' + 10),
235 _ => None,
236 };
237 if text.len() == 64 {
238 let mut seed = [0u8; 32];
239 let mut ok = true;
240 for (i, pair) in text.chunks(2).enumerate() {
241 match (hex(pair[0]), hex(pair[1])) {
242 (Some(h), Some(l)) => seed[i] = h << 4 | l,
243 _ => ok = false,
244 }
245 }
246 if ok {
247 return Ok(SigningKey::from_seed(&seed));
248 }
249 }
250 Err(ConfigError::KeyFile(format!("{}: not a seed (32 bytes, or 64 lowercase hex digits)", path.display())))
251}
252
253impl fmt::Debug for Engine {
254 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
255 f.debug_struct("Engine").finish_non_exhaustive()
256 }
257}
258
259#[non_exhaustive]
261#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
262pub struct MemoStats {
263 pub entries: usize,
265 pub cells: usize,
267 pub dependent_edges: usize,
269 pub dirty: usize,
271 pub stale: usize,
274 pub slice_patterns: usize,
277 pub approx_bytes: usize,
279}
280
281impl Engine {
282 pub fn builder(store: Arc<dyn Store + Send + Sync>, host: impl Into<String>) -> EngineBuilder {
284 EngineBuilder {
285 store,
286 host: host.into(),
287 key: None,
288 key_file: None,
289 runtimes: Vec::new(),
290 depth_limit: DEFAULT_DEPTH_LIMIT,
291 fuel: DEFAULT_FUEL,
292 work_limit: None,
293 memo_capacity: DEFAULT_MEMO_CAPACITY,
294 licence: "CC-BY-4.0".into(),
295 summary: None,
296 clock: Arc::new(SystemClock),
297 }
298 }
299
300 pub fn get(&self, cell: &CellRef, asof: Option<&str>) -> Result<Answer, EngineError> {
305 let now = self.clock.now();
306 let a = asof.or(cell.asof.as_deref()).unwrap_or(&now).to_string();
307 lexical::check_instant(&a, "$.asof").map_err(EngineError::Invalid)?;
308 check_no_slice(&cell.args)?;
309 let fn_name = format!("{}/{}", cell.ns, cell.fn_name);
310 let Some(fn_hash) = self.store.resolve_name(&fn_name, &a).map_err(|e| StoreError::new("resolve_name", e))?
311 else {
312 return Ok(Answer::NOT_COVERED);
313 };
314 let resolved = cell.resolve(&fn_hash).map_err(EngineError::Invalid)?;
315 self.answer(&resolved, &a, &now)
316 }
317
318 pub fn get_cell(&self, cell: &Cell, asof: Option<&str>) -> Result<Answer, EngineError> {
322 let now = self.clock.now();
323 let a = asof.unwrap_or(&now).to_string();
324 lexical::check_instant(&a, "$.asof").map_err(EngineError::Invalid)?;
325 check_no_slice(&cell.args)?;
326 self.answer(cell, &a, &now)
327 }
328
329 fn answer(&self, cell: &Cell, a: &str, now: &str) -> Result<Answer, EngineError> {
331 let Some(desc) = self.descriptor_of(&cell.fn_hash)? else {
332 return Ok(Answer::NOT_COVERED);
333 };
334 check_args(cell, &desc).map_err(EngineError::Invalid)?;
335 let id = cell.id();
336 let store_err = |e: EvalError| match e {
337 EvalError::Store(e) => EngineError::Store(e),
338 other => EngineError::Runtime { cell_id: id.clone(), message: other.to_string() },
339 };
340 if let Some(ans) = Req::new(self, a, now).memo_hit(&id, a).map_err(store_err)? {
342 return Ok(Answer::Receipt(ans.bytes.to_vec()));
343 }
344 let done = if desc.lang == STORED {
345 let start = self.memo.begin();
346 let mut req = Req::new(self, a, now);
347 match req.stored(cell, &id, a) {
348 Ok(d) => {
349 self.memo.end(start, &req.learned);
350 d
351 }
352 Err(e) => {
353 self.memo.abort(start);
354 return Err(store_err(e));
355 }
356 }
357 } else {
358 let _single = self.evaluator.lock().unwrap_or_else(|p| p.into_inner());
359 self.evaluate_cell(cell, desc, a, now, false)?
360 };
361 Ok(match done {
362 Done::Value(ans) => Answer::Receipt(ans.bytes.to_vec()),
363 Done::Fault(fault, _) => Answer::NotCovered { fault: Some(fault) },
364 Done::NotCovered | Done::Unsigned(..) => Answer::NOT_COVERED,
365 })
366 }
367
368 fn evaluate_cell(
372 &self,
373 cell: &Cell,
374 desc: Arc<FunctionDescriptor>,
375 a: &str,
376 now: &str,
377 unsigned: bool,
378 ) -> Result<Done, EngineError> {
379 let start = self.memo.begin();
380 let mut req = Req::new(self, a, now);
381 let result = (|| {
382 if !unsigned {
383 if let Some(ans) =
384 req.memo_hit(&cell.id(), &req.asof()).map_err(|e| crate::eval::engine_error(e, &cell.id()))?
385 {
386 return Ok(Done::Value(ans));
387 }
388 }
389 let done = req.run(cell.clone(), desc, unsigned)?;
390 req.flush()?;
391 Ok(done)
392 })();
393 match &result {
394 Ok(_) => self.memo.end(start, &req.learned),
395 Err(_) => self.memo.abort(start),
396 }
397 result
398 }
399
400 pub(crate) fn parsed(&self, hash: &str, bytes: &[u8]) -> Result<Arc<Parsed>, StoreError> {
402 if let Some(p) = self.receipts.get(hash) {
403 return Ok(p);
404 }
405 let p = Arc::new(parse_stored(hash, bytes)?);
406 self.receipts.insert(hash.to_string(), p.clone());
407 Ok(p)
408 }
409
410 fn evaluate_value(&self, cell: &Cell, asof: &str) -> Result<Option<Evaluation>, EngineError> {
413 if !is_instant(asof) || cell.is_slice() {
414 return Ok(None);
415 }
416 let Some(desc) = self.descriptor_of(&cell.fn_hash)? else { return Ok(None) };
417 if check_args(cell, &desc).is_err() {
418 return Ok(None);
419 }
420 let now = self.clock.now();
421 let done = if desc.lang == STORED {
422 Req::new(self, asof, &now)
423 .stored(cell, &cell.id(), asof)
424 .map_err(|e| EngineError::Runtime { cell_id: cell.id(), message: e.to_string() })?
425 } else {
426 let _single = self.evaluator.lock().unwrap_or_else(|p| p.into_inner());
427 self.evaluate_cell(cell, desc, asof, &now, true)?
428 };
429 Ok(match done {
430 Done::Value(ans) => Some(Evaluation::new(ans.value.clone())),
431 Done::Unsigned(v, lineage) => Some(Evaluation::with_lineage(v, lineage)),
432 Done::NotCovered | Done::Fault(..) => None,
433 })
434 }
435
436 pub(crate) fn descriptor_of(&self, fn_hash: &str) -> Result<Option<Arc<FunctionDescriptor>>, EngineError> {
438 if let Some(d) = self.descriptors.get(fn_hash) {
439 return Ok(Some(d));
440 }
441 let Some(bytes) = self.store.descriptor(fn_hash) else { return Ok(None) };
442 let d = Arc::new(FunctionDescriptor::from_json(&bytes).map_err(EngineError::Invalid)?);
443 self.descriptors.insert(fn_hash.to_string(), d.clone());
444 Ok(Some(d))
445 }
446
447 pub fn stored_version_added(&self, cell_id: &str) {
456 let receipt = if self.memo.wants_receipts() {
457 self.store
458 .latest(cell_id)
459 .ok()
460 .flatten()
461 .and_then(|v| self.store.receipt(&v.receipt_hash).map(|b| b.into_owned()))
462 .and_then(|b| datumesh_core::Receipt::from_json(&b).ok())
463 .map(Arc::new)
464 } else {
465 None
466 };
467 self.memo.invalidate(cell_id, receipt);
468 }
469
470 pub fn clear_memo(&self) {
472 self.memo.clear();
473 }
474
475 pub fn memo_stats(&self) -> MemoStats {
477 self.memo.stats()
478 }
479}
480
481impl Resolver for Engine {
482 fn descriptor(&self, fn_hash: &str) -> Option<Cow<'_, [u8]>> {
483 self.store.descriptor(fn_hash)
484 }
485
486 fn receipt(&self, hash: &str) -> Option<Cow<'_, [u8]>> {
487 self.store.receipt(hash)
488 }
489
490 fn manifest(&self, entry: &str) -> Option<Cow<'_, [u8]>> {
493 self.store.manifest(entry)
494 }
495
496 fn evaluate(&self, cell: &Cell, asof: &str) -> Option<Scalar> {
502 self.evaluation(cell, asof).map(|e| e.value)
503 }
504
505 fn evaluation(&self, cell: &Cell, asof: &str) -> Option<Evaluation> {
510 match self.evaluate_value(cell, asof) {
511 Ok(v) => v,
512 Err(e) => {
513 tracing::debug!(target: "datumesh_engine", cell_id = %cell.id(), asof, error = %e, "evaluate failed");
514 None
515 }
516 }
517 }
518}