1use std::borrow::Cow;
4use std::collections::HashMap;
5use std::fmt;
6use std::path::PathBuf;
7use std::sync::{Arc, Mutex};
8
9use datumesh_core::{
10 lexical, Cell, CellRef, Evaluation, Fault, FunctionDescriptor, KeyDoc, Resolver, Scalar, SigningKey,
11};
12use datumesh_store::Store;
13
14use crate::cache::{Cache, Parsed};
15use crate::check::{check_args, check_no_slice};
16use crate::clock::{Clock, SystemClock};
17use crate::error::{ConfigError, EngineError, EvalError, StoreError};
18use crate::eval::{is_instant, parse_stored, Done, Req};
19use crate::memo::Memo;
20use crate::runtime::Runtime;
21
22pub const DEFAULT_DEPTH_LIMIT: usize = 10_000;
24pub const DEFAULT_FUEL: u64 = 10_000_000;
26pub const DEFAULT_WORK_FACTOR: u64 = 16;
28pub const DEFAULT_MEMO_CAPACITY: usize = 1_000_000;
30
31#[non_exhaustive]
33#[derive(Clone, Debug, PartialEq, Eq)]
34pub enum Answer {
35 Receipt(Vec<u8>),
38 NotCovered {
45 fault: Option<Fault>,
47 },
48}
49
50impl Answer {
51 pub const NOT_COVERED: Answer = Answer::NotCovered { fault: None };
53}
54
55pub type SummaryFn = dyn Fn(&Cell) -> String + Send + Sync;
57
58pub struct EngineBuilder {
60 pub(crate) store: Arc<dyn Store + Send + Sync>,
61 pub(crate) host: String,
62 pub(crate) key: Option<SigningKey>,
63 pub(crate) key_file: Option<PathBuf>,
64 pub(crate) runtimes: Vec<Arc<dyn Runtime>>,
65 pub(crate) depth_limit: usize,
66 pub(crate) fuel: u64,
67 pub(crate) work_limit: Option<u64>,
68 pub(crate) memo_capacity: usize,
69 pub(crate) licence: String,
70 pub(crate) summary: Option<Arc<SummaryFn>>,
71 pub(crate) clock: Arc<dyn Clock>,
72}
73
74impl EngineBuilder {
75 pub fn signing_key(mut self, key: SigningKey) -> Self {
77 self.key = Some(key);
78 self.key_file = None;
79 self
80 }
81
82 pub fn signing_key_file(mut self, path: impl Into<PathBuf>) -> Self {
85 self.key_file = Some(path.into());
86 self.key = None;
87 self
88 }
89
90 pub fn runtime(mut self, runtime: Arc<dyn Runtime>) -> Self {
92 self.runtimes.push(runtime);
93 self
94 }
95
96 pub fn depth_limit(mut self, limit: usize) -> Self {
98 self.depth_limit = limit;
99 self
100 }
101
102 pub fn fuel(mut self, fuel: u64) -> Self {
104 self.fuel = fuel;
105 self
106 }
107
108 pub fn work_limit(mut self, units: u64) -> Self {
113 self.work_limit = Some(units);
114 self
115 }
116
117 pub fn memo_capacity(mut self, entries: usize) -> Self {
120 self.memo_capacity = entries;
121 self
122 }
123
124 pub fn licence(mut self, licence: impl Into<String>) -> Self {
130 self.licence = licence.into();
131 self
132 }
133
134 pub fn summary(mut self, f: impl Fn(&Cell) -> String + Send + Sync + 'static) -> Self {
137 self.summary = Some(Arc::new(f));
138 self
139 }
140
141 pub fn clock(mut self, clock: Arc<dyn Clock>) -> Self {
143 self.clock = clock;
144 self
145 }
146
147 pub fn build(self) -> Result<Engine, ConfigError> {
149 lexical::check_dns_name(&self.host, "$.host").map_err(ConfigError::Host)?;
150 let key = match (self.key, &self.key_file) {
151 (Some(k), _) => k,
152 (None, Some(path)) => read_seed(path)?,
153 (None, None) => return Err(ConfigError::NoSigningKey),
154 };
155 lexical::check_licence(&self.licence, "$.licence").map_err(ConfigError::Licence)?;
156 let mut runtimes: HashMap<String, Arc<dyn Runtime>> = HashMap::new();
157 for rt in self.runtimes {
158 let lang = rt.lang().to_string();
159 if lang == STORED {
160 return Err(ConfigError::Runtime("stored is answered by the engine, not a runtime".into()));
161 }
162 if runtimes.insert(lang.clone(), rt).is_some() {
163 return Err(ConfigError::Runtime(format!("two runtimes for {lang}")));
164 }
165 }
166 let published = match self.store.namespace(&self.host) {
167 Ok(Some(ns)) => KeyDoc::from_json(&self.host, &ns.keydoc)
168 .ok()
169 .and_then(|doc| doc.key(key.key_id()).map(|k| k.public == key.public_key()))
170 .unwrap_or(false),
171 Ok(None) => false,
172 Err(e) => return Err(ConfigError::Store(StoreError::new("namespace", e))),
173 };
174 if !published {
175 return Err(ConfigError::KeyNotPublished { host: self.host, key_id: key.key_id().to_string() });
176 }
177 Ok(Engine {
178 store: self.store,
179 host: self.host,
180 key,
181 runtimes,
182 depth_limit: self.depth_limit,
183 fuel: self.fuel,
184 work_limit: self.work_limit.unwrap_or(self.fuel.saturating_mul(DEFAULT_WORK_FACTOR)),
185 licence: self.licence,
186 summary: self.summary,
187 clock: self.clock,
188 memo: Memo::new(self.memo_capacity),
189 descriptors: Cache::new(DESCRIPTOR_CACHE),
190 receipts: Cache::new(RECEIPT_CACHE),
191 asof_dependent: Cache::new(DESCRIPTOR_CACHE),
192 evaluator: Mutex::new(()),
193 })
194 }
195}
196
197impl fmt::Debug for EngineBuilder {
198 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
199 f.debug_struct("EngineBuilder").field("host", &self.host).finish_non_exhaustive()
200 }
201}
202
203pub struct Engine {
205 pub(crate) store: Arc<dyn Store + Send + Sync>,
206 pub(crate) host: String,
207 pub(crate) key: SigningKey,
208 pub(crate) runtimes: HashMap<String, Arc<dyn Runtime>>,
209 pub(crate) depth_limit: usize,
210 pub(crate) fuel: u64,
211 pub(crate) work_limit: u64,
212 pub(crate) licence: String,
213 pub(crate) summary: Option<Arc<SummaryFn>>,
214 pub(crate) clock: Arc<dyn Clock>,
215 pub(crate) memo: Memo,
216 pub(crate) descriptors: Cache<Arc<FunctionDescriptor>>,
218 pub(crate) receipts: Cache<Arc<Parsed>>,
220 pub(crate) asof_dependent: Cache<bool>,
223 pub(crate) evaluator: Mutex<()>,
225}
226
227pub(crate) const STORED: &str = "stored";
229const DESCRIPTOR_CACHE: usize = 10_000;
230const RECEIPT_CACHE: usize = 100_000;
231
232fn read_seed(path: &std::path::Path) -> Result<SigningKey, ConfigError> {
234 let bytes = std::fs::read(path).map_err(|e| ConfigError::KeyFile(format!("{}: {e}", path.display())))?;
235 if let Ok(seed) = <[u8; 32]>::try_from(bytes.as_slice()) {
236 return Ok(SigningKey::from_seed(&seed));
237 }
238 let text = bytes.strip_suffix(b"\n").unwrap_or(&bytes);
239 let hex = |c: u8| match c {
240 b'0'..=b'9' => Some(c - b'0'),
241 b'a'..=b'f' => Some(c - b'a' + 10),
242 _ => None,
243 };
244 if text.len() == 64 {
245 let mut seed = [0u8; 32];
246 let mut ok = true;
247 for (i, pair) in text.chunks(2).enumerate() {
248 match (hex(pair[0]), hex(pair[1])) {
249 (Some(h), Some(l)) => seed[i] = h << 4 | l,
250 _ => ok = false,
251 }
252 }
253 if ok {
254 return Ok(SigningKey::from_seed(&seed));
255 }
256 }
257 Err(ConfigError::KeyFile(format!("{}: not a seed (32 bytes, or 64 lowercase hex digits)", path.display())))
258}
259
260impl fmt::Debug for Engine {
261 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
262 f.debug_struct("Engine").finish_non_exhaustive()
263 }
264}
265
266#[non_exhaustive]
268#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
269pub struct MemoStats {
270 pub entries: usize,
272 pub cells: usize,
274 pub dependent_edges: usize,
276 pub dirty: usize,
278 pub stale: usize,
281 pub slice_patterns: usize,
284 pub approx_bytes: usize,
286}
287
288impl Engine {
289 pub fn builder(store: Arc<dyn Store + Send + Sync>, host: impl Into<String>) -> EngineBuilder {
291 EngineBuilder {
292 store,
293 host: host.into(),
294 key: None,
295 key_file: None,
296 runtimes: Vec::new(),
297 depth_limit: DEFAULT_DEPTH_LIMIT,
298 fuel: DEFAULT_FUEL,
299 work_limit: None,
300 memo_capacity: DEFAULT_MEMO_CAPACITY,
301 licence: "CC-BY-4.0".into(),
302 summary: None,
303 clock: Arc::new(SystemClock),
304 }
305 }
306
307 pub fn get(&self, cell: &CellRef, asof: Option<&str>) -> Result<Answer, EngineError> {
312 let now = self.clock.now();
313 let a = asof.or(cell.asof.as_deref()).unwrap_or(&now).to_string();
314 lexical::check_instant(&a, "$.asof").map_err(EngineError::Invalid)?;
315 check_no_slice(&cell.args)?;
316 let fn_name = format!("{}/{}", cell.ns, cell.fn_name);
317 let Some(fn_hash) = self.store.resolve_name(&fn_name, &a).map_err(|e| StoreError::new("resolve_name", e))?
318 else {
319 return Ok(Answer::NOT_COVERED);
320 };
321 let resolved = cell.resolve(&fn_hash).map_err(EngineError::Invalid)?;
322 self.answer(&resolved, &a, &now)
323 }
324
325 pub fn resolve(&self, cell: &CellRef, asof: Option<&str>) -> Result<Option<Cell>, EngineError> {
331 let now;
332 let a = match asof.or(cell.asof.as_deref()) {
333 Some(a) => a,
334 None => {
335 now = self.clock.now();
336 &now
337 }
338 };
339 lexical::check_instant(a, "$.asof").map_err(EngineError::Invalid)?;
340 check_no_slice(&cell.args)?;
341 let fn_name = format!("{}/{}", cell.ns, cell.fn_name);
342 let Some(fn_hash) = self.store.resolve_name(&fn_name, a).map_err(|e| StoreError::new("resolve_name", e))?
343 else {
344 return Ok(None);
345 };
346 cell.resolve(&fn_hash).map(Some).map_err(EngineError::Invalid)
347 }
348
349 pub fn get_cell(&self, cell: &Cell, asof: Option<&str>) -> Result<Answer, EngineError> {
353 let now = self.clock.now();
354 let a = asof.unwrap_or(&now).to_string();
355 lexical::check_instant(&a, "$.asof").map_err(EngineError::Invalid)?;
356 check_no_slice(&cell.args)?;
357 self.answer(cell, &a, &now)
358 }
359
360 fn answer(&self, cell: &Cell, a: &str, now: &str) -> Result<Answer, EngineError> {
362 let Some(desc) = self.descriptor_of(&cell.fn_hash)? else {
363 return Ok(Answer::NOT_COVERED);
364 };
365 check_args(cell, &desc).map_err(EngineError::Invalid)?;
366 let id = cell.id();
367 let store_err = |e: EvalError| match e {
368 EvalError::Store(e) => EngineError::Store(e),
369 other => EngineError::Runtime { cell_id: id.clone(), message: other.to_string() },
370 };
371 if let Some(ans) = Req::new(self, a, now).memo_hit(&id, a).map_err(store_err)? {
373 return Ok(Answer::Receipt(ans.bytes.to_vec()));
374 }
375 let done = if desc.lang == STORED {
376 let start = self.memo.begin();
377 let mut req = Req::new(self, a, now);
378 match req.stored(cell, &id, a) {
379 Ok(d) => {
380 self.memo.end(start, &req.learned);
381 d
382 }
383 Err(e) => {
384 self.memo.abort(start);
385 return Err(store_err(e));
386 }
387 }
388 } else {
389 let _single = self.evaluator.lock().unwrap_or_else(|p| p.into_inner());
390 self.evaluate_cell(cell, desc, a, now, false)?
391 };
392 Ok(match done {
393 Done::Value(ans) => Answer::Receipt(ans.bytes.to_vec()),
394 Done::Fault(fault, _) => Answer::NotCovered { fault: Some(fault) },
395 Done::NotCovered | Done::Unsigned(..) => Answer::NOT_COVERED,
396 })
397 }
398
399 fn evaluate_cell(
403 &self,
404 cell: &Cell,
405 desc: Arc<FunctionDescriptor>,
406 a: &str,
407 now: &str,
408 unsigned: bool,
409 ) -> Result<Done, EngineError> {
410 let start = self.memo.begin();
411 let mut req = Req::new(self, a, now);
412 let result = (|| {
413 if !unsigned {
414 if let Some(ans) =
415 req.memo_hit(&cell.id(), &req.asof()).map_err(|e| crate::eval::engine_error(e, &cell.id()))?
416 {
417 return Ok(Done::Value(ans));
418 }
419 }
420 let done = req.run(cell.clone(), desc, unsigned)?;
421 req.flush()?;
422 Ok(done)
423 })();
424 match &result {
425 Ok(_) => self.memo.end(start, &req.learned),
426 Err(_) => self.memo.abort(start),
427 }
428 result
429 }
430
431 pub(crate) fn parsed(&self, hash: &str, bytes: &[u8]) -> Result<Arc<Parsed>, StoreError> {
433 if let Some(p) = self.receipts.get(hash) {
434 return Ok(p);
435 }
436 let p = Arc::new(parse_stored(hash, bytes)?);
437 self.receipts.insert(hash.to_string(), p.clone());
438 Ok(p)
439 }
440
441 fn evaluate_value(&self, cell: &Cell, asof: &str) -> Result<Option<Evaluation>, EngineError> {
444 if !is_instant(asof) || cell.is_slice() {
445 return Ok(None);
446 }
447 let Some(desc) = self.descriptor_of(&cell.fn_hash)? else { return Ok(None) };
448 if check_args(cell, &desc).is_err() {
449 return Ok(None);
450 }
451 let now = self.clock.now();
452 let done = if desc.lang == STORED {
453 Req::new(self, asof, &now)
454 .stored(cell, &cell.id(), asof)
455 .map_err(|e| EngineError::Runtime { cell_id: cell.id(), message: e.to_string() })?
456 } else {
457 let _single = self.evaluator.lock().unwrap_or_else(|p| p.into_inner());
458 self.evaluate_cell(cell, desc, asof, &now, true)?
459 };
460 Ok(match done {
461 Done::Value(ans) => Some(Evaluation::new(ans.value.clone())),
462 Done::Unsigned(v, lineage) => Some(Evaluation::with_lineage(v, lineage)),
463 Done::NotCovered | Done::Fault(..) => None,
464 })
465 }
466
467 pub(crate) fn asof_dependent(&self, fn_hash: &str, desc: &FunctionDescriptor) -> bool {
470 if let Some(b) = self.asof_dependent.get(fn_hash) {
471 return b;
472 }
473 let b = self.runtimes.get(&desc.lang).is_some_and(|rt| rt.asof_dependent(desc));
474 self.asof_dependent.insert(fn_hash.to_string(), b);
475 b
476 }
477
478 pub(crate) fn descriptor_of(&self, fn_hash: &str) -> Result<Option<Arc<FunctionDescriptor>>, EngineError> {
480 if let Some(d) = self.descriptors.get(fn_hash) {
481 return Ok(Some(d));
482 }
483 let Some(bytes) = self.store.descriptor(fn_hash) else { return Ok(None) };
484 let d = Arc::new(FunctionDescriptor::from_json(&bytes).map_err(EngineError::Invalid)?);
485 self.descriptors.insert(fn_hash.to_string(), d.clone());
486 Ok(Some(d))
487 }
488
489 pub fn stored_version_added(&self, cell_id: &str) {
498 let receipt = if self.memo.wants_receipts() {
499 self.store
500 .latest(cell_id)
501 .ok()
502 .flatten()
503 .and_then(|v| self.store.receipt(&v.receipt_hash).map(|b| b.into_owned()))
504 .and_then(|b| datumesh_core::Receipt::from_json(&b).ok())
505 .map(Arc::new)
506 } else {
507 None
508 };
509 self.memo.invalidate(cell_id, receipt);
510 }
511
512 pub fn clear_memo(&self) {
514 self.memo.clear();
515 }
516
517 pub fn memo_stats(&self) -> MemoStats {
519 self.memo.stats()
520 }
521}
522
523impl Resolver for Engine {
524 fn descriptor(&self, fn_hash: &str) -> Option<Cow<'_, [u8]>> {
525 self.store.descriptor(fn_hash)
526 }
527
528 fn receipt(&self, hash: &str) -> Option<Cow<'_, [u8]>> {
529 self.store.receipt(hash)
530 }
531
532 fn manifest(&self, entry: &str) -> Option<Cow<'_, [u8]>> {
535 self.store.manifest(entry)
536 }
537
538 fn evaluate(&self, cell: &Cell, asof: &str) -> Option<Scalar> {
544 self.evaluation(cell, asof).map(|e| e.value)
545 }
546
547 fn evaluation(&self, cell: &Cell, asof: &str) -> Option<Evaluation> {
552 match self.evaluate_value(cell, asof) {
553 Ok(v) => v,
554 Err(e) => {
555 tracing::debug!(target: "datumesh_engine", cell_id = %cell.id(), asof, error = %e, "evaluate failed");
556 None
557 }
558 }
559 }
560}