1use std::borrow::Cow;
4use std::collections::HashMap;
5use std::fmt;
6use std::path::PathBuf;
7use std::sync::{Arc, Mutex};
8
9use datumesh_core::{lexical, Cell, CellRef, Descriptor, KeyDoc, Resolver, Scalar, SigningKey};
10use datumesh_store::Store;
11
12use crate::cache::{Cache, Parsed};
13use crate::check::{check_args, check_no_slice};
14use crate::clock::{Clock, SystemClock};
15use crate::error::{ConfigError, EngineError, EvalError, StoreError};
16use crate::eval::{is_instant, parse_stored, Done, Req};
17use crate::memo::Memo;
18use crate::runtime::Runtime;
19
20pub const DEFAULT_DEPTH_LIMIT: usize = 10_000;
22pub const DEFAULT_FUEL: u64 = 10_000_000;
24pub const DEFAULT_MEMO_CAPACITY: usize = 1_000_000;
26
27#[derive(Clone, Debug, PartialEq, Eq)]
29pub enum Answer {
30 Receipt(Vec<u8>),
33 NotCovered,
37}
38
39pub type SummaryFn = dyn Fn(&Cell) -> String + Send + Sync;
41
42pub struct EngineBuilder {
44 pub(crate) store: Arc<dyn Store + Send + Sync>,
45 pub(crate) host: String,
46 pub(crate) key: Option<SigningKey>,
47 pub(crate) key_file: Option<PathBuf>,
48 pub(crate) runtimes: Vec<Arc<dyn Runtime>>,
49 pub(crate) depth_limit: usize,
50 pub(crate) fuel: u64,
51 pub(crate) memo_capacity: usize,
52 pub(crate) licence: String,
53 pub(crate) summary: Option<Arc<SummaryFn>>,
54 pub(crate) clock: Arc<dyn Clock>,
55}
56
57impl EngineBuilder {
58 pub fn signing_key(mut self, key: SigningKey) -> Self {
60 self.key = Some(key);
61 self.key_file = None;
62 self
63 }
64
65 pub fn signing_key_file(mut self, path: impl Into<PathBuf>) -> Self {
68 self.key_file = Some(path.into());
69 self.key = None;
70 self
71 }
72
73 pub fn runtime(mut self, runtime: Arc<dyn Runtime>) -> Self {
75 self.runtimes.push(runtime);
76 self
77 }
78
79 pub fn depth_limit(mut self, limit: usize) -> Self {
81 self.depth_limit = limit;
82 self
83 }
84
85 pub fn fuel(mut self, fuel: u64) -> Self {
87 self.fuel = fuel;
88 self
89 }
90
91 pub fn memo_capacity(mut self, entries: usize) -> Self {
94 self.memo_capacity = entries;
95 self
96 }
97
98 pub fn licence(mut self, licence: impl Into<String>) -> Self {
101 self.licence = licence.into();
102 self
103 }
104
105 pub fn summary(mut self, f: impl Fn(&Cell) -> String + Send + Sync + 'static) -> Self {
108 self.summary = Some(Arc::new(f));
109 self
110 }
111
112 pub fn clock(mut self, clock: Arc<dyn Clock>) -> Self {
114 self.clock = clock;
115 self
116 }
117
118 pub fn build(self) -> Result<Engine, ConfigError> {
120 lexical::check_dns_name(&self.host, "$.host").map_err(ConfigError::Host)?;
121 let key = match (self.key, &self.key_file) {
122 (Some(k), _) => k,
123 (None, Some(path)) => read_seed(path)?,
124 (None, None) => return Err(ConfigError::NoSigningKey),
125 };
126 lexical::check_licence(&self.licence, "$.licence").map_err(ConfigError::Licence)?;
127 let mut runtimes: HashMap<String, Arc<dyn Runtime>> = HashMap::new();
128 for rt in self.runtimes {
129 let lang = rt.lang().to_string();
130 if lang == STORED {
131 return Err(ConfigError::Runtime("stored is answered by the engine, not a runtime".into()));
132 }
133 if runtimes.insert(lang.clone(), rt).is_some() {
134 return Err(ConfigError::Runtime(format!("two runtimes for {lang}")));
135 }
136 }
137 let published = match self.store.namespace(&self.host) {
138 Ok(Some(ns)) => KeyDoc::from_json(&self.host, &ns.keydoc)
139 .ok()
140 .and_then(|doc| doc.key(key.key_id()).map(|k| k.public == key.public_key()))
141 .unwrap_or(false),
142 Ok(None) => false,
143 Err(e) => return Err(ConfigError::Store(StoreError::new("namespace", e))),
144 };
145 if !published {
146 return Err(ConfigError::KeyNotPublished { host: self.host, key_id: key.key_id().to_string() });
147 }
148 Ok(Engine {
149 store: self.store,
150 host: self.host,
151 key,
152 runtimes,
153 depth_limit: self.depth_limit,
154 fuel: self.fuel,
155 licence: self.licence,
156 summary: self.summary,
157 clock: self.clock,
158 memo: Memo::new(self.memo_capacity),
159 descriptors: Cache::new(DESCRIPTOR_CACHE),
160 receipts: Cache::new(RECEIPT_CACHE),
161 evaluator: Mutex::new(()),
162 })
163 }
164}
165
166impl fmt::Debug for EngineBuilder {
167 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
168 f.debug_struct("EngineBuilder").field("host", &self.host).finish_non_exhaustive()
169 }
170}
171
172pub struct Engine {
174 pub(crate) store: Arc<dyn Store + Send + Sync>,
175 pub(crate) host: String,
176 pub(crate) key: SigningKey,
177 pub(crate) runtimes: HashMap<String, Arc<dyn Runtime>>,
178 pub(crate) depth_limit: usize,
179 pub(crate) fuel: u64,
180 pub(crate) licence: String,
181 pub(crate) summary: Option<Arc<SummaryFn>>,
182 pub(crate) clock: Arc<dyn Clock>,
183 pub(crate) memo: Memo,
184 pub(crate) descriptors: Cache<Arc<Descriptor>>,
186 pub(crate) receipts: Cache<Arc<Parsed>>,
188 pub(crate) evaluator: Mutex<()>,
190}
191
192pub(crate) const STORED: &str = "stored";
194const DESCRIPTOR_CACHE: usize = 10_000;
195const RECEIPT_CACHE: usize = 100_000;
196
197fn read_seed(path: &std::path::Path) -> Result<SigningKey, ConfigError> {
199 let bytes = std::fs::read(path).map_err(|e| ConfigError::KeyFile(format!("{}: {e}", path.display())))?;
200 if let Ok(seed) = <[u8; 32]>::try_from(bytes.as_slice()) {
201 return Ok(SigningKey::from_seed(&seed));
202 }
203 let text = bytes.strip_suffix(b"\n").unwrap_or(&bytes);
204 let hex = |c: u8| match c {
205 b'0'..=b'9' => Some(c - b'0'),
206 b'a'..=b'f' => Some(c - b'a' + 10),
207 _ => None,
208 };
209 if text.len() == 64 {
210 let mut seed = [0u8; 32];
211 let mut ok = true;
212 for (i, pair) in text.chunks(2).enumerate() {
213 match (hex(pair[0]), hex(pair[1])) {
214 (Some(h), Some(l)) => seed[i] = h << 4 | l,
215 _ => ok = false,
216 }
217 }
218 if ok {
219 return Ok(SigningKey::from_seed(&seed));
220 }
221 }
222 Err(ConfigError::KeyFile(format!("{}: not a seed (32 bytes, or 64 lowercase hex digits)", path.display())))
223}
224
225impl fmt::Debug for Engine {
226 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
227 f.debug_struct("Engine").finish_non_exhaustive()
228 }
229}
230
231#[non_exhaustive]
233#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
234pub struct MemoStats {
235 pub entries: usize,
237 pub cells: usize,
239 pub dependent_edges: usize,
241 pub dirty: usize,
243 pub stale: usize,
246 pub approx_bytes: usize,
248}
249
250impl Engine {
251 pub fn builder(store: Arc<dyn Store + Send + Sync>, host: impl Into<String>) -> EngineBuilder {
253 EngineBuilder {
254 store,
255 host: host.into(),
256 key: None,
257 key_file: None,
258 runtimes: Vec::new(),
259 depth_limit: DEFAULT_DEPTH_LIMIT,
260 fuel: DEFAULT_FUEL,
261 memo_capacity: DEFAULT_MEMO_CAPACITY,
262 licence: "CC-BY-4.0".into(),
263 summary: None,
264 clock: Arc::new(SystemClock),
265 }
266 }
267
268 pub fn get(&self, cell: &CellRef, asof: Option<&str>) -> Result<Answer, EngineError> {
273 let now = self.clock.now();
274 let a = asof.or(cell.asof.as_deref()).unwrap_or(&now).to_string();
275 lexical::check_instant(&a, "$.asof").map_err(EngineError::Invalid)?;
276 check_no_slice(&cell.args)?;
277 let fn_name = format!("{}/{}", cell.ns, cell.fn_name);
278 let Some(fn_hash) = self.store.resolve_name(&fn_name, &a).map_err(|e| StoreError::new("resolve_name", e))?
279 else {
280 return Ok(Answer::NotCovered);
281 };
282 let resolved = cell.resolve(&fn_hash).map_err(EngineError::Invalid)?;
283 self.answer(&resolved, &a, &now)
284 }
285
286 pub fn get_cell(&self, cell: &Cell, asof: Option<&str>) -> Result<Answer, EngineError> {
290 let now = self.clock.now();
291 let a = asof.unwrap_or(&now).to_string();
292 lexical::check_instant(&a, "$.asof").map_err(EngineError::Invalid)?;
293 check_no_slice(&cell.args)?;
294 self.answer(cell, &a, &now)
295 }
296
297 fn answer(&self, cell: &Cell, a: &str, now: &str) -> Result<Answer, EngineError> {
299 let Some(desc) = self.descriptor_of(&cell.fn_hash)? else {
300 return Ok(Answer::NotCovered);
301 };
302 check_args(cell, &desc).map_err(EngineError::Invalid)?;
303 let id = cell.id();
304 let store_err = |e: EvalError| match e {
305 EvalError::Store(e) => EngineError::Store(e),
306 other => EngineError::Runtime { cell_id: id.clone(), message: other.to_string() },
307 };
308 if let Some(ans) = Req::new(self, a, now).memo_hit(&id).map_err(store_err)? {
310 return Ok(Answer::Receipt(ans.bytes.to_vec()));
311 }
312 let done = if desc.lang == STORED {
313 let start = self.memo.begin();
314 let mut req = Req::new(self, a, now);
315 match req.stored(cell, &id) {
316 Ok(d) => {
317 self.memo.end(start, &req.learned);
318 d
319 }
320 Err(e) => {
321 self.memo.abort(start);
322 return Err(store_err(e));
323 }
324 }
325 } else {
326 let _single = self.evaluator.lock().unwrap_or_else(|p| p.into_inner());
327 self.evaluate_cell(cell, desc, a, now, false)?
328 };
329 Ok(match done {
330 Done::Value(ans) => Answer::Receipt(ans.bytes.to_vec()),
331 Done::NotCovered | Done::Unsigned(_) => Answer::NotCovered,
332 })
333 }
334
335 fn evaluate_cell(
339 &self,
340 cell: &Cell,
341 desc: Arc<Descriptor>,
342 a: &str,
343 now: &str,
344 unsigned: bool,
345 ) -> Result<Done, EngineError> {
346 let start = self.memo.begin();
347 let mut req = Req::new(self, a, now);
348 let result = (|| {
349 if !unsigned {
350 if let Some(ans) = req.memo_hit(&cell.id()).map_err(|e| crate::eval::engine_error(e, &cell.id()))? {
351 return Ok(Done::Value(ans));
352 }
353 }
354 let done = req.run(cell.clone(), desc, unsigned)?;
355 req.flush()?;
356 Ok(done)
357 })();
358 match &result {
359 Ok(_) => self.memo.end(start, &req.learned),
360 Err(_) => self.memo.abort(start),
361 }
362 result
363 }
364
365 pub(crate) fn parsed(&self, hash: &str, bytes: &[u8]) -> Result<Arc<Parsed>, StoreError> {
367 if let Some(p) = self.receipts.get(hash) {
368 return Ok(p);
369 }
370 let p = Arc::new(parse_stored(hash, bytes)?);
371 self.receipts.insert(hash.to_string(), p.clone());
372 Ok(p)
373 }
374
375 fn evaluate_value(&self, cell: &Cell, asof: &str) -> Result<Option<Scalar>, EngineError> {
378 if !is_instant(asof) || cell.is_slice() {
379 return Ok(None);
380 }
381 let Some(desc) = self.descriptor_of(&cell.fn_hash)? else { return Ok(None) };
382 if check_args(cell, &desc).is_err() {
383 return Ok(None);
384 }
385 let now = self.clock.now();
386 let done = if desc.lang == STORED {
387 Req::new(self, asof, &now)
388 .stored(cell, &cell.id())
389 .map_err(|e| EngineError::Runtime { cell_id: cell.id(), message: e.to_string() })?
390 } else {
391 let _single = self.evaluator.lock().unwrap_or_else(|p| p.into_inner());
392 self.evaluate_cell(cell, desc, asof, &now, true)?
393 };
394 Ok(match done {
395 Done::Value(ans) => Some(ans.value.clone()),
396 Done::Unsigned(v) => Some(v),
397 Done::NotCovered => None,
398 })
399 }
400
401 pub(crate) fn descriptor_of(&self, fn_hash: &str) -> Result<Option<Arc<Descriptor>>, EngineError> {
403 if let Some(d) = self.descriptors.get(fn_hash) {
404 return Ok(Some(d));
405 }
406 let Some(bytes) = self.store.descriptor(fn_hash) else { return Ok(None) };
407 let d = Arc::new(Descriptor::from_json(&bytes).map_err(EngineError::Invalid)?);
408 self.descriptors.insert(fn_hash.to_string(), d.clone());
409 Ok(Some(d))
410 }
411
412 pub fn stored_version_added(&self, cell_id: &str) {
415 self.memo.invalidate(cell_id);
416 }
417
418 pub fn clear_memo(&self) {
420 self.memo.clear();
421 }
422
423 pub fn memo_stats(&self) -> MemoStats {
425 self.memo.stats()
426 }
427}
428
429impl Resolver for Engine {
430 fn descriptor(&self, fn_hash: &str) -> Option<Cow<'_, [u8]>> {
431 self.store.descriptor(fn_hash)
432 }
433
434 fn receipt(&self, hash: &str) -> Option<Cow<'_, [u8]>> {
435 self.store.receipt(hash)
436 }
437
438 fn evaluate(&self, cell: &Cell, asof: &str) -> Option<Scalar> {
444 match self.evaluate_value(cell, asof) {
445 Ok(v) => v,
446 Err(e) => {
447 tracing::debug!(target: "datumesh_engine", cell_id = %cell.id(), asof, error = %e, "evaluate failed");
448 None
449 }
450 }
451 }
452}