Skip to main content

datumesh_http/
mesh.rs

1//! [`Mesh`]: the engine, its store and everything the endpoints share; the blocking
2//! operations each endpoint is built on, usable as library calls.
3
4use std::collections::{BTreeMap, HashMap};
5use std::path::PathBuf;
6use std::sync::{Arc, Mutex, RwLock};
7use std::time::{Duration, Instant};
8
9use datumesh_core::json::{self, Value};
10use datumesh_core::{keys_from_response, verify, ErrorCode, FunctionDescriptor, KeyDoc, Keys, Receipt, SigningKey};
11use datumesh_engine::{Answer, Clock, Engine, EngineError, Runtime, SystemClock};
12use datumesh_formula::runtime::FormulaRuntime;
13use datumesh_store::Store;
14
15use crate::apikeys::Principal;
16use crate::call::{self, Bindings, Bound, Rejection, Target};
17use crate::config::{Config, ConfigError, Rule};
18use crate::envelope::{self, Cache, Reply};
19use crate::foreign::{host_of, CrossMesh, Foreign, ForeignError, KeyCache};
20use crate::jobs::Jobs;
21use crate::limit::Limiter;
22use crate::metrics::Metrics;
23use crate::pool::Pool;
24
25/// Why a mesh could not be built.
26#[derive(Debug)]
27#[non_exhaustive]
28pub enum BuildError {
29    /// The configuration.
30    Config(ConfigError),
31    /// The engine's configuration (the key, its publication in the store, the runtimes).
32    Engine(datumesh_engine::ConfigError),
33}
34
35impl std::fmt::Display for BuildError {
36    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
37        match self {
38            BuildError::Config(e) => write!(f, "{e}"),
39            BuildError::Engine(e) => write!(f, "engine: {e}"),
40        }
41    }
42}
43
44impl std::error::Error for BuildError {}
45
46/// Builds a [`Mesh`].
47pub struct MeshBuilder {
48    config: Config,
49    store: Arc<dyn Store + Send + Sync>,
50    key: Option<SigningKey>,
51    key_file: Option<PathBuf>,
52    runtimes: Vec<Arc<dyn Runtime>>,
53    formula: bool,
54    clock: Arc<dyn Clock>,
55    fuel: Option<u64>,
56    licence: Option<String>,
57    memo_capacity: Option<usize>,
58}
59
60impl std::fmt::Debug for MeshBuilder {
61    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62        f.debug_struct("MeshBuilder").field("host", &self.config.host).finish_non_exhaustive()
63    }
64}
65
66impl MeshBuilder {
67    /// The mesh's signing key.
68    pub fn signing_key(mut self, key: SigningKey) -> Self {
69        self.key = Some(key);
70        self.key_file = None;
71        self
72    }
73
74    /// The mesh's signing key, from a file of its 32-byte seed (32 bytes or 64 hex digits),
75    /// which must have mode 0600.
76    pub fn signing_key_file(mut self, path: impl Into<PathBuf>) -> Self {
77        self.key_file = Some(path.into());
78        self.key = None;
79        self
80    }
81
82    /// Adds a runtime (the formula runtime is added unless [`MeshBuilder::without_formula`]).
83    pub fn runtime(mut self, runtime: Arc<dyn Runtime>) -> Self {
84        self.runtimes.push(runtime);
85        self
86    }
87
88    /// No formula runtime.
89    pub fn without_formula(mut self) -> Self {
90        self.formula = false;
91        self
92    }
93
94    /// The clock (default: the system clock).
95    pub fn clock(mut self, clock: Arc<dyn Clock>) -> Self {
96        self.clock = clock;
97        self
98    }
99
100    /// The engine's fuel per request.
101    pub fn fuel(mut self, fuel: u64) -> Self {
102        self.fuel = Some(fuel);
103        self
104    }
105
106    /// The licence of the receipts the engine signs.
107    pub fn licence(mut self, licence: impl Into<String>) -> Self {
108        self.licence = Some(licence.into());
109        self
110    }
111
112    /// The engine's memo capacity.
113    pub fn memo_capacity(mut self, n: usize) -> Self {
114        self.memo_capacity = Some(n);
115        self
116    }
117
118    /// Checks the configuration, builds the engine (its key must be published in the store's
119    /// namespace record for the host) and the mesh.
120    pub fn build(self) -> Result<Mesh, BuildError> {
121        self.config.check().map_err(BuildError::Config)?;
122        let keys = Arc::new(KeyCache::default());
123        let foreign = Arc::new(Foreign::new(&self.config, self.store.clone(), keys.clone(), self.clock.clone()));
124        let mut b = Engine::builder(self.store.clone(), self.config.host.clone()).clock(self.clock.clone());
125        b = match (self.key, &self.key_file) {
126            (Some(k), _) => b.signing_key(k),
127            (None, Some(p)) => {
128                crate::config::read_secret_file(p).map_err(BuildError::Config)?;
129                b.signing_key_file(p.clone())
130            }
131            (None, None) => b,
132        };
133        let mut runtimes = self.runtimes;
134        if self.formula {
135            runtimes.push(Arc::new(FormulaRuntime::new(self.store.clone())));
136        }
137        for rt in runtimes {
138            b = b.runtime(Arc::new(CrossMesh::new(rt, foreign.clone())));
139        }
140        if let Some(f) = self.fuel {
141            b = b.fuel(f);
142        }
143        if let Some(l) = self.licence {
144            b = b.licence(l);
145        }
146        if let Some(n) = self.memo_capacity {
147            b = b.memo_capacity(n);
148        }
149        let engine = b.build().map_err(BuildError::Engine)?;
150        let config = self.config;
151        let inner = Inner {
152            pool: Arc::new(Pool::new(config.pool_size)),
153            jobs: Jobs::new(config.ttl.job, config.max_jobs),
154            limiter: Limiter::new(config.rate.clone()),
155            metrics: Metrics::default(),
156            engine: Arc::new(engine),
157            store: self.store,
158            clock: self.clock,
159            keys,
160            foreign,
161            descriptors: Mutex::new(HashMap::new()),
162            last_good: Mutex::new(HashMap::new()),
163            infos: (0..32).map(|_| Mutex::new(HashMap::new())).collect(),
164            by_hash: (0..32).map(|_| Mutex::new(HashMap::new())).collect(),
165            latest: Latest::new(),
166            targets: Targets::new(),
167            slices: Mutex::new(Vec::new()),
168            started: Instant::now(),
169            config,
170        };
171        Ok(Mesh { inner: Arc::new(inner) })
172    }
173}
174
175/// The last `ok` receipt served for a cell, for `stale` answers.
176struct LastGood {
177    info: Arc<ReceiptInfo>,
178    at: Instant,
179}
180
181/// What the mesh needs of a receipt it serves, kept by its bytes (content-addressed: never
182/// stale): its hash, `asof`, trust and its `ok` envelope.
183pub(crate) struct ReceiptInfo {
184    pub(crate) hash: String,
185    pub(crate) asof: String,
186    pub(crate) receipt: Vec<u8>,
187    pub(crate) parsed: Receipt,
188    pub(crate) trust: datumesh_core::TrustLabel,
189    /// The `ok` envelope, the function hash echoed.
190    pub(crate) ok: crate::envelope::Bytes,
191}
192
193pub(crate) struct Inner {
194    pub(crate) config: Config,
195    pub(crate) store: Arc<dyn Store + Send + Sync>,
196    pub(crate) engine: Arc<Engine>,
197    pub(crate) clock: Arc<dyn Clock>,
198    pub(crate) pool: Arc<Pool>,
199    pub(crate) jobs: Jobs,
200    pub(crate) limiter: Limiter,
201    pub(crate) metrics: Metrics,
202    pub(crate) keys: Arc<KeyCache>,
203    pub(crate) foreign: Arc<Foreign>,
204    descriptors: Mutex<HashMap<String, Arc<FunctionDescriptor>>>,
205    last_good: Mutex<HashMap<String, LastGood>>,
206    infos: Vec<Mutex<HashMap<u64, Arc<ReceiptInfo>>>>,
207    by_hash: Vec<Mutex<HashMap<String, Arc<ReceiptInfo>>>>,
208    latest: Latest,
209    targets: Targets,
210    #[allow(clippy::type_complexity)]
211    pub(crate) slices: Mutex<Vec<(String, String, Arc<Vec<datumesh_store::SliceValue>>)>>,
212    pub(crate) started: Instant,
213}
214
215/// A Datumesh mesh: the engine behind the endpoints. Cheap to clone (shared).
216#[derive(Clone)]
217pub struct Mesh {
218    pub(crate) inner: Arc<Inner>,
219}
220
221impl std::fmt::Debug for Mesh {
222    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
223        f.debug_struct("Mesh").field("host", &self.inner.config.host).finish_non_exhaustive()
224    }
225}
226
227/// Bindings looked up once per name and asof within one batch.
228pub(crate) struct Memo<'a> {
229    inner: &'a Inner,
230    seen: std::borrow::Cow<'a, Seen>,
231}
232
233/// A binding a batch looks up: the lookup's result once one succeeded, and the lock held
234/// during a lookup, so the batch's chunks, which start together and ask for the same names,
235/// make one store round trip per name and asof, not one each.
236#[derive(Default)]
237struct SeenEntry {
238    found: std::sync::OnceLock<Option<Arc<Bound>>>,
239    lookup: Mutex<()>,
240}
241
242/// The bindings a batch has looked up (shared by its chunks), by name, then asof.
243#[derive(Default)]
244pub(crate) struct Seen(RwLock<HashMap<String, SeenAsofs>>);
245
246/// A name's entries in [`Seen`], by asof.
247type SeenAsofs = Vec<(String, Arc<SeenEntry>)>;
248
249impl Clone for Seen {
250    fn clone(&self) -> Seen {
251        let m = self.0.read().unwrap_or_else(|p| p.into_inner());
252        let copy = m
253            .iter()
254            .map(|(name, v)| {
255                let v = v
256                    .iter()
257                    .map(|(a, e)| (a.clone(), Arc::new(SeenEntry { found: e.found.clone(), lookup: Mutex::new(()) })))
258                    .collect();
259                (name.clone(), v)
260            })
261            .collect();
262        Seen(RwLock::new(copy))
263    }
264}
265
266/// What [`Seen`] has of a name as of an asof.
267enum Held {
268    Found(Option<Arc<Bound>>),
269    Entry(Arc<SeenEntry>),
270    Full,
271}
272
273impl Seen {
274    /// The binding of `fn_name` as of `asof` if found, else its entry (made if there is room:
275    /// 64 asofs per name).
276    fn held(&self, fn_name: &str, asof: &str) -> Held {
277        if let Some(v) = self.0.read().unwrap_or_else(|p| p.into_inner()).get(fn_name) {
278            if let Some((_, e)) = v.iter().find(|(a, _)| a == asof) {
279                return match e.found.get() {
280                    Some(b) => Held::Found(b.clone()),
281                    None => Held::Entry(e.clone()),
282                };
283            }
284        }
285        let mut m = self.0.write().unwrap_or_else(|p| p.into_inner());
286        let v = m.entry(fn_name.to_string()).or_default();
287        if let Some((_, e)) = v.iter().find(|(a, _)| a == asof) {
288            return Held::Entry(e.clone());
289        }
290        if v.len() >= 64 {
291            return Held::Full;
292        }
293        let e = Arc::new(SeenEntry::default());
294        v.push((asof.to_string(), e.clone()));
295        Held::Entry(e)
296    }
297
298    /// The binding of `fn_name` as of `asof`: as found before, else looked up by `look`, at
299    /// most one lookup at a time per name and asof (the others wait for its result). A failed
300    /// lookup is not kept: the next request looks the name up again.
301    pub(crate) fn lookup(
302        &self,
303        fn_name: &str,
304        asof: &str,
305        look: impl FnOnce() -> Result<Option<Arc<Bound>>, ErrorCode>,
306    ) -> Result<Option<Arc<Bound>>, ErrorCode> {
307        let e = match self.held(fn_name, asof) {
308            Held::Found(b) => return Ok(b),
309            Held::Full => return look(),
310            Held::Entry(e) => e,
311        };
312        let _looking = e.lookup.lock().unwrap_or_else(|p| p.into_inner());
313        if let Some(b) = e.found.get() {
314            return Ok(b.clone());
315        }
316        let b = look()?;
317        let _ = e.found.set(b.clone());
318        Ok(b)
319    }
320}
321
322impl<'a> Memo<'a> {
323    pub(crate) fn new(inner: &'a Inner) -> Memo<'a> {
324        Memo { inner, seen: std::borrow::Cow::Owned(Seen::default()) }
325    }
326
327    pub(crate) fn shared(inner: &'a Inner, seen: &'a Seen) -> Memo<'a> {
328        Memo { inner, seen: std::borrow::Cow::Borrowed(seen) }
329    }
330}
331
332impl Bindings for Memo<'_> {
333    fn pinned(&self, fn_hash: &str) -> Result<Option<Arc<Bound>>, ErrorCode> {
334        Bindings::pinned(self.inner, fn_hash)
335    }
336
337    fn alias(&self, alias: &str) -> Option<String> {
338        Bindings::alias(self.inner, alias)
339    }
340
341    fn cached(&self, fn_hash: &str, rest: &str) -> Option<Arc<Target>> {
342        self.inner.targets.get(fn_hash, rest)
343    }
344
345    fn remember(&self, rest: &str, target: &Arc<Target>) {
346        self.inner.targets.put(rest, target);
347    }
348
349    fn binding(&self, fn_name: &str, asof: &str) -> Result<Option<Arc<Bound>>, ErrorCode> {
350        self.seen.lookup(fn_name, asof, || Bindings::binding(self.inner, fn_name, asof))
351    }
352}
353
354impl Bindings for Inner {
355    fn pinned(&self, fn_hash: &str) -> Result<Option<Arc<Bound>>, ErrorCode> {
356        match self.descriptor(fn_hash) {
357            Ok(d) => Ok(Some(Arc::new(Bound::new(fn_hash, "", d)))),
358            Err(ErrorCode::FnUnavailable) => Ok(None),
359            Err(c) => Err(c),
360        }
361    }
362
363    fn alias(&self, alias: &str) -> Option<String> {
364        self.config.aliases.get(alias).cloned()
365    }
366
367    fn cached(&self, fn_hash: &str, rest: &str) -> Option<Arc<Target>> {
368        self.targets.get(fn_hash, rest)
369    }
370
371    fn remember(&self, rest: &str, target: &Arc<Target>) {
372        self.targets.put(rest, target);
373    }
374
375    fn binding(&self, fn_name: &str, asof: &str) -> Result<Option<Arc<Bound>>, ErrorCode> {
376        let mut failed = None;
377        if self.foreign.is_foreign(fn_name) {
378            if let Err(e) = self.foreign.ensure_name(fn_name) {
379                tracing::warn!(target: "datumesh_http", name = fn_name, detail = %e.detail, "cross-mesh binding");
380                failed = Some(e.code);
381            }
382        }
383        let b = self.store.resolve_binding(fn_name, asof).map_err(|e| {
384            tracing::warn!(target: "datumesh_http", error = %e, "resolve_binding");
385            ErrorCode::InputUnavailable
386        })?;
387        // The other mesh is unreachable: a binding imported before still names the cell
388        // (whose answer is then stale or an error), else the name cannot be resolved.
389        if let (Some(code), None) = (failed, &b) {
390            return Err(code);
391        }
392        match b {
393            None => Ok(None),
394            Some(b) => {
395                let d = self.descriptor(&b.fn_hash)?;
396                Ok(Some(Arc::new(Bound::new(b.fn_hash, b.tx, d))))
397            }
398        }
399    }
400}
401
402/// Latest answers by key: the receipt, the cell id and the asof answered at.
403type LatestShard = RwLock<HashMap<String, (Arc<ReceiptInfo>, String, String)>>;
404
405/// The latest answers of local stored cells, as of when they were answered: such an answer
406/// stands for every later asof until a version of the cell is added, which the mesh is told
407/// of ([`Mesh::stored_version_added`]), as the engine's memo is. It saves the engine's work
408/// (the cell id, the memo) on the hottest path: warm gets of stored cells.
409struct Latest {
410    shards: Vec<LatestShard>,
411    by_cell: Mutex<HashMap<String, Vec<String>>>,
412}
413
414impl Latest {
415    fn new() -> Latest {
416        Latest { shards: (0..32).map(|_| RwLock::new(HashMap::new())).collect(), by_cell: Mutex::new(HashMap::new()) }
417    }
418
419    fn shard(&self, key: &str) -> &LatestShard {
420        use std::hash::{Hash, Hasher};
421        let mut h = std::collections::hash_map::DefaultHasher::new();
422        key.hash(&mut h);
423        &self.shards[(h.finish() % self.shards.len() as u64) as usize]
424    }
425
426    /// The answer for `key` as of `a`, if one answered as of `a` or earlier is held.
427    fn get(&self, key: &str, a: &str) -> Option<Arc<ReceiptInfo>> {
428        let m = self.shard(key).read().unwrap_or_else(|p| p.into_inner());
429        let (info, _, at) = m.get(key)?;
430        (a >= at.as_str()).then(|| info.clone())
431    }
432
433    fn put(&self, key: String, cell_id: String, info: Arc<ReceiptInfo>, at: String) {
434        {
435            let mut b = self.by_cell.lock().unwrap_or_else(|p| p.into_inner());
436            if b.len() > 200_000 {
437                b.clear();
438                for s in &self.shards {
439                    s.write().unwrap_or_else(|p| p.into_inner()).clear();
440                }
441            }
442            b.entry(cell_id.clone()).or_default().push(key.clone());
443        }
444        self.shard(&key).write().unwrap_or_else(|p| p.into_inner()).insert(key, (info, cell_id, at));
445    }
446
447    fn forget(&self, cell_id: &str) {
448        let keys = self.by_cell.lock().unwrap_or_else(|p| p.into_inner()).remove(cell_id).unwrap_or_default();
449        for k in keys {
450            self.shard(&k).write().unwrap_or_else(|p| p.into_inner()).remove(&k);
451        }
452    }
453
454    fn clear(&self) {
455        self.by_cell.lock().unwrap_or_else(|p| p.into_inner()).clear();
456        for s in &self.shards {
457            s.write().unwrap_or_else(|p| p.into_inner()).clear();
458        }
459    }
460}
461
462/// Cell URI rests already parsed, by function hash (parsing depends on the signature
463/// alone): resolving a hot cell costs a binding lookup and no parse.
464struct Targets {
465    #[allow(clippy::type_complexity)]
466    shards: Vec<RwLock<HashMap<String, HashMap<String, Arc<Target>>>>>,
467}
468
469impl Targets {
470    fn new() -> Targets {
471        Targets { shards: (0..32).map(|_| RwLock::new(HashMap::new())).collect() }
472    }
473
474    #[allow(clippy::type_complexity)]
475    fn shard(&self, rest: &str) -> &RwLock<HashMap<String, HashMap<String, Arc<Target>>>> {
476        use std::hash::{Hash, Hasher};
477        let mut h = std::collections::hash_map::DefaultHasher::new();
478        rest.hash(&mut h);
479        &self.shards[(h.finish() % self.shards.len() as u64) as usize]
480    }
481
482    fn get(&self, fn_hash: &str, rest: &str) -> Option<Arc<Target>> {
483        self.shard(rest).read().unwrap_or_else(|p| p.into_inner()).get(rest)?.get(fn_hash).cloned()
484    }
485
486    fn put(&self, rest: &str, t: &Arc<Target>) {
487        let mut m = self.shard(rest).write().unwrap_or_else(|p| p.into_inner());
488        if m.len() >= 4_096 {
489            m.clear();
490        }
491        let v = m.entry(rest.to_string()).or_default();
492        if v.len() >= 8 {
493            v.clear();
494        }
495        v.insert(t.bound.fn_hash.clone(), t.clone());
496    }
497}
498
499/// The key of a cell's last good answer: the function hash and the URI (without asof: the
500/// last good answer is kept for requests about now only).
501fn last_good_key(t: &Target) -> String {
502    t.cell_key().to_string()
503}
504
505/// The engine's error as an envelope code (the engine's table, "For the HTTP layer").
506fn engine_code(e: &EngineError) -> (bool, ErrorCode) {
507    match e {
508        EngineError::Invalid(e) if envelope::is_request_code(e.code()) => (true, e.code()),
509        EngineError::Invalid(e) if envelope::is_error_code(e.code()) => (false, e.code()),
510        EngineError::NameUnresolved { .. } => (false, ErrorCode::FnUnavailable),
511        EngineError::UndeclaredInput { .. } => (false, ErrorCode::InputUndeclared),
512        EngineError::Sign(e) if envelope::is_error_code(e.code()) => (false, e.code()),
513        EngineError::Store(_) => (false, ErrorCode::InputUnavailable),
514        _ => (false, ErrorCode::EvalUnavailable),
515    }
516}
517
518impl Inner {
519    pub(crate) fn now(&self) -> String {
520        self.clock.now()
521    }
522
523    /// A descriptor by hash (cached: content-addressed).
524    pub(crate) fn descriptor(&self, fn_hash: &str) -> Result<Arc<FunctionDescriptor>, ErrorCode> {
525        if let Some(d) = self.descriptors.lock().unwrap_or_else(|p| p.into_inner()).get(fn_hash) {
526            return Ok(d.clone());
527        }
528        let bytes = self.store.descriptor(fn_hash).ok_or(ErrorCode::FnUnavailable)?;
529        let d = Arc::new(FunctionDescriptor::from_json(&bytes).map_err(|_| ErrorCode::FnUnavailable)?);
530        let mut m = self.descriptors.lock().unwrap_or_else(|p| p.into_inner());
531        if m.len() > 10_000 {
532            m.clear();
533        }
534        m.insert(fn_hash.to_string(), d.clone());
535        Ok(d)
536    }
537
538    /// The `ok` reply with a receipt, remembered for `stale` answers.
539    fn ok_reply(
540        &self,
541        target: &Target,
542        info: Arc<ReceiptInfo>,
543        a: &str,
544        now: &str,
545        private: bool,
546        link: bool,
547    ) -> Reply {
548        let mut reply = Reply::json(200, info.ok.clone(), Cache::Immutable);
549        if target.asof().is_none() {
550            let mut lg = self.last_good.lock().unwrap_or_else(|p| p.into_inner());
551            if lg.len() > 100_000 {
552                lg.clear();
553            }
554            lg.insert(last_good_key(target), LastGood { info, at: Instant::now() });
555        }
556        self.cache_rules(&mut reply, target, a, now, private, link);
557        reply
558    }
559
560    /// Remembers the answer as of now `a` for a local stored cell, when it is the cell's
561    /// latest version (a version with a later `tx` would answer later asofs).
562    fn remember_latest(&self, target: &Target, info: &Arc<ReceiptInfo>, a: &str) {
563        match self.store.latest(target.cell_id()) {
564            Ok(Some(v)) if v.receipt_hash == info.hash => {
565                self.latest.put(
566                    target.cell_key().to_string(),
567                    target.cell_id().to_string(),
568                    info.clone(),
569                    a.to_string(),
570                );
571            }
572            _ => {}
573        }
574    }
575
576    /// A stored version was added for `cell_id`: the engine's memo, the latest answers and
577    /// the slice cache forget what rests on it.
578    pub(crate) fn version_added(&self, cell_id: &str) {
579        self.engine.stored_version_added(cell_id);
580        self.latest.forget(cell_id);
581        self.slices.lock().unwrap_or_else(|p| p.into_inner()).clear();
582    }
583
584    /// The receipt's hash, asof, trust and `ok` envelope (cached by its bytes).
585    fn receipt_info(&self, bytes: &[u8]) -> Option<Arc<ReceiptInfo>> {
586        use std::hash::{Hash, Hasher};
587        let mut h = std::collections::hash_map::DefaultHasher::new();
588        bytes.hash(&mut h);
589        let key = h.finish();
590        let shard = &self.infos[(key % self.infos.len() as u64) as usize];
591        if let Some(i) = shard.lock().unwrap_or_else(|p| p.into_inner()).get(&key) {
592            if i.receipt == bytes {
593                return Some(i.clone());
594            }
595        }
596        let r = Receipt::from_json(bytes).ok()?;
597        let trust = envelope::trust_label_of(&r, &self.config.host);
598        let ok = envelope::with_fn_hash(datumesh_core::Answer::ok(r.clone(), trust), Some(&r.cell().fn_hash)).to_json();
599        let info = Arc::new(ReceiptInfo {
600            hash: r.hash().to_string(),
601            asof: r.asof().to_string(),
602            receipt: bytes.to_vec(),
603            parsed: r,
604            trust,
605            ok: ok.into(),
606        });
607        let mut m = shard.lock().unwrap_or_else(|p| p.into_inner());
608        if m.len() >= 4_096 {
609            m.clear();
610        }
611        m.insert(key, info.clone());
612        Some(info)
613    }
614
615    /// A receipt's info by its hash: from the cache, else the store.
616    pub(crate) fn receipt_info_by_hash(&self, hash: &str) -> Option<Arc<ReceiptInfo>> {
617        use std::hash::{Hash, Hasher};
618        let mut h = std::collections::hash_map::DefaultHasher::new();
619        hash.hash(&mut h);
620        let shard = &self.by_hash[(h.finish() % self.by_hash.len() as u64) as usize];
621        if let Some(i) = shard.lock().unwrap_or_else(|p| p.into_inner()).get(hash) {
622            return Some(i.clone());
623        }
624        let bytes = self.store.receipt(hash)?;
625        let info = self.receipt_info(&bytes)?;
626        let mut m = shard.lock().unwrap_or_else(|p| p.into_inner());
627        if m.len() >= 2_048 {
628            m.clear();
629        }
630        m.insert(hash.to_string(), info.clone());
631        Some(info)
632    }
633
634    /// The reply to a request the call parser rejected.
635    pub(crate) fn rejection(&self, r: Rejection, asof: Option<&str>) -> Reply {
636        match r {
637            Rejection::BadRequest(c) => envelope::bad_request_reply(c),
638            Rejection::NotCovered => {
639                let _ = asof;
640                Reply::json(404, envelope::not_covered(None), Cache::Public(self.config.ttl.latest_max_age.as_secs()))
641            }
642            Rejection::Error(ErrorCode::NameUnbound) => {
643                envelope::name_unbound_reply(self.config.ttl.latest_max_age.as_secs())
644            }
645            Rejection::Error(c) => envelope::error_reply(c),
646        }
647    }
648
649    /// The licence and payment rules for a target and principal: a refusal, and whether the
650    /// answer depends on who asks.
651    fn policy(&self, target: &Target, principal: Option<&Principal>) -> (Option<Reply>, bool) {
652        if self.config.policy.rules.is_empty() {
653            return (None, false);
654        }
655        match self.config.policy.rule(&target.fn_name()) {
656            None => (None, false),
657            Some(Rule::Principals(ps)) => {
658                if principal.is_some_and(|p| ps.contains(&p.id)) {
659                    (None, true)
660                } else {
661                    let a = envelope::with_fn_hash(datumesh_core::Answer::unlicensed(), Some(&target.bound.fn_hash));
662                    (Some(envelope::answer_reply(&a, Cache::Private)), true)
663                }
664            }
665            Some(Rule::Price { amount, currency }) => {
666                let authorise = format!("{}/pay/{}", self.config.origin(), target.cell_id());
667                let quote = datumesh_core::Quote { amount: amount.clone(), currency: currency.clone(), authorise };
668                let a =
669                    envelope::with_fn_hash(datumesh_core::Answer::payment_required(quote), Some(&target.bound.fn_hash));
670                (Some(envelope::answer_reply(&a, Cache::Private)), true)
671            }
672        }
673    }
674
675    /// Answers a `/cell` request (blocking; no budget: see [`Mesh::cell`]). `on_target` is
676    /// told the request description as soon as it is known (for `pending`).
677    pub(crate) fn answer(
678        &self,
679        rest: &str,
680        principal: Option<&Principal>,
681        bindings: &dyn Bindings,
682        now: &str,
683        on_target: &mut dyn FnMut(&Target),
684    ) -> Reply {
685        self.answer_with(rest, principal, bindings, now, on_target, true)
686    }
687
688    /// As [`Inner::answer`]; without the canonical link when `link` is false (a batch's
689    /// envelopes have no headers).
690    pub(crate) fn answer_with(
691        &self,
692        rest: &str,
693        principal: Option<&Principal>,
694        bindings: &dyn Bindings,
695        now: &str,
696        on_target: &mut dyn FnMut(&Target),
697        link: bool,
698    ) -> Reply {
699        let target = match call::resolve(rest, now, &self.config.aliases, bindings, false) {
700            Ok(t) => t,
701            Err(r) => return self.rejection(r, None),
702        };
703        on_target(&target);
704        let (refused, private) = self.policy(&target, principal);
705        if let Some(r) = refused {
706            return r;
707        }
708        self.answer_target(&target, now, private, link)
709    }
710
711    /// Answers a resolved target as of its asof (or `now`).
712    pub(crate) fn answer_target(&self, target: &Target, now: &str, private: bool, link: bool) -> Reply {
713        let a = target.asof().unwrap_or(now);
714        let stored = target.bound.descriptor.lang == "stored";
715        let local = stored && !self.foreign.is_foreign(&target.cell.ns);
716        if local {
717            if let Some(info) = self.latest.get(target.cell_key(), a) {
718                let mut reply = Reply::json(200, info.ok.clone(), Cache::Immutable);
719                self.cache_rules(&mut reply, target, a, now, private, link);
720                return reply;
721            }
722        }
723        if let Err(e) = self.foreign.prepare(&target.cell, stored, target.asof()) {
724            return self.unavailable(target, e);
725        }
726        let mut attempt = 0;
727        loop {
728            let r = self.engine.get_cell(&target.cell, Some(a));
729            let inval = self.foreign.take_invalidated();
730            for id in &inval {
731                self.version_added(id);
732            }
733            if !inval.is_empty() && attempt < 2 {
734                attempt += 1;
735                continue;
736            }
737            return match r {
738                Ok(Answer::Receipt(bytes)) => {
739                    let Some(info) = self.receipt_info(&bytes) else {
740                        return envelope::error_answer_reply(ErrorCode::EvalUnavailable, Some(&target.bound.fn_hash));
741                    };
742                    if attempt < 2 {
743                        match self.foreign.refresh_deps(&info.hash, a) {
744                            Ok(true) => {
745                                for id in self.foreign.take_invalidated() {
746                                    self.version_added(&id);
747                                }
748                                attempt += 1;
749                                continue;
750                            }
751                            Ok(false) => {}
752                            Err(e) => return self.unavailable(target, e),
753                        }
754                    }
755                    if local && target.asof().is_none() {
756                        self.remember_latest(target, &info, a);
757                    }
758                    self.ok_reply(target, info, a, now, private, link)
759                }
760                Ok(Answer::NotCovered { fault }) => {
761                    let mut ans =
762                        envelope::with_fn_hash(datumesh_core::Answer::not_covered(), Some(&target.bound.fn_hash));
763                    if let Some(f) = fault {
764                        ans = ans.with_fault(f);
765                    }
766                    let mut reply = envelope::answer_reply(&ans, Cache::Public(0));
767                    self.cache_rules(&mut reply, target, a, now, private, link);
768                    if target.asof().is_some() {
769                        reply.cache = if private {
770                            Cache::Private
771                        } else {
772                            Cache::Public(self.config.ttl.latest_max_age.as_secs())
773                        };
774                    }
775                    reply
776                }
777                Ok(_) => envelope::error_answer_reply(ErrorCode::EvalUnavailable, Some(&target.bound.fn_hash)),
778                Err(e) => {
779                    let (bad, code) = engine_code(&e);
780                    tracing::info!(target: "datumesh_http", error = %e, code = code.as_str(), "engine");
781                    if bad {
782                        envelope::bad_request_reply(code)
783                    } else if code == ErrorCode::InputUnavailable {
784                        self.unavailable(target, ForeignError { code, detail: e.to_string() })
785                    } else {
786                        envelope::error_answer_reply(code, Some(&target.bound.fn_hash))
787                    }
788                }
789            };
790        }
791    }
792
793    fn cache_rules(&self, reply: &mut Reply, target: &Target, a: &str, now: &str, private: bool, link: bool) {
794        let latest = self.config.ttl.latest_max_age.as_secs();
795        reply.cache = if private {
796            Cache::Private
797        } else {
798            match target.asof() {
799                Some(x) if x <= now => Cache::Immutable,
800                Some(_) => Cache::Public(latest),
801                None => Cache::Public(latest),
802            }
803        };
804        if link && target.asof().is_none() {
805            reply.canonical = Some(format!("{}/cell/{}", self.config.origin(), target.rest_asof(a)));
806        }
807    }
808
809    /// The answer when something the mesh depends on failed: `stale` with the last receipt
810    /// it served for the cell, when it has one that answers the request; else `error`.
811    fn unavailable(&self, target: &Target, e: ForeignError) -> Reply {
812        tracing::info!(target: "datumesh_http", code = e.code.as_str(), detail = %e.detail, "unavailable");
813        let lg = self.last_good.lock().unwrap_or_else(|p| p.into_inner());
814        if let Some(g) = lg.get(&last_good_key(target)) {
815            if target.asof().is_none_or(|a| g.info.asof.as_str() <= a) {
816                let age = g.at.elapsed().as_secs();
817                let a = datumesh_core::Answer::stale(g.info.parsed.clone(), g.info.trust, age);
818                let a = envelope::with_fn_hash(a, Some(&g.info.parsed.cell().fn_hash));
819                return envelope::answer_reply(&a, Cache::NoStore);
820            }
821        }
822        envelope::error_answer_reply(e.code, Some(&target.bound.fn_hash))
823    }
824
825    /// The keys to verify `receipt` with: the documents of its signer and of the signers of
826    /// its inputs and slice members held in the store, from the cache or the store's
827    /// namespace records.
828    pub(crate) fn keys_for_receipt(&self, receipt: &[u8]) -> Keys {
829        let mut hosts = std::collections::BTreeSet::new();
830        if let Ok(v) = json::parse(receipt) {
831            if let Some(h) = v.get("computed_by").and_then(Value::as_str) {
832                hosts.insert(h.split('#').next().unwrap_or("").to_string());
833            }
834            if let Some(Value::Array(ins)) = v.get("inputs") {
835                let mut members = Vec::new();
836                for i in ins.iter().filter_map(Value::as_str).take(10_000) {
837                    if i.starts_with("slice:") {
838                        if let Some(m) =
839                            self.store.manifest(i).and_then(|m| datumesh_core::Manifest::from_json(&m).ok())
840                        {
841                            members.extend(m.members().iter().take(10_000).cloned());
842                        }
843                    } else {
844                        members.push(i.to_string());
845                    }
846                }
847                for h in members.iter().take(20_000) {
848                    if let Some(r) = self.store.receipt(h).and_then(|b| Receipt::from_json(&b).ok()) {
849                        hosts.insert(r.signer_host().to_string());
850                    }
851                }
852            }
853        }
854        let mut keys = Keys::new();
855        for h in hosts {
856            if let Some(r) = self.keys.result(&h) {
857                keys.insert_result(&h, r);
858            } else if let Some(doc) = self.key_doc(&h) {
859                keys.insert(doc);
860            }
861        }
862        keys
863    }
864
865    /// A host's key document: the cache's, else the store's namespace record.
866    fn key_doc(&self, host: &str) -> Option<KeyDoc> {
867        if let Some((d, _, _)) = self.keys.get(host) {
868            return Some(d);
869        }
870        let ns = self.store.namespace(host).ok()??;
871        keys_from_response(host, 200, Some(&ns.keydoc)).ok()
872    }
873}
874
875fn verified(r: datumesh_core::VerifyResult) -> Verified {
876    let s = |v: Option<&str>| v.map(|x| Value::String(x.to_string())).unwrap_or(Value::Null);
877    let body = Value::Object(vec![
878        ("blob_checked".into(), Value::Bool(r.blob_checked)),
879        ("code".into(), s(r.code.map(|c| c.as_str()))),
880        ("computed_by".into(), s(r.computed_by.as_deref())),
881        ("detail".into(), Value::String(r.detail.clone())),
882        ("hash".into(), s(r.hash.as_deref())),
883        ("level".into(), s(r.level.map(|l| l.as_str()))),
884        ("ok".into(), Value::Bool(r.ok)),
885        ("path".into(), Value::String(r.path.clone())),
886    ]);
887    Verified { body: body.to_jcs() }
888}
889
890/// What `POST /verify` reports.
891#[derive(Clone, Debug, PartialEq, Eq)]
892pub struct Verified {
893    /// The result as JSON (`ok`, `level`, `code`, `hash`, `computed_by`, `path`, `detail`).
894    pub body: Vec<u8>,
895}
896
897impl Mesh {
898    /// A builder for a mesh on `store` with `config`.
899    pub fn builder(config: Config, store: Arc<dyn Store + Send + Sync>) -> MeshBuilder {
900        MeshBuilder {
901            config,
902            store,
903            key: None,
904            key_file: None,
905            runtimes: Vec::new(),
906            formula: true,
907            clock: Arc::new(SystemClock),
908            fuel: None,
909            licence: None,
910            memo_capacity: None,
911        }
912    }
913
914    /// The configuration.
915    pub fn config(&self) -> &Config {
916        &self.inner.config
917    }
918
919    /// The engine.
920    pub fn engine(&self) -> &Arc<Engine> {
921        &self.inner.engine
922    }
923
924    /// Forgets every memoised answer (the engine's memo and the mesh's), for a process that
925    /// wrote versions without telling the mesh.
926    pub fn clear_memo(&self) {
927        self.inner.engine.clear_memo();
928        self.inner.latest.clear();
929        self.inner.slices.lock().unwrap_or_else(|p| p.into_inner()).clear();
930    }
931
932    /// The cross-mesh resolver.
933    pub fn foreign(&self) -> &Foreign {
934        &self.inner.foreign
935    }
936
937    /// The store.
938    pub fn store(&self) -> &Arc<dyn Store + Send + Sync> {
939        &self.inner.store
940    }
941
942    /// The key documents the mesh holds for other hosts (fetched, or inserted here): what
943    /// `POST /verify` verifies with, besides the store's namespace records.
944    pub fn keys(&self) -> &KeyCache {
945        &self.inner.keys
946    }
947
948    /// Tells the engine that a stored version was added for `cell_id` outside the mesh.
949    pub fn stored_version_added(&self, cell_id: &str) {
950        self.inner.version_added(cell_id);
951    }
952
953    /// The envelope for `rest` (a call or a cell URI's rest, query included, as received at
954    /// `/cell/`), for `principal`. Blocking, and without a budget: what a `/cell` request
955    /// gets when it finishes within its budget, or its job's result.
956    pub fn cell(&self, rest: &str, principal: Option<&Principal>) -> Reply {
957        let now = self.inner.now();
958        self.inner.answer(rest, principal, &*self.inner, &now, &mut |_| {})
959    }
960
961    /// The envelopes of a batch, in order (blocking, no budget).
962    pub fn cells(&self, rests: &[String], principal: Option<&Principal>) -> Vec<Reply> {
963        let now = self.inner.now();
964        let memo = Memo::new(&self.inner);
965        rests.iter().map(|r| self.inner.answer(r, principal, &memo, &now, &mut |_| {})).collect()
966    }
967
968    /// Resolves a request to its cell, as `/cell` does (or to its slice, with `slice`).
969    pub fn resolve(&self, rest: &str, slice: bool) -> Result<Arc<Target>, Rejection> {
970        let now = self.inner.now();
971        call::resolve(rest, &now, &self.inner.config.aliases, &*self.inner, slice)
972    }
973
974    /// Verifies a receipt (SPEC 10.1) with the mesh's keys; at level `recomputed` with the
975    /// engine as the resolver unless `signed_only`; `blob` is content for step 7.7.
976    pub fn verify(&self, receipt: &[u8], signed_only: bool, blob: Option<&[u8]>) -> Verified {
977        let keys = self.inner.keys_for_receipt(receipt);
978        let resolver: Option<&dyn datumesh_core::Resolver> =
979            if signed_only { None } else { Some(&*self.inner.engine as &dyn datumesh_core::Resolver) };
980        verified(verify(receipt, &keys, resolver, blob))
981    }
982
983    /// Verifies a receipt with the mesh's keys and what the caller offers for step 7: a
984    /// `recompute` object as SPEC 11 describes it (`descriptor`, `inputs`, `manifests`,
985    /// `lineage`, `value`), the core's `MapResolver` over it. Without one, steps 1-6 only.
986    pub fn verify_offered(&self, receipt: &[u8], recompute: Option<&Value>, blob: Option<&[u8]>) -> Verified {
987        let mut keys = self.inner.keys_for_receipt(receipt);
988        // The signers of the receipts offered for step 7 too.
989        let mut hosts = std::collections::BTreeSet::new();
990        for (_, r) in recompute.and_then(|rc| rc.get("inputs")).and_then(Value::as_object).unwrap_or(&[]) {
991            if let Some(h) = r
992                .as_str()
993                .and_then(|t| json::parse(t.as_bytes()).ok())
994                .and_then(|v| v.get("computed_by").and_then(Value::as_str).map(str::to_string))
995            {
996                hosts.insert(h.split('#').next().unwrap_or("").to_string());
997            }
998        }
999        for h in hosts {
1000            if let Some(r) = self.inner.keys.result(&h) {
1001                keys.insert_result(&h, r);
1002            } else if let Some(doc) = self.inner.key_doc(&h) {
1003                keys.insert(doc);
1004            }
1005        }
1006        let Some(rc) = recompute else {
1007            return verified(verify(receipt, &keys, None, blob));
1008        };
1009        let mut m = datumesh_core::MapResolver::new();
1010        let cell = json::parse(receipt).ok().and_then(|r| r.get("cell").cloned());
1011        if let (Some(d), Some(h)) =
1012            (rc.get("descriptor"), cell.as_ref().and_then(|c| c.get("fn_hash")).and_then(Value::as_str))
1013        {
1014            m.insert_descriptor(h, d.to_jcs());
1015        }
1016        for (h, r) in rc.get("inputs").and_then(Value::as_object).unwrap_or(&[]) {
1017            m.insert_receipt(h.as_str(), r.as_str().unwrap_or("").as_bytes());
1018        }
1019        for (e, t) in rc.get("manifests").and_then(Value::as_object).unwrap_or(&[]) {
1020            m.insert_manifest(e.as_str(), t.as_str().unwrap_or("").as_bytes());
1021        }
1022        let value = rc.get("value").and_then(|v| datumesh_core::Scalar::from_json(&v.to_jcs()).ok());
1023        if let (Some(value), Some(c)) = (value, cell.as_ref().and_then(|c| datumesh_core::Cell::from_value(c).ok())) {
1024            m.insert_value(c.id(), value);
1025            if let Some(Value::Array(l)) = rc.get("lineage") {
1026                m.insert_lineage(c.id(), l.iter().filter_map(Value::as_str));
1027            }
1028        }
1029        verified(verify(receipt, &keys, Some(&m), blob))
1030    }
1031
1032    /// The mesh document (SPEC 9.6) this mesh serves.
1033    pub fn mesh_document(&self) -> Vec<u8> {
1034        let o = self.inner.config.origin();
1035        let ep: BTreeMap<&str, String> = [
1036            ("blob", format!("{o}/blob/{{sha256}}")),
1037            ("cell", format!("{o}/cell/{{cell}}")),
1038            ("cells", format!("{o}/cells")),
1039            ("descriptor", format!("{o}/descriptor/{{hash}}")),
1040            ("explain", format!("{o}/explain/{{cell}}")),
1041            ("find", format!("{o}/find")),
1042            ("jobs", format!("{o}/jobs/{{job}}")),
1043            ("manifest", format!("{o}/manifest/{{slice_id}}/{{members_hash}}")),
1044            ("receipt", format!("{o}/receipt/{{hash}}")),
1045            ("slice", format!("{o}/slice/{{cell}}")),
1046            ("verify", format!("{o}/verify")),
1047            ("versions", format!("{o}/versions/{{cell}}")),
1048        ]
1049        .into_iter()
1050        .collect();
1051        Value::Object(vec![
1052            ("bindings".into(), Value::String(format!("{o}/bindings"))),
1053            (
1054                "endpoints".into(),
1055                Value::Object(ep.into_iter().map(|(k, v)| (k.to_string(), Value::String(v))).collect()),
1056            ),
1057            ("host".into(), Value::String(self.inner.config.host.clone())),
1058            ("proto".into(), Value::String("datumesh/1".into())),
1059        ])
1060        .to_jcs()
1061    }
1062
1063    /// The key document this mesh serves: the store's namespace record for its host.
1064    pub fn key_document(&self) -> Option<Vec<u8>> {
1065        self.inner.store.namespace(&self.inner.config.host).ok().flatten().map(|n| n.keydoc)
1066    }
1067
1068    /// The Prometheus text of `/metrics`.
1069    pub fn metrics(&self) -> String {
1070        crate::http::metrics_text(&self.inner)
1071    }
1072
1073    /// The number of requests counted for a route template (e.g. `/cell/{cell}`).
1074    pub fn requests_of(&self, route: &str) -> u64 {
1075        self.inner.metrics.requests_of(route)
1076    }
1077
1078    /// The host of a function name (its first segment).
1079    pub fn host_of(name: &str) -> &str {
1080        host_of(name)
1081    }
1082
1083    /// How long the mesh has run.
1084    pub fn uptime(&self) -> Duration {
1085        self.inner.started.elapsed()
1086    }
1087}
1088
1089#[cfg(test)]
1090mod tests {
1091    use super::*;
1092    use std::sync::atomic::{AtomicUsize, Ordering};
1093
1094    /// A batch's chunks start together and ask for the same names: one store round trip per
1095    /// name and asof for the whole batch, not one per chunk.
1096    #[test]
1097    fn a_batch_looks_each_binding_up_once() {
1098        let seen = Seen::default();
1099        let lookups = AtomicUsize::new(0);
1100        let start = std::sync::Barrier::new(6);
1101        let look = || {
1102            lookups.fetch_add(1, Ordering::SeqCst);
1103            std::thread::sleep(Duration::from_millis(50));
1104            Ok(None)
1105        };
1106        std::thread::scope(|s| {
1107            for _ in 0..6 {
1108                s.spawn(|| {
1109                    start.wait();
1110                    assert!(matches!(seen.lookup("a.example/f", "2026-01-01T00:00:00Z", look), Ok(None)));
1111                });
1112            }
1113        });
1114        assert_eq!(lookups.load(Ordering::SeqCst), 1, "one lookup for six chunks");
1115        // Found: no lookup.
1116        assert!(matches!(seen.lookup("a.example/f", "2026-01-01T00:00:00Z", look), Ok(None)));
1117        assert_eq!(lookups.load(Ordering::SeqCst), 1);
1118        // Another asof, another name: looked up.
1119        assert!(matches!(seen.lookup("a.example/f", "2026-01-02T00:00:00Z", look), Ok(None)));
1120        assert!(matches!(seen.lookup("a.example/g", "2026-01-01T00:00:00Z", look), Ok(None)));
1121        assert_eq!(lookups.load(Ordering::SeqCst), 3);
1122        // A failed lookup is not kept.
1123        let fail = || Err(ErrorCode::InputUnavailable);
1124        assert!(matches!(seen.lookup("a.example/h", "2026-01-01T00:00:00Z", fail), Err(ErrorCode::InputUnavailable)));
1125        assert!(matches!(seen.lookup("a.example/h", "2026-01-01T00:00:00Z", look), Ok(None)));
1126        assert_eq!(lookups.load(Ordering::SeqCst), 4);
1127    }
1128}