Skip to main content

datumesh_http/
mesh.rs

1//! [`Mesh`]: the engine, its store and everything the endpoints share; the blocking
2//! operations each endpoint is built on, usable as library calls.
3
4use std::collections::{BTreeMap, HashMap};
5use std::path::PathBuf;
6use std::sync::{Arc, Mutex, RwLock};
7use std::time::{Duration, Instant};
8
9use datumesh_core::json::{self, Value};
10use datumesh_core::{
11    keys_from_response, verify, ErrorCode, FunctionDescriptor, KeyDoc, Keys, Receipt, SigningKey, Trust,
12};
13use datumesh_engine::{Answer, Clock, Engine, EngineError, Runtime, SystemClock};
14use datumesh_formula::runtime::FormulaRuntime;
15use datumesh_store::Store;
16
17use crate::apikeys::Principal;
18use crate::call::{self, Bindings, Bound, Rejection, Target};
19use crate::config::{Config, ConfigError, Rule};
20use crate::envelope::{self, Cache, Reply};
21use crate::foreign::{host_of, CrossMesh, Foreign, ForeignError, KeyCache};
22use crate::jobs::Jobs;
23use crate::limit::Limiter;
24use crate::metrics::Metrics;
25use crate::pool::Pool;
26
27/// Why a mesh could not be built.
28#[derive(Debug)]
29#[non_exhaustive]
30pub enum BuildError {
31    /// The configuration.
32    Config(ConfigError),
33    /// The engine's configuration (the key, its publication in the store, the runtimes).
34    Engine(datumesh_engine::ConfigError),
35}
36
37impl std::fmt::Display for BuildError {
38    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39        match self {
40            BuildError::Config(e) => write!(f, "{e}"),
41            BuildError::Engine(e) => write!(f, "engine: {e}"),
42        }
43    }
44}
45
46impl std::error::Error for BuildError {}
47
48/// Builds a [`Mesh`].
49pub struct MeshBuilder {
50    config: Config,
51    store: Arc<dyn Store + Send + Sync>,
52    key: Option<SigningKey>,
53    key_file: Option<PathBuf>,
54    runtimes: Vec<Arc<dyn Runtime>>,
55    formula: bool,
56    clock: Arc<dyn Clock>,
57    fuel: Option<u64>,
58    licence: Option<String>,
59    memo_capacity: Option<usize>,
60}
61
62impl std::fmt::Debug for MeshBuilder {
63    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
64        f.debug_struct("MeshBuilder").field("host", &self.config.host).finish_non_exhaustive()
65    }
66}
67
68impl MeshBuilder {
69    /// The mesh's signing key.
70    pub fn signing_key(mut self, key: SigningKey) -> Self {
71        self.key = Some(key);
72        self.key_file = None;
73        self
74    }
75
76    /// The mesh's signing key, from a file of its 32-byte seed (32 bytes or 64 hex digits),
77    /// which must have mode 0600.
78    pub fn signing_key_file(mut self, path: impl Into<PathBuf>) -> Self {
79        self.key_file = Some(path.into());
80        self.key = None;
81        self
82    }
83
84    /// Adds a runtime (the formula runtime is added unless [`MeshBuilder::without_formula`]).
85    pub fn runtime(mut self, runtime: Arc<dyn Runtime>) -> Self {
86        self.runtimes.push(runtime);
87        self
88    }
89
90    /// No formula runtime.
91    pub fn without_formula(mut self) -> Self {
92        self.formula = false;
93        self
94    }
95
96    /// The clock (default: the system clock).
97    pub fn clock(mut self, clock: Arc<dyn Clock>) -> Self {
98        self.clock = clock;
99        self
100    }
101
102    /// The engine's fuel per request.
103    pub fn fuel(mut self, fuel: u64) -> Self {
104        self.fuel = Some(fuel);
105        self
106    }
107
108    /// The licence of the receipts the engine signs.
109    pub fn licence(mut self, licence: impl Into<String>) -> Self {
110        self.licence = Some(licence.into());
111        self
112    }
113
114    /// The engine's memo capacity.
115    pub fn memo_capacity(mut self, n: usize) -> Self {
116        self.memo_capacity = Some(n);
117        self
118    }
119
120    /// Checks the configuration, builds the engine (its key must be published in the store's
121    /// namespace record for the host) and the mesh.
122    pub fn build(self) -> Result<Mesh, BuildError> {
123        self.config.check().map_err(BuildError::Config)?;
124        let keys = Arc::new(KeyCache::default());
125        let foreign = Arc::new(Foreign::new(&self.config, self.store.clone(), keys.clone(), self.clock.clone()));
126        let mut b = Engine::builder(self.store.clone(), self.config.host.clone()).clock(self.clock.clone());
127        b = match (self.key, &self.key_file) {
128            (Some(k), _) => b.signing_key(k),
129            (None, Some(p)) => {
130                crate::config::read_secret_file(p).map_err(BuildError::Config)?;
131                b.signing_key_file(p.clone())
132            }
133            (None, None) => b,
134        };
135        let mut runtimes = self.runtimes;
136        if self.formula {
137            runtimes.push(Arc::new(FormulaRuntime::new(self.store.clone())));
138        }
139        for rt in runtimes {
140            b = b.runtime(Arc::new(CrossMesh::new(rt, foreign.clone())));
141        }
142        if let Some(f) = self.fuel {
143            b = b.fuel(f);
144        }
145        if let Some(l) = self.licence {
146            b = b.licence(l);
147        }
148        if let Some(n) = self.memo_capacity {
149            b = b.memo_capacity(n);
150        }
151        let engine = b.build().map_err(BuildError::Engine)?;
152        let config = self.config;
153        let inner = Inner {
154            pool: Arc::new(Pool::new(config.pool_size)),
155            jobs: Jobs::new(config.ttl.job, config.max_jobs),
156            limiter: Limiter::new(config.rate.clone()),
157            metrics: Metrics::default(),
158            engine: Arc::new(engine),
159            store: self.store,
160            clock: self.clock,
161            keys,
162            foreign,
163            descriptors: Mutex::new(HashMap::new()),
164            last_good: Mutex::new(HashMap::new()),
165            infos: (0..32).map(|_| Mutex::new(HashMap::new())).collect(),
166            latest: Latest::new(),
167            targets: Targets::new(),
168            slices: Mutex::new(Vec::new()),
169            started: Instant::now(),
170            config,
171        };
172        Ok(Mesh { inner: Arc::new(inner) })
173    }
174}
175
176/// The last `ok` receipt served for a cell, for `stale` answers.
177struct LastGood {
178    info: Arc<ReceiptInfo>,
179    at: Instant,
180}
181
182/// What the mesh needs of a receipt it serves, kept by its bytes (content-addressed: never
183/// stale): its hash, `asof`, trust and its `ok` envelope.
184pub(crate) struct ReceiptInfo {
185    pub(crate) hash: String,
186    pub(crate) asof: String,
187    pub(crate) receipt: Vec<u8>,
188    pub(crate) trust: Trust,
189    pub(crate) ok: crate::envelope::Bytes,
190}
191
192pub(crate) struct Inner {
193    pub(crate) config: Config,
194    pub(crate) store: Arc<dyn Store + Send + Sync>,
195    pub(crate) engine: Arc<Engine>,
196    pub(crate) clock: Arc<dyn Clock>,
197    pub(crate) pool: Arc<Pool>,
198    pub(crate) jobs: Jobs,
199    pub(crate) limiter: Limiter,
200    pub(crate) metrics: Metrics,
201    pub(crate) keys: Arc<KeyCache>,
202    pub(crate) foreign: Arc<Foreign>,
203    descriptors: Mutex<HashMap<String, Arc<FunctionDescriptor>>>,
204    last_good: Mutex<HashMap<String, LastGood>>,
205    infos: Vec<Mutex<HashMap<u64, Arc<ReceiptInfo>>>>,
206    latest: Latest,
207    targets: Targets,
208    #[allow(clippy::type_complexity)]
209    pub(crate) slices: Mutex<Vec<(String, String, Arc<Vec<datumesh_store::SliceValue>>)>>,
210    pub(crate) started: Instant,
211}
212
213/// A Datumesh mesh: the engine behind the endpoints. Cheap to clone (shared).
214#[derive(Clone)]
215pub struct Mesh {
216    pub(crate) inner: Arc<Inner>,
217}
218
219impl std::fmt::Debug for Mesh {
220    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
221        f.debug_struct("Mesh").field("host", &self.inner.config.host).finish_non_exhaustive()
222    }
223}
224
225/// Bindings looked up once per name and asof within one batch.
226pub(crate) struct Memo<'a> {
227    inner: &'a Inner,
228    seen: std::borrow::Cow<'a, Seen>,
229}
230
231/// The bindings a batch has looked up (shared by its chunks).
232#[derive(Default)]
233pub(crate) struct Seen(#[allow(clippy::type_complexity)] RwLock<HashMap<String, Vec<(String, Option<Arc<Bound>>)>>>);
234
235impl Clone for Seen {
236    fn clone(&self) -> Seen {
237        Seen(RwLock::new(self.0.read().unwrap_or_else(|p| p.into_inner()).clone()))
238    }
239}
240
241impl<'a> Memo<'a> {
242    pub(crate) fn new(inner: &'a Inner) -> Memo<'a> {
243        Memo { inner, seen: std::borrow::Cow::Owned(Seen::default()) }
244    }
245
246    pub(crate) fn shared(inner: &'a Inner, seen: &'a Seen) -> Memo<'a> {
247        Memo { inner, seen: std::borrow::Cow::Borrowed(seen) }
248    }
249}
250
251impl Bindings for Memo<'_> {
252    fn cached(&self, fn_hash: &str, rest: &str) -> Option<Arc<Target>> {
253        self.inner.targets.get(fn_hash, rest)
254    }
255
256    fn remember(&self, rest: &str, target: &Arc<Target>) {
257        self.inner.targets.put(rest, target);
258    }
259
260    fn binding(&self, fn_name: &str, asof: &str) -> Result<Option<Arc<Bound>>, ErrorCode> {
261        if let Some(v) = self.seen.0.read().unwrap_or_else(|p| p.into_inner()).get(fn_name) {
262            if let Some((_, b)) = v.iter().find(|(a, _)| a == asof) {
263                return Ok(b.clone());
264            }
265        }
266        let b = Bindings::binding(self.inner, fn_name, asof)?;
267        let mut m = self.seen.0.write().unwrap_or_else(|p| p.into_inner());
268        let v = m.entry(fn_name.to_string()).or_default();
269        if v.len() < 64 {
270            v.push((asof.to_string(), b.clone()));
271        }
272        Ok(b)
273    }
274}
275
276impl Bindings for Inner {
277    fn cached(&self, fn_hash: &str, rest: &str) -> Option<Arc<Target>> {
278        self.targets.get(fn_hash, rest)
279    }
280
281    fn remember(&self, rest: &str, target: &Arc<Target>) {
282        self.targets.put(rest, target);
283    }
284
285    fn binding(&self, fn_name: &str, asof: &str) -> Result<Option<Arc<Bound>>, ErrorCode> {
286        let mut failed = None;
287        if self.foreign.is_foreign(fn_name) {
288            if let Err(e) = self.foreign.ensure_name(fn_name) {
289                tracing::warn!(target: "datumesh_http", name = fn_name, detail = %e.detail, "cross-mesh binding");
290                failed = Some(e.code);
291            }
292        }
293        let b = self.store.resolve_binding(fn_name, asof).map_err(|e| {
294            tracing::warn!(target: "datumesh_http", error = %e, "resolve_binding");
295            ErrorCode::InputUnavailable
296        })?;
297        // The other mesh is unreachable: a binding imported before still names the cell
298        // (whose answer is then stale or an error), else the name cannot be resolved.
299        if let (Some(code), None) = (failed, &b) {
300            return Err(code);
301        }
302        match b {
303            None => Ok(None),
304            Some(b) => {
305                let d = self.descriptor(&b.fn_hash)?;
306                Ok(Some(Arc::new(Bound::new(b.fn_hash, b.tx, d))))
307            }
308        }
309    }
310}
311
312/// Latest answers by key: the receipt, the cell id and the asof answered at.
313type LatestShard = RwLock<HashMap<String, (Arc<ReceiptInfo>, String, String)>>;
314
315/// The latest answers of local stored cells, as of when they were answered: such an answer
316/// stands for every later asof until a version of the cell is added, which the mesh is told
317/// of ([`Mesh::stored_version_added`]), as the engine's memo is. It saves the engine's work
318/// (the cell id, the memo) on the hottest path: warm gets of stored cells.
319struct Latest {
320    shards: Vec<LatestShard>,
321    by_cell: Mutex<HashMap<String, Vec<String>>>,
322}
323
324impl Latest {
325    fn new() -> Latest {
326        Latest { shards: (0..32).map(|_| RwLock::new(HashMap::new())).collect(), by_cell: Mutex::new(HashMap::new()) }
327    }
328
329    fn shard(&self, key: &str) -> &LatestShard {
330        use std::hash::{Hash, Hasher};
331        let mut h = std::collections::hash_map::DefaultHasher::new();
332        key.hash(&mut h);
333        &self.shards[(h.finish() % self.shards.len() as u64) as usize]
334    }
335
336    /// The answer for `key` as of `a`, if one answered as of `a` or earlier is held.
337    fn get(&self, key: &str, a: &str) -> Option<Arc<ReceiptInfo>> {
338        let m = self.shard(key).read().unwrap_or_else(|p| p.into_inner());
339        let (info, _, at) = m.get(key)?;
340        (a >= at.as_str()).then(|| info.clone())
341    }
342
343    fn put(&self, key: String, cell_id: String, info: Arc<ReceiptInfo>, at: String) {
344        {
345            let mut b = self.by_cell.lock().unwrap_or_else(|p| p.into_inner());
346            if b.len() > 200_000 {
347                b.clear();
348                for s in &self.shards {
349                    s.write().unwrap_or_else(|p| p.into_inner()).clear();
350                }
351            }
352            b.entry(cell_id.clone()).or_default().push(key.clone());
353        }
354        self.shard(&key).write().unwrap_or_else(|p| p.into_inner()).insert(key, (info, cell_id, at));
355    }
356
357    fn forget(&self, cell_id: &str) {
358        let keys = self.by_cell.lock().unwrap_or_else(|p| p.into_inner()).remove(cell_id).unwrap_or_default();
359        for k in keys {
360            self.shard(&k).write().unwrap_or_else(|p| p.into_inner()).remove(&k);
361        }
362    }
363
364    fn clear(&self) {
365        self.by_cell.lock().unwrap_or_else(|p| p.into_inner()).clear();
366        for s in &self.shards {
367            s.write().unwrap_or_else(|p| p.into_inner()).clear();
368        }
369    }
370}
371
372/// Cell URI rests already parsed, by function hash (parsing depends on the signature
373/// alone): resolving a hot cell costs a binding lookup and no parse.
374struct Targets {
375    #[allow(clippy::type_complexity)]
376    shards: Vec<RwLock<HashMap<String, HashMap<String, Arc<Target>>>>>,
377}
378
379impl Targets {
380    fn new() -> Targets {
381        Targets { shards: (0..32).map(|_| RwLock::new(HashMap::new())).collect() }
382    }
383
384    #[allow(clippy::type_complexity)]
385    fn shard(&self, rest: &str) -> &RwLock<HashMap<String, HashMap<String, Arc<Target>>>> {
386        use std::hash::{Hash, Hasher};
387        let mut h = std::collections::hash_map::DefaultHasher::new();
388        rest.hash(&mut h);
389        &self.shards[(h.finish() % self.shards.len() as u64) as usize]
390    }
391
392    fn get(&self, fn_hash: &str, rest: &str) -> Option<Arc<Target>> {
393        self.shard(rest).read().unwrap_or_else(|p| p.into_inner()).get(rest)?.get(fn_hash).cloned()
394    }
395
396    fn put(&self, rest: &str, t: &Arc<Target>) {
397        let mut m = self.shard(rest).write().unwrap_or_else(|p| p.into_inner());
398        if m.len() >= 4_096 {
399            m.clear();
400        }
401        let v = m.entry(rest.to_string()).or_default();
402        if v.len() >= 8 {
403            v.clear();
404        }
405        v.insert(t.bound.fn_hash.clone(), t.clone());
406    }
407}
408
409/// The key of a cell's last good answer: the function hash and the URI (without asof: the
410/// last good answer is kept for requests about now only).
411fn last_good_key(t: &Target) -> String {
412    t.cell_key().to_string()
413}
414
415/// The engine's error as an envelope code (the engine's table, "For the HTTP layer").
416fn engine_code(e: &EngineError) -> (bool, ErrorCode) {
417    match e {
418        EngineError::Invalid(e) if envelope::is_request_code(e.code()) => (true, e.code()),
419        EngineError::Invalid(e) if envelope::is_error_code(e.code()) => (false, e.code()),
420        EngineError::NameUnresolved { .. } => (false, ErrorCode::FnUnavailable),
421        EngineError::UndeclaredInput { .. } => (false, ErrorCode::InputUndeclared),
422        EngineError::Sign(e) if envelope::is_error_code(e.code()) => (false, e.code()),
423        EngineError::Store(_) => (false, ErrorCode::InputUnavailable),
424        _ => (false, ErrorCode::EvalUnavailable),
425    }
426}
427
428impl Inner {
429    pub(crate) fn now(&self) -> String {
430        self.clock.now()
431    }
432
433    /// A descriptor by hash (cached: content-addressed).
434    pub(crate) fn descriptor(&self, fn_hash: &str) -> Result<Arc<FunctionDescriptor>, ErrorCode> {
435        if let Some(d) = self.descriptors.lock().unwrap_or_else(|p| p.into_inner()).get(fn_hash) {
436            return Ok(d.clone());
437        }
438        let bytes = self.store.descriptor(fn_hash).ok_or(ErrorCode::FnUnavailable)?;
439        let d = Arc::new(FunctionDescriptor::from_json(&bytes).map_err(|_| ErrorCode::FnUnavailable)?);
440        let mut m = self.descriptors.lock().unwrap_or_else(|p| p.into_inner());
441        if m.len() > 10_000 {
442            m.clear();
443        }
444        m.insert(fn_hash.to_string(), d.clone());
445        Ok(d)
446    }
447
448    /// The `ok` reply with a receipt, remembered for `stale` answers.
449    fn ok_reply(
450        &self,
451        target: &Target,
452        info: Arc<ReceiptInfo>,
453        a: &str,
454        now: &str,
455        private: bool,
456        link: bool,
457    ) -> Reply {
458        let mut reply = Reply::json(200, info.ok.clone(), Cache::Immutable);
459        if target.asof().is_none() {
460            let mut lg = self.last_good.lock().unwrap_or_else(|p| p.into_inner());
461            if lg.len() > 100_000 {
462                lg.clear();
463            }
464            lg.insert(last_good_key(target), LastGood { info, at: Instant::now() });
465        }
466        self.cache_rules(&mut reply, target, a, now, private, link);
467        reply
468    }
469
470    /// Remembers the answer as of now `a` for a local stored cell, when it is the cell's
471    /// latest version (a version with a later `tx` would answer later asofs).
472    fn remember_latest(&self, target: &Target, info: &Arc<ReceiptInfo>, a: &str) {
473        match self.store.latest(target.cell_id()) {
474            Ok(Some(v)) if v.receipt_hash == info.hash => {
475                self.latest.put(
476                    target.cell_key().to_string(),
477                    target.cell_id().to_string(),
478                    info.clone(),
479                    a.to_string(),
480                );
481            }
482            _ => {}
483        }
484    }
485
486    /// A stored version was added for `cell_id`: the engine's memo, the latest answers and
487    /// the slice cache forget what rests on it.
488    pub(crate) fn version_added(&self, cell_id: &str) {
489        self.engine.stored_version_added(cell_id);
490        self.latest.forget(cell_id);
491        self.slices.lock().unwrap_or_else(|p| p.into_inner()).clear();
492    }
493
494    /// The receipt's hash, asof, trust and `ok` envelope (cached by its bytes).
495    fn receipt_info(&self, bytes: &[u8]) -> Option<Arc<ReceiptInfo>> {
496        use std::hash::{Hash, Hasher};
497        let mut h = std::collections::hash_map::DefaultHasher::new();
498        bytes.hash(&mut h);
499        let key = h.finish();
500        let shard = &self.infos[(key % self.infos.len() as u64) as usize];
501        if let Some(i) = shard.lock().unwrap_or_else(|p| p.into_inner()).get(&key) {
502            if i.receipt == bytes {
503                return Some(i.clone());
504            }
505        }
506        let r = Receipt::from_json(bytes).ok()?;
507        let trust = envelope::trust_of(&r, &self.config.host);
508        let info = Arc::new(ReceiptInfo {
509            hash: r.hash().to_string(),
510            asof: r.asof().to_string(),
511            receipt: bytes.to_vec(),
512            trust,
513            ok: envelope::ok(bytes, trust).into(),
514        });
515        let mut m = shard.lock().unwrap_or_else(|p| p.into_inner());
516        if m.len() >= 4_096 {
517            m.clear();
518        }
519        m.insert(key, info.clone());
520        Some(info)
521    }
522
523    /// The reply to a request the call parser rejected.
524    pub(crate) fn rejection(&self, r: Rejection, asof: Option<&str>) -> Reply {
525        match r {
526            Rejection::BadRequest(c) => envelope::bad_request_reply(c),
527            Rejection::NotCovered => {
528                let _ = asof;
529                Reply::json(404, envelope::not_covered(None), Cache::Public(self.config.ttl.latest_max_age.as_secs()))
530            }
531            Rejection::Error(c) => envelope::error_reply(c),
532        }
533    }
534
535    /// The licence and payment rules for a target and principal: a refusal, and whether the
536    /// answer depends on who asks.
537    fn policy(&self, target: &Target, principal: Option<&Principal>) -> (Option<Reply>, bool) {
538        if self.config.policy.rules.is_empty() {
539            return (None, false);
540        }
541        match self.config.policy.rule(&target.fn_name()) {
542            None => (None, false),
543            Some(Rule::Principals(ps)) => {
544                if principal.is_some_and(|p| ps.contains(&p.id)) {
545                    (None, true)
546                } else {
547                    (Some(Reply::json(403, envelope::unlicensed(), Cache::Private)), true)
548                }
549            }
550            Some(Rule::Price { amount, currency }) => {
551                let authorise = format!("{}/pay/{}", self.config.origin(), target.cell_id());
552                (Some(Reply::json(402, envelope::payment_required(amount, currency, &authorise), Cache::Private)), true)
553            }
554        }
555    }
556
557    /// Answers a `/cell` request (blocking; no budget: see [`Mesh::cell`]). `on_target` is
558    /// told the request description as soon as it is known (for `pending`).
559    pub(crate) fn answer(
560        &self,
561        rest: &str,
562        principal: Option<&Principal>,
563        bindings: &dyn Bindings,
564        now: &str,
565        on_target: &mut dyn FnMut(&Target),
566    ) -> Reply {
567        self.answer_with(rest, principal, bindings, now, on_target, true)
568    }
569
570    /// As [`Inner::answer`]; without the canonical link when `link` is false (a batch's
571    /// envelopes have no headers).
572    pub(crate) fn answer_with(
573        &self,
574        rest: &str,
575        principal: Option<&Principal>,
576        bindings: &dyn Bindings,
577        now: &str,
578        on_target: &mut dyn FnMut(&Target),
579        link: bool,
580    ) -> Reply {
581        let target = match call::resolve(rest, now, &self.config.aliases, bindings, false) {
582            Ok(t) => t,
583            Err(r) => return self.rejection(r, None),
584        };
585        on_target(&target);
586        let (refused, private) = self.policy(&target, principal);
587        if let Some(r) = refused {
588            return r;
589        }
590        self.answer_target(&target, now, private, link)
591    }
592
593    /// Answers a resolved target as of its asof (or `now`).
594    pub(crate) fn answer_target(&self, target: &Target, now: &str, private: bool, link: bool) -> Reply {
595        let a = target.asof().unwrap_or(now);
596        let stored = target.bound.descriptor.lang == "stored";
597        let local = stored && !self.foreign.is_foreign(&target.cell.ns);
598        if local {
599            if let Some(info) = self.latest.get(target.cell_key(), a) {
600                let mut reply = Reply::json(200, info.ok.clone(), Cache::Immutable);
601                self.cache_rules(&mut reply, target, a, now, private, link);
602                return reply;
603            }
604        }
605        if let Err(e) = self.foreign.prepare(&target.cell, stored, target.asof()) {
606            return self.unavailable(target, e);
607        }
608        let mut attempt = 0;
609        loop {
610            let r = self.engine.get_cell(&target.cell, Some(a));
611            let inval = self.foreign.take_invalidated();
612            for id in &inval {
613                self.version_added(id);
614            }
615            if !inval.is_empty() && attempt < 2 {
616                attempt += 1;
617                continue;
618            }
619            return match r {
620                Ok(Answer::Receipt(bytes)) => {
621                    let Some(info) = self.receipt_info(&bytes) else {
622                        return envelope::error_reply(ErrorCode::EvalUnavailable);
623                    };
624                    if attempt < 2 {
625                        match self.foreign.refresh_deps(&info.hash, a) {
626                            Ok(true) => {
627                                for id in self.foreign.take_invalidated() {
628                                    self.version_added(&id);
629                                }
630                                attempt += 1;
631                                continue;
632                            }
633                            Ok(false) => {}
634                            Err(e) => return self.unavailable(target, e),
635                        }
636                    }
637                    if local && target.asof().is_none() {
638                        self.remember_latest(target, &info, a);
639                    }
640                    self.ok_reply(target, info, a, now, private, link)
641                }
642                Ok(Answer::NotCovered { fault }) => {
643                    let mut reply = Reply::json(404, envelope::not_covered(fault), Cache::Public(0));
644                    self.cache_rules(&mut reply, target, a, now, private, link);
645                    if target.asof().is_some() {
646                        reply.cache = if private {
647                            Cache::Private
648                        } else {
649                            Cache::Public(self.config.ttl.latest_max_age.as_secs())
650                        };
651                    }
652                    reply
653                }
654                Ok(_) => envelope::error_reply(ErrorCode::EvalUnavailable),
655                Err(e) => {
656                    let (bad, code) = engine_code(&e);
657                    tracing::info!(target: "datumesh_http", error = %e, code = code.as_str(), "engine");
658                    if bad {
659                        envelope::bad_request_reply(code)
660                    } else if code == ErrorCode::InputUnavailable {
661                        self.unavailable(target, ForeignError { code, detail: e.to_string() })
662                    } else {
663                        envelope::error_reply(code)
664                    }
665                }
666            };
667        }
668    }
669
670    fn cache_rules(&self, reply: &mut Reply, target: &Target, a: &str, now: &str, private: bool, link: bool) {
671        let latest = self.config.ttl.latest_max_age.as_secs();
672        reply.cache = if private {
673            Cache::Private
674        } else {
675            match target.asof() {
676                Some(x) if x <= now => Cache::Immutable,
677                Some(_) => Cache::Public(latest),
678                None => Cache::Public(latest),
679            }
680        };
681        if link && target.asof().is_none() {
682            reply.canonical = Some(format!("{}/cell/{}", self.config.origin(), target.rest_asof(a)));
683        }
684    }
685
686    /// The answer when something the mesh depends on failed: `stale` with the last receipt
687    /// it served for the cell, when it has one that answers the request; else `error`.
688    fn unavailable(&self, target: &Target, e: ForeignError) -> Reply {
689        tracing::info!(target: "datumesh_http", code = e.code.as_str(), detail = %e.detail, "unavailable");
690        let lg = self.last_good.lock().unwrap_or_else(|p| p.into_inner());
691        if let Some(g) = lg.get(&last_good_key(target)) {
692            if target.asof().is_none_or(|a| g.info.asof.as_str() <= a) {
693                let age = g.at.elapsed().as_secs();
694                return Reply::json(200, envelope::stale(&g.info.receipt, g.info.trust, age), Cache::NoStore);
695            }
696        }
697        envelope::error_reply(e.code)
698    }
699
700    /// The keys to verify `receipt` with: the documents of its signer and of the signers of
701    /// its inputs and slice members held in the store, from the cache or the store's
702    /// namespace records.
703    pub(crate) fn keys_for_receipt(&self, receipt: &[u8]) -> Keys {
704        let mut hosts = std::collections::BTreeSet::new();
705        if let Ok(v) = json::parse(receipt) {
706            if let Some(h) = v.get("computed_by").and_then(Value::as_str) {
707                hosts.insert(h.split('#').next().unwrap_or("").to_string());
708            }
709            if let Some(Value::Array(ins)) = v.get("inputs") {
710                let mut members = Vec::new();
711                for i in ins.iter().filter_map(Value::as_str).take(10_000) {
712                    if i.starts_with("slice:") {
713                        if let Some(m) =
714                            self.store.manifest(i).and_then(|m| datumesh_core::Manifest::from_json(&m).ok())
715                        {
716                            members.extend(m.members().iter().take(10_000).cloned());
717                        }
718                    } else {
719                        members.push(i.to_string());
720                    }
721                }
722                for h in members.iter().take(20_000) {
723                    if let Some(r) = self.store.receipt(h).and_then(|b| Receipt::from_json(&b).ok()) {
724                        hosts.insert(r.signer_host().to_string());
725                    }
726                }
727            }
728        }
729        let mut keys = Keys::new();
730        for h in hosts {
731            if let Some(r) = self.keys.result(&h) {
732                keys.insert_result(&h, r);
733            } else if let Some(doc) = self.key_doc(&h) {
734                keys.insert(doc);
735            }
736        }
737        keys
738    }
739
740    /// A host's key document: the cache's, else the store's namespace record.
741    fn key_doc(&self, host: &str) -> Option<KeyDoc> {
742        if let Some((d, _, _)) = self.keys.get(host) {
743            return Some(d);
744        }
745        let ns = self.store.namespace(host).ok()??;
746        keys_from_response(host, 200, Some(&ns.keydoc)).ok()
747    }
748}
749
750fn verified(r: datumesh_core::VerifyResult) -> Verified {
751    let s = |v: Option<&str>| v.map(|x| Value::String(x.to_string())).unwrap_or(Value::Null);
752    let body = Value::Object(vec![
753        ("blob_checked".into(), Value::Bool(r.blob_checked)),
754        ("code".into(), s(r.code.map(|c| c.as_str()))),
755        ("computed_by".into(), s(r.computed_by.as_deref())),
756        ("detail".into(), Value::String(r.detail.clone())),
757        ("hash".into(), s(r.hash.as_deref())),
758        ("level".into(), s(r.level.map(|l| l.as_str()))),
759        ("ok".into(), Value::Bool(r.ok)),
760        ("path".into(), Value::String(r.path.clone())),
761    ]);
762    Verified { body: body.to_jcs() }
763}
764
765/// What `POST /verify` reports.
766#[derive(Clone, Debug, PartialEq, Eq)]
767pub struct Verified {
768    /// The result as JSON (`ok`, `level`, `code`, `hash`, `computed_by`, `path`, `detail`).
769    pub body: Vec<u8>,
770}
771
772impl Mesh {
773    /// A builder for a mesh on `store` with `config`.
774    pub fn builder(config: Config, store: Arc<dyn Store + Send + Sync>) -> MeshBuilder {
775        MeshBuilder {
776            config,
777            store,
778            key: None,
779            key_file: None,
780            runtimes: Vec::new(),
781            formula: true,
782            clock: Arc::new(SystemClock),
783            fuel: None,
784            licence: None,
785            memo_capacity: None,
786        }
787    }
788
789    /// The configuration.
790    pub fn config(&self) -> &Config {
791        &self.inner.config
792    }
793
794    /// The engine.
795    pub fn engine(&self) -> &Arc<Engine> {
796        &self.inner.engine
797    }
798
799    /// Forgets every memoised answer (the engine's memo and the mesh's), for a process that
800    /// wrote versions without telling the mesh.
801    pub fn clear_memo(&self) {
802        self.inner.engine.clear_memo();
803        self.inner.latest.clear();
804        self.inner.slices.lock().unwrap_or_else(|p| p.into_inner()).clear();
805    }
806
807    /// The cross-mesh resolver.
808    pub fn foreign(&self) -> &Foreign {
809        &self.inner.foreign
810    }
811
812    /// The store.
813    pub fn store(&self) -> &Arc<dyn Store + Send + Sync> {
814        &self.inner.store
815    }
816
817    /// The key documents the mesh holds for other hosts (fetched, or inserted here): what
818    /// `POST /verify` verifies with, besides the store's namespace records.
819    pub fn keys(&self) -> &KeyCache {
820        &self.inner.keys
821    }
822
823    /// Tells the engine that a stored version was added for `cell_id` outside the mesh.
824    pub fn stored_version_added(&self, cell_id: &str) {
825        self.inner.version_added(cell_id);
826    }
827
828    /// The envelope for `rest` (a call or a cell URI's rest, query included, as received at
829    /// `/cell/`), for `principal`. Blocking, and without a budget: what a `/cell` request
830    /// gets when it finishes within its budget, or its job's result.
831    pub fn cell(&self, rest: &str, principal: Option<&Principal>) -> Reply {
832        let now = self.inner.now();
833        self.inner.answer(rest, principal, &*self.inner, &now, &mut |_| {})
834    }
835
836    /// The envelopes of a batch, in order (blocking, no budget).
837    pub fn cells(&self, rests: &[String], principal: Option<&Principal>) -> Vec<Reply> {
838        let now = self.inner.now();
839        let memo = Memo::new(&self.inner);
840        rests.iter().map(|r| self.inner.answer(r, principal, &memo, &now, &mut |_| {})).collect()
841    }
842
843    /// Resolves a request to its cell, as `/cell` does (or to its slice, with `slice`).
844    pub fn resolve(&self, rest: &str, slice: bool) -> Result<Arc<Target>, Rejection> {
845        let now = self.inner.now();
846        call::resolve(rest, &now, &self.inner.config.aliases, &*self.inner, slice)
847    }
848
849    /// Verifies a receipt (SPEC 10.1) with the mesh's keys; at level `recomputed` with the
850    /// engine as the resolver unless `signed_only`; `blob` is content for step 7.7.
851    pub fn verify(&self, receipt: &[u8], signed_only: bool, blob: Option<&[u8]>) -> Verified {
852        let keys = self.inner.keys_for_receipt(receipt);
853        let resolver: Option<&dyn datumesh_core::Resolver> =
854            if signed_only { None } else { Some(&*self.inner.engine as &dyn datumesh_core::Resolver) };
855        verified(verify(receipt, &keys, resolver, blob))
856    }
857
858    /// Verifies a receipt with the mesh's keys and what the caller offers for step 7: a
859    /// `recompute` object as SPEC 11 describes it (`descriptor`, `inputs`, `manifests`,
860    /// `lineage`, `value`), the core's `MapResolver` over it. Without one, steps 1-6 only.
861    pub fn verify_offered(&self, receipt: &[u8], recompute: Option<&Value>, blob: Option<&[u8]>) -> Verified {
862        let mut keys = self.inner.keys_for_receipt(receipt);
863        // The signers of the receipts offered for step 7 too.
864        let mut hosts = std::collections::BTreeSet::new();
865        for (_, r) in recompute.and_then(|rc| rc.get("inputs")).and_then(Value::as_object).unwrap_or(&[]) {
866            if let Some(h) = r
867                .as_str()
868                .and_then(|t| json::parse(t.as_bytes()).ok())
869                .and_then(|v| v.get("computed_by").and_then(Value::as_str).map(str::to_string))
870            {
871                hosts.insert(h.split('#').next().unwrap_or("").to_string());
872            }
873        }
874        for h in hosts {
875            if let Some(r) = self.inner.keys.result(&h) {
876                keys.insert_result(&h, r);
877            } else if let Some(doc) = self.inner.key_doc(&h) {
878                keys.insert(doc);
879            }
880        }
881        let Some(rc) = recompute else {
882            return verified(verify(receipt, &keys, None, blob));
883        };
884        let mut m = datumesh_core::MapResolver::new();
885        let cell = json::parse(receipt).ok().and_then(|r| r.get("cell").cloned());
886        if let (Some(d), Some(h)) =
887            (rc.get("descriptor"), cell.as_ref().and_then(|c| c.get("fn_hash")).and_then(Value::as_str))
888        {
889            m.insert_descriptor(h, d.to_jcs());
890        }
891        for (h, r) in rc.get("inputs").and_then(Value::as_object).unwrap_or(&[]) {
892            m.insert_receipt(h.as_str(), r.as_str().unwrap_or("").as_bytes());
893        }
894        for (e, t) in rc.get("manifests").and_then(Value::as_object).unwrap_or(&[]) {
895            m.insert_manifest(e.as_str(), t.as_str().unwrap_or("").as_bytes());
896        }
897        let value = rc.get("value").and_then(|v| datumesh_core::Scalar::from_json(&v.to_jcs()).ok());
898        if let (Some(value), Some(c)) = (value, cell.as_ref().and_then(|c| datumesh_core::Cell::from_value(c).ok())) {
899            m.insert_value(c.id(), value);
900            if let Some(Value::Array(l)) = rc.get("lineage") {
901                m.insert_lineage(c.id(), l.iter().filter_map(Value::as_str));
902            }
903        }
904        verified(verify(receipt, &keys, Some(&m), blob))
905    }
906
907    /// The mesh document (SPEC 9.6) this mesh serves.
908    pub fn mesh_document(&self) -> Vec<u8> {
909        let o = self.inner.config.origin();
910        let ep: BTreeMap<&str, String> = [
911            ("blob", format!("{o}/blob/{{sha256}}")),
912            ("cell", format!("{o}/cell/{{cell}}")),
913            ("cells", format!("{o}/cells")),
914            ("descriptor", format!("{o}/descriptor/{{hash}}")),
915            ("explain", format!("{o}/explain/{{cell}}")),
916            ("find", format!("{o}/find")),
917            ("jobs", format!("{o}/jobs/{{job}}")),
918            ("manifest", format!("{o}/manifest/{{slice_id}}/{{members_hash}}")),
919            ("receipt", format!("{o}/receipt/{{hash}}")),
920            ("slice", format!("{o}/slice/{{cell}}")),
921            ("verify", format!("{o}/verify")),
922        ]
923        .into_iter()
924        .collect();
925        Value::Object(vec![
926            ("bindings".into(), Value::String(format!("{o}/bindings"))),
927            (
928                "endpoints".into(),
929                Value::Object(ep.into_iter().map(|(k, v)| (k.to_string(), Value::String(v))).collect()),
930            ),
931            ("host".into(), Value::String(self.inner.config.host.clone())),
932            ("proto".into(), Value::String("datumesh/1".into())),
933        ])
934        .to_jcs()
935    }
936
937    /// The key document this mesh serves: the store's namespace record for its host.
938    pub fn key_document(&self) -> Option<Vec<u8>> {
939        self.inner.store.namespace(&self.inner.config.host).ok().flatten().map(|n| n.keydoc)
940    }
941
942    /// The Prometheus text of `/metrics`.
943    pub fn metrics(&self) -> String {
944        crate::http::metrics_text(&self.inner)
945    }
946
947    /// The number of requests counted for a route template (e.g. `/cell/{cell}`).
948    pub fn requests_of(&self, route: &str) -> u64 {
949        self.inner.metrics.requests_of(route)
950    }
951
952    /// The host of a function name (its first segment).
953    pub fn host_of(name: &str) -> &str {
954        host_of(name)
955    }
956
957    /// How long the mesh has run.
958    pub fn uptime(&self) -> Duration {
959        self.inner.started.elapsed()
960    }
961}